Resource shadow index

.gitattributes

@@ -0,0 +1 @@
+CHANGELOG.md merge=union

.github/dependabot.yml

@@ -8,7 +8,13 @@ updates:
       # For all packages, ignore all major versions to minimize breaking issues
       - dependency-name: "*"
         update-types: ["version-update:semver-major"]
+    labels:
+      - "dependabot"
+      - "dependencies"
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
       interval: "weekly"
+    labels:
+      - "dependabot"
+      - "dependencies"

.github/workflows/ci.yml

@@ -45,7 +45,7 @@ jobs:
       matrix:
         gradle_task: ${{ fromJson(needs.generate-test-list.outputs.separateTestsNames) }}
         platform: [windows-latest]
-        jdk: [21]
+        jdk: [21, 24]
     runs-on: ${{ matrix.platform }}
 
     steps:
@@ -80,7 +80,7 @@ jobs:
       matrix:
         gradle_task: ${{ fromJson(needs.generate-test-list.outputs.separateTestsNames) }}
         platform: [ubuntu-latest]
-        jdk: [21]
+        jdk: [21, 24]
     runs-on: ubuntu-latest
     container:
       # using the same image which is used by opensearch-build to build the OpenSearch Distribution
@@ -144,7 +144,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        jdk: [21]
+        jdk: [21, 24]
         platform: [windows-latest]
     runs-on: ${{ matrix.platform }}
 
@@ -178,7 +178,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        jdk: [21]
+        jdk: [21, 24]
         platform: [ubuntu-latest]
     runs-on: ubuntu-latest
     container:
@@ -221,7 +221,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        jdk: [21]
+        jdk: [21, 24]
         platform: [ubuntu-latest]
     runs-on: ${{ matrix.platform }}
     container:
@@ -263,7 +263,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        jdk: [21]
+        jdk: [21, 24]
         platform: [windows-latest]
     runs-on: ${{ matrix.platform }}
 
@@ -338,7 +338,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        jdk: [21]
+        jdk: [21, 24]
         platform: [windows-latest]
     runs-on: ${{ matrix.platform }}
 
@@ -371,7 +371,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        jdk: [21]
+        jdk: [21, 24]
         platform: [ubuntu-latest]
     runs-on: ${{ matrix.platform }}
 

.github/workflows/dependabot_pr.yml

@@ -21,6 +21,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           token: ${{ steps.github_app_token.outputs.token }}
+          ref: ${{ github.head_ref }}
 
       # See please https://docs.gradle.org/8.10/userguide/upgrading_version_8.html#minimum_daemon_jvm_version
       - name: Set up JDK 21

.github/workflows/integration-tests.yml

@@ -11,7 +11,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        jdk: [21]
+        jdk: [21, 24]
         test-run: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
 
     steps:

.github/workflows/link-checker.yml

@@ -0,0 +1,21 @@
+name: Link Checker
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  linkchecker:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: lychee Link Checker
+        id: lychee
+        uses: lycheeverse/lychee-action@master
+        with:
+          args: --accept=200,403,429  **/*.html **/*.md **/*.txt **/*.json --exclude-path src/test/resources/dlsfls/logs_bulk_data.json
+        env:
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+      - name: Fail if there were link errors
+        run: exit ${{ steps.lychee.outputs.exit_code }}

.github/workflows/plugin_install.yml

@@ -12,7 +12,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest]
-        jdk: [21]
+        jdk: [21, 24]
     runs-on: ${{ matrix.os }}
 
     steps:

.lycheeignore

@@ -0,0 +1,4 @@
+http://localhost:9200/
+http://localhost:33667/
+http://test.entity/
+https://github.com/opendistro-for-elasticsearch/security/

CHANGELOG.md

@@ -8,16 +8,25 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Resource Permissions] Introduces Centralized Resource Access Control Framework ([#5281](https://github.com/opensearch-project/security/pull/5281))
 - Github workflow for changelog verification ([#5318](https://github.com/opensearch-project/security/pull/5318))
 - Register cluster settings listener for `plugins.security.cache.ttl_minutes` ([#5324](https://github.com/opensearch-project/security/pull/5324))
+- Add flush cache endpoint for individual user ([#5337](https://github.com/opensearch-project/security/pull/5337))
+- Handle roles in nested claim for JWT auth backends ([#5355](https://github.com/opensearch-project/security/pull/5355))
+- Integrate search-relevance functionalities with security plugin ([#5376](https://github.com/opensearch-project/security/pull/5376))
+- Add forecast roles and permissions ([#5386](https://github.com/opensearch-project/security/pull/5386))
 
 ### Changed
 - Use extendedPlugins in integrationTest framework for sample resource plugin testing ([#5322](https://github.com/opensearch-project/security/pull/5322))
-- Refactor ResourcePermissions to refer to action groups as access levels ([#5335](https://github.com/opensearch-project/security/pull/5335))
+- [Resource Sharing] Refactor ResourcePermissions to refer to action groups as access levels ([#5335](https://github.com/opensearch-project/security/pull/5335))
+- Introduced new, performance-optimized implementation for tenant privileges ([#5339](https://github.com/opensearch-project/security/pull/5339))
+- Performance improvements: Immutable user object ([#5212](https://github.com/opensearch-project/security/pull/5212))
+- Include mapped roles when setting userInfo in ThreadContext ([#5369](https://github.com/opensearch-project/security/pull/5369))
+- Adds details for debugging Security not initialized error([#5370](https://github.com/opensearch-project/security/pull/5370))
+- [Resource Sharing] Store resource sharing info in indices that map 1-to-1 with resource index ([#5358](https://github.com/opensearch-project/security/pull/5358))
 
 ### Dependencies
 - Bump `guava_version` from 33.4.6-jre to 33.4.8-jre ([#5284](https://github.com/opensearch-project/security/pull/5284))
-- Bump `spring_version` from 6.2.5 to 6.2.6 ([#5283](https://github.com/opensearch-project/security/pull/5283))
+- Bump `spring_version` from 6.2.5 to 6.2.7 ([#5283](https://github.com/opensearch-project/security/pull/5283), [#5345](https://github.com/opensearch-project/security/pull/5345))
 - Bump `com.google.errorprone:error_prone_annotations` from 2.37.0 to 2.38.0 ([#5285](https://github.com/opensearch-project/security/pull/5285))
-- Bump `org.mockito:mockito-core` from 5.15.2 to 5.17.0 ([#5296](https://github.com/opensearch-project/security/pull/5296))
+- Bump `org.mockito:mockito-core` from 5.15.2 to 5.18.0 ([#5296](https://github.com/opensearch-project/security/pull/5296), [#5362](https://github.com/opensearch-project/security/pull/5362))
 - Bump `com.carrotsearch.randomizedtesting:randomizedtesting-runner` from 2.8.2 to 2.8.3 ([#5294](https://github.com/opensearch-project/security/pull/5294))
 - Bump `org.ow2.asm:asm` from 9.7.1 to 9.8 ([#5293](https://github.com/opensearch-project/security/pull/5293))
 - Bump `commons-codec:commons-codec` from 1.16.1 to 1.18.0 ([#5295](https://github.com/opensearch-project/security/pull/5295))
@@ -29,15 +38,26 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `com.google.googlejavaformat:google-java-format` from 1.26.0 to 1.27.0 ([#5330](https://github.com/opensearch-project/security/pull/5330))
 - Bump `io.github.goooler.shadow` from 8.1.7 to 8.1.8 ([#5329](https://github.com/opensearch-project/security/pull/5329))
 - Bump `commons-io:commons-io` from 2.18.0 to 2.19.0 ([#5328](https://github.com/opensearch-project/security/pull/5328))
+- Upgrade kafka_version from 3.7.1 to 4.0.0 ([#5131](https://github.com/opensearch-project/security/pull/5131))
+- Bump `io.dropwizard.metrics:metrics-core` from 4.2.30 to 4.2.32 ([#5361](https://github.com/opensearch-project/security/pull/5361))
+- Bump `org.junit.jupiter:junit-jupiter` from 5.12.2 to 5.13.1 ([#5371](https://github.com/opensearch-project/security/pull/5371), [#5382](https://github.com/opensearch-project/security/pull/5382))
+- Bump `bouncycastle_version` from 1.80 to 1.81 ([#5380](https://github.com/opensearch-project/security/pull/5380))
+- Bump `org.junit.jupiter:junit-jupiter-api` from 5.13.0 to 5.13.1 ([#5383](https://github.com/opensearch-project/security/pull/5383))
+- Bump `org.checkerframework:checker-qual` from 3.49.3 to 3.49.4 ([#5381](https://github.com/opensearch-project/security/pull/5381))
 
 ### Deprecated
 
 ### Removed
 
+- Removed unused support for custom User object serialization ([#5339](https://github.com/opensearch-project/security/pull/5339))
+
 ### Fixed
 - Corrections in DlsFlsFilterLeafReader regarding PointVales and object valued attributes ([#5303](https://github.com/opensearch-project/security/pull/5303))
 - Fix issue computing diffs in compliance audit log when writing to security index ([#5279](https://github.com/opensearch-project/security/pull/5279))
 - Fixing dependabot broken pull_request workflow for changelog update ([#5331](https://github.com/opensearch-project/security/pull/5331))
+- Fixes assemble workflow failure during Jenkins build ([#5334](https://github.com/opensearch-project/security/pull/5334))
+- Fixes security index stale cache issue post snapshot restore ([#5307](https://github.com/opensearch-project/security/pull/5307))
+- Only log Invalid Authentication header when HTTP Basic auth challenge is called ([#5377](https://github.com/opensearch-project/security/pull/5377))
 
 ### Security
 

README.md

@@ -1,5 +1,5 @@
 [![CI](https://github.com/opensearch-project/security/workflows/CI/badge.svg?branch=main)](https://github.com/opensearch-project/security/actions) [![](https://img.shields.io/github/issues/opensearch-project/security/untriaged?labelColor=red)](https://github.com/opensearch-project/security/issues?q=is%3Aissue+is%3Aopen+label%3A"untriaged") [![](https://img.shields.io/github/issues/opensearch-project/security/security%20vulnerability?labelColor=red)](https://github.com/opensearch-project/security/issues?q=is%3Aissue+is%3Aopen+label%3A"security%20vulnerability") [![](https://img.shields.io/github/issues/opensearch-project/security)](https://github.com/opensearch-project/security/issues) [![](https://img.shields.io/github/issues-pr/opensearch-project/security)](https://github.com/opensearch-project/security/pulls)
-[![](https://img.shields.io/codecov/c/gh/opensearch-project/security)](https://app.codecov.io/gh/opensearch-project/security) [![](https://img.shields.io/github/issues/opensearch-project/security/v2.18.0)](https://github.com/opensearch-project/security/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.18.0") [![](https://img.shields.io/github/issues/opensearch-project/security/v3.0.0)](https://github.com/opensearch-project/security/issues?q=is%3Aissue+is%3Aopen+label%3A"v3.0.0")
+[![](https://img.shields.io/codecov/c/gh/opensearch-project/security)](https://app.codecov.io/gh/opensearch-project/security) [![](https://img.shields.io/github/issues/opensearch-project/security/v2.19.3)](https://github.com/opensearch-project/security/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.19.3") [![](https://img.shields.io/github/issues/opensearch-project/security/v3.1.0)](https://github.com/opensearch-project/security/issues?q=is%3Aissue+is%3Aopen+label%3A"v3.1.0")
 [![Slack](https://img.shields.io/badge/Slack-4A154B?&logo=slack&logoColor=white)](https://opensearch.slack.com/archives/C051Y637FKK)
 
 

build.gradle

@@ -26,15 +26,15 @@ buildscript {
 
         common_utils_version = System.getProperty("common_utils.version", '3.1.0.0-SNAPSHOT')
 
-        kafka_version  = '3.7.1'
+        kafka_version  = '4.0.0'
         open_saml_version = '5.1.4'
         open_saml_shib_version = "9.1.4"
         one_login_java_saml = '2.9.0'
         jjwt_version = '0.12.6'
         guava_version = '33.4.8-jre'
         jaxb_version = '2.3.9'
-        spring_version = '6.2.6'
-        bouncycastle_version = '1.80'
+        spring_version = '6.2.7'
+        bouncycastle_version = '1.81'
 
         if (buildVersionQualifier) {
             opensearch_build += "-${buildVersionQualifier}"
@@ -479,12 +479,12 @@ configurations {
             force "org.apache.httpcomponents:httpclient:4.5.14"
             force "org.apache.httpcomponents:httpcore:4.4.16"
             force "com.google.errorprone:error_prone_annotations:2.38.0"
-            force "org.checkerframework:checker-qual:3.49.3"
+            force "org.checkerframework:checker-qual:3.49.4"
             force "ch.qos.logback:logback-classic:1.5.18"
             force "commons-io:commons-io:2.19.0"
             force "com.carrotsearch.randomizedtesting:randomizedtesting-runner:2.8.3"
             force "org.hamcrest:hamcrest:2.2"
-            force "org.mockito:mockito-core:5.17.0"
+            force "org.mockito:mockito-core:5.18.0"
             force "net.bytebuddy:byte-buddy:1.17.5"
             force "org.ow2.asm:asm:9.8"
             force "com.google.j2objc:j2objc-annotations:3.0.0"
@@ -515,7 +515,7 @@ allprojects {
         testImplementation 'org.hamcrest:hamcrest:2.2'
         testImplementation 'junit:junit:4.13.2'
         testImplementation "org.opensearch:opensearch:${opensearch_version}"
-        testImplementation "org.mockito:mockito-core:5.17.0"
+        testImplementation "org.mockito:mockito-core:5.18.0"
 
         //integration test framework:
         integrationTestImplementation('com.carrotsearch.randomizedtesting:randomizedtesting-runner:2.8.3') {
@@ -542,7 +542,7 @@ allprojects {
         integrationTestImplementation "org.apache.httpcomponents:fluent-hc:4.5.14"
         integrationTestImplementation "org.apache.httpcomponents:httpcore:4.4.16"
         integrationTestImplementation "org.apache.httpcomponents:httpasyncclient:4.1.5"
-        integrationTestImplementation "org.mockito:mockito-core:5.17.0"
+        integrationTestImplementation "org.mockito:mockito-core:5.18.0"
         integrationTestImplementation "org.passay:passay:1.6.6"
         integrationTestImplementation "org.opensearch:opensearch:${opensearch_version}"
         integrationTestImplementation "org.opensearch.plugin:transport-netty4-client:${opensearch_version}"
@@ -693,7 +693,7 @@ dependencies {
     implementation "com.onelogin:java-saml:${one_login_java_saml}"
     implementation "com.onelogin:java-saml-core:${one_login_java_saml}"
     //OpenSAML
-    runtimeOnly "io.dropwizard.metrics:metrics-core:4.2.30"
+    runtimeOnly "io.dropwizard.metrics:metrics-core:4.2.32"
     implementation "net.shibboleth:shib-support:${open_saml_shib_version}"
     implementation "net.shibboleth:shib-security:${open_saml_shib_version}"
     implementation "net.shibboleth:shib-networking:${open_saml_shib_version}"
@@ -732,7 +732,7 @@ dependencies {
     runtimeOnly 'org.apache.ws.xmlschema:xmlschema-core:2.3.1'
     runtimeOnly 'org.apache.santuario:xmlsec:2.3.5'
     runtimeOnly "com.github.luben:zstd-jni:${versions.zstd}"
-    runtimeOnly 'org.checkerframework:checker-qual:3.49.3'
+    runtimeOnly 'org.checkerframework:checker-qual:3.49.4'
     runtimeOnly "org.bouncycastle:bcpkix-jdk18on:${bouncycastle_version}"
     runtimeOnly 'org.scala-lang.modules:scala-java8-compat_3:1.0.2'
 
@@ -754,18 +754,25 @@ dependencies {
     testImplementation 'com.unboundid:unboundid-ldapsdk:4.0.14'
     testImplementation 'org.apache.httpcomponents:fluent-hc:4.5.14'
     testImplementation "org.apache.httpcomponents.client5:httpclient5-fluent:${versions.httpclient5}"
+    testImplementation "com.google.re2j:re2j:1.8"
     testImplementation "org.apache.kafka:kafka_2.13:${kafka_version}"
     testImplementation "org.apache.kafka:kafka-server:${kafka_version}"
     testImplementation "org.apache.kafka:kafka-server-common:${kafka_version}"
     testImplementation "org.apache.kafka:kafka-server-common:${kafka_version}:test"
     testImplementation "org.apache.kafka:kafka-group-coordinator:${kafka_version}"
     testImplementation "org.apache.kafka:kafka_2.13:${kafka_version}:test"
     testImplementation "org.apache.kafka:kafka-clients:${kafka_version}:test"
+    testImplementation "org.apache.kafka:test-common:${kafka_version}"
+    testImplementation "org.apache.kafka:kafka-coordinator-common:${kafka_version}"
+    testImplementation "org.apache.kafka:kafka-group-coordinator-api:${kafka_version}"
+    testImplementation "org.apache.kafka:kafka-share-coordinator:${kafka_version}"
+    testImplementation "org.apache.kafka:kafka-test-common-runtime:${kafka_version}"
+    testImplementation "org.apache.kafka:kafka-test-common-internal-api:${kafka_version}"
     testImplementation 'commons-validator:commons-validator:1.9.0'
-    testImplementation 'org.springframework.kafka:spring-kafka-test:3.3.5'
+    testImplementation "org.springframework.kafka:spring-kafka-test:4.0.0-M2"
     testImplementation "org.springframework:spring-beans:${spring_version}"
-    testImplementation 'org.junit.jupiter:junit-jupiter:5.12.2'
-    testImplementation 'org.junit.jupiter:junit-jupiter-api:5.12.2'
+    testImplementation 'org.junit.jupiter:junit-jupiter:5.13.1'
+    testImplementation 'org.junit.jupiter:junit-jupiter-api:5.13.1'
     testImplementation('org.awaitility:awaitility:4.3.0') {
         exclude(group: 'org.hamcrest', module: 'hamcrest')
     }

config/roles.yml

@@ -460,14 +460,77 @@ query_insights_full_access:
 
 # Allow users to execute read only LTR actions
 ltr_read_access:
-    reserved: true
-    cluster_permissions:
-      - cluster:admin/ltr/caches/stats
-      - cluster:admin/ltr/featurestore/list
-      - cluster:admin/ltr/stats
+  reserved: true
+  cluster_permissions:
+    - cluster:admin/ltr/caches/stats
+    - cluster:admin/ltr/featurestore/list
+    - cluster:admin/ltr/stats
 
 # Allow users to execute all LTR actions
 ltr_full_access:
-    reserved: true
-    cluster_permissions:
-      - cluster:admin/ltr/*
+  reserved: true
+  cluster_permissions:
+    - cluster:admin/ltr/*
+
+# Allow users to use all Search Relevance functionalities
+search_relevance_full_access:
+  reserved: true
+  cluster_permissions:
+    - 'cluster:admin/opensearch/search_relevance/*'
+  index_permissions:
+    - index_patterns:
+        - '*'
+      allowed_actions:
+        - 'indices:admin/mappings/get'
+        - 'indices:data/read/search*'
+
+# Allow users to read Search Relevance resources
+search_relevance_read_access:
+  reserved: true
+  cluster_permissions:
+    - 'cluster:admin/opensearch/search_relevance/experiment/get'
+    - 'cluster:admin/opensearch/search_relevance/judgment/get'
+    - 'cluster:admin/opensearch/search_relevance/queryset/get'
+    - 'cluster:admin/opensearch/search_relevance/search_configuration/get'
+
+# Allow users to read Forecast resources
+forecast_read_access:
+  reserved: true
+  cluster_permissions:
+    - 'cluster:admin/plugin/forecast/forecaster/info'
+    - 'cluster:admin/plugin/forecast/forecaster/stats'
+    - 'cluster:admin/plugin/forecast/forecaster/suggest'
+    - 'cluster:admin/plugin/forecast/forecaster/validate'
+    - 'cluster:admin/plugin/forecast/forecasters/get'
+    - 'cluster:admin/plugin/forecast/forecasters/info'
+    - 'cluster:admin/plugin/forecast/forecasters/search'
+    - 'cluster:admin/plugin/forecast/result/topForecasts'
+    - 'cluster:admin/plugin/forecast/tasks/search'
+  index_permissions:
+    - index_patterns:
+        - 'opensearch-forecast-result*'
+      allowed_actions:
+        - 'indices:admin/mappings/fields/get*'
+        - 'indices:admin/resolve/index'
+        - 'indices:data/read*'
+
+# Allows users to use all Forecasting functionality
+forecast_full_access:
+  reserved: true
+  cluster_permissions:
+    - 'cluster:admin/plugin/forecast/*'
+    - 'cluster:admin/settings/update'
+  index_permissions:
+    - index_patterns:
+        - '*'
+      allowed_actions:
+        - 'indices:admin/aliases/get'
+        - 'indices:admin/mapping/get'
+        - 'indices:admin/mapping/put'
+        - 'indices:admin/mappings/fields/get*'
+        - 'indices:admin/resolve/index'
+        - 'indices:data/read*'
+        - 'indices:data/read/field_caps*'
+        - 'indices:data/read/search'
+        - 'indices:data/write*'
+        - 'indices_monitor'