Merge branch 'main' into fix_constant_keyword

Signed-off-by: kkewwei <kkewwei@163.com>
opensearch-project · Jul 13, 2024 · daa4cbb · daa4cbb
2 parents 989979a + 6b8b3ef
commit daa4cbb
Show file tree

Hide file tree

Showing 45 changed files with 2,435 additions and 473 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -12,16 +12,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Workload Management] Add QueryGroup schema ([13669](https://github.com/opensearch-project/OpenSearch/pull/13669))
 - Add batching supported processor base type AbstractBatchingProcessor ([#14554](https://github.com/opensearch-project/OpenSearch/pull/14554))
 - Fix race condition while parsing derived fields from search definition ([14445](https://github.com/opensearch-project/OpenSearch/pull/14445))
+- Add `strict_allow_templates` dynamic mapping option ([#14555](https://github.com/opensearch-project/OpenSearch/pull/14555))
 - Add allowlist setting for ingest-common and search-pipeline-common processors ([#14439](https://github.com/opensearch-project/OpenSearch/issues/14439))
 - Create SystemIndexRegistry with helper method matchesSystemIndex ([#14415](https://github.com/opensearch-project/OpenSearch/pull/14415))
+- Print reason why parent task was cancelled ([#14604](https://github.com/opensearch-project/OpenSearch/issues/14604))
 
 ### Dependencies
 - Bump `org.gradle.test-retry` from 1.5.8 to 1.5.9 ([#13442](https://github.com/opensearch-project/OpenSearch/pull/13442))
 - Update to Apache Lucene 9.11.0 ([#14042](https://github.com/opensearch-project/OpenSearch/pull/14042))
 - Bump `netty` from 4.1.110.Final to 4.1.111.Final ([#14356](https://github.com/opensearch-project/OpenSearch/pull/14356))
 - Bump `org.wiremock:wiremock-standalone` from 3.3.1 to 3.6.0 ([#14361](https://github.com/opensearch-project/OpenSearch/pull/14361))
-- Bump `reactor` from 3.5.17 to 3.5.18 ([#14395](https://github.com/opensearch-project/OpenSearch/pull/14395))
-- Bump `reactor-netty` from 1.1.19 to 1.1.20 ([#14395](https://github.com/opensearch-project/OpenSearch/pull/14395))
+- Bump `reactor` from 3.5.17 to 3.5.19 ([#14395](https://github.com/opensearch-project/OpenSearch/pull/14395), [#14697](https://github.com/opensearch-project/OpenSearch/pull/14697))
+- Bump `reactor-netty` from 1.1.19 to 1.1.21 ([#14395](https://github.com/opensearch-project/OpenSearch/pull/14395), [#14697](https://github.com/opensearch-project/OpenSearch/pull/14697))
 - Bump `commons-net:commons-net` from 3.10.0 to 3.11.1 ([#14396](https://github.com/opensearch-project/OpenSearch/pull/14396))
 - Bump `com.nimbusds:nimbus-jose-jwt` from 9.37.3 to 9.40 ([#14398](https://github.com/opensearch-project/OpenSearch/pull/14398))
 - Bump `org.apache.commons:commons-configuration2` from 2.10.1 to 2.11.0 ([#14399](https://github.com/opensearch-project/OpenSearch/pull/14399))
@@ -64,6 +66,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Handle NPE in GetResult if "found" field is missing ([#14552](https://github.com/opensearch-project/OpenSearch/pull/14552))
 - Refactoring FilterPath.parse by using an iterative approach ([#14200](https://github.com/opensearch-project/OpenSearch/pull/14200))
 - Refactoring Grok.validatePatternBank by using an iterative approach ([#14206](https://github.com/opensearch-project/OpenSearch/pull/14206))
+- Update help output for _cat ([#14722](https://github.com/opensearch-project/OpenSearch/pull/14722))
 - Fix constant_keyword field type ([#14651](https://github.com/opensearch-project/OpenSearch/pull/14651))
 
 ### Security

diff --git a/buildSrc/src/main/java/org/opensearch/gradle/test/rest/RestResourcesPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/test/rest/RestResourcesPlugin.java
@@ -81,50 +81,52 @@ public void apply(Project project) {
         // tests
         Configuration testConfig = project.getConfigurations().create("restTestConfig");
         project.getConfigurations().create("restTests");
+
+        if (BuildParams.isInternal()) {
+            // core
+            Dependency restTestdependency = project.getDependencies().project(new HashMap<String, String>() {
+                {
+                    put("path", ":rest-api-spec");
+                    put("configuration", "restTests");
+                }
+            });
+            testConfig.withDependencies(s -> s.add(restTestdependency));
+        } else {
+            Dependency dependency = project.getDependencies().create("org.opensearch:rest-api-spec:" + VersionProperties.getOpenSearch());
+            testConfig.withDependencies(s -> s.add(dependency));
+        }
+
         Provider<CopyRestTestsTask> copyRestYamlTestTask = project.getTasks()
             .register("copyYamlTestsTask", CopyRestTestsTask.class, task -> {
                 task.includeCore.set(extension.restTests.getIncludeCore());
                 task.coreConfig = testConfig;
                 task.sourceSetName = SourceSet.TEST_SOURCE_SET_NAME;
-                if (BuildParams.isInternal()) {
-                    // core
-                    Dependency restTestdependency = project.getDependencies().project(new HashMap<String, String>() {
-                        {
-                            put("path", ":rest-api-spec");
-                            put("configuration", "restTests");
-                        }
-                    });
-                    project.getDependencies().add(task.coreConfig.getName(), restTestdependency);
-                } else {
-                    Dependency dependency = project.getDependencies()
-                        .create("org.opensearch:rest-api-spec:" + VersionProperties.getOpenSearch());
-                    project.getDependencies().add(task.coreConfig.getName(), dependency);
-                }
                 task.dependsOn(task.coreConfig);
             });
 
         // api
         Configuration specConfig = project.getConfigurations().create("restSpec"); // name chosen for passivity
         project.getConfigurations().create("restSpecs");
+
+        if (BuildParams.isInternal()) {
+            Dependency restSpecDependency = project.getDependencies().project(new HashMap<String, String>() {
+                {
+                    put("path", ":rest-api-spec");
+                    put("configuration", "restSpecs");
+                }
+            });
+            specConfig.withDependencies(s -> s.add(restSpecDependency));
+        } else {
+            Dependency dependency = project.getDependencies().create("org.opensearch:rest-api-spec:" + VersionProperties.getOpenSearch());
+            specConfig.withDependencies(s -> s.add(dependency));
+        }
+
         Provider<CopyRestApiTask> copyRestYamlSpecTask = project.getTasks()
             .register("copyRestApiSpecsTask", CopyRestApiTask.class, task -> {
                 task.includeCore.set(extension.restApi.getIncludeCore());
                 task.dependsOn(copyRestYamlTestTask);
                 task.coreConfig = specConfig;
                 task.sourceSetName = SourceSet.TEST_SOURCE_SET_NAME;
-                if (BuildParams.isInternal()) {
-                    Dependency restSpecDependency = project.getDependencies().project(new HashMap<String, String>() {
-                        {
-                            put("path", ":rest-api-spec");
-                            put("configuration", "restSpecs");
-                        }
-                    });
-                    project.getDependencies().add(task.coreConfig.getName(), restSpecDependency);
-                } else {
-                    Dependency dependency = project.getDependencies()
-                        .create("org.opensearch:rest-api-spec:" + VersionProperties.getOpenSearch());
-                    project.getDependencies().add(task.coreConfig.getName(), dependency);
-                }
                 task.dependsOn(task.coreConfig);
             });
 

diff --git a/buildSrc/version.properties b/buildSrc/version.properties
@@ -33,8 +33,8 @@ netty             = 4.1.111.Final
 joda              = 2.12.7
 
 # project reactor
-reactor_netty     = 1.1.20
-reactor           = 3.5.18
+reactor_netty     = 1.1.21
+reactor           = 3.5.19
 
 # client dependencies
 httpclient5       = 5.2.1

diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar
diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
@@ -11,7 +11,7 @@
 
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.8-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.9-all.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionSha256Sum=f8b4f4772d302c8ff580bc40d0f56e715de69b163546944f787c87abf209c961
+distributionSha256Sum=258e722ec21e955201e31447b0aed14201765a3bfbae296a46cf60b70e66db70
diff --git a/gradlew b/gradlew
@@ -15,6 +15,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+# SPDX-License-Identifier: Apache-2.0
+#
 
 ##############################################################################
 #
@@ -55,7 +57,7 @@
 #       Darwin, MinGW, and NonStop.
 #
 #   (3) This script is generated from the Groovy template
-#       https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
+#       https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
 #       within the Gradle project.
 #
 #       You can find Gradle at https://github.com/gradle/gradle/.
@@ -84,7 +86,8 @@ done
 # shellcheck disable=SC2034
 APP_BASE_NAME=${0##*/}
 # Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
-APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit
+APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s
+' "$PWD" ) || exit
 
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD=maximum

diff --git a/gradlew.bat b/gradlew.bat
@@ -13,6 +13,8 @@
 @rem See the License for the specific language governing permissions and
 @rem limitations under the License.
 @rem
+@rem SPDX-License-Identifier: Apache-2.0
+@rem
 
 @if "%DEBUG%"=="" @echo off
 @rem ##########################################################################

diff --git a/plugins/repository-azure/licenses/reactor-netty-core-1.1.20.jar.sha1 b/plugins/repository-azure/licenses/reactor-netty-core-1.1.20.jar.sha1
diff --git a/plugins/repository-azure/licenses/reactor-netty-core-1.1.21.jar.sha1 b/plugins/repository-azure/licenses/reactor-netty-core-1.1.21.jar.sha1
@@ -0,0 +1 @@
+acb98bd08107287c454ce74e7b1ed8e7a018a662
diff --git a/plugins/repository-azure/licenses/reactor-netty-http-1.1.20.jar.sha1 b/plugins/repository-azure/licenses/reactor-netty-http-1.1.20.jar.sha1
diff --git a/plugins/repository-azure/licenses/reactor-netty-http-1.1.21.jar.sha1 b/plugins/repository-azure/licenses/reactor-netty-http-1.1.21.jar.sha1
@@ -0,0 +1 @@
+b83542bb35630ef815b4177e3c670f62e952e695
diff --git a/plugins/transport-reactor-netty4/licenses/reactor-netty-core-1.1.20.jar.sha1 b/plugins/transport-reactor-netty4/licenses/reactor-netty-core-1.1.20.jar.sha1
diff --git a/plugins/transport-reactor-netty4/licenses/reactor-netty-core-1.1.21.jar.sha1 b/plugins/transport-reactor-netty4/licenses/reactor-netty-core-1.1.21.jar.sha1
@@ -0,0 +1 @@
+acb98bd08107287c454ce74e7b1ed8e7a018a662
diff --git a/plugins/transport-reactor-netty4/licenses/reactor-netty-http-1.1.20.jar.sha1 b/plugins/transport-reactor-netty4/licenses/reactor-netty-http-1.1.20.jar.sha1
diff --git a/plugins/transport-reactor-netty4/licenses/reactor-netty-http-1.1.21.jar.sha1 b/plugins/transport-reactor-netty4/licenses/reactor-netty-http-1.1.21.jar.sha1
@@ -0,0 +1 @@
+b83542bb35630ef815b4177e3c670f62e952e695
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/110_strict_allow_templates.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/110_strict_allow_templates.yml
@@ -0,0 +1,155 @@
+---
+"Index documents with setting dynamic parameter to strict_allow_templates in the mapping of the index":
+  - skip:
+      version: " - 2.99.99"
+      reason: "introduced in 3.0.0"
+
+  - do:
+      indices.create:
+        index:    test_1
+        body:
+          mappings:
+            dynamic: strict_allow_templates
+            dynamic_templates: [
+              {
+                strings: {
+                  "match": "stringField*",
+                  "match_mapping_type": "string",
+                  "mapping": {
+                    "type": "keyword"
+                  }
+                }
+              },
+              {
+                object: {
+                  "match": "objectField*",
+                  "match_mapping_type": "object",
+                  "mapping": {
+                    "type": "object",
+                    "properties": {
+                      "bar1": {
+                        "type": "keyword"
+                      },
+                      "bar2": {
+                        "type": "text"
+                      }
+                    }
+                  }
+                }
+              },
+              {
+                boolean: {
+                  "match": "booleanField*",
+                  "match_mapping_type": "boolean",
+                  "mapping": {
+                    "type": "boolean"
+                  }
+                }
+              },
+              {
+                double: {
+                  "match": "doubleField*",
+                  "match_mapping_type": "double",
+                  "mapping": {
+                    "type": "double"
+                  }
+                }
+              },
+              {
+                long: {
+                  "match": "longField*",
+                  "match_mapping_type": "long",
+                  "mapping": {
+                    "type": "long"
+                  }
+                }
+              },
+              {
+                array: {
+                  "match": "arrayField*",
+                  "mapping": {
+                    "type": "keyword"
+                  }
+                }
+              },
+              {
+                date: {
+                  "match": "dateField*",
+                  "match_mapping_type": "date",
+                  "mapping": {
+                    "type": "date"
+                  }
+                }
+              }
+            ]
+            properties:
+              test1:
+                type: text
+
+  - do:
+      catch: /mapping set to strict_allow_templates, dynamic introduction of \[test2\] within \[\_doc\] is not allowed/
+      index:
+        index:   test_1
+        id:      1
+        body:    {
+          stringField: bar,
+          objectField: {
+            bar1: "bar1",
+            bar2: "bar2"
+          },
+          test1: test1,
+          test2: test2
+        }
+
+  - do:
+      index:
+        index:   test_1
+        id:      1
+        body:    {
+          stringField: bar,
+          objectField: {
+            bar1: "bar1",
+            bar2: "bar2"
+          },
+          booleanField: true,
+          doubleField: 1.0,
+          longField: 100,
+          arrayField: ["1","2"],
+          dateField: "2024-06-25T05:11:51.243Z",
+          test1: test1
+        }
+
+  - do:
+      get:
+        index: test_1
+        id: 1
+  - match: { _source: {
+      stringField: bar,
+      objectField: {
+        bar1: "bar1",
+        bar2: "bar2"
+      },
+      booleanField: true,
+      doubleField: 1.0,
+      longField: 100,
+      arrayField: [ "1","2" ],
+      dateField: "2024-06-25T05:11:51.243Z",
+      test1: test1
+    }
+  }
+
+  - do:
+      indices.get_mapping: {
+        index: test_1
+      }
+
+  - match: {test_1.mappings.dynamic:     strict_allow_templates}
+  - match: {test_1.mappings.properties.stringField.type:     keyword}
+  - match: {test_1.mappings.properties.objectField.properties.bar1.type:     keyword}
+  - match: {test_1.mappings.properties.objectField.properties.bar2.type:     text}
+  - match: {test_1.mappings.properties.booleanField.type:     boolean}
+  - match: {test_1.mappings.properties.doubleField.type:     double}
+  - match: {test_1.mappings.properties.longField.type:     long}
+  - match: {test_1.mappings.properties.arrayField.type:     keyword}
+  - match: {test_1.mappings.properties.dateField.type:     date}
+  - match: {test_1.mappings.properties.test1.type:     text}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/all_path_options.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/all_path_options.yml
@@ -159,3 +159,34 @@ setup:
       indices.get_mapping: {}
 
   - match: {test_index1.mappings.properties.text.type:     text}
+
+---
+"post a mapping with setting dynamic to strict_allow_templates":
+  - skip:
+      version: " - 2.99.99"
+      reason: "introduced in 3.0.0"
+  - do:
+      indices.put_mapping:
+        index: test_index1
+        body:
+          dynamic: strict_allow_templates
+          dynamic_templates: [
+            {
+              strings: {
+                "match": "foo*",
+                "match_mapping_type": "string",
+                "mapping": {
+                  "type": "keyword"
+                }
+              }
+            }
+          ]
+          properties:
+            test1:
+              type: text
+
+  - do:
+      indices.get_mapping: {}
+
+  - match: {test_index1.mappings.dynamic:     strict_allow_templates}
+  - match: {test_index1.mappings.properties.test1.type:     text}