Merge branch 'main' into replace-master-int-server-internalclustertest

Signed-off-by: Tianli Feng <ftianli@amazon.com>

.ci/bwcVersions

@@ -37,4 +37,7 @@ BWC_VERSION:
   - "1.2.4"
   - "1.2.5"
   - "1.3.0"
-  - "1.4.0"
+  - "1.3.1"
+  - "1.3.2"
+  - "2.0.0"
+  - "2.1.0"

.github/workflows/version.yml

@@ -59,6 +59,7 @@ jobs:
           base: ${{ env.BASE }}
           branch: 'create-pull-request/patch-${{ env.BASE }}'
           commit-message: Incremented version to ${{ env.NEXT_VERSION }}
+          signoff: true
           delete-branch: true
           title: '[AUTO] Incremented version to ${{ env.NEXT_VERSION }}.'
           body: |
@@ -83,6 +84,7 @@ jobs:
           base: ${{ env.BASE_X }}
           branch: 'create-pull-request/patch-${{ env.BASE_X }}'
           commit-message: Added bwc version ${{ env.NEXT_VERSION }}
+          signoff: true
           delete-branch: true
           title: '[AUTO] [${{ env.BASE_X }}] Added bwc version ${{ env.NEXT_VERSION }}.'
           body: |
@@ -107,6 +109,7 @@ jobs:
           base: main
           branch: 'create-pull-request/patch-main'
           commit-message: Added bwc version ${{ env.NEXT_VERSION }}
+          signoff: true
           delete-branch: true
           title: '[AUTO] [main] Added bwc version ${{ env.NEXT_VERSION }}.'
           body: |

README.md

@@ -6,6 +6,7 @@
 - [Security](#security)
 - [License](#license)
 - [Copyright](#copyright)
+- [Trademark](#trademark)
 
 ## Welcome!
 

build.gradle

@@ -48,7 +48,8 @@ plugins {
   id 'lifecycle-base'
   id 'opensearch.docker-support'
   id 'opensearch.global-build-info'
-  id "com.diffplug.spotless" version "6.3.0" apply false
+  id "com.diffplug.spotless" version "6.4.1" apply false
+  id "org.gradle.test-retry" version "1.3.1" apply false
 }
 
 apply from: 'gradle/build-complete.gradle'
@@ -232,7 +233,7 @@ allprojects {
   tasks.withType(JavaCompile).configureEach { JavaCompile compile ->
     // See please https://bugs.openjdk.java.net/browse/JDK-8209058
     if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_11) {
-        compile.options.compilerArgs << '-Werror'
+      compile.options.compilerArgs << '-Werror'
     }
     compile.options.compilerArgs << '-Xlint:auxiliaryclass'
     compile.options.compilerArgs << '-Xlint:cast'
@@ -386,6 +387,18 @@ gradle.projectsEvaluated {
   }
 }
 
+// test retry configuration
+subprojects {
+  apply plugin: "org.gradle.test-retry"
+  tasks.withType(Test).configureEach {
+    retry {
+      failOnPassedAfterRetry = false
+      maxRetries = 3
+      maxFailures = 10
+    }
+  }
+}
+
 // eclipse configuration
 allprojects {
   apply plugin: 'eclipse'
@@ -445,9 +458,9 @@ allprojects {
   tasks.named('eclipse') { dependsOn 'cleanEclipse', 'copyEclipseSettings' }
 
   afterEvaluate {
-     tasks.findByName("eclipseJdt")?.configure {
-         dependsOn 'copyEclipseSettings'
-     }
+    tasks.findByName("eclipseJdt")?.configure {
+      dependsOn 'copyEclipseSettings'
+    }
   }
 }
 

buildSrc/build.gradle

@@ -110,19 +110,19 @@ dependencies {
   api 'com.netflix.nebula:gradle-info-plugin:7.1.3'
   api 'org.apache.rat:apache-rat:0.13'
   api 'commons-io:commons-io:2.7'
-  api "net.java.dev.jna:jna:5.5.0"
+  api "net.java.dev.jna:jna:5.10.0"
   api 'gradle.plugin.com.github.johnrengelman:shadow:7.1.2'
-  api 'de.thetaphi:forbiddenapis:3.2'
-  api 'com.avast.gradle:gradle-docker-compose-plugin:0.14.12'
+  api 'de.thetaphi:forbiddenapis:3.3'
+  api 'com.avast.gradle:gradle-docker-compose-plugin:0.15.2'
   api 'org.apache.maven:maven-model:3.6.2'
-  api 'com.networknt:json-schema-validator:1.0.67'
-  api "com.fasterxml.jackson.core:jackson-databind:${props.getProperty('jackson')}"
+  api 'com.networknt:json-schema-validator:1.0.68'
+  api "com.fasterxml.jackson.core:jackson-databind:${props.getProperty('jackson_databind')}"
 
   testFixturesApi "junit:junit:${props.getProperty('junit')}"
   testFixturesApi "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${props.getProperty('randomizedrunner')}"
   testFixturesApi gradleApi()
   testFixturesApi gradleTestKit()
-  testImplementation 'com.github.tomakehurst:wiremock-jre8-standalone:2.23.2'
+  testImplementation 'com.github.tomakehurst:wiremock-jre8-standalone:2.32.0'
   testImplementation "org.mockito:mockito-core:${props.getProperty('mockito')}"
   integTestImplementation('org.spockframework:spock-core:2.1-groovy-3.0') {
     exclude module: "groovy"

buildSrc/reaper/build.gradle

@@ -11,6 +11,9 @@
 
 apply plugin: 'java'
 
+targetCompatibility = JavaVersion.VERSION_11
+sourceCompatibility = JavaVersion.VERSION_11
+
 jar {
   archiveFileName = "${project.name}.jar"
   manifest {

buildSrc/src/main/groovy/org/opensearch/gradle/precommit/LicenseHeadersTask.groovy

@@ -35,7 +35,10 @@ import org.opensearch.gradle.AntTask
 import org.gradle.api.file.FileCollection
 import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.InputFiles
+import org.gradle.api.tasks.IgnoreEmptyDirectories;
 import org.gradle.api.tasks.OutputFile
+import org.gradle.api.tasks.PathSensitive
+import org.gradle.api.tasks.PathSensitivity
 import org.gradle.api.tasks.SkipWhenEmpty
 
 import java.nio.file.Files
@@ -78,6 +81,8 @@ class LicenseHeadersTask extends AntTask {
      */
     @InputFiles
     @SkipWhenEmpty
+    @IgnoreEmptyDirectories
+    @PathSensitive(PathSensitivity.RELATIVE)
     List<FileCollection> getJavaFiles() {
         return project.sourceSets.collect({it.allJava})
     }

buildSrc/src/main/java/org/opensearch/gradle/Version.java

@@ -77,7 +77,7 @@ public Version(int major, int minor, int revision) {
         // currently snapshot is not taken into account
         int id = major * 10000000 + minor * 100000 + revision * 1000;
         // identify if new OpenSearch version 1
-        this.id = major == 1 || major == 2 ? id ^ MASK : id;
+        this.id = major == 1 || major == 2 || major == 3 ? id ^ MASK : id;
     }
 
     private static int parseSuffixNumber(String substring) {

buildSrc/src/main/java/org/opensearch/gradle/precommit/FilePermissionsTask.java

@@ -46,8 +46,11 @@
 import org.gradle.api.GradleException;
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.file.FileTree;
+import org.gradle.api.tasks.IgnoreEmptyDirectories;
 import org.gradle.api.tasks.InputFiles;
 import org.gradle.api.tasks.OutputFile;
+import org.gradle.api.tasks.PathSensitive;
+import org.gradle.api.tasks.PathSensitivity;
 import org.gradle.api.tasks.SkipWhenEmpty;
 import org.gradle.api.tasks.StopExecutionException;
 import org.gradle.api.tasks.TaskAction;
@@ -92,6 +95,8 @@ private static boolean isExecutableFile(File file) {
      */
     @InputFiles
     @SkipWhenEmpty
+    @IgnoreEmptyDirectories
+    @PathSensitive(PathSensitivity.RELATIVE)
     public FileCollection getFiles() {
         return GradleUtils.getJavaSourceSets(getProject())
             .stream()

buildSrc/src/main/java/org/opensearch/gradle/precommit/ForbiddenPatternsTask.java

@@ -37,9 +37,12 @@
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.file.FileTree;
 import org.gradle.api.plugins.JavaPluginConvention;
+import org.gradle.api.tasks.IgnoreEmptyDirectories;
 import org.gradle.api.tasks.Input;
 import org.gradle.api.tasks.InputFiles;
 import org.gradle.api.tasks.OutputFile;
+import org.gradle.api.tasks.PathSensitive;
+import org.gradle.api.tasks.PathSensitivity;
 import org.gradle.api.tasks.SkipWhenEmpty;
 import org.gradle.api.tasks.TaskAction;
 import org.gradle.api.tasks.util.PatternFilterable;
@@ -100,6 +103,8 @@ public ForbiddenPatternsTask() {
 
     @InputFiles
     @SkipWhenEmpty
+    @IgnoreEmptyDirectories
+    @PathSensitive(PathSensitivity.RELATIVE)
     public FileCollection getFiles() {
         return getProject().getConvention()
             .getPlugin(JavaPluginConvention.class)

buildSrc/src/main/java/org/opensearch/gradle/precommit/LoggerUsageTask.java

@@ -37,6 +37,7 @@
 import org.gradle.api.plugins.JavaPluginConvention;
 import org.gradle.api.tasks.CacheableTask;
 import org.gradle.api.tasks.Classpath;
+import org.gradle.api.tasks.IgnoreEmptyDirectories;
 import org.gradle.api.tasks.InputFiles;
 import org.gradle.api.tasks.PathSensitive;
 import org.gradle.api.tasks.PathSensitivity;
@@ -79,6 +80,7 @@ public void setClasspath(FileCollection classpath) {
     @InputFiles
     @PathSensitive(PathSensitivity.RELATIVE)
     @SkipWhenEmpty
+    @IgnoreEmptyDirectories
     public FileCollection getClassDirectories() {
         return getProject().getConvention()
             .getPlugin(JavaPluginConvention.class)

buildSrc/src/main/java/org/opensearch/gradle/precommit/ThirdPartyAuditTask.java

@@ -47,6 +47,7 @@
 import org.gradle.api.tasks.CacheableTask;
 import org.gradle.api.tasks.Classpath;
 import org.gradle.api.tasks.CompileClasspath;
+import org.gradle.api.tasks.IgnoreEmptyDirectories;
 import org.gradle.api.tasks.Input;
 import org.gradle.api.tasks.InputFile;
 import org.gradle.api.tasks.InputFiles;
@@ -195,6 +196,7 @@ public Set<String> getMissingClassExcludes() {
 
     @Classpath
     @SkipWhenEmpty
+    @IgnoreEmptyDirectories
     public Set<File> getJarsToScan() {
         // These are SelfResolvingDependency, and some of them backed by file collections, like the Gradle API files,
         // or dependencies added as `files(...)`, we can't be sure if those are third party or not.

buildSrc/src/main/java/org/opensearch/gradle/test/rest/CopyRestApiTask.java

@@ -43,9 +43,12 @@
 import org.gradle.api.file.FileTree;
 import org.gradle.api.plugins.JavaPluginConvention;
 import org.gradle.api.provider.ListProperty;
+import org.gradle.api.tasks.IgnoreEmptyDirectories;
 import org.gradle.api.tasks.Input;
 import org.gradle.api.tasks.InputFiles;
 import org.gradle.api.tasks.OutputDirectory;
+import org.gradle.api.tasks.PathSensitive;
+import org.gradle.api.tasks.PathSensitivity;
 import org.gradle.api.tasks.SkipWhenEmpty;
 import org.gradle.api.tasks.SourceSet;
 import org.gradle.api.tasks.TaskAction;
@@ -112,8 +115,10 @@ public boolean isSkipHasRestTestCheck() {
         return skipHasRestTestCheck;
     }
 
+    @IgnoreEmptyDirectories
     @SkipWhenEmpty
     @InputFiles
+    @PathSensitive(PathSensitivity.RELATIVE)
     public FileTree getInputDir() {
         FileTree coreFileTree = null;
         boolean projectHasYamlRestTests = skipHasRestTestCheck || projectHasYamlRestTests();

buildSrc/src/main/java/org/opensearch/gradle/test/rest/CopyRestTestsTask.java

@@ -43,9 +43,12 @@
 import org.gradle.api.file.FileTree;
 import org.gradle.api.plugins.JavaPluginConvention;
 import org.gradle.api.provider.ListProperty;
+import org.gradle.api.tasks.IgnoreEmptyDirectories;
 import org.gradle.api.tasks.Input;
 import org.gradle.api.tasks.InputFiles;
 import org.gradle.api.tasks.OutputDirectory;
+import org.gradle.api.tasks.PathSensitive;
+import org.gradle.api.tasks.PathSensitivity;
 import org.gradle.api.tasks.SkipWhenEmpty;
 import org.gradle.api.tasks.SourceSet;
 import org.gradle.api.tasks.TaskAction;
@@ -104,8 +107,10 @@ String getSourceSetName() {
         return sourceSetName;
     }
 
+    @IgnoreEmptyDirectories
     @SkipWhenEmpty
     @InputFiles
+    @PathSensitive(PathSensitivity.RELATIVE)
     public FileTree getInputDir() {
         FileTree coreFileTree = null;
         if (includeCore.get().isEmpty() == false) {

buildSrc/src/main/resources/minimumGradleVersion

@@ -1 +1 @@
-6.6.1
+7.4.1

buildSrc/src/test/java/org/opensearch/gradle/DistributionDownloadPluginTests.java

@@ -53,11 +53,11 @@ public class DistributionDownloadPluginTests extends GradleUnitTestCase {
     private static Project packagesProject;
     private static Project bwcProject;
 
-    private static final Version BWC_MAJOR_VERSION = Version.fromString("5.0.0");
-    private static final Version BWC_MINOR_VERSION = Version.fromString("4.1.0");
-    private static final Version BWC_STAGED_VERSION = Version.fromString("4.0.0");
-    private static final Version BWC_BUGFIX_VERSION = Version.fromString("4.0.1");
-    private static final Version BWC_MAINTENANCE_VERSION = Version.fromString("3.90.1");
+    private static final Version BWC_MAJOR_VERSION = Version.fromString("6.0.0");
+    private static final Version BWC_MINOR_VERSION = Version.fromString("5.1.0");
+    private static final Version BWC_STAGED_VERSION = Version.fromString("5.0.0");
+    private static final Version BWC_BUGFIX_VERSION = Version.fromString("5.0.1");
+    private static final Version BWC_MAINTENANCE_VERSION = Version.fromString("4.90.1");
     private static final BwcVersions BWC_MINOR = new BwcVersions(
         new TreeSet<>(Arrays.asList(BWC_BUGFIX_VERSION, BWC_MINOR_VERSION, BWC_MAJOR_VERSION)),
         BWC_MAJOR_VERSION

buildSrc/src/test/java/org/opensearch/gradle/VersionTests.java

@@ -65,7 +65,7 @@ public void testRelaxedVersionParsing() {
 
     public void testCompareWithStringVersions() {
         // 1.10.2 is now rebased to OpenSearch version; so this needs to report
-        assertTrue("OpenSearch 1.10.20 is not interpreted as after Legacy 3.0.0", Version.fromString("1.10.20").after("3.0.0"));
+        assertTrue("OpenSearch 1.10.20 is not interpreted as after Legacy 6.0.0", Version.fromString("1.10.20").after("6.0.0"));
         assertTrue(
             "7.0.0-alpha1 should be equal to 7.0.0-alpha1",
             Version.fromString("7.0.0-alpha1").equals(Version.fromString("7.0.0-alpha1"))

buildSrc/src/testKit/thirdPartyAudit/build.gradle

@@ -40,7 +40,7 @@ repositories {
 }
 
 dependencies {
-  forbiddenApisCliJar 'de.thetaphi:forbiddenapis:3.2'
+  forbiddenApisCliJar 'de.thetaphi:forbiddenapis:3.3'
   jdkJarHell 'org.opensearch:opensearch-core:current'
   compileOnly "org.${project.properties.compileOnlyGroup}:${project.properties.compileOnlyVersion}"
   implementation "org.${project.properties.compileGroup}:${project.properties.compileVersion}"

buildSrc/version.properties

@@ -1,5 +1,5 @@
-opensearch        = 2.0.0
-lucene            = 9.1.0-snapshot-ea989fe8f30
+opensearch        = 3.0.0
+lucene            = 9.1.0
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 17.0.2+8
@@ -9,7 +9,8 @@ bundled_jdk = 17.0.2+8
 # optional dependencies
 spatial4j         = 0.7
 jts               = 1.15.0
-jackson           = 2.12.6
+jackson           = 2.13.2
+jackson_databind  = 2.13.2.2
 snakeyaml         = 1.26
 icu4j             = 70.1
 supercsv          = 2.4.0

client/rest-high-level/src/main/java/org/opensearch/client/tasks/TaskInfo.java

@@ -57,6 +57,7 @@ public class TaskInfo {
     private TaskId parentTaskId;
     private final Map<String, Object> status = new HashMap<>();
     private final Map<String, String> headers = new HashMap<>();
+    private final Map<String, Object> resourceStats = new HashMap<>();
 
     public TaskInfo(TaskId taskId) {
         this.taskId = taskId;
@@ -150,6 +151,14 @@ public Map<String, Object> getStatus() {
         return status;
     }
 
+    void setResourceStats(Map<String, Object> resourceStats) {
+        this.resourceStats.putAll(resourceStats);
+    }
+
+    public Map<String, Object> getResourceStats() {
+        return resourceStats;
+    }
+
     private void noOpParse(Object s) {}
 
     public static final ObjectParser.NamedObjectParser<TaskInfo, Void> PARSER;
@@ -170,6 +179,7 @@ private void noOpParse(Object s) {}
         parser.declareBoolean(TaskInfo::setCancelled, new ParseField("cancelled"));
         parser.declareString(TaskInfo::setParentTaskId, new ParseField("parent_task_id"));
         parser.declareObject(TaskInfo::setHeaders, (p, c) -> p.mapStrings(), new ParseField("headers"));
+        parser.declareObject(TaskInfo::setResourceStats, (p, c) -> p.map(), new ParseField("resource_stats"));
         PARSER = (XContentParser p, Void v, String name) -> parser.parse(p, new TaskInfo(new TaskId(name)), null);
     }
 
@@ -188,7 +198,8 @@ && isCancelled() == taskInfo.isCancelled()
             && Objects.equals(getDescription(), taskInfo.getDescription())
             && Objects.equals(getParentTaskId(), taskInfo.getParentTaskId())
             && Objects.equals(status, taskInfo.status)
-            && Objects.equals(getHeaders(), taskInfo.getHeaders());
+            && Objects.equals(getHeaders(), taskInfo.getHeaders())
+            && Objects.equals(getResourceStats(), taskInfo.getResourceStats());
     }
 
     @Override
@@ -204,7 +215,8 @@ public int hashCode() {
             isCancelled(),
             getParentTaskId(),
             status,
-            getHeaders()
+            getHeaders(),
+            getResourceStats()
         );
     }
 
@@ -236,6 +248,8 @@ public String toString() {
             + status
             + ", headers="
             + headers
+            + ", resource_stats="
+            + resourceStats
             + '}';
     }
 }