Merge remote-tracking branch 'upstream/main' into manager-timeout-ing…

…est-script-api

Signed-off-by: Tianli Feng <ftianli@amazon.com>

TESTING.md

@@ -245,7 +245,7 @@ The YAML REST tests support all the options provided by the randomized runner, p
 
 -   `tests.rest.suite`: comma separated paths of the test suites to be run (by default loaded from /rest-api-spec/test). It is possible to run only a subset of the tests providing a sub-folder or even a single yaml file (the default /rest-api-spec/test prefix is optional when files are loaded from classpath) e.g. `-Dtests.rest.suite=index,get,create/10_with_id`
 
--   `tests.rest.blacklist`: comma separated globs that identify tests that are denylisted and need to be skipped e.g. `-Dtests.rest.blacklist=index/**/Index document,get/10_basic/**`
+-   `tests.rest.denylist`: comma separated globs that identify tests that are denylisted and need to be skipped e.g. `-Dtests.rest.denylist=index/**/Index document,get/10_basic/**`
 
 Java REST tests can be run with the "javaRestTest" task.
 

build.gradle

@@ -48,7 +48,7 @@ plugins {
   id 'lifecycle-base'
   id 'opensearch.docker-support'
   id 'opensearch.global-build-info'
-  id "com.diffplug.spotless" version "6.4.1" apply false
+  id "com.diffplug.spotless" version "6.4.2" apply false
   id "org.gradle.test-retry" version "1.3.1" apply false
 }
 

buildSrc/build.gradle

@@ -107,7 +107,7 @@ dependencies {
   api 'org.apache.ant:ant:1.10.12'
   api 'com.netflix.nebula:gradle-extra-configurations-plugin:7.0.0'
   api 'com.netflix.nebula:nebula-publishing-plugin:4.4.4'
-  api 'com.netflix.nebula:gradle-info-plugin:7.1.3'
+  api 'com.netflix.nebula:gradle-info-plugin:11.3.3'
   api 'org.apache.rat:apache-rat:0.13'
   api 'commons-io:commons-io:2.7'
   api "net.java.dev.jna:jna:5.10.0"

buildSrc/version.properties

@@ -23,18 +23,20 @@ jna               = 5.5.0
 netty             = 4.1.73.Final
 joda              = 2.10.12
 
+# client dependencies
+httpclient        = 4.5.13
+httpcore          = 4.4.12
+httpasyncclient   = 4.1.4
+commonslogging    = 1.2
+commonscodec      = 1.13
+
 # when updating this version, you need to ensure compatibility with:
 #  - plugins/ingest-attachment (transitive dependency, check the upstream POM)
 #  - distribution/tools/plugin-cli
 bouncycastle=1.70
 # test dependencies
 randomizedrunner  = 2.7.1
 junit             = 4.13.2
-httpclient        = 4.5.13
-httpcore          = 4.4.12
-httpasyncclient   = 4.1.4
-commonslogging    = 1.1.3
-commonscodec      = 1.13
 hamcrest          = 2.1
 mockito           = 4.3.1
 objenesis         = 3.2

client/rest/build.gradle

@@ -89,7 +89,6 @@ thirdPartyAudit.ignoreMissingClasses(
   'org.apache.avalon.framework.logger.Logger',
   'org.apache.log.Hierarchy',
   'org.apache.log.Logger',
-  'org.apache.log4j.Category',
   'org.apache.log4j.Level',
   'org.apache.log4j.Logger',
   'org.apache.log4j.Priority',

client/rest/licenses/commons-logging-1.2.jar.sha1

@@ -0,0 +1 @@
+4bfc12adfe4842bf07b657f0369c4cb522955686

client/sniffer/build.gradle

@@ -88,7 +88,6 @@ thirdPartyAudit.ignoreMissingClasses(
   'org.apache.avalon.framework.logger.Logger',
   'org.apache.log.Hierarchy',
   'org.apache.log.Logger',
-  'org.apache.log4j.Category',
   'org.apache.log4j.Level',
   'org.apache.log4j.Logger',
   'org.apache.log4j.Priority',

client/sniffer/licenses/commons-logging-1.2.jar.sha1

@@ -0,0 +1 @@
+4bfc12adfe4842bf07b657f0369c4cb522955686

libs/grok/build.gradle

@@ -29,7 +29,7 @@
  */
 
 dependencies {
-  api 'org.jruby.joni:joni:2.1.41'
+  api 'org.jruby.joni:joni:2.1.43'
   // joni dependencies:
   api 'org.jruby.jcodings:jcodings:1.0.57'
 

libs/grok/licenses/joni-2.1.43.jar.sha1

@@ -0,0 +1 @@
+9a3bf154469d5ff1d1107755904279081a5fb618

modules/ingest-geoip/build.gradle

@@ -39,7 +39,7 @@ opensearchplugin {
 }
 
 dependencies {
-  api('com.maxmind.geoip2:geoip2:2.16.1')
+  api('com.maxmind.geoip2:geoip2:3.0.1')
   // geoip2 dependencies:
   api("com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}")
   api("com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}")
@@ -67,24 +67,6 @@ tasks.named("bundlePlugin").configure {
   }
 }
 
-tasks.named("thirdPartyAudit").configure {
-  ignoreMissingClasses(
-          // geoip WebServiceClient needs apache http client, but we're not using WebServiceClient:
-          'org.apache.http.HttpEntity',
-          'org.apache.http.HttpResponse',
-          'org.apache.http.StatusLine',
-          'org.apache.http.client.config.RequestConfig$Builder',
-          'org.apache.http.client.config.RequestConfig',
-          'org.apache.http.client.methods.CloseableHttpResponse',
-          'org.apache.http.client.methods.HttpGet',
-          'org.apache.http.client.utils.URIBuilder',
-          'org.apache.http.impl.auth.BasicScheme',
-          'org.apache.http.impl.client.CloseableHttpClient',
-          'org.apache.http.impl.client.HttpClientBuilder',
-          'org.apache.http.util.EntityUtils'
-  )
-}
-
 if (Os.isFamily(Os.FAMILY_WINDOWS)) {
   tasks.named("test").configure {
     // Windows cannot cleanup database files properly unless it loads everything on heap.

modules/ingest-geoip/licenses/geoip2-3.0.1.jar.sha1

@@ -0,0 +1 @@
+8a814ae92a1d8c35f82d0ff76d86927c191b7916

modules/ingest-geoip/src/main/java/org/opensearch/ingest/geoip/GeoIpProcessor.java

@@ -364,7 +364,7 @@ private Map<String, Object> retrieveAsnGeoData(InetAddress ipAddress) {
             })
         );
 
-        Integer asn = response.getAutonomousSystemNumber();
+        Long asn = response.getAutonomousSystemNumber();
         String organization_name = response.getAutonomousSystemOrganization();
         Network network = response.getNetwork();
 

modules/ingest-geoip/src/test/java/org/opensearch/ingest/geoip/GeoIpProcessorTests.java

@@ -308,7 +308,7 @@ public void testAsn() throws Exception {
         Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
         assertThat(geoData.size(), equalTo(4));
         assertThat(geoData.get("ip"), equalTo(ip));
-        assertThat(geoData.get("asn"), equalTo(1136));
+        assertThat(geoData.get("asn"), equalTo(1136L));
         assertThat(geoData.get("organization_name"), equalTo("KPN B.V."));
         assertThat(geoData.get("network"), equalTo("82.168.0.0/14"));
     }

modules/repository-s3/licenses/commons-logging-1.2.jar.sha1

@@ -0,0 +1 @@
+4bfc12adfe4842bf07b657f0369c4cb522955686

plugins/discovery-azure-classic/licenses/commons-logging-1.2.jar.sha1

@@ -0,0 +1 @@
+4bfc12adfe4842bf07b657f0369c4cb522955686

plugins/discovery-ec2/licenses/commons-logging-1.2.jar.sha1

@@ -0,0 +1 @@
+4bfc12adfe4842bf07b657f0369c4cb522955686

plugins/discovery-gce/build.gradle

@@ -24,7 +24,7 @@ versions << [
 dependencies {
   api "com.google.apis:google-api-services-compute:v1-rev160-${versions.google}"
   api "com.google.api-client:google-api-client:${versions.google}"
-  api "com.google.oauth-client:google-oauth-client:1.33.1"
+  api "com.google.oauth-client:google-oauth-client:1.33.2"
   api "com.google.http-client:google-http-client:${versions.google}"
   api "com.google.http-client:google-http-client-jackson2:${versions.google}"
   api 'com.google.code.findbugs:jsr305:3.0.2'

plugins/discovery-gce/licenses/commons-logging-1.2.jar.sha1

@@ -0,0 +1 @@
+4bfc12adfe4842bf07b657f0369c4cb522955686

plugins/discovery-gce/licenses/google-oauth-client-1.33.2.jar.sha1

@@ -0,0 +1 @@
+2810fb515fe110295dc6867fc9f70c401b66daf3

plugins/ingest-attachment/licenses/commons-logging-1.2.jar.sha1

@@ -0,0 +1 @@
+4bfc12adfe4842bf07b657f0369c4cb522955686

plugins/repository-azure/build.gradle

@@ -44,7 +44,7 @@ opensearchplugin {
 }
 
 dependencies {
-  api 'com.azure:azure-core:1.26.0'
+  api 'com.azure:azure-core:1.27.0'
   api 'com.azure:azure-storage-common:12.15.0'
   api 'com.azure:azure-core-http-netty:1.11.9'
   api "io.netty:netty-codec-dns:${versions.netty}"
@@ -54,7 +54,7 @@ dependencies {
   api "io.netty:netty-resolver-dns:${versions.netty}"
   api "io.netty:netty-transport-native-unix-common:${versions.netty}"
   implementation project(':modules:transport-netty4')
-  api 'com.azure:azure-storage-blob:12.14.4'
+  api 'com.azure:azure-storage-blob:12.15.0'
   api 'org.reactivestreams:reactive-streams:1.0.3'
   api 'io.projectreactor:reactor-core:3.4.15'
   api 'io.projectreactor.netty:reactor-netty:1.0.17'

plugins/repository-azure/licenses/azure-core-1.27.0.jar.sha1

@@ -0,0 +1 @@
+75a2db538d218e2bd3c2cbdf04c955b8f6db6626

plugins/repository-azure/licenses/azure-storage-blob-12.15.0.jar.sha1

@@ -0,0 +1 @@
+a53a6bdf7564f4e3a7b0b93cd96b7f5f95c03d36

plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureBlobContainerRetriesTests.java

@@ -231,6 +231,7 @@ public void testReadBlobWithRetries() throws Exception {
                         exchange.getResponseHeaders().add("Content-Type", "application/octet-stream");
                         exchange.getResponseHeaders().add("Content-Length", String.valueOf(length));
                         exchange.getResponseHeaders().add("x-ms-blob-type", "blockblob");
+                        exchange.getResponseHeaders().add("Content-Range", "bytes " + rangeStart + "-" + bytes.length + "/" + bytes.length);
                         exchange.sendResponseHeaders(RestStatus.OK.getStatus(), length);
                         exchange.getResponseBody().write(bytes, rangeStart, length);
                         return;
@@ -247,7 +248,8 @@ public void testReadBlobWithRetries() throws Exception {
         final BlobContainer blobContainer = createBlobContainer(maxRetries);
         try (InputStream inputStream = blobContainer.readBlob("read_blob_max_retries")) {
             assertArrayEquals(bytes, BytesReference.toBytes(Streams.readFully(inputStream)));
-            assertThat(countDownHead.isCountedDown(), is(true));
+            // No more getProperties() calls in BlobClientBase::openInputStream(), HEAD should not be invoked
+            assertThat(countDownHead.isCountedDown(), is(false));
             assertThat(countDownGet.isCountedDown(), is(true));
         }
     }
@@ -278,6 +280,8 @@ public void testReadRangeBlobWithRetries() throws Exception {
                         assertThat(length, lessThanOrEqualTo(bytes.length - rangeStart));
                         exchange.getResponseHeaders().add("Content-Type", "application/octet-stream");
                         exchange.getResponseHeaders().add("Content-Length", String.valueOf(length));
+                        exchange.getResponseHeaders()
+                            .add("Content-Range", "bytes " + rangeStart + "-" + rangeEnd.get() + "/" + bytes.length);
                         exchange.getResponseHeaders().add("x-ms-blob-type", "blockblob");
                         exchange.sendResponseHeaders(RestStatus.OK.getStatus(), length);
                         exchange.getResponseBody().write(bytes, rangeStart, length);

plugins/repository-gcs/build.gradle

@@ -61,7 +61,7 @@ dependencies {
   api 'com.google.api:api-common:1.8.1'
   api 'com.google.api:gax:1.54.0'
   api 'org.threeten:threetenbp:1.4.4'
-  api 'com.google.protobuf:protobuf-java-util:3.19.3'
+  api 'com.google.protobuf:protobuf-java-util:3.20.0'
   api 'com.google.protobuf:protobuf-java:3.19.3'
   api 'com.google.code.gson:gson:2.9.0'
   api 'com.google.api.grpc:proto-google-common-protos:2.8.0'

plugins/repository-gcs/licenses/commons-logging-1.2.jar.sha1

@@ -0,0 +1 @@
+4bfc12adfe4842bf07b657f0369c4cb522955686

plugins/repository-gcs/licenses/protobuf-java-util-3.20.0.jar.sha1

@@ -0,0 +1 @@
+ee4496b296418283cbe7ae784984347fc4717a9a

plugins/repository-hdfs/build.gradle

@@ -67,7 +67,7 @@ dependencies {
   api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
   api 'com.google.code.gson:gson:2.9.0'
   runtimeOnly 'com.google.guava:guava:30.1.1-jre'
-  api 'com.google.protobuf:protobuf-java:3.19.3'
+  api 'com.google.protobuf:protobuf-java:3.20.0'
   api "commons-logging:commons-logging:${versions.commonslogging}"
   api 'commons-cli:commons-cli:1.2'
   api "commons-codec:commons-codec:${versions.commonscodec}"

plugins/repository-hdfs/licenses/commons-logging-1.2.jar.sha1

@@ -0,0 +1 @@
+4bfc12adfe4842bf07b657f0369c4cb522955686

plugins/repository-hdfs/licenses/protobuf-java-3.20.0.jar.sha1

@@ -0,0 +1 @@
+3c72ddaaab7ffafe789e4f732c1fd614eb798bf4

plugins/repository-s3/build.gradle

@@ -190,7 +190,7 @@ internalClusterTest {
 }
 
 yamlRestTest {
-    systemProperty 'tests.rest.blacklist', (
+    systemProperty 'tests.rest.denylist', (
       useFixture ?
         ['repository_s3/50_repository_ecs_credentials/*']
         :
@@ -246,7 +246,7 @@ if (useFixture) {
     setClasspath(yamlRestTestSourceSet.getRuntimeClasspath())
 
     // Minio only supports a single access key, see https://github.com/minio/minio/pull/5968
-    systemProperty 'tests.rest.blacklist', [
+    systemProperty 'tests.rest.denylist', [
       'repository_s3/30_repository_temporary_credentials/*',
       'repository_s3/40_repository_ec2_credentials/*',
       'repository_s3/50_repository_ecs_credentials/*'
@@ -272,7 +272,7 @@ if (useFixture) {
     SourceSet yamlRestTestSourceSet = sourceSets.getByName(YamlRestTestPlugin.SOURCE_SET_NAME)
     setTestClassesDirs(yamlRestTestSourceSet.getOutput().getClassesDirs())
     setClasspath(yamlRestTestSourceSet.getRuntimeClasspath())
-    systemProperty 'tests.rest.blacklist', [
+    systemProperty 'tests.rest.denylist', [
       'repository_s3/10_basic/*',
       'repository_s3/20_repository_permanent_credentials/*',
       'repository_s3/30_repository_temporary_credentials/*',

plugins/repository-s3/licenses/commons-logging-1.2.jar.sha1

@@ -0,0 +1 @@
+4bfc12adfe4842bf07b657f0369c4cb522955686

qa/mixed-cluster/src/test/java/org/opensearch/backwards/ExceptionIT.java

@@ -47,8 +47,13 @@ public void testOpensearchException() throws Exception {
     private void logClusterNodes() throws IOException {
         ObjectPath objectPath = ObjectPath.createFromResponse(client().performRequest(new Request("GET", "_nodes")));
         Map<String, ?> nodes = objectPath.evaluate("nodes");
-        String master = EntityUtils.toString(client().performRequest(new Request("GET", "_cat/master?h=id")).getEntity()).trim();
-        logger.info("cluster discovered: master id='{}'", master);
+        // As of 2.0, 'GET _cat/master' API is deprecated to promote inclusive language.
+        // Allow the deprecation warning for the node running an older version.
+        // TODO: Replace the API with 'GET _cat/cluster_manager' when dropping compatibility with 1.x versions.
+        Request catRequest = new Request("GET", "_cat/master?h=id");
+        catRequest.setOptions(expectWarningsOnce("[GET /_cat/master] is deprecated! Use [GET /_cat/cluster_manager] instead."));
+        String clusterManager = EntityUtils.toString(client().performRequest(catRequest).getEntity()).trim();
+        logger.info("cluster discovered: cluster-manager id='{}'", clusterManager);
         for (String id : nodes.keySet()) {
             logger.info("{}: id='{}', name='{}', version={}",
                 objectPath.evaluate("nodes." + id + ".http.publish_address"),

qa/wildfly/build.gradle

@@ -39,7 +39,7 @@ apply plugin: 'opensearch.internal-distribution-download'
 testFixtures.useFixture()
 
 dependencies {
-  providedCompile 'javax.enterprise:cdi-api:1.2'
+  providedCompile 'javax.enterprise:cdi-api:2.0'
   providedCompile 'org.jboss.spec.javax.annotation:jboss-annotations-api_1.2_spec:1.0.2.Final'
   providedCompile 'org.jboss.spec.javax.ws.rs:jboss-jaxrs-api_2.0_spec:1.0.1.Final'
   api('org.jboss.resteasy:resteasy-jackson2-provider:3.0.19.Final') {