Merge remote-tracking branch 'upstream/main' into 13302

Signed-off-by: Ashish Singh <ssashish@amazon.com>
opensearch-project · Apr 26, 2024 · e92892c · e92892c
2 parents bd3ab77 + a69cd08
commit e92892c
Show file tree

Hide file tree

Showing 30 changed files with 2,059 additions and 143 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -25,6 +25,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Streaming Indexing] Ensure support of the new transport by security plugin ([#13174](https://github.com/opensearch-project/OpenSearch/pull/13174))
 - Add cluster setting to dynamically configure the buckets for filter rewrite optimization. ([#13179](https://github.com/opensearch-project/OpenSearch/pull/13179))
 - [Remote Store] Add capability of doing refresh as determined by the translog ([#12992](https://github.com/opensearch-project/OpenSearch/pull/12992))
+- Batch mode for async fetching shard information in GatewayAllocator for unassigned shards ([#8746](https://github.com/opensearch-project/OpenSearch/pull/8746))
 - [Remote Store] Add settings for remote path type and hash algorithm ([#13225](https://github.com/opensearch-project/OpenSearch/pull/13225))
 - [Remote Store] Upload remote paths during remote enabled index creation ([#13386](https://github.com/opensearch-project/OpenSearch/pull/13386))
 
@@ -48,6 +49,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `com.github.spullara.mustache.java:compiler` from 0.9.10 to 0.9.11 ([#13329](https://github.com/opensearch-project/OpenSearch/pull/13329))
 - Bump `jakarta.enterprise:jakarta.enterprise.cdi-api` from 4.0.1 to 4.1.0 ([#13328](https://github.com/opensearch-project/OpenSearch/pull/13328))
 - Bump `com.google.api.grpc:proto-google-iam-v1` from 0.12.0 to 1.33.0 ([#13332](https://github.com/opensearch-project/OpenSearch/pull/13332))
+- OpenJDK Update (April 2024 Patch releases), update to Eclipse Temurin 21.0.3+9 ([#13389](https://github.com/opensearch-project/OpenSearch/pull/13389))
+- Bump `com.squareup.okio:okio` from 3.8.0 to 3.9.0 ([#12997](https://github.com/opensearch-project/OpenSearch/pull/12997))
 
 ### Changed
 - [BWC and API enforcement] Enforcing the presence of API annotations at build time ([#12872](https://github.com/opensearch-project/OpenSearch/pull/12872))

diff --git a/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java
@@ -77,9 +77,9 @@
 import java.util.stream.Stream;
 
 public class DistroTestPlugin implements Plugin<Project> {
-    private static final String SYSTEM_JDK_VERSION = "21.0.2+13";
+    private static final String SYSTEM_JDK_VERSION = "21.0.3+9";
     private static final String SYSTEM_JDK_VENDOR = "adoptium";
-    private static final String GRADLE_JDK_VERSION = "21.0.2+13";
+    private static final String GRADLE_JDK_VERSION = "21.0.3+9";
     private static final String GRADLE_JDK_VENDOR = "adoptium";
 
     // all distributions used by distro tests. this is temporary until tests are per distribution

diff --git a/buildSrc/version.properties b/buildSrc/version.properties
@@ -2,7 +2,7 @@ opensearch        = 3.0.0
 lucene            = 9.11.0-snapshot-fb97840
 
 bundled_jdk_vendor = adoptium
-bundled_jdk = 21.0.2+13
+bundled_jdk = 21.0.3+9
 
 # optional dependencies
 spatial4j         = 0.7

diff --git a/...ternalClusterTest/java/org/opensearch/cluster/metadata/ClusterIndexRefreshIntervalIT.java b/...ternalClusterTest/java/org/opensearch/cluster/metadata/ClusterIndexRefreshIntervalIT.java
@@ -235,33 +235,19 @@ public void testDefaultRefreshIntervalWithUpdateClusterAndIndexSettings() throws
     }
 
     public void testRefreshIntervalDisabled() throws ExecutionException, InterruptedException {
-        TimeValue clusterMinimumRefreshInterval = client().settings()
-            .getAsTime(IndicesService.CLUSTER_MINIMUM_INDEX_REFRESH_INTERVAL_SETTING.getKey(), TimeValue.MINUS_ONE);
-        boolean createIndexSuccess = clusterMinimumRefreshInterval.equals(TimeValue.MINUS_ONE);
         String clusterManagerName = internalCluster().getClusterManagerName();
         List<String> dataNodes = new ArrayList<>(internalCluster().getDataNodeNames());
         Settings settings = Settings.builder()
             .put(indexSettings())
             .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), IndexSettings.MINIMUM_REFRESH_INTERVAL)
             .build();
-        if (createIndexSuccess) {
-            createIndex(INDEX_NAME, settings);
-            ensureYellowAndNoInitializingShards(INDEX_NAME);
-            ensureGreen(INDEX_NAME);
-            GetIndexResponse getIndexResponse = client(clusterManagerName).admin().indices().getIndex(new GetIndexRequest()).get();
-            IndicesService indicesService = internalCluster().getInstance(IndicesService.class, randomFrom(dataNodes));
-            String uuid = getIndexResponse.getSettings().get(INDEX_NAME).get(IndexMetadata.SETTING_INDEX_UUID);
-            IndexService indexService = indicesService.indexService(new Index(INDEX_NAME, uuid));
-            assertEquals(IndexSettings.MINIMUM_REFRESH_INTERVAL, indexService.getRefreshTaskInterval());
-        } else {
-            IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> createIndex(INDEX_NAME, settings));
-            assertEquals(
-                "invalid index.refresh_interval [-1]: cannot be smaller than cluster.minimum.index.refresh_interval ["
-                    + getMinRefreshIntervalForRefreshDisabled()
-                    + "]",
-                exception.getMessage()
-            );
-        }
+        createIndex(INDEX_NAME, settings);
+        ensureGreen(INDEX_NAME);
+        GetIndexResponse getIndexResponse = client(clusterManagerName).admin().indices().getIndex(new GetIndexRequest()).get();
+        IndicesService indicesService = internalCluster().getInstance(IndicesService.class, randomFrom(dataNodes));
+        String uuid = getIndexResponse.getSettings().get(INDEX_NAME).get(IndexMetadata.SETTING_INDEX_UUID);
+        IndexService indexService = indicesService.indexService(new Index(INDEX_NAME, uuid));
+        assertEquals(IndexSettings.MINIMUM_REFRESH_INTERVAL, indexService.getRefreshTaskInterval());
     }
 
     protected TimeValue getMinRefreshIntervalForRefreshDisabled() {
@@ -366,6 +352,147 @@ public void testClusterMinimumChangeOnIndexWithCustomRefreshInterval() throws Ex
         assertEquals(customRefreshInterval, indexService.getRefreshTaskInterval());
     }
 
+    public void testClusterMinimumRefreshIntervalOfMinusOneFails() {
+        // This test checks that we can not set cluster minimum refresh interval as -1 (or -1ms).
+        String clusterManagerName = internalCluster().getClusterManagerName();
+        String refreshInterval = randomFrom("-1", "-1ms");
+        IllegalArgumentException ex = assertThrows(
+            IllegalArgumentException.class,
+            () -> client(clusterManagerName).admin()
+                .cluster()
+                .prepareUpdateSettings()
+                .setTransientSettings(Settings.builder().put(CLUSTER_MINIMUM_INDEX_REFRESH_INTERVAL_SETTING.getKey(), refreshInterval))
+                .get()
+        );
+        assertEquals(
+            "failed to parse value [" + refreshInterval + "] for setting [cluster.minimum.index.refresh_interval], must be >= [0ms]",
+            ex.getMessage()
+        );
+    }
+
+    public void testClusterMinimumRefreshIntervalOfZero() {
+        // This test checks that we can set the cluster minimum refresh interval as 0.
+        String clusterManagerName = internalCluster().getClusterManagerName();
+        client(clusterManagerName).admin()
+            .cluster()
+            .prepareUpdateSettings()
+            .setTransientSettings(Settings.builder().put(CLUSTER_MINIMUM_INDEX_REFRESH_INTERVAL_SETTING.getKey(), "0"))
+            .get();
+    }
+
+    public void testDefaultRefreshIntervalOfMinusOneIrrespectiveOfMinimum() {
+        // This test checks that we are able to set the cluster default refresh interval to one regardless of what the
+        // minimum is set to. -1 corresponds to no period background refreshes.
+        String clusterManagerName = internalCluster().getClusterManagerName();
+        client(clusterManagerName).admin()
+            .cluster()
+            .prepareUpdateSettings()
+            .setTransientSettings(
+                Settings.builder()
+                    .put(CLUSTER_MINIMUM_INDEX_REFRESH_INTERVAL_SETTING.getKey(), randomFrom("0", "1ms", "1s", "10s"))
+                    .put(CLUSTER_DEFAULT_INDEX_REFRESH_INTERVAL_SETTING.getKey(), randomFrom("-1", "-1ms"))
+            )
+            .get();
+    }
+
+    public void testCreateIndexWithMinusOneRefreshInterval() throws ExecutionException, InterruptedException {
+        // This test checks that we are able to create index with -1 refresh interval using index settings and default interval both.
+        String clusterManagerName = internalCluster().getClusterManagerName();
+        client(clusterManagerName).admin()
+            .cluster()
+            .prepareUpdateSettings()
+            .setTransientSettings(
+                Settings.builder()
+                    .put(CLUSTER_MINIMUM_INDEX_REFRESH_INTERVAL_SETTING.getKey(), "10s")
+                    .put(CLUSTER_DEFAULT_INDEX_REFRESH_INTERVAL_SETTING.getKey(), "10s")
+            )
+            .get();
+
+        Settings indexSettings = Settings.builder()
+            .put(indexSettings())
+            .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), randomFrom("-1", "-1ms"))
+            .build();
+        createIndex(INDEX_NAME, indexSettings);
+        ensureGreen(INDEX_NAME);
+
+        IndexService indexService = getIndexServiceFromRandomDataNode(INDEX_NAME);
+        assertEquals(-1, indexService.getRefreshTaskInterval().millis());
+
+        client(clusterManagerName).admin()
+            .cluster()
+            .prepareUpdateSettings()
+            .setTransientSettings(Settings.builder().put(CLUSTER_DEFAULT_INDEX_REFRESH_INTERVAL_SETTING.getKey(), randomFrom("-1", "-1ms")))
+            .get();
+        createIndex(OTHER_INDEX_NAME);
+        ensureGreen(OTHER_INDEX_NAME);
+        indexService = getIndexServiceFromRandomDataNode(OTHER_INDEX_NAME);
+        assertEquals(-1, indexService.getRefreshTaskInterval().millis());
+    }
+
+    public void testUpdateIndexWithMinusOneRefreshInterval() throws ExecutionException, InterruptedException {
+        // This test checks that we are able to update index with -1 refresh interval using index settings and default interval both.
+        String clusterManagerName = internalCluster().getClusterManagerName();
+        client(clusterManagerName).admin()
+            .cluster()
+            .prepareUpdateSettings()
+            .setTransientSettings(
+                Settings.builder()
+                    .put(CLUSTER_MINIMUM_INDEX_REFRESH_INTERVAL_SETTING.getKey(), "10s")
+                    .put(CLUSTER_DEFAULT_INDEX_REFRESH_INTERVAL_SETTING.getKey(), "10s")
+            )
+            .get();
+
+        createIndex(INDEX_NAME);
+        ensureGreen(INDEX_NAME);
+        IndexService indexService = getIndexServiceFromRandomDataNode(INDEX_NAME);
+        assertEquals(10, indexService.getRefreshTaskInterval().seconds());
+
+        client(clusterManagerName).admin()
+            .indices()
+            .updateSettings(
+                new UpdateSettingsRequest(INDEX_NAME).settings(
+                    Settings.builder().put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), randomFrom("-1", "-1ms"))
+                )
+            )
+            .actionGet();
+        assertEquals(-1, indexService.getRefreshTaskInterval().millis());
+
+        client(clusterManagerName).admin()
+            .indices()
+            .updateSettings(
+                new UpdateSettingsRequest(INDEX_NAME).settings(
+                    Settings.builder().put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "100s")
+                )
+            )
+            .actionGet();
+        assertEquals(100, indexService.getRefreshTaskInterval().seconds());
+
+        client(clusterManagerName).admin()
+            .cluster()
+            .prepareUpdateSettings()
+            .setTransientSettings(Settings.builder().put(CLUSTER_DEFAULT_INDEX_REFRESH_INTERVAL_SETTING.getKey(), randomFrom("-1", "-1ms")))
+            .get();
+
+        client(clusterManagerName).admin()
+            .indices()
+            .updateSettings(
+                new UpdateSettingsRequest(INDEX_NAME).settings(
+                    Settings.builder().putNull(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey())
+                )
+            )
+            .actionGet();
+        assertEquals(-1, indexService.getRefreshTaskInterval().millis());
+    }
+
+    private IndexService getIndexServiceFromRandomDataNode(String indexName) throws ExecutionException, InterruptedException {
+        String clusterManagerName = internalCluster().getClusterManagerName();
+        List<String> dataNodes = new ArrayList<>(internalCluster().getDataNodeNames());
+        GetIndexResponse getIndexResponse = client(clusterManagerName).admin().indices().getIndex(new GetIndexRequest()).get();
+        IndicesService indicesService = internalCluster().getInstance(IndicesService.class, randomFrom(dataNodes));
+        String uuid = getIndexResponse.getSettings().get(indexName).get(IndexMetadata.SETTING_INDEX_UUID);
+        return indicesService.indexService(new Index(indexName, uuid));
+    }
+
     protected TimeValue getDefaultRefreshInterval() {
         return IndexSettings.DEFAULT_REFRESH_INTERVAL;
     }

diff --git a/server/src/internalClusterTest/java/org/opensearch/discovery/ClusterManagerDisruptionIT.java b/server/src/internalClusterTest/java/org/opensearch/discovery/ClusterManagerDisruptionIT.java
@@ -198,11 +198,15 @@ public void testIsolateClusterManagerAndVerifyClusterStateConsensus() throws Exc
                             + nodeState
                     );
                 }
-
             }
-            ClusterStateStats clusterStateStats = internalCluster().clusterService().getClusterManagerService().getClusterStateStats();
-            assertTrue(clusterStateStats.getUpdateFailed() > 0);
+
         });
+
+        ClusterStateStats clusterStateStats = internalCluster().clusterService(isolatedNode)
+            .getClusterManagerService()
+            .getClusterStateStats();
+        assertTrue(clusterStateStats.getUpdateFailed() > 0);
+
     }
 
     /**