Merge branch 'main' into merge_feature_extensions_into_main_2

CHANGELOG.md

@@ -15,8 +15,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Allow mmap to use new JDK-19 preview APIs in Apache Lucene 9.4+ ([#5151](https://github.com/opensearch-project/OpenSearch/pull/5151))
 - Add feature flag for extensions ([#5211](https://github.com/opensearch-project/OpenSearch/pull/5211))
 - Added jackson dependency to server ([#5366] (https://github.com/opensearch-project/OpenSearch/pull/5366))
-- Added experimental extensions to main ([#5347](https://github.com/opensearch-project/OpenSearch/pull/5347))
-- Expanded experimental extensions ([#5518](https://github.com/opensearch-project/OpenSearch/pull/5518))
+- Adding support to register settings dynamically ([#5495](https://github.com/opensearch-project/OpenSearch/pull/5495))
+- Added experimental support for extensions ([#5347](https://github.com/opensearch-project/OpenSearch/pull/5347)), ([#5518](https://github.com/opensearch-project/OpenSearch/pull/5518))
 
 ### Dependencies
 - Bumps `log4j-core` from 2.18.0 to 2.19.0
@@ -60,6 +60,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Support remote translog transfer for request level durability([#4480](https://github.com/opensearch-project/OpenSearch/pull/4480))
 - Changed http code on create index API with bad input raising NotXContentException from 500 to 400 ([#4773](https://github.com/opensearch-project/OpenSearch/pull/4773))
 - Change http code for DecommissioningFailedException from 500 to 400 ([#5283](https://github.com/opensearch-project/OpenSearch/pull/5283))
+- Pre conditions check before updating weighted routing metadata ([#4955](https://github.com/opensearch-project/OpenSearch/pull/4955))
 
 ### Deprecated
 

buildSrc/build.gradle

@@ -103,7 +103,7 @@ dependencies {
   api localGroovy()
 
   api 'commons-codec:commons-codec:1.15'
-  api 'org.apache.commons:commons-compress:1.21'
+  api 'org.apache.commons:commons-compress:1.22'
   api 'org.apache.ant:ant:1.10.12'
   api 'com.netflix.nebula:gradle-extra-configurations-plugin:8.0.0'
   api 'com.netflix.nebula:nebula-publishing-plugin:4.6.0'

buildSrc/version.properties

@@ -24,7 +24,7 @@ kotlin            = 1.7.10
 # when updating the JNA version, also update the version in buildSrc/build.gradle
 jna               = 5.5.0
 
-netty             = 4.1.84.Final
+netty             = 4.1.86.Final
 joda              = 2.10.13
 
 # client dependencies

modules/transport-netty4/build.gradle

@@ -125,7 +125,8 @@ thirdPartyAudit {
     'com.aayushatharva.brotli4j.Brotli4jLoader',
     'com.aayushatharva.brotli4j.decoder.DecoderJNI$Status',
     'com.aayushatharva.brotli4j.decoder.DecoderJNI$Wrapper',
-    'com.aayushatharva.brotli4j.encoder.Encoders',
+    'com.aayushatharva.brotli4j.encoder.BrotliEncoderChannel',
+    'com.aayushatharva.brotli4j.encoder.Encoder',
     'com.aayushatharva.brotli4j.encoder.Encoder$Mode',
     'com.aayushatharva.brotli4j.encoder.Encoder$Parameters',
     // classes are missing

modules/transport-netty4/licenses/netty-buffer-4.1.86.Final.jar.sha1

@@ -0,0 +1 @@
+a66fa0ed2687eb33a2e53a17a6df61bfe3b3f2bd

modules/transport-netty4/licenses/netty-codec-4.1.86.Final.jar.sha1

@@ -0,0 +1 @@
+ee126da926ea202da3b21eb737788ef83b1db772

modules/transport-netty4/licenses/netty-codec-http-4.1.86.Final.jar.sha1

@@ -0,0 +1 @@
+23674593f004959ae002ec348626eecf677191ae

modules/transport-netty4/licenses/netty-codec-http2-4.1.86.Final.jar.sha1

@@ -0,0 +1 @@
+ac0ca067e4118533ad1038776fcd9d5f3058b7d4

modules/transport-netty4/licenses/netty-common-4.1.86.Final.jar.sha1

@@ -0,0 +1 @@
+1dceab4662a9cc93faf87b237bb41103b1bc7f0e

modules/transport-netty4/licenses/netty-handler-4.1.86.Final.jar.sha1

@@ -0,0 +1 @@
+0bcb65230218286e6456b5d085cb42e67776eb70

modules/transport-netty4/licenses/netty-resolver-4.1.86.Final.jar.sha1

@@ -0,0 +1 @@
+bad83d479f7bd8ea84eefd77c316435be4c97270

modules/transport-netty4/licenses/netty-transport-4.1.86.Final.jar.sha1

@@ -0,0 +1 @@
+7c7739c41fd110c3576e9faace332ee957f27203

modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.86.Final.jar.sha1

@@ -0,0 +1 @@
+5e0e7fc1c337485cabcf7971faefe692b76f93a2

plugins/repository-azure/licenses/netty-codec-dns-4.1.86.Final.jar.sha1

@@ -0,0 +1 @@
+c8de479f36a8457541fcbb0016c851bde3e67693

plugins/repository-azure/licenses/netty-codec-http2-4.1.86.Final.jar.sha1

@@ -0,0 +1 @@
+ac0ca067e4118533ad1038776fcd9d5f3058b7d4

plugins/repository-azure/licenses/netty-codec-socks-4.1.86.Final.jar.sha1

@@ -0,0 +1 @@
+854264e7ad75887bc25b82eb38e4ee65c8b44dc3

plugins/repository-azure/licenses/netty-handler-proxy-4.1.86.Final.jar.sha1

@@ -0,0 +1 @@
+2515d76be9671cc248bab77352edddd16bfa9436

plugins/repository-azure/licenses/netty-resolver-dns-4.1.86.Final.jar.sha1

@@ -0,0 +1 @@
+a1e2ef79e4944b5d38092328c36c68e677a4b5f3

plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.86.Final.jar.sha1

@@ -0,0 +1 @@
+5e0e7fc1c337485cabcf7971faefe692b76f93a2

plugins/repository-hdfs/licenses/netty-all-4.1.86.Final.jar.sha1

@@ -0,0 +1 @@
+a6395c3d2f8699e8dc4fd1e38171f82045f4af7b

plugins/transport-nio/build.gradle

@@ -61,7 +61,8 @@ thirdPartyAudit {
     'com.aayushatharva.brotli4j.Brotli4jLoader',
     'com.aayushatharva.brotli4j.decoder.DecoderJNI$Status',
     'com.aayushatharva.brotli4j.decoder.DecoderJNI$Wrapper',
-    'com.aayushatharva.brotli4j.encoder.Encoders',
+    'com.aayushatharva.brotli4j.encoder.BrotliEncoderChannel',
+    'com.aayushatharva.brotli4j.encoder.Encoder',
     'com.aayushatharva.brotli4j.encoder.Encoder$Mode',
     'com.aayushatharva.brotli4j.encoder.Encoder$Parameters',
 

plugins/transport-nio/licenses/netty-buffer-4.1.86.Final.jar.sha1

@@ -0,0 +1 @@
+a66fa0ed2687eb33a2e53a17a6df61bfe3b3f2bd

plugins/transport-nio/licenses/netty-codec-4.1.86.Final.jar.sha1

@@ -0,0 +1 @@
+ee126da926ea202da3b21eb737788ef83b1db772

plugins/transport-nio/licenses/netty-codec-http-4.1.86.Final.jar.sha1

@@ -0,0 +1 @@
+23674593f004959ae002ec348626eecf677191ae

plugins/transport-nio/licenses/netty-common-4.1.86.Final.jar.sha1

@@ -0,0 +1 @@
+1dceab4662a9cc93faf87b237bb41103b1bc7f0e

plugins/transport-nio/licenses/netty-handler-4.1.86.Final.jar.sha1

@@ -0,0 +1 @@
+0bcb65230218286e6456b5d085cb42e67776eb70

plugins/transport-nio/licenses/netty-resolver-4.1.86.Final.jar.sha1

@@ -0,0 +1 @@
+bad83d479f7bd8ea84eefd77c316435be4c97270

plugins/transport-nio/licenses/netty-transport-4.1.86.Final.jar.sha1

@@ -0,0 +1 @@
+7c7739c41fd110c3576e9faace332ee957f27203

release-notes/opensearch.release-notes-1.3.7.md

@@ -1,4 +1,4 @@
-## 2022-11-30 Version 1.3.7 Release Notes
+## 2022-12-13 Version 1.3.7 Release Notes
 
 ### Upgrades
 * Upgrade netty to 4.1.84.Final ([#4919](https://github.com/opensearch-project/OpenSearch/pull/4919))

release-notes/opensearch.release-notes-2.4.1.md

@@ -1,4 +1,4 @@
-## 2022-12-07 Version 2.4.1 Release Notes
+## 2022-12-13 Version 2.4.1 Release Notes
 
 ### Bug Fixes
 * Fix 1.x compatibility bug with stored Tasks ([#5412](https://github.com/opensearch-project/opensearch/pull/5412)) ([#5440](https://github.com/opensearch-project/opensearch/pull/5440))
@@ -20,3 +20,4 @@
 * Update Apache Lucene to 9.4.2 ([#5354](https://github.com/opensearch-project/opensearch/pull/5354)) ([#5361](https://github.com/opensearch-project/opensearch/pull/5361))
 * Update Jackson to 2.14.1 ([#5346](https://github.com/opensearch-project/opensearch/pull/5346)) ([#5358](https://github.com/opensearch-project/opensearch/pull/5358))
 * Bump nebula-publishing-plugin from v4.4.0 to v4.6.0. ([#5127](https://github.com/opensearch-project/opensearch/pull/5127)) ([#5131](https://github.com/opensearch-project/opensearch/pull/5131))
+* Bump commons-compress from 1.21 to 1.22. ([#5520](https://github.com/opensearch-project/OpenSearch/pull/5520)) ([#5522](https://github.com/opensearch-project/opensearch/pull/5522))

server/src/internalClusterTest/java/org/opensearch/cluster/coordination/AwarenessAttributeDecommissionIT.java

@@ -29,7 +29,6 @@
 import org.opensearch.cluster.ClusterStateObserver;
 import org.opensearch.cluster.decommission.DecommissionAttribute;
 import org.opensearch.cluster.decommission.DecommissionAttributeMetadata;
-import org.opensearch.cluster.decommission.DecommissionService;
 import org.opensearch.cluster.decommission.DecommissionStatus;
 import org.opensearch.cluster.decommission.DecommissioningFailedException;
 import org.opensearch.cluster.decommission.NodeDecommissionedException;
@@ -824,24 +823,11 @@ public void testDecommissionFailedWithOnlyOneAttributeValue() throws Exception {
         // and hence due to which the leader won't get abdicated and decommission request should eventually fail.
         // And in this case, to ensure decommission request doesn't leave mutating change in the cluster, we ensure
         // that no exclusion is set to the cluster and state for decommission is marked as FAILED
-        Logger clusterLogger = LogManager.getLogger(DecommissionService.class);
-        MockLogAppender mockLogAppender = MockLogAppender.createForLoggers(clusterLogger);
-        mockLogAppender.addExpectation(
-            new MockLogAppender.SeenEventExpectation(
-                "test",
-                DecommissionService.class.getCanonicalName(),
-                Level.ERROR,
-                "failure in removing to-be-decommissioned cluster manager eligible nodes"
-            )
+        OpenSearchTimeoutException ex = expectThrows(
+            OpenSearchTimeoutException.class,
+            () -> client().execute(DecommissionAction.INSTANCE, decommissionRequest).actionGet()
         );
-
-        assertBusy(() -> {
-            OpenSearchTimeoutException ex = expectThrows(
-                OpenSearchTimeoutException.class,
-                () -> client().execute(DecommissionAction.INSTANCE, decommissionRequest).actionGet()
-            );
-            assertTrue(ex.getMessage().contains("timed out waiting for voting config exclusions"));
-        });
+        assertTrue(ex.getMessage().contains("while removing to-be-decommissioned cluster manager eligible nodes"));
 
         ClusterService leaderClusterService = internalCluster().getInstance(
             ClusterService.class,
@@ -877,7 +863,6 @@ public void testDecommissionFailedWithOnlyOneAttributeValue() throws Exception {
 
         // if the below condition is passed, then we are sure current decommission status is marked FAILED
         assertTrue(expectedStateLatch.await(30, TimeUnit.SECONDS));
-        mockLogAppender.assertAllExpectationsMatched();
 
         // ensure all nodes are part of cluster
         ensureStableCluster(6, TimeValue.timeValueMinutes(2));

server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java

@@ -10,6 +10,8 @@
 
 import com.carrotsearch.randomizedtesting.RandomizedTest;
 import org.junit.BeforeClass;
+import org.opensearch.OpenSearchCorruptionException;
+import org.opensearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.opensearch.action.admin.indices.segments.IndexShardSegments;
 import org.opensearch.action.admin.indices.segments.IndicesSegmentResponse;
 import org.opensearch.action.admin.indices.segments.IndicesSegmentsRequest;
@@ -24,7 +26,9 @@
 import org.opensearch.cluster.routing.ShardRouting;
 import org.opensearch.cluster.routing.allocation.command.CancelAllocationCommand;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.Priority;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.index.Index;
 import org.opensearch.index.IndexModule;
@@ -53,6 +57,7 @@
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
+import static org.hamcrest.Matchers.equalTo;
 import static org.opensearch.index.query.QueryBuilders.matchQuery;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
@@ -194,6 +199,75 @@ public void testCancelPrimaryAllocation() throws Exception {
         assertSegmentStats(REPLICA_COUNT);
     }
 
+    /**
+     * This test verfies that replica shard is not added to the cluster when doing a round of segment replication fails during peer recovery.
+     */
+    public void testAddNewReplicaFailure() throws Exception {
+        logger.info("--> starting [Primary Node] ...");
+        final String primaryNode = internalCluster().startNode();
+
+        logger.info("--> creating test index ...");
+        prepareCreate(
+            INDEX_NAME,
+            Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+        ).get();
+
+        logger.info("--> index 10 docs");
+        for (int i = 0; i < 10; i++) {
+            client().prepareIndex(INDEX_NAME).setId(Integer.toString(i)).setSource("field", "value" + i).execute().actionGet();
+        }
+        logger.info("--> flush so we have some segment files on disk");
+        flush(INDEX_NAME);
+        logger.info("--> index more docs so we have something in the translog");
+        for (int i = 10; i < 20; i++) {
+            client().prepareIndex(INDEX_NAME).setId(Integer.toString(i)).setSource("field", "value" + i).execute().actionGet();
+        }
+        refresh(INDEX_NAME);
+        logger.info("--> verifying count");
+        assertThat(client().prepareSearch(INDEX_NAME).setSize(0).execute().actionGet().getHits().getTotalHits().value, equalTo(20L));
+
+        logger.info("--> start empty node to add replica shard");
+        final String replicaNode = internalCluster().startNode();
+
+        // Mock transport service to add behaviour of throwing corruption exception during segment replication process.
+        MockTransportService mockTransportService = ((MockTransportService) internalCluster().getInstance(
+            TransportService.class,
+            primaryNode
+        ));
+        mockTransportService.addSendBehavior(
+            internalCluster().getInstance(TransportService.class, replicaNode),
+            (connection, requestId, action, request, options) -> {
+                if (action.equals(SegmentReplicationTargetService.Actions.FILE_CHUNK)) {
+                    throw new OpenSearchCorruptionException("expected");
+                }
+                connection.sendRequest(requestId, action, request, options);
+            }
+        );
+        ensureGreen(INDEX_NAME);
+        // Add Replica shard to the new empty replica node
+        assertAcked(
+            client().admin()
+                .indices()
+                .prepareUpdateSettings(INDEX_NAME)
+                .setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1))
+        );
+
+        // Verify that cluster state is not green and replica shard failed during a round of segment replication is not added to the cluster
+        ClusterHealthResponse clusterHealthResponse = client().admin()
+            .cluster()
+            .prepareHealth()
+            .setWaitForEvents(Priority.LANGUID)
+            .setWaitForNodes("2")
+            .setWaitForGreenStatus()
+            .setTimeout(TimeValue.timeValueSeconds(2))
+            .execute()
+            .actionGet();
+        assertTrue(clusterHealthResponse.isTimedOut());
+        ensureYellow(INDEX_NAME);
+        IndicesService indicesService = internalCluster().getInstance(IndicesService.class, replicaNode);
+        assertFalse(indicesService.hasIndex(resolveIndex(INDEX_NAME)));
+    }
+
     public void testReplicationAfterPrimaryRefreshAndFlush() throws Exception {
         final String nodeA = internalCluster().startNode();
         final String nodeB = internalCluster().startNode();
@@ -452,18 +526,14 @@ public void testStartReplicaAfterPrimaryIndexesDocs() throws Exception {
         final String replicaNode = internalCluster().startNode();
         ensureGreen(INDEX_NAME);
 
-        client().prepareIndex(INDEX_NAME).setId("3").setSource("foo", "bar").get();
+        assertHitCount(client(primaryNode).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), 2);
+        assertHitCount(client(replicaNode).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), 2);
 
+        client().prepareIndex(INDEX_NAME).setId("3").setSource("foo", "bar").get();
+        refresh(INDEX_NAME);
         waitForReplicaUpdate();
         assertHitCount(client(primaryNode).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), 3);
         assertHitCount(client(replicaNode).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), 3);
-
-        IndexShard primaryShard = getIndexShard(primaryNode);
-        IndexShard replicaShard = getIndexShard(replicaNode);
-        assertEquals(
-            primaryShard.translogStats().estimatedNumberOfOperations(),
-            replicaShard.translogStats().estimatedNumberOfOperations()
-        );
         assertSegmentStats(REPLICA_COUNT);
     }