Restore ClusterState version during remote state restore

Signed-off-by: bansvaru <bansvaru@amazon.com>

server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java

@@ -21,7 +21,9 @@
 
 import java.nio.charset.StandardCharsets;
 import java.util.Base64;
+import java.util.Locale;
 import java.util.Map;
+import java.util.Optional;
 import java.util.stream.Collectors;
 
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
@@ -90,9 +92,24 @@ public void testFullClusterRestoreStaleDelete() throws Exception {
 
         assertEquals(10, repository.blobStore().blobContainer(baseMetadataPath.add("manifest")).listBlobsByPrefix("manifest").size());
 
+        Optional<ClusterMetadataManifest> clusterMetadataManifest = remoteClusterStateService.getLatestClusterMetadataManifest(
+            clusterService().state().getClusterName().value(),
+            getClusterState().metadata().clusterUUID()
+        );
+        if (clusterMetadataManifest.isEmpty()) {
+            throw new IllegalStateException(
+                String.format(
+                    Locale.ROOT,
+                    "Latest cluster metadata manifest is not present for the provided clusterUUID: %s",
+                    getClusterState().metadata().clusterUUID()
+                )
+            );
+        }
+
         Map<String, IndexMetadata> indexMetadataMap = remoteClusterStateService.getLatestMetadata(
             cluster().getClusterName(),
-            getClusterState().metadata().clusterUUID()
+            getClusterState().metadata().clusterUUID(),
+            clusterMetadataManifest.get()
         ).getIndices();
         assertEquals(0, indexMetadataMap.values().stream().findFirst().get().getNumberOfReplicas());
         assertEquals(shardCount, indexMetadataMap.values().stream().findFirst().get().getNumberOfShards());

server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java

@@ -760,19 +760,14 @@ private IndexMetadata getIndexMetadata(String clusterName, String clusterUUID, U
      * @param clusterName name of the cluster
      * @return {@link IndexMetadata}
      */
-    public Metadata getLatestMetadata(String clusterName, String clusterUUID) {
+    public Metadata getLatestMetadata(String clusterName, String clusterUUID, ClusterMetadataManifest clusterMetadataManifest) {
         start();
-        Optional<ClusterMetadataManifest> clusterMetadataManifest = getLatestClusterMetadataManifest(clusterName, clusterUUID);
-        if (!clusterMetadataManifest.isPresent()) {
-            throw new IllegalStateException(
-                String.format(Locale.ROOT, "Latest cluster metadata manifest is not present for the provided clusterUUID: %s", clusterUUID)
-            );
-        }
+
         // Fetch Global Metadata
-        Metadata globalMetadata = getGlobalMetadata(clusterName, clusterUUID, clusterMetadataManifest.get());
+        Metadata globalMetadata = getGlobalMetadata(clusterName, clusterUUID, clusterMetadataManifest);
 
         // Fetch Index Metadata
-        Map<String, IndexMetadata> indices = getIndexMetadataMap(clusterName, clusterUUID, clusterMetadataManifest.get());
+        Map<String, IndexMetadata> indices = getIndexMetadataMap(clusterName, clusterUUID, clusterMetadataManifest);
 
         Map<String, IndexMetadata> indexMetadataMap = new HashMap<>();
         indices.values().forEach(indexMetadata -> { indexMetadataMap.put(indexMetadata.getIndex().getName(), indexMetadata); });

server/src/main/java/org/opensearch/index/recovery/RemoteStoreRestoreService.java

@@ -32,6 +32,7 @@
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.index.shard.ShardId;
+import org.opensearch.gateway.remote.ClusterMetadataManifest;
 import org.opensearch.gateway.remote.RemoteClusterStateService;
 import org.opensearch.indices.ShardLimitValidator;
 import org.opensearch.repositories.IndexId;
@@ -149,7 +150,24 @@ public RemoteRestoreResult restore(
                 if (currentState.metadata().clusterUUID().equals(restoreClusterUUID)) {
                     throw new IllegalArgumentException("clusterUUID to restore from should be different from current cluster UUID");
                 }
-                remoteMetadata = remoteClusterStateService.getLatestMetadata(currentState.getClusterName().value(), restoreClusterUUID);
+                Optional<ClusterMetadataManifest> clusterMetadataManifest = remoteClusterStateService.getLatestClusterMetadataManifest(
+                    currentState.getClusterName().value(),
+                    restoreClusterUUID
+                );
+                if (clusterMetadataManifest.isEmpty()) {
+                    throw new IllegalStateException(
+                        String.format(
+                            Locale.ROOT,
+                            "Latest cluster metadata manifest is not present for the provided clusterUUID: %s",
+                            restoreClusterUUID
+                        )
+                    );
+                }
+                remoteMetadata = remoteClusterStateService.getLatestMetadata(
+                    currentState.getClusterName().value(),
+                    restoreClusterUUID,
+                    clusterMetadataManifest.get()
+                );
                 remoteMetadata.getIndices().values().forEach(indexMetadata -> {
                     indexMetadataMap.put(indexMetadata.getIndex().getName(), new Tuple<>(true, indexMetadata));
                 });
@@ -255,6 +273,7 @@ private RemoteRestoreResult executeRestore(
     }
 
     private void restoreGlobalMetadata(Metadata.Builder mdBuilder, Metadata remoteMetadata) {
+        mdBuilder.version(remoteMetadata.version());
         if (remoteMetadata.persistentSettings() != null) {
             Settings settings = remoteMetadata.persistentSettings();
             clusterService.getClusterSettings().validateUpdate(settings);

server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java

@@ -659,9 +659,11 @@ public void testReadLatestMetadataManifestSuccessButNoIndexMetadata() throws IOE
 
         remoteClusterStateService.start();
         assertEquals(
-            remoteClusterStateService.getLatestMetadata(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID())
-                .getIndices()
-                .size(),
+            remoteClusterStateService.getLatestMetadata(
+                clusterState.getClusterName().value(),
+                clusterState.metadata().clusterUUID(),
+                expectedManifest
+            ).getIndices().size(),
             0
         );
     }
@@ -688,8 +690,11 @@ public void testReadLatestMetadataManifestSuccessButIndexMetadataFetchIOExceptio
         remoteClusterStateService.start();
         Exception e = assertThrows(
             IllegalStateException.class,
-            () -> remoteClusterStateService.getLatestMetadata(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID())
-                .getIndices()
+            () -> remoteClusterStateService.getLatestMetadata(
+                clusterState.getClusterName().value(),
+                clusterState.metadata().clusterUUID(),
+                expectedManifest
+            ).getIndices()
         );
         assertEquals(e.getMessage(), "Error while downloading IndexMetadata - " + uploadedIndexMetadata.getUploadedFilename());
     }
@@ -752,7 +757,8 @@ public void testReadGlobalMetadata() throws IOException {
 
         Metadata metadata = remoteClusterStateService.getLatestMetadata(
             clusterState.getClusterName().value(),
-            clusterState.metadata().clusterUUID()
+            clusterState.metadata().clusterUUID(),
+            expectedManifest
         );
 
         assertTrue(Metadata.isGlobalStateEquals(metadata, expactedMetadata));
@@ -787,7 +793,11 @@ public void testReadGlobalMetadataIOException() throws IOException {
         remoteClusterStateService.start();
         Exception e = assertThrows(
             IllegalStateException.class,
-            () -> remoteClusterStateService.getLatestMetadata(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID())
+            () -> remoteClusterStateService.getLatestMetadata(
+                clusterState.getClusterName().value(),
+                clusterState.metadata().clusterUUID(),
+                expectedManifest
+            )
         );
         assertEquals(e.getMessage(), "Error while downloading Global Metadata - " + globalIndexMetadataName);
     }
@@ -818,15 +828,16 @@ public void testReadLatestIndexMetadataSuccess() throws IOException {
             .nodeId("nodeA")
             .opensearchVersion(VersionUtils.randomOpenSearchVersion(random()))
             .previousClusterUUID("prev-cluster-uuid")
-            .globalMetadataFileName("global-metadata-file")
+            // .globalMetadataFileName("global-metadata-file")
             .codecVersion(ClusterMetadataManifest.CODEC_V0)
             .build();
 
         mockBlobContainer(mockBlobStoreObjects(), expectedManifest, Map.of(index.getUUID(), indexMetadata));
 
         Map<String, IndexMetadata> indexMetadataMap = remoteClusterStateService.getLatestMetadata(
             clusterState.getClusterName().value(),
-            clusterState.metadata().clusterUUID()
+            clusterState.metadata().clusterUUID(),
+            expectedManifest
         ).getIndices();
 
         assertEquals(indexMetadataMap.size(), 1);