Add ClusterUUID to RepositoryData (#68002)

plugins/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java

@@ -147,7 +147,7 @@ public void testEnforcedCooldownPeriod() throws IOException {
         final BlobStoreRepository repository = (BlobStoreRepository) repositoriesService.repository(repoName);
         final RepositoryData repositoryData = getRepositoryData(repository);
         final RepositoryData modifiedRepositoryData = repositoryData.withVersions(Collections.singletonMap(fakeOldSnapshot,
-            SnapshotsService.SHARD_GEN_IN_REPO_DATA_VERSION.minimumCompatibilityVersion())).withoutUuid();
+            SnapshotsService.SHARD_GEN_IN_REPO_DATA_VERSION.minimumCompatibilityVersion())).withoutRepositoryUUID().withoutClusterUUID();
         final BytesReference serialized = BytesReference.bytes(modifiedRepositoryData.snapshotsToXContent(XContentFactory.jsonBuilder(),
             SnapshotsService.OLD_SNAPSHOT_FORMAT));
         PlainActionFuture.get(f -> repository.threadPool().generic().execute(ActionRunnable.run(f, () ->

qa/repository-multi-version/src/test/java/org/elasticsearch/upgrades/MultiVersionRepositoryAccessIT.java

@@ -9,6 +9,7 @@
 package org.elasticsearch.upgrades;
 
 import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotStatus;
@@ -104,7 +105,7 @@ public void testCreateAndRestoreSnapshot() throws IOException {
             final int shards = 3;
             final String index = "test-index";
             createIndex(client, index, shards);
-            createRepository(client, repoName, false);
+            createRepository(client, repoName, false, true);
             createSnapshot(client, repoName, "snapshot-" + TEST_STEP, index);
             final String snapshotToDeleteName = "snapshot-to-delete";
             // Create a snapshot and delete it right away again to test the impact of each version's cleanup functionality that is run
@@ -149,7 +150,7 @@ public void testReadOnlyRepo() throws IOException {
         try (RestHighLevelClient client = new RestHighLevelClient(RestClient.builder(adminClient().getNodes().toArray(new Node[0])))) {
             final int shards = 3;
             final boolean readOnly = TEST_STEP.ordinal() > 1; // only restore from read-only repo in steps 3 and 4
-            createRepository(client, repoName, readOnly);
+            createRepository(client, repoName, readOnly, true);
             final String index = "test-index";
             if (readOnly == false) {
                 createIndex(client, index, shards);
@@ -181,13 +182,25 @@ public void testReadOnlyRepo() throws IOException {
         }
     }
 
+    private static final List<Class<? extends Exception>> EXPECTED_BWC_EXCEPTIONS =
+            Arrays.asList(ResponseException.class, ElasticsearchStatusException.class);
+
     public void testUpgradeMovesRepoToNewMetaVersion() throws IOException {
         final String repoName = getTestName();
         try (RestHighLevelClient client = new RestHighLevelClient(RestClient.builder(adminClient().getNodes().toArray(new Node[0])))) {
             final int shards = 3;
             final String index=  "test-index";
             createIndex(client, index, shards);
-            createRepository(client, repoName, false);
+            final Version minNodeVersion = minimumNodeVersion();
+            // 7.12.0+ will try to load RepositoryData during repo creation if verify is true, which is impossible in case of version
+            // incompatibility in the downgrade test step. We verify that it is impossible here and then create the repo using verify=false
+            // to check behavior on other operations below.
+            final boolean verify = TEST_STEP != TestStep.STEP3_OLD_CLUSTER || SnapshotsService.includesClusterUUID(minNodeVersion)
+                    || minNodeVersion.before(Version.V_7_12_0);
+            if (verify == false) {
+                expectThrowsAnyOf(EXPECTED_BWC_EXCEPTIONS, () -> createRepository(client, repoName, false, true));
+            }
+            createRepository(client, repoName, false, verify);
             // only create some snapshots in the first two steps
             if (TEST_STEP == TestStep.STEP1_OLD_CLUSTER || TEST_STEP == TestStep.STEP2_NEW_CLUSTER) {
                 createSnapshot(client, repoName, "snapshot-" + TEST_STEP, index);
@@ -208,14 +221,12 @@ public void testUpgradeMovesRepoToNewMetaVersion() throws IOException {
                     ensureSnapshotRestoreWorks(repoName, "snapshot-2", shards);
                 }
             } else {
-                if (SnapshotsService.includesRepositoryUuid(minimumNodeVersion()) == false) {
+                if (SnapshotsService.includesClusterUUID(minNodeVersion) == false) {
                     assertThat(TEST_STEP, is(TestStep.STEP3_OLD_CLUSTER));
-                    final List<Class<? extends Exception>> expectedExceptions =
-                        Arrays.asList(ResponseException.class, ElasticsearchStatusException.class);
-                    expectThrowsAnyOf(expectedExceptions, () -> listSnapshots(repoName));
-                    expectThrowsAnyOf(expectedExceptions, () -> deleteSnapshot(client, repoName, "snapshot-1"));
-                    expectThrowsAnyOf(expectedExceptions, () -> deleteSnapshot(client, repoName, "snapshot-2"));
-                    expectThrowsAnyOf(expectedExceptions, () -> createSnapshot(client, repoName, "snapshot-impossible", index));
+                    expectThrowsAnyOf(EXPECTED_BWC_EXCEPTIONS, () -> listSnapshots(repoName));
+                    expectThrowsAnyOf(EXPECTED_BWC_EXCEPTIONS, () -> deleteSnapshot(client, repoName, "snapshot-1"));
+                    expectThrowsAnyOf(EXPECTED_BWC_EXCEPTIONS, () -> deleteSnapshot(client, repoName, "snapshot-2"));
+                    expectThrowsAnyOf(EXPECTED_BWC_EXCEPTIONS, () -> createSnapshot(client, repoName, "snapshot-impossible", index));
                 } else {
                     assertThat(listSnapshots(repoName), hasSize(2));
                     if (TEST_STEP == TestStep.STEP4_NEW_CLUSTER) {
@@ -273,10 +284,12 @@ private void ensureSnapshotRestoreWorks(String repoName, String name, int shards
         }
     }
 
-    private static void createRepository(RestHighLevelClient client, String repoName, boolean readOnly) throws IOException {
+    private static void createRepository(RestHighLevelClient client, String repoName, boolean readOnly,
+                                         boolean verify) throws IOException {
         assertThat(client.snapshot().createRepository(new PutRepositoryRequest(repoName).type("fs").settings(
-            Settings.builder().put("location", "./" + repoName).put(READONLY_SETTING_KEY, readOnly)), RequestOptions.DEFAULT)
-                        .isAcknowledged(), is(true));
+                Settings.builder().put("location", "./" + repoName).put(READONLY_SETTING_KEY, readOnly))
+                        .verify(verify), RequestOptions.DEFAULT).isAcknowledged(),
+                is(true));
     }
 
     private static void createSnapshot(RestHighLevelClient client, String repoName, String name, String index) throws IOException {

server/src/internalClusterTest/java/org/elasticsearch/snapshots/CorruptedBlobStoreRepositoryIT.java

@@ -256,7 +256,8 @@ public void testHandlingMissingRootLevelSnapshotMetadata() throws Exception {
                 Collections.emptyMap(),
                 Collections.emptyMap(),
                 ShardGenerations.EMPTY,
-                IndexMetaDataGenerations.EMPTY);
+                IndexMetaDataGenerations.EMPTY,
+                RepositoryData.MISSING_UUID); // old-format repository has no cluster UUID
 
         Files.write(repo.resolve(BlobStoreRepository.INDEX_FILE_PREFIX + withoutVersions.getGenId()),
             BytesReference.toBytes(BytesReference.bytes(
@@ -388,7 +389,7 @@ public void testRepairBrokenShardGenerations() throws Exception {
                 snapshotIds.values().stream().collect(Collectors.toMap(SnapshotId::getUUID, repositoryData::getVersion)),
                 repositoryData.getIndices().values().stream().collect(Collectors.toMap(Function.identity(), repositoryData::getSnapshots)),
                 ShardGenerations.builder().putAll(repositoryData.shardGenerations()).put(indexId, 0, "0").build(),
-                repositoryData.indexMetaDataGenerations());
+                repositoryData.indexMetaDataGenerations(), repositoryData.getClusterUUID());
         Files.write(repoPath.resolve(BlobStoreRepository.INDEX_FILE_PREFIX + repositoryData.getGenId()),
                 BytesReference.toBytes(BytesReference.bytes(
                         brokenRepoData.snapshotsToXContent(XContentFactory.jsonBuilder(), Version.CURRENT))),

server/src/internalClusterTest/java/org/elasticsearch/snapshots/MultiClusterRepoAccessIT.java

@@ -94,7 +94,10 @@ public void testConcurrentDeleteFromOtherCluster() throws InterruptedException {
         assertThat(sne.getMessage(), containsString("failed to update snapshot in repository"));
         final RepositoryException cause = (RepositoryException) sne.getCause();
         assertThat(cause.getMessage(), containsString("[" + repoNameOnFirstCluster +
-                "] concurrent modification of the index-N file, expected current generation [2] but it was not found in the repository"));
+                "] concurrent modification of the index-N file, expected current generation [2] but it was not found in the repository."
+                + " The last cluster to write to this repository was ["
+                + secondCluster.client().admin().cluster().prepareState().get().getState().metadata().clusterUUID()
+                + "] at generation [4]."));
         assertAcked(client().admin().cluster().prepareDeleteRepository(repoNameOnFirstCluster).get());
         createRepository(repoNameOnFirstCluster, "fs", repoPath);
         createFullSnapshot(repoNameOnFirstCluster, "snap-5");