Skip to content

Commit a430e18

Browse files
kasundra07kasundra07
authored
Add Changes in Snapshot Delete Flow for remote store interoperability. (#7497)
Signed-off-by: Bansi Kasundra <kasundra@amazon.com>
1 parent c677f35 commit a430e18

File tree

5 files changed

+569
-57
lines changed

5 files changed

+569
-57
lines changed

‎server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java‎

Lines changed: 305 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,305 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.snapshots;
10+
11+
import org.opensearch.action.ActionFuture;
12+
import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
13+
import org.opensearch.client.Client;
14+
import org.opensearch.common.UUIDs;
15+
import org.opensearch.common.settings.Settings;
16+
import org.opensearch.common.unit.TimeValue;
17+
import org.opensearch.common.util.FeatureFlags;
18+
import org.opensearch.test.FeatureFlagSetter;
19+
import org.opensearch.test.OpenSearchIntegTestCase;
20+
21+
import java.nio.file.Path;
22+
import java.util.ArrayList;
23+
import java.util.List;
24+
import java.util.Locale;
25+
import java.util.stream.Stream;
26+
27+
import static org.hamcrest.Matchers.is;
28+
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
29+
30+
@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
31+
public class DeleteSnapshotIT extends AbstractSnapshotIntegTestCase {
32+
33+
public void testDeleteSnapshot() throws Exception {
34+
disableRepoConsistencyCheck("Remote store repository is being used in the test");
35+
FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE);
36+
internalCluster().startClusterManagerOnlyNode();
37+
internalCluster().startDataOnlyNode();
38+
39+
final String snapshotRepoName = "snapshot-repo-name";
40+
final Path snapshotRepoPath = randomRepoPath();
41+
createRepository(snapshotRepoName, "fs", snapshotRepoPath);
42+
43+
final Path remoteStoreRepoPath = randomRepoPath();
44+
final String remoteStoreRepoName = "remote-store-repo-name";
45+
createRepository(remoteStoreRepoName, "fs", remoteStoreRepoPath);
46+
47+
final String indexName = "index-1";
48+
createIndexWithRandomDocs(indexName, randomIntBetween(5, 10));
49+
50+
final String remoteStoreEnabledIndexName = "remote-index-1";
51+
final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(remoteStoreRepoName);
52+
createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings);
53+
indexRandomDocs(remoteStoreEnabledIndexName, randomIntBetween(5, 10));
54+
55+
final String snapshot = "snapshot";
56+
createFullSnapshot(snapshotRepoName, snapshot);
57+
assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 0);
58+
assert (getRepositoryData(snapshotRepoName).getSnapshotIds().size() == 1);
59+
60+
assertAcked(startDeleteSnapshot(snapshotRepoName, snapshot).get());
61+
assert (getRepositoryData(snapshotRepoName).getSnapshotIds().size() == 0);
62+
}
63+
64+
public void testDeleteShallowCopySnapshot() throws Exception {
65+
disableRepoConsistencyCheck("Remote store repository is being used in the test");
66+
FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE);
67+
internalCluster().startClusterManagerOnlyNode();
68+
internalCluster().startDataOnlyNode();
69+
70+
final String snapshotRepoName = "snapshot-repo-name";
71+
createRepository(snapshotRepoName, "fs", snapshotRepoSettingsForShallowCopy());
72+
73+
final Path remoteStoreRepoPath = randomRepoPath();
74+
final String remoteStoreRepoName = "remote-store-repo-name";
75+
createRepository(remoteStoreRepoName, "fs", remoteStoreRepoPath);
76+
77+
final String indexName = "index-1";
78+
createIndexWithRandomDocs(indexName, randomIntBetween(5, 10));
79+
80+
final String remoteStoreEnabledIndexName = "remote-index-1";
81+
final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(remoteStoreRepoName);
82+
createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings);
83+
indexRandomDocs(remoteStoreEnabledIndexName, randomIntBetween(5, 10));
84+
85+
final String shallowSnapshot = "shallow-snapshot";
86+
createFullSnapshot(snapshotRepoName, shallowSnapshot);
87+
assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 1);
88+
assert (getRepositoryData(snapshotRepoName).getSnapshotIds().size() == 1);
89+
90+
assertAcked(startDeleteSnapshot(snapshotRepoName, shallowSnapshot).get());
91+
assert (getRepositoryData(snapshotRepoName).getSnapshotIds().size() == 0);
92+
assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 0);
93+
}
94+
95+
// Deleting multiple shallow copy snapshots as part of single delete call with repo having only shallow copy snapshots.
96+
public void testDeleteMultipleShallowCopySnapshotsCase1() throws Exception {
97+
disableRepoConsistencyCheck("Remote store repository is being used in the test");
98+
FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE);
99+
100+
internalCluster().startClusterManagerOnlyNode();
101+
internalCluster().startDataOnlyNode();
102+
final Client clusterManagerClient = internalCluster().clusterManagerClient();
103+
ensureStableCluster(2);
104+
105+
final String snapshotRepoName = "snapshot-repo-name";
106+
final Path snapshotRepoPath = randomRepoPath();
107+
createRepository(snapshotRepoName, "mock", snapshotRepoSettingsForShallowCopy(snapshotRepoPath));
108+
final String testIndex = "index-test";
109+
createIndexWithContent(testIndex);
110+
111+
final Path remoteStoreRepoPath = randomRepoPath();
112+
final String remoteStoreRepoName = "remote-store-repo-name";
113+
createRepository(remoteStoreRepoName, "fs", remoteStoreRepoPath);
114+
115+
final String remoteStoreEnabledIndexName = "remote-index-1";
116+
final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(remoteStoreRepoName);
117+
createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings);
118+
indexRandomDocs(remoteStoreEnabledIndexName, randomIntBetween(5, 10));
119+
120+
// Creating some shallow copy snapshots
121+
int totalShallowCopySnapshotsCount = randomIntBetween(4, 10);
122+
List<String> shallowCopySnapshots = createNSnapshots(snapshotRepoName, totalShallowCopySnapshotsCount);
123+
List<String> snapshotsToBeDeleted = shallowCopySnapshots.subList(0, randomIntBetween(2, totalShallowCopySnapshotsCount));
124+
int tobeDeletedSnapshotsCount = snapshotsToBeDeleted.size();
125+
assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == totalShallowCopySnapshotsCount);
126+
assert (getRepositoryData(snapshotRepoName).getSnapshotIds().size() == totalShallowCopySnapshotsCount);
127+
// Deleting subset of shallow copy snapshots
128+
assertAcked(
129+
clusterManagerClient.admin()
130+
.cluster()
131+
.prepareDeleteSnapshot(snapshotRepoName, snapshotsToBeDeleted.toArray(new String[0]))
132+
.get()
133+
);
134+
assert (getRepositoryData(snapshotRepoName).getSnapshotIds().size() == totalShallowCopySnapshotsCount - tobeDeletedSnapshotsCount);
135+
assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == totalShallowCopySnapshotsCount
136+
- tobeDeletedSnapshotsCount);
137+
}
138+
139+
// Deleting multiple shallow copy snapshots as part of single delete call with both partial and full copy snapshot present in the repo
140+
// And then deleting multiple full copy snapshots as part of single delete call with both partial and shallow copy snapshots present in
141+
// the repo
142+
@AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8610")
143+
public void testDeleteMultipleShallowCopySnapshotsCase2() throws Exception {
144+
disableRepoConsistencyCheck("Remote store repository is being used in the test");
145+
FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE);
146+
147+
internalCluster().startClusterManagerOnlyNode();
148+
final String dataNode = internalCluster().startDataOnlyNode();
149+
ensureStableCluster(2);
150+
final String clusterManagerNode = internalCluster().getClusterManagerName();
151+
152+
final String snapshotRepoName = "snapshot-repo-name";
153+
final Path snapshotRepoPath = randomRepoPath();
154+
createRepository(snapshotRepoName, "mock", snapshotRepoSettingsForShallowCopy(snapshotRepoPath));
155+
final String testIndex = "index-test";
156+
createIndexWithContent(testIndex);
157+
158+
final Path remoteStoreRepoPath = randomRepoPath();
159+
final String remoteStoreRepoName = "remote-store-repo-name";
160+
createRepository(remoteStoreRepoName, "fs", remoteStoreRepoPath);
161+
162+
final String remoteStoreEnabledIndexName = "remote-index-1";
163+
final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(remoteStoreRepoName);
164+
createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings);
165+
indexRandomDocs(remoteStoreEnabledIndexName, randomIntBetween(5, 10));
166+
167+
// Creating a partial shallow copy snapshot
168+
final String snapshot = "snapshot";
169+
blockNodeWithIndex(snapshotRepoName, testIndex);
170+
blockDataNode(snapshotRepoName, dataNode);
171+
172+
final Client clusterManagerClient = internalCluster().clusterManagerClient();
173+
final ActionFuture<CreateSnapshotResponse> snapshotFuture = clusterManagerClient.admin()
174+
.cluster()
175+
.prepareCreateSnapshot(snapshotRepoName, snapshot)
176+
.setWaitForCompletion(true)
177+
.execute();
178+
179+
awaitNumberOfSnapshotsInProgress(1);
180+
waitForBlock(dataNode, snapshotRepoName, TimeValue.timeValueSeconds(30L));
181+
internalCluster().restartNode(dataNode);
182+
assertThat(snapshotFuture.get().getSnapshotInfo().state(), is(SnapshotState.PARTIAL));
183+
184+
unblockAllDataNodes(snapshotRepoName);
185+
186+
ensureStableCluster(2, clusterManagerNode);
187+
188+
// Creating some shallow copy snapshots
189+
int totalShallowCopySnapshotsCount = randomIntBetween(4, 10);
190+
List<String> shallowCopySnapshots = createNSnapshots(snapshotRepoName, totalShallowCopySnapshotsCount);
191+
List<String> shallowCopySnapshotsToBeDeleted = shallowCopySnapshots.subList(0, randomIntBetween(2, totalShallowCopySnapshotsCount));
192+
int tobeDeletedShallowCopySnapshotsCount = shallowCopySnapshotsToBeDeleted.size();
193+
totalShallowCopySnapshotsCount += 1; // Adding partial shallow snapshot here
194+
// Updating the snapshot repository flag to disable shallow snapshots
195+
createRepository(snapshotRepoName, "mock", snapshotRepoPath);
196+
// Creating some full copy snapshots
197+
int totalFullCopySnapshotsCount = randomIntBetween(4, 10);
198+
List<String> fullCopySnapshots = createNSnapshots(snapshotRepoName, totalFullCopySnapshotsCount);
199+
List<String> fullCopySnapshotsToBeDeleted = fullCopySnapshots.subList(0, randomIntBetween(2, totalFullCopySnapshotsCount));
200+
int tobeDeletedFullCopySnapshotsCount = fullCopySnapshotsToBeDeleted.size();
201+
202+
int totalSnapshotsCount = totalFullCopySnapshotsCount + totalShallowCopySnapshotsCount;
203+
204+
assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == totalShallowCopySnapshotsCount);
205+
assert (getRepositoryData(snapshotRepoName).getSnapshotIds().size() == totalSnapshotsCount);
206+
// Deleting subset of shallow copy snapshots
207+
assertAcked(
208+
clusterManagerClient.admin()
209+
.cluster()
210+
.prepareDeleteSnapshot(snapshotRepoName, shallowCopySnapshotsToBeDeleted.toArray(new String[0]))
211+
.get()
212+
);
213+
totalSnapshotsCount -= tobeDeletedShallowCopySnapshotsCount;
214+
totalShallowCopySnapshotsCount -= tobeDeletedShallowCopySnapshotsCount;
215+
assert (getRepositoryData(snapshotRepoName).getSnapshotIds().size() == totalSnapshotsCount);
216+
assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == totalShallowCopySnapshotsCount);
217+
218+
// Deleting subset of full copy snapshots
219+
assertAcked(
220+
clusterManagerClient.admin()
221+
.cluster()
222+
.prepareDeleteSnapshot(snapshotRepoName, fullCopySnapshotsToBeDeleted.toArray(new String[0]))
223+
.get()
224+
);
225+
totalSnapshotsCount -= tobeDeletedFullCopySnapshotsCount;
226+
assert (getRepositoryData(snapshotRepoName).getSnapshotIds().size() == totalSnapshotsCount);
227+
assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == totalShallowCopySnapshotsCount);
228+
}
229+
230+
// Deleting subset of shallow and full copy snapshots as part of single delete call and then deleting all snapshots in the repo.
231+
@AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8610")
232+
public void testDeleteMultipleShallowCopySnapshotsCase3() throws Exception {
233+
disableRepoConsistencyCheck("Remote store repository is being used in the test");
234+
FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE);
235+
236+
internalCluster().startClusterManagerOnlyNode();
237+
internalCluster().startDataOnlyNode();
238+
final Client clusterManagerClient = internalCluster().clusterManagerClient();
239+
ensureStableCluster(2);
240+
241+
final String snapshotRepoName = "snapshot-repo-name";
242+
final Path snapshotRepoPath = randomRepoPath();
243+
createRepository(snapshotRepoName, "mock", snapshotRepoSettingsForShallowCopy(snapshotRepoPath));
244+
final String testIndex = "index-test";
245+
createIndexWithContent(testIndex);
246+
247+
final Path remoteStoreRepoPath = randomRepoPath();
248+
final String remoteStoreRepoName = "remote-store-repo-name";
249+
createRepository(remoteStoreRepoName, "fs", remoteStoreRepoPath);
250+
251+
final String remoteStoreEnabledIndexName = "remote-index-1";
252+
final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(remoteStoreRepoName);
253+
createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings);
254+
indexRandomDocs(remoteStoreEnabledIndexName, randomIntBetween(5, 10));
255+
256+
// Creating some shallow copy snapshots
257+
int totalShallowCopySnapshotsCount = randomIntBetween(4, 10);
258+
List<String> shallowCopySnapshots = createNSnapshots(snapshotRepoName, totalShallowCopySnapshotsCount);
259+
List<String> shallowCopySnapshotsToBeDeleted = shallowCopySnapshots.subList(0, randomIntBetween(2, totalShallowCopySnapshotsCount));
260+
int tobeDeletedShallowCopySnapshotsCount = shallowCopySnapshotsToBeDeleted.size();
261+
// Updating the snapshot repository flag to disable shallow snapshots
262+
createRepository(snapshotRepoName, "mock", snapshotRepoPath);
263+
// Creating some full copy snapshots
264+
int totalFullCopySnapshotsCount = randomIntBetween(4, 10);
265+
List<String> fullCopySnapshots = createNSnapshots(snapshotRepoName, totalFullCopySnapshotsCount);
266+
List<String> fullCopySnapshotsToBeDeleted = fullCopySnapshots.subList(0, randomIntBetween(2, totalFullCopySnapshotsCount));
267+
int tobeDeletedFullCopySnapshotsCount = fullCopySnapshotsToBeDeleted.size();
268+
269+
int totalSnapshotsCount = totalFullCopySnapshotsCount + totalShallowCopySnapshotsCount;
270+
271+
assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == totalShallowCopySnapshotsCount);
272+
assert (getRepositoryData(snapshotRepoName).getSnapshotIds().size() == totalSnapshotsCount);
273+
// Deleting subset of shallow copy snapshots and full copy snapshots
274+
assertAcked(
275+
clusterManagerClient.admin()
276+
.cluster()
277+
.prepareDeleteSnapshot(
278+
snapshotRepoName,
279+
Stream.concat(shallowCopySnapshotsToBeDeleted.stream(), fullCopySnapshotsToBeDeleted.stream()).toArray(String[]::new)
280+
)
281+
.get()
282+
);
283+
totalSnapshotsCount -= (tobeDeletedShallowCopySnapshotsCount + tobeDeletedFullCopySnapshotsCount);
284+
totalShallowCopySnapshotsCount -= tobeDeletedShallowCopySnapshotsCount;
285+
assert (getRepositoryData(snapshotRepoName).getSnapshotIds().size() == totalSnapshotsCount);
286+
assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == totalShallowCopySnapshotsCount);
287+
288+
// Deleting all the remaining snapshots
289+
assertAcked(clusterManagerClient.admin().cluster().prepareDeleteSnapshot(snapshotRepoName, "*").get());
290+
assert (getRepositoryData(snapshotRepoName).getSnapshotIds().size() == 0);
291+
assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 0);
292+
}
293+
294+
private List<String> createNSnapshots(String repoName, int count) {
295+
final List<String> snapshotNames = new ArrayList<>(count);
296+
final String prefix = "snap-" + UUIDs.randomBase64UUID(random()).toLowerCase(Locale.ROOT) + "-";
297+
for (int i = 0; i < count; i++) {
298+
final String name = prefix + i;
299+
createFullSnapshot(repoName, name);
300+
snapshotNames.add(name);
301+
}
302+
logger.info("--> created {} in [{}]", snapshotNames, repoName);
303+
return snapshotNames;
304+
}
305+
}

‎server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@
5252
import org.opensearch.common.Nullable;
5353
import org.opensearch.common.inject.Inject;
5454
import org.opensearch.common.io.stream.StreamInput;
55+
import org.opensearch.index.store.lockmanager.RemoteStoreLockManagerFactory;
5556
import org.opensearch.repositories.RepositoriesService;
5657
import org.opensearch.repositories.Repository;
5758
import org.opensearch.repositories.RepositoryCleanupResult;
@@ -93,6 +94,8 @@ public final class TransportCleanupRepositoryAction extends TransportClusterMana
9394

9495
private final SnapshotsService snapshotsService;
9596

97+
private final RemoteStoreLockManagerFactory remoteStoreLockManagerFactory;
98+
9699
@Override
97100
protected String executor() {
98101
return ThreadPool.Names.SAME;
@@ -119,6 +122,7 @@ public TransportCleanupRepositoryAction(
119122
);
120123
this.repositoriesService = repositoriesService;
121124
this.snapshotsService = snapshotsService;
125+
this.remoteStoreLockManagerFactory = new RemoteStoreLockManagerFactory(() -> repositoriesService);
122126
// We add a state applier that will remove any dangling repository cleanup actions on cluster-manager failover.
123127
// This is safe to do since cleanups will increment the repository state id before executing any operations to prevent concurrent
124128
// operations from corrupting the repository. This is the same safety mechanism used by snapshot deletes.
@@ -267,6 +271,7 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS
267271
l -> blobStoreRepository.cleanup(
268272
repositoryStateId,
269273
snapshotsService.minCompatibleVersion(newState.nodes().getMinNodeVersion(), repositoryData, null),
274+
remoteStoreLockManagerFactory,
270275
ActionListener.wrap(result -> after(null, result), e -> after(e, null))
271276
)
272277
)

0 commit comments

Comments
 (0)