Skip to content

Commit 3bb56f7

Browse files
martijnvgmartijnvg
committed
Make DataStreamsSnapshotsIT resilient to failures because of local time. (elastic#73516)
Backing index names contain a date component. Instead of defining what the expected backing index names are before creating the data streams, resolve these expected backing index names after the data streams are created. This way we avoid failures that can occur around midnight (local time), where the expected names are created before midnight and the data streams are created after midnight. Closes elastic#73510
1 parent f454cef commit 3bb56f7

File tree

1 file changed

+29
-21
lines changed

1 file changed

+29
-21
lines changed

‎x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java

Lines changed: 29 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -70,13 +70,13 @@
7070
@ESIntegTestCase.ClusterScope(transportClientRatio = 0)
7171
public class DataStreamsSnapshotsIT extends AbstractSnapshotIntegTestCase {
7272

73-
private static final String DS_BACKING_INDEX_NAME = DataStream.getDefaultBackingIndexName("ds", 1);
74-
private static final String DS2_BACKING_INDEX_NAME = DataStream.getDefaultBackingIndexName("ds2", 1);
7573
private static final Map<String, Integer> DOCUMENT_SOURCE = Collections.singletonMap("@timestamp", 123);
7674
public static final String REPO = "repo";
7775
public static final String SNAPSHOT = "snap";
7876
private Client client;
7977

78+
private String dsBackingIndexName;
79+
private String ds2BackingIndexName;
8080
private String id;
8181

8282
@Override
@@ -100,6 +100,14 @@ public void setup() throws Exception {
100100
response = client.execute(CreateDataStreamAction.INSTANCE, request).get();
101101
assertTrue(response.isAcknowledged());
102102

103+
// Resolve backing index names after data streams have been created:
104+
// (these names have a date component, and running around midnight could lead to test failures otherwise)
105+
GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { "*" });
106+
GetDataStreamAction.Response getDataStreamResponse = client.execute(GetDataStreamAction.INSTANCE, getDataStreamRequest).actionGet();
107+
dsBackingIndexName = getDataStreamResponse.getDataStreams().get(0).getDataStream().getIndices().get(0).getName();
108+
// Will be used in some tests, to test renaming while restoring a snapshot:
109+
ds2BackingIndexName = dsBackingIndexName.replace("-ds-", "-ds2-");
110+
103111
IndexResponse indexResponse = client.prepareIndex("ds", "_doc")
104112
.setOpType(DocWriteRequest.OpType.CREATE)
105113
.setSource(DOCUMENT_SOURCE)
@@ -134,7 +142,7 @@ public void testSnapshotAndRestore() throws Exception {
134142
RestStatus status = createSnapshotResponse.getSnapshotInfo().status();
135143
assertEquals(RestStatus.OK, status);
136144

137-
assertEquals(Collections.singletonList(DS_BACKING_INDEX_NAME), getSnapshot(REPO, SNAPSHOT).indices());
145+
assertEquals(Collections.singletonList(dsBackingIndexName), getSnapshot(REPO, SNAPSHOT).indices());
138146

139147
assertTrue(
140148
client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(new String[] { "ds" }))
@@ -151,7 +159,7 @@ public void testSnapshotAndRestore() throws Exception {
151159

152160
assertEquals(1, restoreSnapshotResponse.getRestoreInfo().successfulShards());
153161

154-
assertEquals(DOCUMENT_SOURCE, client.prepareGet(DS_BACKING_INDEX_NAME, "_doc", id).get().getSourceAsMap());
162+
assertEquals(DOCUMENT_SOURCE, client.prepareGet(dsBackingIndexName, "_doc", id).get().getSourceAsMap());
155163
SearchHit[] hits = client.prepareSearch("ds").get().getHits().getHits();
156164
assertEquals(1, hits.length);
157165
assertEquals(DOCUMENT_SOURCE, hits[0].getSourceAsMap());
@@ -162,7 +170,7 @@ public void testSnapshotAndRestore() throws Exception {
162170
).get();
163171
assertEquals(1, ds.getDataStreams().size());
164172
assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size());
165-
assertEquals(DS_BACKING_INDEX_NAME, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName());
173+
assertEquals(dsBackingIndexName, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName());
166174

167175
GetAliasesResponse getAliasesResponse = client.admin().indices().getAliases(new GetAliasesRequest("my-alias")).actionGet();
168176
assertThat(getAliasesResponse.getDataStreamAliases().keySet(), containsInAnyOrder("ds", "other-ds"));
@@ -184,7 +192,7 @@ public void testSnapshotAndRestoreAllDataStreamsInPlace() throws Exception {
184192
RestStatus status = createSnapshotResponse.getSnapshotInfo().status();
185193
assertEquals(RestStatus.OK, status);
186194

187-
assertEquals(Collections.singletonList(DS_BACKING_INDEX_NAME), getSnapshot(REPO, SNAPSHOT).indices());
195+
assertEquals(Collections.singletonList(dsBackingIndexName), getSnapshot(REPO, SNAPSHOT).indices());
188196

189197
// Close all indices:
190198
CloseIndexRequest closeIndexRequest = new CloseIndexRequest("*");
@@ -199,7 +207,7 @@ public void testSnapshotAndRestoreAllDataStreamsInPlace() throws Exception {
199207
.get();
200208
assertEquals(1, restoreSnapshotResponse.getRestoreInfo().successfulShards());
201209

202-
assertEquals(DOCUMENT_SOURCE, client.prepareGet(DS_BACKING_INDEX_NAME, "_doc", id).get().getSourceAsMap());
210+
assertEquals(DOCUMENT_SOURCE, client.prepareGet(dsBackingIndexName, "_doc", id).get().getSourceAsMap());
203211
SearchHit[] hits = client.prepareSearch("ds").get().getHits().getHits();
204212
assertEquals(1, hits.length);
205213
assertEquals(DOCUMENT_SOURCE, hits[0].getSourceAsMap());
@@ -211,7 +219,7 @@ public void testSnapshotAndRestoreAllDataStreamsInPlace() throws Exception {
211219
contains(equalTo("ds"), equalTo("other-ds"))
212220
);
213221
java.util.List<Index> backingIndices = ds.getDataStreams().get(0).getDataStream().getIndices();
214-
assertThat(backingIndices.stream().map(Index::getName).collect(Collectors.toList()), contains(DS_BACKING_INDEX_NAME));
222+
assertThat(backingIndices.stream().map(Index::getName).collect(Collectors.toList()), contains(dsBackingIndexName));
215223
backingIndices = ds.getDataStreams().get(1).getDataStream().getIndices();
216224
String expectedBackingIndexName = DataStream.getDefaultBackingIndexName("other-ds", 1);
217225
assertThat(backingIndices.stream().map(Index::getName).collect(Collectors.toList()), contains(expectedBackingIndexName));
@@ -229,7 +237,7 @@ public void testSnapshotAndRestoreInPlace() throws Exception {
229237
RestStatus status = createSnapshotResponse.getSnapshotInfo().status();
230238
assertEquals(RestStatus.OK, status);
231239

232-
assertEquals(Collections.singletonList(DS_BACKING_INDEX_NAME), getSnapshot(REPO, SNAPSHOT).indices());
240+
assertEquals(Collections.singletonList(dsBackingIndexName), getSnapshot(REPO, SNAPSHOT).indices());
233241

234242
// A rollover after taking snapshot. The new backing index should be a standalone index after restoring
235243
// and not part of the data stream:
@@ -251,7 +259,7 @@ public void testSnapshotAndRestoreInPlace() throws Exception {
251259
.get();
252260
assertEquals(1, restoreSnapshotResponse.getRestoreInfo().successfulShards());
253261

254-
assertEquals(DOCUMENT_SOURCE, client.prepareGet(DS_BACKING_INDEX_NAME, "_doc", id).get().getSourceAsMap());
262+
assertEquals(DOCUMENT_SOURCE, client.prepareGet(dsBackingIndexName, "_doc", id).get().getSourceAsMap());
255263
SearchHit[] hits = client.prepareSearch("ds").get().getHits().getHits();
256264
assertEquals(1, hits.length);
257265
assertEquals(DOCUMENT_SOURCE, hits[0].getSourceAsMap());
@@ -264,7 +272,7 @@ public void testSnapshotAndRestoreInPlace() throws Exception {
264272
);
265273
java.util.List<Index> backingIndices = ds.getDataStreams().get(0).getDataStream().getIndices();
266274
assertThat(ds.getDataStreams().get(0).getDataStream().getIndices(), hasSize(1));
267-
assertThat(backingIndices.stream().map(Index::getName).collect(Collectors.toList()), contains(equalTo(DS_BACKING_INDEX_NAME)));
275+
assertThat(backingIndices.stream().map(Index::getName).collect(Collectors.toList()), contains(equalTo(dsBackingIndexName)));
268276

269277
// The backing index created as part of rollover should still exist (but just not part of the data stream)
270278
assertThat(indexExists(DataStream.getDefaultBackingIndexName("ds", 2)), is(true));
@@ -287,7 +295,7 @@ public void testSnapshotAndRestoreAll() throws Exception {
287295
RestStatus status = createSnapshotResponse.getSnapshotInfo().status();
288296
assertEquals(RestStatus.OK, status);
289297

290-
assertEquals(Collections.singletonList(DS_BACKING_INDEX_NAME), getSnapshot(REPO, SNAPSHOT).indices());
298+
assertEquals(Collections.singletonList(dsBackingIndexName), getSnapshot(REPO, SNAPSHOT).indices());
291299

292300
assertAcked(client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(new String[] { "*" })).get());
293301
assertAcked(client.admin().indices().prepareDelete("*").setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN));
@@ -301,7 +309,7 @@ public void testSnapshotAndRestoreAll() throws Exception {
301309

302310
assertEquals(1, restoreSnapshotResponse.getRestoreInfo().successfulShards());
303311

304-
assertEquals(DOCUMENT_SOURCE, client.prepareGet(DS_BACKING_INDEX_NAME, "_doc", id).get().getSourceAsMap());
312+
assertEquals(DOCUMENT_SOURCE, client.prepareGet(dsBackingIndexName, "_doc", id).get().getSourceAsMap());
305313
SearchHit[] hits = client.prepareSearch("ds").get().getHits().getHits();
306314
assertEquals(1, hits.length);
307315
assertEquals(DOCUMENT_SOURCE, hits[0].getSourceAsMap());
@@ -312,7 +320,7 @@ public void testSnapshotAndRestoreAll() throws Exception {
312320
).get();
313321
assertEquals(1, ds.getDataStreams().size());
314322
assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size());
315-
assertEquals(DS_BACKING_INDEX_NAME, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName());
323+
assertEquals(dsBackingIndexName, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName());
316324

317325
GetAliasesResponse getAliasesResponse = client.admin().indices().getAliases(new GetAliasesRequest("my-alias")).actionGet();
318326
assertThat(getAliasesResponse.getDataStreamAliases().keySet(), containsInAnyOrder("ds"));
@@ -353,9 +361,9 @@ public void testRename() throws Exception {
353361
).get();
354362
assertEquals(1, ds.getDataStreams().size());
355363
assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size());
356-
assertEquals(DS2_BACKING_INDEX_NAME, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName());
364+
assertEquals(ds2BackingIndexName, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName());
357365
assertEquals(DOCUMENT_SOURCE, client.prepareSearch("ds2").get().getHits().getHits()[0].getSourceAsMap());
358-
assertEquals(DOCUMENT_SOURCE, client.prepareGet(DS2_BACKING_INDEX_NAME, "_doc", id).get().getSourceAsMap());
366+
assertEquals(DOCUMENT_SOURCE, client.prepareGet(ds2BackingIndexName, "_doc", id).get().getSourceAsMap());
359367

360368
GetAliasesResponse getAliasesResponse = client.admin().indices().getAliases(new GetAliasesRequest("my-alias")).actionGet();
361369
assertThat(getAliasesResponse.getDataStreamAliases().keySet(), containsInAnyOrder("ds", "ds2", "other-ds"));
@@ -393,15 +401,15 @@ public void testBackingIndexIsNotRenamedWhenRestoringDataStream() {
393401
.prepareRestoreSnapshot(REPO, SNAPSHOT)
394402
.setWaitForCompletion(true)
395403
.setIndices("ds")
396-
.setRenamePattern(DS_BACKING_INDEX_NAME)
404+
.setRenamePattern(dsBackingIndexName)
397405
.setRenameReplacement("new_index_name")
398406
.get();
399407

400408
assertThat(restoreSnapshotResponse.status(), is(RestStatus.OK));
401409

402410
GetDataStreamAction.Request getDSRequest = new GetDataStreamAction.Request(new String[] { "ds" });
403411
GetDataStreamAction.Response response = client.execute(GetDataStreamAction.INSTANCE, getDSRequest).actionGet();
404-
assertThat(response.getDataStreams().get(0).getDataStream().getIndices().get(0).getName(), is(DS_BACKING_INDEX_NAME));
412+
assertThat(response.getDataStreams().get(0).getDataStream().getIndices().get(0).getName(), is(dsBackingIndexName));
405413
}
406414

407415
public void testDataStreamAndBackingIndicesAreRenamedUsingRegex() {
@@ -444,7 +452,7 @@ public void testDataStreamAndBackingIndicesAreRenamedUsingRegex() {
444452
// data stream "ds" should still exist in the system
445453
GetDataStreamAction.Request getDSRequest = new GetDataStreamAction.Request(new String[] { "ds" });
446454
response = client.execute(GetDataStreamAction.INSTANCE, getDSRequest).actionGet();
447-
assertThat(response.getDataStreams().get(0).getDataStream().getIndices().get(0).getName(), is(DS_BACKING_INDEX_NAME));
455+
assertThat(response.getDataStreams().get(0).getDataStream().getIndices().get(0).getName(), is(dsBackingIndexName));
448456
}
449457

450458
public void testWildcards() throws Exception {
@@ -476,7 +484,7 @@ public void testWildcards() throws Exception {
476484
).get();
477485
assertEquals(1, ds.getDataStreams().size());
478486
assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size());
479-
assertEquals(DS2_BACKING_INDEX_NAME, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName());
487+
assertEquals(ds2BackingIndexName, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName());
480488
assertThat(
481489
"we renamed the restored data stream to one that doesn't match any existing composable template",
482490
ds.getDataStreams().get(0).getIndexTemplate(),
@@ -489,7 +497,7 @@ public void testDataStreamNotStoredWhenIndexRequested() {
489497
.cluster()
490498
.prepareCreateSnapshot(REPO, "snap2")
491499
.setWaitForCompletion(true)
492-
.setIndices(DS_BACKING_INDEX_NAME)
500+
.setIndices(dsBackingIndexName)
493501
.setIncludeGlobalState(false)
494502
.get();
495503

0 commit comments

Comments
 (0)