Skip to content

Ensure ILM policies run safely on leader indices #38140

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Feb 2, 2019
Merged

Ensure ILM policies run safely on leader indices #38140

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
962d5ba
Ensure ILM policies run safely on leader indices
gwbrown Jan 31, 2019
fa2e3a9
Fix test
gwbrown Jan 31, 2019
2c175ab
Fix integration test
gwbrown Feb 1, 2019
1de1466
Clean up unused functions
gwbrown Feb 1, 2019
906da63
Review feedback
gwbrown Feb 1, 2019
99a2acf
Merge remote-tracking branch 'elastic/master' into pr/38140
jasontedor Feb 1, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 10 additions & 3 deletions ...k/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/DeleteAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;

import java.io.IOException;
import java.util.Collections;
import java.util.Arrays;
import java.util.List;

/**
Expand Down Expand Up @@ -59,13 +59,20 @@ public boolean isSafeAction() {

@Override
public List<Step> toSteps(Client client, String phase, Step.StepKey nextStepKey) {
Step.StepKey waitForNoFollowerStepKey = new Step.StepKey(phase, NAME, WaitForNoFollowersStep.NAME);
Step.StepKey deleteStepKey = new Step.StepKey(phase, NAME, DeleteStep.NAME);
return Collections.singletonList(new DeleteStep(deleteStepKey, nextStepKey, client));

WaitForNoFollowersStep waitForNoFollowersStep = new WaitForNoFollowersStep(waitForNoFollowerStepKey, deleteStepKey, client);
DeleteStep deleteStep = new DeleteStep(deleteStepKey, nextStepKey, client);
return Arrays.asList(waitForNoFollowersStep, deleteStep);
}

@Override
public List<StepKey> toStepKeys(String phase) {
return Collections.singletonList(new Step.StepKey(phase, NAME, DeleteStep.NAME));
Step.StepKey waitForNoFollowerStepKey = new Step.StepKey(phase, NAME, WaitForNoFollowersStep.NAME);
Step.StepKey deleteStepKey = new Step.StepKey(phase, NAME, DeleteStep.NAME);

return Arrays.asList(waitForNoFollowerStepKey, deleteStepKey);
}

@Override
Expand Down
13 changes: 8 additions & 5 deletions ...k/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ShrinkAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ public List<Step> toSteps(Client client, String phase, Step.StepKey nextStepKey)
Settings readOnlySettings = Settings.builder().put(IndexMetaData.SETTING_BLOCKS_WRITE, true).build();

StepKey branchingKey = new StepKey(phase, NAME, BranchingStep.NAME);
StepKey waitForNoFollowerStepKey = new StepKey(phase, NAME, WaitForNoFollowersStep.NAME);
StepKey readOnlyKey = new StepKey(phase, NAME, ReadOnlyAction.NAME);
StepKey setSingleNodeKey = new StepKey(phase, NAME, SetSingleNodeAllocateStep.NAME);
StepKey allocationRoutedKey = new StepKey(phase, NAME, CheckShrinkReadyStep.NAME);
Expand All @@ -95,8 +96,9 @@ public List<Step> toSteps(Client client, String phase, Step.StepKey nextStepKey)
StepKey aliasKey = new StepKey(phase, NAME, ShrinkSetAliasStep.NAME);
StepKey isShrunkIndexKey = new StepKey(phase, NAME, ShrunkenIndexCheckStep.NAME);

BranchingStep conditionalSkipShrinkStep = new BranchingStep(branchingKey, readOnlyKey, nextStepKey,
BranchingStep conditionalSkipShrinkStep = new BranchingStep(branchingKey, waitForNoFollowerStepKey, nextStepKey,
(index, clusterState) -> clusterState.getMetaData().index(index).getNumberOfShards() == numberOfShards);
WaitForNoFollowersStep waitForNoFollowersStep = new WaitForNoFollowersStep(waitForNoFollowerStepKey, readOnlyKey, client);
UpdateSettingsStep readOnlyStep = new UpdateSettingsStep(readOnlyKey, setSingleNodeKey, client, readOnlySettings);
SetSingleNodeAllocateStep setSingleNodeStep = new SetSingleNodeAllocateStep(setSingleNodeKey, allocationRoutedKey, client);
CheckShrinkReadyStep checkShrinkReadyStep = new CheckShrinkReadyStep(allocationRoutedKey, shrinkKey);
Expand All @@ -105,13 +107,14 @@ public List<Step> toSteps(Client client, String phase, Step.StepKey nextStepKey)
CopyExecutionStateStep copyMetadata = new CopyExecutionStateStep(copyMetadataKey, aliasKey, SHRUNKEN_INDEX_PREFIX);
ShrinkSetAliasStep aliasSwapAndDelete = new ShrinkSetAliasStep(aliasKey, isShrunkIndexKey, client, SHRUNKEN_INDEX_PREFIX);
ShrunkenIndexCheckStep waitOnShrinkTakeover = new ShrunkenIndexCheckStep(isShrunkIndexKey, nextStepKey, SHRUNKEN_INDEX_PREFIX);
return Arrays.asList(conditionalSkipShrinkStep, readOnlyStep, setSingleNodeStep, checkShrinkReadyStep, shrink, allocated,
copyMetadata, aliasSwapAndDelete, waitOnShrinkTakeover);
return Arrays.asList(conditionalSkipShrinkStep, waitForNoFollowersStep, readOnlyStep, setSingleNodeStep, checkShrinkReadyStep,
shrink, allocated, copyMetadata, aliasSwapAndDelete, waitOnShrinkTakeover);
}

@Override
public List<StepKey> toStepKeys(String phase) {
StepKey conditionalSkipKey = new StepKey(phase, NAME, BranchingStep.NAME);
StepKey waitForNoFollowerStepKey = new StepKey(phase, NAME, WaitForNoFollowersStep.NAME);
StepKey readOnlyKey = new StepKey(phase, NAME, ReadOnlyAction.NAME);
StepKey setSingleNodeKey = new StepKey(phase, NAME, SetSingleNodeAllocateStep.NAME);
StepKey checkShrinkReadyKey = new StepKey(phase, NAME, CheckShrinkReadyStep.NAME);
Expand All @@ -120,8 +123,8 @@ public List<StepKey> toStepKeys(String phase) {
StepKey copyMetadataKey = new StepKey(phase, NAME, CopyExecutionStateStep.NAME);
StepKey aliasKey = new StepKey(phase, NAME, ShrinkSetAliasStep.NAME);
StepKey isShrunkIndexKey = new StepKey(phase, NAME, ShrunkenIndexCheckStep.NAME);
return Arrays.asList(conditionalSkipKey, readOnlyKey, setSingleNodeKey, checkShrinkReadyKey, shrinkKey, enoughShardsKey,
copyMetadataKey, aliasKey, isShrunkIndexKey);
return Arrays.asList(conditionalSkipKey, waitForNoFollowerStepKey, readOnlyKey, setSingleNodeKey, checkShrinkReadyKey, shrinkKey,
enoughShardsKey, copyMetadataKey, aliasKey, isShrunkIndexKey);
}

@Override
Expand Down
108 changes: 108 additions & 0 deletions ...ore/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForNoFollowersStep.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

package org.elasticsearch.xpack.core.indexlifecycle;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.indices.stats.IndexStats;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest;
import org.elasticsearch.action.admin.indices.stats.ShardStats;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;

import java.io.IOException;
import java.util.Arrays;
import java.util.Objects;

/**
* A step that waits until the index it's used on is no longer a leader index.
* This is necessary as there are some actions which are not safe to perform on
* a leader index, such as those which delete the index, including Shrink and
* Delete.
*/
public class WaitForNoFollowersStep extends AsyncWaitStep {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you add javadoc please?


private static final Logger logger = LogManager.getLogger(WaitForNoFollowersStep.class);

static final String NAME = "wait-for-shard-history-leases";
static final String CCR_LEASE_KEY = "ccr";

WaitForNoFollowersStep(StepKey key, StepKey nextStepKey, Client client) {
super(key, nextStepKey, client);
}

@Override
public void evaluateCondition(IndexMetaData indexMetaData, Listener listener) {
IndicesStatsRequest request = new IndicesStatsRequest();
request.clear();
String indexName = indexMetaData.getIndex().getName();
request.indices(indexName);
getClient().admin().indices().stats(request, ActionListener.wrap((response) -> {
IndexStats indexStats = response.getIndex(indexName);
if (indexStats == null) {
// Index was probably deleted
logger.debug("got null shard stats for index {}, proceeding on the assumption it has been deleted",
indexMetaData.getIndex());
listener.onResponse(true, null);
return;
}

boolean isCurrentlyLeaderIndex = Arrays.stream(indexStats.getShards())
.map(ShardStats::getRetentionLeaseStats)
.flatMap(retentionLeaseStats -> retentionLeaseStats.retentionLeases().leases().stream())
.anyMatch(lease -> CCR_LEASE_KEY.equals(lease.source()));

if (isCurrentlyLeaderIndex) {
listener.onResponse(false, new Info());
} else {
listener.onResponse(true, null);
}
}, listener::onFailure));
}

static final class Info implements ToXContentObject {

static final ParseField MESSAGE_FIELD = new ParseField("message");

private static final String message = "this index is a leader index; waiting for all following indices to cease " +
"following before proceeding";

Info() { }

String getMessage() {
return message;
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.field(MESSAGE_FIELD.getPreferredName(), message);
builder.endObject();
return builder;
}

@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
return true;
}

@Override
public int hashCode() {
return Objects.hash(getMessage());
}
}
}
12 changes: 8 additions & 4 deletions ...gin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/DeleteActionTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,14 @@ public void testToSteps() {
randomAlphaOfLengthBetween(1, 10));
List<Step> steps = action.toSteps(null, phase, nextStepKey);
assertNotNull(steps);
assertEquals(1, steps.size());
StepKey expectedFirstStepKey = new StepKey(phase, DeleteAction.NAME, DeleteStep.NAME);
DeleteStep firstStep = (DeleteStep) steps.get(0);
assertEquals(2, steps.size());
StepKey expectedFirstStepKey = new StepKey(phase, DeleteAction.NAME, WaitForNoFollowersStep.NAME);
StepKey expectedSecondStepKey = new StepKey(phase, DeleteAction.NAME, DeleteStep.NAME);
WaitForNoFollowersStep firstStep = (WaitForNoFollowersStep) steps.get(0);
DeleteStep secondStep = (DeleteStep) steps.get(1);
assertEquals(expectedFirstStepKey, firstStep.getKey());
assertEquals(nextStepKey, firstStep.getNextStepKey());
assertEquals(expectedSecondStepKey, firstStep.getNextStepKey());
assertEquals(expectedSecondStepKey, secondStep.getKey());
assertEquals(nextStepKey, secondStep.getNextStepKey());
}
}
53 changes: 29 additions & 24 deletions ...gin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ShrinkActionTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,60 +126,65 @@ public void testToSteps() {
StepKey nextStepKey = new StepKey(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10),
randomAlphaOfLengthBetween(1, 10));
List<Step> steps = action.toSteps(null, phase, nextStepKey);
assertThat(steps.size(), equalTo(9));
assertThat(steps.size(), equalTo(10));
StepKey expectedFirstKey = new StepKey(phase, ShrinkAction.NAME, BranchingStep.NAME);
StepKey expectedSecondKey = new StepKey(phase, ShrinkAction.NAME, ReadOnlyAction.NAME);
StepKey expectedThirdKey = new StepKey(phase, ShrinkAction.NAME, SetSingleNodeAllocateStep.NAME);
StepKey expectedFourthKey = new StepKey(phase, ShrinkAction.NAME, CheckShrinkReadyStep.NAME);
StepKey expectedFifthKey = new StepKey(phase, ShrinkAction.NAME, ShrinkStep.NAME);
StepKey expectedSixthKey = new StepKey(phase, ShrinkAction.NAME, ShrunkShardsAllocatedStep.NAME);
StepKey expectedSeventhKey = new StepKey(phase, ShrinkAction.NAME, CopyExecutionStateStep.NAME);
StepKey expectedEighthKey = new StepKey(phase, ShrinkAction.NAME, ShrinkSetAliasStep.NAME);
StepKey expectedNinthKey = new StepKey(phase, ShrinkAction.NAME, ShrunkenIndexCheckStep.NAME);
StepKey expectedSecondKey = new StepKey(phase, ShrinkAction.NAME, WaitForNoFollowersStep.NAME);
StepKey expectedThirdKey = new StepKey(phase, ShrinkAction.NAME, ReadOnlyAction.NAME);
StepKey expectedFourthKey = new StepKey(phase, ShrinkAction.NAME, SetSingleNodeAllocateStep.NAME);
StepKey expectedFifthKey = new StepKey(phase, ShrinkAction.NAME, CheckShrinkReadyStep.NAME);
StepKey expectedSixthKey = new StepKey(phase, ShrinkAction.NAME, ShrinkStep.NAME);
StepKey expectedSeventhKey = new StepKey(phase, ShrinkAction.NAME, ShrunkShardsAllocatedStep.NAME);
StepKey expectedEighthKey = new StepKey(phase, ShrinkAction.NAME, CopyExecutionStateStep.NAME);
StepKey expectedNinthKey = new StepKey(phase, ShrinkAction.NAME, ShrinkSetAliasStep.NAME);
StepKey expectedTenthKey = new StepKey(phase, ShrinkAction.NAME, ShrunkenIndexCheckStep.NAME);

assertTrue(steps.get(0) instanceof BranchingStep);
assertThat(steps.get(0).getKey(), equalTo(expectedFirstKey));
expectThrows(IllegalStateException.class, () -> steps.get(0).getNextStepKey());
assertThat(((BranchingStep) steps.get(0)).getNextStepKeyOnFalse(), equalTo(expectedSecondKey));
assertThat(((BranchingStep) steps.get(0)).getNextStepKeyOnTrue(), equalTo(nextStepKey));

assertTrue(steps.get(1) instanceof UpdateSettingsStep);
assertTrue(steps.get(1) instanceof WaitForNoFollowersStep);
assertThat(steps.get(1).getKey(), equalTo(expectedSecondKey));
assertThat(steps.get(1).getNextStepKey(), equalTo(expectedThirdKey));
assertTrue(IndexMetaData.INDEX_BLOCKS_WRITE_SETTING.get(((UpdateSettingsStep)steps.get(1)).getSettings()));

assertTrue(steps.get(2) instanceof SetSingleNodeAllocateStep);
assertTrue(steps.get(2) instanceof UpdateSettingsStep);
assertThat(steps.get(2).getKey(), equalTo(expectedThirdKey));
assertThat(steps.get(2).getNextStepKey(), equalTo(expectedFourthKey));
assertTrue(IndexMetaData.INDEX_BLOCKS_WRITE_SETTING.get(((UpdateSettingsStep)steps.get(2)).getSettings()));

assertTrue(steps.get(3) instanceof CheckShrinkReadyStep);
assertTrue(steps.get(3) instanceof SetSingleNodeAllocateStep);
assertThat(steps.get(3).getKey(), equalTo(expectedFourthKey));
assertThat(steps.get(3).getNextStepKey(), equalTo(expectedFifthKey));

assertTrue(steps.get(4) instanceof ShrinkStep);
assertTrue(steps.get(4) instanceof CheckShrinkReadyStep);
assertThat(steps.get(4).getKey(), equalTo(expectedFifthKey));
assertThat(steps.get(4).getNextStepKey(), equalTo(expectedSixthKey));
assertThat(((ShrinkStep) steps.get(4)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX));

assertTrue(steps.get(5) instanceof ShrunkShardsAllocatedStep);
assertTrue(steps.get(5) instanceof ShrinkStep);
assertThat(steps.get(5).getKey(), equalTo(expectedSixthKey));
assertThat(steps.get(5).getNextStepKey(), equalTo(expectedSeventhKey));
assertThat(((ShrunkShardsAllocatedStep) steps.get(5)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX));
assertThat(((ShrinkStep) steps.get(5)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX));

assertTrue(steps.get(6) instanceof CopyExecutionStateStep);
assertTrue(steps.get(6) instanceof ShrunkShardsAllocatedStep);
assertThat(steps.get(6).getKey(), equalTo(expectedSeventhKey));
assertThat(steps.get(6).getNextStepKey(), equalTo(expectedEighthKey));
assertThat(((CopyExecutionStateStep) steps.get(6)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX));
assertThat(((ShrunkShardsAllocatedStep) steps.get(6)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX));

assertTrue(steps.get(7) instanceof ShrinkSetAliasStep);
assertTrue(steps.get(7) instanceof CopyExecutionStateStep);
assertThat(steps.get(7).getKey(), equalTo(expectedEighthKey));
assertThat(steps.get(7).getNextStepKey(), equalTo(expectedNinthKey));
assertThat(((ShrinkSetAliasStep) steps.get(7)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX));
assertThat(((CopyExecutionStateStep) steps.get(7)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX));

assertTrue(steps.get(8) instanceof ShrunkenIndexCheckStep);
assertTrue(steps.get(8) instanceof ShrinkSetAliasStep);
assertThat(steps.get(8).getKey(), equalTo(expectedNinthKey));
assertThat(steps.get(8).getNextStepKey(), equalTo(nextStepKey));
assertThat(((ShrunkenIndexCheckStep) steps.get(8)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX));
assertThat(steps.get(8).getNextStepKey(), equalTo(expectedTenthKey));
assertThat(((ShrinkSetAliasStep) steps.get(8)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX));

assertTrue(steps.get(9) instanceof ShrunkenIndexCheckStep);
assertThat(steps.get(9).getKey(), equalTo(expectedTenthKey));
assertThat(steps.get(9).getNextStepKey(), equalTo(nextStepKey));
assertThat(((ShrunkenIndexCheckStep) steps.get(9)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX));
}

@Override
Expand Down
Loading