Skip to content

Commit 9504286

Browse files
gwbrowngwbrown
committed
Use custom index metadata for ILM state (#33783)
Using index settings for ILM state is fragile and exposes too much information that doesn't need to be exposed. Using custom index metadata is more resilient and allows more controlled access to internal information. As part of these changes, moves away from using defaults for ILM-related values, in favor of using null values to clearly indicate that the value is not present.
1 parent ac06b1f commit 9504286

File tree

37 files changed

+1352
-609
lines changed

37 files changed

+1352
-609
lines changed

‎client/rest-high-level/src/test/java/org/elasticsearch/client/IndexLifecycleIT.java

Lines changed: 7 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,13 @@
2525
import org.elasticsearch.action.support.master.AcknowledgedResponse;
2626
import org.elasticsearch.client.indexlifecycle.AllocateAction;
2727
import org.elasticsearch.client.indexlifecycle.DeleteAction;
28+
import org.elasticsearch.client.indexlifecycle.DeleteLifecyclePolicyRequest;
29+
import org.elasticsearch.client.indexlifecycle.ExplainLifecycleRequest;
30+
import org.elasticsearch.client.indexlifecycle.ExplainLifecycleResponse;
2831
import org.elasticsearch.client.indexlifecycle.ForceMergeAction;
2932
import org.elasticsearch.client.indexlifecycle.GetLifecyclePolicyRequest;
3033
import org.elasticsearch.client.indexlifecycle.GetLifecyclePolicyResponse;
34+
import org.elasticsearch.client.indexlifecycle.IndexLifecycleExplainResponse;
3135
import org.elasticsearch.client.indexlifecycle.LifecycleAction;
3236
import org.elasticsearch.client.indexlifecycle.LifecycleManagementStatusRequest;
3337
import org.elasticsearch.client.indexlifecycle.LifecycleManagementStatusResponse;
@@ -38,17 +42,13 @@
3842
import org.elasticsearch.client.indexlifecycle.PhaseExecutionInfo;
3943
import org.elasticsearch.client.indexlifecycle.PutLifecyclePolicyRequest;
4044
import org.elasticsearch.client.indexlifecycle.RolloverAction;
41-
import org.elasticsearch.client.indexlifecycle.ShrinkAction;
42-
import org.elasticsearch.common.settings.Settings;
43-
import org.elasticsearch.client.indexlifecycle.DeleteLifecyclePolicyRequest;
44-
import org.elasticsearch.common.unit.TimeValue;
45-
import org.elasticsearch.client.indexlifecycle.ExplainLifecycleRequest;
46-
import org.elasticsearch.client.indexlifecycle.ExplainLifecycleResponse;
47-
import org.elasticsearch.client.indexlifecycle.IndexLifecycleExplainResponse;
4845
import org.elasticsearch.client.indexlifecycle.SetIndexLifecyclePolicyRequest;
4946
import org.elasticsearch.client.indexlifecycle.SetIndexLifecyclePolicyResponse;
47+
import org.elasticsearch.client.indexlifecycle.ShrinkAction;
5048
import org.elasticsearch.client.indexlifecycle.StartILMRequest;
5149
import org.elasticsearch.client.indexlifecycle.StopILMRequest;
50+
import org.elasticsearch.common.settings.Settings;
51+
import org.elasticsearch.common.unit.TimeValue;
5252
import org.hamcrest.Matchers;
5353

5454
import java.io.IOException;
@@ -167,14 +167,6 @@ public void testExplainLifecycle() throws Exception {
167167
.put("index.lifecycle.rollover_alias", "baz-alias").build(), "", "\"baz-alias\" : {}");
168168

169169
createIndex("squash", Settings.EMPTY);
170-
assertBusy(() -> {
171-
GetSettingsRequest getSettingsRequest = new GetSettingsRequest().indices("foo-01", "baz-01");
172-
GetSettingsResponse settingsResponse = highLevelClient().indices().getSettings(getSettingsRequest, RequestOptions.DEFAULT);
173-
assertThat(settingsResponse.getSetting("foo-01", "index.lifecycle.name"), equalTo(policy.getName()));
174-
assertThat(settingsResponse.getSetting("baz-01", "index.lifecycle.name"), equalTo(policy.getName()));
175-
assertThat(settingsResponse.getSetting("foo-01", "index.lifecycle.phase"), equalTo("hot"));
176-
assertThat(settingsResponse.getSetting("baz-01", "index.lifecycle.phase"), equalTo("hot"));
177-
});
178170

179171
ExplainLifecycleRequest req = new ExplainLifecycleRequest();
180172
req.indices("foo-01", "baz-01", "squash");

‎server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -964,6 +964,11 @@ public Builder putCustom(String type, Map<String, String> customIndexMetaData) {
964964
return this;
965965
}
966966

967+
public Builder removeCustom(String type) {
968+
this.customMetaData.remove(type);
969+
return this;
970+
}
971+
967972
public Set<String> getInSyncAllocationIds(int shardId) {
968973
return inSyncAllocationIds.get(shardId);
969974
}

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/CopyExecutionStateStep.java

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
package org.elasticsearch.xpack.core.indexlifecycle;
8+
9+
import org.elasticsearch.cluster.ClusterState;
10+
import org.elasticsearch.cluster.metadata.IndexMetaData;
11+
import org.elasticsearch.cluster.metadata.MetaData;
12+
import org.elasticsearch.index.Index;
13+
14+
import java.util.Objects;
15+
16+
import static org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
17+
18+
/**
19+
* Copies the execution state data from one index to another, typically after a
20+
* new index has been created. Useful for actions such as shrink.
21+
*/
22+
public class CopyExecutionStateStep extends ClusterStateActionStep {
23+
public static final String NAME = "copy_execution_state";
24+
private String shrunkIndexPrefix;
25+
26+
public CopyExecutionStateStep(StepKey key, StepKey nextStepKey, String shrunkIndexPrefix) {
27+
super(key, nextStepKey);
28+
this.shrunkIndexPrefix = shrunkIndexPrefix;
29+
}
30+
31+
String getShrunkIndexPrefix() {
32+
return shrunkIndexPrefix;
33+
}
34+
35+
@Override
36+
public ClusterState performAction(Index index, ClusterState clusterState) {
37+
IndexMetaData indexMetaData = clusterState.metaData().index(index);
38+
// get source index
39+
String indexName = indexMetaData.getIndex().getName();
40+
// get target shrink index
41+
String targetIndexName = shrunkIndexPrefix + indexName;
42+
IndexMetaData targetIndexMetaData = clusterState.metaData().index(targetIndexName);
43+
44+
LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(indexMetaData);
45+
String phase = lifecycleState.getPhase();
46+
String action = lifecycleState.getAction();
47+
long lifecycleDate = lifecycleState.getLifecycleDate();
48+
49+
LifecycleExecutionState.Builder relevantTargetCustomData = LifecycleExecutionState.builder();
50+
relevantTargetCustomData.setIndexCreationDate(lifecycleDate);
51+
relevantTargetCustomData.setPhase(phase);
52+
relevantTargetCustomData.setAction(action);
53+
relevantTargetCustomData.setStep(ShrunkenIndexCheckStep.NAME);
54+
55+
MetaData.Builder newMetaData = MetaData.builder(clusterState.getMetaData())
56+
.put(IndexMetaData.builder(targetIndexMetaData)
57+
.putCustom(ILM_CUSTOM_METADATA_KEY, relevantTargetCustomData.build().asMap()));
58+
59+
return ClusterState.builder(clusterState).metaData(newMetaData).build();
60+
}
61+
62+
@Override
63+
public boolean equals(Object o) {
64+
if (this == o) return true;
65+
if (o == null || getClass() != o.getClass()) return false;
66+
if (!super.equals(o)) return false;
67+
CopyExecutionStateStep that = (CopyExecutionStateStep) o;
68+
return Objects.equals(shrunkIndexPrefix, that.shrunkIndexPrefix);
69+
}
70+
71+
@Override
72+
public int hashCode() {
73+
return Objects.hash(super.hashCode(), shrunkIndexPrefix);
74+
}
75+
}

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/IndexLifecycleExplainResponse.java

Lines changed: 38 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -48,14 +48,14 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl
4848
(boolean) a[1],
4949
(String) a[2],
5050
(boolean) (a[3] == null ? false: a[3]),
51-
(long) (a[4] == null ? -1L: a[4]),
51+
(Long) (a[4]),
5252
(String) a[5],
5353
(String) a[6],
5454
(String) a[7],
5555
(String) a[8],
56-
(long) (a[9] == null ? -1L: a[9]),
57-
(long) (a[10] == null ? -1L: a[10]),
58-
(long) (a[11] == null ? -1L: a[11]),
56+
(Long) (a[9]),
57+
(Long) (a[10]),
58+
(Long) (a[11]),
5959
(BytesReference) a[12],
6060
(PhaseExecutionInfo) a[13]));
6161
static {
@@ -86,36 +86,36 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl
8686
private final String action;
8787
private final String step;
8888
private final String failedStep;
89-
private final long lifecycleDate;
90-
private final long phaseTime;
91-
private final long actionTime;
92-
private final long stepTime;
89+
private final Long lifecycleDate;
90+
private final Long phaseTime;
91+
private final Long actionTime;
92+
private final Long stepTime;
9393
private final boolean skip;
9494
private final boolean managedByILM;
9595
private final BytesReference stepInfo;
9696
private final PhaseExecutionInfo phaseExecutionInfo;
9797

98-
public static IndexLifecycleExplainResponse newManagedIndexResponse(String index, String policyName, boolean skip, long lifecycleDate,
99-
String phase, String action, String step, String failedStep, long phaseTime, long actionTime, long stepTime,
98+
public static IndexLifecycleExplainResponse newManagedIndexResponse(String index, String policyName, boolean skip, Long lifecycleDate,
99+
String phase, String action, String step, String failedStep, Long phaseTime, Long actionTime, Long stepTime,
100100
BytesReference stepInfo, PhaseExecutionInfo phaseExecutionInfo) {
101101
return new IndexLifecycleExplainResponse(index, true, policyName, skip, lifecycleDate, phase, action, step, failedStep, phaseTime,
102102
actionTime, stepTime, stepInfo, phaseExecutionInfo);
103103
}
104104

105105
public static IndexLifecycleExplainResponse newUnmanagedIndexResponse(String index) {
106-
return new IndexLifecycleExplainResponse(index, false, null, false, -1L, null, null, null, null, -1L, -1L, -1L, null, null);
106+
return new IndexLifecycleExplainResponse(index, false, null, false, null, null, null, null, null, null, null, null, null, null);
107107
}
108108

109-
private IndexLifecycleExplainResponse(String index, boolean managedByILM, String policyName, boolean skip, long lifecycleDate,
110-
String phase, String action, String step, String failedStep, long phaseTime, long actionTime,
111-
long stepTime, BytesReference stepInfo, PhaseExecutionInfo phaseExecutionInfo) {
109+
private IndexLifecycleExplainResponse(String index, boolean managedByILM, String policyName, boolean skip, Long lifecycleDate,
110+
String phase, String action, String step, String failedStep, Long phaseTime, Long actionTime,
111+
Long stepTime, BytesReference stepInfo, PhaseExecutionInfo phaseExecutionInfo) {
112112
if (managedByILM) {
113113
if (policyName == null) {
114114
throw new IllegalArgumentException("[" + POLICY_NAME_FIELD.getPreferredName() + "] cannot be null for managed index");
115115
}
116116
} else {
117-
if (policyName != null || lifecycleDate >= 0 || phase != null || action != null || step != null || failedStep != null
118-
|| phaseTime >= 0 || actionTime >= 0 || stepTime >= 0 || stepInfo != null || phaseExecutionInfo != null) {
117+
if (policyName != null || lifecycleDate != null || phase != null || action != null || step != null || failedStep != null
118+
|| phaseTime != null || actionTime != null || stepTime != null || stepInfo != null || phaseExecutionInfo != null) {
119119
throw new IllegalArgumentException(
120120
"Unmanaged index response must only contain fields: [" + MANAGED_BY_ILM_FIELD + ", " + INDEX_FIELD + "]");
121121
}
@@ -142,27 +142,27 @@ public IndexLifecycleExplainResponse(StreamInput in) throws IOException {
142142
if (managedByILM) {
143143
policyName = in.readString();
144144
skip = in.readBoolean();
145-
lifecycleDate = in.readZLong();
146-
phase = in.readString();
147-
action = in.readString();
148-
step = in.readString();
145+
lifecycleDate = in.readOptionalLong();
146+
phase = in.readOptionalString();
147+
action = in.readOptionalString();
148+
step = in.readOptionalString();
149149
failedStep = in.readOptionalString();
150-
phaseTime = in.readZLong();
151-
actionTime = in.readZLong();
152-
stepTime = in.readZLong();
150+
phaseTime = in.readOptionalLong();
151+
actionTime = in.readOptionalLong();
152+
stepTime = in.readOptionalLong();
153153
stepInfo = in.readOptionalBytesReference();
154154
phaseExecutionInfo = in.readOptionalWriteable(PhaseExecutionInfo::new);
155155
} else {
156156
policyName = null;
157157
skip = false;
158-
lifecycleDate = -1L;
158+
lifecycleDate = null;
159159
phase = null;
160160
action = null;
161161
step = null;
162162
failedStep = null;
163-
phaseTime = -1L;
164-
actionTime = -1L;
165-
stepTime = -1L;
163+
phaseTime = null;
164+
actionTime = null;
165+
stepTime = null;
166166
stepInfo = null;
167167
phaseExecutionInfo = null;
168168
}
@@ -175,14 +175,14 @@ public void writeTo(StreamOutput out) throws IOException {
175175
if (managedByILM) {
176176
out.writeString(policyName);
177177
out.writeBoolean(skip);
178-
out.writeZLong(lifecycleDate);
179-
out.writeString(phase);
180-
out.writeString(action);
181-
out.writeString(step);
178+
out.writeOptionalLong(lifecycleDate);
179+
out.writeOptionalString(phase);
180+
out.writeOptionalString(action);
181+
out.writeOptionalString(step);
182182
out.writeOptionalString(failedStep);
183-
out.writeZLong(phaseTime);
184-
out.writeZLong(actionTime);
185-
out.writeZLong(stepTime);
183+
out.writeOptionalLong(phaseTime);
184+
out.writeOptionalLong(actionTime);
185+
out.writeOptionalLong(stepTime);
186186
out.writeOptionalBytesReference(stepInfo);
187187
out.writeOptionalWriteable(phaseExecutionInfo);
188188
}
@@ -204,31 +204,31 @@ public boolean skip() {
204204
return skip;
205205
}
206206

207-
public long getLifecycleDate() {
207+
public Long getLifecycleDate() {
208208
return lifecycleDate;
209209
}
210210

211211
public String getPhase() {
212212
return phase;
213213
}
214214

215-
public long getPhaseTime() {
215+
public Long getPhaseTime() {
216216
return phaseTime;
217217
}
218218

219219
public String getAction() {
220220
return action;
221221
}
222222

223-
public long getActionTime() {
223+
public Long getActionTime() {
224224
return actionTime;
225225
}
226226

227227
public String getStep() {
228228
return step;
229229
}
230230

231-
public long getStepTime() {
231+
public Long getStepTime() {
232232
return stepTime;
233233
}
234234

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/InitializePolicyContextStep.java

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,10 @@
1010
import org.elasticsearch.cluster.ClusterState;
1111
import org.elasticsearch.cluster.metadata.IndexMetaData;
1212
import org.elasticsearch.cluster.metadata.MetaData;
13-
import org.elasticsearch.common.settings.Settings;
1413
import org.elasticsearch.index.Index;
1514

15+
import static org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY;
16+
1617
public final class InitializePolicyContextStep extends ClusterStateActionStep {
1718
public static final String INITIALIZATION_PHASE = "new";
1819
public static final StepKey KEY = new StepKey(INITIALIZATION_PHASE, "init", "init");
@@ -30,18 +31,19 @@ public ClusterState performAction(Index index, ClusterState clusterState) {
3031
// Index must have been since deleted, ignore it
3132
return clusterState;
3233
}
33-
Settings settings = indexMetaData.getSettings();
34-
if (settings.hasValue(LifecycleSettings.LIFECYCLE_INDEX_CREATION_DATE)) {
34+
LifecycleExecutionState lifecycleState = LifecycleExecutionState
35+
.fromIndexMetadata(indexMetaData);
36+
if (lifecycleState.getLifecycleDate() != null) {
3537
return clusterState;
3638
}
3739

3840
ClusterState.Builder newClusterStateBuilder = ClusterState.builder(clusterState);
39-
IndexMetaData idxMeta = clusterState.getMetaData().index(index);
40-
Settings.Builder indexSettings = Settings.builder().put(idxMeta.getSettings())
41-
.put(LifecycleSettings.LIFECYCLE_INDEX_CREATION_DATE, idxMeta.getCreationDate());
41+
42+
LifecycleExecutionState.Builder newCustomData = LifecycleExecutionState.builder(lifecycleState);
43+
newCustomData.setIndexCreationDate(indexMetaData.getCreationDate());
4244
newClusterStateBuilder.metaData(MetaData.builder(clusterState.getMetaData()).put(IndexMetaData
43-
.builder(clusterState.getMetaData().index(index))
44-
.settings(indexSettings)));
45+
.builder(indexMetaData)
46+
.putCustom(ILM_CUSTOM_METADATA_KEY, newCustomData.build().asMap())));
4547
return newClusterStateBuilder.build();
4648
}
4749
}

0 commit comments

Comments
 (0)