Readd index.lifecycle.skip setting

docs/changelog/128736.yaml

@@ -0,0 +1,5 @@
+pr: 128736
+summary: Add `index.lifecycle.skip` index-scoped setting to instruct ILM to skip processing specific indices
+area: ILM+SLM
+type: enhancement
+issues: []

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -279,6 +279,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion ESQL_REGEX_MATCH_WITH_CASE_INSENSITIVITY = def(9_086_0_00);
     public static final TransportVersion IDP_CUSTOM_SAML_ATTRIBUTES = def(9_087_0_00);
     public static final TransportVersion JOIN_ON_ALIASES = def(9_088_0_00);
+    public static final TransportVersion ILM_ADD_SKIP_SETTING = def(9_089_0_00);
 
     /*
      * STOP! READ THIS FIRST! No, really,

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleExplainResponse.java

@@ -55,6 +55,7 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl
     private static final ParseField REPOSITORY_NAME = new ParseField("repository_name");
     private static final ParseField SHRINK_INDEX_NAME = new ParseField("shrink_index_name");
     private static final ParseField SNAPSHOT_NAME = new ParseField("snapshot_name");
+    private static final ParseField SKIP_NAME = new ParseField("skip");
 
     public static final ConstructingObjectParser<IndexLifecycleExplainResponse, Void> PARSER = new ConstructingObjectParser<>(
         "index_lifecycle_explain_response",
@@ -78,7 +79,8 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl
             (String) a[18],
             (BytesReference) a[11],
             (BytesReference) a[21],
-            (PhaseExecutionInfo) a[12]
+            (PhaseExecutionInfo) a[12],
+            Objects.requireNonNullElse((Boolean) a[22], false)
             // a[13] == "age"
             // a[20] == "time_since_index_creation"
         )
@@ -118,6 +120,7 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl
             builder.copyCurrentStructure(p);
             return BytesReference.bytes(builder);
         }, PREVIOUS_STEP_INFO_FIELD);
+        PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), SKIP_NAME);
     }
 
     private final String index;
@@ -140,6 +143,7 @@ public class IndexLifecycleExplainResponse implements ToXContentObject, Writeabl
     private final String repositoryName;
     private final String snapshotName;
     private final String shrinkIndexName;
+    private final boolean skip;
 
     Supplier<Long> nowSupplier = System::currentTimeMillis; // Can be changed for testing
 
@@ -162,7 +166,8 @@ public static IndexLifecycleExplainResponse newManagedIndexResponse(
         String shrinkIndexName,
         BytesReference stepInfo,
         BytesReference previousStepInfo,
-        PhaseExecutionInfo phaseExecutionInfo
+        PhaseExecutionInfo phaseExecutionInfo,
+        boolean skip
     ) {
         return new IndexLifecycleExplainResponse(
             index,
@@ -184,7 +189,8 @@ public static IndexLifecycleExplainResponse newManagedIndexResponse(
             shrinkIndexName,
             stepInfo,
             previousStepInfo,
-            phaseExecutionInfo
+            phaseExecutionInfo,
+            skip
         );
     }
 
@@ -209,7 +215,8 @@ public static IndexLifecycleExplainResponse newUnmanagedIndexResponse(String ind
             null,
             null,
             null,
-            null
+            null,
+            false
         );
     }
 
@@ -233,7 +240,8 @@ private IndexLifecycleExplainResponse(
         String shrinkIndexName,
         BytesReference stepInfo,
         BytesReference previousStepInfo,
-        PhaseExecutionInfo phaseExecutionInfo
+        PhaseExecutionInfo phaseExecutionInfo,
+        boolean skip
     ) {
         if (managedByILM) {
             if (policyName == null) {
@@ -301,6 +309,7 @@ private IndexLifecycleExplainResponse(
         this.repositoryName = repositoryName;
         this.snapshotName = snapshotName;
         this.shrinkIndexName = shrinkIndexName;
+        this.skip = skip;
     }
 
     public IndexLifecycleExplainResponse(StreamInput in) throws IOException {
@@ -333,6 +342,11 @@ public IndexLifecycleExplainResponse(StreamInput in) throws IOException {
             } else {
                 previousStepInfo = null;
             }
+            if (in.getTransportVersion().onOrAfter(TransportVersions.ILM_ADD_SKIP_SETTING)) {
+                skip = in.readBoolean();
+            } else {
+                skip = false;
+            }
         } else {
             policyName = null;
             lifecycleDate = null;
@@ -352,6 +366,7 @@ public IndexLifecycleExplainResponse(StreamInput in) throws IOException {
             snapshotName = null;
             shrinkIndexName = null;
             indexCreationDate = null;
+            skip = false;
         }
     }
 
@@ -382,6 +397,9 @@ public void writeTo(StreamOutput out) throws IOException {
             if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) {
                 out.writeOptionalBytesReference(previousStepInfo);
             }
+            if (out.getTransportVersion().onOrAfter(TransportVersions.ILM_ADD_SKIP_SETTING)) {
+                out.writeBoolean(skip);
+            }
         }
     }
 
@@ -481,6 +499,10 @@ public String getShrinkIndexName() {
         return shrinkIndexName;
     }
 
+    public boolean getSkip() {
+        return skip;
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
@@ -564,6 +586,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             if (phaseExecutionInfo != null) {
                 builder.field(PHASE_EXECUTION_INFO.getPreferredName(), phaseExecutionInfo);
             }
+            builder.field(SKIP_NAME.getPreferredName(), skip);
         }
         builder.endObject();
         return builder;
@@ -591,7 +614,8 @@ public int hashCode() {
             shrinkIndexName,
             stepInfo,
             previousStepInfo,
-            phaseExecutionInfo
+            phaseExecutionInfo,
+            skip
         );
     }
 
@@ -623,7 +647,8 @@ public boolean equals(Object obj) {
             && Objects.equals(shrinkIndexName, other.shrinkIndexName)
             && Objects.equals(stepInfo, other.stepInfo)
             && Objects.equals(previousStepInfo, other.previousStepInfo)
-            && Objects.equals(phaseExecutionInfo, other.phaseExecutionInfo);
+            && Objects.equals(phaseExecutionInfo, other.phaseExecutionInfo)
+            && Objects.equals(skip, other.skip);
     }
 
     @Override

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecycleSettings.java

@@ -23,6 +23,7 @@ public class LifecycleSettings {
     public static final String LIFECYCLE_STEP_MASTER_TIMEOUT = "indices.lifecycle.step.master_timeout";
     public static final String LIFECYCLE_STEP_WAIT_TIME_THRESHOLD = "index.lifecycle.step.wait_time_threshold";
     public static final String LIFECYCLE_ROLLOVER_ONLY_IF_HAS_DOCUMENTS = "indices.lifecycle.rollover.only_if_has_documents";
+    public static final String LIFECYCLE_SKIP = "index.lifecycle.skip";
 
     public static final String SLM_HISTORY_INDEX_ENABLED = "slm.history_index_enabled";
     public static final String SLM_RETENTION_SCHEDULE = "slm.retention_schedule";
@@ -82,6 +83,12 @@ public class LifecycleSettings {
         Setting.Property.NodeScope,
         Setting.Property.DeprecatedWarning
     );
+    public static final Setting<Boolean> LIFECYCLE_SKIP_SETTING = Setting.boolSetting(
+        LIFECYCLE_SKIP,
+        false,
+        Setting.Property.Dynamic,
+        Setting.Property.IndexScope
+    );
 
     public static final Setting<Boolean> SLM_HISTORY_INDEX_ENABLED_SETTING = Setting.boolSetting(
         SLM_HISTORY_INDEX_ENABLED,

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleExplainResponseTests.java

@@ -74,7 +74,8 @@ private static IndexLifecycleExplainResponse randomManagedIndexExplainResponse()
             stepNull ? null : randomAlphaOfLength(10),
             randomBoolean() ? null : new BytesArray(new RandomStepInfo(() -> randomAlphaOfLength(10)).toString()),
             randomBoolean() ? null : new BytesArray(new RandomStepInfo(() -> randomAlphaOfLength(10)).toString()),
-            randomBoolean() ? null : PhaseExecutionInfoTests.randomPhaseExecutionInfo("")
+            randomBoolean() ? null : PhaseExecutionInfoTests.randomPhaseExecutionInfo(""),
+            randomBoolean()
         );
     }
 
@@ -101,7 +102,8 @@ public void testInvalidStepDetails() {
                 randomBoolean() ? null : randomAlphaOfLength(10),
                 randomBoolean() ? null : new BytesArray(new RandomStepInfo(() -> randomAlphaOfLength(10)).toString()),
                 randomBoolean() ? null : new BytesArray(new RandomStepInfo(() -> randomAlphaOfLength(10)).toString()),
-                randomBoolean() ? null : PhaseExecutionInfoTests.randomPhaseExecutionInfo("")
+                randomBoolean() ? null : PhaseExecutionInfoTests.randomPhaseExecutionInfo(""),
+                randomBoolean()
             )
         );
         assertThat(exception.getMessage(), startsWith("managed index response must have complete step details"));
@@ -135,7 +137,8 @@ public void testIndexAges() {
             null,
             null,
             null,
-            null
+            null,
+            false
         );
         assertThat(managedExplainResponse.getLifecycleDate(), is(notNullValue()));
         Long now = 1_000_000L;
@@ -196,9 +199,10 @@ protected IndexLifecycleExplainResponse mutateInstance(IndexLifecycleExplainResp
         BytesReference stepInfo = instance.getStepInfo();
         BytesReference previousStepInfo = instance.getPreviousStepInfo();
         PhaseExecutionInfo phaseExecutionInfo = instance.getPhaseExecutionInfo();
+        boolean skip = instance.getSkip();
 
         if (managed) {
-            switch (between(0, 15)) {
+            switch (between(0, 16)) {
                 case 0 -> index += randomAlphaOfLengthBetween(1, 5);
                 case 1 -> policy += randomAlphaOfLengthBetween(1, 5);
                 case 2 -> {
@@ -257,6 +261,7 @@ protected IndexLifecycleExplainResponse mutateInstance(IndexLifecycleExplainResp
                 case 13 -> repositoryName = randomValueOtherThan(repositoryName, () -> randomAlphaOfLengthBetween(5, 10));
                 case 14 -> snapshotName = randomValueOtherThan(snapshotName, () -> randomAlphaOfLengthBetween(5, 10));
                 case 15 -> shrinkIndexName = randomValueOtherThan(shrinkIndexName, () -> randomAlphaOfLengthBetween(5, 10));
+                case 16 -> skip = skip == false;
                 default -> throw new AssertionError("Illegal randomisation branch");
             }
 
@@ -279,7 +284,8 @@ protected IndexLifecycleExplainResponse mutateInstance(IndexLifecycleExplainResp
                 shrinkIndexName,
                 stepInfo,
                 previousStepInfo,
-                phaseExecutionInfo
+                phaseExecutionInfo,
+                skip
             );
         } else {
             return switch (between(0, 1)) {

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java

@@ -126,6 +126,7 @@ public List<Setting<?>> getSettings() {
             LifecycleSettings.LIFECYCLE_STEP_MASTER_TIMEOUT_SETTING,
             LifecycleSettings.LIFECYCLE_STEP_WAIT_TIME_THRESHOLD_SETTING,
             LifecycleSettings.LIFECYCLE_ROLLOVER_ONLY_IF_HAS_DOCUMENTS_SETTING,
+            LifecycleSettings.LIFECYCLE_SKIP_SETTING,
             RolloverAction.LIFECYCLE_ROLLOVER_ALIAS_SETTING,
             IlmHealthIndicatorService.MAX_TIME_ON_ACTION_SETTING,
             IlmHealthIndicatorService.MAX_TIME_ON_STEP_SETTING,

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleRunner.java

@@ -31,6 +31,7 @@
 import org.elasticsearch.xpack.core.ilm.ClusterStateActionStep;
 import org.elasticsearch.xpack.core.ilm.ClusterStateWaitStep;
 import org.elasticsearch.xpack.core.ilm.ErrorStep;
+import org.elasticsearch.xpack.core.ilm.LifecycleSettings;
 import org.elasticsearch.xpack.core.ilm.PhaseCompleteStep;
 import org.elasticsearch.xpack.core.ilm.Step;
 import org.elasticsearch.xpack.core.ilm.Step.StepKey;
@@ -176,6 +177,10 @@ boolean isReadyToTransitionToThisPhase(final String policy, final IndexMetadata
      */
     void runPeriodicStep(String policy, Metadata metadata, IndexMetadata indexMetadata) {
         String index = indexMetadata.getIndex().getName();
+        if (LifecycleSettings.LIFECYCLE_SKIP_SETTING.get(indexMetadata.getSettings())) {
+            logger.info("[{}] skipping policy [{}] because [{}] is true", index, policy, LifecycleSettings.LIFECYCLE_SKIP);
+            return;
+        }
         LifecycleExecutionState lifecycleState = indexMetadata.getLifecycleExecutionState();
         final Step currentStep;
         try {
@@ -303,6 +308,10 @@ void onErrorMaybeRetryFailedStep(String policy, StepKey currentStep, IndexMetada
      */
     void maybeRunAsyncAction(ClusterState currentState, IndexMetadata indexMetadata, String policy, StepKey expectedStepKey) {
         String index = indexMetadata.getIndex().getName();
+        if (LifecycleSettings.LIFECYCLE_SKIP_SETTING.get(indexMetadata.getSettings())) {
+            logger.info("[{}] skipping policy [{}] because [{}] is true", index, policy, LifecycleSettings.LIFECYCLE_SKIP);
+            return;
+        }
         LifecycleExecutionState lifecycleState = indexMetadata.getLifecycleExecutionState();
         final Step currentStep;
         try {
@@ -383,6 +392,10 @@ public void onFailure(Exception e) {
      */
     void runPolicyAfterStateChange(String policy, IndexMetadata indexMetadata) {
         String index = indexMetadata.getIndex().getName();
+        if (LifecycleSettings.LIFECYCLE_SKIP_SETTING.get(indexMetadata.getSettings())) {
+            logger.info("[{}] skipping policy [{}] because [{}] is true", index, policy, LifecycleSettings.LIFECYCLE_SKIP);
+            return;
+        }
         LifecycleExecutionState lifecycleState = indexMetadata.getLifecycleExecutionState();
         final StepKey currentStepKey = Step.getCurrentStepKey(lifecycleState);
         if (busyIndices.contains(Tuple.tuple(indexMetadata.getIndex(), currentStepKey))) {

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleTransition.java

@@ -494,6 +494,7 @@ private static IndexMetadata.Builder removePolicyForIndex(IndexMetadata indexMet
 
         notChanged &= Strings.isNullOrEmpty(newSettings.remove(LifecycleSettings.LIFECYCLE_NAME_SETTING.getKey()));
         notChanged &= Strings.isNullOrEmpty(newSettings.remove(LifecycleSettings.LIFECYCLE_INDEXING_COMPLETE_SETTING.getKey()));
+        notChanged &= Strings.isNullOrEmpty(newSettings.remove(LifecycleSettings.LIFECYCLE_SKIP_SETTING.getKey()));
         notChanged &= Strings.isNullOrEmpty(newSettings.remove(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS_SETTING.getKey()));
         long newSettingsVersion = notChanged ? indexMetadata.getSettingsVersion() : 1 + indexMetadata.getSettingsVersion();
 

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportExplainLifecycleAction.java

@@ -222,7 +222,8 @@ static IndexLifecycleExplainResponse getIndexLifecycleExplainResponse(
                     lifecycleState.shrinkIndexName(),
                     stepInfoBytes,
                     previousStepInfoBytes,
-                    phaseExecutionInfo
+                    phaseExecutionInfo,
+                    LifecycleSettings.LIFECYCLE_SKIP_SETTING.get(idxSettings)
                 );
             } else {
                 indexResponse = null;