Skip to content

[7.x] Add max_single_primary_size as a condition for the ILM rollover action #69202

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Feb 18, 2021
Merged

[7.x] Add max_single_primary_size as a condition for the ILM rollover action #69202

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 26 additions & 9 deletions ...rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/RolloverAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,43 +20,56 @@
import java.io.IOException;
import java.util.Objects;


public class RolloverAction implements LifecycleAction, ToXContentObject {
public static final String NAME = "rollover";
private static final ParseField MAX_SIZE_FIELD = new ParseField("max_size");
private static final ParseField MAX_DOCS_FIELD = new ParseField("max_docs");
private static final ParseField MAX_SINGLE_PRIMARY_SIZE_FIELD = new ParseField("max_single_primary_size");
private static final ParseField MAX_AGE_FIELD = new ParseField("max_age");
private static final ParseField MAX_DOCS_FIELD = new ParseField("max_docs");

private static final ConstructingObjectParser<RolloverAction, Void> PARSER = new ConstructingObjectParser<>(NAME, true,
a -> new RolloverAction((ByteSizeValue) a[0], (TimeValue) a[1], (Long) a[2]));
a -> new RolloverAction((ByteSizeValue) a[0], (ByteSizeValue) a[1], (TimeValue) a[2], (Long) a[3]));

static {
PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(),
(p, c) -> ByteSizeValue.parseBytesSizeValue(p.text(), MAX_SIZE_FIELD.getPreferredName()), MAX_SIZE_FIELD, ValueType.VALUE);
(p, c) -> ByteSizeValue.parseBytesSizeValue(p.text(), MAX_SIZE_FIELD.getPreferredName()),
MAX_SIZE_FIELD, ValueType.VALUE);
PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(),
(p, c) -> ByteSizeValue.parseBytesSizeValue(p.text(), MAX_SINGLE_PRIMARY_SIZE_FIELD.getPreferredName()),
MAX_SINGLE_PRIMARY_SIZE_FIELD, ValueType.VALUE);
PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(),
(p, c) -> TimeValue.parseTimeValue(p.text(), MAX_AGE_FIELD.getPreferredName()), MAX_AGE_FIELD, ValueType.VALUE);
(p, c) -> TimeValue.parseTimeValue(p.text(), MAX_AGE_FIELD.getPreferredName()),
MAX_AGE_FIELD, ValueType.VALUE);
PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), MAX_DOCS_FIELD);
}

private final ByteSizeValue maxSize;
private final Long maxDocs;
private final ByteSizeValue maxSinglePrimarySize;
private final TimeValue maxAge;
private final Long maxDocs;

public static RolloverAction parse(XContentParser parser) {
return PARSER.apply(parser, null);
}

public RolloverAction(ByteSizeValue maxSize, TimeValue maxAge, Long maxDocs) {
if (maxSize == null && maxAge == null && maxDocs == null) {
public RolloverAction(ByteSizeValue maxSize, ByteSizeValue maxSinglePrimarySize, TimeValue maxAge, Long maxDocs) {
if (maxSize == null && maxSinglePrimarySize == null && maxAge == null && maxDocs == null) {
throw new IllegalArgumentException("At least one rollover condition must be set.");
}
this.maxSize = maxSize;
this.maxSinglePrimarySize = maxSinglePrimarySize;
this.maxAge = maxAge;
this.maxDocs = maxDocs;
}

public ByteSizeValue getMaxSize() {
return maxSize;
}

public ByteSizeValue getMaxSinglePrimarySize() {
return maxSinglePrimarySize;
}

public TimeValue getMaxAge() {
return maxAge;
}
Expand All @@ -76,6 +89,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
if (maxSize != null) {
builder.field(MAX_SIZE_FIELD.getPreferredName(), maxSize.getStringRep());
}
if (maxSinglePrimarySize != null) {
builder.field(MAX_SINGLE_PRIMARY_SIZE_FIELD.getPreferredName(), maxSinglePrimarySize.getStringRep());
}
if (maxAge != null) {
builder.field(MAX_AGE_FIELD.getPreferredName(), maxAge.getStringRep());
}
Expand All @@ -88,7 +104,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws

@Override
public int hashCode() {
return Objects.hash(maxSize, maxAge, maxDocs);
return Objects.hash(maxSize, maxSinglePrimarySize, maxAge, maxDocs);
}

@Override
Expand All @@ -101,6 +117,7 @@ public boolean equals(Object obj) {
}
RolloverAction other = (RolloverAction) obj;
return Objects.equals(maxSize, other.maxSize) &&
Objects.equals(maxSinglePrimarySize, other.maxSinglePrimarySize) &&
Objects.equals(maxAge, other.maxAge) &&
Objects.equals(maxDocs, other.maxDocs);
}
Expand Down
2 changes: 1 addition & 1 deletion client/rest-high-level/src/test/java/org/elasticsearch/client/IndexLifecycleIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ public void testStartStopILM() throws Exception {
public void testExplainLifecycle() throws Exception {
Map<String, Phase> lifecyclePhases = new HashMap<>();
Map<String, LifecycleAction> hotActions = new HashMap<>();
hotActions.put(RolloverAction.NAME, new RolloverAction(null, TimeValue.timeValueHours(50 * 24), null));
hotActions.put(RolloverAction.NAME, new RolloverAction(null, null, TimeValue.timeValueHours(50 * 24), null));
Phase hotPhase = new Phase("hot", randomFrom(TimeValue.ZERO, null), hotActions);
lifecyclePhases.put("hot", hotPhase);

Expand Down
8 changes: 4 additions & 4 deletions ...t-high-level/src/test/java/org/elasticsearch/client/documentation/ILMDocumentationIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ public void testPutLifecyclePolicy() throws Exception {
Map<String, Phase> phases = new HashMap<>();
Map<String, LifecycleAction> hotActions = new HashMap<>();
hotActions.put(RolloverAction.NAME, new RolloverAction(
new ByteSizeValue(50, ByteSizeUnit.GB), null, null));
new ByteSizeValue(50, ByteSizeUnit.GB), null, null, null));
phases.put("hot", new Phase("hot", TimeValue.ZERO, hotActions)); // <1>

Map<String, LifecycleAction> deleteActions =
Expand Down Expand Up @@ -164,7 +164,7 @@ public void testDeletePolicy() throws IOException, InterruptedException {
Map<String, Phase> phases = new HashMap<>();
Map<String, LifecycleAction> hotActions = new HashMap<>();
hotActions.put(RolloverAction.NAME, new RolloverAction(
new ByteSizeValue(50, ByteSizeUnit.GB), null, null));
new ByteSizeValue(50, ByteSizeUnit.GB), null, null, null));
phases.put("hot", new Phase("hot", TimeValue.ZERO, hotActions));
Map<String, LifecycleAction> deleteActions =
Collections.singletonMap(DeleteAction.NAME,
Expand Down Expand Up @@ -239,7 +239,7 @@ public void testGetLifecyclePolicy() throws IOException, InterruptedException {
Map<String, Phase> phases = new HashMap<>();
Map<String, LifecycleAction> hotActions = new HashMap<>();
hotActions.put(RolloverAction.NAME, new RolloverAction(
new ByteSizeValue(50, ByteSizeUnit.GB), null, null));
new ByteSizeValue(50, ByteSizeUnit.GB), null, null, null));
phases.put("hot", new Phase("hot", TimeValue.ZERO, hotActions));

Map<String, LifecycleAction> deleteActions =
Expand Down Expand Up @@ -339,7 +339,7 @@ public void testExplainLifecycle() throws Exception {
Map<String, Phase> phases = new HashMap<>();
Map<String, LifecycleAction> hotActions = new HashMap<>();
hotActions.put(RolloverAction.NAME, new RolloverAction(
new ByteSizeValue(50, ByteSizeUnit.GB), null, null));
new ByteSizeValue(50, ByteSizeUnit.GB), null, null, null));
phases.put("hot", new Phase("hot", TimeValue.ZERO, hotActions));

LifecyclePolicy policy = new LifecyclePolicy("my_policy",
Expand Down
18 changes: 11 additions & 7 deletions ...high-level/src/test/java/org/elasticsearch/client/indexlifecycle/RolloverActionTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,17 +32,21 @@ protected RolloverAction createTestInstance() {

static RolloverAction randomInstance() {
ByteSizeUnit maxSizeUnit = randomFrom(ByteSizeUnit.values());
ByteSizeValue maxSize = randomBoolean() ? null : new ByteSizeValue(randomNonNegativeLong() / maxSizeUnit.toBytes(1), maxSizeUnit);
Long maxDocs = randomBoolean() ? null : randomNonNegativeLong();
TimeValue maxAge = (maxDocs == null && maxSize == null || randomBoolean())
? TimeValue.parseTimeValue(randomPositiveTimeValue(), "rollover_action_test")
: null;
return new RolloverAction(maxSize, maxAge, maxDocs);
ByteSizeValue maxSize = randomBoolean()
? null : new ByteSizeValue(randomNonNegativeLong() / maxSizeUnit.toBytes(1), maxSizeUnit);
ByteSizeUnit maxSinglePrimarySizeUnit = randomFrom(ByteSizeUnit.values());
ByteSizeValue maxSinglePrimarySize = randomBoolean()
? null : new ByteSizeValue(randomNonNegativeLong() / maxSinglePrimarySizeUnit.toBytes(1), maxSinglePrimarySizeUnit);
TimeValue maxAge = randomBoolean()
? null : TimeValue.parseTimeValue(randomPositiveTimeValue(), "rollover_action_test");
Long maxDocs = (maxSize == null && maxSinglePrimarySize == null && maxAge == null || randomBoolean())
? randomNonNegativeLong() : null;
return new RolloverAction(maxSize, maxSinglePrimarySize, maxAge, maxDocs);
}

public void testNoConditions() {
IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
() -> new RolloverAction(null, null, null));
() -> new RolloverAction(null, null, null, null));
assertEquals("At least one rollover condition must be set.", exception.getMessage());
}
}
10 changes: 10 additions & 0 deletions docs/reference/ilm/actions/ilm-rollover.asciidoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,16 @@ Replicas are not counted toward the maximum index size.
TIP: To see the current index size, use the <<cat-indices, _cat indices>> API.
The `pri.store.size` value shows the combined size of all primary shards.

`max_single_primary_size`::
(Optional, <<byte-units, byte units>>)
Triggers roll over when the largest primary shard in the index reaches a certain size.
This is the maximum size of the primary shards in the index. As with `max_size`,
replicas are ignored.
+
TIP: To see the current shard size, use the <<cat-shards, _cat shards>> API.
The `store` value shows the size each shard, and `prirep` indicates whether a
shard is a primary (`p`) or a replica (`r`).

[[ilm-rollover-ex]]
==== Example

Expand Down
59 changes: 41 additions & 18 deletions x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RolloverAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
*/
package org.elasticsearch.xpack.core.ilm;

import org.elasticsearch.Version;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.ParseField;
Expand Down Expand Up @@ -34,35 +35,45 @@ public class RolloverAction implements LifecycleAction {
public static final String NAME = "rollover";
public static final String INDEXING_COMPLETE_STEP_NAME = "set-indexing-complete";
public static final ParseField MAX_SIZE_FIELD = new ParseField("max_size");
public static final ParseField MAX_SINGLE_PRIMARY_SIZE_FIELD = new ParseField("max_single_primary_size");
public static final ParseField MAX_DOCS_FIELD = new ParseField("max_docs");
public static final ParseField MAX_AGE_FIELD = new ParseField("max_age");
public static final String LIFECYCLE_ROLLOVER_ALIAS = "index.lifecycle.rollover_alias";
public static final Setting<String> LIFECYCLE_ROLLOVER_ALIAS_SETTING = Setting.simpleString(LIFECYCLE_ROLLOVER_ALIAS,
Setting.Property.Dynamic, Setting.Property.IndexScope);

private static final ConstructingObjectParser<RolloverAction, Void> PARSER = new ConstructingObjectParser<>(NAME,
a -> new RolloverAction((ByteSizeValue) a[0], (TimeValue) a[1], (Long) a[2]));
a -> new RolloverAction((ByteSizeValue) a[0], (ByteSizeValue) a[1], (TimeValue) a[2], (Long) a[3]));

static {
PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(),
(p, c) -> ByteSizeValue.parseBytesSizeValue(p.text(), MAX_SIZE_FIELD.getPreferredName()), MAX_SIZE_FIELD, ValueType.VALUE);
(p, c) -> ByteSizeValue.parseBytesSizeValue(p.text(), MAX_SIZE_FIELD.getPreferredName()),
MAX_SIZE_FIELD, ValueType.VALUE);
PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(),
(p, c) -> ByteSizeValue.parseBytesSizeValue(p.text(), MAX_SINGLE_PRIMARY_SIZE_FIELD.getPreferredName()),
MAX_SINGLE_PRIMARY_SIZE_FIELD, ValueType.VALUE);
PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(),
(p, c) -> TimeValue.parseTimeValue(p.text(), MAX_AGE_FIELD.getPreferredName()), MAX_AGE_FIELD, ValueType.VALUE);
(p, c) -> TimeValue.parseTimeValue(p.text(), MAX_AGE_FIELD.getPreferredName()),
MAX_AGE_FIELD, ValueType.VALUE);
PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), MAX_DOCS_FIELD);
}

private final ByteSizeValue maxSize;
private final ByteSizeValue maxSinglePrimarySize;
private final Long maxDocs;
private final TimeValue maxAge;

public static RolloverAction parse(XContentParser parser) {
return PARSER.apply(parser, null);
}

public RolloverAction(@Nullable ByteSizeValue maxSize, @Nullable TimeValue maxAge, @Nullable Long maxDocs) {
if (maxSize == null && maxAge == null && maxDocs == null) {
public RolloverAction(@Nullable ByteSizeValue maxSize, @Nullable ByteSizeValue maxSinglePrimarySize, @Nullable TimeValue maxAge,
@Nullable Long maxDocs) {
if (maxSize == null && maxSinglePrimarySize == null && maxAge == null && maxDocs == null) {
throw new IllegalArgumentException("At least one rollover condition must be set.");
}
this.maxSize = maxSize;
this.maxSinglePrimarySize = maxSinglePrimarySize;
this.maxAge = maxAge;
this.maxDocs = maxDocs;
}
Expand All @@ -73,12 +84,13 @@ public RolloverAction(StreamInput in) throws IOException {
} else {
maxSize = null;
}
maxAge = in.readOptionalTimeValue();
if (in.readBoolean()) {
maxDocs = in.readVLong();
if (in.getVersion().onOrAfter(Version.V_7_13_0) && in.readBoolean()) {
maxSinglePrimarySize = new ByteSizeValue(in);
} else {
maxDocs = null;
maxSinglePrimarySize = null;
}
maxAge = in.readOptionalTimeValue();
maxDocs = in.readOptionalVLong();
}

@Override
Expand All @@ -88,12 +100,15 @@ public void writeTo(StreamOutput out) throws IOException {
if (hasMaxSize) {
maxSize.writeTo(out);
}
out.writeOptionalTimeValue(maxAge);
boolean hasMaxDocs = maxDocs != null;
out.writeBoolean(hasMaxDocs);
if (hasMaxDocs) {
out.writeVLong(maxDocs);
if (out.getVersion().onOrAfter(Version.V_7_13_0)) {
boolean hasMaxSinglePrimarySize = maxSinglePrimarySize != null;
out.writeBoolean(hasMaxSinglePrimarySize);
if (hasMaxSinglePrimarySize) {
maxSinglePrimarySize.writeTo(out);
}
}
out.writeOptionalTimeValue(maxAge);
out.writeOptionalVLong(maxDocs);
}

@Override
Expand All @@ -105,6 +120,10 @@ public ByteSizeValue getMaxSize() {
return maxSize;
}

public ByteSizeValue getMaxSinglePrimarySize() {
return maxSinglePrimarySize;
}

public TimeValue getMaxAge() {
return maxAge;
}
Expand All @@ -119,6 +138,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
if (maxSize != null) {
builder.field(MAX_SIZE_FIELD.getPreferredName(), maxSize.getStringRep());
}
if (maxSinglePrimarySize != null) {
builder.field(MAX_SINGLE_PRIMARY_SIZE_FIELD.getPreferredName(), maxSinglePrimarySize.getStringRep());
}
if (maxAge != null) {
builder.field(MAX_AGE_FIELD.getPreferredName(), maxAge.getStringRep());
}
Expand All @@ -145,7 +167,7 @@ public List<Step> toSteps(Client client, String phase, Step.StepKey nextStepKey)
StepKey setIndexingCompleteStepKey = new StepKey(phase, NAME, INDEXING_COMPLETE_STEP_NAME);

WaitForRolloverReadyStep waitForRolloverReadyStep = new WaitForRolloverReadyStep(waitForRolloverReadyStepKey, rolloverStepKey,
client, maxSize, maxAge, maxDocs);
client, maxSize, maxSinglePrimarySize, maxAge, maxDocs);
RolloverStep rolloverStep = new RolloverStep(rolloverStepKey, waitForActiveShardsKey, client);
WaitForActiveShardsStep waitForActiveShardsStep = new WaitForActiveShardsStep(waitForActiveShardsKey, updateDateStepKey);
UpdateRolloverLifecycleDateStep updateDateStep = new UpdateRolloverLifecycleDateStep(updateDateStepKey, setIndexingCompleteStepKey,
Expand All @@ -157,7 +179,7 @@ public List<Step> toSteps(Client client, String phase, Step.StepKey nextStepKey)

@Override
public int hashCode() {
return Objects.hash(maxSize, maxAge, maxDocs);
return Objects.hash(maxSize, maxSinglePrimarySize, maxAge, maxDocs);
}

@Override
Expand All @@ -170,8 +192,9 @@ public boolean equals(Object obj) {
}
RolloverAction other = (RolloverAction) obj;
return Objects.equals(maxSize, other.maxSize) &&
Objects.equals(maxAge, other.maxAge) &&
Objects.equals(maxDocs, other.maxDocs);
Objects.equals(maxSinglePrimarySize, other.maxSinglePrimarySize) &&
Objects.equals(maxAge, other.maxAge) &&
Objects.equals(maxDocs, other.maxDocs);
}

@Override
Expand Down
22 changes: 16 additions & 6 deletions .../plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForRolloverReadyStep.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,13 +35,15 @@ public class WaitForRolloverReadyStep extends AsyncWaitStep {
public static final String NAME = "check-rollover-ready";

private final ByteSizeValue maxSize;
private final ByteSizeValue maxSinglePrimarySize;
private final TimeValue maxAge;
private final Long maxDocs;

public WaitForRolloverReadyStep(StepKey key, StepKey nextStepKey, Client client, ByteSizeValue maxSize, TimeValue maxAge,
Long maxDocs) {
public WaitForRolloverReadyStep(StepKey key, StepKey nextStepKey, Client client,
ByteSizeValue maxSize, ByteSizeValue maxSinglePrimarySize, TimeValue maxAge, Long maxDocs) {
super(key, nextStepKey, client);
this.maxSize = maxSize;
this.maxSinglePrimarySize = maxSinglePrimarySize;
this.maxAge = maxAge;
this.maxDocs = maxDocs;
}
Expand Down Expand Up @@ -139,12 +141,15 @@ public void evaluateCondition(Metadata metadata, Index index, Listener listener,

RolloverRequest rolloverRequest = new RolloverRequest(rolloverTarget, null).masterNodeTimeout(masterTimeout);
rolloverRequest.dryRun(true);
if (maxAge != null) {
rolloverRequest.addMaxIndexAgeCondition(maxAge);
}
if (maxSize != null) {
rolloverRequest.addMaxIndexSizeCondition(maxSize);
}
if (maxSinglePrimarySize != null) {
rolloverRequest.addMaxSinglePrimarySizeCondition(maxSinglePrimarySize);
}
if (maxAge != null) {
rolloverRequest.addMaxIndexAgeCondition(maxAge);
}
if (maxDocs != null) {
rolloverRequest.addMaxIndexDocsCondition(maxDocs);
}
Expand All @@ -157,6 +162,10 @@ ByteSizeValue getMaxSize() {
return maxSize;
}

ByteSizeValue getMaxSinglePrimarySize() {
return maxSinglePrimarySize;
}

TimeValue getMaxAge() {
return maxAge;
}
Expand All @@ -167,7 +176,7 @@ Long getMaxDocs() {

@Override
public int hashCode() {
return Objects.hash(super.hashCode(), maxSize, maxAge, maxDocs);
return Objects.hash(super.hashCode(), maxSize, maxSinglePrimarySize, maxAge, maxDocs);
}

@Override
Expand All @@ -181,6 +190,7 @@ public boolean equals(Object obj) {
WaitForRolloverReadyStep other = (WaitForRolloverReadyStep) obj;
return super.equals(obj) &&
Objects.equals(maxSize, other.maxSize) &&
Objects.equals(maxSinglePrimarySize, other.maxSinglePrimarySize) &&
Objects.equals(maxAge, other.maxAge) &&
Objects.equals(maxDocs, other.maxDocs);
}
Expand Down
Loading