Skip to content

Commit 4d18334

Browse files
joegallojoegallo
authored
Add max_single_primary_size as a condition for the rollover index API (#67842)
1 parent 509a154 commit 4d18334

File tree

25 files changed

+450
-155
lines changed

25 files changed

+450
-155
lines changed

‎client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverRequest.java

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import org.elasticsearch.action.admin.indices.rollover.Condition;
1111
import org.elasticsearch.action.admin.indices.rollover.MaxAgeCondition;
1212
import org.elasticsearch.action.admin.indices.rollover.MaxDocsCondition;
13+
import org.elasticsearch.action.admin.indices.rollover.MaxSinglePrimarySizeCondition;
1314
import org.elasticsearch.action.admin.indices.rollover.MaxSizeCondition;
1415
import org.elasticsearch.client.TimedRequest;
1516
import org.elasticsearch.client.indices.CreateIndexRequest;
@@ -94,6 +95,7 @@ public RolloverRequest addMaxIndexDocsCondition(long numDocs) {
9495
this.conditions.put(maxDocsCondition.name(), maxDocsCondition);
9596
return this;
9697
}
98+
9799
/**
98100
* Adds a size-based condition to check if the index size is at least <code>size</code>.
99101
*/
@@ -105,6 +107,19 @@ public RolloverRequest addMaxIndexSizeCondition(ByteSizeValue size) {
105107
this.conditions.put(maxSizeCondition.name(), maxSizeCondition);
106108
return this;
107109
}
110+
111+
/**
112+
* Adds a size-based condition to check if the size of the largest primary shard is at least <code>size</code>.
113+
*/
114+
public RolloverRequest addMaxSinglePrimarySizeCondition(ByteSizeValue size) {
115+
MaxSinglePrimarySizeCondition maxSinglePrimarySizeCondition = new MaxSinglePrimarySizeCondition(size);
116+
if (this.conditions.containsKey(maxSinglePrimarySizeCondition.name())) {
117+
throw new IllegalArgumentException(maxSinglePrimarySizeCondition + " condition is already set");
118+
}
119+
this.conditions.put(maxSinglePrimarySizeCondition.name(), maxSinglePrimarySizeCondition);
120+
return this;
121+
}
122+
108123
/**
109124
* Returns all set conditions
110125
*/

‎client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -958,15 +958,17 @@ public void testRollover() throws IOException {
958958
rolloverRequest.getCreateIndexRequest().mapping(mappings, XContentType.JSON);
959959
rolloverRequest.dryRun(false);
960960
rolloverRequest.addMaxIndexSizeCondition(new ByteSizeValue(1, ByteSizeUnit.MB));
961+
rolloverRequest.addMaxSinglePrimarySizeCondition(new ByteSizeValue(1, ByteSizeUnit.MB));
961962
RolloverResponse rolloverResponse = execute(rolloverRequest, highLevelClient().indices()::rollover,
962963
highLevelClient().indices()::rolloverAsync);
963964
assertTrue(rolloverResponse.isRolledOver());
964965
assertFalse(rolloverResponse.isDryRun());
965966
Map<String, Boolean> conditionStatus = rolloverResponse.getConditionStatus();
966-
assertEquals(3, conditionStatus.size());
967+
assertEquals(4, conditionStatus.size());
967968
assertTrue(conditionStatus.get("[max_docs: 1]"));
968969
assertTrue(conditionStatus.get("[max_age: 1ms]"));
969970
assertFalse(conditionStatus.get("[max_size: 1mb]"));
971+
assertFalse(conditionStatus.get("[max_single_primary_size: 1mb]"));
970972
assertEquals("test", rolloverResponse.getOldIndex());
971973
assertEquals("test_new", rolloverResponse.getNewIndex());
972974
}

‎client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1802,6 +1802,7 @@ public void testRolloverIndex() throws Exception {
18021802
request.addMaxIndexAgeCondition(new TimeValue(7, TimeUnit.DAYS)); // <2>
18031803
request.addMaxIndexDocsCondition(1000); // <3>
18041804
request.addMaxIndexSizeCondition(new ByteSizeValue(5, ByteSizeUnit.GB)); // <4>
1805+
request.addMaxSinglePrimarySizeCondition(new ByteSizeValue(2, ByteSizeUnit.GB)); // <5>
18051806
// end::rollover-index-request
18061807

18071808
// tag::rollover-index-request-timeout
@@ -1848,7 +1849,7 @@ public void testRolloverIndex() throws Exception {
18481849
assertEquals("index-2", newIndex);
18491850
assertFalse(isRolledOver);
18501851
assertTrue(isDryRun);
1851-
assertEquals(3, conditionStatus.size());
1852+
assertEquals(4, conditionStatus.size());
18521853

18531854
// tag::rollover-index-execute-listener
18541855
ActionListener<RolloverResponse> listener = new ActionListener<RolloverResponse>() {

‎client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverRequestTests.java

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import org.elasticsearch.action.admin.indices.rollover.Condition;
1212
import org.elasticsearch.action.admin.indices.rollover.MaxAgeCondition;
1313
import org.elasticsearch.action.admin.indices.rollover.MaxDocsCondition;
14+
import org.elasticsearch.action.admin.indices.rollover.MaxSinglePrimarySizeCondition;
1415
import org.elasticsearch.action.admin.indices.rollover.MaxSizeCondition;
1516
import org.elasticsearch.common.unit.ByteSizeValue;
1617
import org.elasticsearch.common.unit.TimeValue;
@@ -36,12 +37,16 @@ public void testConstructorAndFieldAssignments() {
3637

3738
// test assignment of conditions
3839
MaxAgeCondition maxAgeCondition = new MaxAgeCondition(new TimeValue(10));
39-
MaxSizeCondition maxSizeCondition = new MaxSizeCondition(new ByteSizeValue(2000));
4040
MaxDocsCondition maxDocsCondition = new MaxDocsCondition(10000L);
41-
Condition<?>[] expectedConditions = new Condition<?>[] {maxAgeCondition, maxSizeCondition, maxDocsCondition};
41+
MaxSizeCondition maxSizeCondition = new MaxSizeCondition(new ByteSizeValue(2000));
42+
MaxSinglePrimarySizeCondition maxSinglePrimarySizeCondition = new MaxSinglePrimarySizeCondition(new ByteSizeValue(3000));
43+
Condition<?>[] expectedConditions = new Condition<?>[]{
44+
maxAgeCondition, maxDocsCondition, maxSizeCondition, maxSinglePrimarySizeCondition
45+
};
4246
rolloverRequest.addMaxIndexAgeCondition(maxAgeCondition.value());
43-
rolloverRequest.addMaxIndexSizeCondition(maxSizeCondition.value());
4447
rolloverRequest.addMaxIndexDocsCondition(maxDocsCondition.value());
48+
rolloverRequest.addMaxIndexSizeCondition(maxSizeCondition.value());
49+
rolloverRequest.addMaxSinglePrimarySizeCondition(maxSinglePrimarySizeCondition.value());
4550
List<Condition<?>> requestConditions = new ArrayList<>(rolloverRequest.getConditions().values());
4651
assertThat(requestConditions, containsInAnyOrder(expectedConditions));
4752
}

‎client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverResponseTests.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import org.elasticsearch.action.admin.indices.rollover.Condition;
1212
import org.elasticsearch.action.admin.indices.rollover.MaxAgeCondition;
1313
import org.elasticsearch.action.admin.indices.rollover.MaxDocsCondition;
14+
import org.elasticsearch.action.admin.indices.rollover.MaxSinglePrimarySizeCondition;
1415
import org.elasticsearch.action.admin.indices.rollover.MaxSizeCondition;
1516
import org.elasticsearch.common.unit.ByteSizeValue;
1617
import org.elasticsearch.common.unit.TimeValue;
@@ -32,8 +33,9 @@ public class RolloverResponseTests extends ESTestCase {
3233
private static final List<Supplier<Condition<?>>> conditionSuppliers = new ArrayList<>();
3334
static {
3435
conditionSuppliers.add(() -> new MaxAgeCondition(new TimeValue(randomNonNegativeLong())));
35-
conditionSuppliers.add(() -> new MaxSizeCondition(new ByteSizeValue(randomNonNegativeLong())));
3636
conditionSuppliers.add(() -> new MaxDocsCondition(randomNonNegativeLong()));
37+
conditionSuppliers.add(() -> new MaxSizeCondition(new ByteSizeValue(randomNonNegativeLong())));
38+
conditionSuppliers.add(() -> new MaxSinglePrimarySizeCondition(new ByteSizeValue(randomNonNegativeLong())));
3739
}
3840

3941
public void testFromXContent() throws IOException {

‎docs/java-rest/high-level/indices/rollover.asciidoc

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ The new index argument is optional, and can be set to null
2424
<2> Condition on the age of the index
2525
<3> Condition on the number of documents in the index
2626
<4> Condition on the size of the index
27+
<5> Condition on the size of the largest primary shard of the index
2728

2829
==== Optional arguments
2930
The following arguments can optionally be provided:

‎docs/reference/indices/rollover-index.asciidoc

Lines changed: 24 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,8 @@ POST /alias1/_rollover/my-index-000002
1818
"conditions": {
1919
"max_age": "7d",
2020
"max_docs": 1000,
21-
"max_size": "5gb"
21+
"max_size": "5gb",
22+
"max_single_primary_size": "2gb"
2223
}
2324
}
2425
----
@@ -163,6 +164,16 @@ Replicas are not counted toward the maximum index size.
163164

164165
TIP: To see the current index size, use the <<cat-indices, _cat indices>> API.
165166
The `pri.store.size` value shows the combined size of all primary shards.
167+
168+
`max_single_primary_size`::
169+
(Optional, <<byte-units, byte units>>)
170+
Maximum primary shard size.
171+
This is the maximum size of the primary shards in the index. As with `max_size`,
172+
replicas are ignored.
173+
174+
TIP: To see the current shard size, use the <<cat-shards, _cat shards>> API.
175+
The `store` value shows the size each shard, and `prirep` indicates whether a
176+
shard is a primary (`p`) or a replica (`r`).
166177
--
167178

168179
include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=mappings]
@@ -192,7 +203,8 @@ POST /logs_write/_rollover <2>
192203
"conditions": {
193204
"max_age": "7d",
194205
"max_docs": 1000,
195-
"max_size": "5gb"
206+
"max_size": "5gb",
207+
"max_single_primary_size": "2gb"
196208
}
197209
}
198210
--------------------------------------------------
@@ -218,6 +230,7 @@ The API returns the following response:
218230
"[max_age: 7d]": false,
219231
"[max_docs: 1000]": true,
220232
"[max_size: 5gb]": false,
233+
"[max_single_primary_size: 2gb]": false
221234
}
222235
}
223236
--------------------------------------------------
@@ -250,7 +263,8 @@ POST /my-data-stream/_rollover <2>
250263
"conditions" : {
251264
"max_age": "7d",
252265
"max_docs": 1000,
253-
"max_size": "5gb"
266+
"max_size": "5gb",
267+
"max_single_primary_size": "2gb"
254268
}
255269
}
256270
--------------------------------------------------
@@ -284,6 +298,7 @@ The API returns the following response:
284298
"[max_age: 7d]": false,
285299
"[max_docs: 1000]": true,
286300
"[max_size: 5gb]": false,
301+
"[max_single_primary_size: 2gb]": false
287302
}
288303
}
289304
--------------------------------------------------
@@ -329,7 +344,8 @@ POST /logs_write/_rollover
329344
"conditions" : {
330345
"max_age": "7d",
331346
"max_docs": 1000,
332-
"max_size": "5gb"
347+
"max_size": "5gb",
348+
"max_single_primary_size": "2gb"
333349
},
334350
"settings": {
335351
"index.number_of_shards": 2
@@ -359,7 +375,8 @@ POST /my_alias/_rollover/my_new_index_name
359375
"conditions": {
360376
"max_age": "7d",
361377
"max_docs": 1000,
362-
"max_size": "5gb"
378+
"max_size": "5gb",
379+
"max_single_primary_size": "2gb"
363380
}
364381
}
365382
--------------------------------------------------
@@ -456,7 +473,8 @@ POST /logs_write/_rollover?dry_run
456473
"conditions" : {
457474
"max_age": "7d",
458475
"max_docs": 1000,
459-
"max_size": "5gb"
476+
"max_size": "5gb",
477+
"max_single_primary_size": "2gb"
460478
}
461479
}
462480
--------------------------------------------------

‎rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/35_max_single_primary_size_condition.yml

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
---
2+
"Rollover with max_single_primary_size condition":
3+
- skip:
4+
version: " - 7.99.99"
5+
reason: max_single_primary_size condition was introduced in 8.0.0
6+
7+
# create index with alias and replica
8+
- do:
9+
indices.create:
10+
index: logs-1
11+
wait_for_active_shards: 1
12+
body:
13+
aliases:
14+
logs_search: {}
15+
16+
# index a document
17+
- do:
18+
index:
19+
index: logs-1
20+
id: "1"
21+
body: { "foo": "hello world" }
22+
refresh: true
23+
24+
# perform alias rollover with a large max_single_primary_size, no action.
25+
- do:
26+
indices.rollover:
27+
alias: "logs_search"
28+
wait_for_active_shards: 1
29+
body:
30+
conditions:
31+
max_single_primary_size: 100mb
32+
33+
- match: { conditions: { "[max_single_primary_size: 100mb]": false } }
34+
- match: { rolled_over: false }
35+
36+
# perform alias rollover with a small max_single_primary_size, got action.
37+
- do:
38+
indices.rollover:
39+
alias: "logs_search"
40+
wait_for_active_shards: 1
41+
body:
42+
conditions:
43+
max_single_primary_size: 10b
44+
45+
- match: { conditions: { "[max_single_primary_size: 10b]": true } }
46+
- match: { rolled_over: true }
47+
48+
# perform alias rollover on an empty index, no action.
49+
- do:
50+
indices.rollover:
51+
alias: "logs_search"
52+
wait_for_active_shards: 1
53+
body:
54+
conditions:
55+
max_single_primary_size: 1b
56+
57+
- match: { conditions: { "[max_single_primary_size: 1b]": false } }
58+
- match: { rolled_over: false }

‎server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -434,6 +434,62 @@ public void testRolloverMaxSize() throws Exception {
434434
}
435435
}
436436

437+
public void testRolloverMaxSinglePrimarySize() throws Exception {
438+
assertAcked(prepareCreate("test-1").addAlias(new Alias("test_alias")).get());
439+
int numDocs = randomIntBetween(10, 20);
440+
for (int i = 0; i < numDocs; i++) {
441+
indexDoc("test-1", Integer.toString(i), "field", "foo-" + i);
442+
}
443+
flush("test-1");
444+
refresh("test_alias");
445+
446+
// A large max_single_primary_size
447+
{
448+
final RolloverResponse response = client().admin().indices()
449+
.prepareRolloverIndex("test_alias")
450+
.addMaxSinglePrimarySizeCondition(new ByteSizeValue(randomIntBetween(100, 50 * 1024), ByteSizeUnit.MB))
451+
.get();
452+
assertThat(response.getOldIndex(), equalTo("test-1"));
453+
assertThat(response.getNewIndex(), equalTo("test-000002"));
454+
assertThat("No rollover with a large max_single_primary_size condition", response.isRolledOver(), equalTo(false));
455+
final IndexMetadata oldIndex = client().admin().cluster().prepareState().get().getState().metadata().index("test-1");
456+
assertThat(oldIndex.getRolloverInfos().size(), equalTo(0));
457+
}
458+
459+
// A small max_single_primary_size
460+
{
461+
ByteSizeValue maxSinglePrimarySizeCondition = new ByteSizeValue(randomIntBetween(1, 20), ByteSizeUnit.BYTES);
462+
long beforeTime = client().threadPool().absoluteTimeInMillis() - 1000L;
463+
final RolloverResponse response = client().admin().indices()
464+
.prepareRolloverIndex("test_alias")
465+
.addMaxSinglePrimarySizeCondition(maxSinglePrimarySizeCondition)
466+
.get();
467+
assertThat(response.getOldIndex(), equalTo("test-1"));
468+
assertThat(response.getNewIndex(), equalTo("test-000002"));
469+
assertThat("Should rollover with a small max_single_primary_size condition", response.isRolledOver(), equalTo(true));
470+
final IndexMetadata oldIndex = client().admin().cluster().prepareState().get().getState().metadata().index("test-1");
471+
List<Condition<?>> metConditions = oldIndex.getRolloverInfos().get("test_alias").getMetConditions();
472+
assertThat(metConditions.size(), equalTo(1));
473+
assertThat(metConditions.get(0).toString(),
474+
equalTo(new MaxSinglePrimarySizeCondition(maxSinglePrimarySizeCondition).toString()));
475+
assertThat(oldIndex.getRolloverInfos().get("test_alias").getTime(),
476+
is(both(greaterThanOrEqualTo(beforeTime)).and(lessThanOrEqualTo(client().threadPool().absoluteTimeInMillis() + 1000L))));
477+
}
478+
479+
// An empty index
480+
{
481+
final RolloverResponse response = client().admin().indices()
482+
.prepareRolloverIndex("test_alias")
483+
.addMaxSinglePrimarySizeCondition(new ByteSizeValue(randomNonNegativeLong(), ByteSizeUnit.BYTES))
484+
.get();
485+
assertThat(response.getOldIndex(), equalTo("test-000002"));
486+
assertThat(response.getNewIndex(), equalTo("test-000003"));
487+
assertThat("No rollover with an empty index", response.isRolledOver(), equalTo(false));
488+
final IndexMetadata oldIndex = client().admin().cluster().prepareState().get().getState().metadata().index("test-000002");
489+
assertThat(oldIndex.getRolloverInfos().size(), equalTo(0));
490+
}
491+
}
492+
437493
public void testRejectIfAliasFoundInTemplate() throws Exception {
438494
client().admin().indices().preparePutTemplate("logs")
439495
.setPatterns(Collections.singletonList("logs-*")).addAlias(new Alias("logs-write")).get();

‎server/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88

99
package org.elasticsearch.action.admin.indices.rollover;
1010

11+
import org.elasticsearch.Version;
1112
import org.elasticsearch.common.io.stream.NamedWriteable;
1213
import org.elasticsearch.common.unit.ByteSizeValue;
1314
import org.elasticsearch.common.xcontent.ToXContentFragment;
@@ -26,6 +27,14 @@ protected Condition(String name) {
2627
this.name = name;
2728
}
2829

30+
/**
31+
* Checks if this condition is available in a specific version.
32+
* This makes sure BWC when introducing a new condition which is not recognized by older versions.
33+
*/
34+
boolean includedInVersion(Version version) {
35+
return true;
36+
}
37+
2938
public abstract Result evaluate(Stats stats);
3039

3140
@Override
@@ -66,11 +75,13 @@ public static class Stats {
6675
public final long numDocs;
6776
public final long indexCreated;
6877
public final ByteSizeValue indexSize;
78+
public final ByteSizeValue maxSinglePrimarySize;
6979

70-
public Stats(long numDocs, long indexCreated, ByteSizeValue indexSize) {
80+
public Stats(long numDocs, long indexCreated, ByteSizeValue indexSize, ByteSizeValue maxSinglePrimarySize) {
7181
this.numDocs = numDocs;
7282
this.indexCreated = indexCreated;
7383
this.indexSize = indexSize;
84+
this.maxSinglePrimarySize = maxSinglePrimarySize;
7485
}
7586
}
7687

0 commit comments

Comments
 (0)