Skip to content

Commit a06b65c

Browse files
probakowskiprobakowski
authored
[7.x] Hidden data streams (#63987) (#64402)
Backports the following commits to 7.x: Hidden data streams (#63987)
1 parent eeeff09 commit a06b65c

File tree

27 files changed

+410
-60
lines changed

27 files changed

+410
-60
lines changed

‎client/rest-high-level/src/main/java/org/elasticsearch/client/indices/DataStream.java

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ public final class DataStream {
3636
private final String timeStampField;
3737
private final List<String> indices;
3838
private final long generation;
39+
private final boolean hidden;
3940
ClusterHealthStatus dataStreamStatus;
4041
@Nullable
4142
String indexTemplate;
@@ -45,7 +46,8 @@ public final class DataStream {
4546
private final Map<String, Object> metadata;
4647

4748
public DataStream(String name, String timeStampField, List<String> indices, long generation, ClusterHealthStatus dataStreamStatus,
48-
@Nullable String indexTemplate, @Nullable String ilmPolicyName, @Nullable Map<String, Object> metadata) {
49+
@Nullable String indexTemplate, @Nullable String ilmPolicyName, @Nullable Map<String, Object> metadata,
50+
boolean hidden) {
4951
this.name = name;
5052
this.timeStampField = timeStampField;
5153
this.indices = indices;
@@ -54,6 +56,7 @@ public DataStream(String name, String timeStampField, List<String> indices, long
5456
this.indexTemplate = indexTemplate;
5557
this.ilmPolicyName = ilmPolicyName;
5658
this.metadata = metadata;
59+
this.hidden = hidden;
5760
}
5861

5962
public String getName() {
@@ -88,6 +91,10 @@ public Map<String, Object> getMetadata() {
8891
return metadata;
8992
}
9093

94+
public boolean isHidden() {
95+
return hidden;
96+
}
97+
9198
public static final ParseField NAME_FIELD = new ParseField("name");
9299
public static final ParseField TIMESTAMP_FIELD_FIELD = new ParseField("timestamp_field");
93100
public static final ParseField INDICES_FIELD = new ParseField("indices");
@@ -96,6 +103,7 @@ public Map<String, Object> getMetadata() {
96103
public static final ParseField INDEX_TEMPLATE_FIELD = new ParseField("template");
97104
public static final ParseField ILM_POLICY_FIELD = new ParseField("ilm_policy");
98105
public static final ParseField METADATA_FIELD = new ParseField("_meta");
106+
public static final ParseField HIDDEN_FIELD = new ParseField("hidden");
99107

100108
@SuppressWarnings("unchecked")
101109
private static final ConstructingObjectParser<DataStream, Void> PARSER = new ConstructingObjectParser<>("data_stream",
@@ -110,7 +118,9 @@ public Map<String, Object> getMetadata() {
110118
String indexTemplate = (String) args[5];
111119
String ilmPolicy = (String) args[6];
112120
Map<String, Object> metadata = (Map<String, Object>) args[7];
113-
return new DataStream(dataStreamName, timeStampField, indices, generation, status, indexTemplate, ilmPolicy, metadata);
121+
Boolean hidden = (Boolean) args[8];
122+
hidden = hidden != null && hidden;
123+
return new DataStream(dataStreamName, timeStampField, indices, generation, status, indexTemplate, ilmPolicy, metadata, hidden);
114124
});
115125

116126
static {
@@ -122,6 +132,7 @@ public Map<String, Object> getMetadata() {
122132
PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), INDEX_TEMPLATE_FIELD);
123133
PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), ILM_POLICY_FIELD);
124134
PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> p.map(), METADATA_FIELD);
135+
PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), HIDDEN_FIELD);
125136
}
126137

127138
public static DataStream fromXContent(XContentParser parser) throws IOException {

‎docs/reference/data-streams/change-mappings-and-settings.asciidoc

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -572,7 +572,8 @@ contains information about the stream's oldest backing index,
572572
],
573573
"generation": 2,
574574
"status": "GREEN",
575-
"template": "my-data-stream-template"
575+
"template": "my-data-stream-template",
576+
"hidden": false
576577
}
577578
]
578579
}

‎docs/reference/indices/get-data-stream.asciidoc

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -209,7 +209,8 @@ The API returns the following response:
209209
"generation": 2,
210210
"status": "GREEN",
211211
"template": "my-index-template",
212-
"ilm_policy": "my-lifecycle-policy"
212+
"ilm_policy": "my-lifecycle-policy",
213+
"hidden": false
213214
},
214215
{
215216
"name": "my-data-stream_two",
@@ -225,7 +226,8 @@ The API returns the following response:
225226
"generation": 1,
226227
"status": "YELLOW",
227228
"template": "my-index-template",
228-
"ilm_policy": "my-lifecycle-policy"
229+
"ilm_policy": "my-lifecycle-policy",
230+
"hidden": false
229231
}
230232
]
231233
}

‎server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -228,6 +228,13 @@ public EnumSet<WildcardStates> getExpandWildcards() {
228228
return EnumSet.copyOf(expandWildcards);
229229
}
230230

231+
/**
232+
* @return a copy of the {@link Option}s that these indices options will use
233+
*/
234+
public EnumSet<Option> getOptions() {
235+
return EnumSet.copyOf(options);
236+
}
237+
231238
public void writeIndicesOptions(StreamOutput out) throws IOException {
232239
EnumSet<Option> options = this.options;
233240
// never write this out to a pre 6.6 version

‎server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java

Lines changed: 32 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@
2929
import org.elasticsearch.common.io.stream.StreamOutput;
3030
import org.elasticsearch.common.io.stream.Writeable;
3131
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
32-
import org.elasticsearch.common.xcontent.ObjectParser;
3332
import org.elasticsearch.common.xcontent.ToXContentObject;
3433
import org.elasticsearch.common.xcontent.XContentBuilder;
3534
import org.elasticsearch.common.xcontent.XContentParser;
@@ -250,20 +249,33 @@ public String toString() {
250249

251250
public static class DataStreamTemplate implements Writeable, ToXContentObject {
252251

253-
private static final ObjectParser<DataStreamTemplate, Void> PARSER = new ObjectParser<>(
252+
private static final ParseField HIDDEN = new ParseField("hidden");
253+
254+
public static final ConstructingObjectParser<DataStreamTemplate, Void> PARSER = new ConstructingObjectParser<>(
254255
"data_stream_template",
255-
DataStreamTemplate::new
256-
);
256+
false,
257+
a -> new DataStreamTemplate(a[0] != null && (boolean) a[0]));
258+
259+
static {
260+
PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), HIDDEN);
261+
}
262+
263+
private final boolean hidden;
257264

258265
public DataStreamTemplate() {
266+
this(false);
259267
}
260268

261-
public String getTimestampField() {
262-
return FIXED_TIMESTAMP_FIELD;
269+
public DataStreamTemplate(boolean hidden) {
270+
this.hidden = hidden;
263271
}
264272

265-
DataStreamTemplate(StreamInput in) {
266-
this();
273+
DataStreamTemplate(StreamInput in) throws IOException {
274+
hidden = in.getVersion().onOrAfter(DataStream.HIDDEN_VERSION) && in.readBoolean();
275+
}
276+
277+
public String getTimestampField() {
278+
return FIXED_TIMESTAMP_FIELD;
267279
}
268280

269281
/**
@@ -275,26 +287,36 @@ public Map<String, Object> getDataStreamMappingSnippet() {
275287
singletonMap("enabled", true)));
276288
}
277289

290+
public boolean isHidden() {
291+
return hidden;
292+
}
293+
278294
@Override
279295
public void writeTo(StreamOutput out) throws IOException {
296+
if (out.getVersion().onOrAfter(DataStream.HIDDEN_VERSION)) {
297+
out.writeBoolean(hidden);
298+
}
280299
}
281300

282301
@Override
283302
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
284303
builder.startObject();
304+
builder.field("hidden", hidden);
285305
builder.endObject();
286306
return builder;
287307
}
288308

289309
@Override
290310
public boolean equals(Object o) {
291311
if (this == o) return true;
292-
return o != null && getClass() == o.getClass();
312+
if (o == null || getClass() != o.getClass()) return false;
313+
DataStreamTemplate that = (DataStreamTemplate) o;
314+
return hidden == that.hidden;
293315
}
294316

295317
@Override
296318
public int hashCode() {
297-
return DataStreamTemplate.class.hashCode();
319+
return Objects.hash(hidden);
298320
}
299321
}
300322
}

‎server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java

Lines changed: 24 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -43,19 +43,27 @@
4343
public final class DataStream extends AbstractDiffable<DataStream> implements ToXContentObject {
4444

4545
public static final String BACKING_INDEX_PREFIX = ".ds-";
46+
public static final Version HIDDEN_VERSION = Version.V_7_11_0;
4647

4748
private final String name;
4849
private final TimestampField timeStampField;
4950
private final List<Index> indices;
5051
private final long generation;
5152
private final Map<String, Object> metadata;
53+
private final boolean hidden;
5254

5355
public DataStream(String name, TimestampField timeStampField, List<Index> indices, long generation, Map<String, Object> metadata) {
56+
this(name, timeStampField, indices, generation, metadata, false);
57+
}
58+
59+
public DataStream(String name, TimestampField timeStampField, List<Index> indices, long generation, Map<String, Object> metadata,
60+
boolean hidden) {
5461
this.name = name;
5562
this.timeStampField = timeStampField;
5663
this.indices = Collections.unmodifiableList(indices);
5764
this.generation = generation;
5865
this.metadata = metadata;
66+
this.hidden = hidden;
5967
assert indices.size() > 0;
6068
assert indices.get(indices.size() - 1).getName().equals(getDefaultBackingIndexName(name, generation));
6169
}
@@ -85,6 +93,10 @@ public Map<String, Object> getMetadata() {
8593
return metadata;
8694
}
8795

96+
public boolean isHidden() {
97+
return hidden;
98+
}
99+
88100
/**
89101
* Performs a rollover on a {@code DataStream} instance and returns a new instance containing
90102
* the updated list of backing indices and incremented generation.
@@ -97,7 +109,7 @@ public DataStream rollover(Index newWriteIndex) {
97109
assert newWriteIndex.getName().equals(getDefaultBackingIndexName(name, generation + 1));
98110
List<Index> backingIndices = new ArrayList<>(indices);
99111
backingIndices.add(newWriteIndex);
100-
return new DataStream(name, timeStampField, backingIndices, generation + 1, metadata);
112+
return new DataStream(name, timeStampField, backingIndices, generation + 1, metadata, hidden);
101113
}
102114

103115
/**
@@ -111,7 +123,7 @@ public DataStream removeBackingIndex(Index index) {
111123
List<Index> backingIndices = new ArrayList<>(indices);
112124
backingIndices.remove(index);
113125
assert backingIndices.size() == indices.size() - 1;
114-
return new DataStream(name, timeStampField, backingIndices, generation, metadata);
126+
return new DataStream(name, timeStampField, backingIndices, generation, metadata, hidden);
115127
}
116128

117129
/**
@@ -136,7 +148,7 @@ public DataStream replaceBackingIndex(Index existingBackingIndex, Index newBacki
136148
"it is the write index", existingBackingIndex.getName(), name));
137149
}
138150
backingIndices.set(backingIndexPosition, newBackingIndex);
139-
return new DataStream(name, timeStampField, backingIndices, generation, metadata);
151+
return new DataStream(name, timeStampField, backingIndices, generation, metadata, hidden);
140152
}
141153

142154
/**
@@ -153,7 +165,8 @@ public static String getDefaultBackingIndexName(String dataStreamName, long gene
153165

154166
public DataStream(StreamInput in) throws IOException {
155167
this(in.readString(), new TimestampField(in), in.readList(Index::new), in.readVLong(),
156-
in.getVersion().onOrAfter(Version.V_7_11_0) ? in.readMap(): null);
168+
in.getVersion().onOrAfter(Version.V_7_11_0) ? in.readMap(): null,
169+
in.getVersion().onOrAfter(HIDDEN_VERSION) && in.readBoolean());
157170
}
158171

159172
public static Diff<DataStream> readDiffFrom(StreamInput in) throws IOException {
@@ -169,25 +182,30 @@ public void writeTo(StreamOutput out) throws IOException {
169182
if (out.getVersion().onOrAfter(Version.V_7_11_0)) {
170183
out.writeMap(metadata);
171184
}
185+
if (out.getVersion().onOrAfter(HIDDEN_VERSION)) {
186+
out.writeBoolean(hidden);
187+
}
172188
}
173189

174190
public static final ParseField NAME_FIELD = new ParseField("name");
175191
public static final ParseField TIMESTAMP_FIELD_FIELD = new ParseField("timestamp_field");
176192
public static final ParseField INDICES_FIELD = new ParseField("indices");
177193
public static final ParseField GENERATION_FIELD = new ParseField("generation");
178194
public static final ParseField METADATA_FIELD = new ParseField("_meta");
195+
public static final ParseField HIDDEN_FIELD = new ParseField("hidden");
179196

180197
@SuppressWarnings("unchecked")
181198
private static final ConstructingObjectParser<DataStream, Void> PARSER = new ConstructingObjectParser<>("data_stream",
182199
args -> new DataStream((String) args[0], (TimestampField) args[1], (List<Index>) args[2], (Long) args[3],
183-
(Map<String, Object>) args[4]));
200+
(Map<String, Object>) args[4], args[5] != null && (boolean) args[5]));
184201

185202
static {
186203
PARSER.declareString(ConstructingObjectParser.constructorArg(), NAME_FIELD);
187204
PARSER.declareObject(ConstructingObjectParser.constructorArg(), TimestampField.PARSER, TIMESTAMP_FIELD_FIELD);
188205
PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), (p, c) -> Index.fromXContent(p), INDICES_FIELD);
189206
PARSER.declareLong(ConstructingObjectParser.constructorArg(), GENERATION_FIELD);
190207
PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> p.map(), METADATA_FIELD);
208+
PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), HIDDEN_FIELD);
191209
}
192210

193211
public static DataStream fromXContent(XContentParser parser) throws IOException {
@@ -204,6 +222,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
204222
if (metadata != null) {
205223
builder.field(METADATA_FIELD.getPreferredName(), metadata);
206224
}
225+
builder.field(HIDDEN_FIELD.getPreferredName(), hidden);
207226
builder.endObject();
208227
return builder;
209228
}

‎server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -339,7 +339,7 @@ public DataStream getParentDataStream() {
339339

340340
@Override
341341
public boolean isHidden() {
342-
return false;
342+
return dataStream.isHidden();
343343
}
344344

345345
@Override

‎server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -131,23 +131,18 @@ public static boolean isIndexVisible(String expression, String index, IndicesOpt
131131
throw new IllegalStateException("could not resolve index abstraction [" + index + "]");
132132
}
133133
final boolean isHidden = indexAbstraction.isHidden();
134+
boolean isVisible = isHidden == false || indicesOptions.expandWildcardsHidden() || isVisibleDueToImplicitHidden(expression, index);
134135
if (indexAbstraction.getType() == IndexAbstraction.Type.ALIAS) {
135136
//it's an alias, ignore expandWildcardsOpen and expandWildcardsClosed.
136137
//complicated to support those options with aliases pointing to multiple indices...
137-
if (indicesOptions.ignoreAliases()) {
138-
return false;
139-
} else if (isHidden == false || indicesOptions.expandWildcardsHidden() || isVisibleDueToImplicitHidden(expression, index)) {
140-
return true;
141-
} else {
142-
return false;
143-
}
138+
return isVisible && indicesOptions.ignoreAliases() == false;
144139
}
145140
if (indexAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM) {
146-
return includeDataStreams;
141+
return isVisible && includeDataStreams;
147142
}
148143
assert indexAbstraction.getIndices().size() == 1 : "concrete index must point to a single index";
149144
IndexMetadata indexMetadata = indexAbstraction.getIndices().get(0);
150-
if (isHidden && indicesOptions.expandWildcardsHidden() == false && isVisibleDueToImplicitHidden(expression, index) == false) {
145+
if (isVisible == false) {
151146
return false;
152147
}
153148

‎server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -137,8 +137,9 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn
137137
if (request.name.toLowerCase(Locale.ROOT).equals(request.name) == false) {
138138
throw new IllegalArgumentException("data_stream [" + request.name + "] must be lowercase");
139139
}
140-
if (request.name.startsWith(".")) {
141-
throw new IllegalArgumentException("data_stream [" + request.name + "] must not start with '.'");
140+
if (request.name.startsWith(DataStream.BACKING_INDEX_PREFIX)) {
141+
throw new IllegalArgumentException("data_stream [" + request.name + "] must not start with '"
142+
+ DataStream.BACKING_INDEX_PREFIX + "'");
142143
}
143144

144145
ComposableIndexTemplate template = lookupTemplateForDataStream(request.name, currentState.metadata());
@@ -163,10 +164,11 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn
163164

164165
String fieldName = template.getDataStreamTemplate().getTimestampField();
165166
DataStream.TimestampField timestampField = new DataStream.TimestampField(fieldName);
167+
boolean hidden = template.getDataStreamTemplate().isHidden();
166168
DataStream newDataStream =
167169
new DataStream(request.name, timestampField,
168170
Collections.singletonList(firstBackingIndex.getIndex()), 1L,
169-
template.metadata() != null ? Collections.unmodifiableMap(new HashMap<>(template.metadata())) : null);
171+
template.metadata() != null ? Collections.unmodifiableMap(new HashMap<>(template.metadata())) : null, hidden);
170172
Metadata.Builder builder = Metadata.builder(currentState.metadata()).put(newDataStream);
171173
logger.info("adding data stream [{}]", request.name);
172174
return ClusterState.builder(currentState).metadata(builder).build();

‎server/src/main/java/org/elasticsearch/snapshots/RestoreService.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -620,7 +620,7 @@ static DataStream updateDataStream(DataStream dataStream, Metadata.Builder metad
620620
.map(i -> metadata.get(renameIndex(i.getName(), request, true)).getIndex())
621621
.collect(Collectors.toList());
622622
return new DataStream(dataStreamName, dataStream.getTimeStampField(), updatedIndices, dataStream.getGeneration(),
623-
dataStream.getMetadata());
623+
dataStream.getMetadata(), dataStream.isHidden());
624624
}
625625

626626
public static RestoreInProgress updateRestoreStateWithDeletedIndices(RestoreInProgress oldRestore, Set<Index> deletedIndices) {

0 commit comments

Comments
 (0)