Hidden data streams (#63987)

* Hidden data streams

* whitespace reverted

* stricter ds name

* Revert "stricter ds name"

This reverts commit 100dba5.

* String.format removed

* fix test

* fix GetDataStream action

* fix test

* fix test

* rest test

* rest test

* spotless

* tests

* Delete a.json

* added expand_wildcards for GetDataStream and DeleteDataStream

* unused imports

* add hidden setting to data stream template

* fix expand_wildcards

* spotless

* fix compilation

* unused import

* yaml test

* fix test

* fix cleanup

* review

* compilation fix

* fix javadoc

* fix javadoc

client/rest-high-level/src/main/java/org/elasticsearch/client/indices/DataStream.java

@@ -36,6 +36,7 @@ public final class DataStream {
     private final String timeStampField;
     private final List<String> indices;
     private final long generation;
+    private final boolean hidden;
     ClusterHealthStatus dataStreamStatus;
     @Nullable
     String indexTemplate;
@@ -45,7 +46,8 @@ public final class DataStream {
     private final Map<String, Object> metadata;
 
     public DataStream(String name, String timeStampField, List<String> indices, long generation, ClusterHealthStatus dataStreamStatus,
-                      @Nullable String indexTemplate, @Nullable String ilmPolicyName, @Nullable  Map<String, Object> metadata) {
+                      @Nullable String indexTemplate, @Nullable String ilmPolicyName, @Nullable  Map<String, Object> metadata,
+                      boolean hidden) {
         this.name = name;
         this.timeStampField = timeStampField;
         this.indices = indices;
@@ -54,6 +56,7 @@ public DataStream(String name, String timeStampField, List<String> indices, long
         this.indexTemplate = indexTemplate;
         this.ilmPolicyName = ilmPolicyName;
         this.metadata = metadata;
+        this.hidden = hidden;
     }
 
     public String getName() {
@@ -88,6 +91,10 @@ public Map<String, Object> getMetadata() {
         return metadata;
     }
 
+    public boolean isHidden() {
+        return hidden;
+    }
+
     public static final ParseField NAME_FIELD = new ParseField("name");
     public static final ParseField TIMESTAMP_FIELD_FIELD = new ParseField("timestamp_field");
     public static final ParseField INDICES_FIELD = new ParseField("indices");
@@ -96,6 +103,7 @@ public Map<String, Object> getMetadata() {
     public static final ParseField INDEX_TEMPLATE_FIELD = new ParseField("template");
     public static final ParseField ILM_POLICY_FIELD = new ParseField("ilm_policy");
     public static final ParseField METADATA_FIELD = new ParseField("_meta");
+    public static final ParseField HIDDEN_FIELD = new ParseField("hidden");
 
     @SuppressWarnings("unchecked")
     private static final ConstructingObjectParser<DataStream, Void> PARSER = new ConstructingObjectParser<>("data_stream",
@@ -110,7 +118,9 @@ public Map<String, Object> getMetadata() {
             String indexTemplate = (String) args[5];
             String ilmPolicy = (String) args[6];
             Map<String, Object> metadata = (Map<String, Object>) args[7];
-            return new DataStream(dataStreamName, timeStampField, indices, generation, status, indexTemplate, ilmPolicy, metadata);
+            Boolean hidden = (Boolean) args[8];
+            hidden = hidden != null && hidden;
+            return new DataStream(dataStreamName, timeStampField, indices, generation, status, indexTemplate, ilmPolicy, metadata, hidden);
         });
 
     static {
@@ -122,6 +132,7 @@ public Map<String, Object> getMetadata() {
         PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), INDEX_TEMPLATE_FIELD);
         PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), ILM_POLICY_FIELD);
         PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> p.map(), METADATA_FIELD);
+        PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), HIDDEN_FIELD);
     }
 
     public static DataStream fromXContent(XContentParser parser) throws IOException {

docs/reference/data-streams/change-mappings-and-settings.asciidoc

@@ -572,7 +572,8 @@ contains information about the stream's oldest backing index,
       ],
       "generation": 2,
       "status": "GREEN",
-      "template": "my-data-stream-template"
+      "template": "my-data-stream-template",
+      "hidden": false
     }
   ]
 }

docs/reference/indices/get-data-stream.asciidoc

@@ -209,7 +209,8 @@ The API returns the following response:
       "generation": 2,
       "status": "GREEN",
       "template": "my-index-template",
-      "ilm_policy": "my-lifecycle-policy"
+      "ilm_policy": "my-lifecycle-policy",
+      "hidden": false
     },
     {
       "name": "my-data-stream_two",
@@ -225,7 +226,8 @@ The API returns the following response:
       "generation": 1,
       "status": "YELLOW",
       "template": "my-index-template",
-      "ilm_policy": "my-lifecycle-policy"
+      "ilm_policy": "my-lifecycle-policy",
+      "hidden": false
     }
   ]
 }

server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java

@@ -225,6 +225,13 @@ public EnumSet<WildcardStates> getExpandWildcards() {
         return EnumSet.copyOf(expandWildcards);
     }
 
+    /**
+     * @return a copy of the {@link Option}s that these indices options will use
+     */
+    public EnumSet<Option> getOptions() {
+        return EnumSet.copyOf(options);
+    }
+
     public void writeIndicesOptions(StreamOutput out) throws IOException {
         out.writeEnumSet(options);
         out.writeEnumSet(expandWildcards);

server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java

@@ -29,7 +29,6 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
-import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -279,20 +278,33 @@ public String toString() {
 
     public static class DataStreamTemplate implements Writeable, ToXContentObject {
 
-        private static final ObjectParser<DataStreamTemplate, Void> PARSER = new ObjectParser<>(
+        private static final ParseField HIDDEN = new ParseField("hidden");
+
+        public static final ConstructingObjectParser<DataStreamTemplate, Void> PARSER = new ConstructingObjectParser<>(
             "data_stream_template",
-            DataStreamTemplate::new
-        );
+            false,
+            a -> new DataStreamTemplate(a[0] != null && (boolean) a[0]));
+
+        static {
+            PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), HIDDEN);
+        }
+
+        private final boolean hidden;
 
         public DataStreamTemplate() {
+            this(false);
         }
 
-        public String getTimestampField() {
-            return FIXED_TIMESTAMP_FIELD;
+        public DataStreamTemplate(boolean hidden) {
+            this.hidden = hidden;
         }
 
-        DataStreamTemplate(StreamInput in) {
-            this();
+        DataStreamTemplate(StreamInput in) throws IOException {
+            hidden = in.getVersion().onOrAfter(DataStream.HIDDEN_VERSION) && in.readBoolean();
+        }
+
+        public String getTimestampField() {
+            return FIXED_TIMESTAMP_FIELD;
         }
 
         /**
@@ -303,26 +315,36 @@ public Map<String, Object> getDataStreamMappingSnippet() {
             return Map.of(MapperService.SINGLE_MAPPING_NAME, Map.of("_data_stream_timestamp", Map.of("enabled", true)));
         }
 
+        public boolean isHidden() {
+            return hidden;
+        }
+
         @Override
         public void writeTo(StreamOutput out) throws IOException {
+            if (out.getVersion().onOrAfter(DataStream.HIDDEN_VERSION)) {
+                out.writeBoolean(hidden);
+            }
         }
 
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.startObject();
+            builder.field("hidden", hidden);
             builder.endObject();
             return builder;
         }
 
         @Override
         public boolean equals(Object o) {
             if (this == o) return true;
-            return o != null && getClass() == o.getClass();
+            if (o == null || getClass() != o.getClass()) return false;
+            DataStreamTemplate that = (DataStreamTemplate) o;
+            return hidden == that.hidden;
         }
 
         @Override
         public int hashCode() {
-            return DataStreamTemplate.class.hashCode();
+            return Objects.hash(hidden);
         }
     }
 }

server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java

@@ -43,19 +43,27 @@
 public final class DataStream extends AbstractDiffable<DataStream> implements ToXContentObject {
 
     public static final String BACKING_INDEX_PREFIX = ".ds-";
+    public static final Version HIDDEN_VERSION = Version.V_8_0_0;
 
     private final String name;
     private final TimestampField timeStampField;
     private final List<Index> indices;
     private final long generation;
     private final Map<String, Object> metadata;
+    private final boolean hidden;
 
     public DataStream(String name, TimestampField timeStampField, List<Index> indices, long generation, Map<String, Object> metadata) {
+        this(name, timeStampField, indices, generation, metadata, false);
+    }
+
+    public DataStream(String name, TimestampField timeStampField, List<Index> indices, long generation, Map<String, Object> metadata,
+                      boolean hidden) {
         this.name = name;
         this.timeStampField = timeStampField;
         this.indices = Collections.unmodifiableList(indices);
         this.generation = generation;
         this.metadata = metadata;
+        this.hidden = hidden;
         assert indices.size() > 0;
     }
 
@@ -88,6 +96,10 @@ public Map<String, Object> getMetadata() {
         return metadata;
     }
 
+    public boolean isHidden() {
+        return hidden;
+    }
+
     /**
      * Performs a rollover on a {@code DataStream} instance and returns a new instance containing
      * the updated list of backing indices and incremented generation.
@@ -100,7 +112,7 @@ public DataStream rollover(Index newWriteIndex) {
         assert newWriteIndex.getName().equals(getDefaultBackingIndexName(name, generation + 1));
         List<Index> backingIndices = new ArrayList<>(indices);
         backingIndices.add(newWriteIndex);
-        return new DataStream(name, timeStampField, backingIndices, generation + 1, metadata);
+        return new DataStream(name, timeStampField, backingIndices, generation + 1, metadata, hidden);
     }
 
     /**
@@ -114,7 +126,7 @@ public DataStream removeBackingIndex(Index index) {
         List<Index> backingIndices = new ArrayList<>(indices);
         backingIndices.remove(index);
         assert backingIndices.size() == indices.size() - 1;
-        return new DataStream(name, timeStampField, backingIndices, generation, metadata);
+        return new DataStream(name, timeStampField, backingIndices, generation, metadata, hidden);
     }
 
     /**
@@ -139,7 +151,7 @@ public DataStream replaceBackingIndex(Index existingBackingIndex, Index newBacki
                 "it is the write index", existingBackingIndex.getName(), name));
         }
         backingIndices.set(backingIndexPosition, newBackingIndex);
-        return new DataStream(name, timeStampField, backingIndices, generation, metadata);
+        return new DataStream(name, timeStampField, backingIndices, generation, metadata, hidden);
     }
 
     /**
@@ -156,7 +168,8 @@ public static String getDefaultBackingIndexName(String dataStreamName, long gene
 
     public DataStream(StreamInput in) throws IOException {
         this(in.readString(), new TimestampField(in), in.readList(Index::new), in.readVLong(),
-            in.getVersion().onOrAfter(Version.V_7_11_0) ? in.readMap(): null);
+            in.getVersion().onOrAfter(Version.V_7_11_0) ? in.readMap(): null,
+            in.getVersion().onOrAfter(HIDDEN_VERSION) && in.readBoolean());
     }
 
     public static Diff<DataStream> readDiffFrom(StreamInput in) throws IOException {
@@ -172,25 +185,30 @@ public void writeTo(StreamOutput out) throws IOException {
         if (out.getVersion().onOrAfter(Version.V_7_11_0)) {
             out.writeMap(metadata);
         }
+        if (out.getVersion().onOrAfter(HIDDEN_VERSION)) {
+            out.writeBoolean(hidden);
+        }
     }
 
     public static final ParseField NAME_FIELD = new ParseField("name");
     public static final ParseField TIMESTAMP_FIELD_FIELD = new ParseField("timestamp_field");
     public static final ParseField INDICES_FIELD = new ParseField("indices");
     public static final ParseField GENERATION_FIELD = new ParseField("generation");
     public static final ParseField METADATA_FIELD = new ParseField("_meta");
+    public static final ParseField HIDDEN_FIELD = new ParseField("hidden");
 
     @SuppressWarnings("unchecked")
     private static final ConstructingObjectParser<DataStream, Void> PARSER = new ConstructingObjectParser<>("data_stream",
         args -> new DataStream((String) args[0], (TimestampField) args[1], (List<Index>) args[2], (Long) args[3],
-            (Map<String, Object>) args[4]));
+            (Map<String, Object>) args[4], args[5] != null && (boolean) args[5]));
 
     static {
         PARSER.declareString(ConstructingObjectParser.constructorArg(), NAME_FIELD);
         PARSER.declareObject(ConstructingObjectParser.constructorArg(), TimestampField.PARSER, TIMESTAMP_FIELD_FIELD);
         PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), (p, c) -> Index.fromXContent(p), INDICES_FIELD);
         PARSER.declareLong(ConstructingObjectParser.constructorArg(), GENERATION_FIELD);
         PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> p.map(), METADATA_FIELD);
+        PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), HIDDEN_FIELD);
     }
 
     public static DataStream fromXContent(XContentParser parser) throws IOException {
@@ -207,6 +225,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (metadata != null) {
             builder.field(METADATA_FIELD.getPreferredName(), metadata);
         }
+        builder.field(HIDDEN_FIELD.getPreferredName(), hidden);
         builder.endObject();
         return builder;
     }

server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java

@@ -358,7 +358,7 @@ public DataStream getParentDataStream() {
 
         @Override
         public boolean isHidden() {
-            return false;
+            return dataStream.isHidden();
         }
 
         @Override

server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java

@@ -131,23 +131,18 @@ public static boolean isIndexVisible(String expression, String index, IndicesOpt
             throw new IllegalStateException("could not resolve index abstraction [" + index + "]");
         }
         final boolean isHidden = indexAbstraction.isHidden();
+        boolean isVisible = isHidden == false || indicesOptions.expandWildcardsHidden() || isVisibleDueToImplicitHidden(expression, index);
         if (indexAbstraction.getType() == IndexAbstraction.Type.ALIAS) {
             //it's an alias, ignore expandWildcardsOpen and expandWildcardsClosed.
             //complicated to support those options with aliases pointing to multiple indices...
-            if (indicesOptions.ignoreAliases()) {
-                return false;
-            } else if (isHidden == false || indicesOptions.expandWildcardsHidden() || isVisibleDueToImplicitHidden(expression, index)) {
-                return true;
-            } else {
-                return false;
-            }
+            return isVisible && indicesOptions.ignoreAliases() == false;
         }
         if (indexAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM) {
-            return includeDataStreams;
+            return isVisible && includeDataStreams;
         }
         assert indexAbstraction.getIndices().size() == 1 : "concrete index must point to a single index";
         IndexMetadata indexMetadata = indexAbstraction.getIndices().get(0);
-        if (isHidden && indicesOptions.expandWildcardsHidden() == false && isVisibleDueToImplicitHidden(expression, index) == false) {
+        if (isVisible == false) {
             return false;
         }
 

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java

@@ -159,8 +159,9 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn
         if (dataStreamName.toLowerCase(Locale.ROOT).equals(dataStreamName) == false) {
             throw new IllegalArgumentException("data_stream [" + dataStreamName + "] must be lowercase");
         }
-        if (dataStreamName.startsWith(".")) {
-            throw new IllegalArgumentException("data_stream [" + dataStreamName + "] must not start with '.'");
+        if (dataStreamName.startsWith(DataStream.BACKING_INDEX_PREFIX)) {
+            throw new IllegalArgumentException("data_stream [" + dataStreamName + "] must not start with '"
+                + DataStream.BACKING_INDEX_PREFIX + "'");
         }
 
         ComposableIndexTemplate template = lookupTemplateForDataStream(dataStreamName, currentState.metadata());
@@ -189,8 +190,9 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn
         DataStream.TimestampField timestampField = new DataStream.TimestampField(fieldName);
         List<Index> dsBackingIndices = backingIndices.stream().map(IndexMetadata::getIndex).collect(Collectors.toList());
         dsBackingIndices.add(writeIndex.getIndex());
+        boolean hidden = template.getDataStreamTemplate().isHidden();
         DataStream newDataStream = new DataStream(dataStreamName, timestampField, dsBackingIndices, 1L,
-                                                  template.metadata() != null ? Map.copyOf(template.metadata()) : null);
+                                                  template.metadata() != null ? Map.copyOf(template.metadata()) : null, hidden);
         Metadata.Builder builder = Metadata.builder(currentState.metadata()).put(newDataStream);
         logger.info("adding data stream [{}] with write index [{}] and backing indices [{}]", dataStreamName,
             writeIndex.getIndex().getName(),

server/src/main/java/org/elasticsearch/snapshots/RestoreService.java

@@ -604,7 +604,7 @@ static DataStream updateDataStream(DataStream dataStream, Metadata.Builder metad
             .map(i -> metadata.get(renameIndex(i.getName(), request, true)).getIndex())
             .collect(Collectors.toList());
         return new DataStream(dataStreamName, dataStream.getTimeStampField(), updatedIndices, dataStream.getGeneration(),
-            dataStream.getMetadata());
+            dataStream.getMetadata(), dataStream.isHidden());
     }
 
     public static RestoreInProgress updateRestoreStateWithDeletedIndices(RestoreInProgress oldRestore, Set<Index> deletedIndices) {