[8.10] Set default index mode for TimeSeries to null (#98808)

docs/changelog/98808.yaml

@@ -0,0 +1,6 @@
+pr: 98808
+summary: Set default index mode for `TimeSeries` to `null`
+area: Aggregations
+type: enhancement
+issues:
+ - 97429

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml

@@ -415,6 +415,37 @@ nested fields:
                             type: long
                             time_series_metric: gauge
 
+---
+"Synthetic source":
+  - skip:
+      version: " - 8.9.99"
+      reason: "Synthetic source shows up in the mapping in 8.10 and on"
+
+  - do:
+      indices.create:
+        index: tsdb-synthetic
+        body:
+          settings:
+            number_of_replicas: 0
+            mode: time_series
+            routing_path: [field1]
+            time_series:
+              start_time: 2000-01-01T00:00:00Z
+              end_time: 2099-12-31T23:59:59Z
+          mappings:
+            properties:
+              field1:
+                type: keyword
+                time_series_dimension: true
+              field2:
+                type: long
+                time_series_metric: gauge
+
+  - do:
+      indices.get_mapping: {}
+
+  - match: {tsdb-synthetic.mappings._source.mode: synthetic}
+
 ---
 regular source:
   - skip:

server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java

@@ -10,6 +10,7 @@
 
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.IndexVersion;
 
 import java.util.List;
 
@@ -30,18 +31,30 @@ public static DocumentMapper createEmpty(MapperService mapperService) {
         );
         MetadataFieldMapper[] metadata = mapperService.getMetadataMappers().values().toArray(new MetadataFieldMapper[0]);
         Mapping mapping = new Mapping(root, metadata, null);
-        return new DocumentMapper(mapperService.documentParser(), mapping, mapping.toCompressedXContent());
+        return new DocumentMapper(mapperService.documentParser(), mapping, mapping.toCompressedXContent(), IndexVersion.current());
     }
 
-    DocumentMapper(DocumentParser documentParser, Mapping mapping, CompressedXContent source) {
+    DocumentMapper(DocumentParser documentParser, Mapping mapping, CompressedXContent source, IndexVersion version) {
         this.documentParser = documentParser;
         this.type = mapping.getRoot().name();
         this.mappingLookup = MappingLookup.fromMapping(mapping);
         this.mappingSource = source;
-        assert mapping.toCompressedXContent().equals(source)
+
+        assert mapping.toCompressedXContent().equals(source) || isSyntheticSourceMalformed(source, version)
             : "provided source [" + source + "] differs from mapping [" + mapping.toCompressedXContent() + "]";
     }
 
+    /**
+     * Indexes built at v.8.7 were missing an explicit entry for synthetic_source.
+     * This got restored in v.8.10 to avoid confusion. The change is only restricted to mapping printout, it has no
+     * functional effect as the synthetic source already applies.
+     */
+    boolean isSyntheticSourceMalformed(CompressedXContent source, IndexVersion version) {
+        return sourceMapper().isSynthetic()
+            && source.string().contains("\"_source\":{\"mode\":\"synthetic\"}") == false
+            && version.onOrBefore(IndexVersion.V_8_10_0);
+    }
+
     public Mapping mapping() {
         return mappingLookup.getMapping();
     }

server/src/main/java/org/elasticsearch/index/mapper/MapperService.java

@@ -348,13 +348,15 @@ boolean assertNoUpdateRequired(final IndexMetadata newIndexMetadata) {
             Mapping newMapping = parseMapping(mapping.type(), mapping.source());
             final CompressedXContent currentSource = this.mapper.mappingSource();
             final CompressedXContent newSource = newMapping.toCompressedXContent();
-            if (Objects.equals(currentSource, newSource) == false) {
+            if (Objects.equals(currentSource, newSource) == false
+                && mapper.isSyntheticSourceMalformed(currentSource, indexVersionCreated) == false) {
                 throw new IllegalStateException(
                     "expected current mapping [" + currentSource + "] to be the same as new mapping [" + newSource + "]"
                 );
             }
         }
         return true;
+
     }
 
     public void merge(IndexMetadata indexMetadata, MergeReason reason) {
@@ -386,7 +388,7 @@ public DocumentMapper merge(String type, CompressedXContent mappingSource, Merge
     }
 
     private DocumentMapper newDocumentMapper(Mapping mapping, MergeReason reason, CompressedXContent mappingSource) {
-        DocumentMapper newMapper = new DocumentMapper(documentParser, mapping, mappingSource);
+        DocumentMapper newMapper = new DocumentMapper(documentParser, mapping, mappingSource, indexVersionCreated);
         newMapper.validate(indexSettings, reason != MergeReason.MAPPING_RECOVERY);
         return newMapper;
     }

server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java

@@ -20,6 +20,7 @@
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.IndexMode;
+import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.query.QueryShardException;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.search.lookup.Source;
@@ -61,6 +62,18 @@ private enum Mode {
         IndexMode.TIME_SERIES
     );
 
+    /*
+     * Synthetic source was added as the default for TSDB in v.8.7. The legacy field mapper below
+     * is used in bwc tests and mixed clusters containing time series indexes created in an earlier version.
+     */
+    private static final SourceFieldMapper TSDB_LEGACY_DEFAULT = new SourceFieldMapper(
+        null,
+        Explicit.IMPLICIT_TRUE,
+        Strings.EMPTY_ARRAY,
+        Strings.EMPTY_ARRAY,
+        IndexMode.TIME_SERIES
+    );
+
     public static class Defaults {
         public static final String NAME = SourceFieldMapper.NAME;
 
@@ -86,10 +99,15 @@ public static class Builder extends MetadataFieldMapper.Builder {
             .setMergeValidator(
                 (previous, current, conflicts) -> (previous.value() == current.value()) || (previous.value() && current.value() == false)
             );
+
+        /*
+         * The default mode for TimeSeries is left empty on purpose, so that mapping printings include the synthetic
+         * source mode.
+         */
         private final Parameter<Mode> mode = new Parameter<>(
             "mode",
             true,
-            () -> getIndexMode() == IndexMode.TIME_SERIES ? Mode.SYNTHETIC : null,
+            () -> null,
             (n, c, o) -> Mode.valueOf(o.toString().toUpperCase(Locale.ROOT)),
             m -> toType(m).enabled.explicit() ? null : toType(m).mode,
             (b, n, v) -> b.field(n, v.toString().toLowerCase(Locale.ROOT)),
@@ -136,10 +154,11 @@ private boolean isDefault() {
 
         @Override
         public SourceFieldMapper build() {
-            if (enabled.getValue().explicit() && mode.get() != null) {
+            if (enabled.getValue().explicit()) {
                 if (indexMode == IndexMode.TIME_SERIES) {
                     throw new MapperParsingException("Time series indices only support synthetic source");
-                } else {
+                }
+                if (mode.get() != null) {
                     throw new MapperParsingException("Cannot set both [mode] and [enabled] parameters");
                 }
             }
@@ -165,7 +184,9 @@ private IndexMode getIndexMode() {
     }
 
     public static final TypeParser PARSER = new ConfigurableTypeParser(
-        c -> c.getIndexSettings().getMode() == IndexMode.TIME_SERIES ? TSDB_DEFAULT : DEFAULT,
+        c -> c.getIndexSettings().getMode() == IndexMode.TIME_SERIES
+            ? c.getIndexSettings().getIndexVersionCreated().onOrAfter(IndexVersion.V_8_7_0) ? TSDB_DEFAULT : TSDB_LEGACY_DEFAULT
+            : DEFAULT,
         c -> new Builder(c.getIndexSettings().getMode())
     );
 
@@ -217,7 +238,7 @@ private SourceFieldMapper(Mode mode, Explicit<Boolean> enabled, String[] include
         this.sourceFilter = buildSourceFilter(includes, excludes);
         this.includes = includes;
         this.excludes = excludes;
-        if (this.sourceFilter != null && mode == Mode.SYNTHETIC) {
+        if (this.sourceFilter != null && (mode == Mode.SYNTHETIC || indexMode == IndexMode.TIME_SERIES)) {
             throw new IllegalArgumentException("filtering the stored _source is incompatible with synthetic source");
         }
         this.complete = stored() && sourceFilter == null;

server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java

@@ -69,7 +69,7 @@ public void testAddFields() throws Exception {
         assertThat(stage1.mappers().getMapper("obj1.prop1"), nullValue());
         // but merged should
         DocumentParser documentParser = new DocumentParser(null, null, () -> DocumentParsingObserver.EMPTY_INSTANCE);
-        DocumentMapper mergedMapper = new DocumentMapper(documentParser, merged, merged.toCompressedXContent());
+        DocumentMapper mergedMapper = new DocumentMapper(documentParser, merged, merged.toCompressedXContent(), IndexVersion.current());
         assertThat(mergedMapper.mappers().getMapper("age"), notNullValue());
         assertThat(mergedMapper.mappers().getMapper("obj1.prop1"), notNullValue());
     }

server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java

@@ -20,6 +20,7 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.CompositeFieldScript;
@@ -2530,7 +2531,12 @@ same name need to be part  of the same mappings (hence the same document). If th
 
         // merge without going through toXContent and reparsing, otherwise the potential leaf path issue gets fixed on its own
         Mapping newMapping = MapperService.mergeMappings(mapperService.documentMapper(), mapping, MapperService.MergeReason.MAPPING_UPDATE);
-        DocumentMapper newDocMapper = new DocumentMapper(mapperService.documentParser(), newMapping, newMapping.toCompressedXContent());
+        DocumentMapper newDocMapper = new DocumentMapper(
+            mapperService.documentParser(),
+            newMapping,
+            newMapping.toCompressedXContent(),
+            IndexVersion.current()
+        );
         ParsedDocument doc2 = newDocMapper.parse(source("""
             {
               "foo" : {

server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java

@@ -12,6 +12,7 @@
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentType;
@@ -227,4 +228,14 @@ public void testSyntheticUpdates() throws Exception {
         assertFalse(mapper.enabled());
         assertFalse(mapper.isSynthetic());
     }
+
+    public void testSyntheticSourceInTimeSeries() throws IOException {
+        XContentBuilder mapping = fieldMapping(b -> {
+            b.field("type", "keyword");
+            b.field("time_series_dimension", true);
+        });
+        DocumentMapper mapper = createTimeSeriesModeDocumentMapper(mapping);
+        assertTrue(mapper.sourceMapper().isSynthetic());
+        assertEquals("{\"_source\":{\"mode\":\"synthetic\"}}", mapper.sourceMapper().toString());
+    }
 }