[7.x][Transform] use ISO dates in output instead of epoch millis (#65584)

client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/SettingsConfig.java

@@ -21,6 +21,7 @@
 
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -34,30 +35,43 @@ public class SettingsConfig implements ToXContentObject {
 
     private static final ParseField MAX_PAGE_SEARCH_SIZE = new ParseField("max_page_search_size");
     private static final ParseField DOCS_PER_SECOND = new ParseField("docs_per_second");
+    private static final ParseField DATES_AS_EPOCH_MILLIS = new ParseField("dates_as_epoch_millis");
     private static final int DEFAULT_MAX_PAGE_SEARCH_SIZE = -1;
     private static final float DEFAULT_DOCS_PER_SECOND = -1F;
 
+    // use an integer as we need to code 4 states: true, false, null (unchanged), default (defined server side)
+    private static final int DEFAULT_DATES_AS_EPOCH_MILLIS = -1;
+
     private final Integer maxPageSearchSize;
     private final Float docsPerSecond;
+    private final Integer datesAsEpochMillis;
 
     private static final ConstructingObjectParser<SettingsConfig, Void> PARSER = new ConstructingObjectParser<>(
         "settings_config",
         true,
-        args -> new SettingsConfig((Integer) args[0], (Float) args[1])
+        args -> new SettingsConfig((Integer) args[0], (Float) args[1], (Integer) args[2])
     );
 
     static {
         PARSER.declareIntOrNull(optionalConstructorArg(), DEFAULT_MAX_PAGE_SEARCH_SIZE, MAX_PAGE_SEARCH_SIZE);
         PARSER.declareFloatOrNull(optionalConstructorArg(), DEFAULT_DOCS_PER_SECOND, DOCS_PER_SECOND);
+        // this boolean requires 4 possible values: true, false, not_specified, default, therefore using a custom parser
+        PARSER.declareField(
+            optionalConstructorArg(),
+            p -> p.currentToken() == XContentParser.Token.VALUE_NULL ? DEFAULT_DATES_AS_EPOCH_MILLIS : p.booleanValue() ? 1 : 0,
+            DATES_AS_EPOCH_MILLIS,
+            ValueType.BOOLEAN_OR_NULL
+        );
     }
 
     public static SettingsConfig fromXContent(final XContentParser parser) {
         return PARSER.apply(parser, null);
     }
 
-    SettingsConfig(Integer maxPageSearchSize, Float docsPerSecond) {
+    SettingsConfig(Integer maxPageSearchSize, Float docsPerSecond, Integer datesAsEpochMillis) {
         this.maxPageSearchSize = maxPageSearchSize;
         this.docsPerSecond = docsPerSecond;
+        this.datesAsEpochMillis = datesAsEpochMillis;
     }
 
     @Override
@@ -77,6 +91,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                 builder.field(DOCS_PER_SECOND.getPreferredName(), docsPerSecond);
             }
         }
+        if (datesAsEpochMillis != null) {
+            if (datesAsEpochMillis.equals(DEFAULT_DATES_AS_EPOCH_MILLIS)) {
+                builder.field(DATES_AS_EPOCH_MILLIS.getPreferredName(), (Boolean) null);
+            } else {
+                builder.field(DATES_AS_EPOCH_MILLIS.getPreferredName(), datesAsEpochMillis > 0 ? true : false);
+            }
+        }
         builder.endObject();
         return builder;
     }
@@ -89,6 +110,10 @@ public Float getDocsPerSecond() {
         return docsPerSecond;
     }
 
+    public Boolean getDatesAsEpochMillis() {
+        return datesAsEpochMillis != null ? datesAsEpochMillis > 0 : null;
+    }
+
     @Override
     public boolean equals(Object other) {
         if (other == this) {
@@ -99,12 +124,14 @@ public boolean equals(Object other) {
         }
 
         SettingsConfig that = (SettingsConfig) other;
-        return Objects.equals(maxPageSearchSize, that.maxPageSearchSize) && Objects.equals(docsPerSecond, that.docsPerSecond);
+        return Objects.equals(maxPageSearchSize, that.maxPageSearchSize)
+            && Objects.equals(docsPerSecond, that.docsPerSecond)
+            && Objects.equals(datesAsEpochMillis, that.datesAsEpochMillis);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(maxPageSearchSize, docsPerSecond);
+        return Objects.hash(maxPageSearchSize, docsPerSecond, datesAsEpochMillis);
     }
 
     public static Builder builder() {
@@ -114,6 +141,7 @@ public static Builder builder() {
     public static class Builder {
         private Integer maxPageSearchSize;
         private Float docsPerSecond;
+        private Integer datesAsEpochMillis;
 
         /**
          * Sets the paging maximum paging maxPageSearchSize that transform can use when
@@ -143,8 +171,24 @@ public Builder setRequestsPerSecond(Float docsPerSecond) {
             return this;
         }
 
+        /**
+         * Whether to write the output of a date aggregation as millis since epoch or as formatted string (ISO format).
+         *
+         * Transforms created before 7.11 write dates as epoch_millis. The new default is ISO string.
+         * You can use this setter to configure the old style writing as epoch millis.
+         *
+         * An explicit `null` resets to default.
+         *
+         * @param datesAsEpochMillis true if dates should be written as epoch_millis.
+         * @return the {@link Builder} with datesAsEpochMilli set.
+         */
+        public Builder setDatesAsEpochMillis(Boolean datesAsEpochMillis) {
+            this.datesAsEpochMillis = datesAsEpochMillis == null ? DEFAULT_DATES_AS_EPOCH_MILLIS : datesAsEpochMillis ? 1 : 0;
+            return this;
+        }
+
         public SettingsConfig build() {
-            return new SettingsConfig(maxPageSearchSize, docsPerSecond);
+            return new SettingsConfig(maxPageSearchSize, docsPerSecond, datesAsEpochMillis);
         }
     }
 }

client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/SettingsConfigTests.java

@@ -38,7 +38,11 @@
 public class SettingsConfigTests extends AbstractXContentTestCase<SettingsConfig> {
 
     public static SettingsConfig randomSettingsConfig() {
-        return new SettingsConfig(randomBoolean() ? null : randomIntBetween(10, 10_000), randomBoolean() ? null : randomFloat());
+        return new SettingsConfig(
+            randomBoolean() ? null : randomIntBetween(10, 10_000),
+            randomBoolean() ? null : randomFloat(),
+            randomBoolean() ? null : randomIntBetween(-1, 1)
+        );
     }
 
     @Override
@@ -67,6 +71,7 @@ public void testExplicitNullOnWriteParser() throws IOException {
 
         SettingsConfig emptyConfig = fromString("{}");
         assertNull(emptyConfig.getMaxPageSearchSize());
+        assertNull(emptyConfig.getDatesAsEpochMillis());
 
         settingsAsMap = xContentToMap(emptyConfig);
         assertTrue(settingsAsMap.isEmpty());
@@ -77,6 +82,15 @@ public void testExplicitNullOnWriteParser() throws IOException {
         settingsAsMap = xContentToMap(config);
         assertThat(settingsAsMap.getOrDefault("max_page_search_size", "not_set"), equalTo("not_set"));
         assertNull(settingsAsMap.getOrDefault("docs_per_second", "not_set"));
+        assertThat(settingsAsMap.getOrDefault("dates_as_epoch_millis", "not_set"), equalTo("not_set"));
+
+        config = fromString("{\"dates_as_epoch_millis\" : null}");
+        assertFalse(config.getDatesAsEpochMillis());
+
+        settingsAsMap = xContentToMap(config);
+        assertThat(settingsAsMap.getOrDefault("max_page_search_size", "not_set"), equalTo("not_set"));
+        assertThat(settingsAsMap.getOrDefault("docs_per_second", "not_set"), equalTo("not_set"));
+        assertNull(settingsAsMap.getOrDefault("dates_as_epoch_millis", "not_set"));
     }
 
     public void testExplicitNullOnWriteBuilder() throws IOException {
@@ -87,9 +101,11 @@ public void testExplicitNullOnWriteBuilder() throws IOException {
         Map<String, Object> settingsAsMap = xContentToMap(config);
         assertNull(settingsAsMap.getOrDefault("max_page_search_size", "not_set"));
         assertThat(settingsAsMap.getOrDefault("docs_per_second", "not_set"), equalTo("not_set"));
+        assertThat(settingsAsMap.getOrDefault("dates_as_epoch_millis", "not_set"), equalTo("not_set"));
 
         SettingsConfig emptyConfig = new SettingsConfig.Builder().build();
         assertNull(emptyConfig.getMaxPageSearchSize());
+        assertNull(emptyConfig.getDatesAsEpochMillis());
 
         settingsAsMap = xContentToMap(emptyConfig);
         assertTrue(settingsAsMap.isEmpty());
@@ -100,6 +116,16 @@ public void testExplicitNullOnWriteBuilder() throws IOException {
         settingsAsMap = xContentToMap(config);
         assertThat(settingsAsMap.getOrDefault("max_page_search_size", "not_set"), equalTo("not_set"));
         assertNull(settingsAsMap.getOrDefault("docs_per_second", "not_set"));
+        assertThat(settingsAsMap.getOrDefault("dates_as_epoch_millis", "not_set"), equalTo("not_set"));
+
+        config = new SettingsConfig.Builder().setDatesAsEpochMillis(null).build();
+        // returns false, however it's `null` as in "use default", checked next
+        assertFalse(config.getDatesAsEpochMillis());
+
+        settingsAsMap = xContentToMap(config);
+        assertThat(settingsAsMap.getOrDefault("max_page_search_size", "not_set"), equalTo("not_set"));
+        assertThat(settingsAsMap.getOrDefault("docs_per_second", "not_set"), equalTo("not_set"));
+        assertNull(settingsAsMap.getOrDefault("dates_as_epoch_millis", "not_set"));
     }
 
     private Map<String, Object> xContentToMap(ToXContent xcontent) throws IOException {

client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/SettingsConfigTests.java

@@ -33,7 +33,8 @@ public class SettingsConfigTests extends AbstractResponseTestCase<
     public static org.elasticsearch.xpack.core.transform.transforms.SettingsConfig randomSettingsConfig() {
         return new org.elasticsearch.xpack.core.transform.transforms.SettingsConfig(
             randomBoolean() ? null : randomIntBetween(10, 10_000),
-            randomBoolean() ? null : randomFloat()
+            randomBoolean() ? null : randomFloat(),
+            randomBoolean() ? null : randomIntBetween(0, 1)
         );
     }
 
@@ -43,6 +44,7 @@ public static void assertHlrcEquals(
     ) {
         assertEquals(serverTestInstance.getMaxPageSearchSize(), clientInstance.getMaxPageSearchSize());
         assertEquals(serverTestInstance.getDocsPerSecond(), clientInstance.getDocsPerSecond());
+        assertEquals(serverTestInstance.getDatesAsEpochMillis(), clientInstance.getDatesAsEpochMillis());
     }
 
     @Override

docs/reference/migration/migrate_7_11.asciidoc

@@ -32,3 +32,13 @@ will be applied to the values of a keyword field; for example, a
 field configured with a lowercase normalizer will return highlighted
 snippets in lower case.
 ====
+[discrete]
+[[breaking_711_api_changes]]
+=== API changes
+
+[discrete]
+==== {dataframe-transform-cap} API changes
+
+Transform no longer writes dates used in a `group_by` as `epoch_millis` but as
+formatted ISO string. Previously constructed transforms will still use `epoch_millis`.
+You can configure and change the output format in the settings of the transform.

docs/reference/rest-api/common-parms.asciidoc

@@ -158,10 +158,10 @@ The destination for the {transform}.
 end::dest[]
 
 tag::dest-index[]
-The _destination index_ for the {transform}. The mappings of the destination 
-index are deduced based on the source fields when possible. If alternate 
-mappings are required, use the 
-https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html[Create index API] 
+The _destination index_ for the {transform}. The mappings of the destination
+index are deduced based on the source fields when possible. If alternate
+mappings are required, use the
+https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html[Create index API]
 prior to starting the {transform}.
 end::dest-index[]
 
@@ -971,6 +971,13 @@ tag::transform-settings[]
 Defines optional {transform} settings.
 end::transform-settings[]
 
+tag::transform-settings-dates-as-epoch-milli[]
+Defines if dates in the ouput should be written as ISO formatted string (default)
+or as millis since epoch. `epoch_millis` has been the default for transforms created
+before version `7.11`. For compatible output set this to `true`.
+The default value is `false`.
+end::transform-settings-dates-as-epoch-milli[]
+
 tag::transform-settings-docs-per-second[]
 Specifies a limit on the number of input documents per second. This setting
 throttles the {transform} by adding a wait time between search requests. The

docs/reference/transform/apis/put-transform.asciidoc

@@ -18,7 +18,7 @@ Instantiates a {transform}.
 [[put-transform-prereqs]]
 == {api-prereq-title}
 
-If the {es} {security-features} are enabled, you must have the following 
+If the {es} {security-features} are enabled, you must have the following
 built-in roles and privileges:
 
 * `transform_admin`
@@ -134,6 +134,9 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings]
 .Properties of `settings`
 [%collapsible%open]
 ====
+`dates_as_epoch_millis`:::
+(Optional, boolean)
+include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-dates-as-epoch-milli]
 `docs_per_second`:::
 (Optional, float)
 include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-docs-per-second]
@@ -183,7 +186,7 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=sync-time]
 `delay`::::
 (Optional, <<time-units, time units>>)
 include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=sync-time-delay]
-    
+
 `field`::::
 (Required, string)
 include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=sync-time-field]

docs/reference/transform/apis/update-transform.asciidoc

@@ -18,7 +18,7 @@ Updates certain properties of a {transform}.
 [[update-transform-prereqs]]
 == {api-prereq-title}
 
-If the {es} {security-features} are enabled, you must have the following 
+If the {es} {security-features} are enabled, you must have the following
 built-in roles and privileges:
 
 * `transform_admin`
@@ -110,6 +110,9 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings]
 .Properties of `settings`
 [%collapsible%open]
 ====
+`dates_as_epoch_millis`:::
+(Optional, boolean)
+include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-dates-as-epoch-milli]
 `docs_per_second`:::
 (Optional, float)
 include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-docs-per-second]
@@ -131,7 +134,7 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=source-transforms]
 `index`:::
 (Required, string or array)
 include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=source-index-transforms]
-    
+
 `query`:::
 (Optional, object)
 include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=source-query-transforms]

libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java

@@ -632,6 +632,7 @@ public enum ValueType {
         INT(VALUE_NUMBER, VALUE_STRING),
         INT_OR_NULL(VALUE_NUMBER, VALUE_STRING, VALUE_NULL),
         BOOLEAN(VALUE_BOOLEAN, VALUE_STRING),
+        BOOLEAN_OR_NULL(VALUE_BOOLEAN, VALUE_STRING, VALUE_NULL),
         STRING_ARRAY(START_ARRAY, VALUE_STRING),
         FLOAT_ARRAY(START_ARRAY, VALUE_NUMBER, VALUE_STRING),
         DOUBLE_ARRAY(START_ARRAY, VALUE_NUMBER, VALUE_STRING),

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java

@@ -35,6 +35,7 @@ public final class TransformField {
     public static final ParseField FORCE = new ParseField("force");
     public static final ParseField MAX_PAGE_SEARCH_SIZE = new ParseField("max_page_search_size");
     public static final ParseField DOCS_PER_SECOND = new ParseField("docs_per_second");
+    public static final ParseField DATES_AS_EPOCH_MILLIS = new ParseField("dates_as_epoch_millis");
     public static final ParseField FIELD = new ParseField("field");
     public static final ParseField SYNC = new ParseField("sync");
     public static final ParseField TIME_BASED_SYNC = new ParseField("time");