[ML-DataFrame] set meta data on data frame index

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/DataFrameField.java

@@ -29,6 +29,15 @@ public final class DataFrameField {
     // note: this is used to match tasks
     public static final String PERSISTENT_TASK_DESCRIPTION_PREFIX = "data_frame_";
 
+    // strings for meta information
+    public static final String META_FIELDNAME = "_data_frame";
+    public static final String CREATION_DATE_MILLIS = "creation_date_in_millis";
+    public static final String VERSION = "version";
+    public static final String CREATED = "created";
+    public static final String CREATED_BY = "created_by";
+    public static final String TRANSFORM = "transform";
+    public static final String DATA_FRAME_SIGNATURE = "data-frame-transform";
+
     private DataFrameField() {
     }
 }

x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameMetaDataIT.java

@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.dataframe.integration;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.Map;
+
+public class DataFrameMetaDataIT extends DataFrameRestTestCase {
+
+    private boolean indicesCreated = false;
+
+    // preserve indices in order to reuse source indices in several test cases
+    @Override
+    protected boolean preserveIndicesUponCompletion() {
+        return true;
+    }
+
+    @Before
+    public void createIndexes() throws IOException {
+
+        // it's not possible to run it as @BeforeClass as clients aren't initialized then, so we need this little hack
+        if (indicesCreated) {
+            return;
+        }
+
+        createReviewsIndex();
+        indicesCreated = true;
+    }
+
+    public void testMetaData() throws IOException {
+        long testStarted = System.currentTimeMillis();
+        createPivotReviewsTransform("test_meta", "pivot_reviews", null);
+
+        Response mappingResponse = client().performRequest(new Request("GET", "pivot_reviews/_mapping"));
+
+        Map<?, ?> mappingAsMap = entityAsMap(mappingResponse);
+        assertEquals(Version.CURRENT.toString(),
+                XContentMapValues.extractValue("pivot_reviews.mappings._meta._data_frame.version.created", mappingAsMap));
+        assertTrue((Long) XContentMapValues.extractValue("pivot_reviews.mappings._meta._data_frame.creation_date_in_millis",
+                mappingAsMap) < System.currentTimeMillis());
+        assertTrue((Long) XContentMapValues.extractValue("pivot_reviews.mappings._meta._data_frame.creation_date_in_millis",
+                mappingAsMap) > testStarted);
+        assertEquals("test_meta",
+                XContentMapValues.extractValue("pivot_reviews.mappings._meta._data_frame.transform", mappingAsMap));
+        assertEquals("data-frame-transform",
+                XContentMapValues.extractValue("pivot_reviews.mappings._meta.created_by", mappingAsMap));
+    }
+
+}

x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/persistence/DataframeIndex.java

@@ -8,12 +8,14 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.create.CreateIndexAction;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.core.dataframe.DataFrameField;
 import org.elasticsearch.xpack.core.dataframe.DataFrameMessages;
 import org.elasticsearch.xpack.dataframe.transforms.DataFrameTransformConfig;
 
@@ -29,6 +31,7 @@ public final class DataframeIndex {
     public static final String DOC_TYPE = "_doc";
     private static final String PROPERTIES = "properties";
     private static final String TYPE = "type";
+    private static final String META = "_meta";
 
     private DataframeIndex() {
     }
@@ -41,7 +44,7 @@ public static void createDestinationIndex(Client client, DataFrameTransformConfi
         request.settings(Settings.builder() // <1>
                 .put("index.number_of_shards", 1).put("index.number_of_replicas", 0));
 
-        request.mapping(DOC_TYPE, createMappingXContent(mappings));
+        request.mapping(DOC_TYPE, createMappingXContent(mappings, transformConfig.getId()));
 
         client.execute(CreateIndexAction.INSTANCE, request, ActionListener.wrap(createIndexResponse -> {
             listener.onResponse(true);
@@ -53,10 +56,11 @@ public static void createDestinationIndex(Client client, DataFrameTransformConfi
         }));
     }
 
-    private static XContentBuilder createMappingXContent(Map<String, String> mappings) {
+    private static XContentBuilder createMappingXContent(Map<String, String> mappings, String id) {
         try {
             XContentBuilder builder = jsonBuilder().startObject();
             builder.startObject(DOC_TYPE);
+            addMetaData(builder, id);
             builder.startObject(PROPERTIES);
             for (Entry<String, String> field : mappings.entrySet()) {
                 builder.startObject(field.getKey()).field(TYPE, field.getValue()).endObject();
@@ -68,4 +72,18 @@ private static XContentBuilder createMappingXContent(Map<String, String> mapping
             throw new RuntimeException(e);
         }
     }
+
+    private static XContentBuilder addMetaData(XContentBuilder builder, String id) throws IOException {
+        builder.startObject(META);
+        builder.field(DataFrameField.CREATED_BY, DataFrameField.DATA_FRAME_SIGNATURE);
+        builder.startObject(DataFrameField.META_FIELDNAME);
+        builder.field(DataFrameField.CREATION_DATE_MILLIS, System.currentTimeMillis());
+        builder.startObject(DataFrameField.VERSION);
+        builder.field(DataFrameField.CREATED, Version.CURRENT);
+        builder.endObject();
+        builder.field(DataFrameField.TRANSFORM, id);
+        builder.endObject(); // META_FIELDNAME
+        builder.endObject(); // META
+        return builder;
+    }
 }