Better out-of-the-box mappings for logs, metrics and synthetics

x-pack/plugin/core/src/main/resources/data-streams-mappings.json

@@ -0,0 +1,67 @@
+{
+  "template": {
+    "mappings": {
+      "dynamic_templates": [
+        {
+          "match_ip": {
+            "match_mapping_type": "string",
+            "match": "ip",
+            "mapping": {
+              "type": "ip"
+            }
+          }
+        },
+        {
+          "match_message": {
+            "match_mapping_type": "string",
+            "match": "message",
+            "mapping": {
+              "type": "match_only_text"
+            }
+          }
+        },
+        {
+          "strings_as_keyword": {
+            "mapping": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "match_mapping_type": "string"
+          }
+        }
+      ],
+      "date_detection": false,
+      "properties": {
+        "@timestamp": {
+          "type": "date"
+        },
+        "data_stream": {
+          "properties": {
+            "dataset": {
+              "type": "constant_keyword"
+            },
+            "namespace": {
+              "type": "constant_keyword"
+            }
+          }
+        },
+        "ecs": {
+          "properties": {
+            "version": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            }
+          }
+        },
+        "host": {
+          "type": "object"
+        }
+      }
+    }
+  },
+  "_meta": {
+    "description": "general mapping conventions for data streams",
+    "managed": true
+  },
+  "version": ${xpack.stack.template.version}
+}

x-pack/plugin/core/src/main/resources/logs-mappings.json

@@ -1,53 +1,14 @@
 {
   "template": {
     "mappings": {
-      "dynamic_templates": [
-        {
-          "strings_as_keyword": {
-            "mapping": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "match_mapping_type": "string"
-          }
-        }
-      ],
-      "date_detection": false,
       "properties": {
-        "@timestamp": {
-          "type": "date"
-        },
         "data_stream": {
           "properties": {
             "type": {
               "type": "constant_keyword",
               "value": "logs"
-            },
-            "dataset": {
-              "type": "constant_keyword"
-            },
-            "namespace": {
-              "type": "constant_keyword"
-            }
-          }
-        },
-        "ecs": {
-          "properties": {
-            "version": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            }
-          }
-        },
-        "host": {
-          "properties": {
-            "ip": {
-              "type": "ip"
             }
           }
-        },
-        "message": {
-          "type": "text"
         }
       }
     }

x-pack/plugin/core/src/main/resources/logs-template.json

@@ -4,6 +4,7 @@
   "data_stream": {},
   "composed_of": [
     "logs-mappings",
+    "data-streams-mappings",
     "logs-settings"
   ],
   "allow_auto_create": true,

x-pack/plugin/core/src/main/resources/metrics-mappings.json

@@ -1,48 +1,12 @@
 {
   "template": {
     "mappings": {
-      "dynamic_templates": [
-        {
-          "strings_as_keyword": {
-            "mapping": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "match_mapping_type": "string"
-          }
-        }
-      ],
-      "date_detection": false,
       "properties": {
-        "@timestamp": {
-          "type": "date"
-        },
         "data_stream": {
           "properties": {
             "type": {
               "type": "constant_keyword",
               "value": "metrics"
-            },
-            "dataset": {
-              "type": "constant_keyword"
-            },
-            "namespace": {
-              "type": "constant_keyword"
-            }
-          }
-        },
-        "ecs": {
-          "properties": {
-            "version": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            }
-          }
-        },
-        "host": {
-          "properties": {
-            "ip": {
-              "type": "ip"
             }
           }
         }

x-pack/plugin/core/src/main/resources/metrics-template.json

@@ -4,6 +4,7 @@
   "data_stream": {},
   "composed_of": [
     "metrics-mappings",
+    "data-streams-mappings",
     "metrics-settings"
   ],
   "allow_auto_create": true,

x-pack/plugin/core/src/main/resources/synthetics-mappings.json

@@ -1,55 +1,12 @@
 {
   "template": {
     "mappings": {
-      "dynamic_templates": [
-        {
-          "strings_as_keyword": {
-            "mapping": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "match_mapping_type": "string"
-          }
-        }
-      ],
-      "date_detection": false,
       "properties": {
-        "@timestamp": {
-          "type": "date"
-        },
         "data_stream": {
           "properties": {
             "type": {
               "type": "constant_keyword",
               "value": "synthetics"
-            },
-            "dataset": {
-              "type": "constant_keyword"
-            },
-            "namespace": {
-              "type": "constant_keyword"
-            }
-          }
-        },
-        "ecs": {
-          "properties": {
-            "version": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            }
-          }
-        },
-        "host": {
-          "properties": {
-            "ip": {
-              "type": "ip"
-            }
-          }
-        },
-        "observer": {
-          "properties": {
-            "ip": {
-              "type": "ip"
             }
           }
         }

x-pack/plugin/core/src/main/resources/synthetics-template.json

@@ -4,6 +4,7 @@
   "data_stream": {},
   "composed_of": [
     "synthetics-mappings",
+    "data-streams-mappings",
     "synthetics-settings"
   ],
   "allow_auto_create": true,

x-pack/plugin/stack/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/stack/10_basic.yml

@@ -14,6 +14,10 @@ setup:
       ilm.get_lifecycle:
         policy: "metrics"
 
+  - do:
+      cluster.get_component_template:
+        name: data-streams-mappings
+
   - do:
       cluster.get_component_template:
         name: logs-mappings
@@ -46,6 +50,11 @@ setup:
         body:
           "@timestamp": "2020-01-01"
           message: "test-log-message"
+          source.ip: "10.1.2.3"
+          log.file.path: "/var/log/web/access.log"
+          data_stream.type: "logs"
+          data_stream.dataset: "foo"
+          data_stream.namespace: "bar"
 
   - do:
       indices.get_data_stream:
@@ -66,7 +75,16 @@ setup:
   - is_true: .$idx0name.settings
   - is_true: .$idx0name.mappings
   - match: { .$idx0name.settings.index.lifecycle.name: "logs" }
+  - match: { .$idx0name.mappings.properties.data_stream.properties.type.type: "constant_keyword" }
+  - match: { .$idx0name.mappings.properties.data_stream.properties.type.value: "logs" }
+  - match: { .$idx0name.mappings.properties.data_stream.properties.dataset.type: "constant_keyword" }
+  - match: { .$idx0name.mappings.properties.data_stream.properties.dataset.value: "foo" }
+  - match: { .$idx0name.mappings.properties.data_stream.properties.namespace.type: "constant_keyword" }
+  - match: { .$idx0name.mappings.properties.data_stream.properties.namespace.value: "bar" }
   - is_true: .$idx0name.mappings.properties.message
+  - match: { .$idx0name.mappings.properties.message.type: "match_only_text" }
+  - match: { .$idx0name.mappings.properties.source.properties.ip.type: "ip" }
+  - match: { .$idx0name.mappings.properties.log.properties.file.properties.path.type: "keyword" }
   - match: { .$idx0name.data_stream: "logs-foo-bar" }
 
   - do:
@@ -80,7 +98,10 @@ setup:
         index: metrics-foo-bar
         body:
           "@timestamp": "2020-01-01"
-          message: "test-log-message"
+          source.ip: "10.1.2.3"
+          data_stream.type: "metrics"
+          data_stream.dataset: "foo"
+          data_stream.namespace: "bar"
 
   - do:
       indices.get_data_stream:
@@ -101,9 +122,92 @@ setup:
   - is_true: .$idx0name.settings
   - is_true: .$idx0name.mappings
   - match: { .$idx0name.settings.index.lifecycle.name: "metrics" }
-  - is_true: .$idx0name.mappings.properties.message
+  - match: { .$idx0name.mappings.properties.data_stream.properties.type.type: "constant_keyword" }
+  - match: { .$idx0name.mappings.properties.data_stream.properties.type.value: "metrics" }
+  - match: { .$idx0name.mappings.properties.data_stream.properties.dataset.type: "constant_keyword" }
+  - match: { .$idx0name.mappings.properties.data_stream.properties.dataset.value: "foo" }
+  - match: { .$idx0name.mappings.properties.data_stream.properties.namespace.type: "constant_keyword" }
+  - match: { .$idx0name.mappings.properties.data_stream.properties.namespace.value: "bar" }
+  - match: { .$idx0name.mappings.properties.source.properties.ip.type: "ip" }
   - match: { .$idx0name.data_stream: "metrics-foo-bar" }
 
   - do:
       indices.delete_data_stream:
         name: metrics-foo-bar
+
+---
+"Test synthetics index auto creation":
+  - do:
+      index:
+        index: synthetics-foo-bar
+        body:
+          "@timestamp": "2020-01-01"
+          source.ip: "10.1.2.3"
+          data_stream.type: "synthetics"
+          data_stream.dataset: "foo"
+          data_stream.namespace: "bar"
+
+  - do:
+      indices.get_data_stream:
+        name: synthetics-foo-bar
+
+  - match: { data_streams.0.name: synthetics-foo-bar }
+  - match: { data_streams.0.timestamp_field.name: '@timestamp' }
+  - match: { data_streams.0.generation: 1 }
+  - length: { data_streams.0.indices: 1 }
+  - match: { data_streams.0.indices.0.index_name: '/\.ds-synthetics-foo-bar-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
+
+  - set: { data_streams.0.indices.0.index_name: idx0name }
+
+  - do:
+      indices.get:
+        index: $idx0name
+
+  - is_true: .$idx0name.settings
+  - is_true: .$idx0name.mappings
+  - match: { .$idx0name.settings.index.lifecycle.name: "synthetics" }
+  - match: { .$idx0name.mappings.properties.data_stream.properties.type.type: "constant_keyword" }
+  - match: { .$idx0name.mappings.properties.data_stream.properties.type.value: "synthetics" }
+  - match: { .$idx0name.mappings.properties.data_stream.properties.dataset.type: "constant_keyword" }
+  - match: { .$idx0name.mappings.properties.data_stream.properties.dataset.value: "foo" }
+  - match: { .$idx0name.mappings.properties.data_stream.properties.namespace.type: "constant_keyword" }
+  - match: { .$idx0name.mappings.properties.data_stream.properties.namespace.value: "bar" }
+  - match: { .$idx0name.mappings.properties.source.properties.ip.type: "ip" }
+  - match: { .$idx0name.data_stream: "synthetics-foo-bar" }
+
+  - do:
+      indices.delete_data_stream:
+        name: synthetics-foo-bar
+
+---
+"Test wrong data_stream type":
+
+ - do:
+      catch: bad_request
+      index:
+        index: synthetics-dataset0-namespace1
+        body:
+          "@timestamp": "2020-01-01"
+          data_stream.type: "logs"
+          data_stream.dataset: "dataset0"
+          data_stream.namespace: "namespace1"
+
+ - do:
+      catch: bad_request
+      index:
+        index: logs-dataset0-namespace1
+        body:
+          "@timestamp": "2020-01-01"
+          data_stream.type: "metrics"
+          data_stream.dataset: "dataset0"
+          data_stream.namespace: "namespace1"
+
+ - do:
+      catch: bad_request
+      index:
+        index: metrics-dataset0-namespace1
+        body:
+          "@timestamp": "2020-01-01"
+          data_stream.type: "synthetics"
+          data_stream.dataset: "dataset0"
+          data_stream.namespace: "namespace1"