add heartbeat ds (#396)

Author: joeypoon

custom_subsets/elastic_endpoint/heartbeat/heartbeat.yaml

@@ -0,0 +1,15 @@
+---
+name: heartbeat
+fields:
+  base:
+    fields:
+      "@timestamp": {}
+      message: {}
+  agent:
+    fields:
+      id: {}
+  data_stream:
+    fields: "*"
+  event:
+    fields:
+      ingested: {}

package/endpoint/data_stream/heartbeat/elasticsearch/ingest_pipeline/default.json

@@ -0,0 +1,12 @@
+{
+  "description": "Pipeline for setting event.ingested",
+  "processors": [
+    {
+      "set": {
+        "field": "event.ingested",
+        "value": "{{ _ingest.timestamp }}",
+        "ignore_failure": true
+      }
+    }
+  ]
+}

package/endpoint/data_stream/heartbeat/fields/fields.yml

@@ -0,0 +1,85 @@
+- name: '@timestamp'
+  level: core
+  required: true
+  type: date
+  description: 'Date/time when the event originated.
+
+    This is the date/time extracted from the event, typically representing when the event was generated by the source.
+
+    If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline.
+
+    Required field for all events.'
+  example: '2016-05-23T08:05:34.853Z'
+  default_field: true
+- name: message
+  level: core
+  type: match_only_text
+  description: 'For log events the message field contains the log message, optimized for viewing in a log viewer.
+
+    For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event.
+
+    If multiple messages exist, they can be combined into one message.'
+  example: Hello World
+  default_field: true
+- name: agent
+  title: Agent
+  group: 2
+  description: 'The agent fields contain the data about the software entity, if any, that collects, detects, or observes events on a host, or takes measurements on a host.
+
+    Examples include Beats. Agents may also run on observers. ECS agent.* fields shall be populated with details of the agent running on the host or observer where the event happened or the measurement was taken.'
+  footnote: 'Examples: In the case of Beats for logs, the agent.name is filebeat. For APM, it is the agent running in the app/service. The agent information does not change if data is sent through queuing systems like Kafka, Redis, or processing systems such as Logstash or APM Server.'
+  type: group
+  default_field: true
+  fields:
+    - name: id
+      level: core
+      type: keyword
+      ignore_above: 1024
+      description: 'Unique identifier of this agent (if one exists).
+
+        Example: For Beats this would be beat.id.'
+      example: 8a4f500d
+- name: data_stream
+  title: data_stream
+  group: 2
+  description: Fields describing the new indexing strategy <type>-<dataset>-<namespace>
+  type: group
+  default_field: true
+  fields:
+    - name: dataset
+      level: custom
+      type: constant_keyword
+      description: Data stream dataset name.
+      example: nginx.access
+      default_field: false
+    - name: namespace
+      level: custom
+      type: constant_keyword
+      description: Data stream namespace.
+      example: production
+      default_field: false
+    - name: type
+      level: custom
+      type: constant_keyword
+      description: Data stream type.
+      example: logs
+      default_field: false
+- name: event
+  title: Event
+  group: 2
+  description: 'The event fields are used for context information about the log or metric event itself.
+
+    A log is defined as an event containing details of something that happened. Log events must include the time at which the thing happened. Examples of log events include a process starting on a host, a network packet being sent from a source to a destination, or a network connection between a client and a server being initiated or closed. A metric is defined as an event containing one or more numerical measurements and the time at which the measurement was taken. Examples of metric events include memory pressure measured on a host and device temperature. See the `event.kind` definition in this section for additional details about metric and state events.'
+  type: group
+  default_field: true
+  fields:
+    - name: ingested
+      level: core
+      type: date
+      description: 'Timestamp when an event arrived in the central data store.
+
+        This is different from `@timestamp`, which is when the event originally occurred.  It''s also different from `event.created`, which is meant to capture the first time an agent saw the event.
+
+        In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` < `event.created` < `event.ingested`.'
+      example: '2016-05-23T08:05:35.101Z'
+      default_field: false

package/endpoint/data_stream/heartbeat/manifest.yml

@@ -0,0 +1,14 @@
+title: Endpoint Heartbeat
+type: logs
+dataset: endpoint.heartbeat
+hidden: true
+elasticsearch:
+  index_template:
+    mappings:
+      dynamic: false
+    settings:
+      index:
+        sort.field:
+          - event.ingested
+        sort.order:
+          - desc

package/endpoint/data_stream/heartbeat/sample_event.json

@@ -0,0 +1,15 @@
+{
+    "@timestamp": "2023-07-18T20:40:09.279939Z",
+    "agent": {
+        "id": "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"
+    },
+    "data_stream": {
+        "dataset": "endpoint.heartbeat",
+        "namespace": "default",
+        "type": ".logs"
+    },
+    "message": "Endpoint heartbeat",
+    "event": {
+        "ingested": "2023-07-18T20:40:09.279939Z"
+    }
+}