[elasticsearch] Extension of the Elasticsearch integration with datastream-centric stats

packages/elasticsearch/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.16.0"
+  changes:
+    - description: Add transform job for usage analysis
+      type: enhancement
+      link: "" # TODO
 - version: "1.15.0"
   changes:
     - description: Add support shard_stats.total_count in node metricset

packages/elasticsearch/elasticsearch/ingest_pipeline/monitoring_indices.yml

@@ -0,0 +1,145 @@
+---
+processors:
+  - set:
+      field: event.ingested
+      tag: set_event_ingested
+      value: "{{_ingest.timestamp}}"
+  - grok:
+      field: elasticsearch.index.name
+      tag: grok_parse_index_name
+      patterns:
+        - '^(?:partial-)?(?:restored-)?(?:shrink-.{4}-)?(?:\.ds-)?(?<elasticsearch.index.datastream>[a-z_0-9\-\.]+?)(-(?<elasticsearch.index.creation_date>\d{4}\.\d{2}(\.\d{2})?))?(?:-\d+)?$'
+      ignore_failure: true
+  - date:
+      field: elasticsearch.index.creation_date
+      target_field: elasticsearch.index.creation_date
+      ignore_failure: true
+      formats:
+        - yyyy.MM.dd
+        - yyyy.MM
+      tag: date_parse_index_creation_date
+  - script:
+      source: |
+        def creationDate = ctx.elasticsearch.index.creation_date;
+        if (creationDate != null) {
+          def creationDateTime = ZonedDateTime.parse(creationDate);
+          def now = ZonedDateTime.parse(ctx['@timestamp']);
+          ctx.elasticsearch.index.age = ChronoUnit.DAYS.between(creationDateTime, now);
+        }
+      ignore_failure: true
+      tag: script_compute_index_age
+  - script:
+      source: |
+        def preference = ctx.end['elasticsearch.index.tier_preference'];
+        if (preference.contains("data_frozen")) {
+          ctx.elasticsearch.index.tier = "frozen";
+        } else if (preference.contains("data_cold")) {
+          ctx.elasticsearch.index.tier = "cold";
+        } else if (preference.contains("data_warm")) {
+          ctx.elasticsearch.index.tier = "warm";
+        } else if (preference.contains("data_hot") || preference.contains("data_content")) {
+          ctx.elasticsearch.index.tier = "hot/content";
+        }
+        ctx.end.remove('elasticsearch.index.tier_preference');
+      ignore_failure: true
+      tag: script_parse_index_tier
+  # Failure to identify the tier preference will result in the index tier being set to unknown
+  # This is also the "default" case when tier preference is not available.
+  - set:
+      field: elasticsearch.index.tier
+      value: "unknown"
+      tag: set_index_tier_unknown
+      if: "ctx.elasticsearch.index.tier == null"
+  - foreach:
+      field: end
+      processor:
+        set:
+          field: "{{ _ingest._key }}"
+          value: "{{ _ingest._value }}"
+      tag: set_end_fields
+  - dot_expander:
+      field: "*"
+      tag: dot_expander
+  - convert:
+      field: elasticsearch.index.primaries.docs.count
+      type: long
+      ignore_failure: true
+      tag: convert_primaries_docs_count
+  - convert:
+      field: elasticsearch.index.primaries.docs.count_delta
+      type: long
+      ignore_failure: true
+      tag: convert_primaries_docs_count_delta
+  - convert:
+      field: elasticsearch.index.primaries.store.total_data_set_size_in_bytes
+      type: long
+      ignore_failure: true
+      tag: convert_primaries_store_total_data_set_size_in_bytes
+  - convert:
+      field: elasticsearch.index.primaries.store.total_data_set_size_in_bytes_delta
+      type: long
+      ignore_failure: true
+      tag: convert_primaries_store_total_data_set_size_in_bytes_delta
+  - convert:
+      field: elasticsearch.index.total.store.size_in_bytes
+      type: long
+      ignore_failure: true
+      tag: convert_total_store_size_in_bytes
+  - convert:
+      field: elasticsearch.index.total.store.size_in_bytes_delta
+      type: long
+      ignore_failure: true
+      tag: convert_total_store_size_in_bytes_delta
+  - convert:
+      field: elasticsearch.index.total.search.query_total
+      type: long
+      ignore_failure: true
+      tag: convert_total_search_query_total
+  - convert:
+      field: elasticsearch.index.total.search.query_total_delta
+      type: long
+      ignore_failure: true
+      tag: convert_total_search_query_total_delta
+  - convert:
+      field: elasticsearch.index.total.search.query_time_in_millis
+      type: long
+      ignore_failure: true
+      tag: convert_total_search_query_time_in_millis
+  - convert:
+      field: elasticsearch.index.total.search.query_time_in_millis_delta
+      type: long
+      ignore_failure: true
+      tag: convert_total_search_query_time_in_millis_delta
+  - convert:
+      field: elasticsearch.index.total.indexing.index_total
+      type: long
+      ignore_failure: true
+      tag: convert_total_indexing_index_total
+  - convert:
+      field: elasticsearch.index.total.indexing.index_total_delta
+      type: long
+      ignore_failure: true
+      tag: convert_total_indexing_index_total_delta
+  - convert:
+      field: elasticsearch.index.total.indexing.index_time_in_millis
+      type: long
+      ignore_failure: true
+      tag: convert_total_indexing_index_time_in_millis
+  - convert:
+      field: elasticsearch.index.total.indexing.index_time_in_millis_delta
+      type: long
+      ignore_failure: true
+      tag: convert_total_indexing_index_time_in_millis_delta
+  - remove:
+      field:
+        - start
+        - end
+      tag: remove_start_end_fields
+
+on_failure:
+  - set:
+      field: event.kind
+      value: "pipeline_error"
+  - append:
+      field: error.message
+      value: "Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} failed with message {{ _ingest.on_failure_message }}"

packages/elasticsearch/elasticsearch/transform/index_pivot/fields/fields.yml

@@ -0,0 +1,118 @@
+- name: "@timestamp"
+  type: date
+  description: The time the event was recorded.
+
+- name: elasticsearch
+  type: group
+  description: Elasticsearch metrics and metadata
+  fields:
+    - name: cluster
+      type: group
+      description: Information about the Elasticsearch cluster.
+      fields:
+        - name: name
+          type: keyword
+          description: The name of the Elasticsearch cluster.
+
+    - name: index
+      type: group
+      description: Information about the specific index being monitored.
+      fields:
+        - name: datastream
+          type: keyword
+          description: The data stream associated with the index.
+        - name: tier
+          type: keyword
+          description: The data tier of the index (e.g., hot, warm, cold, frozen, or unknown).
+        - name: name
+          type: keyword
+          description: Name of the Elasticsearch index.
+        - name: creation_date
+          type: date
+          description: The creation date of the index.
+        - name: age
+          type: long
+          description: Age of the index in days.
+
+        - name: primaries
+          type: group
+          description: Metrics related to the primary shards of the index.
+          fields:
+            - name: docs
+              type: group
+              description: Metrics related to the number of documents in the index.
+              fields:
+                - name: count
+                  type: long
+                  description: Number of primary documents in the index.
+                - name: count_delta
+                  type: long
+                  description: Change in the number of primary documents in the index since the last measurement.
+
+            - name: store
+              type: group
+              description: Metrics related to data storage.
+              fields:
+                - name: total_data_set_size_in_bytes
+                  type: long
+                  description: Primary size of the data set in bytes.
+                - name: total_data_set_size_in_bytes_delta
+                  type: long
+                  description: Change in the primary size of the data set in bytes since the last measurement.
+
+        - name: total
+          type: group
+          description: Aggregated metrics related to the index.
+          fields:
+            - name: search
+              type: group
+              description: Metrics related to search operations.
+              fields:
+                - name: query_total
+                  type: long
+                  description: Total number of search queries executed.
+                - name: query_time_in_millis
+                  type: long
+                  description: Total time in milliseconds spent on search queries.
+                - name: query_total_delta
+                  type: long
+                  description: Change in the number of search queries since the last measurement.
+                - name: query_time_in_millis_delta
+                  type: long
+                  description: Change in the total time in milliseconds for search queries since the last measurement.
+
+            - name: indexing
+              type: group
+              description: Metrics related to document indexing.
+              fields:
+                - name: index_total
+                  type: long
+                  description: Total number of documents indexed.
+                - name: index_time_in_millis
+                  type: long
+                  description: Total time in milliseconds spent on indexing documents.
+                - name: index_total_delta
+                  type: long
+                  description: Change in the number of documents indexed since the last measurement.
+                - name: index_time_in_millis_delta
+                  type: long
+                  description: Change in the total time in milliseconds for indexing documents since the last measurement.
+
+            - name: store
+              type: group
+              description: Metrics related to data storage.
+              fields:
+                - name: size_in_bytes
+                  type: long
+                  description: Total size of the data set in bytes.
+                - name: size_in_bytes_delta
+                  type: long
+                  description: Change in the total size of the data set in bytes since the last measurement.
+
+- name: event
+  type: group
+  description: Information about the event.
+  fields:
+    - name: ingested
+      type: date
+      description: Timestamp when the event was ingested by Elasticsearch.

packages/elasticsearch/elasticsearch/transform/index_pivot/manifest.yml

@@ -0,0 +1,11 @@
+start: false
+destination_index_template:
+  # All these settings aim to optimize the index for storage space, as we can't use rollovers or monthly indices.
+  # In most cases, this should be enough for 1+ year of usage insights.
+  settings:
+    index:
+      codec: best_compression
+      number_of_shards: 3
+  mappings:
+    _source:
+      mode: synthetic