diff --git a/elastic/assets/configuration/spec.yaml b/elastic/assets/configuration/spec.yaml index d530126886543..e278dd27c1997 100644 --- a/elastic/assets/configuration/spec.yaml +++ b/elastic/assets/configuration/spec.yaml @@ -35,6 +35,14 @@ files: value: type: boolean example: false + - name: detailed_index_stats + description: | + If you want to obtain index-specific stats, use this flag with `cluster_stats` and `pshard_stats` set to true. + Without this flag you only get stats from `_all`. + Do not use it if you are pointing to localhost. + value: + type: boolean + example: false - name: index_stats description: Set "index_stats" to true to collect metrics for individual indices. value: diff --git a/elastic/datadog_checks/elastic/config.py b/elastic/datadog_checks/elastic/config.py index b81d17bc6179a..213db630ad60f 100644 --- a/elastic/datadog_checks/elastic/config.py +++ b/elastic/datadog_checks/elastic/config.py @@ -15,6 +15,7 @@ 'pshard_graceful_to', 'node_name_as_host', 'cluster_stats', + 'detailed_index_stats', 'slm_stats', 'index_stats', 'service_check_tags', @@ -39,6 +40,7 @@ def from_instance(instance): node_name_as_host = is_affirmative(instance.get('node_name_as_host', False)) index_stats = is_affirmative(instance.get('index_stats', False)) cluster_stats = is_affirmative(instance.get('cluster_stats', False)) + detailed_index_stats = is_affirmative(instance.get('detailed_index_stats', False)) slm_stats = is_affirmative(instance.get('slm_stats', False)) if 'is_external' in instance: cluster_stats = is_affirmative(instance.get('is_external', False)) @@ -69,6 +71,7 @@ def from_instance(instance): pshard_graceful_to=pshard_graceful_to, node_name_as_host=node_name_as_host, cluster_stats=cluster_stats, + detailed_index_stats=detailed_index_stats, slm_stats=slm_stats, index_stats=index_stats, service_check_tags=service_check_tags, diff --git a/elastic/datadog_checks/elastic/config_models/defaults.py b/elastic/datadog_checks/elastic/config_models/defaults.py index 59657526f5068..fb92ae541234e 100644 --- a/elastic/datadog_checks/elastic/config_models/defaults.py +++ b/elastic/datadog_checks/elastic/config_models/defaults.py @@ -60,6 +60,10 @@ def instance_connect_timeout(field, value): return get_default_field_value(field, value) +def instance_detailed_index_stats(field, value): + return False + + def instance_disable_generic_tags(field, value): return False diff --git a/elastic/datadog_checks/elastic/config_models/instance.py b/elastic/datadog_checks/elastic/config_models/instance.py index bc2cb002017bd..7dc9b8c5072ba 100644 --- a/elastic/datadog_checks/elastic/config_models/instance.py +++ b/elastic/datadog_checks/elastic/config_models/instance.py @@ -44,6 +44,7 @@ class Config: cat_allocation_stats: Optional[bool] cluster_stats: Optional[bool] connect_timeout: Optional[float] + detailed_index_stats: Optional[bool] disable_generic_tags: Optional[bool] disable_legacy_cluster_tag: Optional[bool] empty_default_hostname: Optional[bool] diff --git a/elastic/datadog_checks/elastic/data/conf.yaml.example b/elastic/datadog_checks/elastic/data/conf.yaml.example index c671a1fa69044..eae694b017ba3 100644 --- a/elastic/datadog_checks/elastic/data/conf.yaml.example +++ b/elastic/datadog_checks/elastic/data/conf.yaml.example @@ -68,6 +68,13 @@ instances: # # cluster_stats: false + ## @param detailed_index_stats - boolean - optional - default: false + ## If you want to obtain index-specific stats, use this flag with `cluster_stats` and `pshard_stats` set to true. + ## Without this flag you only get stats from `_all`. + ## Do not use it if you are pointing to localhost. + # + # detailed_index_stats: false + ## @param index_stats - boolean - optional - default: false ## Set "index_stats" to true to collect metrics for individual indices. # diff --git a/elastic/datadog_checks/elastic/elastic.py b/elastic/datadog_checks/elastic/elastic.py index 7d122d5885e20..8cbab3b9d7a58 100644 --- a/elastic/datadog_checks/elastic/elastic.py +++ b/elastic/datadog_checks/elastic/elastic.py @@ -1,6 +1,7 @@ # (C) Datadog, Inc. 2018-present # All rights reserved # Licensed under Simplified BSD License (see LICENSE) +import re import time from collections import defaultdict @@ -307,7 +308,23 @@ def _process_stats_data(self, data, stats_metrics, base_tags): def _process_pshard_stats_data(self, data, pshard_stats_metrics, base_tags): for metric, desc in iteritems(pshard_stats_metrics): - self._process_metric(data, metric, *desc, tags=base_tags) + pshard_tags = base_tags + if desc[1].startswith('_all.'): + pshard_tags = pshard_tags + ['index_name:_all'] + self._process_metric(data, metric, *desc, tags=pshard_tags) + # process index-level metrics + if self._config.cluster_stats and self._config.detailed_index_stats: + for metric, desc in iteritems(pshard_stats_metrics): + if desc[1].startswith('_all.'): + for index in data['indices']: + self.log.debug("Processing index %s", index) + escaped_index = index.replace('.', '\.') # noqa: W605 + index_desc = ( + desc[0], + 'indices.' + escaped_index + '.' + desc[1].replace('_all.', ''), + desc[2] if 2 < len(desc) else None, + ) + self._process_metric(data, metric, *index_desc, tags=base_tags + ['index_name:' + index]) def _process_metric(self, data, metric, xtype, path, xform=None, tags=None, hostname=None): """ @@ -319,9 +336,9 @@ def _process_metric(self, data, metric, xtype, path, xform=None, tags=None, host value = data # Traverse the nested dictionaries - for key in path.split('.'): + for key in re.split(r'(?