Add shard count to the Elasticsearch index stats scrapper #35858
Labels
Comments
nathan-maves
added
enhancement
|
Pinging code owners:
See Adding Labels via Comments if you do not have permissions to add labels yourself. |
This was referenced Oct 29, 2024
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Component(s)
receiver/elasticsearch
Is your feature request related to a problem? Please describe.
I was trying to be able to compare 3 dimensions of Elasticsearch index stats document count, shards size and shard count but this receiver only scapes 2 of the 3 from the
/_statsendpoint.Describe the solution you'd like
Add the shard count to the collected metrics for each index. The
/users/_statsendpoint contains thetotal_countfor shards.{ "_shards": { "total": 6, "successful": 6, "failed": 0 }, "_all": { "primaries": { "docs": { "count": 599007114, "deleted": 6255158, "total_size_in_bytes": 106202644463 }, "shard_stats": { "total_count": 3 }, "store": { "size_in_bytes": 106202656601, "total_data_set_size_in_bytes": 106202656601, "reserved_in_bytes": 0 }, "indexing": { "index_total": 15653444, "index_time_in_millis": 5137004, "index_current": 0, "index_failed": 0, "delete_total": 0, "delete_time_in_millis": 0, "delete_current": 0, "noop_update_total": 75007, "is_throttled": false, "throttle_time_in_millis": 0, "write_load": 0.002295616505724265 }, "get": { "total": 1077709, "time_in_millis": 217716, "exists_total": 1077709, "exists_time_in_millis": 217716, "missing_total": 0, "missing_time_in_millis": 0, "current": 0 }, "search": { "open_contexts": 0, "query_total": 1, "query_time_in_millis": 0, "query_current": 0, "fetch_total": 0, "fetch_time_in_millis": 0, "fetch_current": 0, "scroll_total": 1, "scroll_time_in_millis": 1294, "scroll_current": 0, "suggest_total": 0, "suggest_time_in_millis": 0, "suggest_current": 0 }, "merges": { "current": 0, "current_docs": 0, "current_size_in_bytes": 0, "total": 1052, "total_time_in_millis": 17872746, "total_docs": 568163196, "total_size_in_bytes": 105534994910, "total_stopped_time_in_millis": 0, "total_throttled_time_in_millis": 14177420, "total_auto_throttle_in_bytes": 45279418 }, "refresh": { "total": 3070, "total_time_in_millis": 100790, "external_total": 1064, "external_total_time_in_millis": 834, "listeners": 0 }, "flush": { "total": 904, "periodic": 902, "total_time_in_millis": 434681, "total_time_excluding_waiting_on_lock_in_millis": 466358 }, "warmer": { "current": 0, "total": 1061, "total_time_in_millis": 313 }, "query_cache": { "memory_size_in_bytes": 0, "total_count": 99, "hit_count": 0, "miss_count": 99, "cache_size": 0, "cache_count": 0, "evictions": 0 }, "fielddata": { "memory_size_in_bytes": 0, "evictions": 0, "global_ordinals": { "build_time_in_millis": 0 } }, "completion": { "size_in_bytes": 0 }, "segments": { "count": 99, "memory_in_bytes": 0, "terms_memory_in_bytes": 0, "stored_fields_memory_in_bytes": 0, "term_vectors_memory_in_bytes": 0, "norms_memory_in_bytes": 0, "points_memory_in_bytes": 0, "doc_values_memory_in_bytes": 0, "index_writer_memory_in_bytes": 0, "version_map_memory_in_bytes": 0, "fixed_bit_set_memory_in_bytes": 75662136, "max_unsafe_auto_id_timestamp": -1, "file_sizes": {} }, "translog": { "operations": 0, "size_in_bytes": 165, "uncommitted_operations": 0, "uncommitted_size_in_bytes": 165, "earliest_last_modified_age": 408423717 }, "request_cache": { "memory_size_in_bytes": 0, "evictions": 0, "hit_count": 0, "miss_count": 0 }, "recovery": { "current_as_source": 0, "current_as_target": 0, "throttle_time_in_millis": 309022 }, "bulk": { "total_operations": 183844, "total_time_in_millis": 5419278, "total_size_in_bytes": 26490845523, "avg_time_in_millis": 16, "avg_size_in_bytes": 63504 }, "dense_vector": { "value_count": 0 }, "sparse_vector": { "value_count": 0 } }, "total": { "docs": { "count": 1198014228, "deleted": 12506844, "total_size_in_bytes": 212356147240 }, "shard_stats": { "total_count": 6 }, "store": { "size_in_bytes": 212356170745, "total_data_set_size_in_bytes": 212356170745, "reserved_in_bytes": 0 }, "indexing": { "index_total": 15653444, "index_time_in_millis": 5137004, "index_current": 0, "index_failed": 0, "delete_total": 0, "delete_time_in_millis": 0, "delete_current": 0, "noop_update_total": 75007, "is_throttled": false, "throttle_time_in_millis": 0, "write_load": 0.0012662688070682509 }, "get": { "total": 1077709, "time_in_millis": 217716, "exists_total": 1077709, "exists_time_in_millis": 217716, "missing_total": 0, "missing_time_in_millis": 0, "current": 0 }, "search": { "open_contexts": 0, "query_total": 1, "query_time_in_millis": 0, "query_current": 0, "fetch_total": 0, "fetch_time_in_millis": 0, "fetch_current": 0, "scroll_total": 1, "scroll_time_in_millis": 1294, "scroll_current": 0, "suggest_total": 0, "suggest_time_in_millis": 0, "suggest_current": 0 }, "merges": { "current": 0, "current_docs": 0, "current_size_in_bytes": 0, "total": 1053, "total_time_in_millis": 17873029, "total_docs": 568207232, "total_size_in_bytes": 105545476271, "total_stopped_time_in_millis": 0, "total_throttled_time_in_millis": 14177420, "total_auto_throttle_in_bytes": 108193978 }, "refresh": { "total": 3091, "total_time_in_millis": 101623, "external_total": 1075, "external_total_time_in_millis": 987, "listeners": 0 }, "flush": { "total": 910, "periodic": 907, "total_time_in_millis": 434756, "total_time_excluding_waiting_on_lock_in_millis": 466433 }, "warmer": { "current": 0, "total": 1069, "total_time_in_millis": 451 }, "query_cache": { "memory_size_in_bytes": 0, "total_count": 99, "hit_count": 0, "miss_count": 99, "cache_size": 0, "cache_count": 0, "evictions": 0 }, "fielddata": { "memory_size_in_bytes": 0, "evictions": 0, "global_ordinals": { "build_time_in_millis": 0 } }, "completion": { "size_in_bytes": 0 }, "segments": { "count": 191, "memory_in_bytes": 0, "terms_memory_in_bytes": 0, "stored_fields_memory_in_bytes": 0, "term_vectors_memory_in_bytes": 0, "norms_memory_in_bytes": 0, "points_memory_in_bytes": 0, "doc_values_memory_in_bytes": 0, "index_writer_memory_in_bytes": 0, "version_map_memory_in_bytes": 0, "fixed_bit_set_memory_in_bytes": 151323528, "max_unsafe_auto_id_timestamp": -1, "file_sizes": {} }, "translog": { "operations": 0, "size_in_bytes": 330, "uncommitted_operations": 0, "uncommitted_size_in_bytes": 330, "earliest_last_modified_age": 408423717 }, "request_cache": { "memory_size_in_bytes": 0, "evictions": 0, "hit_count": 0, "miss_count": 0 }, "recovery": { "current_as_source": 0, "current_as_target": 0, "throttle_time_in_millis": 1678130 }, "bulk": { "total_operations": 183844, "total_time_in_millis": 5419278, "total_size_in_bytes": 26490845523, "avg_time_in_millis": 16, "avg_size_in_bytes": 63504 }, "dense_vector": { "value_count": 0 }, "sparse_vector": { "value_count": 0 } } }, "indices": { "users": { "uuid": "6UqMYhVwS4OwyngTqKG8ow", "health": "green", "status": "open", "primaries": { "docs": { "count": 599007114, "deleted": 6255158, "total_size_in_bytes": 106202644463 }, "shard_stats": { "total_count": 3 }, "store": { "size_in_bytes": 106202656601, "total_data_set_size_in_bytes": 106202656601, "reserved_in_bytes": 0 }, "indexing": { "index_total": 15653444, "index_time_in_millis": 5137004, "index_current": 0, "index_failed": 0, "delete_total": 0, "delete_time_in_millis": 0, "delete_current": 0, "noop_update_total": 75007, "is_throttled": false, "throttle_time_in_millis": 0, "write_load": 0.002295616505724265 }, "get": { "total": 1077709, "time_in_millis": 217716, "exists_total": 1077709, "exists_time_in_millis": 217716, "missing_total": 0, "missing_time_in_millis": 0, "current": 0 }, "search": { "open_contexts": 0, "query_total": 1, "query_time_in_millis": 0, "query_current": 0, "fetch_total": 0, "fetch_time_in_millis": 0, "fetch_current": 0, "scroll_total": 1, "scroll_time_in_millis": 1294, "scroll_current": 0, "suggest_total": 0, "suggest_time_in_millis": 0, "suggest_current": 0 }, "merges": { "current": 0, "current_docs": 0, "current_size_in_bytes": 0, "total": 1052, "total_time_in_millis": 17872746, "total_docs": 568163196, "total_size_in_bytes": 105534994910, "total_stopped_time_in_millis": 0, "total_throttled_time_in_millis": 14177420, "total_auto_throttle_in_bytes": 45279418 }, "refresh": { "total": 3070, "total_time_in_millis": 100790, "external_total": 1064, "external_total_time_in_millis": 834, "listeners": 0 }, "flush": { "total": 904, "periodic": 902, "total_time_in_millis": 434681, "total_time_excluding_waiting_on_lock_in_millis": 466358 }, "warmer": { "current": 0, "total": 1061, "total_time_in_millis": 313 }, "query_cache": { "memory_size_in_bytes": 0, "total_count": 99, "hit_count": 0, "miss_count": 99, "cache_size": 0, "cache_count": 0, "evictions": 0 }, "fielddata": { "memory_size_in_bytes": 0, "evictions": 0, "global_ordinals": { "build_time_in_millis": 0 } }, "completion": { "size_in_bytes": 0 }, "segments": { "count": 99, "memory_in_bytes": 0, "terms_memory_in_bytes": 0, "stored_fields_memory_in_bytes": 0, "term_vectors_memory_in_bytes": 0, "norms_memory_in_bytes": 0, "points_memory_in_bytes": 0, "doc_values_memory_in_bytes": 0, "index_writer_memory_in_bytes": 0, "version_map_memory_in_bytes": 0, "fixed_bit_set_memory_in_bytes": 75662136, "max_unsafe_auto_id_timestamp": -1, "file_sizes": {} }, "translog": { "operations": 0, "size_in_bytes": 165, "uncommitted_operations": 0, "uncommitted_size_in_bytes": 165, "earliest_last_modified_age": 408423717 }, "request_cache": { "memory_size_in_bytes": 0, "evictions": 0, "hit_count": 0, "miss_count": 0 }, "recovery": { "current_as_source": 0, "current_as_target": 0, "throttle_time_in_millis": 309022 }, "bulk": { "total_operations": 183844, "total_time_in_millis": 5419278, "total_size_in_bytes": 26490845523, "avg_time_in_millis": 16, "avg_size_in_bytes": 63504 }, "dense_vector": { "value_count": 0 }, "sparse_vector": { "value_count": 0 } }, "total": { "docs": { "count": 1198014228, "deleted": 12506844, "total_size_in_bytes": 212356147240 }, "shard_stats": { "total_count": 6 }, "store": { "size_in_bytes": 212356170745, "total_data_set_size_in_bytes": 212356170745, "reserved_in_bytes": 0 }, "indexing": { "index_total": 15653444, "index_time_in_millis": 5137004, "index_current": 0, "index_failed": 0, "delete_total": 0, "delete_time_in_millis": 0, "delete_current": 0, "noop_update_total": 75007, "is_throttled": false, "throttle_time_in_millis": 0, "write_load": 0.0012662688070682509 }, "get": { "total": 1077709, "time_in_millis": 217716, "exists_total": 1077709, "exists_time_in_millis": 217716, "missing_total": 0, "missing_time_in_millis": 0, "current": 0 }, "search": { "open_contexts": 0, "query_total": 1, "query_time_in_millis": 0, "query_current": 0, "fetch_total": 0, "fetch_time_in_millis": 0, "fetch_current": 0, "scroll_total": 1, "scroll_time_in_millis": 1294, "scroll_current": 0, "suggest_total": 0, "suggest_time_in_millis": 0, "suggest_current": 0 }, "merges": { "current": 0, "current_docs": 0, "current_size_in_bytes": 0, "total": 1053, "total_time_in_millis": 17873029, "total_docs": 568207232, "total_size_in_bytes": 105545476271, "total_stopped_time_in_millis": 0, "total_throttled_time_in_millis": 14177420, "total_auto_throttle_in_bytes": 108193978 }, "refresh": { "total": 3091, "total_time_in_millis": 101623, "external_total": 1075, "external_total_time_in_millis": 987, "listeners": 0 }, "flush": { "total": 910, "periodic": 907, "total_time_in_millis": 434756, "total_time_excluding_waiting_on_lock_in_millis": 466433 }, "warmer": { "current": 0, "total": 1069, "total_time_in_millis": 451 }, "query_cache": { "memory_size_in_bytes": 0, "total_count": 99, "hit_count": 0, "miss_count": 99, "cache_size": 0, "cache_count": 0, "evictions": 0 }, "fielddata": { "memory_size_in_bytes": 0, "evictions": 0, "global_ordinals": { "build_time_in_millis": 0 } }, "completion": { "size_in_bytes": 0 }, "segments": { "count": 191, "memory_in_bytes": 0, "terms_memory_in_bytes": 0, "stored_fields_memory_in_bytes": 0, "term_vectors_memory_in_bytes": 0, "norms_memory_in_bytes": 0, "points_memory_in_bytes": 0, "doc_values_memory_in_bytes": 0, "index_writer_memory_in_bytes": 0, "version_map_memory_in_bytes": 0, "fixed_bit_set_memory_in_bytes": 151323528, "max_unsafe_auto_id_timestamp": -1, "file_sizes": {} }, "translog": { "operations": 0, "size_in_bytes": 330, "uncommitted_operations": 0, "uncommitted_size_in_bytes": 330, "earliest_last_modified_age": 408423717 }, "request_cache": { "memory_size_in_bytes": 0, "evictions": 0, "hit_count": 0, "miss_count": 0 }, "recovery": { "current_as_source": 0, "current_as_target": 0, "throttle_time_in_millis": 1678130 }, "bulk": { "total_operations": 183844, "total_time_in_millis": 5419278, "total_size_in_bytes": 26490845523, "avg_time_in_millis": 16, "avg_size_in_bytes": 63504 }, "dense_vector": { "value_count": 0 }, "sparse_vector": { "value_count": 0 } } } } }Describe alternatives you've considered
No response
Additional context
No response
The text was updated successfully, but these errors were encountered: