diff --git a/.chloggen/es_add_metric.yaml b/.chloggen/es_add_metric.yaml new file mode 100644 index 000000000000..09722a7d8888 --- /dev/null +++ b/.chloggen/es_add_metric.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: elasticsearchreceiver + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Add metric for active index merges + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [34387] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] diff --git a/receiver/elasticsearchreceiver/documentation.md b/receiver/elasticsearchreceiver/documentation.md index f35723dd0490..e73a153d52a7 100644 --- a/receiver/elasticsearchreceiver/documentation.md +++ b/receiver/elasticsearchreceiver/documentation.md @@ -852,6 +852,20 @@ The number of documents for an index. | state | The state of the document. | Str: ``active``, ``deleted`` | | aggregation | Type of shard aggregation for index statistics | Str: ``primary_shards``, ``total`` | +### elasticsearch.index.operations.merge.current + +The number of currently active segment merges + +| Unit | Metric Type | Value Type | +| ---- | ----------- | ---------- | +| {merges} | Gauge | Int | + +#### Attributes + +| Name | Description | Values | +| ---- | ----------- | ------ | +| aggregation | Type of shard aggregation for index statistics | Str: ``primary_shards``, ``total`` | + ### elasticsearch.index.operations.merge.docs_count The total number of documents in merge operations for an index. diff --git a/receiver/elasticsearchreceiver/internal/metadata/generated_config.go b/receiver/elasticsearchreceiver/internal/metadata/generated_config.go index c1f3b8b170d6..d572d9c9ef21 100644 --- a/receiver/elasticsearchreceiver/internal/metadata/generated_config.go +++ b/receiver/elasticsearchreceiver/internal/metadata/generated_config.go @@ -48,6 +48,7 @@ type MetricsConfig struct { ElasticsearchIndexCacheSize MetricConfig `mapstructure:"elasticsearch.index.cache.size"` ElasticsearchIndexDocuments MetricConfig `mapstructure:"elasticsearch.index.documents"` ElasticsearchIndexOperationsCompleted MetricConfig `mapstructure:"elasticsearch.index.operations.completed"` + ElasticsearchIndexOperationsMergeCurrent MetricConfig `mapstructure:"elasticsearch.index.operations.merge.current"` ElasticsearchIndexOperationsMergeDocsCount MetricConfig `mapstructure:"elasticsearch.index.operations.merge.docs_count"` ElasticsearchIndexOperationsMergeSize MetricConfig `mapstructure:"elasticsearch.index.operations.merge.size"` ElasticsearchIndexOperationsTime MetricConfig `mapstructure:"elasticsearch.index.operations.time"` @@ -183,6 +184,9 @@ func DefaultMetricsConfig() MetricsConfig { ElasticsearchIndexOperationsCompleted: MetricConfig{ Enabled: true, }, + ElasticsearchIndexOperationsMergeCurrent: MetricConfig{ + Enabled: false, + }, ElasticsearchIndexOperationsMergeDocsCount: MetricConfig{ Enabled: false, }, diff --git a/receiver/elasticsearchreceiver/internal/metadata/generated_config_test.go b/receiver/elasticsearchreceiver/internal/metadata/generated_config_test.go index 0533e8e88ee0..b274c10eb483 100644 --- a/receiver/elasticsearchreceiver/internal/metadata/generated_config_test.go +++ b/receiver/elasticsearchreceiver/internal/metadata/generated_config_test.go @@ -45,6 +45,7 @@ func TestMetricsBuilderConfig(t *testing.T) { ElasticsearchIndexCacheSize: MetricConfig{Enabled: true}, ElasticsearchIndexDocuments: MetricConfig{Enabled: true}, ElasticsearchIndexOperationsCompleted: MetricConfig{Enabled: true}, + ElasticsearchIndexOperationsMergeCurrent: MetricConfig{Enabled: true}, ElasticsearchIndexOperationsMergeDocsCount: MetricConfig{Enabled: true}, ElasticsearchIndexOperationsMergeSize: MetricConfig{Enabled: true}, ElasticsearchIndexOperationsTime: MetricConfig{Enabled: true}, @@ -149,6 +150,7 @@ func TestMetricsBuilderConfig(t *testing.T) { ElasticsearchIndexCacheSize: MetricConfig{Enabled: false}, ElasticsearchIndexDocuments: MetricConfig{Enabled: false}, ElasticsearchIndexOperationsCompleted: MetricConfig{Enabled: false}, + ElasticsearchIndexOperationsMergeCurrent: MetricConfig{Enabled: false}, ElasticsearchIndexOperationsMergeDocsCount: MetricConfig{Enabled: false}, ElasticsearchIndexOperationsMergeSize: MetricConfig{Enabled: false}, ElasticsearchIndexOperationsTime: MetricConfig{Enabled: false}, diff --git a/receiver/elasticsearchreceiver/internal/metadata/generated_metrics.go b/receiver/elasticsearchreceiver/internal/metadata/generated_metrics.go index 76236872ba06..c43a76e53e77 100644 --- a/receiver/elasticsearchreceiver/internal/metadata/generated_metrics.go +++ b/receiver/elasticsearchreceiver/internal/metadata/generated_metrics.go @@ -1591,6 +1591,57 @@ func newMetricElasticsearchIndexOperationsCompleted(cfg MetricConfig) metricElas return m } +type metricElasticsearchIndexOperationsMergeCurrent struct { + data pmetric.Metric // data buffer for generated metric. + config MetricConfig // metric config provided by user. + capacity int // max observed number of data points added to the metric. +} + +// init fills elasticsearch.index.operations.merge.current metric with initial data. +func (m *metricElasticsearchIndexOperationsMergeCurrent) init() { + m.data.SetName("elasticsearch.index.operations.merge.current") + m.data.SetDescription("The number of currently active segment merges") + m.data.SetUnit("{merges}") + m.data.SetEmptyGauge() + m.data.Gauge().DataPoints().EnsureCapacity(m.capacity) +} + +func (m *metricElasticsearchIndexOperationsMergeCurrent) recordDataPoint(start pcommon.Timestamp, ts pcommon.Timestamp, val int64, indexAggregationTypeAttributeValue string) { + if !m.config.Enabled { + return + } + dp := m.data.Gauge().DataPoints().AppendEmpty() + dp.SetStartTimestamp(start) + dp.SetTimestamp(ts) + dp.SetIntValue(val) + dp.Attributes().PutStr("aggregation", indexAggregationTypeAttributeValue) +} + +// updateCapacity saves max length of data point slices that will be used for the slice capacity. +func (m *metricElasticsearchIndexOperationsMergeCurrent) updateCapacity() { + if m.data.Gauge().DataPoints().Len() > m.capacity { + m.capacity = m.data.Gauge().DataPoints().Len() + } +} + +// emit appends recorded metric data to a metrics slice and prepares it for recording another set of data points. +func (m *metricElasticsearchIndexOperationsMergeCurrent) emit(metrics pmetric.MetricSlice) { + if m.config.Enabled && m.data.Gauge().DataPoints().Len() > 0 { + m.updateCapacity() + m.data.MoveTo(metrics.AppendEmpty()) + m.init() + } +} + +func newMetricElasticsearchIndexOperationsMergeCurrent(cfg MetricConfig) metricElasticsearchIndexOperationsMergeCurrent { + m := metricElasticsearchIndexOperationsMergeCurrent{config: cfg} + if cfg.Enabled { + m.data = pmetric.NewMetric() + m.init() + } + return m +} + type metricElasticsearchIndexOperationsMergeDocsCount struct { data pmetric.Metric // data buffer for generated metric. config MetricConfig // metric config provided by user. @@ -5274,6 +5325,7 @@ type MetricsBuilder struct { metricElasticsearchIndexCacheSize metricElasticsearchIndexCacheSize metricElasticsearchIndexDocuments metricElasticsearchIndexDocuments metricElasticsearchIndexOperationsCompleted metricElasticsearchIndexOperationsCompleted + metricElasticsearchIndexOperationsMergeCurrent metricElasticsearchIndexOperationsMergeCurrent metricElasticsearchIndexOperationsMergeDocsCount metricElasticsearchIndexOperationsMergeDocsCount metricElasticsearchIndexOperationsMergeSize metricElasticsearchIndexOperationsMergeSize metricElasticsearchIndexOperationsTime metricElasticsearchIndexOperationsTime @@ -5383,6 +5435,7 @@ func NewMetricsBuilder(mbc MetricsBuilderConfig, settings receiver.Settings, opt metricElasticsearchIndexCacheSize: newMetricElasticsearchIndexCacheSize(mbc.Metrics.ElasticsearchIndexCacheSize), metricElasticsearchIndexDocuments: newMetricElasticsearchIndexDocuments(mbc.Metrics.ElasticsearchIndexDocuments), metricElasticsearchIndexOperationsCompleted: newMetricElasticsearchIndexOperationsCompleted(mbc.Metrics.ElasticsearchIndexOperationsCompleted), + metricElasticsearchIndexOperationsMergeCurrent: newMetricElasticsearchIndexOperationsMergeCurrent(mbc.Metrics.ElasticsearchIndexOperationsMergeCurrent), metricElasticsearchIndexOperationsMergeDocsCount: newMetricElasticsearchIndexOperationsMergeDocsCount(mbc.Metrics.ElasticsearchIndexOperationsMergeDocsCount), metricElasticsearchIndexOperationsMergeSize: newMetricElasticsearchIndexOperationsMergeSize(mbc.Metrics.ElasticsearchIndexOperationsMergeSize), metricElasticsearchIndexOperationsTime: newMetricElasticsearchIndexOperationsTime(mbc.Metrics.ElasticsearchIndexOperationsTime), @@ -5562,6 +5615,7 @@ func (mb *MetricsBuilder) EmitForResource(rmo ...ResourceMetricsOption) { mb.metricElasticsearchIndexCacheSize.emit(ils.Metrics()) mb.metricElasticsearchIndexDocuments.emit(ils.Metrics()) mb.metricElasticsearchIndexOperationsCompleted.emit(ils.Metrics()) + mb.metricElasticsearchIndexOperationsMergeCurrent.emit(ils.Metrics()) mb.metricElasticsearchIndexOperationsMergeDocsCount.emit(ils.Metrics()) mb.metricElasticsearchIndexOperationsMergeSize.emit(ils.Metrics()) mb.metricElasticsearchIndexOperationsTime.emit(ils.Metrics()) @@ -5764,6 +5818,11 @@ func (mb *MetricsBuilder) RecordElasticsearchIndexOperationsCompletedDataPoint(t mb.metricElasticsearchIndexOperationsCompleted.recordDataPoint(mb.startTime, ts, val, operationAttributeValue.String(), indexAggregationTypeAttributeValue.String()) } +// RecordElasticsearchIndexOperationsMergeCurrentDataPoint adds a data point to elasticsearch.index.operations.merge.current metric. +func (mb *MetricsBuilder) RecordElasticsearchIndexOperationsMergeCurrentDataPoint(ts pcommon.Timestamp, val int64, indexAggregationTypeAttributeValue AttributeIndexAggregationType) { + mb.metricElasticsearchIndexOperationsMergeCurrent.recordDataPoint(mb.startTime, ts, val, indexAggregationTypeAttributeValue.String()) +} + // RecordElasticsearchIndexOperationsMergeDocsCountDataPoint adds a data point to elasticsearch.index.operations.merge.docs_count metric. func (mb *MetricsBuilder) RecordElasticsearchIndexOperationsMergeDocsCountDataPoint(ts pcommon.Timestamp, val int64, indexAggregationTypeAttributeValue AttributeIndexAggregationType) { mb.metricElasticsearchIndexOperationsMergeDocsCount.recordDataPoint(mb.startTime, ts, val, indexAggregationTypeAttributeValue.String()) diff --git a/receiver/elasticsearchreceiver/internal/metadata/generated_metrics_test.go b/receiver/elasticsearchreceiver/internal/metadata/generated_metrics_test.go index db9b994b6219..73e139fc2c6c 100644 --- a/receiver/elasticsearchreceiver/internal/metadata/generated_metrics_test.go +++ b/receiver/elasticsearchreceiver/internal/metadata/generated_metrics_test.go @@ -143,6 +143,9 @@ func TestMetricsBuilder(t *testing.T) { allMetricsCount++ mb.RecordElasticsearchIndexOperationsCompletedDataPoint(ts, 1, AttributeOperationIndex, AttributeIndexAggregationTypePrimaryShards) + allMetricsCount++ + mb.RecordElasticsearchIndexOperationsMergeCurrentDataPoint(ts, 1, AttributeIndexAggregationTypePrimaryShards) + allMetricsCount++ mb.RecordElasticsearchIndexOperationsMergeDocsCountDataPoint(ts, 1, AttributeIndexAggregationTypePrimaryShards) @@ -776,6 +779,21 @@ func TestMetricsBuilder(t *testing.T) { attrVal, ok = dp.Attributes().Get("aggregation") assert.True(t, ok) assert.EqualValues(t, "primary_shards", attrVal.Str()) + case "elasticsearch.index.operations.merge.current": + assert.False(t, validatedMetrics["elasticsearch.index.operations.merge.current"], "Found a duplicate in the metrics slice: elasticsearch.index.operations.merge.current") + validatedMetrics["elasticsearch.index.operations.merge.current"] = true + assert.Equal(t, pmetric.MetricTypeGauge, ms.At(i).Type()) + assert.Equal(t, 1, ms.At(i).Gauge().DataPoints().Len()) + assert.Equal(t, "The number of currently active segment merges", ms.At(i).Description()) + assert.Equal(t, "{merges}", ms.At(i).Unit()) + dp := ms.At(i).Gauge().DataPoints().At(0) + assert.Equal(t, start, dp.StartTimestamp()) + assert.Equal(t, ts, dp.Timestamp()) + assert.Equal(t, pmetric.NumberDataPointValueTypeInt, dp.ValueType()) + assert.Equal(t, int64(1), dp.IntValue()) + attrVal, ok := dp.Attributes().Get("aggregation") + assert.True(t, ok) + assert.EqualValues(t, "primary_shards", attrVal.Str()) case "elasticsearch.index.operations.merge.docs_count": assert.False(t, validatedMetrics["elasticsearch.index.operations.merge.docs_count"], "Found a duplicate in the metrics slice: elasticsearch.index.operations.merge.docs_count") validatedMetrics["elasticsearch.index.operations.merge.docs_count"] = true diff --git a/receiver/elasticsearchreceiver/internal/metadata/testdata/config.yaml b/receiver/elasticsearchreceiver/internal/metadata/testdata/config.yaml index ebb918407bab..3607bcd34b67 100644 --- a/receiver/elasticsearchreceiver/internal/metadata/testdata/config.yaml +++ b/receiver/elasticsearchreceiver/internal/metadata/testdata/config.yaml @@ -41,6 +41,8 @@ all_set: enabled: true elasticsearch.index.operations.completed: enabled: true + elasticsearch.index.operations.merge.current: + enabled: true elasticsearch.index.operations.merge.docs_count: enabled: true elasticsearch.index.operations.merge.size: @@ -234,6 +236,8 @@ none_set: enabled: false elasticsearch.index.operations.completed: enabled: false + elasticsearch.index.operations.merge.current: + enabled: false elasticsearch.index.operations.merge.docs_count: enabled: false elasticsearch.index.operations.merge.size: diff --git a/receiver/elasticsearchreceiver/internal/model/nodestats.go b/receiver/elasticsearchreceiver/internal/model/nodestats.go index 28923700f871..02a9d1e2cdb8 100644 --- a/receiver/elasticsearchreceiver/internal/model/nodestats.go +++ b/receiver/elasticsearchreceiver/internal/model/nodestats.go @@ -215,6 +215,7 @@ type BasicIndexOperation struct { type MergeOperations struct { BasicIndexOperation + Current int64 `json:"current"` TotalSizeInBytes int64 `json:"total_size_in_bytes"` TotalDocs int64 `json:"total_docs"` } diff --git a/receiver/elasticsearchreceiver/metadata.yaml b/receiver/elasticsearchreceiver/metadata.yaml index 9c50affa8cd2..12844da7fa5d 100644 --- a/receiver/elasticsearchreceiver/metadata.yaml +++ b/receiver/elasticsearchreceiver/metadata.yaml @@ -890,6 +890,13 @@ metrics: value_type: int attributes: [index_aggregation_type] enabled: false + elasticsearch.index.operations.merge.current: + description: The number of currently active segment merges + unit: "{merges}" + gauge: + value_type: int + attributes: [ index_aggregation_type ] + enabled: false elasticsearch.index.segments.count: description: Number of segments of an index. unit: "{segments}" diff --git a/receiver/elasticsearchreceiver/scraper.go b/receiver/elasticsearchreceiver/scraper.go index 43f32214913c..ad1807498d28 100644 --- a/receiver/elasticsearchreceiver/scraper.go +++ b/receiver/elasticsearchreceiver/scraper.go @@ -562,6 +562,10 @@ func (r *elasticsearchScraper) scrapeOneIndexMetrics(now pcommon.Timestamp, name now, stats.Total.MergeOperations.TotalDocs, metadata.AttributeIndexAggregationTypeTotal, ) + r.mb.RecordElasticsearchIndexOperationsMergeCurrentDataPoint( + now, stats.Total.MergeOperations.Current, metadata.AttributeIndexAggregationTypeTotal, + ) + r.mb.RecordElasticsearchIndexShardsSizeDataPoint( now, stats.Total.StoreInfo.SizeInBy, metadata.AttributeIndexAggregationTypeTotal, ) diff --git a/receiver/elasticsearchreceiver/scraper_test.go b/receiver/elasticsearchreceiver/scraper_test.go index e2404d15fd45..72e99092d9ba 100644 --- a/receiver/elasticsearchreceiver/scraper_test.go +++ b/receiver/elasticsearchreceiver/scraper_test.go @@ -43,6 +43,7 @@ func TestScraper(t *testing.T) { config.Metrics.ElasticsearchIndexOperationsMergeSize.Enabled = true config.Metrics.ElasticsearchIndexOperationsMergeDocsCount.Enabled = true + config.Metrics.ElasticsearchIndexOperationsMergeCurrent.Enabled = true config.Metrics.ElasticsearchIndexSegmentsCount.Enabled = true config.Metrics.ElasticsearchIndexSegmentsSize.Enabled = true config.Metrics.ElasticsearchIndexSegmentsMemory.Enabled = true @@ -100,6 +101,7 @@ func TestScraperNoIOStats(t *testing.T) { config.Metrics.ElasticsearchIndexOperationsMergeSize.Enabled = true config.Metrics.ElasticsearchIndexOperationsMergeDocsCount.Enabled = true + config.Metrics.ElasticsearchIndexOperationsMergeCurrent.Enabled = true config.Metrics.ElasticsearchIndexSegmentsCount.Enabled = true config.Metrics.ElasticsearchIndexSegmentsSize.Enabled = true config.Metrics.ElasticsearchIndexSegmentsMemory.Enabled = true diff --git a/receiver/elasticsearchreceiver/testdata/expected_metrics/full_linux.yaml b/receiver/elasticsearchreceiver/testdata/expected_metrics/full_linux.yaml index 97af08fca79a..8808a30e6802 100644 --- a/receiver/elasticsearchreceiver/testdata/expected_metrics/full_linux.yaml +++ b/receiver/elasticsearchreceiver/testdata/expected_metrics/full_linux.yaml @@ -505,6 +505,18 @@ resourceMetrics: timeUnixNano: "2000000" isMonotonic: true unit: '{operations}' + - description: The number of currently active segment merges + gauge: + dataPoints: + - asInt: "0" + attributes: + - key: aggregation + value: + stringValue: total + startTimeUnixNano: "1000000" + timeUnixNano: "2000000" + name: elasticsearch.index.operations.merge.current + unit: '{merges}' - description: The total number of documents in merge operations for an index. name: elasticsearch.index.operations.merge.docs_count sum: @@ -1285,6 +1297,18 @@ resourceMetrics: timeUnixNano: "2000000" isMonotonic: true unit: '{operations}' + - description: The number of currently active segment merges + gauge: + dataPoints: + - asInt: "0" + attributes: + - key: aggregation + value: + stringValue: total + startTimeUnixNano: "1000000" + timeUnixNano: "2000000" + name: elasticsearch.index.operations.merge.current + unit: '{merges}' - description: The total number of documents in merge operations for an index. name: elasticsearch.index.operations.merge.docs_count sum: @@ -2298,8 +2322,8 @@ resourceMetrics: aggregationTemporality: 2 dataPoints: - asInt: "15746158592" - startTimeUnixNano: "2000000" - timeUnixNano: "1000000" + startTimeUnixNano: "1000000" + timeUnixNano: "2000000" unit: By - description: The amount of disk space across all file stores for this node. name: elasticsearch.node.fs.disk.total @@ -2307,8 +2331,8 @@ resourceMetrics: aggregationTemporality: 2 dataPoints: - asInt: "67371577344" - startTimeUnixNano: "2000000" - timeUnixNano: "1000000" + startTimeUnixNano: "1000000" + timeUnixNano: "2000000" unit: By - description: The number of HTTP connections to the node. name: elasticsearch.node.http.connections diff --git a/receiver/elasticsearchreceiver/testdata/expected_metrics/full_other.yaml b/receiver/elasticsearchreceiver/testdata/expected_metrics/full_other.yaml index 5c1f5d380225..6744af363336 100644 --- a/receiver/elasticsearchreceiver/testdata/expected_metrics/full_other.yaml +++ b/receiver/elasticsearchreceiver/testdata/expected_metrics/full_other.yaml @@ -505,6 +505,18 @@ resourceMetrics: timeUnixNano: "2000000" isMonotonic: true unit: '{operations}' + - description: The number of currently active segment merges + gauge: + dataPoints: + - asInt: "0" + attributes: + - key: aggregation + value: + stringValue: total + startTimeUnixNano: "1000000" + timeUnixNano: "2000000" + name: elasticsearch.index.operations.merge.current + unit: '{merges}' - description: The total number of documents in merge operations for an index. name: elasticsearch.index.operations.merge.docs_count sum: @@ -1285,6 +1297,18 @@ resourceMetrics: timeUnixNano: "2000000" isMonotonic: true unit: '{operations}' + - description: The number of currently active segment merges + gauge: + dataPoints: + - asInt: "0" + attributes: + - key: aggregation + value: + stringValue: total + startTimeUnixNano: "1000000" + timeUnixNano: "2000000" + name: elasticsearch.index.operations.merge.current + unit: '{merges}' - description: The total number of documents in merge operations for an index. name: elasticsearch.index.operations.merge.docs_count sum: @@ -2280,8 +2304,8 @@ resourceMetrics: aggregationTemporality: 2 dataPoints: - asInt: "15746158592" - startTimeUnixNano: "2000000" - timeUnixNano: "1000000" + startTimeUnixNano: "1000000" + timeUnixNano: "2000000" unit: By - description: The amount of disk space across all file stores for this node. name: elasticsearch.node.fs.disk.total @@ -2289,8 +2313,8 @@ resourceMetrics: aggregationTemporality: 2 dataPoints: - asInt: "67371577344" - startTimeUnixNano: "2000000" - timeUnixNano: "1000000" + startTimeUnixNano: "1000000" + timeUnixNano: "2000000" unit: By - description: The number of HTTP connections to the node. name: elasticsearch.node.http.connections