From 350f4bdf5385832cebcc5f3ef8f4205a8f820561 Mon Sep 17 00:00:00 2001 From: Carson Ip Date: Wed, 25 Sep 2024 10:59:55 +0100 Subject: [PATCH] [exporter/elasticsearch] Revert TSDB array dimension workaround for metrics OTel mode (#35291) **Description:** Remove the workaround in #35009 to stringify array dimensions as the limitation has been lifted in Elasticsearch in https://github.com/elastic/elasticsearch/issues/110387 This change is not breaking as array will be coerced as string, meaning that there will be no indexing error. Additionally, the changes are acceptable as OTel mapping mode explicitly marked as in development. **Link to tracking Issue:** **Testing:** **Documentation:** --------- Co-authored-by: Vishal Raj --- ...ter_revert-array-dimension-workaround.yaml | 27 ++++++++++ .../elasticsearchexporter/exporter_test.go | 7 ++- exporter/elasticsearchexporter/model.go | 52 ++++++------------- 3 files changed, 46 insertions(+), 40 deletions(-) create mode 100644 .chloggen/elasticsearchexporter_revert-array-dimension-workaround.yaml diff --git a/.chloggen/elasticsearchexporter_revert-array-dimension-workaround.yaml b/.chloggen/elasticsearchexporter_revert-array-dimension-workaround.yaml new file mode 100644 index 000000000000..24b331afe0cf --- /dev/null +++ b/.chloggen/elasticsearchexporter_revert-array-dimension-workaround.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: elasticsearchexporter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Revert TSDB array dimension workaround for metrics OTel mode + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [35291] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: Remove the workaround to stringify array dimensions as the limitation has been lifted in Elasticsearch v8.16.0. + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] diff --git a/exporter/elasticsearchexporter/exporter_test.go b/exporter/elasticsearchexporter/exporter_test.go index 1f45b456a9b8..65894511b626 100644 --- a/exporter/elasticsearchexporter/exporter_test.go +++ b/exporter/elasticsearchexporter/exporter_test.go @@ -942,10 +942,9 @@ func TestExporterMetrics(t *testing.T) { assert.Len(t, rec.Items(), 1) doc := rec.Items()[0].Document - // Workaround TSDB limitation by stringifying array values - assert.Equal(t, `{"some.record.attribute":"[\"foo\",\"bar\"]"}`, gjson.GetBytes(doc, `attributes`).Raw) - assert.Equal(t, `{"some.scope.attribute":"[\"foo\",\"bar\"]"}`, gjson.GetBytes(doc, `scope.attributes`).Raw) - assert.Equal(t, `{"some.resource.attribute":"[\"foo\",\"bar\"]"}`, gjson.GetBytes(doc, `resource.attributes`).Raw) + assert.Equal(t, `{"some.record.attribute":["foo","bar"]}`, gjson.GetBytes(doc, `attributes`).Raw) + assert.Equal(t, `{"some.scope.attribute":["foo","bar"]}`, gjson.GetBytes(doc, `scope.attributes`).Raw) + assert.Equal(t, `{"some.resource.attribute":["foo","bar"]}`, gjson.GetBytes(doc, `resource.attributes`).Raw) }) t.Run("publish summary", func(t *testing.T) { diff --git a/exporter/elasticsearchexporter/model.go b/exporter/elasticsearchexporter/model.go index 3280042a5fe7..73b583a93330 100644 --- a/exporter/elasticsearchexporter/model.go +++ b/exporter/elasticsearchexporter/model.go @@ -152,9 +152,9 @@ func (m *encodeModel) encodeLogOTelMode(resource pcommon.Resource, resourceSchem document.AddInt("severity_number", int64(record.SeverityNumber())) document.AddInt("dropped_attributes_count", int64(record.DroppedAttributesCount())) - m.encodeAttributesOTelMode(&document, record.Attributes(), false) - m.encodeResourceOTelMode(&document, resource, resourceSchemaURL, false) - m.encodeScopeOTelMode(&document, scope, scopeSchemaURL, false) + m.encodeAttributesOTelMode(&document, record.Attributes()) + m.encodeResourceOTelMode(&document, resource, resourceSchemaURL) + m.encodeScopeOTelMode(&document, scope, scopeSchemaURL) // Body setOTelLogBody(&document, record.Body()) @@ -297,9 +297,9 @@ func (m *encodeModel) upsertMetricDataPointValueOTelMode(documents map[uint32]ob } document.AddString("unit", metric.Unit()) - m.encodeAttributesOTelMode(&document, dp.Attributes(), true) - m.encodeResourceOTelMode(&document, resource, resourceSchemaURL, true) - m.encodeScopeOTelMode(&document, scope, scopeSchemaURL, true) + m.encodeAttributesOTelMode(&document, dp.Attributes()) + m.encodeResourceOTelMode(&document, resource, resourceSchemaURL) + m.encodeScopeOTelMode(&document, scope, scopeSchemaURL) } switch value.Type() { @@ -477,7 +477,7 @@ func (dp numberDataPoint) DynamicTemplate(metric pmetric.Metric) string { var errInvalidNumberDataPoint = errors.New("invalid number data point") -func (m *encodeModel) encodeResourceOTelMode(document *objmodel.Document, resource pcommon.Resource, resourceSchemaURL string, stringifyArrayValues bool) { +func (m *encodeModel) encodeResourceOTelMode(document *objmodel.Document, resource pcommon.Resource, resourceSchemaURL string) { resourceMapVal := pcommon.NewValueMap() resourceMap := resourceMapVal.Map() if resourceSchemaURL != "" { @@ -493,13 +493,11 @@ func (m *encodeModel) encodeResourceOTelMode(document *objmodel.Document, resour } return false }) - if stringifyArrayValues { - mapStringifyArrayValues(resourceAttrMap) - } + document.Add("resource", objmodel.ValueFromAttribute(resourceMapVal)) } -func (m *encodeModel) encodeScopeOTelMode(document *objmodel.Document, scope pcommon.InstrumentationScope, scopeSchemaURL string, stringifyArrayValues bool) { +func (m *encodeModel) encodeScopeOTelMode(document *objmodel.Document, scope pcommon.InstrumentationScope, scopeSchemaURL string) { scopeMapVal := pcommon.NewValueMap() scopeMap := scopeMapVal.Map() if scope.Name() != "" { @@ -521,13 +519,10 @@ func (m *encodeModel) encodeScopeOTelMode(document *objmodel.Document, scope pco } return false }) - if stringifyArrayValues { - mapStringifyArrayValues(scopeAttrMap) - } document.Add("scope", objmodel.ValueFromAttribute(scopeMapVal)) } -func (m *encodeModel) encodeAttributesOTelMode(document *objmodel.Document, attributeMap pcommon.Map, stringifyArrayValues bool) { +func (m *encodeModel) encodeAttributesOTelMode(document *objmodel.Document, attributeMap pcommon.Map) { attrsCopy := pcommon.NewMap() // Copy to avoid mutating original map attributeMap.CopyTo(attrsCopy) attrsCopy.RemoveIf(func(key string, val pcommon.Value) bool { @@ -541,24 +536,9 @@ func (m *encodeModel) encodeAttributesOTelMode(document *objmodel.Document, attr } return false }) - if stringifyArrayValues { - mapStringifyArrayValues(attrsCopy) - } document.AddAttributes("attributes", attrsCopy) } -// mapStringifyArrayValues replaces all slice values within an attribute map to their string representation. -// It is useful to workaround Elasticsearch TSDB not supporting arrays as dimensions. -// See https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/35004 -func mapStringifyArrayValues(m pcommon.Map) { - m.Range(func(_ string, v pcommon.Value) bool { - if v.Type() == pcommon.ValueTypeSlice { - v.SetStr(v.AsString()) - } - return true - }) -} - func (m *encodeModel) encodeSpan(resource pcommon.Resource, resourceSchemaURL string, span ptrace.Span, scope pcommon.InstrumentationScope, scopeSchemaURL string) ([]byte, error) { var document objmodel.Document switch m.mode { @@ -584,7 +564,7 @@ func (m *encodeModel) encodeSpanOTelMode(resource pcommon.Resource, resourceSche document.AddString("kind", span.Kind().String()) document.AddInt("duration", int64(span.EndTimestamp()-span.StartTimestamp())) - m.encodeAttributesOTelMode(&document, span.Attributes(), false) + m.encodeAttributesOTelMode(&document, span.Attributes()) document.AddInt("dropped_attributes_count", int64(span.DroppedAttributesCount())) document.AddInt("dropped_events_count", int64(span.DroppedEventsCount())) @@ -608,8 +588,8 @@ func (m *encodeModel) encodeSpanOTelMode(resource pcommon.Resource, resourceSche document.AddString("status.message", span.Status().Message()) document.AddString("status.code", span.Status().Code().String()) - m.encodeResourceOTelMode(&document, resource, resourceSchemaURL, false) - m.encodeScopeOTelMode(&document, scope, scopeSchemaURL, false) + m.encodeResourceOTelMode(&document, resource, resourceSchemaURL) + m.encodeScopeOTelMode(&document, scope, scopeSchemaURL) return document } @@ -647,9 +627,9 @@ func (m *encodeModel) encodeSpanEvent(resource pcommon.Resource, resourceSchemaU document.AddTraceID("trace_id", span.TraceID()) document.AddInt("dropped_attributes_count", int64(spanEvent.DroppedAttributesCount())) - m.encodeAttributesOTelMode(&document, spanEvent.Attributes(), false) - m.encodeResourceOTelMode(&document, resource, resourceSchemaURL, false) - m.encodeScopeOTelMode(&document, scope, scopeSchemaURL, false) + m.encodeAttributesOTelMode(&document, spanEvent.Attributes()) + m.encodeResourceOTelMode(&document, resource, resourceSchemaURL) + m.encodeScopeOTelMode(&document, scope, scopeSchemaURL) return &document }