From 0217d55cbaf5e9fd5486c897b84127ca01a870de Mon Sep 17 00:00:00 2001 From: Raphael Koh <72152843+kohrapha@users.noreply.github.com> Date: Fri, 13 Nov 2020 08:32:06 -0500 Subject: [PATCH] [awsemfexporter] Implement metric filtering and dimension setting (#1503) * Create MetricDeclaration struct * Implement dimension dedup logic when adding rolledup dimensions (#6) * Implement dimension dedup logic when adding rolledup dimensions * Remove duplicated dimensions in metric declaration * Create benchmark for filtering with metric declarations and use assertCwMeasurementEqual for tests * Move helper test code to the top of file * Update dimRollup test * Aggregate dimensions and perform dedup * Minor code change * Fix test failure from merging new changes --- exporter/awsemfexporter/README.md | 10 +- exporter/awsemfexporter/config.go | 2 + exporter/awsemfexporter/config_test.go | 2 + exporter/awsemfexporter/emf_exporter.go | 15 +- exporter/awsemfexporter/emf_exporter_test.go | 57 + exporter/awsemfexporter/factory.go | 1 + exporter/awsemfexporter/metric_declaration.go | 171 ++ .../awsemfexporter/metric_declaration_test.go | 388 +++++ exporter/awsemfexporter/metric_translator.go | 140 +- .../awsemfexporter/metric_translator_test.go | 1397 ++++++++++++----- 10 files changed, 1718 insertions(+), 465 deletions(-) create mode 100644 exporter/awsemfexporter/metric_declaration.go create mode 100644 exporter/awsemfexporter/metric_declaration_test.go diff --git a/exporter/awsemfexporter/README.md b/exporter/awsemfexporter/README.md index 737dba1cdcc9..6405f2571c63 100644 --- a/exporter/awsemfexporter/README.md +++ b/exporter/awsemfexporter/README.md @@ -25,7 +25,15 @@ The following exporter configuration parameters are supported. | `max_retries` | Maximum number of retries before abandoning an attempt to post data. | 1 | | `dimension_rollup_option`| DimensionRollupOption is the option for metrics dimension rollup. Three options are available. |"ZeroAndSingleDimensionRollup" (Enable both zero dimension rollup and single dimension rollup)| | `resource_to_telemetry_conversion` | "resource_to_telemetry_conversion" is the option for converting resource attributes to telemetry attributes. It has only one config onption- `enabled`. For metrics, if `enabled=true`, all the resource attributes will be converted to metric labels by default. See `Resource Attributes to Metric Labels` section below for examples. | `enabled=false` | - +| [`metric_declarations`](#metric_declaration) | List of rules for filtering exported metrics and their dimensions. | [ ] | + +### +A metric_declaration section characterizes a rule to be used to set dimensions for exported metrics, filtered by the incoming metrics' metric names. +| Name | Description | Default | +| :---------------- | :--------------------------------------------------------------------- | ------- | +| `dimensions` | List of dimension sets to be exported. | [[ ]] | +| `metric_name_selectors` | List of regex strings to filter metric names by. | [ ] | + ## AWS Credential Configuration diff --git a/exporter/awsemfexporter/config.go b/exporter/awsemfexporter/config.go index 8c3aeee9015b..a322022ab62c 100644 --- a/exporter/awsemfexporter/config.go +++ b/exporter/awsemfexporter/config.go @@ -55,6 +55,8 @@ type Config struct { // "SingleDimensionRollupOnly" - Enable single dimension rollup // "NoDimensionRollup" - No dimension rollup (only keep original metrics which contain all dimensions) DimensionRollupOption string `mapstructure:"dimension_rollup_option"` + // MetricDeclarations is a list of rules to be used to set dimensions for exported metrics. + MetricDeclarations []*MetricDeclaration `mapstructure:"metric_declarations"` // ResourceToTelemetrySettings is the option for converting resource attrihutes to telemetry attributes. // "Enabled" - A boolean field to enable/disable this option. Default is `false`. diff --git a/exporter/awsemfexporter/config_test.go b/exporter/awsemfexporter/config_test.go index 4923f1648ae6..4347ec1eef18 100644 --- a/exporter/awsemfexporter/config_test.go +++ b/exporter/awsemfexporter/config_test.go @@ -58,6 +58,7 @@ func TestLoadConfig(t *testing.T) { Region: "us-west-2", RoleARN: "arn:aws:iam::123456789:role/monitoring-EKS-NodeInstanceRole", DimensionRollupOption: "ZeroAndSingleDimensionRollup", + MetricDeclarations: []*MetricDeclaration{}, }) r2 := cfg.Exporters["awsemf/resource_attr_to_label"].(*Config) @@ -75,5 +76,6 @@ func TestLoadConfig(t *testing.T) { RoleARN: "", DimensionRollupOption: "ZeroAndSingleDimensionRollup", ResourceToTelemetrySettings: exporterhelper.ResourceToTelemetrySettings{Enabled: true}, + MetricDeclarations: []*MetricDeclaration{}, }) } diff --git a/exporter/awsemfexporter/emf_exporter.go b/exporter/awsemfexporter/emf_exporter.go index d418b3465cd8..5cbbaa61d44a 100644 --- a/exporter/awsemfexporter/emf_exporter.go +++ b/exporter/awsemfexporter/emf_exporter.go @@ -67,6 +67,19 @@ func New( svcStructuredLog := NewCloudWatchLogsClient(logger, awsConfig, session) collectorIdentifier, _ := uuid.NewRandom() + // Initialize metric declarations and filter out invalid ones + emfConfig := config.(*Config) + var validDeclarations []*MetricDeclaration + for _, declaration := range emfConfig.MetricDeclarations { + err := declaration.Init(logger) + if err != nil { + logger.Warn("Dropped metric declaration. Error: " + err.Error() + ".") + } else { + validDeclarations = append(validDeclarations, declaration) + } + } + emfConfig.MetricDeclarations = validDeclarations + emfExporter := &emfExporter{ svcStructuredLog: svcStructuredLog, config: config, @@ -212,7 +225,7 @@ func generateLogEventFromMetric(metric pdata.Metrics, config *Config) ([]*LogEve cwMetricLists = append(cwMetricLists, cwm...) } - return TranslateCWMetricToEMF(cwMetricLists), totalDroppedMetrics, namespace + return TranslateCWMetricToEMF(cwMetricLists, config.logger), totalDroppedMetrics, namespace } func wrapErrorIfBadRequest(err *error) error { diff --git a/exporter/awsemfexporter/emf_exporter_test.go b/exporter/awsemfexporter/emf_exporter_test.go index 6dde5e5bc5ab..db708626afe0 100644 --- a/exporter/awsemfexporter/emf_exporter_test.go +++ b/exporter/awsemfexporter/emf_exporter_test.go @@ -32,6 +32,8 @@ import ( "go.opentelemetry.io/collector/consumer/consumererror" "go.opentelemetry.io/collector/translator/internaldata" "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" ) func init() { @@ -236,6 +238,61 @@ func TestNewExporterWithoutConfig(t *testing.T) { assert.NotNil(t, expCfg.logger) } +func TestNewExporterWithMetricDeclarations(t *testing.T) { + factory := NewFactory() + expCfg := factory.CreateDefaultConfig().(*Config) + expCfg.Region = "us-west-2" + expCfg.MaxRetries = defaultRetryCount + expCfg.LogGroupName = "test-logGroupName" + expCfg.LogStreamName = "test-logStreamName" + mds := []*MetricDeclaration{ + { + MetricNameSelectors: []string{"a", "b"}, + }, + { + MetricNameSelectors: []string{"c", "d"}, + }, + { + MetricNameSelectors: nil, + }, + { + Dimensions: [][]string{ + {"foo"}, + {"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k"}, + }, + MetricNameSelectors: []string{"a"}, + }, + } + expCfg.MetricDeclarations = mds + + obs, logs := observer.New(zap.WarnLevel) + logger := zap.New(obs) + exp, err := New(expCfg, component.ExporterCreateParams{Logger: logger}) + assert.Nil(t, err) + assert.NotNil(t, exp) + + emfExporter := exp.(*emfExporter) + config := emfExporter.config.(*Config) + // Invalid metric declaration should be filtered out + assert.Equal(t, 3, len(config.MetricDeclarations)) + // Invalid dimensions (> 10 dims) should be filtered out + assert.Equal(t, 1, len(config.MetricDeclarations[2].Dimensions)) + + // Test output warning logs + expectedLogs := []observer.LoggedEntry{ + { + Entry: zapcore.Entry{Level: zap.WarnLevel, Message: "Dropped metric declaration. Error: invalid metric declaration: no metric name selectors defined."}, + Context: []zapcore.Field{}, + }, + { + Entry: zapcore.Entry{Level: zap.WarnLevel, Message: "Dropped dimension set: > 10 dimensions specified."}, + Context: []zapcore.Field{zap.String("dimensions", "a,b,c,d,e,f,g,h,i,j,k")}, + }, + } + assert.Equal(t, 2, logs.Len()) + assert.Equal(t, expectedLogs, logs.AllUntimed()) +} + func TestNewExporterWithoutSession(t *testing.T) { exp, err := New(nil, component.ExporterCreateParams{Logger: zap.NewNop()}) assert.NotNil(t, err) diff --git a/exporter/awsemfexporter/factory.go b/exporter/awsemfexporter/factory.go index 117b0e6f86d8..6f6050844136 100644 --- a/exporter/awsemfexporter/factory.go +++ b/exporter/awsemfexporter/factory.go @@ -53,6 +53,7 @@ func createDefaultConfig() configmodels.Exporter { Region: "", RoleARN: "", DimensionRollupOption: "ZeroAndSingleDimensionRollup", + MetricDeclarations: make([]*MetricDeclaration, 0), logger: nil, } } diff --git a/exporter/awsemfexporter/metric_declaration.go b/exporter/awsemfexporter/metric_declaration.go new file mode 100644 index 000000000000..b4b7aa034ffc --- /dev/null +++ b/exporter/awsemfexporter/metric_declaration.go @@ -0,0 +1,171 @@ +// Copyright 2020, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package awsemfexporter + +import ( + "errors" + "regexp" + "sort" + "strings" + + "go.opentelemetry.io/collector/consumer/pdata" + "go.uber.org/zap" +) + +// MetricDeclaration characterizes a rule to be used to set dimensions for certain +// incoming metrics, filtered by their metric names. +type MetricDeclaration struct { + // Dimensions is a list of dimension sets (which are lists of dimension names) to be + // included in exported metrics. If the metric does not contain any of the specified + // dimensions, the metric would be dropped (will only show up in logs). + Dimensions [][]string `mapstructure:"dimensions"` + // MetricNameSelectors is a list of regex strings to be matched against metric names + // to determine which metrics should be included with this metric declaration rule. + MetricNameSelectors []string `mapstructure:"metric_name_selectors"` + + // metricRegexList is a list of compiled regexes for metric name selectors. + metricRegexList []*regexp.Regexp +} + +// Remove duplicated entries from dimension set. +func dedupDimensionSet(dimensions []string) (deduped []string, hasDuplicate bool) { + seen := make(map[string]bool, len(dimensions)) + for _, v := range dimensions { + seen[v] = true + } + hasDuplicate = (len(seen) < len(dimensions)) + if !hasDuplicate { + deduped = dimensions + return + } + deduped = make([]string, len(seen)) + idx := 0 + for dim := range seen { + deduped[idx] = dim + idx++ + } + return +} + +// Init initializes the MetricDeclaration struct. Performs validation and compiles +// regex strings. Dimensions are deduped and sorted. +func (m *MetricDeclaration) Init(logger *zap.Logger) (err error) { + // Return error if no metric name selectors are defined + if len(m.MetricNameSelectors) == 0 { + return errors.New("invalid metric declaration: no metric name selectors defined") + } + + // Filter out duplicate dimension sets and those with more than 10 elements + validDims := make([][]string, 0, len(m.Dimensions)) + seen := make(map[string]bool, len(m.Dimensions)) + for _, dimSet := range m.Dimensions { + concatenatedDims := strings.Join(dimSet, ",") + if len(dimSet) > 10 { + logger.Warn("Dropped dimension set: > 10 dimensions specified.", zap.String("dimensions", concatenatedDims)) + continue + } + + // Dedup dimensions within dimension set + dedupedDims, hasDuplicate := dedupDimensionSet(dimSet) + if hasDuplicate { + logger.Warn("Removed duplicates from dimension set.", zap.String("dimensions", concatenatedDims)) + } + + // Sort dimensions + sort.Strings(dedupedDims) + + // Dedup dimension sets + key := strings.Join(dedupedDims, ",") + if _, ok := seen[key]; ok { + logger.Warn("Dropped dimension set: duplicated dimension set.", zap.String("dimensions", concatenatedDims)) + continue + } + seen[key] = true + validDims = append(validDims, dedupedDims) + } + m.Dimensions = validDims + + m.metricRegexList = make([]*regexp.Regexp, len(m.MetricNameSelectors)) + for i, selector := range m.MetricNameSelectors { + m.metricRegexList[i] = regexp.MustCompile(selector) + } + return +} + +// Matches returns true if the given OTLP Metric's name matches any of the Metric +// Declaration's metric name selectors. +func (m *MetricDeclaration) Matches(metric *pdata.Metric) bool { + for _, regex := range m.metricRegexList { + if regex.MatchString(metric.Name()) { + return true + } + } + return false +} + +// ExtractDimensions extracts dimensions within the MetricDeclaration that only +// contains labels found in `labels`. Returned order of dimensions are preserved. +func (m *MetricDeclaration) ExtractDimensions(labels map[string]string) (dimensions [][]string) { + for _, dimensionSet := range m.Dimensions { + if len(dimensionSet) == 0 { + continue + } + includeSet := true + for _, dim := range dimensionSet { + if _, ok := labels[dim]; !ok { + includeSet = false + break + } + } + if includeSet { + dimensions = append(dimensions, dimensionSet) + } + } + return +} + +// processMetricDeclarations processes a list of MetricDeclarations and creates a +// list of dimension sets that matches the given `metric`. This list is then aggregated +// together with the rolled-up dimensions. Returned dimension sets +// are deduped and the dimensions in each dimension set are sorted. +// Prerequisite: +// 1. metricDeclarations' dimensions are sorted. +func processMetricDeclarations(metricDeclarations []*MetricDeclaration, metric *pdata.Metric, + labels map[string]string, rolledUpDimensions [][]string) (dimensions [][]string) { + seen := make(map[string]bool) + addDimSet := func(dimSet []string) { + key := strings.Join(dimSet, ",") + // Only add dimension set if not a duplicate + if _, ok := seen[key]; !ok { + dimensions = append(dimensions, dimSet) + seen[key] = true + } + } + // Extract and append dimensions from metric declarations + for _, m := range metricDeclarations { + if m.Matches(metric) { + extractedDims := m.ExtractDimensions(labels) + for _, dimSet := range extractedDims { + addDimSet(dimSet) + } + } + } + // Add on rolled-up dimensions + for _, dimSet := range rolledUpDimensions { + sort.Strings(dimSet) + addDimSet(dimSet) + } + return +} diff --git a/exporter/awsemfexporter/metric_declaration_test.go b/exporter/awsemfexporter/metric_declaration_test.go new file mode 100644 index 000000000000..035545e04f15 --- /dev/null +++ b/exporter/awsemfexporter/metric_declaration_test.go @@ -0,0 +1,388 @@ +// Copyright 2020, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package awsemfexporter + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "go.opentelemetry.io/collector/consumer/pdata" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" +) + +func TestInit(t *testing.T) { + logger := zap.NewNop() + t.Run("no dimensions", func(t *testing.T) { + m := &MetricDeclaration{ + MetricNameSelectors: []string{"a", "b", "aa"}, + } + err := m.Init(logger) + assert.Nil(t, err) + assert.Equal(t, 3, len(m.metricRegexList)) + }) + + t.Run("with dimensions", func(t *testing.T) { + m := &MetricDeclaration{ + Dimensions: [][]string{ + {"foo"}, + {"a", "b", "c", "d", "e", "f", "g", "h", "i", "j"}, + }, + MetricNameSelectors: []string{"a.*", "b$", "aa+"}, + } + err := m.Init(logger) + assert.Nil(t, err) + assert.Equal(t, 3, len(m.metricRegexList)) + assert.Equal(t, 2, len(m.Dimensions)) + }) + + // Test removal of dimension sets with more than 10 elements + t.Run("dimension set with more than 10 elements", func(t *testing.T) { + m := &MetricDeclaration{ + Dimensions: [][]string{ + {"foo"}, + {"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k"}, + }, + MetricNameSelectors: []string{"a.*", "b$", "aa+"}, + } + obs, logs := observer.New(zap.WarnLevel) + obsLogger := zap.New(obs) + err := m.Init(obsLogger) + assert.Nil(t, err) + assert.Equal(t, 3, len(m.metricRegexList)) + assert.Equal(t, 1, len(m.Dimensions)) + // Check logged warning message + expectedLogs := []observer.LoggedEntry{{ + Entry: zapcore.Entry{Level: zap.WarnLevel, Message: "Dropped dimension set: > 10 dimensions specified."}, + Context: []zapcore.Field{zap.String("dimensions", "a,b,c,d,e,f,g,h,i,j,k")}, + }} + assert.Equal(t, 1, logs.Len()) + assert.Equal(t, expectedLogs, logs.AllUntimed()) + }) + + // Test removal of duplicate dimensions within a dimension set, and removal of + // duplicate dimension sets + t.Run("remove duplicate dimensions and dimension sets", func(t *testing.T) { + m := &MetricDeclaration{ + Dimensions: [][]string{ + {"a", "c", "b", "c"}, + {"c", "b", "a"}, + }, + MetricNameSelectors: []string{"a.*", "b$", "aa+"}, + } + obs, logs := observer.New(zap.WarnLevel) + obsLogger := zap.New(obs) + err := m.Init(obsLogger) + assert.Nil(t, err) + assert.Equal(t, 1, len(m.Dimensions)) + assert.Equal(t, []string{"a", "b", "c"}, m.Dimensions[0]) + // Check logged warning message + expectedLogs := []observer.LoggedEntry{ + { + Entry: zapcore.Entry{Level: zap.WarnLevel, Message: "Removed duplicates from dimension set."}, + Context: []zapcore.Field{zap.String("dimensions", "a,c,b,c")}, + }, + { + Entry: zapcore.Entry{Level: zap.WarnLevel, Message: "Dropped dimension set: duplicated dimension set."}, + Context: []zapcore.Field{zap.String("dimensions", "c,b,a")}, + }, + } + assert.Equal(t, 2, logs.Len()) + assert.Equal(t, expectedLogs, logs.AllUntimed()) + }) + + // Test invalid metric declaration + t.Run("invalid metric declaration", func(t *testing.T) { + m := &MetricDeclaration{} + err := m.Init(logger) + assert.NotNil(t, err) + assert.EqualError(t, err, "invalid metric declaration: no metric name selectors defined") + }) +} + +func TestMatches(t *testing.T) { + m := &MetricDeclaration{ + MetricNameSelectors: []string{"^a+$", "^b.*$", "^ac+a$"}, + } + logger := zap.NewNop() + err := m.Init(logger) + assert.Nil(t, err) + + metric := pdata.NewMetric() + metric.InitEmpty() + metric.SetName("a") + assert.True(t, m.Matches(&metric)) + + metric.SetName("aa") + assert.True(t, m.Matches(&metric)) + + metric.SetName("aaaa") + assert.True(t, m.Matches(&metric)) + + metric.SetName("aaab") + assert.False(t, m.Matches(&metric)) + + metric.SetName("b") + assert.True(t, m.Matches(&metric)) + + metric.SetName("ba") + assert.True(t, m.Matches(&metric)) + + metric.SetName("c") + assert.False(t, m.Matches(&metric)) + + metric.SetName("aca") + assert.True(t, m.Matches(&metric)) + + metric.SetName("accca") + assert.True(t, m.Matches(&metric)) +} + +func TestExtractDimensions(t *testing.T) { + testCases := []struct { + testName string + dimensions [][]string + labels map[string]string + extractedDimensions [][]string + }{ + { + "matches single dimension set exactly", + [][]string{{"a", "b"}}, + map[string]string{ + "a": "foo", + "b": "bar", + }, + [][]string{{"a", "b"}}, + }, + { + "matches subset of single dimension set", + [][]string{{"a"}}, + map[string]string{ + "a": "foo", + "b": "bar", + }, + [][]string{{"a"}}, + }, + { + "does not match single dimension set", + [][]string{{"a", "b"}}, + map[string]string{ + "b": "bar", + }, + nil, + }, + { + "matches multiple dimension sets", + [][]string{{"a", "b"}, {"a"}}, + map[string]string{ + "a": "foo", + "b": "bar", + }, + [][]string{{"a", "b"}, {"a"}}, + }, + { + "matches one of multiple dimension sets", + [][]string{{"a", "b"}, {"a"}}, + map[string]string{ + "a": "foo", + }, + [][]string{{"a"}}, + }, + { + "no dimensions", + [][]string{}, + map[string]string{ + "a": "foo", + }, + nil, + }, + { + "empty dimension set", + [][]string{{}}, + map[string]string{ + "a": "foo", + }, + nil, + }, + } + logger := zap.NewNop() + + for _, tc := range testCases { + m := MetricDeclaration{ + Dimensions: tc.dimensions, + MetricNameSelectors: []string{"foo"}, + } + t.Run(tc.testName, func(t *testing.T) { + err := m.Init(logger) + assert.Nil(t, err) + dimensions := m.ExtractDimensions(tc.labels) + assert.Equal(t, tc.extractedDimensions, dimensions) + }) + } +} + +func TestProcessMetricDeclarations(t *testing.T) { + metricDeclarations := []*MetricDeclaration{ + { + Dimensions: [][]string{{"dim1", "dim2"}}, + MetricNameSelectors: []string{"a", "b", "c"}, + }, + { + Dimensions: [][]string{{"dim1"}}, + MetricNameSelectors: []string{"aa", "b"}, + }, + { + Dimensions: [][]string{{"dim2", "dim1"}, {"dim1"}}, + MetricNameSelectors: []string{"a"}, + }, + } + logger := zap.NewNop() + for _, m := range metricDeclarations { + err := m.Init(logger) + assert.Nil(t, err) + } + testCases := []struct { + testName string + metricName string + labels map[string]string + rollUpDims [][]string + expectedDims [][]string + }{ + { + "Matching single declaration", + "c", + map[string]string{ + "dim1": "foo", + "dim2": "bar", + }, + nil, + [][]string{ + {"dim1", "dim2"}, + }, + }, + { + "Match single dimension set", + "a", + map[string]string{ + "dim1": "foo", + }, + nil, + [][]string{ + {"dim1"}, + }, + }, + { + "Match single dimension set w/ rolled-up dims", + "a", + map[string]string{ + "dim1": "foo", + "dim3": "car", + }, + [][]string{{"dim1"}, {"dim3"}}, + [][]string{ + {"dim1"}, + {"dim3"}, + }, + }, + { + "Matching multiple declarations", + "b", + map[string]string{ + "dim1": "foo", + "dim2": "bar", + }, + nil, + [][]string{ + {"dim1", "dim2"}, + {"dim1"}, + }, + }, + { + "Matching multiple declarations w/ duplicate", + "a", + map[string]string{ + "dim1": "foo", + "dim2": "bar", + }, + nil, + [][]string{ + {"dim1", "dim2"}, + {"dim1"}, + }, + }, + { + "Matching multiple declarations w/ duplicate + rolled-up dims", + "a", + map[string]string{ + "dim1": "foo", + "dim2": "bar", + "dim3": "car", + }, + [][]string{{"dim2", "dim1"}, {"dim3"}}, + [][]string{ + {"dim1", "dim2"}, + {"dim1"}, + {"dim3"}, + }, + }, + { + "No matching dimension set", + "a", + map[string]string{ + "dim2": "bar", + }, + nil, + nil, + }, + { + "No matching dimension set w/ rolled-up dims", + "a", + map[string]string{ + "dim2": "bar", + }, + [][]string{{"dim2"}}, + [][]string{{"dim2"}}, + }, + { + "No matching metric name", + "c", + map[string]string{ + "dim1": "foo", + }, + nil, + nil, + }, + { + "No matching metric name w/ rolled-up dims", + "c", + map[string]string{ + "dim1": "foo", + }, + [][]string{{"dim1"}}, + [][]string{{"dim1"}}, + }, + } + + for _, tc := range testCases { + metric := pdata.NewMetric() + metric.InitEmpty() + metric.SetName(tc.metricName) + t.Run(tc.testName, func(t *testing.T) { + dimensions := processMetricDeclarations(metricDeclarations, &metric, tc.labels, tc.rollUpDims) + assert.Equal(t, tc.expectedDims, dimensions) + }) + } +} diff --git a/exporter/awsemfexporter/metric_translator.go b/exporter/awsemfexporter/metric_translator.go index 0003122d89cc..36b3b24c6b79 100644 --- a/exporter/awsemfexporter/metric_translator.go +++ b/exporter/awsemfexporter/metric_translator.go @@ -88,7 +88,7 @@ type DataPoints interface { At(int) DataPoint } -// Wrapper interface for: +// DataPoint is a wrapper interface for: // - pdata.IntDataPoint // - pdata.DoubleDataPoint // - pdata.IntHistogramDataPoint @@ -175,15 +175,23 @@ func TranslateOtToCWMetric(rm *pdata.ResourceMetrics, config *Config) ([]*CWMetr return cwMetricList, totalDroppedMetrics } -func TranslateCWMetricToEMF(cwMetricLists []*CWMetrics) []*LogEvent { +// TranslateCWMetricToEMF converts CloudWatch Metric format to EMF. +func TranslateCWMetricToEMF(cwMetricLists []*CWMetrics, logger *zap.Logger) []*LogEvent { // convert CWMetric into map format for compatible with PLE input ples := make([]*LogEvent, 0, maximumLogEventsPerPut) for _, met := range cwMetricLists { cwmMap := make(map[string]interface{}) fieldMap := met.Fields - cwmMap["CloudWatchMetrics"] = met.Measurements - cwmMap["Timestamp"] = met.Timestamp - fieldMap["_aws"] = cwmMap + + if len(met.Measurements) > 0 { + // Create `_aws` section only if there are measurements + cwmMap["CloudWatchMetrics"] = met.Measurements + cwmMap["Timestamp"] = met.Timestamp + fieldMap["_aws"] = cwmMap + } else { + str, _ := json.Marshal(fieldMap) + logger.Warn("Dropped metric due to no matching metric declarations", zap.String("labels", string(str))) + } pleMsg, err := json.Marshal(fieldMap) if err != nil { @@ -201,10 +209,8 @@ func TranslateCWMetricToEMF(cwMetricLists []*CWMetrics) []*LogEvent { return ples } -// Translates OTLP Metric to list of CW Metrics +// getCWMetrics translates OTLP Metric to a list of CW Metrics func getCWMetrics(metric *pdata.Metric, namespace string, instrumentationLibName string, config *Config) (cwMetrics []*CWMetrics) { - var dps DataPoints - if metric == nil { return } @@ -217,6 +223,7 @@ func getCWMetrics(metric *pdata.Metric, namespace string, instrumentationLibName metricSlice := []map[string]string{metricMeasure} // Retrieve data points + var dps DataPoints switch metric.DataType() { case pdata.MetricDataTypeIntGauge: dps = IntDataPointSlice{metric.IntGauge().DataPoints()} @@ -248,7 +255,7 @@ func getCWMetrics(metric *pdata.Metric, namespace string, instrumentationLibName if dp.IsNil() { continue } - cwMetric := buildCWMetric(dp, metric, namespace, metricSlice, instrumentationLibName, config.DimensionRollupOption) + cwMetric := buildCWMetric(dp, metric, namespace, metricSlice, instrumentationLibName, config) if cwMetric != nil { cwMetrics = append(cwMetrics, cwMetric) } @@ -256,15 +263,74 @@ func getCWMetrics(metric *pdata.Metric, namespace string, instrumentationLibName return } -// Build CWMetric from DataPoint -func buildCWMetric(dp DataPoint, pmd *pdata.Metric, namespace string, metricSlice []map[string]string, instrumentationLibName string, dimensionRollupOption string) *CWMetrics { - dimensions, fields := createDimensions(dp, instrumentationLibName, dimensionRollupOption) - cwMeasurement := &CwMeasurement{ - Namespace: namespace, - Dimensions: dimensions, - Metrics: metricSlice, +// buildCWMetric builds CWMetric from DataPoint +func buildCWMetric(dp DataPoint, pmd *pdata.Metric, namespace string, metricSlice []map[string]string, instrumentationLibName string, config *Config) *CWMetrics { + dimensionRollupOption := config.DimensionRollupOption + metricDeclarations := config.MetricDeclarations + + labelsMap := dp.LabelsMap() + labelsSlice := make([]string, labelsMap.Len(), labelsMap.Len()+1) + // `labels` contains label key/value pairs + labels := make(map[string]string, labelsMap.Len()+1) + // `fields` contains metric and dimensions key/value pairs + fields := make(map[string]interface{}, labelsMap.Len()+2) + idx := 0 + labelsMap.ForEach(func(k, v string) { + fields[k] = v + labels[k] = v + labelsSlice[idx] = k + idx++ + }) + + // Apply single/zero dimension rollup to labels + rollupDimensionArray := dimensionRollup(dimensionRollupOption, labelsSlice, instrumentationLibName) + + // Add OTel instrumentation lib name as an additional dimension if it is defined + if instrumentationLibName != noInstrumentationLibraryName { + labels[OTellibDimensionKey] = instrumentationLibName + fields[OTellibDimensionKey] = instrumentationLibName } - metricList := []CwMeasurement{*cwMeasurement} + + // Create list of dimension sets + var dimensions [][]string + if len(metricDeclarations) > 0 { + // If metric declarations are defined, extract dimension sets from them + dimensions = processMetricDeclarations(metricDeclarations, pmd, labels, rollupDimensionArray) + } else { + // If no metric declarations defined, create a single dimension set containing + // the list of labels + dims := labelsSlice + if instrumentationLibName != noInstrumentationLibraryName { + // If OTel instrumentation lib name is defined, add instrumentation lib + // name as a dimension + dims = append(dims, OTellibDimensionKey) + } + + if len(rollupDimensionArray) > 0 { + // Perform de-duplication check for edge case with a single label and single roll-up + // is activated + if len(labelsSlice) > 1 || (dimensionRollupOption != SingleDimensionRollupOnly && + dimensionRollupOption != ZeroAndSingleDimensionRollup) { + dimensions = [][]string{dims} + } + dimensions = append(dimensions, rollupDimensionArray...) + } else { + dimensions = [][]string{dims} + } + } + + // Build list of CW Measurements + var cwMeasurements []CwMeasurement + if len(dimensions) > 0 { + cwMeasurements = []CwMeasurement{ + { + Namespace: namespace, + Dimensions: dimensions, + Metrics: metricSlice, + }, + } + } + timestamp := time.Now().UnixNano() / int64(time.Millisecond) // Extract metric @@ -307,44 +373,13 @@ func buildCWMetric(dp DataPoint, pmd *pdata.Metric, namespace string, metricSlic fields[pmd.Name()] = metricVal cwMetric := &CWMetrics{ - Measurements: metricList, + Measurements: cwMeasurements, Timestamp: timestamp, Fields: fields, } return cwMetric } -// Create dimensions from DataPoint labels, where dimensions is a 2D array of dimension names, -// and initialize fields with dimension key/value pairs -func createDimensions(dp DataPoint, instrumentationLibName string, dimensionRollupOption string) (dimensions [][]string, fields map[string]interface{}) { - // fields contains metric and dimensions key/value pairs - fields = make(map[string]interface{}) - dimensionKV := dp.LabelsMap() - - dimensionSlice := make([]string, dimensionKV.Len(), dimensionKV.Len()+1) - idx := 0 - dimensionKV.ForEach(func(k string, v string) { - fields[k] = v - dimensionSlice[idx] = k - idx++ - }) - // Add OTel instrumentation lib name as an additional dimension if it is defined - if instrumentationLibName != noInstrumentationLibraryName { - fields[OTellibDimensionKey] = instrumentationLibName - dimensions = append(dimensions, append(dimensionSlice, OTellibDimensionKey)) - } else { - dimensions = append(dimensions, dimensionSlice) - } - - // EMF dimension attr takes list of list on dimensions. Including single/zero dimension rollup - rollupDimensionArray := dimensionRollup(dimensionRollupOption, dimensionSlice, instrumentationLibName) - if len(rollupDimensionArray) > 0 { - dimensions = append(dimensions, rollupDimensionArray...) - } - - return -} - // rate is calculated by valDelta / timeDelta func calculateRate(fields map[string]interface{}, val interface{}, timestamp int64) interface{} { keys := make([]string, 0, len(fields)) @@ -402,6 +437,7 @@ func calculateRate(fields map[string]interface{}, val interface{}, timestamp int return metricRate } +// dimensionRollup creates rolled-up dimensions from the metric's label set. func dimensionRollup(dimensionRollupOption string, originalDimensionSlice []string, instrumentationLibName string) [][]string { var rollupDimensionArray [][]string dimensionZero := []string{} @@ -416,10 +452,8 @@ func dimensionRollup(dimensionRollupOption string, originalDimensionSlice []stri } if dimensionRollupOption == ZeroAndSingleDimensionRollup || dimensionRollupOption == SingleDimensionRollupOnly { //"One" dimension rollup - if len(originalDimensionSlice) > 1 { - for _, dimensionKey := range originalDimensionSlice { - rollupDimensionArray = append(rollupDimensionArray, append(dimensionZero, dimensionKey)) - } + for _, dimensionKey := range originalDimensionSlice { + rollupDimensionArray = append(rollupDimensionArray, append(dimensionZero, dimensionKey)) } } diff --git a/exporter/awsemfexporter/metric_translator_test.go b/exporter/awsemfexporter/metric_translator_test.go index 0d74ec6cea9b..4d8313573f69 100644 --- a/exporter/awsemfexporter/metric_translator_test.go +++ b/exporter/awsemfexporter/metric_translator_test.go @@ -15,6 +15,7 @@ package awsemfexporter import ( + "encoding/json" "io/ioutil" "sort" "strings" @@ -36,6 +37,245 @@ import ( "go.uber.org/zap/zaptest/observer" ) +func readFromFile(filename string) string { + data, err := ioutil.ReadFile(filename) + if err != nil { + panic(err) + } + str := string(data) + return str +} + +func createMetricTestData() consumerdata.MetricsData { + return consumerdata.MetricsData{ + Node: &commonpb.Node{ + LibraryInfo: &commonpb.LibraryInfo{ExporterVersion: "SomeVersion"}, + }, + Resource: &resourcepb.Resource{ + Labels: map[string]string{ + conventions.AttributeServiceName: "myServiceName", + conventions.AttributeServiceNamespace: "myServiceNS", + }, + }, + Metrics: []*metricspb.Metric{ + { + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: "spanCounter", + Description: "Counting all the spans", + Unit: "Count", + Type: metricspb.MetricDescriptor_CUMULATIVE_INT64, + LabelKeys: []*metricspb.LabelKey{ + {Key: "spanName"}, + {Key: "isItAnError"}, + }, + }, + Timeseries: []*metricspb.TimeSeries{ + { + LabelValues: []*metricspb.LabelValue{ + {Value: "testSpan", HasValue: true}, + {Value: "false", HasValue: true}, + }, + Points: []*metricspb.Point{ + { + Timestamp: ×tamp.Timestamp{ + Seconds: 100, + }, + Value: &metricspb.Point_Int64Value{ + Int64Value: 1, + }, + }, + }, + }, + }, + }, + { + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: "spanGaugeCounter", + Description: "Counting all the spans", + Unit: "Count", + Type: metricspb.MetricDescriptor_GAUGE_INT64, + LabelKeys: []*metricspb.LabelKey{ + {Key: "spanName"}, + {Key: "isItAnError"}, + }, + }, + Timeseries: []*metricspb.TimeSeries{ + { + LabelValues: []*metricspb.LabelValue{ + {Value: "testSpan", HasValue: true}, + {Value: "false", HasValue: true}, + }, + Points: []*metricspb.Point{ + { + Timestamp: ×tamp.Timestamp{ + Seconds: 100, + }, + Value: &metricspb.Point_Int64Value{ + Int64Value: 1, + }, + }, + }, + }, + }, + }, + { + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: "spanDoubleCounter", + Description: "Counting all the spans", + Unit: "Count", + Type: metricspb.MetricDescriptor_CUMULATIVE_DOUBLE, + LabelKeys: []*metricspb.LabelKey{ + {Key: "spanName"}, + {Key: "isItAnError"}, + }, + }, + Timeseries: []*metricspb.TimeSeries{ + { + LabelValues: []*metricspb.LabelValue{ + {Value: "testSpan", HasValue: true}, + {Value: "false", HasValue: true}, + }, + Points: []*metricspb.Point{ + { + Timestamp: ×tamp.Timestamp{ + Seconds: 100, + }, + Value: &metricspb.Point_DoubleValue{ + DoubleValue: 0.1, + }, + }, + }, + }, + }, + }, + { + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: "spanGaugeDoubleCounter", + Description: "Counting all the spans", + Unit: "Count", + Type: metricspb.MetricDescriptor_GAUGE_DOUBLE, + LabelKeys: []*metricspb.LabelKey{ + {Key: "spanName"}, + {Key: "isItAnError"}, + }, + }, + Timeseries: []*metricspb.TimeSeries{ + { + LabelValues: []*metricspb.LabelValue{ + {Value: "testSpan", HasValue: true}, + {Value: "false", HasValue: true}, + }, + Points: []*metricspb.Point{ + { + Timestamp: ×tamp.Timestamp{ + Seconds: 100, + }, + Value: &metricspb.Point_DoubleValue{ + DoubleValue: 0.1, + }, + }, + }, + }, + }, + }, + { + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: "spanTimer", + Description: "How long the spans take", + Unit: "Seconds", + Type: metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION, + LabelKeys: []*metricspb.LabelKey{ + {Key: "spanName"}, + }, + }, + Timeseries: []*metricspb.TimeSeries{ + { + LabelValues: []*metricspb.LabelValue{ + {Value: "testSpan", HasValue: true}, + }, + Points: []*metricspb.Point{ + { + Timestamp: ×tamp.Timestamp{ + Seconds: 100, + }, + Value: &metricspb.Point_DistributionValue{ + DistributionValue: &metricspb.DistributionValue{ + Sum: 15.0, + Count: 5, + BucketOptions: &metricspb.DistributionValue_BucketOptions{ + Type: &metricspb.DistributionValue_BucketOptions_Explicit_{ + Explicit: &metricspb.DistributionValue_BucketOptions_Explicit{ + Bounds: []float64{0, 10}, + }, + }, + }, + Buckets: []*metricspb.DistributionValue_Bucket{ + { + Count: 0, + }, + { + Count: 4, + }, + { + Count: 1, + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: "spanTimer", + Description: "How long the spans take", + Unit: "Seconds", + Type: metricspb.MetricDescriptor_SUMMARY, + LabelKeys: []*metricspb.LabelKey{ + {Key: "spanName"}, + }, + }, + Timeseries: []*metricspb.TimeSeries{ + { + LabelValues: []*metricspb.LabelValue{ + {Value: "testSpan"}, + }, + Points: []*metricspb.Point{ + { + Timestamp: ×tamp.Timestamp{ + Seconds: 100, + }, + Value: &metricspb.Point_SummaryValue{ + SummaryValue: &metricspb.SummaryValue{ + Sum: &wrappers.DoubleValue{ + Value: 15.0, + }, + Count: &wrappers.Int64Value{ + Value: 5, + }, + Snapshot: &metricspb.SummaryValue_Snapshot{ + PercentileValues: []*metricspb.SummaryValue_Snapshot_ValueAtPercentile{{ + Percentile: 0, + Value: 1, + }, + { + Percentile: 100, + Value: 5, + }}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } +} + // Asserts whether dimension sets are equal (i.e. has same sets of dimensions) func assertDimsEqual(t *testing.T, expected, actual [][]string) { // Convert to string for easier sorting @@ -55,6 +295,13 @@ func assertDimsEqual(t *testing.T, expected, actual [][]string) { assert.Equal(t, expectedStringified, actualStringified) } +// Asserts whether CW Measurements are equal. +func assertCwMeasurementEqual(t *testing.T, expected, actual CwMeasurement) { + assert.Equal(t, expected.Namespace, actual.Namespace) + assert.Equal(t, expected.Metrics, actual.Metrics) + assertDimsEqual(t, expected.Dimensions, actual.Dimensions) +} + func TestTranslateOtToCWMetricWithInstrLibrary(t *testing.T) { config := &Config{ Namespace: "", @@ -76,25 +323,23 @@ func TestTranslateOtToCWMetricWithInstrLibrary(t *testing.T) { met := cwm[0] assert.Equal(t, met.Fields["spanCounter"], 0) - assert.Equal(t, "myServiceNS/myServiceName", met.Measurements[0].Namespace) - assert.Equal(t, 4, len(met.Measurements[0].Dimensions)) - dimensionSetOne := met.Measurements[0].Dimensions[0] - sort.Strings(dimensionSetOne) - assert.Equal(t, []string{OTellibDimensionKey, "isItAnError", "spanName"}, dimensionSetOne) - assert.Equal(t, 1, len(met.Measurements[0].Metrics)) - assert.Equal(t, "spanCounter", met.Measurements[0].Metrics[0]["Name"]) - assert.Equal(t, "Count", met.Measurements[0].Metrics[0]["Unit"]) - - dimensionSetTwo := met.Measurements[0].Dimensions[1] - assert.Equal(t, []string{OTellibDimensionKey}, dimensionSetTwo) - - dimensionSetThree := met.Measurements[0].Dimensions[2] - sort.Strings(dimensionSetThree) - assert.Equal(t, []string{OTellibDimensionKey, "spanName"}, dimensionSetThree) - - dimensionSetFour := met.Measurements[0].Dimensions[3] - sort.Strings(dimensionSetFour) - assert.Equal(t, []string{OTellibDimensionKey, "isItAnError"}, dimensionSetFour) + + expectedMeasurement := CwMeasurement{ + Namespace: "myServiceNS/myServiceName", + Dimensions: [][]string{ + {OTellibDimensionKey, "isItAnError", "spanName"}, + {OTellibDimensionKey}, + {OTellibDimensionKey, "spanName"}, + {OTellibDimensionKey, "isItAnError"}, + }, + Metrics: []map[string]string{ + { + "Name": "spanCounter", + "Unit": "Count", + }, + }, + } + assertCwMeasurementEqual(t, expectedMeasurement, met.Measurements[0]) } func TestTranslateOtToCWMetricWithoutInstrLibrary(t *testing.T) { @@ -115,26 +360,22 @@ func TestTranslateOtToCWMetricWithoutInstrLibrary(t *testing.T) { assert.NotContains(t, met.Fields, OTellibDimensionKey) assert.Equal(t, met.Fields["spanCounter"], 0) - assert.Equal(t, "myServiceNS/myServiceName", met.Measurements[0].Namespace) - assert.Equal(t, 4, len(met.Measurements[0].Dimensions)) - dimensionSetOne := met.Measurements[0].Dimensions[0] - sort.Strings(dimensionSetOne) - assert.Equal(t, []string{"isItAnError", "spanName"}, dimensionSetOne) - assert.Equal(t, 1, len(met.Measurements[0].Metrics)) - assert.Equal(t, "spanCounter", met.Measurements[0].Metrics[0]["Name"]) - assert.Equal(t, "Count", met.Measurements[0].Metrics[0]["Unit"]) - - // zero dimension metric - dimensionSetTwo := met.Measurements[0].Dimensions[1] - assert.Equal(t, []string{}, dimensionSetTwo) - - dimensionSetThree := met.Measurements[0].Dimensions[2] - sort.Strings(dimensionSetTwo) - assert.Equal(t, []string{"spanName"}, dimensionSetThree) - - dimensionSetFour := met.Measurements[0].Dimensions[3] - sort.Strings(dimensionSetFour) - assert.Equal(t, []string{"isItAnError"}, dimensionSetFour) + expectedMeasurement := CwMeasurement{ + Namespace: "myServiceNS/myServiceName", + Dimensions: [][]string{ + {"isItAnError", "spanName"}, + {}, + {"spanName"}, + {"isItAnError"}, + }, + Metrics: []map[string]string{ + { + "Name": "spanCounter", + "Unit": "Count", + }, + }, + } + assertCwMeasurementEqual(t, expectedMeasurement, met.Measurements[0]) } func TestTranslateOtToCWMetricWithNameSpace(t *testing.T) { @@ -255,18 +496,174 @@ func TestTranslateOtToCWMetricWithNameSpace(t *testing.T) { assert.Equal(t, "myServiceNS", met.Measurements[0].Namespace) } -func TestTranslateCWMetricToEMF(t *testing.T) { - cwMeasurement := CwMeasurement{ - Namespace: "test-emf", - Dimensions: [][]string{{"OTelLib"}, {"OTelLib", "spanName"}}, - Metrics: []map[string]string{{ +func TestTranslateOtToCWMetricWithFiltering(t *testing.T) { + md := consumerdata.MetricsData{ + Node: &commonpb.Node{ + LibraryInfo: &commonpb.LibraryInfo{ExporterVersion: "SomeVersion"}, + }, + Resource: &resourcepb.Resource{ + Labels: map[string]string{ + conventions.AttributeServiceName: "myServiceName", + conventions.AttributeServiceNamespace: "myServiceNS", + }, + }, + Metrics: []*metricspb.Metric{ + { + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: "spanCounter", + Description: "Counting all the spans", + Unit: "Count", + Type: metricspb.MetricDescriptor_CUMULATIVE_INT64, + LabelKeys: []*metricspb.LabelKey{ + {Key: "spanName"}, + {Key: "isItAnError"}, + }, + }, + Timeseries: []*metricspb.TimeSeries{ + { + LabelValues: []*metricspb.LabelValue{ + {Value: "testSpan", HasValue: true}, + {Value: "false", HasValue: true}, + }, + Points: []*metricspb.Point{ + { + Timestamp: ×tamp.Timestamp{ + Seconds: 100, + }, + Value: &metricspb.Point_Int64Value{ + Int64Value: 1, + }, + }, + }, + }, + }, + }, + }, + } + + rm := internaldata.OCToMetrics(md).ResourceMetrics().At(0) + ilm := rm.InstrumentationLibraryMetrics().At(0) + ilm.InstrumentationLibrary().InitEmpty() + ilm.InstrumentationLibrary().SetName("cloudwatch-lib") + + testCases := []struct { + testName string + metricNameSelectors []string + dimensionRollupOption string + expectedDimensions [][]string + numMeasurements int + }{ + { + "has match w/ Zero + Single dim rollup", + []string{"spanCounter"}, + ZeroAndSingleDimensionRollup, + [][]string{ + {"spanName", "isItAnError"}, + {"spanName", OTellibDimensionKey}, + {OTellibDimensionKey, "isItAnError"}, + {OTellibDimensionKey}, + }, + 1, + }, + { + "has match w/ no dim rollup", + []string{"spanCounter"}, + "", + [][]string{ + {"spanName", "isItAnError"}, + {"spanName", OTellibDimensionKey}, + }, + 1, + }, + { + "No match w/ rollup", + []string{"invalid"}, + ZeroAndSingleDimensionRollup, + [][]string{ + {OTellibDimensionKey, "spanName"}, + {OTellibDimensionKey, "isItAnError"}, + {OTellibDimensionKey}, + }, + 1, + }, + { + "No match w/ no rollup", + []string{"invalid"}, + "", + nil, + 0, + }, + } + logger := zap.NewNop() + + for _, tc := range testCases { + m := MetricDeclaration{ + Dimensions: [][]string{{"isItAnError", "spanName"}, {"spanName", OTellibDimensionKey}}, + MetricNameSelectors: tc.metricNameSelectors, + } + config := &Config{ + Namespace: "", + DimensionRollupOption: tc.dimensionRollupOption, + MetricDeclarations: []*MetricDeclaration{&m}, + } + t.Run(tc.testName, func(t *testing.T) { + err := m.Init(logger) + assert.Nil(t, err) + cwm, totalDroppedMetrics := TranslateOtToCWMetric(&rm, config) + assert.Equal(t, 0, totalDroppedMetrics) + assert.Equal(t, 1, len(cwm)) + assert.NotNil(t, cwm) + + assert.Equal(t, tc.numMeasurements, len(cwm[0].Measurements)) + + if tc.numMeasurements > 0 { + dimensions := cwm[0].Measurements[0].Dimensions + assertDimsEqual(t, tc.expectedDimensions, dimensions) + } + }) + } + + t.Run("No instrumentation library name w/ no dim rollup", func(t *testing.T) { + rm = internaldata.OCToMetrics(md).ResourceMetrics().At(0) + m := MetricDeclaration{ + Dimensions: [][]string{{"isItAnError", "spanName"}, {"spanName", OTellibDimensionKey}}, + MetricNameSelectors: []string{"spanCounter"}, + } + config := &Config{ + Namespace: "", + DimensionRollupOption: "", + MetricDeclarations: []*MetricDeclaration{&m}, + } + err := m.Init(logger) + assert.Nil(t, err) + cwm, totalDroppedMetrics := TranslateOtToCWMetric(&rm, config) + assert.Equal(t, 0, totalDroppedMetrics) + assert.Equal(t, 1, len(cwm)) + assert.NotNil(t, cwm) + + assert.Equal(t, 1, len(cwm[0].Measurements)) + + // No OTelLib present + expectedDims := [][]string{ + {"spanName", "isItAnError"}, + } + dimensions := cwm[0].Measurements[0].Dimensions + assertDimsEqual(t, expectedDims, dimensions) + }) +} + +func TestTranslateCWMetricToEMF(t *testing.T) { + cwMeasurement := CwMeasurement{ + Namespace: "test-emf", + Dimensions: [][]string{{OTellibDimensionKey}, {OTellibDimensionKey, "spanName"}}, + Metrics: []map[string]string{{ "Name": "spanCounter", "Unit": "Count", }}, } timestamp := int64(1596151098037) fields := make(map[string]interface{}) - fields["OTelLib"] = "cloudwatch-otel" + fields[OTellibDimensionKey] = "cloudwatch-otel" fields["spanName"] = "test" fields["spanCounter"] = 0 @@ -275,16 +672,47 @@ func TestTranslateCWMetricToEMF(t *testing.T) { Fields: fields, Measurements: []CwMeasurement{cwMeasurement}, } - inputLogEvent := TranslateCWMetricToEMF([]*CWMetrics{met}) + logger := zap.NewNop() + inputLogEvent := TranslateCWMetricToEMF([]*CWMetrics{met}, logger) assert.Equal(t, readFromFile("testdata/testTranslateCWMetricToEMF.json"), *inputLogEvent[0].InputLogEvent.Message, "Expect to be equal") } +func TestTranslateCWMetricToEMFNoMeasurements(t *testing.T) { + timestamp := int64(1596151098037) + fields := make(map[string]interface{}) + fields[OTellibDimensionKey] = "cloudwatch-otel" + fields["spanName"] = "test" + fields["spanCounter"] = 0 + + met := &CWMetrics{ + Timestamp: timestamp, + Fields: fields, + Measurements: nil, + } + obs, logs := observer.New(zap.WarnLevel) + logger := zap.New(obs) + inputLogEvent := TranslateCWMetricToEMF([]*CWMetrics{met}, logger) + expected := "{\"OTelLib\":\"cloudwatch-otel\",\"spanCounter\":0,\"spanName\":\"test\"}" + + assert.Equal(t, expected, *inputLogEvent[0].InputLogEvent.Message) + + // Check logged warning message + fieldsStr, _ := json.Marshal(fields) + expectedLogs := []observer.LoggedEntry{{ + Entry: zapcore.Entry{Level: zap.WarnLevel, Message: "Dropped metric due to no matching metric declarations"}, + Context: []zapcore.Field{zap.String("labels", string(fieldsStr))}, + }} + assert.Equal(t, 1, logs.Len()) + assert.Equal(t, expectedLogs, logs.AllUntimed()) +} + func TestGetCWMetrics(t *testing.T) { namespace := "Namespace" - OTelLib := "OTelLib" + OTelLib := OTellibDimensionKey instrumentationLibName := "InstrLibName" config := &Config{ + Namespace: "", DimensionRollupOption: "", } @@ -917,7 +1345,9 @@ func TestGetCWMetrics(t *testing.T) { for i, expected := range tc.expected { cwMetric := cwMetrics[i] assert.Equal(t, len(expected.Measurements), len(cwMetric.Measurements)) - assert.Equal(t, expected.Measurements, cwMetric.Measurements) + for i, expectedMeasurement := range expected.Measurements { + assertCwMeasurementEqual(t, expectedMeasurement, cwMetric.Measurements[i]) + } assert.Equal(t, len(expected.Fields), len(cwMetric.Fields)) assert.Equal(t, expected.Fields, cwMetric.Fields) } @@ -964,13 +1394,19 @@ func TestGetCWMetrics(t *testing.T) { func TestBuildCWMetric(t *testing.T) { namespace := "Namespace" instrLibName := "InstrLibName" - OTelLib := "OTelLib" + OTelLib := OTellibDimensionKey + config := &Config{ + Namespace: "", + DimensionRollupOption: "", + } metricSlice := []map[string]string{ { "Name": "foo", "Unit": "", }, } + + // Test data types metric := pdata.NewMetric() metric.InitEmpty() metric.SetName("foo") @@ -984,7 +1420,7 @@ func TestBuildCWMetric(t *testing.T) { }) dp.SetValue(int64(-17)) - cwMetric := buildCWMetric(dp, &metric, namespace, metricSlice, instrLibName, "") + cwMetric := buildCWMetric(dp, &metric, namespace, metricSlice, instrLibName, config) assert.NotNil(t, cwMetric) assert.Equal(t, 1, len(cwMetric.Measurements)) @@ -993,7 +1429,7 @@ func TestBuildCWMetric(t *testing.T) { Dimensions: [][]string{{"label1", OTelLib}}, Metrics: metricSlice, } - assert.Equal(t, expectedMeasurement, cwMetric.Measurements[0]) + assertCwMeasurementEqual(t, expectedMeasurement, cwMetric.Measurements[0]) expectedFields := map[string]interface{}{ OTelLib: instrLibName, "foo": int64(-17), @@ -1011,7 +1447,7 @@ func TestBuildCWMetric(t *testing.T) { }) dp.SetValue(0.3) - cwMetric := buildCWMetric(dp, &metric, namespace, metricSlice, instrLibName, "") + cwMetric := buildCWMetric(dp, &metric, namespace, metricSlice, instrLibName, config) assert.NotNil(t, cwMetric) assert.Equal(t, 1, len(cwMetric.Measurements)) @@ -1020,7 +1456,7 @@ func TestBuildCWMetric(t *testing.T) { Dimensions: [][]string{{"label1", OTelLib}}, Metrics: metricSlice, } - assert.Equal(t, expectedMeasurement, cwMetric.Measurements[0]) + assertCwMeasurementEqual(t, expectedMeasurement, cwMetric.Measurements[0]) expectedFields := map[string]interface{}{ OTelLib: instrLibName, "foo": 0.3, @@ -1040,7 +1476,7 @@ func TestBuildCWMetric(t *testing.T) { }) dp.SetValue(int64(-17)) - cwMetric := buildCWMetric(dp, &metric, namespace, metricSlice, instrLibName, "") + cwMetric := buildCWMetric(dp, &metric, namespace, metricSlice, instrLibName, config) assert.NotNil(t, cwMetric) assert.Equal(t, 1, len(cwMetric.Measurements)) @@ -1049,7 +1485,7 @@ func TestBuildCWMetric(t *testing.T) { Dimensions: [][]string{{"label1", OTelLib}}, Metrics: metricSlice, } - assert.Equal(t, expectedMeasurement, cwMetric.Measurements[0]) + assertCwMeasurementEqual(t, expectedMeasurement, cwMetric.Measurements[0]) expectedFields := map[string]interface{}{ OTelLib: instrLibName, "foo": 0, @@ -1069,7 +1505,7 @@ func TestBuildCWMetric(t *testing.T) { }) dp.SetValue(0.3) - cwMetric := buildCWMetric(dp, &metric, namespace, metricSlice, instrLibName, "") + cwMetric := buildCWMetric(dp, &metric, namespace, metricSlice, instrLibName, config) assert.NotNil(t, cwMetric) assert.Equal(t, 1, len(cwMetric.Measurements)) @@ -1078,7 +1514,7 @@ func TestBuildCWMetric(t *testing.T) { Dimensions: [][]string{{"label1", OTelLib}}, Metrics: metricSlice, } - assert.Equal(t, expectedMeasurement, cwMetric.Measurements[0]) + assertCwMeasurementEqual(t, expectedMeasurement, cwMetric.Measurements[0]) expectedFields := map[string]interface{}{ OTelLib: instrLibName, "foo": 0, @@ -1099,7 +1535,7 @@ func TestBuildCWMetric(t *testing.T) { dp.SetBucketCounts([]uint64{1, 2, 3}) dp.SetExplicitBounds([]float64{1, 2, 3}) - cwMetric := buildCWMetric(dp, &metric, namespace, metricSlice, instrLibName, "") + cwMetric := buildCWMetric(dp, &metric, namespace, metricSlice, instrLibName, config) assert.NotNil(t, cwMetric) assert.Equal(t, 1, len(cwMetric.Measurements)) @@ -1108,7 +1544,7 @@ func TestBuildCWMetric(t *testing.T) { Dimensions: [][]string{{"label1", OTelLib}}, Metrics: metricSlice, } - assert.Equal(t, expectedMeasurement, cwMetric.Measurements[0]) + assertCwMeasurementEqual(t, expectedMeasurement, cwMetric.Measurements[0]) expectedFields := map[string]interface{}{ OTelLib: instrLibName, "foo": &CWMetricStats{ @@ -1125,40 +1561,67 @@ func TestBuildCWMetric(t *testing.T) { dp := pdata.NewIntHistogramDataPoint() dp.InitEmpty() - cwMetric := buildCWMetric(dp, &metric, namespace, metricSlice, instrLibName, "") + cwMetric := buildCWMetric(dp, &metric, namespace, metricSlice, instrLibName, config) assert.Nil(t, cwMetric) }) -} -func TestCreateDimensions(t *testing.T) { - OTelLib := "OTelLib" + // Test rollup options and labels testCases := []struct { - testName string - labels map[string]string - dims [][]string + testName string + labels map[string]string + dimensionRollupOption string + expectedDims [][]string }{ { - "single label", + "Single label w/ no rollup", + map[string]string{"a": "foo"}, + "", + [][]string{ + {"a", OTelLib}, + }, + }, + { + "Single label w/ single rollup", map[string]string{"a": "foo"}, + SingleDimensionRollupOnly, [][]string{ {"a", OTelLib}, - {OTelLib}, }, }, { - "multiple labels", - map[string]string{"a": "foo", "b": "bar"}, + "Single label w/ zero + single rollup", + map[string]string{"a": "foo"}, + ZeroAndSingleDimensionRollup, [][]string{ - {"a", "b", OTelLib}, + {"a", OTelLib}, {OTelLib}, - {OTelLib, "a"}, - {OTelLib, "b"}, }, }, { - "no labels", - map[string]string{}, + "Multiple label w/ no rollup", + map[string]string{ + "a": "foo", + "b": "bar", + "c": "car", + }, + "", + [][]string{ + {"a", "b", "c", OTelLib}, + }, + }, + { + "Multiple label w/ rollup", + map[string]string{ + "a": "foo", + "b": "bar", + "c": "car", + }, + ZeroAndSingleDimensionRollup, [][]string{ + {"a", "b", "c", OTelLib}, + {OTelLib, "a"}, + {OTelLib, "b"}, + {OTelLib, "c"}, {OTelLib}, }, }, @@ -1169,388 +1632,500 @@ func TestCreateDimensions(t *testing.T) { dp := pdata.NewIntDataPoint() dp.InitEmpty() dp.LabelsMap().InitFromMap(tc.labels) - dimensions, fields := createDimensions(dp, OTelLib, ZeroAndSingleDimensionRollup) - - assertDimsEqual(t, tc.dims, dimensions) + dp.SetValue(int64(-17)) + config = &Config{ + Namespace: namespace, + DimensionRollupOption: tc.dimensionRollupOption, + } - expectedFields := make(map[string]interface{}) + expectedFields := map[string]interface{}{ + OTellibDimensionKey: OTelLib, + "foo": int64(-17), + } for k, v := range tc.labels { expectedFields[k] = v } - expectedFields[OTellibDimensionKey] = OTelLib - - assert.Equal(t, expectedFields, fields) - }) - } + expectedMeasurement := CwMeasurement{ + Namespace: namespace, + Dimensions: tc.expectedDims, + Metrics: metricSlice, + } -} + cwMetric := buildCWMetric(dp, &metric, namespace, metricSlice, OTelLib, config) -func TestCalculateRate(t *testing.T) { - prevValue := int64(0) - curValue := int64(10) - fields := make(map[string]interface{}) - fields["OTelLib"] = "cloudwatch-otel" - fields["spanName"] = "test" - fields["spanCounter"] = prevValue - fields["type"] = "Int64" - prevTime := time.Now().UnixNano() / int64(time.Millisecond) - curTime := time.Unix(0, prevTime*int64(time.Millisecond)).Add(time.Second*10).UnixNano() / int64(time.Millisecond) - rate := calculateRate(fields, prevValue, prevTime) - assert.Equal(t, 0, rate) - rate = calculateRate(fields, curValue, curTime) - assert.Equal(t, int64(1), rate) + // Check fields + assert.Equal(t, expectedFields, cwMetric.Fields) - prevDoubleValue := 0.0 - curDoubleValue := 5.0 - fields["type"] = "Float64" - rate = calculateRate(fields, prevDoubleValue, prevTime) - assert.Equal(t, 0, rate) - rate = calculateRate(fields, curDoubleValue, curTime) - assert.Equal(t, 0.5, rate) + // Check CW measurement + assert.Equal(t, 1, len(cwMetric.Measurements)) + assertCwMeasurementEqual(t, expectedMeasurement, cwMetric.Measurements[0]) + }) + } } -func TestDimensionRollup(t *testing.T) { +func TestBuildCWMetricWithMetricDeclarations(t *testing.T) { + namespace := "Namespace" + OTelLib := OTellibDimensionKey + instrumentationLibName := "cloudwatch-otel" + metricName := "metric1" + metricValue := int64(-17) + metric := pdata.NewMetric() + metric.InitEmpty() + metric.SetName(metricName) + metricSlice := []map[string]string{{"Name": metricName}} testCases := []struct { - testName string - dimensionRollupOption string - dims []string - instrumentationLibName string - expected [][]string + testName string + labels map[string]string + metricDeclarations []*MetricDeclaration + dimensionRollupOption string + expectedDims [][]string }{ { - "no rollup w/o instrumentation library name", + "Single label w/ no rollup", + map[string]string{"a": "foo"}, + []*MetricDeclaration{ + { + Dimensions: [][]string{{"a"}}, + MetricNameSelectors: []string{metricName}, + }, + }, "", - []string{"a", "b", "c"}, - noInstrumentationLibraryName, - nil, + [][]string{{"a"}}, }, { - "no rollup w/ instrumentation library name", + "Single label + OTelLib w/ no rollup", + map[string]string{"a": "foo"}, + []*MetricDeclaration{ + { + Dimensions: [][]string{{"a", OTelLib}}, + MetricNameSelectors: []string{metricName}, + }, + }, "", - []string{"a", "b", "c"}, - "cloudwatch-otel", - nil, + [][]string{{"a", OTelLib}}, }, { - "single dim w/o instrumentation library name", - SingleDimensionRollupOnly, - []string{"a", "b", "c"}, - noInstrumentationLibraryName, - [][]string{ - {"a"}, - {"b"}, - {"c"}, + "Single label w/ single rollup", + map[string]string{"a": "foo"}, + []*MetricDeclaration{ + { + Dimensions: [][]string{{"a"}}, + MetricNameSelectors: []string{metricName}, + }, }, + SingleDimensionRollupOnly, + [][]string{{"a"}, {"a", OTelLib}}, }, { - "single dim w/ instrumentation library name", - SingleDimensionRollupOnly, - []string{"a", "b", "c"}, - "cloudwatch-otel", - [][]string{ - {OTellibDimensionKey, "a"}, - {OTellibDimensionKey, "b"}, - {OTellibDimensionKey, "c"}, + "Single label w/ zero/single rollup", + map[string]string{"a": "foo"}, + []*MetricDeclaration{ + { + Dimensions: [][]string{{"a"}}, + MetricNameSelectors: []string{metricName}, + }, }, + ZeroAndSingleDimensionRollup, + [][]string{{"a"}, {"a", OTelLib}, {OTelLib}}, }, { - "single dim w/o instrumentation library name and only one label", - SingleDimensionRollupOnly, - []string{"a"}, - noInstrumentationLibraryName, + "No matching metric name", + map[string]string{"a": "foo"}, + []*MetricDeclaration{ + { + Dimensions: [][]string{{"a"}}, + MetricNameSelectors: []string{"invalid"}, + }, + }, + "", nil, }, { - "single dim w/ instrumentation library name and only one label", - SingleDimensionRollupOnly, - []string{"a"}, - "cloudwatch-otel", - nil, + "multiple labels w/ no rollup", + map[string]string{"a": "foo", "b": "bar"}, + []*MetricDeclaration{ + { + Dimensions: [][]string{{"a"}}, + MetricNameSelectors: []string{metricName}, + }, + }, + "", + [][]string{{"a"}}, }, { - "zero + single dim w/o instrumentation library name", + "multiple labels w/ rollup", + map[string]string{"a": "foo", "b": "bar"}, + []*MetricDeclaration{ + { + Dimensions: [][]string{{"a"}}, + MetricNameSelectors: []string{metricName}, + }, + }, ZeroAndSingleDimensionRollup, - []string{"a", "b", "c"}, - noInstrumentationLibraryName, [][]string{ - {}, {"a"}, - {"b"}, - {"c"}, + {OTelLib, "a"}, + {OTelLib, "b"}, + {OTelLib}, }, }, { - "zero + single dim w/ instrumentation library name", - ZeroAndSingleDimensionRollup, - []string{"a", "b", "c"}, - "cloudwatch-otel", - [][]string{ - {OTellibDimensionKey}, - {OTellibDimensionKey, "a"}, - {OTellibDimensionKey, "b"}, - {OTellibDimensionKey, "c"}, + "multiple labels + multiple dimensions w/ no rollup", + map[string]string{"a": "foo", "b": "bar"}, + []*MetricDeclaration{ + { + Dimensions: [][]string{{"a", "b"}, {"b"}}, + MetricNameSelectors: []string{metricName}, + }, }, + "", + [][]string{{"a", "b"}, {"b"}}, }, { - "zero dim rollup w/o instrumentation library name and no labels", + "multiple labels + multiple dimensions + OTelLib w/ no rollup", + map[string]string{"a": "foo", "b": "bar"}, + []*MetricDeclaration{ + { + Dimensions: [][]string{{"a", "b"}, {"b", OTelLib}, {OTelLib}}, + MetricNameSelectors: []string{metricName}, + }, + }, + "", + [][]string{{"a", "b"}, {"b", OTelLib}, {OTelLib}}, + }, + { + "multiple labels + multiple dimensions w/ rollup", + map[string]string{"a": "foo", "b": "bar"}, + []*MetricDeclaration{ + { + Dimensions: [][]string{{"a", "b"}, {"b"}}, + MetricNameSelectors: []string{metricName}, + }, + }, ZeroAndSingleDimensionRollup, - []string{}, - noInstrumentationLibraryName, + [][]string{ + {"a", "b"}, + {"b"}, + {OTelLib, "a"}, + {OTelLib, "b"}, + {OTelLib}, + }, + }, + { + "multiple labels, multiple dimensions w/ invalid dimension", + map[string]string{"a": "foo", "b": "bar"}, + []*MetricDeclaration{ + { + Dimensions: [][]string{{"a", "b", "c"}, {"b"}}, + MetricNameSelectors: []string{metricName}, + }, + }, + ZeroAndSingleDimensionRollup, + [][]string{ + {"b"}, + {OTelLib, "a"}, + {OTelLib, "b"}, + {OTelLib}, + }, + }, + { + "multiple labels, multiple dimensions w/ missing dimension", + map[string]string{"a": "foo", "b": "bar", "c": "car"}, + []*MetricDeclaration{ + { + Dimensions: [][]string{{"a", "b"}, {"b"}}, + MetricNameSelectors: []string{metricName}, + }, + }, + ZeroAndSingleDimensionRollup, + [][]string{ + {"a", "b"}, + {"b"}, + {OTelLib, "a"}, + {OTelLib, "b"}, + {OTelLib, "c"}, + {OTelLib}, + }, + }, + { + "multiple metric declarations w/ no rollup", + map[string]string{"a": "foo", "b": "bar", "c": "car"}, + []*MetricDeclaration{ + { + Dimensions: [][]string{{"a", "b"}, {"b"}}, + MetricNameSelectors: []string{metricName}, + }, + { + Dimensions: [][]string{{"a", "c"}, {"b"}, {"c"}}, + MetricNameSelectors: []string{metricName}, + }, + { + Dimensions: [][]string{{"a", "d"}, {"b", "c"}}, + MetricNameSelectors: []string{metricName}, + }, + }, + "", + [][]string{ + {"a", "b"}, + {"b"}, + {"a", "c"}, + {"c"}, + {"b", "c"}, + }, + }, + { + "multiple metric declarations w/ rollup", + map[string]string{"a": "foo", "b": "bar", "c": "car"}, + []*MetricDeclaration{ + { + Dimensions: [][]string{{"a", "b"}, {"b"}}, + MetricNameSelectors: []string{metricName}, + }, + { + Dimensions: [][]string{{"a", "c"}, {"b"}, {"c"}}, + MetricNameSelectors: []string{metricName}, + }, + { + Dimensions: [][]string{{"a", "d"}, {"b", "c"}}, + MetricNameSelectors: []string{metricName}, + }, + }, + ZeroAndSingleDimensionRollup, + [][]string{ + {"a", "b"}, + {"b"}, + {OTelLib, "a"}, + {OTelLib, "b"}, + {OTelLib, "c"}, + {OTelLib}, + {"a", "c"}, + {"c"}, + {"b", "c"}, + }, + }, + { + "remove measurements with no dimensions", + map[string]string{"a": "foo", "b": "bar", "c": "car"}, + []*MetricDeclaration{ + { + Dimensions: [][]string{{"a", "b"}, {"b"}}, + MetricNameSelectors: []string{metricName}, + }, + { + Dimensions: [][]string{{"a", "d"}}, + MetricNameSelectors: []string{metricName}, + }, + }, + "", + [][]string{ + {"a", "b"}, + {"b"}, + }, + }, + { + "multiple declarations w/ no dimensions", + map[string]string{"a": "foo", "b": "bar", "c": "car"}, + []*MetricDeclaration{ + { + Dimensions: [][]string{{"a", "e"}, {"d"}}, + MetricNameSelectors: []string{metricName}, + }, + { + Dimensions: [][]string{{"a", "d"}}, + MetricNameSelectors: []string{metricName}, + }, + }, + "", nil, }, { - "zero dim rollup w/ instrumentation library name and no labels", + "no labels", + map[string]string{}, + []*MetricDeclaration{ + { + Dimensions: [][]string{{"a", "b", "c"}, {"b"}}, + MetricNameSelectors: []string{metricName}, + }, + }, ZeroAndSingleDimensionRollup, - []string{}, - "cloudwatch-otel", nil, }, } for _, tc := range testCases { t.Run(tc.testName, func(t *testing.T) { - rolledUp := dimensionRollup(tc.dimensionRollupOption, tc.dims, tc.instrumentationLibName) - assert.Equal(t, tc.expected, rolledUp) + dp := pdata.NewIntDataPoint() + dp.InitEmpty() + dp.LabelsMap().InitFromMap(tc.labels) + dp.SetValue(metricValue) + config := &Config{ + Namespace: namespace, + DimensionRollupOption: tc.dimensionRollupOption, + MetricDeclarations: tc.metricDeclarations, + } + logger := zap.NewNop() + for _, m := range tc.metricDeclarations { + err := m.Init(logger) + assert.Nil(t, err) + } + + expectedFields := map[string]interface{}{ + OTellibDimensionKey: instrumentationLibName, + metricName: metricValue, + } + for k, v := range tc.labels { + expectedFields[k] = v + } + + cwMetric := buildCWMetric(dp, &metric, namespace, metricSlice, instrumentationLibName, config) + + // Check fields + assert.Equal(t, expectedFields, cwMetric.Fields) + + // Check CW measurement + if tc.expectedDims == nil { + assert.Equal(t, 0, len(cwMetric.Measurements)) + } else { + assert.Equal(t, 1, len(cwMetric.Measurements)) + expectedMeasurement := CwMeasurement{ + Namespace: namespace, + Dimensions: tc.expectedDims, + Metrics: metricSlice, + } + assertCwMeasurementEqual(t, expectedMeasurement, cwMetric.Measurements[0]) + } }) } } -func readFromFile(filename string) string { - data, err := ioutil.ReadFile(filename) - if err != nil { - panic(err) - } - str := string(data) - return str +func TestCalculateRate(t *testing.T) { + prevValue := int64(0) + curValue := int64(10) + fields := make(map[string]interface{}) + fields[OTellibDimensionKey] = "cloudwatch-otel" + fields["spanName"] = "test" + fields["spanCounter"] = prevValue + fields["type"] = "Int64" + prevTime := time.Now().UnixNano() / int64(time.Millisecond) + curTime := time.Unix(0, prevTime*int64(time.Millisecond)).Add(time.Second*10).UnixNano() / int64(time.Millisecond) + rate := calculateRate(fields, prevValue, prevTime) + assert.Equal(t, 0, rate) + rate = calculateRate(fields, curValue, curTime) + assert.Equal(t, int64(1), rate) + + prevDoubleValue := 0.0 + curDoubleValue := 5.0 + fields["type"] = "Float64" + rate = calculateRate(fields, prevDoubleValue, prevTime) + assert.Equal(t, 0, rate) + rate = calculateRate(fields, curDoubleValue, curTime) + assert.Equal(t, 0.5, rate) } -func createMetricTestData() consumerdata.MetricsData { - return consumerdata.MetricsData{ - Node: &commonpb.Node{ - LibraryInfo: &commonpb.LibraryInfo{ExporterVersion: "SomeVersion"}, +func TestDimensionRollup(t *testing.T) { + testCases := []struct { + testName string + dimensionRollupOption string + dims []string + instrumentationLibName string + expected [][]string + }{ + { + "no rollup w/o instrumentation library name", + "", + []string{"a", "b", "c"}, + noInstrumentationLibraryName, + nil, }, - Resource: &resourcepb.Resource{ - Labels: map[string]string{ - conventions.AttributeServiceName: "myServiceName", - conventions.AttributeServiceNamespace: "myServiceNS", - }, + { + "no rollup w/ instrumentation library name", + "", + []string{"a", "b", "c"}, + "cloudwatch-otel", + nil, }, - Metrics: []*metricspb.Metric{ - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "spanCounter", - Description: "Counting all the spans", - Unit: "Count", - Type: metricspb.MetricDescriptor_CUMULATIVE_INT64, - LabelKeys: []*metricspb.LabelKey{ - {Key: "spanName"}, - {Key: "isItAnError"}, - }, - }, - Timeseries: []*metricspb.TimeSeries{ - { - LabelValues: []*metricspb.LabelValue{ - {Value: "testSpan", HasValue: true}, - {Value: "false", HasValue: true}, - }, - Points: []*metricspb.Point{ - { - Timestamp: ×tamp.Timestamp{ - Seconds: 100, - }, - Value: &metricspb.Point_Int64Value{ - Int64Value: 1, - }, - }, - }, - }, - }, - }, - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "spanGaugeCounter", - Description: "Counting all the spans", - Unit: "Count", - Type: metricspb.MetricDescriptor_GAUGE_INT64, - LabelKeys: []*metricspb.LabelKey{ - {Key: "spanName"}, - {Key: "isItAnError"}, - }, - }, - Timeseries: []*metricspb.TimeSeries{ - { - LabelValues: []*metricspb.LabelValue{ - {Value: "testSpan", HasValue: true}, - {Value: "false", HasValue: true}, - }, - Points: []*metricspb.Point{ - { - Timestamp: ×tamp.Timestamp{ - Seconds: 100, - }, - Value: &metricspb.Point_Int64Value{ - Int64Value: 1, - }, - }, - }, - }, - }, - }, - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "spanDoubleCounter", - Description: "Counting all the spans", - Unit: "Count", - Type: metricspb.MetricDescriptor_CUMULATIVE_DOUBLE, - LabelKeys: []*metricspb.LabelKey{ - {Key: "spanName"}, - {Key: "isItAnError"}, - }, - }, - Timeseries: []*metricspb.TimeSeries{ - { - LabelValues: []*metricspb.LabelValue{ - {Value: "testSpan", HasValue: true}, - {Value: "false", HasValue: true}, - }, - Points: []*metricspb.Point{ - { - Timestamp: ×tamp.Timestamp{ - Seconds: 100, - }, - Value: &metricspb.Point_DoubleValue{ - DoubleValue: 0.1, - }, - }, - }, - }, - }, + { + "single dim w/o instrumentation library name", + SingleDimensionRollupOnly, + []string{"a", "b", "c"}, + noInstrumentationLibraryName, + [][]string{ + {"a"}, + {"b"}, + {"c"}, }, - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "spanGaugeDoubleCounter", - Description: "Counting all the spans", - Unit: "Count", - Type: metricspb.MetricDescriptor_GAUGE_DOUBLE, - LabelKeys: []*metricspb.LabelKey{ - {Key: "spanName"}, - {Key: "isItAnError"}, - }, - }, - Timeseries: []*metricspb.TimeSeries{ - { - LabelValues: []*metricspb.LabelValue{ - {Value: "testSpan", HasValue: true}, - {Value: "false", HasValue: true}, - }, - Points: []*metricspb.Point{ - { - Timestamp: ×tamp.Timestamp{ - Seconds: 100, - }, - Value: &metricspb.Point_DoubleValue{ - DoubleValue: 0.1, - }, - }, - }, - }, - }, + }, + { + "single dim w/ instrumentation library name", + SingleDimensionRollupOnly, + []string{"a", "b", "c"}, + "cloudwatch-otel", + [][]string{ + {OTellibDimensionKey, "a"}, + {OTellibDimensionKey, "b"}, + {OTellibDimensionKey, "c"}, }, - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "spanTimer", - Description: "How long the spans take", - Unit: "Seconds", - Type: metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION, - LabelKeys: []*metricspb.LabelKey{ - {Key: "spanName"}, - }, - }, - Timeseries: []*metricspb.TimeSeries{ - { - LabelValues: []*metricspb.LabelValue{ - {Value: "testSpan", HasValue: true}, - }, - Points: []*metricspb.Point{ - { - Timestamp: ×tamp.Timestamp{ - Seconds: 100, - }, - Value: &metricspb.Point_DistributionValue{ - DistributionValue: &metricspb.DistributionValue{ - Sum: 15.0, - Count: 5, - BucketOptions: &metricspb.DistributionValue_BucketOptions{ - Type: &metricspb.DistributionValue_BucketOptions_Explicit_{ - Explicit: &metricspb.DistributionValue_BucketOptions_Explicit{ - Bounds: []float64{0, 10}, - }, - }, - }, - Buckets: []*metricspb.DistributionValue_Bucket{ - { - Count: 0, - }, - { - Count: 4, - }, - { - Count: 1, - }, - }, - }, - }, - }, - }, - }, - }, + }, + { + "single dim w/o instrumentation library name and only one label", + SingleDimensionRollupOnly, + []string{"a"}, + noInstrumentationLibraryName, + [][]string{{"a"}}, + }, + { + "single dim w/ instrumentation library name and only one label", + SingleDimensionRollupOnly, + []string{"a"}, + "cloudwatch-otel", + [][]string{{OTellibDimensionKey, "a"}}, + }, + { + "zero + single dim w/o instrumentation library name", + ZeroAndSingleDimensionRollup, + []string{"a", "b", "c"}, + noInstrumentationLibraryName, + [][]string{ + {}, + {"a"}, + {"b"}, + {"c"}, }, - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "spanTimer", - Description: "How long the spans take", - Unit: "Seconds", - Type: metricspb.MetricDescriptor_SUMMARY, - LabelKeys: []*metricspb.LabelKey{ - {Key: "spanName"}, - }, - }, - Timeseries: []*metricspb.TimeSeries{ - { - LabelValues: []*metricspb.LabelValue{ - {Value: "testSpan"}, - }, - Points: []*metricspb.Point{ - { - Timestamp: ×tamp.Timestamp{ - Seconds: 100, - }, - Value: &metricspb.Point_SummaryValue{ - SummaryValue: &metricspb.SummaryValue{ - Sum: &wrappers.DoubleValue{ - Value: 15.0, - }, - Count: &wrappers.Int64Value{ - Value: 5, - }, - Snapshot: &metricspb.SummaryValue_Snapshot{ - PercentileValues: []*metricspb.SummaryValue_Snapshot_ValueAtPercentile{{ - Percentile: 0, - Value: 1, - }, - { - Percentile: 100, - Value: 5, - }}, - }, - }, - }, - }, - }, - }, - }, + }, + { + "zero + single dim w/ instrumentation library name", + ZeroAndSingleDimensionRollup, + []string{"a", "b", "c", "A"}, + "cloudwatch-otel", + [][]string{ + {OTellibDimensionKey}, + {OTellibDimensionKey, "a"}, + {OTellibDimensionKey, "b"}, + {OTellibDimensionKey, "c"}, + {OTellibDimensionKey, "A"}, }, }, + { + "zero dim rollup w/o instrumentation library name and no labels", + ZeroAndSingleDimensionRollup, + []string{}, + noInstrumentationLibraryName, + nil, + }, + { + "zero dim rollup w/ instrumentation library name and no labels", + ZeroAndSingleDimensionRollup, + []string{}, + "cloudwatch-otel", + nil, + }, + } + + for _, tc := range testCases { + t.Run(tc.testName, func(t *testing.T) { + rolledUp := dimensionRollup(tc.dimensionRollupOption, tc.dims, tc.instrumentationLibName) + assertDimsEqual(t, tc.expected, rolledUp) + }) } } @@ -1614,22 +2189,23 @@ func BenchmarkTranslateOtToCWMetricWithoutInstrLibrary(b *testing.B) { } } -func BenchmarkTranslateOtToCWMetricWithNamespace(b *testing.B) { - md := consumerdata.MetricsData{ - Node: &commonpb.Node{ - LibraryInfo: &commonpb.LibraryInfo{ExporterVersion: "SomeVersion"}, - }, - Resource: &resourcepb.Resource{ - Labels: map[string]string{ - conventions.AttributeServiceName: "myServiceName", - }, - }, - Metrics: []*metricspb.Metric{}, - } +func BenchmarkTranslateOtToCWMetricWithFiltering(b *testing.B) { + md := createMetricTestData() rm := internaldata.OCToMetrics(md).ResourceMetrics().At(0) + ilms := rm.InstrumentationLibraryMetrics() + ilm := ilms.At(0) + ilm.InstrumentationLibrary().InitEmpty() + ilm.InstrumentationLibrary().SetName("cloudwatch-lib") + m := MetricDeclaration{ + Dimensions: [][]string{{"spanName"}}, + MetricNameSelectors: []string{"spanCounter", "spanGaugeCounter"}, + } + logger := zap.NewNop() + m.Init(logger) config := &Config{ Namespace: "", DimensionRollupOption: ZeroAndSingleDimensionRollup, + MetricDeclarations: []*MetricDeclaration{&m}, } b.ResetTimer() @@ -1641,7 +2217,7 @@ func BenchmarkTranslateOtToCWMetricWithNamespace(b *testing.B) { func BenchmarkTranslateCWMetricToEMF(b *testing.B) { cwMeasurement := CwMeasurement{ Namespace: "test-emf", - Dimensions: [][]string{{"OTelLib"}, {"OTelLib", "spanName"}}, + Dimensions: [][]string{{OTellibDimensionKey}, {OTellibDimensionKey, "spanName"}}, Metrics: []map[string]string{{ "Name": "spanCounter", "Unit": "Count", @@ -1649,7 +2225,7 @@ func BenchmarkTranslateCWMetricToEMF(b *testing.B) { } timestamp := int64(1596151098037) fields := make(map[string]interface{}) - fields["OTelLib"] = "cloudwatch-otel" + fields[OTellibDimensionKey] = "cloudwatch-otel" fields["spanName"] = "test" fields["spanCounter"] = 0 @@ -1658,10 +2234,11 @@ func BenchmarkTranslateCWMetricToEMF(b *testing.B) { Fields: fields, Measurements: []CwMeasurement{cwMeasurement}, } + logger := zap.NewNop() b.ResetTimer() for n := 0; n < b.N; n++ { - TranslateCWMetricToEMF([]*CWMetrics{met}) + TranslateCWMetricToEMF([]*CWMetrics{met}, logger) } }