Skip to content

Commit

Permalink
[processor/transform] support substring function (#17085)
Browse files Browse the repository at this point in the history
  • Loading branch information
@ralphgj
ralphgj authored Jan 10, 2023
1 parent ead3675 commit 74ad8bd
Show file tree
Hide file tree
Showing 5 changed files with 97 additions and 0 deletions.
16 changes: 16 additions & 0 deletions .chloggen/transform_processor_support_substring_func.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
change_type: enhancement

# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
component: processor/transform

# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
note: Add substring function.

# One or more tracking issues related to the change
issues: [17038]

# (Optional) One or more lines of additional information to render under the primary note.
# These lines will be padded with 2 spaces and then inserted directly into the document.
# Use pipe (|) for multiline entries.
subtext:
1 change: 1 addition & 0 deletions processor/transformprocessor/internal/common/functions.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ func Functions[K any]() map[string]interface{} {
"Int": ottlfuncs.Int[K],
"ConvertCase": ottlfuncs.ConvertCase[K],
"ParseJSON": ottlfuncs.ParseJSON[K],
"Substring": ottlfuncs.Substring[K],
"keep_keys": ottlfuncs.KeepKeys[K],
"set": ottlfuncs.Set[K],
"truncate_all": ottlfuncs.TruncateAll[K],
Expand Down
25 changes: 25 additions & 0 deletions processor/transformprocessor/internal/logs/processor_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,11 +163,13 @@ func Test_ProcessLogs_LogContext(t *testing.T) {
td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().PutStr("http.path", "/health")
td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().PutStr("url", "http://localhost/health")
td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().PutStr("flags", "A|B|C")
td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().PutStr("total.string", "123456789")
td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(1).Attributes().Clear()
td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(1).Attributes().PutStr("http.method", "get")
td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(1).Attributes().PutStr("http.path", "/health")
td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(1).Attributes().PutStr("url", "http://localhost/health")
td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(1).Attributes().PutStr("flags", "C|D")
td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(1).Attributes().PutStr("total.string", "345678")
},
},
{
Expand Down Expand Up @@ -220,6 +222,8 @@ func Test_ProcessLogs_LogContext(t *testing.T) {
"get")
td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().PutStr("http.path",
"/health")
td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().PutStr("total.string",
"123456789")
td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().PutStr("flags",
"A|B|C")
},
Expand All @@ -232,6 +236,8 @@ func Test_ProcessLogs_LogContext(t *testing.T) {
"http://localhost/health")
td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().PutStr("flags",
"A|B|C")
td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().PutStr("total.string",
"123456789")
},
},
{
Expand Down Expand Up @@ -265,6 +271,23 @@ func Test_ProcessLogs_LogContext(t *testing.T) {
statement: `set(attributes["test"], Split(attributes["not_exist"], "|"))`,
want: func(td plog.Logs) {},
},
{
statement: `set(attributes["test"], Substring(attributes["total.string"], 3, 3))`,
want: func(td plog.Logs) {
td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().PutStr("test", "456")
td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(1).Attributes().PutStr("test", "678")
},
},
{
statement: `set(attributes["test"], Substring(attributes["total.string"], 3, 3)) where body == "operationA"`,
want: func(td plog.Logs) {
td.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().PutStr("test", "456")
},
},
{
statement: `set(attributes["test"], Substring(attributes["not_exist"], 3, 3))`,
want: func(td plog.Logs) {},
},
{
statement: `set(attributes["test"], ["A", "B", "C"]) where body == "operationA"`,
want: func(td plog.Logs) {
Expand Down Expand Up @@ -464,6 +487,7 @@ func fillLogOne(log plog.LogRecord) {
log.Attributes().PutStr("http.path", "/health")
log.Attributes().PutStr("http.url", "http://localhost/health")
log.Attributes().PutStr("flags", "A|B|C")
log.Attributes().PutStr("total.string", "123456789")

}

Expand All @@ -475,5 +499,6 @@ func fillLogTwo(log plog.LogRecord) {
log.Attributes().PutStr("http.path", "/health")
log.Attributes().PutStr("http.url", "http://localhost/health")
log.Attributes().PutStr("flags", "C|D")
log.Attributes().PutStr("total.string", "345678")

}
32 changes: 32 additions & 0 deletions processor/transformprocessor/internal/metrics/processor_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -253,24 +253,28 @@ func Test_ProcessMetrics_DataPointContext(t *testing.T) {
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(0).Attributes().PutStr("attr2", "test2")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(0).Attributes().PutStr("attr4", "test3")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(0).Attributes().PutStr("flags", "A|B|C")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(0).Attributes().PutStr("total.string", "123456789")

td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(1).Attributes().Clear()
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(1).Attributes().PutStr("attr1", "test1")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(1).Attributes().PutStr("attr2", "test2")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(1).Attributes().PutStr("attr4", "test3")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(1).Attributes().PutStr("flags", "A|B|C")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(1).Attributes().PutStr("total.string", "123456789")

td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(1).Histogram().DataPoints().At(0).Attributes().Clear()
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(1).Histogram().DataPoints().At(0).Attributes().PutStr("attr1", "test1")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(1).Histogram().DataPoints().At(0).Attributes().PutStr("attr2", "test2")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(1).Histogram().DataPoints().At(0).Attributes().PutStr("attr4", "test3")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(1).Histogram().DataPoints().At(0).Attributes().PutStr("flags", "C|D")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(1).Histogram().DataPoints().At(0).Attributes().PutStr("total.string", "345678")

td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(1).Histogram().DataPoints().At(1).Attributes().Clear()
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(1).Histogram().DataPoints().At(1).Attributes().PutStr("attr1", "test1")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(1).Histogram().DataPoints().At(1).Attributes().PutStr("attr2", "test2")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(1).Histogram().DataPoints().At(1).Attributes().PutStr("attr4", "test3")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(1).Histogram().DataPoints().At(1).Attributes().PutStr("flags", "C|D")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(1).Histogram().DataPoints().At(1).Attributes().PutStr("total.string", "345678")

td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(2).ExponentialHistogram().DataPoints().At(0).Attributes().Clear()
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(2).ExponentialHistogram().DataPoints().At(0).Attributes().PutStr("attr1", "test1")
Expand Down Expand Up @@ -377,10 +381,12 @@ func Test_ProcessMetrics_DataPointContext(t *testing.T) {
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(0).Attributes().Clear()
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(0).Attributes().PutStr("attr1", "test1")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(0).Attributes().PutStr("attr2", "test2")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(0).Attributes().PutStr("total.string", "123456789")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(0).Attributes().PutStr("flags", "A|B|C")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(1).Attributes().Clear()
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(1).Attributes().PutStr("attr1", "test1")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(1).Attributes().PutStr("attr2", "test2")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(1).Attributes().PutStr("total.string", "123456789")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(1).Attributes().PutStr("flags", "A|B|C")
},
},
Expand All @@ -390,9 +396,11 @@ func Test_ProcessMetrics_DataPointContext(t *testing.T) {
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(0).Attributes().Clear()
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(0).Attributes().PutStr("attr1", "test1")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(0).Attributes().PutStr("flags", "A|B|C")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(0).Attributes().PutStr("total.string", "123456789")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(1).Attributes().Clear()
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(1).Attributes().PutStr("attr1", "test1")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(1).Attributes().PutStr("flags", "A|B|C")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(1).Attributes().PutStr("total.string", "123456789")
},
},
{
Expand Down Expand Up @@ -438,6 +446,26 @@ func Test_ProcessMetrics_DataPointContext(t *testing.T) {
statements: []string{`set(attributes["test"], Split(attributes["not_exist"], "|"))`},
want: func(td pmetric.Metrics) {},
},
{
statements: []string{`set(attributes["test"], Substring(attributes["total.string"], 3, 3))`},
want: func(td pmetric.Metrics) {
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(0).Attributes().PutStr("test", "456")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(1).Attributes().PutStr("test", "456")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(1).Histogram().DataPoints().At(0).Attributes().PutStr("test", "678")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(1).Histogram().DataPoints().At(1).Attributes().PutStr("test", "678")
},
},
{
statements: []string{`set(attributes["test"], Substring(attributes["total.string"], 3, 3)) where metric.name == "operationA"`},
want: func(td pmetric.Metrics) {
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(0).Attributes().PutStr("test", "456")
td.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(1).Attributes().PutStr("test", "456")
},
},
{
statements: []string{`set(attributes["test"], Substring(attributes["not_exist"], 3, 3))`},
want: func(td pmetric.Metrics) {},
},
{
statements: []string{
`set(attributes["test_lower"], ConvertCase(metric.name, "lower")) where metric.name == "operationA"`,
Expand Down Expand Up @@ -652,6 +680,7 @@ func fillMetricOne(m pmetric.Metric) {
dataPoint0.Attributes().PutStr("attr2", "test2")
dataPoint0.Attributes().PutStr("attr3", "test3")
dataPoint0.Attributes().PutStr("flags", "A|B|C")
dataPoint0.Attributes().PutStr("total.string", "123456789")

dataPoint1 := m.Sum().DataPoints().AppendEmpty()
dataPoint1.SetStartTimestamp(StartTimestamp)
Expand All @@ -660,6 +689,7 @@ func fillMetricOne(m pmetric.Metric) {
dataPoint1.Attributes().PutStr("attr2", "test2")
dataPoint1.Attributes().PutStr("attr3", "test3")
dataPoint1.Attributes().PutStr("flags", "A|B|C")
dataPoint1.Attributes().PutStr("total.string", "123456789")
}

func fillMetricTwo(m pmetric.Metric) {
Expand All @@ -673,6 +703,7 @@ func fillMetricTwo(m pmetric.Metric) {
dataPoint0.Attributes().PutStr("attr2", "test2")
dataPoint0.Attributes().PutStr("attr3", "test3")
dataPoint0.Attributes().PutStr("flags", "C|D")
dataPoint0.Attributes().PutStr("total.string", "345678")
dataPoint0.SetCount(1)

dataPoint1 := m.Histogram().DataPoints().AppendEmpty()
Expand All @@ -681,6 +712,7 @@ func fillMetricTwo(m pmetric.Metric) {
dataPoint1.Attributes().PutStr("attr2", "test2")
dataPoint1.Attributes().PutStr("attr3", "test3")
dataPoint1.Attributes().PutStr("flags", "C|D")
dataPoint1.Attributes().PutStr("total.string", "345678")
}

func fillMetricThree(m pmetric.Metric) {
Expand Down
23 changes: 23 additions & 0 deletions processor/transformprocessor/internal/traces/processor_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -205,12 +205,14 @@ func Test_ProcessTraces_TraceContext(t *testing.T) {
td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0).Attributes().PutStr("http.path", "/health")
td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0).Attributes().PutStr("url", "http://localhost/health")
td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0).Attributes().PutStr("flags", "A|B|C")
td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0).Attributes().PutStr("total.string", "123456789")

td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(1).Attributes().Clear()
td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(1).Attributes().PutStr("http.method", "get")
td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(1).Attributes().PutStr("http.path", "/health")
td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(1).Attributes().PutStr("url", "http://localhost/health")
td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(1).Attributes().PutStr("flags", "C|D")
td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(1).Attributes().PutStr("total.string", "345678")
},
},
{
Expand All @@ -232,6 +234,7 @@ func Test_ProcessTraces_TraceContext(t *testing.T) {
td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0).Attributes().Clear()
td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0).Attributes().PutStr("http.method", "get")
td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0).Attributes().PutStr("http.path", "/health")
td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0).Attributes().PutStr("total.string", "123456789")
td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0).Attributes().PutStr("flags", "A|B|C")
},
},
Expand All @@ -241,6 +244,7 @@ func Test_ProcessTraces_TraceContext(t *testing.T) {
td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0).Attributes().Clear()
td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0).Attributes().PutStr("http.url", "http://localhost/health")
td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0).Attributes().PutStr("flags", "A|B|C")
td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0).Attributes().PutStr("total.string", "123456789")
},
},
{
Expand Down Expand Up @@ -306,6 +310,23 @@ func Test_ProcessTraces_TraceContext(t *testing.T) {
statement: `set(attributes["test"], Split(attributes["not_exist"], "|"))`,
want: func(td ptrace.Traces) {},
},
{
statement: `set(attributes["test"], Substring(attributes["total.string"], 3, 3))`,
want: func(td ptrace.Traces) {
td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0).Attributes().PutStr("test", "456")
td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(1).Attributes().PutStr("test", "678")
},
},
{
statement: `set(attributes["test"], Substring(attributes["total.string"], 3, 3)) where name == "operationA"`,
want: func(td ptrace.Traces) {
td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0).Attributes().PutStr("test", "456")
},
},
{
statement: `set(attributes["test"], Substring(attributes["not_exist"], 3, 3))`,
want: func(td ptrace.Traces) {},
},
{
statement: `set(attributes["test"], ["A", "B", "C"]) where name == "operationA"`,
want: func(td ptrace.Traces) {
Expand Down Expand Up @@ -644,6 +665,7 @@ func fillSpanOne(span ptrace.Span) {
span.Attributes().PutStr("http.path", "/health")
span.Attributes().PutStr("http.url", "http://localhost/health")
span.Attributes().PutStr("flags", "A|B|C")
span.Attributes().PutStr("total.string", "123456789")
status := span.Status()
status.SetCode(ptrace.StatusCodeError)
status.SetMessage("status-cancelled")
Expand All @@ -659,6 +681,7 @@ func fillSpanTwo(span ptrace.Span) {
span.Attributes().PutStr("http.path", "/health")
span.Attributes().PutStr("http.url", "http://localhost/health")
span.Attributes().PutStr("flags", "C|D")
span.Attributes().PutStr("total.string", "345678")
link0 := span.Links().AppendEmpty()
link0.SetDroppedAttributesCount(4)
link1 := span.Links().AppendEmpty()
Expand Down

0 comments on commit 74ad8bd

Please sign in to comment.