From fd61bbbc9b5759b32f8e3218a522494c87ad008b Mon Sep 17 00:00:00 2001 From: Traian Schiau <55734665+trasc@users.noreply.github.com> Date: Wed, 27 Oct 2021 20:18:54 +0300 Subject: [PATCH] pdatagen - Add MoveTo method for struct types (#4240) * [pdatagen] Add MoveTo method for struct types * [pdatagen] Add MoveTo method for struct types - regenerate * [batchprocessor] Split benchmarks rework - Fix metric split benchmark - Ignore clone counters in split benchmarks * [batchprocessor] Use MoveTo in split * [batchprocessor] Skip long running split benchmarks * [batchprocessor] Improve coverage --- .../cmd/pdatagen/internal/base_structs.go | 15 +++ model/pdata/generated_common.go | 7 + model/pdata/generated_common_test.go | 8 ++ model/pdata/generated_log.go | 21 +++ model/pdata/generated_log_test.go | 24 ++++ model/pdata/generated_metrics.go | 105 +++++++++++++++ model/pdata/generated_metrics_test.go | 120 ++++++++++++++++++ model/pdata/generated_resource.go | 7 + model/pdata/generated_resource_test.go | 8 ++ model/pdata/generated_trace.go | 42 ++++++ model/pdata/generated_trace_test.go | 48 +++++++ processor/batchprocessor/splitlogs.go | 26 +++- processor/batchprocessor/splitlogs_test.go | 41 +++++- processor/batchprocessor/splitmetrics.go | 83 +++++++----- processor/batchprocessor/splitmetrics_test.go | 42 +++++- processor/batchprocessor/splittraces.go | 26 +++- processor/batchprocessor/splittraces_test.go | 40 +++++- 17 files changed, 620 insertions(+), 43 deletions(-) diff --git a/model/internal/cmd/pdatagen/internal/base_structs.go b/model/internal/cmd/pdatagen/internal/base_structs.go index 4bf219d89ff..1312f941f4f 100644 --- a/model/internal/cmd/pdatagen/internal/base_structs.go +++ b/model/internal/cmd/pdatagen/internal/base_structs.go @@ -40,6 +40,13 @@ func new${structName}(orig *${originName}) ${structName} { // This must be used only in testing code since no "Set" method available. func New${structName}() ${structName} { return new${structName}(&${originName}{}) +} + +// MoveTo moves all properties from the current struct to dest +// reseting the current instance to its zero value +func (ms ${structName}) MoveTo(dest ${structName}) { + *dest.orig = *ms.orig + *ms.orig = ${originName}{} }` const messageValueCopyToHeaderTemplate = `// CopyTo copies all properties from the current struct to the dest. @@ -48,6 +55,14 @@ func (ms ${structName}) CopyTo(dest ${structName}) {` const messageValueCopyToFooterTemplate = `}` const messageValueTestTemplate = ` +func Test${structName}_MoveTo(t *testing.T) { + ms := generateTest${structName}() + dest := New${structName}() + ms.MoveTo(dest) + assert.EqualValues(t, New${structName}(), ms) + assert.EqualValues(t, generateTest${structName}(), dest) +} + func Test${structName}_CopyTo(t *testing.T) { ms := New${structName}() generateTest${structName}().CopyTo(ms) diff --git a/model/pdata/generated_common.go b/model/pdata/generated_common.go index 0f970e6c6ae..4839690d6a1 100644 --- a/model/pdata/generated_common.go +++ b/model/pdata/generated_common.go @@ -44,6 +44,13 @@ func NewInstrumentationLibrary() InstrumentationLibrary { return newInstrumentationLibrary(&otlpcommon.InstrumentationLibrary{}) } +// MoveTo moves all properties from the current struct to dest +// reseting the current instance to its zero value +func (ms InstrumentationLibrary) MoveTo(dest InstrumentationLibrary) { + *dest.orig = *ms.orig + *ms.orig = otlpcommon.InstrumentationLibrary{} +} + // Name returns the name associated with this InstrumentationLibrary. func (ms InstrumentationLibrary) Name() string { return (*ms.orig).Name diff --git a/model/pdata/generated_common_test.go b/model/pdata/generated_common_test.go index 1a61fe3a24d..ac07ddf6708 100644 --- a/model/pdata/generated_common_test.go +++ b/model/pdata/generated_common_test.go @@ -25,6 +25,14 @@ import ( otlpcommon "go.opentelemetry.io/collector/model/internal/data/protogen/common/v1" ) +func TestInstrumentationLibrary_MoveTo(t *testing.T) { + ms := generateTestInstrumentationLibrary() + dest := NewInstrumentationLibrary() + ms.MoveTo(dest) + assert.EqualValues(t, NewInstrumentationLibrary(), ms) + assert.EqualValues(t, generateTestInstrumentationLibrary(), dest) +} + func TestInstrumentationLibrary_CopyTo(t *testing.T) { ms := NewInstrumentationLibrary() generateTestInstrumentationLibrary().CopyTo(ms) diff --git a/model/pdata/generated_log.go b/model/pdata/generated_log.go index 110c9f88df5..ee46494ecf8 100644 --- a/model/pdata/generated_log.go +++ b/model/pdata/generated_log.go @@ -183,6 +183,13 @@ func NewResourceLogs() ResourceLogs { return newResourceLogs(&otlplogs.ResourceLogs{}) } +// MoveTo moves all properties from the current struct to dest +// reseting the current instance to its zero value +func (ms ResourceLogs) MoveTo(dest ResourceLogs) { + *dest.orig = *ms.orig + *ms.orig = otlplogs.ResourceLogs{} +} + // Resource returns the resource associated with this ResourceLogs. func (ms ResourceLogs) Resource() Resource { return newResource(&(*ms.orig).Resource) @@ -370,6 +377,13 @@ func NewInstrumentationLibraryLogs() InstrumentationLibraryLogs { return newInstrumentationLibraryLogs(&otlplogs.InstrumentationLibraryLogs{}) } +// MoveTo moves all properties from the current struct to dest +// reseting the current instance to its zero value +func (ms InstrumentationLibraryLogs) MoveTo(dest InstrumentationLibraryLogs) { + *dest.orig = *ms.orig + *ms.orig = otlplogs.InstrumentationLibraryLogs{} +} + // InstrumentationLibrary returns the instrumentationlibrary associated with this InstrumentationLibraryLogs. func (ms InstrumentationLibraryLogs) InstrumentationLibrary() InstrumentationLibrary { return newInstrumentationLibrary(&(*ms.orig).InstrumentationLibrary) @@ -558,6 +572,13 @@ func NewLogRecord() LogRecord { return newLogRecord(&otlplogs.LogRecord{}) } +// MoveTo moves all properties from the current struct to dest +// reseting the current instance to its zero value +func (ms LogRecord) MoveTo(dest LogRecord) { + *dest.orig = *ms.orig + *ms.orig = otlplogs.LogRecord{} +} + // Timestamp returns the timestamp associated with this LogRecord. func (ms LogRecord) Timestamp() Timestamp { return Timestamp((*ms.orig).TimeUnixNano) diff --git a/model/pdata/generated_log_test.go b/model/pdata/generated_log_test.go index 001c7b7fc40..5af8d4d7270 100644 --- a/model/pdata/generated_log_test.go +++ b/model/pdata/generated_log_test.go @@ -135,6 +135,14 @@ func TestResourceLogsSlice_RemoveIf(t *testing.T) { assert.Equal(t, 5, filtered.Len()) } +func TestResourceLogs_MoveTo(t *testing.T) { + ms := generateTestResourceLogs() + dest := NewResourceLogs() + ms.MoveTo(dest) + assert.EqualValues(t, NewResourceLogs(), ms) + assert.EqualValues(t, generateTestResourceLogs(), dest) +} + func TestResourceLogs_CopyTo(t *testing.T) { ms := NewResourceLogs() generateTestResourceLogs().CopyTo(ms) @@ -273,6 +281,14 @@ func TestInstrumentationLibraryLogsSlice_RemoveIf(t *testing.T) { assert.Equal(t, 5, filtered.Len()) } +func TestInstrumentationLibraryLogs_MoveTo(t *testing.T) { + ms := generateTestInstrumentationLibraryLogs() + dest := NewInstrumentationLibraryLogs() + ms.MoveTo(dest) + assert.EqualValues(t, NewInstrumentationLibraryLogs(), ms) + assert.EqualValues(t, generateTestInstrumentationLibraryLogs(), dest) +} + func TestInstrumentationLibraryLogs_CopyTo(t *testing.T) { ms := NewInstrumentationLibraryLogs() generateTestInstrumentationLibraryLogs().CopyTo(ms) @@ -411,6 +427,14 @@ func TestLogSlice_RemoveIf(t *testing.T) { assert.Equal(t, 5, filtered.Len()) } +func TestLogRecord_MoveTo(t *testing.T) { + ms := generateTestLogRecord() + dest := NewLogRecord() + ms.MoveTo(dest) + assert.EqualValues(t, NewLogRecord(), ms) + assert.EqualValues(t, generateTestLogRecord(), dest) +} + func TestLogRecord_CopyTo(t *testing.T) { ms := NewLogRecord() generateTestLogRecord().CopyTo(ms) diff --git a/model/pdata/generated_metrics.go b/model/pdata/generated_metrics.go index 01e3449ab45..10fc5736cc1 100644 --- a/model/pdata/generated_metrics.go +++ b/model/pdata/generated_metrics.go @@ -183,6 +183,13 @@ func NewResourceMetrics() ResourceMetrics { return newResourceMetrics(&otlpmetrics.ResourceMetrics{}) } +// MoveTo moves all properties from the current struct to dest +// reseting the current instance to its zero value +func (ms ResourceMetrics) MoveTo(dest ResourceMetrics) { + *dest.orig = *ms.orig + *ms.orig = otlpmetrics.ResourceMetrics{} +} + // Resource returns the resource associated with this ResourceMetrics. func (ms ResourceMetrics) Resource() Resource { return newResource(&(*ms.orig).Resource) @@ -370,6 +377,13 @@ func NewInstrumentationLibraryMetrics() InstrumentationLibraryMetrics { return newInstrumentationLibraryMetrics(&otlpmetrics.InstrumentationLibraryMetrics{}) } +// MoveTo moves all properties from the current struct to dest +// reseting the current instance to its zero value +func (ms InstrumentationLibraryMetrics) MoveTo(dest InstrumentationLibraryMetrics) { + *dest.orig = *ms.orig + *ms.orig = otlpmetrics.InstrumentationLibraryMetrics{} +} + // InstrumentationLibrary returns the instrumentationlibrary associated with this InstrumentationLibraryMetrics. func (ms InstrumentationLibraryMetrics) InstrumentationLibrary() InstrumentationLibrary { return newInstrumentationLibrary(&(*ms.orig).InstrumentationLibrary) @@ -558,6 +572,13 @@ func NewMetric() Metric { return newMetric(&otlpmetrics.Metric{}) } +// MoveTo moves all properties from the current struct to dest +// reseting the current instance to its zero value +func (ms Metric) MoveTo(dest Metric) { + *dest.orig = *ms.orig + *ms.orig = otlpmetrics.Metric{} +} + // Name returns the name associated with this Metric. func (ms Metric) Name() string { return (*ms.orig).Name @@ -619,6 +640,13 @@ func NewGauge() Gauge { return newGauge(&otlpmetrics.Gauge{}) } +// MoveTo moves all properties from the current struct to dest +// reseting the current instance to its zero value +func (ms Gauge) MoveTo(dest Gauge) { + *dest.orig = *ms.orig + *ms.orig = otlpmetrics.Gauge{} +} + // DataPoints returns the DataPoints associated with this Gauge. func (ms Gauge) DataPoints() NumberDataPointSlice { return newNumberDataPointSlice(&(*ms.orig).DataPoints) @@ -652,6 +680,13 @@ func NewSum() Sum { return newSum(&otlpmetrics.Sum{}) } +// MoveTo moves all properties from the current struct to dest +// reseting the current instance to its zero value +func (ms Sum) MoveTo(dest Sum) { + *dest.orig = *ms.orig + *ms.orig = otlpmetrics.Sum{} +} + // AggregationTemporality returns the aggregationtemporality associated with this Sum. func (ms Sum) AggregationTemporality() MetricAggregationTemporality { return MetricAggregationTemporality((*ms.orig).AggregationTemporality) @@ -707,6 +742,13 @@ func NewHistogram() Histogram { return newHistogram(&otlpmetrics.Histogram{}) } +// MoveTo moves all properties from the current struct to dest +// reseting the current instance to its zero value +func (ms Histogram) MoveTo(dest Histogram) { + *dest.orig = *ms.orig + *ms.orig = otlpmetrics.Histogram{} +} + // AggregationTemporality returns the aggregationtemporality associated with this Histogram. func (ms Histogram) AggregationTemporality() MetricAggregationTemporality { return MetricAggregationTemporality((*ms.orig).AggregationTemporality) @@ -752,6 +794,13 @@ func NewExponentialHistogram() ExponentialHistogram { return newExponentialHistogram(&otlpmetrics.ExponentialHistogram{}) } +// MoveTo moves all properties from the current struct to dest +// reseting the current instance to its zero value +func (ms ExponentialHistogram) MoveTo(dest ExponentialHistogram) { + *dest.orig = *ms.orig + *ms.orig = otlpmetrics.ExponentialHistogram{} +} + // AggregationTemporality returns the aggregationtemporality associated with this ExponentialHistogram. func (ms ExponentialHistogram) AggregationTemporality() MetricAggregationTemporality { return MetricAggregationTemporality((*ms.orig).AggregationTemporality) @@ -796,6 +845,13 @@ func NewSummary() Summary { return newSummary(&otlpmetrics.Summary{}) } +// MoveTo moves all properties from the current struct to dest +// reseting the current instance to its zero value +func (ms Summary) MoveTo(dest Summary) { + *dest.orig = *ms.orig + *ms.orig = otlpmetrics.Summary{} +} + // DataPoints returns the DataPoints associated with this Summary. func (ms Summary) DataPoints() SummaryDataPointSlice { return newSummaryDataPointSlice(&(*ms.orig).DataPoints) @@ -966,6 +1022,13 @@ func NewNumberDataPoint() NumberDataPoint { return newNumberDataPoint(&otlpmetrics.NumberDataPoint{}) } +// MoveTo moves all properties from the current struct to dest +// reseting the current instance to its zero value +func (ms NumberDataPoint) MoveTo(dest NumberDataPoint) { + *dest.orig = *ms.orig + *ms.orig = otlpmetrics.NumberDataPoint{} +} + // Attributes returns the Attributes associated with this NumberDataPoint. func (ms NumberDataPoint) Attributes() AttributeMap { return newAttributeMap(&(*ms.orig).Attributes) @@ -1206,6 +1269,13 @@ func NewHistogramDataPoint() HistogramDataPoint { return newHistogramDataPoint(&otlpmetrics.HistogramDataPoint{}) } +// MoveTo moves all properties from the current struct to dest +// reseting the current instance to its zero value +func (ms HistogramDataPoint) MoveTo(dest HistogramDataPoint) { + *dest.orig = *ms.orig + *ms.orig = otlpmetrics.HistogramDataPoint{} +} + // Attributes returns the Attributes associated with this HistogramDataPoint. func (ms HistogramDataPoint) Attributes() AttributeMap { return newAttributeMap(&(*ms.orig).Attributes) @@ -1462,6 +1532,13 @@ func NewExponentialHistogramDataPoint() ExponentialHistogramDataPoint { return newExponentialHistogramDataPoint(&otlpmetrics.ExponentialHistogramDataPoint{}) } +// MoveTo moves all properties from the current struct to dest +// reseting the current instance to its zero value +func (ms ExponentialHistogramDataPoint) MoveTo(dest ExponentialHistogramDataPoint) { + *dest.orig = *ms.orig + *ms.orig = otlpmetrics.ExponentialHistogramDataPoint{} +} + // Attributes returns the Attributes associated with this ExponentialHistogramDataPoint. func (ms ExponentialHistogramDataPoint) Attributes() AttributeMap { return newAttributeMap(&(*ms.orig).Attributes) @@ -1590,6 +1667,13 @@ func NewBuckets() Buckets { return newBuckets(&otlpmetrics.ExponentialHistogramDataPoint_Buckets{}) } +// MoveTo moves all properties from the current struct to dest +// reseting the current instance to its zero value +func (ms Buckets) MoveTo(dest Buckets) { + *dest.orig = *ms.orig + *ms.orig = otlpmetrics.ExponentialHistogramDataPoint_Buckets{} +} + // Offset returns the offset associated with this Buckets. func (ms Buckets) Offset() int32 { return int32((*ms.orig).Offset) @@ -1776,6 +1860,13 @@ func NewSummaryDataPoint() SummaryDataPoint { return newSummaryDataPoint(&otlpmetrics.SummaryDataPoint{}) } +// MoveTo moves all properties from the current struct to dest +// reseting the current instance to its zero value +func (ms SummaryDataPoint) MoveTo(dest SummaryDataPoint) { + *dest.orig = *ms.orig + *ms.orig = otlpmetrics.SummaryDataPoint{} +} + // Attributes returns the Attributes associated with this SummaryDataPoint. func (ms SummaryDataPoint) Attributes() AttributeMap { return newAttributeMap(&(*ms.orig).Attributes) @@ -2007,6 +2098,13 @@ func NewValueAtQuantile() ValueAtQuantile { return newValueAtQuantile(&otlpmetrics.SummaryDataPoint_ValueAtQuantile{}) } +// MoveTo moves all properties from the current struct to dest +// reseting the current instance to its zero value +func (ms ValueAtQuantile) MoveTo(dest ValueAtQuantile) { + *dest.orig = *ms.orig + *ms.orig = otlpmetrics.SummaryDataPoint_ValueAtQuantile{} +} + // Quantile returns the quantile associated with this ValueAtQuantile. func (ms ValueAtQuantile) Quantile() float64 { return (*ms.orig).Quantile @@ -2177,6 +2275,13 @@ func NewExemplar() Exemplar { return newExemplar(&otlpmetrics.Exemplar{}) } +// MoveTo moves all properties from the current struct to dest +// reseting the current instance to its zero value +func (ms Exemplar) MoveTo(dest Exemplar) { + *dest.orig = *ms.orig + *ms.orig = otlpmetrics.Exemplar{} +} + // Timestamp returns the timestamp associated with this Exemplar. func (ms Exemplar) Timestamp() Timestamp { return Timestamp((*ms.orig).TimeUnixNano) diff --git a/model/pdata/generated_metrics_test.go b/model/pdata/generated_metrics_test.go index 71b03c4cb04..1fa88bdfe76 100644 --- a/model/pdata/generated_metrics_test.go +++ b/model/pdata/generated_metrics_test.go @@ -135,6 +135,14 @@ func TestResourceMetricsSlice_RemoveIf(t *testing.T) { assert.Equal(t, 5, filtered.Len()) } +func TestResourceMetrics_MoveTo(t *testing.T) { + ms := generateTestResourceMetrics() + dest := NewResourceMetrics() + ms.MoveTo(dest) + assert.EqualValues(t, NewResourceMetrics(), ms) + assert.EqualValues(t, generateTestResourceMetrics(), dest) +} + func TestResourceMetrics_CopyTo(t *testing.T) { ms := NewResourceMetrics() generateTestResourceMetrics().CopyTo(ms) @@ -273,6 +281,14 @@ func TestInstrumentationLibraryMetricsSlice_RemoveIf(t *testing.T) { assert.Equal(t, 5, filtered.Len()) } +func TestInstrumentationLibraryMetrics_MoveTo(t *testing.T) { + ms := generateTestInstrumentationLibraryMetrics() + dest := NewInstrumentationLibraryMetrics() + ms.MoveTo(dest) + assert.EqualValues(t, NewInstrumentationLibraryMetrics(), ms) + assert.EqualValues(t, generateTestInstrumentationLibraryMetrics(), dest) +} + func TestInstrumentationLibraryMetrics_CopyTo(t *testing.T) { ms := NewInstrumentationLibraryMetrics() generateTestInstrumentationLibraryMetrics().CopyTo(ms) @@ -411,6 +427,14 @@ func TestMetricSlice_RemoveIf(t *testing.T) { assert.Equal(t, 5, filtered.Len()) } +func TestMetric_MoveTo(t *testing.T) { + ms := generateTestMetric() + dest := NewMetric() + ms.MoveTo(dest) + assert.EqualValues(t, NewMetric(), ms) + assert.EqualValues(t, generateTestMetric(), dest) +} + func TestMetric_CopyTo(t *testing.T) { ms := NewMetric() generateTestMetric().CopyTo(ms) @@ -441,6 +465,14 @@ func TestMetric_Unit(t *testing.T) { assert.EqualValues(t, testValUnit, ms.Unit()) } +func TestGauge_MoveTo(t *testing.T) { + ms := generateTestGauge() + dest := NewGauge() + ms.MoveTo(dest) + assert.EqualValues(t, NewGauge(), ms) + assert.EqualValues(t, generateTestGauge(), dest) +} + func TestGauge_CopyTo(t *testing.T) { ms := NewGauge() generateTestGauge().CopyTo(ms) @@ -455,6 +487,14 @@ func TestGauge_DataPoints(t *testing.T) { assert.EqualValues(t, testValDataPoints, ms.DataPoints()) } +func TestSum_MoveTo(t *testing.T) { + ms := generateTestSum() + dest := NewSum() + ms.MoveTo(dest) + assert.EqualValues(t, NewSum(), ms) + assert.EqualValues(t, generateTestSum(), dest) +} + func TestSum_CopyTo(t *testing.T) { ms := NewSum() generateTestSum().CopyTo(ms) @@ -485,6 +525,14 @@ func TestSum_DataPoints(t *testing.T) { assert.EqualValues(t, testValDataPoints, ms.DataPoints()) } +func TestHistogram_MoveTo(t *testing.T) { + ms := generateTestHistogram() + dest := NewHistogram() + ms.MoveTo(dest) + assert.EqualValues(t, NewHistogram(), ms) + assert.EqualValues(t, generateTestHistogram(), dest) +} + func TestHistogram_CopyTo(t *testing.T) { ms := NewHistogram() generateTestHistogram().CopyTo(ms) @@ -507,6 +555,14 @@ func TestHistogram_DataPoints(t *testing.T) { assert.EqualValues(t, testValDataPoints, ms.DataPoints()) } +func TestExponentialHistogram_MoveTo(t *testing.T) { + ms := generateTestExponentialHistogram() + dest := NewExponentialHistogram() + ms.MoveTo(dest) + assert.EqualValues(t, NewExponentialHistogram(), ms) + assert.EqualValues(t, generateTestExponentialHistogram(), dest) +} + func TestExponentialHistogram_CopyTo(t *testing.T) { ms := NewExponentialHistogram() generateTestExponentialHistogram().CopyTo(ms) @@ -529,6 +585,14 @@ func TestExponentialHistogram_DataPoints(t *testing.T) { assert.EqualValues(t, testValDataPoints, ms.DataPoints()) } +func TestSummary_MoveTo(t *testing.T) { + ms := generateTestSummary() + dest := NewSummary() + ms.MoveTo(dest) + assert.EqualValues(t, NewSummary(), ms) + assert.EqualValues(t, generateTestSummary(), dest) +} + func TestSummary_CopyTo(t *testing.T) { ms := NewSummary() generateTestSummary().CopyTo(ms) @@ -653,6 +717,14 @@ func TestNumberDataPointSlice_RemoveIf(t *testing.T) { assert.Equal(t, 5, filtered.Len()) } +func TestNumberDataPoint_MoveTo(t *testing.T) { + ms := generateTestNumberDataPoint() + dest := NewNumberDataPoint() + ms.MoveTo(dest) + assert.EqualValues(t, NewNumberDataPoint(), ms) + assert.EqualValues(t, generateTestNumberDataPoint(), dest) +} + func TestNumberDataPoint_CopyTo(t *testing.T) { ms := NewNumberDataPoint() generateTestNumberDataPoint().CopyTo(ms) @@ -824,6 +896,14 @@ func TestHistogramDataPointSlice_RemoveIf(t *testing.T) { assert.Equal(t, 5, filtered.Len()) } +func TestHistogramDataPoint_MoveTo(t *testing.T) { + ms := generateTestHistogramDataPoint() + dest := NewHistogramDataPoint() + ms.MoveTo(dest) + assert.EqualValues(t, NewHistogramDataPoint(), ms) + assert.EqualValues(t, generateTestHistogramDataPoint(), dest) +} + func TestHistogramDataPoint_CopyTo(t *testing.T) { ms := NewHistogramDataPoint() generateTestHistogramDataPoint().CopyTo(ms) @@ -1012,6 +1092,14 @@ func TestExponentialHistogramDataPointSlice_RemoveIf(t *testing.T) { assert.Equal(t, 5, filtered.Len()) } +func TestExponentialHistogramDataPoint_MoveTo(t *testing.T) { + ms := generateTestExponentialHistogramDataPoint() + dest := NewExponentialHistogramDataPoint() + ms.MoveTo(dest) + assert.EqualValues(t, NewExponentialHistogramDataPoint(), ms) + assert.EqualValues(t, generateTestExponentialHistogramDataPoint(), dest) +} + func TestExponentialHistogramDataPoint_CopyTo(t *testing.T) { ms := NewExponentialHistogramDataPoint() generateTestExponentialHistogramDataPoint().CopyTo(ms) @@ -1102,6 +1190,14 @@ func TestExponentialHistogramDataPoint_Flags(t *testing.T) { assert.EqualValues(t, testValFlags, ms.Flags()) } +func TestBuckets_MoveTo(t *testing.T) { + ms := generateTestBuckets() + dest := NewBuckets() + ms.MoveTo(dest) + assert.EqualValues(t, NewBuckets(), ms) + assert.EqualValues(t, generateTestBuckets(), dest) +} + func TestBuckets_CopyTo(t *testing.T) { ms := NewBuckets() generateTestBuckets().CopyTo(ms) @@ -1234,6 +1330,14 @@ func TestSummaryDataPointSlice_RemoveIf(t *testing.T) { assert.Equal(t, 5, filtered.Len()) } +func TestSummaryDataPoint_MoveTo(t *testing.T) { + ms := generateTestSummaryDataPoint() + dest := NewSummaryDataPoint() + ms.MoveTo(dest) + assert.EqualValues(t, NewSummaryDataPoint(), ms) + assert.EqualValues(t, generateTestSummaryDataPoint(), dest) +} + func TestSummaryDataPoint_CopyTo(t *testing.T) { ms := NewSummaryDataPoint() generateTestSummaryDataPoint().CopyTo(ms) @@ -1406,6 +1510,14 @@ func TestValueAtQuantileSlice_RemoveIf(t *testing.T) { assert.Equal(t, 5, filtered.Len()) } +func TestValueAtQuantile_MoveTo(t *testing.T) { + ms := generateTestValueAtQuantile() + dest := NewValueAtQuantile() + ms.MoveTo(dest) + assert.EqualValues(t, NewValueAtQuantile(), ms) + assert.EqualValues(t, generateTestValueAtQuantile(), dest) +} + func TestValueAtQuantile_CopyTo(t *testing.T) { ms := NewValueAtQuantile() generateTestValueAtQuantile().CopyTo(ms) @@ -1529,6 +1641,14 @@ func TestExemplarSlice_RemoveIf(t *testing.T) { assert.Equal(t, 5, filtered.Len()) } +func TestExemplar_MoveTo(t *testing.T) { + ms := generateTestExemplar() + dest := NewExemplar() + ms.MoveTo(dest) + assert.EqualValues(t, NewExemplar(), ms) + assert.EqualValues(t, generateTestExemplar(), dest) +} + func TestExemplar_CopyTo(t *testing.T) { ms := NewExemplar() generateTestExemplar().CopyTo(ms) diff --git a/model/pdata/generated_resource.go b/model/pdata/generated_resource.go index 6e44dd276a4..1a29265e489 100644 --- a/model/pdata/generated_resource.go +++ b/model/pdata/generated_resource.go @@ -44,6 +44,13 @@ func NewResource() Resource { return newResource(&otlpresource.Resource{}) } +// MoveTo moves all properties from the current struct to dest +// reseting the current instance to its zero value +func (ms Resource) MoveTo(dest Resource) { + *dest.orig = *ms.orig + *ms.orig = otlpresource.Resource{} +} + // Attributes returns the Attributes associated with this Resource. func (ms Resource) Attributes() AttributeMap { return newAttributeMap(&(*ms.orig).Attributes) diff --git a/model/pdata/generated_resource_test.go b/model/pdata/generated_resource_test.go index bc8295cd0db..5e9207eb83b 100644 --- a/model/pdata/generated_resource_test.go +++ b/model/pdata/generated_resource_test.go @@ -23,6 +23,14 @@ import ( "github.com/stretchr/testify/assert" ) +func TestResource_MoveTo(t *testing.T) { + ms := generateTestResource() + dest := NewResource() + ms.MoveTo(dest) + assert.EqualValues(t, NewResource(), ms) + assert.EqualValues(t, generateTestResource(), dest) +} + func TestResource_CopyTo(t *testing.T) { ms := NewResource() generateTestResource().CopyTo(ms) diff --git a/model/pdata/generated_trace.go b/model/pdata/generated_trace.go index e72a55b6329..93909bf4e95 100644 --- a/model/pdata/generated_trace.go +++ b/model/pdata/generated_trace.go @@ -183,6 +183,13 @@ func NewResourceSpans() ResourceSpans { return newResourceSpans(&otlptrace.ResourceSpans{}) } +// MoveTo moves all properties from the current struct to dest +// reseting the current instance to its zero value +func (ms ResourceSpans) MoveTo(dest ResourceSpans) { + *dest.orig = *ms.orig + *ms.orig = otlptrace.ResourceSpans{} +} + // Resource returns the resource associated with this ResourceSpans. func (ms ResourceSpans) Resource() Resource { return newResource(&(*ms.orig).Resource) @@ -370,6 +377,13 @@ func NewInstrumentationLibrarySpans() InstrumentationLibrarySpans { return newInstrumentationLibrarySpans(&otlptrace.InstrumentationLibrarySpans{}) } +// MoveTo moves all properties from the current struct to dest +// reseting the current instance to its zero value +func (ms InstrumentationLibrarySpans) MoveTo(dest InstrumentationLibrarySpans) { + *dest.orig = *ms.orig + *ms.orig = otlptrace.InstrumentationLibrarySpans{} +} + // InstrumentationLibrary returns the instrumentationlibrary associated with this InstrumentationLibrarySpans. func (ms InstrumentationLibrarySpans) InstrumentationLibrary() InstrumentationLibrary { return newInstrumentationLibrary(&(*ms.orig).InstrumentationLibrary) @@ -558,6 +572,13 @@ func NewSpan() Span { return newSpan(&otlptrace.Span{}) } +// MoveTo moves all properties from the current struct to dest +// reseting the current instance to its zero value +func (ms Span) MoveTo(dest Span) { + *dest.orig = *ms.orig + *ms.orig = otlptrace.Span{} +} + // TraceID returns the traceid associated with this Span. func (ms Span) TraceID() TraceID { return TraceID{orig: ((*ms.orig).TraceId)} @@ -868,6 +889,13 @@ func NewSpanEvent() SpanEvent { return newSpanEvent(&otlptrace.Span_Event{}) } +// MoveTo moves all properties from the current struct to dest +// reseting the current instance to its zero value +func (ms SpanEvent) MoveTo(dest SpanEvent) { + *dest.orig = *ms.orig + *ms.orig = otlptrace.Span_Event{} +} + // Timestamp returns the timestamp associated with this SpanEvent. func (ms SpanEvent) Timestamp() Timestamp { return Timestamp((*ms.orig).TimeUnixNano) @@ -1073,6 +1101,13 @@ func NewSpanLink() SpanLink { return newSpanLink(&otlptrace.Span_Link{}) } +// MoveTo moves all properties from the current struct to dest +// reseting the current instance to its zero value +func (ms SpanLink) MoveTo(dest SpanLink) { + *dest.orig = *ms.orig + *ms.orig = otlptrace.Span_Link{} +} + // TraceID returns the traceid associated with this SpanLink. func (ms SpanLink) TraceID() TraceID { return TraceID{orig: ((*ms.orig).TraceId)} @@ -1151,6 +1186,13 @@ func NewSpanStatus() SpanStatus { return newSpanStatus(&otlptrace.Status{}) } +// MoveTo moves all properties from the current struct to dest +// reseting the current instance to its zero value +func (ms SpanStatus) MoveTo(dest SpanStatus) { + *dest.orig = *ms.orig + *ms.orig = otlptrace.Status{} +} + // Code returns the code associated with this SpanStatus. func (ms SpanStatus) Code() StatusCode { return StatusCode((*ms.orig).Code) diff --git a/model/pdata/generated_trace_test.go b/model/pdata/generated_trace_test.go index 3eb4b35ccc3..7e0575110af 100644 --- a/model/pdata/generated_trace_test.go +++ b/model/pdata/generated_trace_test.go @@ -135,6 +135,14 @@ func TestResourceSpansSlice_RemoveIf(t *testing.T) { assert.Equal(t, 5, filtered.Len()) } +func TestResourceSpans_MoveTo(t *testing.T) { + ms := generateTestResourceSpans() + dest := NewResourceSpans() + ms.MoveTo(dest) + assert.EqualValues(t, NewResourceSpans(), ms) + assert.EqualValues(t, generateTestResourceSpans(), dest) +} + func TestResourceSpans_CopyTo(t *testing.T) { ms := NewResourceSpans() generateTestResourceSpans().CopyTo(ms) @@ -273,6 +281,14 @@ func TestInstrumentationLibrarySpansSlice_RemoveIf(t *testing.T) { assert.Equal(t, 5, filtered.Len()) } +func TestInstrumentationLibrarySpans_MoveTo(t *testing.T) { + ms := generateTestInstrumentationLibrarySpans() + dest := NewInstrumentationLibrarySpans() + ms.MoveTo(dest) + assert.EqualValues(t, NewInstrumentationLibrarySpans(), ms) + assert.EqualValues(t, generateTestInstrumentationLibrarySpans(), dest) +} + func TestInstrumentationLibrarySpans_CopyTo(t *testing.T) { ms := NewInstrumentationLibrarySpans() generateTestInstrumentationLibrarySpans().CopyTo(ms) @@ -411,6 +427,14 @@ func TestSpanSlice_RemoveIf(t *testing.T) { assert.Equal(t, 5, filtered.Len()) } +func TestSpan_MoveTo(t *testing.T) { + ms := generateTestSpan() + dest := NewSpan() + ms.MoveTo(dest) + assert.EqualValues(t, NewSpan(), ms) + assert.EqualValues(t, generateTestSpan(), dest) +} + func TestSpan_CopyTo(t *testing.T) { ms := NewSpan() generateTestSpan().CopyTo(ms) @@ -645,6 +669,14 @@ func TestSpanEventSlice_RemoveIf(t *testing.T) { assert.Equal(t, 5, filtered.Len()) } +func TestSpanEvent_MoveTo(t *testing.T) { + ms := generateTestSpanEvent() + dest := NewSpanEvent() + ms.MoveTo(dest) + assert.EqualValues(t, NewSpanEvent(), ms) + assert.EqualValues(t, generateTestSpanEvent(), dest) +} + func TestSpanEvent_CopyTo(t *testing.T) { ms := NewSpanEvent() generateTestSpanEvent().CopyTo(ms) @@ -793,6 +825,14 @@ func TestSpanLinkSlice_RemoveIf(t *testing.T) { assert.Equal(t, 5, filtered.Len()) } +func TestSpanLink_MoveTo(t *testing.T) { + ms := generateTestSpanLink() + dest := NewSpanLink() + ms.MoveTo(dest) + assert.EqualValues(t, NewSpanLink(), ms) + assert.EqualValues(t, generateTestSpanLink(), dest) +} + func TestSpanLink_CopyTo(t *testing.T) { ms := NewSpanLink() generateTestSpanLink().CopyTo(ms) @@ -839,6 +879,14 @@ func TestSpanLink_DroppedAttributesCount(t *testing.T) { assert.EqualValues(t, testValDroppedAttributesCount, ms.DroppedAttributesCount()) } +func TestSpanStatus_MoveTo(t *testing.T) { + ms := generateTestSpanStatus() + dest := NewSpanStatus() + ms.MoveTo(dest) + assert.EqualValues(t, NewSpanStatus(), ms) + assert.EqualValues(t, generateTestSpanStatus(), dest) +} + func TestSpanStatus_CopyTo(t *testing.T) { ms := NewSpanStatus() generateTestSpanStatus().CopyTo(ms) diff --git a/processor/batchprocessor/splitlogs.go b/processor/batchprocessor/splitlogs.go index b95ce446e44..9bc8bcfc89f 100644 --- a/processor/batchprocessor/splitlogs.go +++ b/processor/batchprocessor/splitlogs.go @@ -32,6 +32,14 @@ func splitLogs(size int, src pdata.Logs) pdata.Logs { return false } + // If it fully fits + srcRsCount := resourceLogsCount(srcRs) + if (totalCopiedLogs + srcRsCount) <= size { + totalCopiedLogs += srcRsCount + srcRs.MoveTo(dest.ResourceLogs().AppendEmpty()) + return true + } + destRs := dest.ResourceLogs().AppendEmpty() srcRs.Resource().CopyTo(destRs.Resource()) @@ -41,23 +49,23 @@ func splitLogs(size int, src pdata.Logs) pdata.Logs { return false } - destIlm := destRs.InstrumentationLibraryLogs().AppendEmpty() - srcIlm.InstrumentationLibrary().CopyTo(destIlm.InstrumentationLibrary()) - // If possible to move all metrics do that. srcLogsLen := srcIlm.Logs().Len() if size >= srcLogsLen+totalCopiedLogs { totalCopiedLogs += srcLogsLen - srcIlm.Logs().MoveAndAppendTo(destIlm.Logs()) + srcIlm.MoveTo(destRs.InstrumentationLibraryLogs().AppendEmpty()) return true } + destIlm := destRs.InstrumentationLibraryLogs().AppendEmpty() + srcIlm.InstrumentationLibrary().CopyTo(destIlm.InstrumentationLibrary()) + srcIlm.Logs().RemoveIf(func(srcMetric pdata.LogRecord) bool { // If we are done skip everything else. if totalCopiedLogs == size { return false } - srcMetric.CopyTo(destIlm.Logs().AppendEmpty()) + srcMetric.MoveTo(destIlm.Logs().AppendEmpty()) totalCopiedLogs++ return true }) @@ -68,3 +76,11 @@ func splitLogs(size int, src pdata.Logs) pdata.Logs { return dest } + +// resourceLogsCount calculates the total number of logs. +func resourceLogsCount(rs pdata.ResourceLogs) (count int) { + for k := 0; k < rs.InstrumentationLibraryLogs().Len(); k++ { + count += rs.InstrumentationLibraryLogs().At(k).Logs().Len() + } + return +} diff --git a/processor/batchprocessor/splitlogs_test.go b/processor/batchprocessor/splitlogs_test.go index 5e3e3837a5c..229ccf5aff6 100644 --- a/processor/batchprocessor/splitlogs_test.go +++ b/processor/batchprocessor/splitlogs_test.go @@ -138,10 +138,19 @@ func BenchmarkSplitLogs(b *testing.B) { } } + if b.N > 100000 { + b.Skipf("SKIP: b.N too high, set -benchtine=x with n < 100000") + } + + clones := make([]pdata.Logs, b.N) + for n := 0; n < b.N; n++ { + clones[n] = md.Clone() + } + b.ReportAllocs() b.ResetTimer() for n := 0; n < b.N; n++ { - cloneReq := md.Clone() + cloneReq := clones[n] split := splitLogs(128, cloneReq) if split.LogRecordCount() != 128 || cloneReq.LogRecordCount() != 400-128 { b.Fail() @@ -169,3 +178,33 @@ func BenchmarkCloneLogs(b *testing.B) { } } } + +func TestSplitLogsMultipleILL(t *testing.T) { + td := testdata.GenerateLogsManyLogRecordsSameResource(20) + logs := td.ResourceLogs().At(0).InstrumentationLibraryLogs().At(0).Logs() + for i := 0; i < logs.Len(); i++ { + logs.At(i).SetName(getTestLogName(0, i)) + } + // add second index to ILL + td.ResourceLogs().At(0).InstrumentationLibraryLogs().At(0). + CopyTo(td.ResourceLogs().At(0).InstrumentationLibraryLogs().AppendEmpty()) + logs = td.ResourceLogs().At(0).InstrumentationLibraryLogs().At(1).Logs() + for i := 0; i < logs.Len(); i++ { + logs.At(i).SetName(getTestLogName(1, i)) + } + + // add third index to ILL + td.ResourceLogs().At(0).InstrumentationLibraryLogs().At(0). + CopyTo(td.ResourceLogs().At(0).InstrumentationLibraryLogs().AppendEmpty()) + logs = td.ResourceLogs().At(0).InstrumentationLibraryLogs().At(2).Logs() + for i := 0; i < logs.Len(); i++ { + logs.At(i).SetName(getTestLogName(2, i)) + } + + splitSize := 40 + split := splitLogs(splitSize, td) + assert.Equal(t, splitSize, split.LogRecordCount()) + assert.Equal(t, 20, td.LogRecordCount()) + assert.Equal(t, "test-log-int-0-0", split.ResourceLogs().At(0).InstrumentationLibraryLogs().At(0).Logs().At(0).Name()) + assert.Equal(t, "test-log-int-0-4", split.ResourceLogs().At(0).InstrumentationLibraryLogs().At(0).Logs().At(4).Name()) +} diff --git a/processor/batchprocessor/splitmetrics.go b/processor/batchprocessor/splitmetrics.go index b842929c113..04a33f65ce6 100644 --- a/processor/batchprocessor/splitmetrics.go +++ b/processor/batchprocessor/splitmetrics.go @@ -33,6 +33,14 @@ func splitMetrics(size int, src pdata.Metrics) pdata.Metrics { return false } + // If it fully fits + srcRsDataPointCount := resourceMetricsDataPointCount(srcRs) + if (totalCopiedDataPoints + srcRsDataPointCount) <= size { + totalCopiedDataPoints += srcRsDataPointCount + srcRs.MoveTo(dest.ResourceMetrics().AppendEmpty()) + return true + } + destRs := dest.ResourceMetrics().AppendEmpty() srcRs.Resource().CopyTo(destRs.Resource()) @@ -42,17 +50,17 @@ func splitMetrics(size int, src pdata.Metrics) pdata.Metrics { return false } - destIlm := destRs.InstrumentationLibraryMetrics().AppendEmpty() - srcIlm.InstrumentationLibrary().CopyTo(destIlm.InstrumentationLibrary()) - // If possible to move all metrics do that. srcDataPointCount := metricSliceDataPointCount(srcIlm.Metrics()) if size-totalCopiedDataPoints >= srcDataPointCount { totalCopiedDataPoints += srcDataPointCount - srcIlm.Metrics().MoveAndAppendTo(destIlm.Metrics()) + srcIlm.MoveTo(destRs.InstrumentationLibraryMetrics().AppendEmpty()) return true } + destIlm := destRs.InstrumentationLibraryMetrics().AppendEmpty() + srcIlm.InstrumentationLibrary().CopyTo(destIlm.InstrumentationLibrary()) + srcIlm.Metrics().RemoveIf(func(srcMetric pdata.Metric) bool { // If we are done skip everything else. if totalCopiedDataPoints == size { @@ -71,6 +79,14 @@ func splitMetrics(size int, src pdata.Metrics) pdata.Metrics { return dest } +// resourceMetricsDataPointCount calculates the total number of data points. +func resourceMetricsDataPointCount(rs pdata.ResourceMetrics) (dataPointCount int) { + for k := 0; k < rs.InstrumentationLibraryMetrics().Len(); k++ { + dataPointCount += metricSliceDataPointCount(rs.InstrumentationLibraryMetrics().At(k).Metrics()) + } + return +} + // metricSliceDataPointCount calculates the total number of data points. func metricSliceDataPointCount(ms pdata.MetricSlice) (dataPointCount int) { for k := 0; k < ms.Len(); k++ { @@ -98,13 +114,10 @@ func metricDataPointCount(ms pdata.Metric) (dataPointCount int) { // Returns size of moved data and boolean describing, whether the metric should be removed from original slice. func splitMetric(ms, dest pdata.Metric, size int) (int, bool) { if metricDataPointCount(ms) <= size { - ms.CopyTo(dest) - return metricDataPointCount(ms), true + ms.MoveTo(dest) + return metricDataPointCount(dest), true } - msSize, i := metricDataPointCount(ms)-size, 0 - filterDataPoints := func() bool { i++; return i <= msSize } - dest.SetDataType(ms.DataType()) dest.SetName(ms.Name()) dest.SetDescription(ms.Description()) @@ -115,41 +128,53 @@ func splitMetric(ms, dest pdata.Metric, size int) (int, bool) { src := ms.Gauge().DataPoints() dst := dest.Gauge().DataPoints() dst.EnsureCapacity(size) - for j := 0; j < size; j++ { - src.At(j).CopyTo(dst.AppendEmpty()) - } - src.RemoveIf(func(_ pdata.NumberDataPoint) bool { - return filterDataPoints() + i := 0 + src.RemoveIf(func(dp pdata.NumberDataPoint) bool { + defer func() { i++ }() + if i < size { + dp.MoveTo(dst.AppendEmpty()) + return true + } + return false }) case pdata.MetricDataTypeSum: src := ms.Sum().DataPoints() dst := dest.Sum().DataPoints() dst.EnsureCapacity(size) - for j := 0; j < size; j++ { - src.At(j).CopyTo(dst.AppendEmpty()) - } - src.RemoveIf(func(_ pdata.NumberDataPoint) bool { - return filterDataPoints() + i := 0 + src.RemoveIf(func(dp pdata.NumberDataPoint) bool { + defer func() { i++ }() + if i < size { + dp.MoveTo(dst.AppendEmpty()) + return true + } + return false }) case pdata.MetricDataTypeHistogram: src := ms.Histogram().DataPoints() dst := dest.Histogram().DataPoints() dst.EnsureCapacity(size) - for j := 0; j < size; j++ { - src.At(j).CopyTo(dst.AppendEmpty()) - } - src.RemoveIf(func(_ pdata.HistogramDataPoint) bool { - return filterDataPoints() + i := 0 + src.RemoveIf(func(dp pdata.HistogramDataPoint) bool { + defer func() { i++ }() + if i < size { + dp.MoveTo(dst.AppendEmpty()) + return true + } + return false }) case pdata.MetricDataTypeSummary: src := ms.Summary().DataPoints() dst := dest.Summary().DataPoints() dst.EnsureCapacity(size) - for j := 0; j < size; j++ { - src.At(j).CopyTo(dst.AppendEmpty()) - } - src.RemoveIf(func(_ pdata.SummaryDataPoint) bool { - return filterDataPoints() + i := 0 + src.RemoveIf(func(dp pdata.SummaryDataPoint) bool { + defer func() { i++ }() + if i < size { + dp.MoveTo(dst.AppendEmpty()) + return true + } + return false }) } return size, false diff --git a/processor/batchprocessor/splitmetrics_test.go b/processor/batchprocessor/splitmetrics_test.go index 98e860fe6be..fee708927c2 100644 --- a/processor/batchprocessor/splitmetrics_test.go +++ b/processor/batchprocessor/splitmetrics_test.go @@ -147,11 +147,20 @@ func BenchmarkSplitMetrics(b *testing.B) { } } + if b.N > 100000 { + b.Skipf("SKIP: b.N too high, set -benchtine=x with n < 100000") + } + + dataPointCount := metricDataPointCount(md.ResourceMetrics().At(0).InstrumentationLibraryMetrics().At(0).Metrics().At(0)) + clones := make([]pdata.Metrics, b.N) + for n := 0; n < b.N; n++ { + clones[n] = md.Clone() + } b.ReportAllocs() b.ResetTimer() for n := 0; n < b.N; n++ { - cloneReq := md.Clone() - split := splitMetrics(128, cloneReq) + cloneReq := clones[n] + split := splitMetrics(128*dataPointCount, cloneReq) if split.MetricCount() != 128 || cloneReq.MetricCount() != 400-128 { b.Fail() } @@ -236,3 +245,32 @@ func TestSplitMetricsBatchSizeSmallerThanDataPointCount(t *testing.T) { assert.Equal(t, 1, md.MetricCount()) assert.Equal(t, "test-metric-int-0-1", split.ResourceMetrics().At(0).InstrumentationLibraryMetrics().At(0).Metrics().At(0).Name()) } + +func TestSplitMetricsMultipleILM(t *testing.T) { + md := testdata.GenerateMetricsManyMetricsSameResource(20) + metrics := md.ResourceMetrics().At(0).InstrumentationLibraryMetrics().At(0).Metrics() + dataPointCount := metricDataPointCount(metrics.At(0)) + for i := 0; i < metrics.Len(); i++ { + metrics.At(i).SetName(getTestMetricName(0, i)) + assert.Equal(t, dataPointCount, metricDataPointCount(metrics.At(i))) + } + // add second index to ilm + md.ResourceMetrics().At(0).InstrumentationLibraryMetrics().At(0). + CopyTo(md.ResourceMetrics().At(0).InstrumentationLibraryMetrics().AppendEmpty()) + + // add a third index to ilm + md.ResourceMetrics().At(0).InstrumentationLibraryMetrics().At(0). + CopyTo(md.ResourceMetrics().At(0).InstrumentationLibraryMetrics().AppendEmpty()) + metrics = md.ResourceMetrics().At(0).InstrumentationLibraryMetrics().At(2).Metrics() + for i := 0; i < metrics.Len(); i++ { + metrics.At(i).SetName(getTestMetricName(2, i)) + } + + splitMetricCount := 40 + splitSize := splitMetricCount * dataPointCount + split := splitMetrics(splitSize, md) + assert.Equal(t, splitMetricCount, split.MetricCount()) + assert.Equal(t, 20, md.MetricCount()) + assert.Equal(t, "test-metric-int-0-0", split.ResourceMetrics().At(0).InstrumentationLibraryMetrics().At(0).Metrics().At(0).Name()) + assert.Equal(t, "test-metric-int-0-4", split.ResourceMetrics().At(0).InstrumentationLibraryMetrics().At(0).Metrics().At(4).Name()) +} diff --git a/processor/batchprocessor/splittraces.go b/processor/batchprocessor/splittraces.go index 11465bb442b..95b23a42abc 100644 --- a/processor/batchprocessor/splittraces.go +++ b/processor/batchprocessor/splittraces.go @@ -32,6 +32,14 @@ func splitTraces(size int, src pdata.Traces) pdata.Traces { return false } + // If it fully fits + srcRsCount := resourceSpamsCount(srcRs) + if (totalCopiedSpans + srcRsCount) <= size { + totalCopiedSpans += srcRsCount + srcRs.MoveTo(dest.ResourceSpans().AppendEmpty()) + return true + } + destRs := dest.ResourceSpans().AppendEmpty() srcRs.Resource().CopyTo(destRs.Resource()) @@ -41,23 +49,23 @@ func splitTraces(size int, src pdata.Traces) pdata.Traces { return false } - destIls := destRs.InstrumentationLibrarySpans().AppendEmpty() - srcIls.InstrumentationLibrary().CopyTo(destIls.InstrumentationLibrary()) - // If possible to move all metrics do that. srcSpansLen := srcIls.Spans().Len() if size-totalCopiedSpans >= srcSpansLen { totalCopiedSpans += srcSpansLen - srcIls.Spans().MoveAndAppendTo(destIls.Spans()) + srcIls.MoveTo(destRs.InstrumentationLibrarySpans().AppendEmpty()) return true } + destIls := destRs.InstrumentationLibrarySpans().AppendEmpty() + srcIls.InstrumentationLibrary().CopyTo(destIls.InstrumentationLibrary()) + srcIls.Spans().RemoveIf(func(srcSpan pdata.Span) bool { // If we are done skip everything else. if totalCopiedSpans == size { return false } - srcSpan.CopyTo(destIls.Spans().AppendEmpty()) + srcSpan.MoveTo(destIls.Spans().AppendEmpty()) totalCopiedSpans++ return true }) @@ -68,3 +76,11 @@ func splitTraces(size int, src pdata.Traces) pdata.Traces { return dest } + +// resourceSpamsCount calculates the total number of spans. +func resourceSpamsCount(rs pdata.ResourceSpans) (count int) { + for k := 0; k < rs.InstrumentationLibrarySpans().Len(); k++ { + count += rs.InstrumentationLibrarySpans().At(k).Spans().Len() + } + return +} diff --git a/processor/batchprocessor/splittraces_test.go b/processor/batchprocessor/splittraces_test.go index dcfdb5877a4..6de10da9598 100644 --- a/processor/batchprocessor/splittraces_test.go +++ b/processor/batchprocessor/splittraces_test.go @@ -138,10 +138,18 @@ func BenchmarkSplitTraces(b *testing.B) { } } + if b.N > 100000 { + b.Skipf("SKIP: b.N too high, set -benchtine=x with n < 100000") + } + + clones := make([]pdata.Traces, b.N) + for n := 0; n < b.N; n++ { + clones[n] = td.Clone() + } b.ReportAllocs() b.ResetTimer() for n := 0; n < b.N; n++ { - cloneReq := td.Clone() + cloneReq := clones[n] split := splitTraces(128, cloneReq) if split.SpanCount() != 128 || cloneReq.SpanCount() != 400-128 { b.Fail() @@ -169,3 +177,33 @@ func BenchmarkCloneSpans(b *testing.B) { } } } + +func TestSplitTracesMultipleILS(t *testing.T) { + td := testdata.GenerateTracesManySpansSameResource(20) + spans := td.ResourceSpans().At(0).InstrumentationLibrarySpans().At(0).Spans() + for i := 0; i < spans.Len(); i++ { + spans.At(i).SetName(getTestSpanName(0, i)) + } + // add second index to ILS + td.ResourceSpans().At(0).InstrumentationLibrarySpans().At(0). + CopyTo(td.ResourceSpans().At(0).InstrumentationLibrarySpans().AppendEmpty()) + spans = td.ResourceSpans().At(0).InstrumentationLibrarySpans().At(1).Spans() + for i := 0; i < spans.Len(); i++ { + spans.At(i).SetName(getTestSpanName(1, i)) + } + + // add third index to ILS + td.ResourceSpans().At(0).InstrumentationLibrarySpans().At(0). + CopyTo(td.ResourceSpans().At(0).InstrumentationLibrarySpans().AppendEmpty()) + spans = td.ResourceSpans().At(0).InstrumentationLibrarySpans().At(2).Spans() + for i := 0; i < spans.Len(); i++ { + spans.At(i).SetName(getTestSpanName(2, i)) + } + + splitSize := 40 + split := splitTraces(splitSize, td) + assert.Equal(t, splitSize, split.SpanCount()) + assert.Equal(t, 20, td.SpanCount()) + assert.Equal(t, "test-span-0-0", split.ResourceSpans().At(0).InstrumentationLibrarySpans().At(0).Spans().At(0).Name()) + assert.Equal(t, "test-span-0-4", split.ResourceSpans().At(0).InstrumentationLibrarySpans().At(0).Spans().At(4).Name()) +}