Merge branch 'main' into fix/prometheus-exporter-resource-labels

.github/workflows/codeql-analysis.yml

@@ -1,18 +1,8 @@
 name: "CodeQL Analysis"
-
 on:
-  workflow_dispatch:
-  schedule:
-    #        ┌───────────── minute (0 - 59)
-    #        │  ┌───────────── hour (0 - 23)
-    #        │  │ ┌───────────── day of the month (1 - 31)
-    #        │  │ │ ┌───────────── month (1 - 12 or JAN-DEC)
-    #        │  │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT)
-    #        │  │ │ │ │
-    #        │  │ │ │ │
-    #        │  │ │ │ │
-    #        *  * * * *
-    - cron: '30 1 * * *'
+  push:
+    branches: [ main ]
+  pull_request:
 
 jobs:
   CodeQL-Build:

.golangci.yml

@@ -99,25 +99,22 @@ linters:
   disable:
     - errcheck
   enable:
+    - exportloopref
+    - gocritic
     - gofmt
     - goimports
     - golint
     - gosec
     - govet
-    - staticcheck
     - misspell
-    - scopelint
+    - staticcheck
     - unconvert
-    - gocritic
     - unparam
 
 issues:
   # Excluding configuration per-path, per-linter, per-text and per-source
   exclude-rules:
     # Exclude some linters from running on tests files.
-    - path: _test\.go
-      linters:
-        - scopelint
     - path: otlp_test.go
       linters:
         # See https://github.com/golangci/golangci-lint/issues/537#issuecomment-545170007

CHANGELOG.md

@@ -2,6 +2,56 @@
 
 ## Unreleased
 
+## 🛑 Breaking changes 🛑
+
+## 💡 Enhancements 💡
+
+## 🧰 Bug fixes 🧰
+
+- Fix Jaeger receiver to honor TLS Settings (#2866)
+
+## v0.25.0 Beta
+
+## 🛑 Breaking changes 🛑
+
+- Rename ForEach (in pdata) with Range to be consistent with sync.Map (#2931)
+- Rename `componenthelper.Start` to `componenthelper.StartFunc` (#2880)
+- Rename `componenthelper.Stop` to `componenthelper.StopFunc` (#2880)
+- Remove `exporterheleper.WithCustomUnmarshaler`, `processorheleper.WithCustomUnmarshaler`, `receiverheleper.WithCustomUnmarshaler`, `extensionheleper.WithCustomUnmarshaler`, implement `config.CustomUnmarshaler` interface instead (#2867)
+- Remove `component.CustomUnmarshaler` implement `config.CustomUnmarshaler` interface instead (#2867)
+- Remove `testutil.HostPortFromAddr`, users can write their own parsing helper (#2919)
+- Remove `configparser.DecodeTypeAndName`, use `config.IDFromString` (#2869)
+- Remove `config.NewViper`, users should use `config.NewParser` (#2917)
+- Remove `testutil.WaitFor`, use `testify.Eventually` helper if needed (#2920)
+- Remove testutil.WaitForPort, users can use testify.Eventually (#2926)
+- Rename `processorhelper.NewTraceProcessor` to `processorhelper.NewTracesProcessor` (#2935)
+- Rename `exporterhelper.NewTraceExporter` to `exporterhelper.NewTracesExporter` (#2937)
+- Remove InitEmptyWithCapacity, add EnsureCapacity and Clear (#2845)
+- Rename traces methods/objects to include Traces in Kafka receiver (#2966)
+
+## 💡 Enhancements 💡
+
+- Add `validatable` interface with `Validate()` to all `config.<component>` (#2898)
+  - add the empty `Validate()` implementation for all component configs
+- **Experimental**: Add a config source manager that wraps the interaction with config sources (#2857, #2903, #2948)
+- `kafka` exporter: Key jaeger messages on traceid (#2855)
+- `scraperhelper`: Don't try to count metrics if scraper returns an error (#2902)
+- Extract ConfigFactory in a ParserProvider interface (#2868)
+- `prometheus` exporter: Allows Summary metrics to be exported to Prometheus (#2900)
+- `prometheus` receiver: Optimize `dpgSignature` function (#2945)
+- `kafka` receiver: Add logs support (#2944)
+
+## 🧰 Bug fixes 🧰
+
+- `prometheus` receiver:
+  - Treat Summary and Histogram metrics without "_sum" counter as valid metric (#2812)
+  - Add `job` and `instance` as well-known labels (#2897)
+- `prometheusremotewrite` exporter:
+  - Sort Sample by Timestamp to avoid out of order errors (#2941)
+  - Remove incompatible queued retry (#2951)
+- `kafka` receiver: Fix data race with batchprocessor (#2957)
+- `jaeger` receiver: Jaeger agent should not report ErrServerClosed (#2965)
+
 ## v0.24.0 Beta
 
 ## 🛑 Breaking changes 🛑
@@ -20,9 +70,9 @@
   - Rename `NewComponent()` to `New()`
 - obsReport.NewExporter accepts a settings struct (#2668)
 - Remove ErrorWaitingHost from `componenttest` (#2582)
-- Move `config.Load` to use `configparser.Load` (#2796)
+- Move `config.Load` to `configparser.Load` (#2796)
 - Remove `configtest.NewViperFromYamlFile()`, use `config.Parser.NewParserFromFile()` (#2806)
-- Move `config.ViperSubExact()` to use `config.Parser.Sub()` (#2806)
+- Remove `config.ViperSubExact()`, use `config.Parser.Sub()` (#2806)
 - Update LoadReceiver signature to remove unused params (#2823)
 - Move `configerror.ErrDataTypeIsNotSupported` to `componenterror.ErrDataTypeIsNotSupported` (#2886)
 - Rename`CreateTraceExporter` type to `CreateTracesExporter` in `exporterhelper` (#2779)

CONTRIBUTING.md

@@ -246,6 +246,16 @@ the event happens.
 Make log message human readable and also include data that is needed for easier
 understanding of what happened and in what context.
 
+### Observability
+
+Out of the box, your users should be able to observe the state of your component. 
+The collector exposes an OpenMetrics endpoint at `http://localhost:8888/metrics`
+where your data will land.
+
+When using the regular helpers, you should have some metrics added around key 
+events automatically. For instance, exporters should have `otelcol_exporter_sent_spans` 
+tracked without your exporter doing anything.
+
 ### Resource Usage
 
 Limit usage of CPU, RAM or other resources that the code can use. Do not write code

cmd/mdatagen/go.mod

@@ -5,7 +5,7 @@ go 1.15
 require (
 	github.com/go-playground/locales v0.13.0
 	github.com/go-playground/universal-translator v0.17.0
-	github.com/go-playground/validator/v10 v10.4.1
+	github.com/go-playground/validator/v10 v10.5.0
 	github.com/stretchr/testify v1.7.0
 	gopkg.in/yaml.v2 v2.4.0
 )

cmd/mdatagen/go.sum

@@ -6,8 +6,8 @@ github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8c
 github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
 github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no=
 github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
-github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE=
-github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
+github.com/go-playground/validator/v10 v10.5.0 h1:X9rflw/KmpACwT8zdrm1upefpvdy6ur8d1kWyq6sg3E=
+github.com/go-playground/validator/v10 v10.5.0/go.mod h1:xm76BBt941f7yWdGnI2DVPFFg1UK3YY04qifoXU3lOk=
 github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -24,6 +24,7 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=

cmd/pdatagen/internal/base_fields.go

@@ -76,6 +76,16 @@ func (ms ${structName}) ${fieldName}() ${returnType} {
 	return ${returnType}((*ms.orig).${originFieldName})
 }`
 
+const accessorsPrimitiveStructTemplate = `// ${fieldName} returns the ${lowerFieldName} associated with this ${structName}.
+func (ms ${structName}) ${fieldName}() ${returnType} {
+	return ${returnType}{orig: ((*ms.orig).${originFieldName})}
+}
+
+// Set${fieldName} replaces the ${lowerFieldName} associated with this ${structName}.
+func (ms ${structName}) Set${fieldName}(v ${returnType}) {
+	(*ms.orig).${originFieldName} = v.orig
+}`
+
 type baseField interface {
 	generateAccessors(ms baseStruct, sb *strings.Builder)
 
@@ -305,6 +315,62 @@ func (ptf *primitiveTypedField) generateCopyToValue(sb *strings.Builder) {
 
 var _ baseField = (*primitiveTypedField)(nil)
 
+// Types that has defined a custom type (e.g. "type TraceID struct {}")
+type primitiveStructField struct {
+	fieldName       string
+	originFieldName string
+	returnType      string
+	defaultVal      string
+	testVal         string
+}
+
+func (ptf *primitiveStructField) generateAccessors(ms baseStruct, sb *strings.Builder) {
+	template := accessorsPrimitiveStructTemplate
+	sb.WriteString(os.Expand(template, func(name string) string {
+		switch name {
+		case "structName":
+			return ms.getName()
+		case "fieldName":
+			return ptf.fieldName
+		case "lowerFieldName":
+			return strings.ToLower(ptf.fieldName)
+		case "returnType":
+			return ptf.returnType
+		case "originFieldName":
+			return ptf.originFieldName
+		default:
+			panic(name)
+		}
+	}))
+}
+
+func (ptf *primitiveStructField) generateAccessorsTest(ms baseStruct, sb *strings.Builder) {
+	sb.WriteString(os.Expand(accessorsPrimitiveTestTemplate, func(name string) string {
+		switch name {
+		case "structName":
+			return ms.getName()
+		case "defaultVal":
+			return ptf.defaultVal
+		case "fieldName":
+			return ptf.fieldName
+		case "testValue":
+			return ptf.testVal
+		default:
+			panic(name)
+		}
+	}))
+}
+
+func (ptf *primitiveStructField) generateSetWithTestValue(sb *strings.Builder) {
+	sb.WriteString("\ttv.Set" + ptf.fieldName + "(" + ptf.testVal + ")")
+}
+
+func (ptf *primitiveStructField) generateCopyToValue(sb *strings.Builder) {
+	sb.WriteString("\tdest.Set" + ptf.fieldName + "(ms." + ptf.fieldName + "())")
+}
+
+var _ baseField = (*primitiveStructField)(nil)
+
 // oneofField is used in case where the proto defines an "oneof".
 type oneofField struct {
 	copyFuncName    string

cmd/pdatagen/internal/base_slices.go

@@ -129,8 +129,16 @@ func (es ${structName}) Resize(newLen int) {
 // given ${elementName} at that new position.  The original ${elementName}
 // could still be referenced so do not reuse it after passing it to this
 // method.
+// Deprecated: Use AppendEmpty.
 func (es ${structName}) Append(e ${elementName}) {
 	*es.orig = append(*es.orig, e.orig)
+}
+
+// AppendEmpty will append to the end of the slice an empty ${elementName}.
+// It returns the newly added ${elementName}.
+func (es ${structName}) AppendEmpty() ${elementName} {
+	*es.orig = append(*es.orig, &${originName}{})
+	return es.At(es.Len() - 1)
 }`
 
 const slicePtrTestTemplate = `func Test${structName}(t *testing.T) {
@@ -140,7 +148,7 @@ const slicePtrTestTemplate = `func Test${structName}(t *testing.T) {
 	assert.EqualValues(t, 0, es.Len())
 
 	es.Resize(7)
-	emptyVal := New${elementName}()
+	emptyVal := new${elementName}(&${originName}{})
 	testVal := generateTest${elementName}()
 	assert.EqualValues(t, 7, es.Len())
 	for i := 0; i < es.Len(); i++ {
@@ -192,7 +200,7 @@ func Test${structName}_CopyTo(t *testing.T) {
 
 func Test${structName}_Resize(t *testing.T) {
 	es := generateTest${structName}()
-	emptyVal := New${elementName}()
+	emptyVal := new${elementName}(&${originName}{})
 	// Test Resize less elements.
 	const resizeSmallLen = 4
 	expectedEs := make(map[*${originName}]bool, resizeSmallLen)
@@ -235,12 +243,10 @@ func Test${structName}_Resize(t *testing.T) {
 func Test${structName}_Append(t *testing.T) {
 	es := generateTest${structName}()
 
-	emptyVal := New${elementName}()
-	es.Append(emptyVal)
-	assert.EqualValues(t, emptyVal.orig, es.At(7).orig)
+	es.AppendEmpty()
+	assert.EqualValues(t, &${originName}{}, es.At(7).orig)
 
-	value := New${elementName}()
-	fillTest${elementName}(value)
+	value := generateTest${elementName}()
 	es.Append(value)
 	assert.EqualValues(t, value.orig, es.At(8).orig)
 
@@ -365,8 +371,16 @@ func (es ${structName}) Resize(newLen int) {
 // given ${elementName} at that new position.  The original ${elementName}
 // could still be referenced so do not reuse it after passing it to this
 // method.
+// Deprecated: Use AppendEmpty.
 func (es ${structName}) Append(e ${elementName}) {
 	*es.orig = append(*es.orig, *e.orig)
+}
+
+// AppendEmpty will append to the end of the slice an empty ${elementName}.
+// It returns the newly added ${elementName}.
+func (es ${structName}) AppendEmpty() ${elementName} {
+	*es.orig = append(*es.orig, ${originName}{})
+	return es.At(es.Len() - 1)
 }`
 
 const sliceValueTestTemplate = `func Test${structName}(t *testing.T) {
@@ -376,7 +390,7 @@ const sliceValueTestTemplate = `func Test${structName}(t *testing.T) {
 	assert.EqualValues(t, 0, es.Len())
 
 	es.Resize(7)
-	emptyVal := New${elementName}()
+	emptyVal := new${elementName}(&${originName}{})
 	testVal := generateTest${elementName}()
 	assert.EqualValues(t, 7, es.Len())
 	for i := 0; i < es.Len(); i++ {
@@ -428,7 +442,7 @@ func Test${structName}_CopyTo(t *testing.T) {
 
 func Test${structName}_Resize(t *testing.T) {
 	es := generateTest${structName}()
-	emptyVal := New${elementName}()
+	emptyVal := new${elementName}(&${originName}{})
 	// Test Resize less elements.
 	const resizeSmallLen = 4
 	expectedEs := make(map[*${originName}]bool, resizeSmallLen)
@@ -471,12 +485,10 @@ func Test${structName}_Resize(t *testing.T) {
 func Test${structName}_Append(t *testing.T) {
 	es := generateTest${structName}()
 
-	emptyVal := New${elementName}()
-	es.Append(emptyVal)
-	assert.EqualValues(t, emptyVal, es.At(7))
+	es.AppendEmpty()
+	assert.EqualValues(t, new${elementName}(&${originName}{}), es.At(7))
 
-	value := New${elementName}()
-	fillTest${elementName}(value)
+	value := generateTest${elementName}()
 	es.Append(value)
 	assert.EqualValues(t, value, es.At(8))
 

cmd/pdatagen/internal/trace_structs.go

@@ -206,29 +206,26 @@ var spanStatus = &messageValueStruct{
 	},
 }
 
-var traceIDField = &primitiveTypedField{
+var traceIDField = &primitiveStructField{
 	fieldName:       "TraceID",
 	originFieldName: "TraceId",
 	returnType:      "TraceID",
-	rawType:         "data.TraceID",
 	defaultVal:      "NewTraceID([16]byte{})",
 	testVal:         "NewTraceID([16]byte{1, 2, 3, 4, 5, 6, 7, 8, 8, 7, 6, 5, 4, 3, 2, 1})",
 }
 
-var spanIDField = &primitiveTypedField{
+var spanIDField = &primitiveStructField{
 	fieldName:       "SpanID",
 	originFieldName: "SpanId",
 	returnType:      "SpanID",
-	rawType:         "data.SpanID",
 	defaultVal:      "NewSpanID([8]byte{})",
 	testVal:         "NewSpanID([8]byte{1, 2, 3, 4, 5, 6, 7, 8})",
 }
 
-var parentSpanIDField = &primitiveTypedField{
+var parentSpanIDField = &primitiveStructField{
 	fieldName:       "ParentSpanID",
 	originFieldName: "ParentSpanId",
 	returnType:      "SpanID",
-	rawType:         "data.SpanID",
 	defaultVal:      "NewSpanID([8]byte{})",
 	testVal:         "NewSpanID([8]byte{8, 7, 6, 5, 4, 3, 2, 1})",
 }