From b991f117e9b6e7033bf667641b5cadec0e23bff0 Mon Sep 17 00:00:00 2001 From: Ryan Fitzpatrick Date: Mon, 27 Feb 2023 16:25:02 +0000 Subject: [PATCH] Add discoverybundler, initial embedded bundle.d, and enabled properties --- Makefile | 4 + go.mod | 2 +- internal/confmapprovider/discovery/README.md | 75 ++++++++-- .../discovery/bundle/README.md | 72 +++++++++ .../extensions/docker-observer.discovery.yaml | 4 + .../docker-observer.discovery.yaml.tmpl | 1 + .../extensions/host-observer.discovery.yaml | 4 + .../host-observer.discovery.yaml.tmpl | 1 + .../extensions/k8s-observer.discovery.yaml | 5 + .../k8s-observer.discovery.yaml.tmpl | 2 + .../smartagent-postgresql.discovery.yaml | 49 +++++++ .../smartagent-postgresql.discovery.yaml.tmpl | 46 ++++++ .../discovery/bundle/bundle.go | 29 ++++ .../discovery/bundle/bundle_gen.go | 27 ++++ .../discovery/bundle/bundle_test.go | 38 +++++ .../bundle/cmd/discoverybundler/main.go | 84 +++++++++++ .../discovery/bundle/templatefunctions.go | 138 ++++++++++++++++++ .../bundle/templatefunctions_test.go | 111 ++++++++++++++ internal/confmapprovider/discovery/config.go | 97 +++++++++--- .../confmapprovider/discovery/config_test.go | 8 +- .../confmapprovider/discovery/discoverer.go | 85 ++++++++--- .../discovery/properties/env_var.go | 7 +- .../discovery/properties/env_var_test.go | 55 ++++++- .../discovery/properties/property.go | 60 +++++--- .../discovery/properties/property_test.go | 56 ++++++- .../confmapprovider/discovery/provider.go | 34 ++++- .../docker_observer_discovery_test.go | 2 +- .../host_observer_discovery_test.go | 2 +- .../k8s_observer_discovery_test.go | 2 +- 29 files changed, 1003 insertions(+), 97 deletions(-) create mode 100644 internal/confmapprovider/discovery/bundle/README.md create mode 100644 internal/confmapprovider/discovery/bundle/bundle.d/extensions/docker-observer.discovery.yaml create mode 100644 internal/confmapprovider/discovery/bundle/bundle.d/extensions/docker-observer.discovery.yaml.tmpl create mode 100644 internal/confmapprovider/discovery/bundle/bundle.d/extensions/host-observer.discovery.yaml create mode 100644 internal/confmapprovider/discovery/bundle/bundle.d/extensions/host-observer.discovery.yaml.tmpl create mode 100644 internal/confmapprovider/discovery/bundle/bundle.d/extensions/k8s-observer.discovery.yaml create mode 100644 internal/confmapprovider/discovery/bundle/bundle.d/extensions/k8s-observer.discovery.yaml.tmpl create mode 100644 internal/confmapprovider/discovery/bundle/bundle.d/receivers/smartagent-postgresql.discovery.yaml create mode 100644 internal/confmapprovider/discovery/bundle/bundle.d/receivers/smartagent-postgresql.discovery.yaml.tmpl create mode 100644 internal/confmapprovider/discovery/bundle/bundle.go create mode 100644 internal/confmapprovider/discovery/bundle/bundle_gen.go create mode 100644 internal/confmapprovider/discovery/bundle/bundle_test.go create mode 100644 internal/confmapprovider/discovery/bundle/cmd/discoverybundler/main.go create mode 100644 internal/confmapprovider/discovery/bundle/templatefunctions.go create mode 100644 internal/confmapprovider/discovery/bundle/templatefunctions_test.go diff --git a/Makefile b/Makefile index 905bcf33f4..64a9fb6669 100644 --- a/Makefile +++ b/Makefile @@ -149,6 +149,10 @@ migratecheckpoint: GO111MODULE=on CGO_ENABLED=0 go build -trimpath -o ./bin/migratecheckpoint_$(GOOS)_$(GOARCH)$(EXTENSION) $(BUILD_INFO) ./cmd/migratecheckpoint ln -sf migratecheckpoint_$(GOOS)_$(GOARCH)$(EXTENSION) ./bin/migratecheckpoint +.PHONY: bundle.d +bundle.d: + go generate -tags bundle.d ./... + .PHONY: add-tag add-tag: @[ "${TAG}" ] || ( echo ">> env var TAG is not set"; exit 1 ) diff --git a/go.mod b/go.mod index ff78f810a0..bfbbb1297a 100644 --- a/go.mod +++ b/go.mod @@ -471,7 +471,7 @@ require ( gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/square/go-jose.v2 v2.6.0 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect + gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.26.1 // indirect k8s.io/apimachinery v0.26.1 // indirect k8s.io/client-go v0.26.1 // indirect diff --git a/internal/confmapprovider/discovery/README.md b/internal/confmapprovider/discovery/README.md index a885abf510..e9090db878 100644 --- a/internal/confmapprovider/discovery/README.md +++ b/internal/confmapprovider/discovery/README.md @@ -16,7 +16,7 @@ graph LR 2 --> 2a1>otlp.yaml] 2a1 --> 2b1[[otlp:
endpoint: 1.2.3.4:2345]] 2 --> 2a2>logging.yaml] - 2a2 --> 2b2[[logging:
logLevel: debug]] + 2a2 --> 2b2[[logging:
verbosity: detailed]] end config.d --> 3[/extensions/] subgraph 3a[extensions] @@ -35,17 +35,70 @@ graph LR config.d --> 5[/receivers/] subgraph 5a[receivers] 5 --> 5a1>otlp.yaml] - 5a1 --> 5b1[[otlp:
protocols:
grpc]] + 5a1 --> 5b1[[otlp:
protocols:
grpc:]] end ``` -This component is currently exposed in the Collector via the `--configd` option with corresponding -`--config-dir ` and `SPLUNK_CONFIG_DIR` option and environment variable to load -additional components and service configuration from the specified `config.d` directory (`/etc/otel/collector/config.d` -by default). +This component is currently supported in the Collector settings via the following commandline options: -This component is also exposed in the Collector via the `--discovery [--dry-run]` option that also uses the -`--config-dir ` and `SPLUNK_CONFIG_DIR` option and environment variable that attempts to -instantiate any `.discovery.yaml` receivers using corresponding `.discovery.yaml` observers in a "preflight" -Collector service, using any successfully discovered entities in the final config, or writing it to stdout -if `--dry-run` was specified. +| option | environment variable | default | description | +|----------------|----------------------|--------------------------------|----------------------------------------------------------------------------------------------------------------------------------------| +| `--configd` | none | disabled | Whether to enable `config.d` functionality for final Collector config content. | +| `--config-dir` | `SPLUNK_CONFIG_DIR` | `/etc/otel/collector/config.d` | The root `config.d` directory to walk for component directories and yaml mapping files. | +| `--dry-run` | none | disabled | Whether to report the final assembled config contents to stdout before immediately exiting. This can be used with or without `config.d` | + +To source only `config.d` content and not an additional or default configuration file, the `--config` option or +`SPLUNK_CONFIG` environment variable must be set to `/dev/null` or an arbitrary empty file: + +```bash +$ # run the Collector without a config file using components from a local ./config.d config directory, +$ # printing the config to stdout before exiting instead of starting the Collector service: +$ bin/otelcol --config /dev/null --configd --config-dir ./config.d --dry-run +2023/02/24 19:54:23 settings.go:331: Set config to [/dev/null] +2023/02/24 19:54:23 settings.go:384: Set ballast to 168 MiB +2023/02/24 19:54:23 settings.go:400: Set memory limit to 460 MiB +exporters: + logging: + verbosity: detailed + otlp: + endpoint: 1.2.3.4:2345 +extensions: + health_check: + path: /health + zpages: + endpoint: 0.0.0.0:1234 +processors: + batch: {} + resourcedetection: + detectors: + - system +receivers: + otlp: + protocols: + grpc: null +service: + pipelines: + metrics: + exporters: + - logging + receivers: + - otlp +``` + +## Discovery Mode + +This component also provides a `--discovery [--dry-run]` option compatible with `config.d` that attempts to instantiate +any `.discovery.yaml` receivers using corresponding `.discovery.yaml` observers in a "preflight" Collector service. +Discovery mode will: + +1. Load and attempt to start any observers in `config.d/extensions/.discovery.yaml`. +1. Load and attempt to start any receiver blocks in `config.d/receivers/.discovery.yaml` in a +[Discovery Receiver](../../receiver/discoveryreceiver/README.md) instance to receive discovery events from all +successfully started observers. +1. Wait 10s or the configured `SPLUNK_DISCOVERY_DURATION` environment variable [`time.Duration`](https://pkg.go.dev/time#ParseDuration). +1. Embed any receiver instances' configs resulting in a `discovery.status` of `successful` inside a `receiver_creator/discovery` receiver's configuration to be passed to the final Collector service config (or outputted w/ `--dry-run`). +1. Log any receiver resulting in a `discovery.status` of `partial` with the configured guidance for setting any relevant discovery properties. +1. Stop all temporary components before continuing on to the actual Collector service (or exiting early with `--dry-run`). + + +By default the Discovery mode is provided with premade discovery config components in [`bundle.d`](./bundle/README.md). \ No newline at end of file diff --git a/internal/confmapprovider/discovery/bundle/README.md b/internal/confmapprovider/discovery/bundle/README.md new file mode 100644 index 0000000000..54428be8a6 --- /dev/null +++ b/internal/confmapprovider/discovery/bundle/README.md @@ -0,0 +1,72 @@ +## bundle.d + +`bundle.d` refers to the [`embed.FS`](https://pkg.go.dev/embed#hdr-File_Systems) config directory made available by the +[`bundle.BundledFS`](./bundle.go). It currently consists of all `./bundle.d/extensions/*.discovery.yaml` and +`./bundle.d/receivers/*.discovery.yaml` files that are generated by the `discoverybundler` cmd as used by `go:generate` +directives in [bundle_gen.go](./bundle_gen.go). + +To construct the latest bundle.d contents before building the collector run: + +```bash +$ make bundle.d +``` + +### *.discovery.yaml.tmpl + +All discovery config component discovery.yaml files are generated from [`text/template`](https://pkg.go.dev/text/template) +`discovery.yaml.tmpl` files using built-in validators and property guidance helpers: + +Example `redis.discovery.yaml.tmpl`: + +```yaml +{{ receiver "redis" }}: + rule: + docker_observer: type == "container" and port == 6379 + <...> + status: + <...> + statements: + partial: + - regexp: 'ERR AUTH.*' + first_only: true + log_record: + severity_text: info + body: >- + Please ensure your redis password is correctly specified with + `--set {{ configProperty "password" "" }}` or + `{{ configPropertyEnvVar "password" "" }}` environment variable. +``` + +After adding the required generate directive to `bundle_gen.go` and running `make bundle.d`: + +```go +//go:generate discoverybundler -r -t bundle.d/receivers/redis.discovery.yaml.tmpl +``` + +There is now a corresponding `bundle.d/receiver/redis.discovery.yaml`: + +```yaml +##################################################################################### +# This file is generated by the Splunk Distribution of the OpenTelemetry Collector. # +##################################################################################### +redis: + rule: + docker_observer: type == "container" and port == 6379 + <...> + status: + <...> + statements: + partial: + - regexp: 'ERR AUTH.*' + first_only: true + log_record: + severity_text: info + body: >- + Please ensure your redis password is correctly specified with + `--set splunk.discovery.receivers.redis.config.password=""` or + `SPLUNK_DISCOVERY_RECEIVERS_redis_CONFIG_password=""` environment variable. +``` + +When building the collector afterward, this redis receiver discovery config is now made available to discovery mode, and +it can be disabled by `--set splunk.discovery.receivers.redis.enabled=false` or +`SPLUNK_DISCOVERY_RECEIVERS_redis_ENABLED=false`. diff --git a/internal/confmapprovider/discovery/bundle/bundle.d/extensions/docker-observer.discovery.yaml b/internal/confmapprovider/discovery/bundle/bundle.d/extensions/docker-observer.discovery.yaml new file mode 100644 index 0000000000..1ee84bca81 --- /dev/null +++ b/internal/confmapprovider/discovery/bundle/bundle.d/extensions/docker-observer.discovery.yaml @@ -0,0 +1,4 @@ +##################################################################################### +# This file is generated by the Splunk Distribution of the OpenTelemetry Collector. # +##################################################################################### +docker_observer: diff --git a/internal/confmapprovider/discovery/bundle/bundle.d/extensions/docker-observer.discovery.yaml.tmpl b/internal/confmapprovider/discovery/bundle/bundle.d/extensions/docker-observer.discovery.yaml.tmpl new file mode 100644 index 0000000000..7dfb614d23 --- /dev/null +++ b/internal/confmapprovider/discovery/bundle/bundle.d/extensions/docker-observer.discovery.yaml.tmpl @@ -0,0 +1 @@ +{{ extension "docker_observer" }}: diff --git a/internal/confmapprovider/discovery/bundle/bundle.d/extensions/host-observer.discovery.yaml b/internal/confmapprovider/discovery/bundle/bundle.d/extensions/host-observer.discovery.yaml new file mode 100644 index 0000000000..18ee16d584 --- /dev/null +++ b/internal/confmapprovider/discovery/bundle/bundle.d/extensions/host-observer.discovery.yaml @@ -0,0 +1,4 @@ +##################################################################################### +# This file is generated by the Splunk Distribution of the OpenTelemetry Collector. # +##################################################################################### +host_observer: diff --git a/internal/confmapprovider/discovery/bundle/bundle.d/extensions/host-observer.discovery.yaml.tmpl b/internal/confmapprovider/discovery/bundle/bundle.d/extensions/host-observer.discovery.yaml.tmpl new file mode 100644 index 0000000000..aa3f30e6b0 --- /dev/null +++ b/internal/confmapprovider/discovery/bundle/bundle.d/extensions/host-observer.discovery.yaml.tmpl @@ -0,0 +1 @@ +{{ extension "host_observer" }}: diff --git a/internal/confmapprovider/discovery/bundle/bundle.d/extensions/k8s-observer.discovery.yaml b/internal/confmapprovider/discovery/bundle/bundle.d/extensions/k8s-observer.discovery.yaml new file mode 100644 index 0000000000..bfc729c1da --- /dev/null +++ b/internal/confmapprovider/discovery/bundle/bundle.d/extensions/k8s-observer.discovery.yaml @@ -0,0 +1,5 @@ +##################################################################################### +# This file is generated by the Splunk Distribution of the OpenTelemetry Collector. # +##################################################################################### +k8s_observer: + auth_type: serviceAccount diff --git a/internal/confmapprovider/discovery/bundle/bundle.d/extensions/k8s-observer.discovery.yaml.tmpl b/internal/confmapprovider/discovery/bundle/bundle.d/extensions/k8s-observer.discovery.yaml.tmpl new file mode 100644 index 0000000000..d4a541396b --- /dev/null +++ b/internal/confmapprovider/discovery/bundle/bundle.d/extensions/k8s-observer.discovery.yaml.tmpl @@ -0,0 +1,2 @@ +{{ extension "k8s_observer" }}: + auth_type: serviceAccount diff --git a/internal/confmapprovider/discovery/bundle/bundle.d/receivers/smartagent-postgresql.discovery.yaml b/internal/confmapprovider/discovery/bundle/bundle.d/receivers/smartagent-postgresql.discovery.yaml new file mode 100644 index 0000000000..156f692393 --- /dev/null +++ b/internal/confmapprovider/discovery/bundle/bundle.d/receivers/smartagent-postgresql.discovery.yaml @@ -0,0 +1,49 @@ +##################################################################################### +# This file is generated by the Splunk Distribution of the OpenTelemetry Collector. # +##################################################################################### +smartagent/postgresql: + rule: + docker_observer: type == "container" and port == 5432 + host_observer: type == "hostport" and command contains "pg" and port == 5432 + config: + default: + type: postgresql + connectionString: 'sslmode=disable user={{.username}} password={{.password}}' + params: + username: bundle.default + password: bundle.default + masterDBName: postgres + status: + metrics: + successful: + - strict: postgres_block_hit_ratio + first_only: true + log_record: + severity_text: info + body: postgresql SA receiver working! + statements: + failed: + - regexp: '.* connect: connection refused' + first_only: true + log_record: + severity_text: info + body: container appears to not be accepting postgres connections + partial: + - regexp: '.*pq: password authentication failed for user.*' + first_only: true + log_record: + severity_text: info + body: >- + Please ensure your user credentials are correctly specified with + `--set splunk.discovery.receivers.smartagent/postgresql.config.params::username=""` and + `--set splunk.discovery.receivers.smartagent/postgresql.config.params::password=""` or + `SPLUNK_DISCOVERY_RECEIVERS_smartagent_x2f_postgresql_CONFIG_params_x3a__x3a_username=""` and + `SPLUNK_DISCOVERY_RECEIVERS_smartagent_x2f_postgresql_CONFIG_params_x3a__x3a_password=""` environment variables. + - regexp: '.*pq: database ".*" does not exist.*' + first_only: true + log_record: + severity_text: info + body: >- + Please ensure your target database is correctly specified with + `--set splunk.discovery.receivers.smartagent/postgresql.config.masterDBName=""` or + `SPLUNK_DISCOVERY_RECEIVERS_smartagent_x2f_postgresql_CONFIG_masterDBName=""` environment variable. diff --git a/internal/confmapprovider/discovery/bundle/bundle.d/receivers/smartagent-postgresql.discovery.yaml.tmpl b/internal/confmapprovider/discovery/bundle/bundle.d/receivers/smartagent-postgresql.discovery.yaml.tmpl new file mode 100644 index 0000000000..739ab1b97a --- /dev/null +++ b/internal/confmapprovider/discovery/bundle/bundle.d/receivers/smartagent-postgresql.discovery.yaml.tmpl @@ -0,0 +1,46 @@ +{{ receiver "smartagent/postgresql" }}: + rule: + docker_observer: type == "container" and port == 5432 + host_observer: type == "hostport" and command contains "pg" and port == 5432 + config: + default: + type: postgresql + connectionString: 'sslmode=disable user={{ "{{.username}}" }} password={{ "{{.password}}" }}' + params: + username: bundle.default + password: bundle.default + masterDBName: postgres + status: + metrics: + successful: + - strict: postgres_block_hit_ratio + first_only: true + log_record: + severity_text: info + body: postgresql SA receiver working! + statements: + failed: + - regexp: '.* connect: connection refused' + first_only: true + log_record: + severity_text: info + body: container appears to not be accepting postgres connections + partial: + - regexp: '.*pq: password authentication failed for user.*' + first_only: true + log_record: + severity_text: info + body: >- + Please ensure your user credentials are correctly specified with + `--set {{ configProperty "params" "username" "" }}` and + `--set {{ configProperty "params" "password" "" }}` or + `{{ configPropertyEnvVar "params" "username" "" }}` and + `{{ configPropertyEnvVar "params" "password" "" }}` environment variables. + - regexp: '.*pq: database ".*" does not exist.*' + first_only: true + log_record: + severity_text: info + body: >- + Please ensure your target database is correctly specified with + `--set {{ configProperty "masterDBName" "" }}` or + `{{ configPropertyEnvVar "masterDBName" "" }}` environment variable. diff --git a/internal/confmapprovider/discovery/bundle/bundle.go b/internal/confmapprovider/discovery/bundle/bundle.go new file mode 100644 index 0000000000..40104661ae --- /dev/null +++ b/internal/confmapprovider/discovery/bundle/bundle.go @@ -0,0 +1,29 @@ +// Copyright Splunk, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bundle + +import ( + "embed" +) + +// BundledFS is the in-executable filesystem that contains all bundled discovery config.d components. +// +// If you are bootstrapping bundle_gen.go or the `discoverybundler` cmd without any rendered files in bundle.d, +// comment out the below embed directives before installing to prevent "no matching files found" +// build errors. +// +//go:embed bundle.d/extensions/*.discovery.yaml +//go:embed bundle.d/receivers/*.discovery.yaml +var BundledFS embed.FS diff --git a/internal/confmapprovider/discovery/bundle/bundle_gen.go b/internal/confmapprovider/discovery/bundle/bundle_gen.go new file mode 100644 index 0000000000..99580e4717 --- /dev/null +++ b/internal/confmapprovider/discovery/bundle/bundle_gen.go @@ -0,0 +1,27 @@ +// Copyright Splunk, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build bundle.d + +// These are the discovery config component generating statements. +// In order to update run go generate -tags bundle.d ./... +//go:generate go install github.com/signalfx/splunk-otel-collector/internal/confmapprovider/discovery/bundle/cmd/discoverybundler + +//go:generate discoverybundler -r -t bundle.d/receivers/smartagent-postgresql.discovery.yaml.tmpl + +//go:generate discoverybundler -r -t bundle.d/extensions/docker-observer.discovery.yaml.tmpl +//go:generate discoverybundler -r -t bundle.d/extensions/host-observer.discovery.yaml.tmpl +//go:generate discoverybundler -r -t bundle.d/extensions/k8s-observer.discovery.yaml.tmpl + +package bundle diff --git a/internal/confmapprovider/discovery/bundle/bundle_test.go b/internal/confmapprovider/discovery/bundle/bundle_test.go new file mode 100644 index 0000000000..ecb804f1fd --- /dev/null +++ b/internal/confmapprovider/discovery/bundle/bundle_test.go @@ -0,0 +1,38 @@ +// Copyright Splunk, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bundle + +import ( + "io/fs" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestBundleDir(t *testing.T) { + receivers, err := fs.Glob(BundledFS, "bundle.d/receivers/*.discovery.yaml") + require.NoError(t, err) + require.Equal(t, []string{ + "bundle.d/receivers/smartagent-postgresql.discovery.yaml", + }, receivers) + + extensions, err := fs.Glob(BundledFS, "bundle.d/extensions/*.discovery.yaml") + require.NoError(t, err) + require.Equal(t, []string{ + "bundle.d/extensions/docker-observer.discovery.yaml", + "bundle.d/extensions/host-observer.discovery.yaml", + "bundle.d/extensions/k8s-observer.discovery.yaml", + }, extensions) +} diff --git a/internal/confmapprovider/discovery/bundle/cmd/discoverybundler/main.go b/internal/confmapprovider/discovery/bundle/cmd/discoverybundler/main.go new file mode 100644 index 0000000000..41065fd253 --- /dev/null +++ b/internal/confmapprovider/discovery/bundle/cmd/discoverybundler/main.go @@ -0,0 +1,84 @@ +// Copyright Splunk, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "fmt" + "os" + "strings" + "text/template" + + flag "github.com/spf13/pflag" + "gopkg.in/yaml.v3" + + "github.com/signalfx/splunk-otel-collector/internal/confmapprovider/discovery/bundle" +) + +const ( + genHeader = `##################################################################################### +# This file is generated by the Splunk Distribution of the OpenTelemetry Collector. # +##################################################################################### +` +) + +type settings struct { + templateFile string + renderInParentDir bool +} + +func panicOnError(err error) { + if err != nil { + panic(err) + } +} + +func loadSettings() *settings { + s := &settings{} + flagSet := flag.NewFlagSet("discoverybundler", flag.ContinueOnError) + flagSet.StringVarP(&s.templateFile, "template", "t", "", "the discovery config template (.tmpl) to render") + flagSet.BoolVarP(&s.renderInParentDir, "render", "r", false, `whether to render in parent dir (sans ".tmpl")`) + panicOnError(flagSet.Parse(os.Args[1:])) + return s +} + +func main() { + s := loadSettings() + if s.templateFile == "" { + panic("empty templateFile") + } + if !strings.HasSuffix(s.templateFile, ".tmpl") { + panic(fmt.Errorf(`%q must end in ".tmpl"`, s.templateFile)) + } + tmpl, err := os.ReadFile(s.templateFile) + panicOnError(err) + t, err := template.New("discoverybundler").Funcs(bundle.FuncMap()).Parse(string(tmpl)) + panicOnError(err) + + out := &bytes.Buffer{} + out.WriteString(genHeader) + panicOnError(t.Execute(out, nil)) + + var rendered map[string]any + // confirm rendered is valid yaml + panicOnError(yaml.Unmarshal(out.Bytes(), &rendered)) + + outFilename := strings.TrimSuffix(s.templateFile, ".tmpl") + if s.renderInParentDir { + panicOnError(os.WriteFile(outFilename, out.Bytes(), 0600)) + } else { + fmt.Fprint(os.Stdout, out.String()) + } +} diff --git a/internal/confmapprovider/discovery/bundle/templatefunctions.go b/internal/confmapprovider/discovery/bundle/templatefunctions.go new file mode 100644 index 0000000000..dbe145e26e --- /dev/null +++ b/internal/confmapprovider/discovery/bundle/templatefunctions.go @@ -0,0 +1,138 @@ +// Copyright Splunk, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bundle + +import ( + "fmt" + "strings" + "text/template" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/otelcol" + + "github.com/signalfx/splunk-otel-collector/internal/components" + "github.com/signalfx/splunk-otel-collector/internal/confmapprovider/discovery/properties" +) + +func FuncMap() template.FuncMap { + dc := newDiscoveryConfig() + return map[string]any{ + "configProperty": dc.configProperty, + "configPropertyEnvVar": dc.configPropertyEnvVar, + "extension": dc.extension, + "receiver": dc.receiver, + } +} + +type discoveryConfig struct { + factories otelcol.Factories + componentID component.ID + componentKind component.Kind +} + +func newDiscoveryConfig() *discoveryConfig { + factories, err := components.Get() + if err != nil { + panic(fmt.Errorf("failed accessing distribution components: %w", err)) + } + return &discoveryConfig{ + factories: factories, + } +} + +func (dc *discoveryConfig) extension(id string) (string, error) { + return dc.setComponentType(id, component.KindExtension) +} + +func (dc *discoveryConfig) receiver(id string) (string, error) { + return dc.setComponentType(id, component.KindReceiver) +} + +func (dc *discoveryConfig) setComponentType(id string, kind component.Kind) (string, error) { + cid := &component.ID{} + if err := cid.UnmarshalText([]byte(id)); err != nil { + return "", err + } + dc.componentKind = kind + dc.componentID = *cid + switch kind { + case component.KindExtension: + if _, ok := dc.factories.Extensions[cid.Type()]; !ok { + return "", fmt.Errorf("no extension %q available in this distribution", cid.Type()) + } + case component.KindReceiver: + if _, ok := dc.factories.Receivers[cid.Type()]; !ok { + return "", fmt.Errorf("no receiver %q available in this distribution", cid.Type()) + } + default: + return "", fmt.Errorf("unsupported discovery config component kind %#v", kind) + } + return dc.componentID.String(), nil +} + +func (dc *discoveryConfig) configProperty(args ...string) (string, error) { + return dc.configPropertyWithStringer(args, "configProperty", func(property *properties.Property) string { + return fmt.Sprintf("%s=%q", property.Input, property.Val) + }) +} + +func (dc *discoveryConfig) configPropertyEnvVar(args ...string) (string, error) { + return dc.configPropertyWithStringer(args, "configPropertyEnvVar", func(property *properties.Property) string { + return fmt.Sprintf("%s=%q", property.ToEnvVar(), property.Val) + }) +} + +func (dc *discoveryConfig) configPropertyWithStringer(args []string, methodName string, stringer func(property *properties.Property) string) (string, error) { + prefix, err := dc.configPropertyPrefix(methodName, args) + if err != nil { + return "", err + } + property, err := configProperty(methodName, prefix, args) + if err != nil { + return "", err + } + + return stringer(property), nil +} + +func (dc *discoveryConfig) configPropertyPrefix(methodName string, args []string) (string, error) { + l := len(args) + if l < 2 { + return "", fmt.Errorf("%s takes key+ and value{1} arguments (minimum 2)", methodName) + } + var prefix string + switch dc.componentKind { + case component.KindReceiver: + prefix = fmt.Sprintf("splunk.discovery.receivers.%s.config", dc.componentID) + case component.KindExtension: + prefix = fmt.Sprintf("splunk.discovery.extensions.%s.config", dc.componentID) + default: + return "", fmt.Errorf("invalid discovery config component type %d", dc.componentKind) + } + return prefix, nil +} + +func configProperty(methodName, prefix string, args []string) (*properties.Property, error) { + la := len(args) + if la < 1 { + return nil, fmt.Errorf("%s requires at least a value", methodName) + } + key := "" + if la > 1 { + key = fmt.Sprintf(".%s", strings.Join(args[:la-1], "::")) + } + property := fmt.Sprintf("%s%s", prefix, key) + return properties.NewProperty(property, args[la-1]) +} diff --git a/internal/confmapprovider/discovery/bundle/templatefunctions_test.go b/internal/confmapprovider/discovery/bundle/templatefunctions_test.go new file mode 100644 index 0000000000..7f10a1f403 --- /dev/null +++ b/internal/confmapprovider/discovery/bundle/templatefunctions_test.go @@ -0,0 +1,111 @@ +// Copyright Splunk, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bundle + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestFuncMap(t *testing.T) { + fm := FuncMap() + functions := []string{ + "configProperty", + "configPropertyEnvVar", + "extension", + "receiver", + } + for _, function := range functions { + require.Contains(t, fm, function) + } + for function := range fm { + require.Contains(t, functions, function) + } +} + +func TestReceiverValidatesComponentType(t *testing.T) { + dc := newDiscoveryConfig() + cid, err := dc.receiver("not.real") + require.EqualError(t, err, `no receiver "not.real" available in this distribution`) + require.Empty(t, cid) + + cid, err = dc.receiver("otlp") + require.NoError(t, err) + require.Equal(t, "otlp", cid) + + cid, err = dc.receiver("otlp/name") + require.NoError(t, err) + require.Equal(t, "otlp/name", cid) +} + +func TestExtensionValidatesComponentType(t *testing.T) { + dc := newDiscoveryConfig() + cid, err := dc.extension("not.real") + require.EqualError(t, err, `no extension "not.real" available in this distribution`) + require.Empty(t, cid) + + cid, err = dc.extension("docker_observer") + require.NoError(t, err) + require.Equal(t, "docker_observer", cid) + + cid, err = dc.extension("docker_observer/name") + require.NoError(t, err) + require.Equal(t, "docker_observer/name", cid) +} + +func TestReceiverConfigProperties(t *testing.T) { + dc := newDiscoveryConfig() + cid, err := dc.receiver("otlp") + require.NoError(t, err) + require.Equal(t, "otlp", cid) + prop, err := dc.configProperty("one", "two", "three", "") + require.NoError(t, err) + require.Equal(t, `splunk.discovery.receivers.otlp.config.one::two::three=""`, prop) + + prop, err = dc.configProperty("invalid") + require.EqualError(t, err, "configProperty takes key+ and value{1} arguments (minimum 2)") + require.Empty(t, prop) + + prop, err = dc.configPropertyEnvVar("one", "two", "three", "") + require.NoError(t, err) + require.Equal(t, `SPLUNK_DISCOVERY_RECEIVERS_otlp_CONFIG_one_x3a__x3a_two_x3a__x3a_three=""`, prop) + + prop, err = dc.configPropertyEnvVar("invalid") + require.EqualError(t, err, "configPropertyEnvVar takes key+ and value{1} arguments (minimum 2)") + require.Empty(t, prop) +} + +func TestExtensionConfigProperties(t *testing.T) { + dc := newDiscoveryConfig() + cid, err := dc.extension("host_observer/name") + require.NoError(t, err) + require.Equal(t, "host_observer/name", cid) + prop, err := dc.configProperty("one", "two", "three", "") + require.NoError(t, err) + require.Equal(t, `splunk.discovery.extensions.host_observer/name.config.one::two::three=""`, prop) + + prop, err = dc.configProperty("invalid") + require.EqualError(t, err, "configProperty takes key+ and value{1} arguments (minimum 2)") + require.Empty(t, prop) + + prop, err = dc.configPropertyEnvVar("one", "two", "three", "") + require.NoError(t, err) + require.Equal(t, `SPLUNK_DISCOVERY_EXTENSIONS_host_x5f_observer_x2f_name_CONFIG_one_x3a__x3a_two_x3a__x3a_three=""`, prop) + + prop, err = dc.configPropertyEnvVar("invalid") + require.EqualError(t, err, "configPropertyEnvVar takes key+ and value{1} arguments (minimum 2)") + require.Empty(t, prop) +} diff --git a/internal/confmapprovider/discovery/config.go b/internal/confmapprovider/discovery/config.go index b359cf3bff..af4ad0ebb2 100644 --- a/internal/confmapprovider/discovery/config.go +++ b/internal/confmapprovider/discovery/config.go @@ -16,9 +16,9 @@ package discovery import ( "fmt" + "io" "io/fs" "os" - "path/filepath" "regexp" "sort" @@ -45,7 +45,7 @@ var ( defaultType = component.NewID("default") discoveryDirRegex = fmt.Sprintf("[^%s]*", compilablePathSeparator) - serviceEntryRegex = regexp.MustCompile(fmt.Sprintf("%s%sservice\\.(yaml|yml)$", discoveryDirRegex, compilablePathSeparator)) + serviceEntryRegex = regexp.MustCompile(fmt.Sprintf("%s%s*service\\.(yaml|yml)$", discoveryDirRegex, compilablePathSeparator)) _, exporterEntryRegex = dirAndEntryRegex("exporters") extensionsDirRegex, extensionEntryRegex = dirAndEntryRegex("extensions") @@ -100,7 +100,7 @@ func NewConfig(logger *zap.Logger) *Config { } func dirAndEntryRegex(dirName string) (*regexp.Regexp, *regexp.Regexp) { - dirRegex := regexp.MustCompile(fmt.Sprintf("%s%s%s", discoveryDirRegex, compilablePathSeparator, dirName)) + dirRegex := regexp.MustCompile(fmt.Sprintf("%s%s*%s", discoveryDirRegex, compilablePathSeparator, dirName)) entryRegex := regexp.MustCompile(fmt.Sprintf("%s%s[^%s]*\\.(yaml|yml)$", dirRegex, compilablePathSeparator, compilablePathSeparator)) return dirRegex, entryRegex } @@ -216,31 +216,41 @@ func (c *Config) Load(configDPath string) error { if c == nil { return fmt.Errorf("config must not be nil to be loaded (use NewConfig())") } - err := filepath.WalkDir(configDPath, func(path string, d fs.DirEntry, err error) error { + return c.LoadFS(os.DirFS(configDPath)) +} + +// LoadFS will walk the provided filesystem, loading the component files as they are discovered, +// determined by their parent directory and filename. +func (c *Config) LoadFS(dirfs fs.FS) error { + if c == nil { + return fmt.Errorf("config must not be nil to be loaded (use NewConfig())") + } + err := fs.WalkDir(dirfs, ".", func(path string, d fs.DirEntry, err error) error { c.logger.Debug("loading component", zap.String("path", path), zap.String("DirEntry", fmt.Sprintf("%#v", d)), zap.Error(err)) if err != nil { return err } + switch { case isServiceEntryPath(path): // c.Service is not a map[string]ServiceEntry, so we form a tmp // and unmarshal to the underlying ServiceEntry tmpSEMap := map[string]ServiceEntry{typeService: c.Service} - return loadEntry(typeService, path, tmpSEMap) + return loadEntry(typeService, dirfs, path, tmpSEMap) case isExporterEntryPath(path): - return loadEntry(typeExporter, path, c.Exporters) + return loadEntry(typeExporter, dirfs, path, c.Exporters) case isExtensionEntryPath(path): if isDiscoveryObserverEntryPath(path) { - return loadEntry(typeDiscoveryObserver, path, c.DiscoveryObservers) + return loadEntry(typeDiscoveryObserver, dirfs, path, c.DiscoveryObservers) } - return loadEntry(typeExtension, path, c.Extensions) + return loadEntry(typeExtension, dirfs, path, c.Extensions) case isProcessorEntryPath(path): - return loadEntry(typeProcessor, path, c.Processors) + return loadEntry(typeProcessor, dirfs, path, c.Processors) case isReceiverEntryPath(path): if isReceiverToDiscoverEntryPath(path) { - return loadEntry(typeReceiverToDiscover, path, c.ReceiversToDiscover) + return loadEntry(typeReceiverToDiscover, dirfs, path, c.ReceiversToDiscover) } - return loadEntry(typeReceiver, path, c.Receivers) + return loadEntry(typeReceiver, dirfs, path, c.Receivers) default: c.logger.Debug("Disregarding path", zap.String("path", path)) } @@ -325,10 +335,10 @@ func isReceiverToDiscoverEntryPath(path string) bool { return receiverToDiscoverEntryRegex.MatchString(path) } -func loadEntry[K keyType, V entryType](componentType, path string, target map[K]V) error { +func loadEntry[K keyType, V entryType](componentType string, fs fs.FS, path string, target map[K]V) error { tmpDest := map[K]V{} - componentID, err := unmarshalEntry(componentType, path, &tmpDest) + componentID, err := unmarshalEntry(componentType, fs, path, &tmpDest) noTypeK, err2 := stringToKeyType(discovery.NoType.String(), componentID) if err2 != nil { return err2 @@ -363,7 +373,7 @@ func loadEntry[K keyType, V entryType](componentType, path string, target map[K] return nil } -func unmarshalEntry[K keyType, V entryType](componentType, path string, dst *map[K]V) (componentID K, err error) { +func unmarshalEntry[K keyType, V entryType](componentType string, fs fs.FS, path string, dst *map[K]V) (componentID K, err error) { if dst == nil { err = fmt.Errorf("cannot load %s into nil entry", componentType) return @@ -379,7 +389,7 @@ func unmarshalEntry[K keyType, V entryType](componentType, path string, dst *map unmarshalDst = &se } - if err = unmarshalYaml(path, unmarshalDst); err != nil { + if err = unmarshalYaml(fs, path, unmarshalDst); err != nil { err = fmt.Errorf("failed unmarshalling component %s: %w", componentType, err) return } @@ -422,8 +432,13 @@ func unmarshalEntry[K keyType, V entryType](componentType, path string, dst *map return componentIDs[0], nil } -func unmarshalYaml(path string, out any) error { - contents, err := os.ReadFile(filepath.Clean(path)) +func unmarshalYaml(fs fs.FS, path string, out any) error { + f, err := fs.Open(path) + if err != nil { + return err + } + defer f.Close() + contents, err := io.ReadAll(f) if err != nil { return fmt.Errorf("failed reading file %q: %w", path, err) } @@ -477,7 +492,53 @@ func keyTypeToString[K keyType](key K) string { var compilablePathSeparator = func() string { if os.PathSeparator == '\\' { - return "\\\\" + // fs.Stat doesn't use os.PathSeparator so accept '/' as well. + // TODO: determine if we even need anything but "/" + return "(\\\\|/)" } return string(os.PathSeparator) }() + +func mergeConfigWithBundle(userCfg *Config, bundleCfg *Config) error { + for obs, bundledObs := range bundleCfg.DiscoveryObservers { + userObs, ok := userCfg.DiscoveryObservers[obs] + if !ok { + userCfg.DiscoveryObservers[obs] = bundledObs + continue + } + bundledConfMap := confmap.NewFromStringMap(bundledObs.ToStringMap()) + userConfMap := confmap.NewFromStringMap(userObs.ToStringMap()) + if err := bundledConfMap.Merge(userConfMap); err != nil { + return fmt.Errorf("failed merged user and bundled observer %q discovery configs: %w", obs, err) + } + userCfg.DiscoveryObservers[obs] = ExtensionEntry{Entry: bundledConfMap.ToStringMap()} + } + for rec, bundledRec := range bundleCfg.ReceiversToDiscover { + userRec, ok := userCfg.ReceiversToDiscover[rec] + if !ok { + userCfg.ReceiversToDiscover[rec] = bundledRec + continue + } + bundledConfMap := confmap.NewFromStringMap(bundledRec.ToStringMap()) + userConfMap := confmap.NewFromStringMap(userRec.ToStringMap()) + if err := bundledConfMap.Merge(userConfMap); err != nil { + return fmt.Errorf("failed merged user and bundled receiver %q discovery configs: %w", rec, err) + } + receiver := ReceiverToDiscoverEntry{ + Rule: bundledRec.Rule, Config: bundledRec.Config, Entry: bundledConfMap.ToStringMap(), + } + for cid, rule := range userRec.Rule { + receiver.Rule[cid] = rule + } + for obs, config := range userRec.Config { + if bundledConfig, ok := bundledRec.Config[obs]; ok { + bundledConf := confmap.NewFromStringMap(bundledConfig) + bundledConf.Merge(confmap.NewFromStringMap(config)) + config = bundledConf.ToStringMap() + } + receiver.Config[obs] = config + } + userCfg.ReceiversToDiscover[rec] = receiver + } + return nil +} diff --git a/internal/confmapprovider/discovery/config_test.go b/internal/confmapprovider/discovery/config_test.go index 9b009b488d..20188c8cea 100644 --- a/internal/confmapprovider/discovery/config_test.go +++ b/internal/confmapprovider/discovery/config_test.go @@ -23,7 +23,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/component" - "go.uber.org/zap/zaptest" + "go.uber.org/zap" ) func TestServiceEntryPath(t *testing.T) { @@ -280,7 +280,7 @@ var expectedConfig = Config{ func TestConfig(t *testing.T) { configDir := filepath.Join(".", "testdata", "config.d") - cfg := NewConfig(zaptest.NewLogger(t)) + cfg := NewConfig(zap.NewNop()) require.NotNil(t, cfg) require.NoError(t, cfg.Load(configDir)) cfg.logger = nil // unset for equality check @@ -323,7 +323,7 @@ var expectedServiceConfig = map[string]any{ func TestToServiceConfig(t *testing.T) { configDir := filepath.Join(".", "testdata", "config.d") - cfg := NewConfig(zaptest.NewLogger(t)) + cfg := NewConfig(zap.NewNop()) require.NotNil(t, cfg) require.NoError(t, cfg.Load(configDir)) sc := cfg.toServiceConfig() @@ -332,7 +332,7 @@ func TestToServiceConfig(t *testing.T) { func TestConfigWithTwoReceiversInOneFile(t *testing.T) { configDir := filepath.Join(".", "testdata", "double-receiver-item-config.d") - logger := zaptest.NewLogger(t) + logger := zap.NewNop() cfg := NewConfig(logger) require.NotNil(t, cfg) err := cfg.Load(configDir) diff --git a/internal/confmapprovider/discovery/discoverer.go b/internal/confmapprovider/discovery/discoverer.go index 8484b17962..db3a4b0501 100644 --- a/internal/confmapprovider/discovery/discoverer.go +++ b/internal/confmapprovider/discovery/discoverer.go @@ -143,6 +143,11 @@ func (d *discoverer) discover(cfg *Config) (map[string]any, error) { return nil, err } + if len(discoveryObservers) == 0 { + fmt.Fprintf(os.Stderr, "No discovery observers have been configured.\n") + return nil, nil + } + var cancels []context.CancelFunc defer func() { @@ -152,30 +157,30 @@ func (d *discoverer) discover(cfg *Config) (map[string]any, error) { }() for observerID, observer := range discoveryObservers { - d.logger.Debug(fmt.Sprintf("starting observer %s", observerID.String())) + d.logger.Debug(fmt.Sprintf("starting observer %q", observerID)) ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) cancels = append(cancels, cancel) if e := observer.Start(ctx, d); e != nil { d.logger.Warn( - fmt.Sprintf("%s startup failed. Won't proceed with %s-based discovery", observerID.String(), observerID.Type()), + fmt.Sprintf("%q startup failed. Won't proceed with %q-based discovery", observerID, observerID.Type()), zap.Error(e), ) } } for receiverID, receiver := range discoveryReceivers { - d.logger.Debug(fmt.Sprintf("starting receiver %s", receiverID.String())) + d.logger.Debug(fmt.Sprintf("starting receiver %q", receiverID)) ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) cancels = append(cancels, cancel) if err = receiver.Start(ctx, d); err != nil { d.logger.Warn( - fmt.Sprintf("%s startup failed.", receiverID.String()), + fmt.Sprintf("%q startup failed.", receiverID), zap.Error(err), ) } } - _, _ = fmt.Fprintf(os.Stderr, "Discovering for next %s...\n", d.duration.String()) + _, _ = fmt.Fprintf(os.Stderr, "Discovering for next %s...\n", d.duration) select { case <-time.After(d.duration): case <-context.Background().Done(): @@ -186,14 +191,14 @@ func (d *discoverer) discover(cfg *Config) (map[string]any, error) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) cancels = append(cancels, cancel) if e := receiver.Shutdown(ctx); e != nil { - d.logger.Warn(fmt.Sprintf("error shutting down receiver %s", receiverID.String()), zap.Error(e)) + d.logger.Warn(fmt.Sprintf("error shutting down receiver %q", receiverID), zap.Error(e)) } } for observerID, observer := range discoveryObservers { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) cancels = append(cancels, cancel) if e := observer.Shutdown(ctx); e != nil { - d.logger.Warn(fmt.Sprintf("error shutting down observer %s", observerID.String()), zap.Error(e)) + d.logger.Warn(fmt.Sprintf("error shutting down observer %q", observerID), zap.Error(e)) } } @@ -212,7 +217,12 @@ func (d *discoverer) createDiscoveryReceiversAndObservers(cfg *Config) (map[comp for _, observerID := range cfg.observersForDiscoveryMode() { observer, err := d.createObserver(observerID, cfg) if err != nil { - return nil, nil, err + d.logger.Info(fmt.Sprintf("failed creating %q extension. no service discovery possible on this platform", observerID), zap.Error(err)) + continue + } + if observer == nil { + // disabled by property + continue } d.extensions[observerID] = observer discoveryObservers[observerID] = observer @@ -220,7 +230,7 @@ func (d *discoverer) createDiscoveryReceiversAndObservers(cfg *Config) (map[comp discoveryReceiverDefaultConfig := discoveryReceiverFactory.CreateDefaultConfig() discoveryReceiverConfig, ok := discoveryReceiverDefaultConfig.(*discoveryreceiver.Config) if !ok { - return nil, nil, fmt.Errorf("failed to coerce to receivercreator.Config") + return nil, nil, fmt.Errorf("failed to coerce to discoveryreceiver.Config") } discoveryReceiverRaw := map[string]any{} @@ -246,8 +256,22 @@ func (d *discoverer) createDiscoveryReceiversAndObservers(cfg *Config) (map[comp return nil, nil, fmt.Errorf("failed obtaining receiver properties config: %w", e) } entryConf := confmap.NewFromStringMap(receiverEntry) + + if receiverPropertiesConf.IsSet("enabled") { + enabled := true + if strings.ToLower(fmt.Sprintf("%v", receiverPropertiesConf.Get("enabled"))) == "false" { + enabled = false + } + if !enabled { + continue + } + pc := receiverPropertiesConf.ToStringMap() + delete(pc, "enabled") + receiverPropertiesConf = confmap.NewFromStringMap(pc) + } + if err = entryConf.Merge(receiverPropertiesConf); err != nil { - return nil, nil, fmt.Errorf("failed merging receiver properties config: %w", err) + return nil, nil, fmt.Errorf("failed merging receiver %q properties config: %w", receiverID, err) } receiverEntry = entryConf.ToStringMap() } @@ -300,6 +324,19 @@ func (d *discoverer) createObserver(observerID component.ID, cfg *Config) (otelc if e != nil { return nil, fmt.Errorf("failed obtaining observer properties config: %w", e) } + if propertiesConf.IsSet("enabled") { + enabled := true + if strings.ToLower(fmt.Sprintf("%v", propertiesConf.Get("enabled"))) == "false" { + enabled = false + } + if !enabled { + return nil, nil + } + // delete enabled property since it's not valid config field + pc := propertiesConf.ToStringMap() + delete(pc, "enabled") + propertiesConf = confmap.NewFromStringMap(pc) + } if err = observerCfgMap.Merge(propertiesConf); err != nil { return nil, fmt.Errorf("failed merging observer properties config: %w", err) } @@ -308,11 +345,11 @@ func (d *discoverer) createObserver(observerID component.ID, cfg *Config) (otelc } if err = d.expandConverter.Convert(context.Background(), observerCfgMap); err != nil { - return nil, fmt.Errorf("error converting environment variables in %q config: %w", observerID.String(), err) + return nil, fmt.Errorf("error converting environment variables in %q config: %w", observerID, err) } if err = component.UnmarshalConfig(observerCfgMap, observerConfig); err != nil { - return nil, fmt.Errorf("failed unmarshaling %s config: %w", observerID.String(), err) + return nil, fmt.Errorf("failed unmarshaling %q config: %w", observerID, err) } if ce := d.logger.Check(zap.DebugLevel, "unmarshalled observer config"); ce != nil { @@ -322,7 +359,7 @@ func (d *discoverer) createObserver(observerID component.ID, cfg *Config) (otelc observerSettings := d.createExtensionCreateSettings(observerID.String()) observer, err := observerFactory.CreateExtension(context.Background(), observerSettings, observerConfig) if err != nil { - return nil, fmt.Errorf("failed creating %s extension: %w", observerID.String(), err) + return nil, fmt.Errorf("failed creating %q extension: %w", observerID, err) } return observer, nil } @@ -330,7 +367,7 @@ func (d *discoverer) createObserver(observerID component.ID, cfg *Config) (otelc func (d *discoverer) updateReceiverForObserver(receiverID component.ID, receiver ReceiverToDiscoverEntry, observerID component.ID) (bool, error) { observerRule, hasRule := receiver.Rule[observerID] if !hasRule { - d.logger.Debug(fmt.Sprintf("disregarding %s without a %s rule", receiverID.String(), observerID.String())) + d.logger.Debug(fmt.Sprintf("disregarding %q without a %q rule", receiverID, observerID)) return false, nil } receiver.Entry["rule"] = observerRule @@ -342,13 +379,13 @@ func (d *discoverer) updateReceiverForObserver(receiverID component.ID, receiver } observerConfigBlock, hasObserverConfigBlock := receiver.Config[observerID] if !hasObserverConfigBlock && !hasDefault { - d.logger.Debug(fmt.Sprintf("disregarding %s without a default and %s config", receiverID.String(), observerID.String())) + d.logger.Debug(fmt.Sprintf("disregarding %q without a default and %q config", receiverID, observerID)) return false, nil } if hasObserverConfigBlock { if hasDefault { if err := mergeMaps(defaultConfig, observerConfigBlock); err != nil { - return false, fmt.Errorf("failed merging %s config for %s: %w", receiverID.String(), observerID.String(), err) + return false, fmt.Errorf("failed merging %q config for %q: %w", receiverID, observerID, err) } } else { receiver.Entry["config"] = observerConfigBlock @@ -375,7 +412,7 @@ func (d *discoverer) discoveryConfig(cfg *Config) (map[string]any, error) { dCfg := confmap.New() receiverAdded := false for receiverID, receiverStatus := range d.discoveredReceivers { - if receiverStatus == discovery.Failed { + if receiverStatus != discovery.Successful { continue } if receiverCfgMap, ok := d.discoveredConfig[receiverID]; ok { @@ -415,7 +452,7 @@ func (d *discoverer) discoveryConfig(cfg *Config) (map[string]any, error) { }, } if err := extensions.Merge(confmap.NewFromStringMap(obsMap)); err != nil { - return nil, fmt.Errorf("failure merging %q with suggested config: %w", observerID.String(), err) + return nil, fmt.Errorf("failure merging %q with suggested config: %w", observerID, err) } observers = append(observers, observerID.String()) } @@ -474,7 +511,7 @@ func (c *Config) observersForDiscoveryMode() []component.ID { } func (d *discoverer) addUnexpandedReceiverConfig(receiverID, observerID component.ID, cfg map[string]any) { - d.logger.Debug(fmt.Sprintf("adding unexpanded config[%s][%s]: %v\n", receiverID.String(), observerID.String(), cfg)) + d.logger.Debug(fmt.Sprintf("adding unexpanded config[%q][%q]: %v\n", receiverID, observerID, cfg)) observerMap, ok := d.unexpandedReceiverEntries[receiverID] if !ok { observerMap = map[component.ID]map[string]any{} @@ -490,7 +527,7 @@ func (d *discoverer) getUnexpandedReceiverConfig(receiverID, observerID componen if hasReceiver { cfg, found = observerMap[observerID] } - d.logger.Debug(fmt.Sprintf("getting unexpanded config[%s][%s](%v): %v\n", receiverID.String(), observerID.String(), found, cfg)) + d.logger.Debug(fmt.Sprintf("getting unexpanded config[%q][%q](%v): %v\n", receiverID, observerID, found, cfg)) return cfg, found } @@ -628,7 +665,13 @@ func (d *discoverer) ConsumeLogs(_ context.Context, ld plog.Logs) error { d.logger.Debug("invalid status from log record", zap.Error(err), zap.Any("lr", lr.Body().AsRaw())) continue } - d.discoveredReceivers[receiverID] = determineCurrentStatus(currentReceiverStatus, rStatus) + receiverStatus := determineCurrentStatus(currentReceiverStatus, rStatus) + if receiverStatus == discovery.Partial { + fmt.Fprintf(os.Stderr, "Partially discovered %q using %q: %s\n", receiverID, observerID, lr.Body().AsString()) + } else if receiverStatus == discovery.Successful { + fmt.Fprintf(os.Stderr, "Successfully discovered %q using %q.\n", receiverID, observerID) + } + d.discoveredReceivers[receiverID] = receiverStatus d.discoveredObservers[observerID] = determineCurrentStatus(currentObserverStatus, rStatus) } } diff --git a/internal/confmapprovider/discovery/properties/env_var.go b/internal/confmapprovider/discovery/properties/env_var.go index 7e5173445a..d488a8ee0f 100644 --- a/internal/confmapprovider/discovery/properties/env_var.go +++ b/internal/confmapprovider/discovery/properties/env_var.go @@ -37,14 +37,15 @@ var envVarParser = participle.MustBuild[EnvVarProperty]( type EnvVarProperty struct { ComponentType string `parser:"'SPLUNK' Underscore 'DISCOVERY' Underscore @('RECEIVERS' | 'EXTENSIONS') Underscore"` Component EnvVarComponentID `parser:"@@"` - Key string `parser:"Underscore 'CONFIG' Underscore @(String|Underscore)+"` + Type string `parser:"Underscore @('CONFIG'|'ENABLED')"` + Key string `parser:"(Underscore @(String|Underscore)+)*"` Val string } type EnvVarComponentID struct { - Type string `parser:"@~(Underscore (?= 'CONFIG'))+"` + Type string `parser:"@~(Underscore (?= ('CONFIG'|'ENABLED')))+"` // _x2f_ -> '/' - Name string `parser:"(Underscore 'x2f' Underscore @(~(?= Underscore (?= 'CONFIG'))+|''))?"` + Name string `parser:"(Underscore 'x2f' Underscore @(~(?= Underscore (?= ('CONFIG'|'ENABLED')))+|''))?"` } func NewEnvVarProperty(property, val string) (*EnvVarProperty, error) { diff --git a/internal/confmapprovider/discovery/properties/env_var_test.go b/internal/confmapprovider/discovery/properties/env_var_test.go index 1a69a61d77..1d878899e6 100644 --- a/internal/confmapprovider/discovery/properties/env_var_test.go +++ b/internal/confmapprovider/discovery/properties/env_var_test.go @@ -21,16 +21,18 @@ import ( ) func TestEnvVarPropertyEBNF(t *testing.T) { - require.Equal(t, `EnvVarProperty = "SPLUNK" "DISCOVERY" ("RECEIVERS" | "EXTENSIONS") EnvVarComponentID "CONFIG" ( | )+ . -EnvVarComponentID = ~( (?= "CONFIG"))+ ( "x2f" (~(?= (?= "CONFIG"))+ | ""))? .`, envVarParser.String()) + require.Equal(t, `EnvVarProperty = "SPLUNK" "DISCOVERY" ("RECEIVERS" | "EXTENSIONS") EnvVarComponentID ("CONFIG" | "ENABLED") ( ( | )+)* . +EnvVarComponentID = ~( (?= ("CONFIG" | "ENABLED")))+ ( "x2f" (~(?= (?= ("CONFIG" | "ENABLED")))+ | ""))? .`, envVarParser.String()) } func TestValidEnvVarProperties(t *testing.T) { for _, tt := range []struct { expected *Property envVar string + val string }{ {envVar: "SPLUNK_DISCOVERY_RECEIVERS_receiver_x2d_type_x2f__CONFIG_one", + val: "val", expected: &Property{ stringMap: map[string]any{ "receivers": map[string]any{ @@ -42,30 +44,69 @@ func TestValidEnvVarProperties(t *testing.T) { }, ComponentType: "receivers", Component: ComponentID{Type: "receiver-type"}, + Type: "config", Key: "one", Val: "val", + Input: "splunk.discovery.receivers.receiver-type/.config.one", }, }, {envVar: "SPLUNK_DISCOVERY_EXTENSIONS_extension_x2e_type_x2f_extension____name_CONFIG_one_x3a__x3a_two", + val: "a.val", expected: &Property{ stringMap: map[string]any{ "extensions": map[string]any{ "extension.type/extension____name": map[string]any{ "one": map[string]any{ - "two": "val", + "two": "a.val", }, }, }, }, ComponentType: "extensions", Component: ComponentID{Type: "extension.type", Name: "extension____name"}, + Type: "config", Key: "one::two", - Val: "val", + Val: "a.val", + Input: "splunk.discovery.extensions.extension.type/extension____name.config.one::two", + }, + }, + {envVar: "SPLUNK_DISCOVERY_EXTENSIONS_extension_x2e_type_x2f_extension____name_ENABLED", + val: "False", + expected: &Property{ + stringMap: map[string]any{ + "extensions": map[string]any{ + "extension.type/extension____name": map[string]any{ + "enabled": "false", + }, + }, + }, + ComponentType: "extensions", + Component: ComponentID{Type: "extension.type", Name: "extension____name"}, + Type: "enabled", + Val: "false", + Input: "splunk.discovery.extensions.extension.type/extension____name.enabled", + }, + }, + {envVar: "SPLUNK_DISCOVERY_RECEIVERS_receiver_x2d_type_x2f__ENABLED", + val: "true", + expected: &Property{ + stringMap: map[string]any{ + "receivers": map[string]any{ + "receiver-type": map[string]any{ + "enabled": "true", + }, + }, + }, + ComponentType: "receivers", + Component: ComponentID{Type: "receiver-type"}, + Type: "enabled", + Val: "true", + Input: "splunk.discovery.receivers.receiver-type/.enabled", }, }, } { t.Run(tt.envVar, func(t *testing.T) { - p, ok, err := NewPropertyFromEnvVar(tt.envVar, "val") + p, ok, err := NewPropertyFromEnvVar(tt.envVar, tt.val) require.True(t, ok) require.NoError(t, err) require.NotNil(t, p) @@ -78,8 +119,8 @@ func TestInvalidEnvVarProperties(t *testing.T) { for _, tt := range []struct { envVar, expectedError string }{ - {envVar: "SPLUNK_DISCOVERY_NOTVALIDCOMPONENT_TYPE_CONFIG_ONE", expectedError: "invalid env var property (parsing error): invalid property env var (parsing error): SPLUNK_DISCOVERY:1:18: unexpected token \"NOTVALIDCOMPONENT\" (expected (\"RECEIVERS\" | \"EXTENSIONS\") EnvVarComponentID \"CONFIG\" ( | )+)"}, - {envVar: "SPLUNK_DISCOVERY_RECEIVERS_TYPE_NOTCONFIG_ONE", expectedError: "invalid env var property (parsing error): invalid property env var (parsing error): SPLUNK_DISCOVERY:1:46: unexpected token \"\" (expected \"CONFIG\" ( | )+)"}, + {envVar: "SPLUNK_DISCOVERY_NOTVALIDCOMPONENT_TYPE_CONFIG_ONE", expectedError: "invalid env var property (parsing error): invalid property env var (parsing error): SPLUNK_DISCOVERY:1:18: unexpected token \"NOTVALIDCOMPONENT\" (expected (\"RECEIVERS\" | \"EXTENSIONS\") EnvVarComponentID (\"CONFIG\" | \"ENABLED\") ( ( | )+)*)"}, + {envVar: "SPLUNK_DISCOVERY_RECEIVERS_TYPE_NOTCONFIG_ONE", expectedError: "invalid env var property (parsing error): invalid property env var (parsing error): SPLUNK_DISCOVERY:1:46: unexpected token \"\" (expected (\"CONFIG\" | \"ENABLED\") ( ( | )+)*)"}, {envVar: "SPLUNK_DISCOVERY_EXTENSIONS_TYPE_x2f_NAME_CONFIG_", expectedError: "invalid env var property (parsing error): invalid property env var (parsing error): SPLUNK_DISCOVERY:1:50: sub-expression ( | )+ must match at least once"}, } { t.Run(tt.envVar, func(t *testing.T) { diff --git a/internal/confmapprovider/discovery/properties/property.go b/internal/confmapprovider/discovery/properties/property.go index b2e4afbc0d..38c64af6ea 100644 --- a/internal/confmapprovider/discovery/properties/property.go +++ b/internal/confmapprovider/discovery/properties/property.go @@ -19,6 +19,7 @@ import ( "encoding/hex" "fmt" "regexp" + "strconv" "strings" "github.com/alecthomas/participle/v2" @@ -26,7 +27,7 @@ import ( "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/confmap" "go.uber.org/multierr" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) // Discovery properties are the method of configuring individual components for discovery mode. @@ -58,14 +59,16 @@ var parser = participle.MustBuild[Property]( type Property struct { stringMap map[string]any ComponentType string `parser:"'splunk' Dot 'discovery' Dot @('receivers' | 'extensions') Dot"` - Component ComponentID `parser:"@@"` - Key string `parser:"Dot 'config' Dot @(String|Dot|ForwardSlash)+"` + Component ComponentID `parser:"@@ Dot"` + Type string `parser:"((@'config' Dot)|@('enabled'))"` + Key string `parser:"@(String|Dot|ForwardSlash)*"` Val string + Input string } type ComponentID struct { - Type string `parser:"@~(ForwardSlash | (Dot (?= 'config')))+"` - Name string `parser:"(ForwardSlash @(~(Dot (?= 'config'))+)*)?"` + Type string `parser:"@~(ForwardSlash | (Dot (?= ('config'|'enabled'))))+"` + Name string `parser:"(ForwardSlash @(~(Dot (?= ('config'|'enabled')))+)*)?"` } func NewProperty(property, val string) (*Property, error) { @@ -74,20 +77,33 @@ func NewProperty(property, val string) (*Property, error) { return nil, fmt.Errorf("invalid property (parsing error): %w", err) } p.Val = val - var dst map[string]any - cfgItem := []byte(fmt.Sprintf("%s: %s", p.Key, val)) - if err = yaml.Unmarshal(cfgItem, &dst); err != nil { - return nil, fmt.Errorf("failed unmarshaling property %q: %w", p.Key, err) - } - config := confmap.NewFromStringMap(dst).ToStringMap() - if p.ComponentType == "receivers" { - config = map[string]any{"config": config} + + var subStringMap map[string]any + switch p.Type { + case "enabled": + bVal, e := strconv.ParseBool(p.Val) + if e != nil { + return nil, fmt.Errorf("failed parsing %q bool: %w", property, e) + } + p.Val = fmt.Sprintf("%t", bVal) + subStringMap = map[string]any{"enabled": p.Val} + case "config": + var dst map[string]any + cfgItem := []byte(fmt.Sprintf("%s: %s", p.Key, val)) + if err = yaml.Unmarshal(cfgItem, &dst); err != nil { + return nil, fmt.Errorf("failed unmarshaling property %q: %w", p.Key, err) + } + subStringMap = confmap.NewFromStringMap(dst).ToStringMap() + if p.ComponentType == "receivers" { + subStringMap = map[string]any{"config": subStringMap} + } } p.stringMap = map[string]any{ p.ComponentType: map[string]any{ - component.NewIDWithName(component.Type(p.Component.Type), p.Component.Name).String(): config, + component.NewIDWithName(component.Type(p.Component.Type), p.Component.Name).String(): subStringMap, }, } + p.Input = property return p, nil } @@ -96,11 +112,15 @@ func (p *Property) ToEnvVar() string { envVar := envVarPrefixS envVar = fmt.Sprintf("%s%s_", envVar, strings.ToUpper(p.ComponentType)) envVar = fmt.Sprintf("%s%s", envVar, wordify(p.Component.Type)) - if p.Component.Name != "" { + // preserve input of `component.type/` with no name + if p.Component.Name != "" || strings.HasPrefix(p.Input, fmt.Sprintf("splunk.discovery.%s.%s/.", p.ComponentType, p.Component.Type)) { envVar = fmt.Sprintf("%s%s", envVar, wordify(fmt.Sprintf("/%s", p.Component.Name))) } - envVar = fmt.Sprintf("%s_CONFIG_", envVar) - return fmt.Sprintf("%s%s", envVar, wordify(p.Key)) + envVar = fmt.Sprintf("%s_%s", envVar, strings.ToUpper(p.Type)) + if p.Type == "config" { + envVar = fmt.Sprintf("%s_%s", envVar, wordify(p.Key)) + } + return envVar } // ToStringMap() will return a map[string]any equivalent to the property's root-level confmap.ToStringMap() @@ -147,7 +167,11 @@ func NewPropertyFromEnvVar(envVar, val string) (*Property, bool, error) { return nil, true, fmt.Errorf("failed parsing env var property key: %w", err) } - property := fmt.Sprintf("splunk.discovery.%s.%s.config.%s", strings.ToLower(evp.ComponentType), cid, key) + pType := strings.ToLower(evp.Type) + property := fmt.Sprintf("splunk.discovery.%s.%s.%s", strings.ToLower(evp.ComponentType), cid, pType) + if pType == "config" { + property = fmt.Sprintf("%s.%s", property, key) + } prop, err := NewProperty(property, val) return prop, true, err diff --git a/internal/confmapprovider/discovery/properties/property_test.go b/internal/confmapprovider/discovery/properties/property_test.go index 59bb5d7997..d0f10400fb 100644 --- a/internal/confmapprovider/discovery/properties/property_test.go +++ b/internal/confmapprovider/discovery/properties/property_test.go @@ -26,8 +26,8 @@ import ( ) func TestPropertyEBNF(t *testing.T) { - require.Equal(t, `Property = "splunk" "discovery" ("receivers" | "extensions") ComponentID "config" ( | | )+ . -ComponentID = ~( | ( (?= "config")))+ ( ~( (?= "config"))+*)? .`, parser.String()) + require.Equal(t, `Property = "splunk" "discovery" ("receivers" | "extensions") ComponentID (("config" ) | "enabled") ( | | )* . +ComponentID = ~( | ( (?= ("config" | "enabled"))))+ ( ~( (?= ("config" | "enabled")))+*)? .`, parser.String()) } func TestWordifyHappyPath(t *testing.T) { @@ -70,6 +70,7 @@ func TestValidProperties(t *testing.T) { expected: &Property{ ComponentType: "receivers", Component: ComponentID{Type: "receivertype"}, + Type: "config", Key: "key", Val: "val", stringMap: map[string]any{ @@ -81,12 +82,14 @@ func TestValidProperties(t *testing.T) { }, }, }, + Input: "splunk.discovery.receivers.receivertype.config.key", }, }, {key: "splunk.discovery.extensions.extension-type/extensionname.config.key", val: "val", expected: &Property{ ComponentType: "extensions", Component: ComponentID{Type: "extension-type", Name: "extensionname"}, + Type: "config", Key: "key", Val: "val", stringMap: map[string]any{ @@ -96,12 +99,14 @@ func TestValidProperties(t *testing.T) { }, }, }, + Input: "splunk.discovery.extensions.extension-type/extensionname.config.key", }, }, {key: "splunk.discovery.receivers.receivertype/.config.key", val: "val", expected: &Property{ ComponentType: "receivers", Component: ComponentID{Type: "receivertype"}, + Type: "config", Key: "key", Val: "val", stringMap: map[string]any{ @@ -113,12 +118,14 @@ func TestValidProperties(t *testing.T) { }, }, }, + Input: "splunk.discovery.receivers.receivertype/.config.key", }, }, {key: "splunk.discovery.receivers.receiver_type/config.config.one::two::three", val: "val", expected: &Property{ ComponentType: "receivers", Component: ComponentID{Type: "receiver_type", Name: "config"}, + Type: "config", Key: "one::two::three", Val: "val", stringMap: map[string]any{ @@ -130,12 +137,14 @@ func TestValidProperties(t *testing.T) { }, }, }, + Input: "splunk.discovery.receivers.receiver_type/config.config.one::two::three", }, }, {key: "splunk.discovery.receivers.receiver.type////.config.one::config", val: "val", expected: &Property{ ComponentType: "receivers", Component: ComponentID{Type: "receiver.type", Name: "///"}, + Type: "config", Key: "one::config", Val: "val", stringMap: map[string]any{ @@ -146,12 +155,14 @@ func TestValidProperties(t *testing.T) { }, }, }, + Input: "splunk.discovery.receivers.receiver.type////.config.one::config", }, }, {key: "splunk.discovery.extensions.extension--0-1-with-config-in-type-_x64__x86_🙈🙉🙊4:000x0;;0;;0;;-___-----type/e/x/t/e%nso<=n=>nam/e-with-config.config.o::n::e.config", val: "val", expected: &Property{ ComponentType: "extensions", Component: ComponentID{Type: "extension--0-1-with-config-in-type-_x64__x86_🙈🙉🙊4:000x0;;0;;0;;-___-----type", Name: "e/x/t/e%nso<=n=>nam/e-with-config"}, + Type: "config", Key: "o::n::e.config", Val: "val", stringMap: map[string]any{ @@ -160,6 +171,41 @@ func TestValidProperties(t *testing.T) { "o": map[string]any{"n": map[string]any{"e.config": "val"}}}, }, }, + Input: "splunk.discovery.extensions.extension--0-1-with-config-in-type-_x64__x86_🙈🙉🙊4:000x0;;0;;0;;-___-----type/e/x/t/e%nso<=n=>nam/e-with-config.config.o::n::e.config", + }, + }, + {key: "splunk.discovery.receivers.receiver.type////.enabled", val: "false", + expected: &Property{ + stringMap: map[string]any{ + "receivers": map[string]any{ + "receiver.type////": map[string]any{ + "enabled": "false", + }, + }, + }, + ComponentType: "receivers", + Component: ComponentID{Type: "receiver.type", Name: "///"}, + Type: "enabled", + Key: "", + Val: "false", + Input: "splunk.discovery.receivers.receiver.type////.enabled", + }, + }, + {key: "splunk.discovery.receivers.receiver.type////.enabled", val: "T", + expected: &Property{ + stringMap: map[string]any{ + "receivers": map[string]any{ + "receiver.type////": map[string]any{ + "enabled": "true", + }, + }, + }, + ComponentType: "receivers", + Component: ComponentID{Type: "receiver.type", Name: "///"}, + Type: "enabled", + Key: "", + Val: "true", + Input: "splunk.discovery.receivers.receiver.type////.enabled", }, }, } { @@ -186,9 +232,9 @@ func TestInvalidProperties(t *testing.T) { for _, tt := range []struct { property, expectedError string }{ - {property: "splunk.discovery.invalid", expectedError: "invalid property (parsing error): splunk.discovery:1:18: unexpected token \"invalid\" (expected (\"receivers\" | \"extensions\") ComponentID \"config\" ( | | )+)"}, - {property: "splunk.discovery.extensions.config.one.two", expectedError: "invalid property (parsing error): splunk.discovery:1:43: unexpected token \"\" (expected \"config\" ( | | )+)"}, - {property: "splunk.discovery.receivers.type/name.config", expectedError: "invalid property (parsing error): splunk.discovery:1:44: unexpected token \"\" (expected ( | | )+)"}, + {property: "splunk.discovery.invalid", expectedError: "invalid property (parsing error): splunk.discovery:1:18: unexpected token \"invalid\" (expected (\"receivers\" | \"extensions\") ComponentID ((\"config\" ) | \"enabled\") ( | | )*)"}, + {property: "splunk.discovery.extensions.config.one.two", expectedError: "invalid property (parsing error): splunk.discovery:1:43: unexpected token \"\" (expected ((\"config\" ) | \"enabled\") ( | | )*)"}, + {property: "splunk.discovery.receivers.type/name.config", expectedError: "invalid property (parsing error): splunk.discovery:1:44: unexpected token \"\" (expected )"}, } { t.Run(tt.property, func(t *testing.T) { p, err := NewProperty(tt.property, "val") diff --git a/internal/confmapprovider/discovery/provider.go b/internal/confmapprovider/discovery/provider.go index b9e173fc4d..1417e7b5b8 100644 --- a/internal/confmapprovider/discovery/provider.go +++ b/internal/confmapprovider/discovery/provider.go @@ -24,6 +24,7 @@ import ( "go.uber.org/zap" "go.uber.org/zap/zapcore" + "github.com/signalfx/splunk-otel-collector/internal/confmapprovider/discovery/bundle" "github.com/signalfx/splunk-otel-collector/internal/confmapprovider/discovery/properties" "github.com/signalfx/splunk-otel-collector/internal/settings" ) @@ -118,13 +119,20 @@ func (m *mapProvider) retrieve(scheme string) func(context.Context, string, conf var cfg *Config var ok bool - configDir := uriVal - if cfg, ok = m.configs[configDir]; !ok { - cfg = NewConfig(m.logger) - if err := cfg.Load(configDir); err != nil { - return nil, err + if uriVal != "" { + if cfg, ok = m.configs[uriVal]; !ok { + cfg = NewConfig(m.logger) + m.logger.Debug("loading config.d", zap.String("config-dir", uriVal)) + if err := cfg.Load(uriVal); err != nil { + m.logger.Error("failed loading config.d", zap.String("config-dir", uriVal), zap.Error(err)) + return nil, err + } + m.logger.Debug("successfully loaded config.d", zap.String("config-dir", uriVal)) + m.configs[uriVal] = cfg } - m.configs[configDir] = cfg + } else { + // empty config to be noop for config.d or base for bundle.d + cfg = NewConfig(m.logger) } if strings.HasPrefix(uri, settings.ConfigDScheme) { @@ -132,6 +140,20 @@ func (m *mapProvider) retrieve(scheme string) func(context.Context, string, conf } if strings.HasPrefix(uri, settings.DiscoveryModeScheme) { + var bundledCfg *Config + if bundledCfg, ok = m.configs[""]; !ok { + m.logger.Debug("loading bundle.d") + bundledCfg = NewConfig(m.logger) + if err := bundledCfg.LoadFS(bundle.BundledFS); err != nil { + m.logger.Error("failed loading bundle.d", zap.Error(err)) + return nil, err + } + m.logger.Debug("successfully loaded bundle.d") + m.configs[""] = bundledCfg + } + if err := mergeConfigWithBundle(cfg, bundledCfg); err != nil { + return nil, fmt.Errorf("failed merging user and bundled discovery configs: %w", err) + } discoveryCfg, err := m.discoverer.discover(cfg) if err != nil { return nil, fmt.Errorf("failed to successfully discover target services: %w", err) diff --git a/tests/general/discoverymode/docker_observer_discovery_test.go b/tests/general/discoverymode/docker_observer_discovery_test.go index 2340ce9f24..b2686c206e 100644 --- a/tests/general/discoverymode/docker_observer_discovery_test.go +++ b/tests/general/discoverymode/docker_observer_discovery_test.go @@ -239,6 +239,6 @@ service: address: "" level: none `, stdout) - require.Contains(t, stderr, "Discovering for next 20s...\nDiscovery complete.") + require.Contains(t, stderr, "Discovering for next 20s...\nSuccessfully discovered \"prometheus_simple\" using \"docker_observer\".\nDiscovery complete.\n") require.Zero(t, sc) } diff --git a/tests/general/discoverymode/host_observer_discovery_test.go b/tests/general/discoverymode/host_observer_discovery_test.go index 27bce7c72c..0afd9d507a 100644 --- a/tests/general/discoverymode/host_observer_discovery_test.go +++ b/tests/general/discoverymode/host_observer_discovery_test.go @@ -266,6 +266,6 @@ service: address: "" level: none `, stdout, fmt.Sprintf("unexpected --dry-run: %s", stderr)) - require.Contains(t, stderr, "Discovering for next 9s...\nDiscovery complete.\n") + require.Contains(t, stderr, "Discovering for next 9s...\nSuccessfully discovered \"prometheus_simple\" using \"host_observer\".\nDiscovery complete.\n") require.Zero(t, sc) } diff --git a/tests/general/discoverymode/k8s_observer_discovery_test.go b/tests/general/discoverymode/k8s_observer_discovery_test.go index e782ccbbe6..99f688ae32 100644 --- a/tests/general/discoverymode/k8s_observer_discovery_test.go +++ b/tests/general/discoverymode/k8s_observer_discovery_test.go @@ -144,7 +144,7 @@ service: address: "" level: none `, stdout.String()) - require.Contains(t, stderr.String(), "Discovering for next 10s...\nDiscovery complete.\n") + require.Contains(t, stderr.String(), "Discovering for next 10s...\nSuccessfully discovered \"smartagent\" using \"k8s_observer\".\nDiscovery complete.\n") } func createRedis(cluster *kubeutils.KindCluster, name, namespace, serviceAccount string) string {