Add bundled discovery configs to default config.d for rpm and deb pac…

…kages (#3311)

* update headers for config.d

* add default config.d to rpm and deb packages

* testutils helper updates

* deb and rpm installation tests

* give discovery provider development stability

* update config.d header and stability level

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### 💡 Enhancements 💡
 
+- (Splunk) Package default discovery configuration in reference form in `/etc/otel/collector/config.d` ([#3311](https://github.com/signalfx/splunk-otel-collector/pull/3311))
 - (Splunk) Add bundled collectd/nginx Smart Agent receiver discovery rules ([#3321](https://github.com/signalfx/splunk-otel-collector/pull/3321))
 
 ## v0.80.0

cmd/otelcol/config/collector/config.d.linux/extensions/docker-observer.discovery.yaml

@@ -1,5 +1,10 @@
 #####################################################################################
 # This file is generated by the Splunk Distribution of the OpenTelemetry Collector. #
+#                                                                                   #
+# It reflects the default configuration bundled in the Collector executable for use #
+# in discovery mode (--discovery) and is provided for reference or customization.   #
+# Please note that any changes made to this file will need to be reconciled during  #
+# upgrades of the Collector.                                                        #
 #####################################################################################
 # docker_observer:
 #   enabled: true

cmd/otelcol/config/collector/config.d.linux/extensions/host-observer.discovery.yaml

@@ -1,5 +1,10 @@
 #####################################################################################
 # This file is generated by the Splunk Distribution of the OpenTelemetry Collector. #
+#                                                                                   #
+# It reflects the default configuration bundled in the Collector executable for use #
+# in discovery mode (--discovery) and is provided for reference or customization.   #
+# Please note that any changes made to this file will need to be reconciled during  #
+# upgrades of the Collector.                                                        #
 #####################################################################################
 # host_observer:
 #   enabled: true

cmd/otelcol/config/collector/config.d.linux/extensions/k8s-observer.discovery.yaml

@@ -1,5 +1,10 @@
 #####################################################################################
 # This file is generated by the Splunk Distribution of the OpenTelemetry Collector. #
+#                                                                                   #
+# It reflects the default configuration bundled in the Collector executable for use #
+# in discovery mode (--discovery) and is provided for reference or customization.   #
+# Please note that any changes made to this file will need to be reconciled during  #
+# upgrades of the Collector.                                                        #
 #####################################################################################
 # k8s_observer:
 #   enabled: true

cmd/otelcol/config/collector/config.d.linux/properties.discovery.yaml.example

@@ -0,0 +1,19 @@
+# Copy this file to properties.discovery.yaml and add any discovery properties for use in discovery mode:
+# For more information:
+# https://github.com/signalfx/splunk-otel-collector/tree/main/internal/confmapprovider/discovery#discovery-properties
+
+splunk.discovery:
+  extensions:
+    docker_observer:
+      enabled: true
+    host_observer:
+      enabled: true
+    k8s_observer:
+      enabled: true
+  receivers:
+    smartagent/collectd/mysql:
+      enabled: true
+    smartagent/collectd/nginx:
+      enabled: true
+    smartagent/postgresql:
+      enabled: true

cmd/otelcol/config/collector/config.d.linux/receivers/smartagent-collectd-mysql.discovery.yaml

@@ -1,5 +1,10 @@
 #####################################################################################
 # This file is generated by the Splunk Distribution of the OpenTelemetry Collector. #
+#                                                                                   #
+# It reflects the default configuration bundled in the Collector executable for use #
+# in discovery mode (--discovery) and is provided for reference or customization.   #
+# Please note that any changes made to this file will need to be reconciled during  #
+# upgrades of the Collector.                                                        #
 #####################################################################################
 # smartagent/collectd/mysql:
 #   enabled: true

cmd/otelcol/config/collector/config.d.linux/receivers/smartagent-collectd-nginx.discovery.yaml

@@ -1,5 +1,10 @@
 #####################################################################################
 # This file is generated by the Splunk Distribution of the OpenTelemetry Collector. #
+#                                                                                   #
+# It reflects the default configuration bundled in the Collector executable for use #
+# in discovery mode (--discovery) and is provided for reference or customization.   #
+# Please note that any changes made to this file will need to be reconciled during  #
+# upgrades of the Collector.                                                        #
 #####################################################################################
 # smartagent/collectd/nginx:
 #   enabled: true

cmd/otelcol/config/collector/config.d.linux/receivers/smartagent-postgresql.discovery.yaml

@@ -1,5 +1,10 @@
 #####################################################################################
 # This file is generated by the Splunk Distribution of the OpenTelemetry Collector. #
+#                                                                                   #
+# It reflects the default configuration bundled in the Collector executable for use #
+# in discovery mode (--discovery) and is provided for reference or customization.   #
+# Please note that any changes made to this file will need to be reconciled during  #
+# upgrades of the Collector.                                                        #
 #####################################################################################
 # smartagent/postgresql:
 #   enabled: true

internal/buildscripts/packaging/fpm/common.sh

@@ -30,6 +30,8 @@ SERVICE_GROUP="splunk-otel-collector"
 
 OTELCOL_INSTALL_PATH="/usr/bin/otelcol"
 TRANSLATESFX_INSTALL_PATH="/usr/bin/translatesfx"
+CONFIG_DIR_REPO_PATH="$REPO_DIR/cmd/otelcol/config/collector/config.d.linux"
+CONFIG_DIR_INSTALL_PATH="/etc/otel/collector/config.d"
 AGENT_CONFIG_REPO_PATH="$REPO_DIR/cmd/otelcol/config/collector/agent_config.yaml"
 AGENT_CONFIG_INSTALL_PATH="/etc/otel/collector/agent_config.yaml"
 GATEWAY_CONFIG_REPO_PATH="$REPO_DIR/cmd/otelcol/config/collector/gateway_config.yaml"
@@ -84,6 +86,7 @@ setup_files_and_permissions() {
     sudo chmod 755 "$buildroot/$TRANSLATESFX_INSTALL_PATH"
 
     cp -r "$FPM_DIR/etc" "$buildroot/etc"
+    cp -r "$CONFIG_DIR_REPO_PATH" "$buildroot/$CONFIG_DIR_INSTALL_PATH"
     cp -f "$AGENT_CONFIG_REPO_PATH" "$buildroot/$AGENT_CONFIG_INSTALL_PATH"
     cp -f "$GATEWAY_CONFIG_REPO_PATH" "$buildroot/$GATEWAY_CONFIG_INSTALL_PATH"
     sudo chown -R $SERVICE_USER:$SERVICE_GROUP "$buildroot/etc/otel"

internal/buildscripts/packaging/fpm/deb/build.sh

@@ -52,6 +52,7 @@ sudo fpm -s dir -t deb -n "$PKG_NAME" -v "$VERSION" -f -p "$OUTPUT_DIR" \
     --after-install "$POSTINSTALL_PATH" \
     --before-remove "$PREUNINSTALL_PATH" \
     --deb-no-default-config-files \
+    --config-files "$CONFIG_DIR_INSTALL_PATH" \
     --config-files "$AGENT_CONFIG_INSTALL_PATH" \
     --config-files "$GATEWAY_CONFIG_INSTALL_PATH" \
     --config-files "$FLUENTD_CONFIG_INSTALL_DIR" \

internal/confmapprovider/discovery/README.md

@@ -1,6 +1,7 @@
-# Discovery confmap.Provider (Experimental)
+# Discovery confmap.Provider
 
-**This component should not be considered stable. At this time its functionality is provided for testing and validation purposes only.**
+**This feature currently has an [alpha](https://github.com/open-telemetry/opentelemetry-collector#alpha) stability level.<br>
+Backwards incompatible changes to components and custom discovery configuration may occur.**
 
 The Discovery [confmap.Provider](https://pkg.go.dev/go.opentelemetry.io/collector/confmap#readme-provider) provides
 the ability to define Collector service config through individual component yaml mappings in a `config.d` directory:
@@ -100,8 +101,6 @@ successfully started observers.
 1. Log any receiver resulting in a `discovery.status` of `partial` with the configured guidance for setting any relevant discovery properties.
 1. Stop all temporary components before continuing on to the actual Collector service (or exiting early with `--dry-run`).
 
-By default, the Discovery mode is provided with pre-made discovery config components in [`bundle.d`](./bundle/README.md).
-
 Unlike `config.d` component files, which are direct configuration entries for the desired component, Discovery component
 configs have an `enabled` boolean and `config` parent mapping field to determine use and configure the functionality of
 the components:
@@ -132,6 +131,20 @@ the components:
       <discovery receiver statement status entries>
 ```
 
+By default, the discovery mode is provided with pre-made discovery config components in [`bundle.d`](./bundle/README.md).
+
+The following components have bundled discovery configurations in the last Splunk OpenTelemetry Collector release:
+
+I. Smart Agent receiver
+* `collectd/mysql` monitor type ([Linux](./bundle/bundle.d/receivers/smartagent-collectd-mysql.discovery.yaml))
+* `collectd/nginx` monitor type ([Linux](./bundle/bundle.d/receivers/smartagent-collectd-nginx.discovery.yaml))
+* `postgresql` monitor type ([Linux and Windows](./bundle/bundle.d/receivers/smartagent-postgresql.discovery.yaml))
+
+II. Extensions
+* `docker_observer` ([Linux and Windows](./bundle/bundle.d/extensions/docker-observer.discovery.yaml))
+* `host_observer` ([Linux and Windows](./bundle/bundle.d/extensions/host-observer.discovery.yaml))
+* `k8s_observer` ([Linux and Windows](./bundle/bundle.d/extensions/k8s-observer.discovery.yaml))
+
 ### Discovery properties
 
 Configuring discovery components is performed by merging discovery properties with the config.d receivers

internal/confmapprovider/discovery/bundle/bundle.d/extensions/docker-observer.discovery.yaml

@@ -1,5 +1,6 @@
 #####################################################################################
-# This file is generated by the Splunk Distribution of the OpenTelemetry Collector. #
+#                               Do not edit manually!                               #
+# All changes must be made to associated .tmpl file before running 'make bundle.d'. #
 #####################################################################################
 docker_observer:
   enabled: true

internal/confmapprovider/discovery/bundle/bundle.d/extensions/host-observer.discovery.yaml

@@ -1,5 +1,6 @@
 #####################################################################################
-# This file is generated by the Splunk Distribution of the OpenTelemetry Collector. #
+#                               Do not edit manually!                               #
+# All changes must be made to associated .tmpl file before running 'make bundle.d'. #
 #####################################################################################
 host_observer:
   enabled: true

internal/confmapprovider/discovery/bundle/bundle.d/extensions/k8s-observer.discovery.yaml

@@ -1,5 +1,6 @@
 #####################################################################################
-# This file is generated by the Splunk Distribution of the OpenTelemetry Collector. #
+#                               Do not edit manually!                               #
+# All changes must be made to associated .tmpl file before running 'make bundle.d'. #
 #####################################################################################
 k8s_observer:
   enabled: true

internal/confmapprovider/discovery/bundle/bundle.d/receivers/smartagent-collectd-mysql.discovery.yaml

@@ -1,5 +1,6 @@
 #####################################################################################
-# This file is generated by the Splunk Distribution of the OpenTelemetry Collector. #
+#                               Do not edit manually!                               #
+# All changes must be made to associated .tmpl file before running 'make bundle.d'. #
 #####################################################################################
 smartagent/collectd/mysql:
   enabled: true

internal/confmapprovider/discovery/bundle/bundle.d/receivers/smartagent-collectd-nginx.discovery.yaml

@@ -1,5 +1,6 @@
 #####################################################################################
-# This file is generated by the Splunk Distribution of the OpenTelemetry Collector. #
+#                               Do not edit manually!                               #
+# All changes must be made to associated .tmpl file before running 'make bundle.d'. #
 #####################################################################################
 smartagent/collectd/nginx:
   enabled: true

internal/confmapprovider/discovery/bundle/bundle.d/receivers/smartagent-postgresql.discovery.yaml

@@ -1,5 +1,6 @@
 #####################################################################################
-# This file is generated by the Splunk Distribution of the OpenTelemetry Collector. #
+#                               Do not edit manually!                               #
+# All changes must be made to associated .tmpl file before running 'make bundle.d'. #
 #####################################################################################
 smartagent/postgresql:
   enabled: true

internal/confmapprovider/discovery/bundle/cmd/discoverybundler/main.go

@@ -29,8 +29,19 @@ import (
 )
 
 const (
-	genHeader = `#####################################################################################
+	bundledHeader = `#####################################################################################
+#                               Do not edit manually!                               #
+# All changes must be made to associated .tmpl file before running 'make bundle.d'. #
+#####################################################################################
+`
+
+	commentedHeader = `#####################################################################################
 # This file is generated by the Splunk Distribution of the OpenTelemetry Collector. #
+#                                                                                   #
+# It reflects the default configuration bundled in the Collector executable for use #
+# in discovery mode (--discovery) and is provided for reference or customization.   #
+# Please note that any changes made to this file will need to be reconciled during  #
+# upgrades of the Collector.                                                        #
 #####################################################################################
 `
 )
@@ -70,15 +81,17 @@ func main() {
 	tmpl, err := os.ReadFile(s.templateFile)
 	panicOnError(err)
 
+	out := &bytes.Buffer{}
 	if s.commented {
+		out.WriteString(commentedHeader)
 		tmpl = commentedTemplate(tmpl)
+	} else {
+		out.WriteString(bundledHeader)
 	}
 
 	t, err := template.New("discoverybundler").Funcs(bundle.FuncMap()).Parse(string(tmpl))
 	panicOnError(err)
 
-	out := &bytes.Buffer{}
-	out.WriteString(genHeader)
 	panicOnError(t.Execute(out, nil))
 
 	var rendered map[string]any