[config] Default config env variable expansion (#2231)

Adds the ability for components to use environment variables in string fields in the default configuration returned by `CreateDefaultConfig`. These fields get expanded when the component is loaded, in the same way the fields of configuration yaml files are (`$FOO` is replaced by the value of the `FOO` environment variable, `$$FOO` is replaced by `$FOO`, `$$$FOO` is replaced by `$`followed by the contents of `FOO`).

For instance, if `CreateDefaultConfig` of a component returns:
```
&Config{
  ...
  TagsConfig: &TagsConfig{
    Env: "$DD_ENV",
  }
  ...
}
```

and the `DD_ENV` environment variable is set to `prod`, then the resulting struct will contain:
```
&Config{
  ...
  TagsConfig: &TagsConfig{
    Env: "prod",
  }
  ...
}
```

**Note:** The default config is expanded _before_ it's merged with the user-provided config, so as to not mess with the latter.

**Link to tracking Issue:** n/a

**Testing:** 

Added unit tests to check that the variable expansion works, and that it doesn't crash in edge cases (unexported private fields that can't be modified, uninitialised config object).

Tested behavior with the Datadog exporter.

config/config.go

@@ -275,6 +275,7 @@ func loadExtensions(exts map[string]interface{}, factories map[configmodels.Type
 		// Create the default config for this extension
 		extensionCfg := factory.CreateDefaultConfig()
 		extensionCfg.SetName(fullName)
+		expandEnvLoadedConfig(extensionCfg)
 
 		// Now that the default config struct is created we can Unmarshal into it
 		// and it will apply user-defined config on top of the default.
@@ -310,6 +311,7 @@ func LoadReceiver(componentConfig *viper.Viper, typeStr configmodels.Type, fullN
 	// Create the default config for this receiver.
 	receiverCfg := factory.CreateDefaultConfig()
 	receiverCfg.SetName(fullName)
+	expandEnvLoadedConfig(receiverCfg)
 
 	// Now that the default config struct is created we can Unmarshal into it
 	// and it will apply user-defined config on top of the default.
@@ -382,6 +384,7 @@ func loadExporters(exps map[string]interface{}, factories map[configmodels.Type]
 		// Create the default config for this exporter
 		exporterCfg := factory.CreateDefaultConfig()
 		exporterCfg.SetName(fullName)
+		expandEnvLoadedConfig(exporterCfg)
 
 		// Now that the default config struct is created we can Unmarshal into it
 		// and it will apply user-defined config on top of the default.
@@ -424,6 +427,7 @@ func loadProcessors(procs map[string]interface{}, factories map[configmodels.Typ
 		// Create the default config for this processor.
 		processorCfg := factory.CreateDefaultConfig()
 		processorCfg.SetName(fullName)
+		expandEnvLoadedConfig(processorCfg)
 
 		// Now that the default config struct is created we can Unmarshal into it
 		// and it will apply user-defined config on top of the default.
@@ -676,6 +680,45 @@ func expandStringValues(value interface{}) interface{} {
 	}
 }
 
+// expandEnvLoadedConfig is a utility function that goes recursively through a config object
+// and tries to expand environment variables in its string fields.
+func expandEnvLoadedConfig(s interface{}) {
+	expandEnvLoadedConfigPointer(s)
+}
+
+func expandEnvLoadedConfigPointer(s interface{}) {
+	// Check that the value given is indeed a pointer, otherwise safely stop the search here
+	value := reflect.ValueOf(s)
+	if value.Kind() != reflect.Ptr {
+		return
+	}
+	// Run expandLoadedConfigValue on the value behind the pointer
+	expandEnvLoadedConfigValue(value.Elem())
+}
+
+func expandEnvLoadedConfigValue(value reflect.Value) {
+	// The value given is a string, we expand it (if allowed)
+	if value.Kind() == reflect.String && value.CanSet() {
+		value.SetString(expandEnv(value.String()))
+	}
+	// The value given is a struct, we go through its fields
+	if value.Kind() == reflect.Struct {
+		for i := 0; i < value.NumField(); i++ {
+			field := value.Field(i) // Returns the content of the field
+			if field.CanSet() {     // Only try to modify a field if it can be modified (eg. skip unexported private fields)
+				switch field.Kind() {
+				case reflect.String: // The current field is a string, we want to expand it
+					field.SetString(expandEnv(field.String())) // Expand env variables in the string
+				case reflect.Ptr: // The current field is a pointer
+					expandEnvLoadedConfigPointer(field.Interface()) // Run the expansion function on the pointer
+				case reflect.Struct: // The current field is a nested struct
+					expandEnvLoadedConfigValue(field) // Go through the nested struct
+				}
+			}
+		}
+	}
+}
+
 func expandEnv(s string) string {
 	return os.Expand(s, func(str string) string {
 		// This allows escaping environment variable substitution via $$, e.g.

config/config_test.go

@@ -571,3 +571,227 @@ func loadConfigFile(t *testing.T, fileName string, factories component.Factories
 	}
 	return cfg, ValidateConfig(cfg, zap.NewNop())
 }
+
+type nestedConfig struct {
+	NestedStringValue string
+	NestedIntValue    int
+}
+
+type testConfig struct {
+	configmodels.ExporterSettings
+
+	NestedConfigPtr   *nestedConfig
+	NestedConfigValue nestedConfig
+	StringValue       string
+	StringPtrValue    *string
+	IntValue          int
+}
+
+func TestExpandEnvLoadedConfig(t *testing.T) {
+	assert.NoError(t, os.Setenv("NESTED_VALUE", "replaced_nested_value"))
+	assert.NoError(t, os.Setenv("VALUE", "replaced_value"))
+	assert.NoError(t, os.Setenv("PTR_VALUE", "replaced_ptr_value"))
+
+	defer func() {
+		assert.NoError(t, os.Unsetenv("NESTED_VALUE"))
+		assert.NoError(t, os.Unsetenv("VALUE"))
+		assert.NoError(t, os.Unsetenv("PTR_VALUE"))
+	}()
+
+	testString := "$PTR_VALUE"
+
+	config := &testConfig{
+		ExporterSettings: configmodels.ExporterSettings{
+			TypeVal: configmodels.Type("test"),
+			NameVal: "test",
+		},
+		NestedConfigPtr: &nestedConfig{
+			NestedStringValue: "$NESTED_VALUE",
+			NestedIntValue:    1,
+		},
+		NestedConfigValue: nestedConfig{
+			NestedStringValue: "$NESTED_VALUE",
+			NestedIntValue:    2,
+		},
+		StringValue:    "$VALUE",
+		StringPtrValue: &testString,
+		IntValue:       3,
+	}
+
+	expandEnvLoadedConfig(config)
+
+	replacedTestString := "replaced_ptr_value"
+
+	assert.Equal(t, &testConfig{
+		ExporterSettings: configmodels.ExporterSettings{
+			TypeVal: configmodels.Type("test"),
+			NameVal: "test",
+		},
+		NestedConfigPtr: &nestedConfig{
+			NestedStringValue: "replaced_nested_value",
+			NestedIntValue:    1,
+		},
+		NestedConfigValue: nestedConfig{
+			NestedStringValue: "replaced_nested_value",
+			NestedIntValue:    2,
+		},
+		StringValue:    "replaced_value",
+		StringPtrValue: &replacedTestString,
+		IntValue:       3,
+	}, config)
+}
+
+func TestExpandEnvLoadedConfigEscapedEnv(t *testing.T) {
+	assert.NoError(t, os.Setenv("NESTED_VALUE", "replaced_nested_value"))
+	assert.NoError(t, os.Setenv("ESCAPED_VALUE", "replaced_escaped_value"))
+	assert.NoError(t, os.Setenv("ESCAPED_PTR_VALUE", "replaced_escaped_pointer_value"))
+
+	defer func() {
+		assert.NoError(t, os.Unsetenv("NESTED_VALUE"))
+		assert.NoError(t, os.Unsetenv("ESCAPED_VALUE"))
+		assert.NoError(t, os.Unsetenv("ESCAPED_PTR_VALUE"))
+	}()
+
+	testString := "$$ESCAPED_PTR_VALUE"
+
+	config := &testConfig{
+		ExporterSettings: configmodels.ExporterSettings{
+			TypeVal: configmodels.Type("test"),
+			NameVal: "test",
+		},
+		NestedConfigPtr: &nestedConfig{
+			NestedStringValue: "$NESTED_VALUE",
+			NestedIntValue:    1,
+		},
+		NestedConfigValue: nestedConfig{
+			NestedStringValue: "$NESTED_VALUE",
+			NestedIntValue:    2,
+		},
+		StringValue:    "$$ESCAPED_VALUE",
+		StringPtrValue: &testString,
+		IntValue:       3,
+	}
+
+	expandEnvLoadedConfig(config)
+
+	replacedTestString := "$ESCAPED_PTR_VALUE"
+
+	assert.Equal(t, &testConfig{
+		ExporterSettings: configmodels.ExporterSettings{
+			TypeVal: configmodels.Type("test"),
+			NameVal: "test",
+		},
+		NestedConfigPtr: &nestedConfig{
+			NestedStringValue: "replaced_nested_value",
+			NestedIntValue:    1,
+		},
+		NestedConfigValue: nestedConfig{
+			NestedStringValue: "replaced_nested_value",
+			NestedIntValue:    2,
+		},
+		StringValue:    "$ESCAPED_VALUE",
+		StringPtrValue: &replacedTestString,
+		IntValue:       3,
+	}, config)
+}
+
+func TestExpandEnvLoadedConfigMissingEnv(t *testing.T) {
+	assert.NoError(t, os.Setenv("NESTED_VALUE", "replaced_nested_value"))
+
+	defer func() {
+		assert.NoError(t, os.Unsetenv("NESTED_VALUE"))
+	}()
+
+	testString := "$PTR_VALUE"
+
+	config := &testConfig{
+		ExporterSettings: configmodels.ExporterSettings{
+			TypeVal: configmodels.Type("test"),
+			NameVal: "test",
+		},
+		NestedConfigPtr: &nestedConfig{
+			NestedStringValue: "$NESTED_VALUE",
+			NestedIntValue:    1,
+		},
+		NestedConfigValue: nestedConfig{
+			NestedStringValue: "$NESTED_VALUE",
+			NestedIntValue:    2,
+		},
+		StringValue:    "$VALUE",
+		StringPtrValue: &testString,
+		IntValue:       3,
+	}
+
+	expandEnvLoadedConfig(config)
+
+	replacedTestString := ""
+
+	assert.Equal(t, &testConfig{
+		ExporterSettings: configmodels.ExporterSettings{
+			TypeVal: configmodels.Type("test"),
+			NameVal: "test",
+		},
+		NestedConfigPtr: &nestedConfig{
+			NestedStringValue: "replaced_nested_value",
+			NestedIntValue:    1,
+		},
+		NestedConfigValue: nestedConfig{
+			NestedStringValue: "replaced_nested_value",
+			NestedIntValue:    2,
+		},
+		StringValue:    "",
+		StringPtrValue: &replacedTestString,
+		IntValue:       3,
+	}, config)
+}
+
+func TestExpandEnvLoadedConfigNil(t *testing.T) {
+	var config *testConfig
+
+	// This should safely do nothing
+	expandEnvLoadedConfig(config)
+
+	assert.Equal(t, (*testConfig)(nil), config)
+}
+
+func TestExpandEnvLoadedConfigNoPointer(t *testing.T) {
+	assert.NoError(t, os.Setenv("VALUE", "replaced_value"))
+
+	config := testConfig{
+		StringValue: "$VALUE",
+	}
+
+	// This should do nothing as config is not a pointer
+	expandEnvLoadedConfig(config)
+
+	assert.Equal(t, testConfig{
+		StringValue: "$VALUE",
+	}, config)
+}
+
+type testUnexportedConfig struct {
+	configmodels.ExporterSettings
+
+	unexportedStringValue string
+	ExportedStringValue   string
+}
+
+func TestExpandEnvLoadedConfigUnexportedField(t *testing.T) {
+	assert.NoError(t, os.Setenv("VALUE", "replaced_value"))
+
+	defer func() {
+		assert.NoError(t, os.Unsetenv("VALUE"))
+	}()
+
+	config := &testUnexportedConfig{
+		unexportedStringValue: "$VALUE",
+		ExportedStringValue:   "$VALUE",
+	}
+
+	expandEnvLoadedConfig(config)
+
+	assert.Equal(t, &testUnexportedConfig{
+		unexportedStringValue: "$VALUE",
+		ExportedStringValue:   "replaced_value",
+	}, config)
+}