feat: solve secrets on config and backends setup data

agent/agent.go

@@ -75,25 +75,24 @@ func New(logger *slog.Logger, c config.Config) (Agent, error) {
 		return nil, err
 	}
 
-	cm := configmgr.New(logger, pm, c.OrbAgent.ConfigManager)
+	cm := configmgr.New(logger, pm, c.OrbAgent.ConfigManager.Active)
 
 	return &orbAgent{
 		logger: logger, config: c, policyManager: pm, configManager: cm,
 		secretsManager: sm, groupsInfos: make(map[string]groupInfo),
 	}, nil
 }
 
-func (a *orbAgent) startBackends(agentCtx context.Context) error {
+func (a *orbAgent) startBackends(agentCtx context.Context, cfgBackends map[string]any, labels map[string]string) error {
 	a.logger.Info("registered backends", slog.Any("values", backend.GetList()))
-	a.logger.Info("requested backends", slog.Any("values", a.config.OrbAgent.Backends))
-	if len(a.config.OrbAgent.Backends) == 0 {
+	if len(cfgBackends) == 0 {
 		return errors.New("no backends specified")
 	}
-	a.backends = make(map[string]backend.Backend, len(a.config.OrbAgent.Backends))
+	a.backends = make(map[string]backend.Backend, len(cfgBackends))
 	a.backendState = make(map[string]*backend.State)
 
 	var commonConfig config.BackendCommons
-	if v, prs := a.config.OrbAgent.Backends["common"]; prs {
+	if v, prs := cfgBackends["common"]; prs {
 		bytes, err := yaml.Marshal(v)
 		if err != nil {
 			return err
@@ -106,11 +105,11 @@ func (a *orbAgent) startBackends(agentCtx context.Context) error {
 	} else {
 		commonConfig = config.BackendCommons{}
 	}
-	commonConfig.Otel.AgentLabels = a.config.OrbAgent.Labels
+	commonConfig.Otel.AgentLabels = labels
 	a.backendsCommon = commonConfig
-	delete(a.config.OrbAgent.Backends, "common")
+	delete(cfgBackends, "common")
 
-	for name, configurationEntry := range a.config.OrbAgent.Backends {
+	for name, configurationEntry := range cfgBackends {
 		var cEntity map[string]any
 		if configurationEntry != nil {
 			var ok bool
@@ -164,17 +163,25 @@ func (a *orbAgent) Start(ctx context.Context, cancelFunc context.CancelFunc) err
 	a.rpcFromCancelFunc = cancelAllAsync
 	a.cancelFunction = cancelFunc
 	a.logger.Info("agent started", slog.String("version", version.GetBuildVersion()), slog.Any("routine", agentCtx.Value(routineKey)))
+	a.logger.Info("requested backends", slog.Any("values", a.config.OrbAgent.Backends))
 
 	if err := a.secretsManager.Start(ctx); err != nil {
 		a.logger.Error("error during start secrets manager", slog.Any("error", err))
 		return err
 	}
 
-	if err := a.startBackends(ctx); err != nil {
+	var err error
+	if a.config.OrbAgent.Backends,
+		a.config.OrbAgent.ConfigManager,
+		err = a.secretsManager.SolveConfigSecrets(a.config.OrbAgent.Backends, a.config.OrbAgent.ConfigManager); err != nil {
+		return err
+	}
+
+	if err = a.startBackends(ctx, a.config.OrbAgent.Backends, a.config.OrbAgent.Labels); err != nil {
 		return err
 	}
 
-	if err := a.configManager.Start(a.config, a.backends); err != nil {
+	if err = a.configManager.Start(a.config, a.backends); err != nil {
 		return err
 	}
 

agent/config/config_test.go

@@ -0,0 +1,164 @@
+package config_test
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/netboxlabs/orb-agent/agent/config"
+)
+
+func TestResolveEnv(t *testing.T) {
+	err := os.Setenv("TEST_VAR", "test_value")
+	assert.NoError(t, err, "failed to set environment variable")
+	defer func() {
+		err := os.Unsetenv("TEST_VAR")
+		assert.NoError(t, err, "failed to unset environment variable")
+	}()
+
+	tests := []struct {
+		input    string
+		expected string
+		hasError bool
+	}{
+		{"${TEST_VAR}", "test_value", false},
+		{"${UNSET_VAR}", "", true},
+		{"no_env_var", "no_env_var", false},
+	}
+
+	for _, test := range tests {
+		result, err := config.ResolveEnv(test.input)
+		if test.hasError {
+			assert.Error(t, err, "expected error for input %s", test.input)
+		} else {
+			assert.NoError(t, err, "unexpected error for input %s", test.input)
+		}
+		assert.Equal(t, test.expected, result, "unexpected result for input %s", test.input)
+	}
+}
+
+func TestResolveEnvError(t *testing.T) {
+	err := os.Unsetenv("UNSET_VAR") // Ensure the variable is not set
+	assert.NoError(t, err, "failed to unset environment variable")
+
+	_, err = config.ResolveEnv("${UNSET_VAR}")
+	assert.Error(t, err, "expected error for unset environment variable")
+}
+
+func TestResolveEnvInMap(t *testing.T) {
+	err := os.Setenv("TEST_VAR", "test_value")
+	assert.NoError(t, err, "failed to set environment variable")
+	defer func() {
+		err := os.Unsetenv("TEST_VAR")
+		assert.NoError(t, err, "failed to unset environment variable")
+	}()
+
+	data := map[string]any{
+		"key1": "${TEST_VAR}",
+		"key2": "static_value",
+		"nested": map[string]any{
+			"key3": "${TEST_VAR}",
+		},
+		"list": []any{
+			"${TEST_VAR}",
+		},
+	}
+
+	err = config.ResolveEnvInMap(data)
+	assert.NoError(t, err, "unexpected error")
+
+	assert.Equal(t, "test_value", data["key1"], "unexpected value for key1")
+	assert.Equal(t, "static_value", data["key2"], "unexpected value for key2")
+
+	nested, ok := data["nested"].(map[string]any)
+	assert.True(t, ok, "expected nested to be a map")
+	assert.Equal(t, "test_value", nested["key3"], "unexpected value for nested.key3")
+}
+
+func TestResolveEnvInMapError(t *testing.T) {
+	err := os.Unsetenv("UNSET_VAR") // Ensure the variable is not set
+	assert.NoError(t, err, "failed to unset environment variable")
+
+	data := map[string]any{
+		"key1": "${UNSET_VAR}",
+	}
+
+	err = config.ResolveEnvInMap(data)
+	assert.Error(t, err, "expected error for unset environment variable in map")
+
+	// Test with a nested map
+	data = map[string]any{
+		"key1": map[string]any{
+			"nested_key": "${UNSET_VAR}",
+		},
+	}
+	err = config.ResolveEnvInMap(data)
+	assert.Error(t, err, "expected error for unset environment variable in nested map")
+
+	// Test with a nested slice
+	data = map[string]any{
+		"key1": []any{"${UNSET_VAR}"},
+	}
+	err = config.ResolveEnvInMap(data)
+	assert.Error(t, err, "expected error for unset environment variable in nested slice")
+}
+
+func TestResolveEnvInSlice(t *testing.T) {
+	err := os.Setenv("TEST_VAR", "test_value")
+	assert.NoError(t, err, "failed to set environment variable")
+	defer func() {
+		err := os.Unsetenv("TEST_VAR")
+		assert.NoError(t, err, "failed to unset environment variable")
+	}()
+
+	data := []any{
+		"${TEST_VAR}",
+		"static_value",
+		map[string]any{
+			"key1": "${TEST_VAR}",
+		},
+		[]any{"${TEST_VAR}"},
+	}
+
+	err = config.ResolveEnvInSlice(data)
+	assert.NoError(t, err, "unexpected error")
+
+	assert.Equal(t, "test_value", data[0], "unexpected value at index 0")
+	assert.Equal(t, "static_value", data[1], "unexpected value at index 1")
+
+	nestedMap, ok := data[2].(map[string]any)
+	assert.True(t, ok, "expected index 2 to be a map")
+	assert.Equal(t, "test_value", nestedMap["key1"], "unexpected value for nestedMap.key1")
+
+	nestedSlice, ok := data[3].([]any)
+	assert.True(t, ok, "expected index 3 to be a slice")
+	assert.Equal(t, "test_value", nestedSlice[0], "unexpected value for nestedSlice[0]")
+}
+
+func TestResolveEnvInSliceError(t *testing.T) {
+	err := os.Unsetenv("UNSET_VAR") // Ensure the variable is not set
+	assert.NoError(t, err, "failed to unset environment variable")
+
+	data := []any{
+		"${UNSET_VAR}",
+	}
+
+	err = config.ResolveEnvInSlice(data)
+	assert.Error(t, err, "expected error for unset environment variable in slice")
+
+	// Test with a nested map
+	data = []any{
+		map[string]any{
+			"key1": "${UNSET_VAR}",
+		},
+	}
+	err = config.ResolveEnvInSlice(data)
+	assert.Error(t, err, "expected error for unset environment variable in nested map")
+	// Test with a nested slice
+	data = []any{
+		[]any{"${UNSET_VAR}"},
+	}
+	err = config.ResolveEnvInSlice(data)
+	assert.Error(t, err, "expected error for unset environment variable in nested slice")
+}

agent/config/env.go

@@ -0,0 +1,70 @@
+package config
+
+import (
+	"errors"
+	"os"
+	"strings"
+)
+
+// ResolveEnv replaces environment variable placeholders in the format ${VAR} with their actual values.
+func ResolveEnv(value string) (string, error) {
+	// Check if the value starts with "${" and ends with "}"
+	if strings.HasPrefix(value, "${") && strings.HasSuffix(value, "}") {
+		// Extract the environment variable name
+		envVar := value[2 : len(value)-1]
+		// Get the value of the environment variable
+		envValue := os.Getenv(envVar)
+		if envValue != "" {
+			return envValue, nil
+		}
+		return "", errors.New("a provided environment variable is not set")
+	}
+	// Return the original value if no substitution occurs
+	return value, nil
+}
+
+// ResolveEnvInMap recursively traverses and resolves env vars in map[string]any structures.
+func ResolveEnvInMap(data map[string]any) error {
+	for key, val := range data {
+		switch v := val.(type) {
+		case string:
+			resolved, err := ResolveEnv(v)
+			if err != nil {
+				return err
+			}
+			data[key] = resolved
+		case map[string]any:
+			if err := ResolveEnvInMap(v); err != nil {
+				return err
+			}
+		case []any:
+			if err := ResolveEnvInSlice(v); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// ResolveEnvInSlice handles []any elements recursively.
+func ResolveEnvInSlice(slice []any) error {
+	for i, val := range slice {
+		switch v := val.(type) {
+		case string:
+			resolved, err := ResolveEnv(v)
+			if err != nil {
+				return err
+			}
+			slice[i] = resolved
+		case map[string]any:
+			if err := ResolveEnvInMap(v); err != nil {
+				return err
+			}
+		case []any:
+			if err := ResolveEnvInSlice(v); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}

agent/configmgr/git.go

@@ -5,9 +5,7 @@ import (
 	"errors"
 	"io"
 	"log/slog"
-	"os"
 	"slices"
-	"strings"
 
 	"github.com/go-co-op/gocron/v2"
 	gitv5 "github.com/go-git/go-git/v5"
@@ -50,22 +48,6 @@ type (
 	}
 )
 
-func resolveEnv(value string) (string, error) {
-	// Check if the value starts with ${ and ends with }
-	if strings.HasPrefix(value, "${") && strings.HasSuffix(value, "}") {
-		// Extract the environment variable name
-		envVar := value[2 : len(value)-1]
-		// Get the value of the environment variable
-		envValue := os.Getenv(envVar)
-		if envValue != "" {
-			return envValue, nil
-		}
-		return "", errors.New("a provided environment variable is not set")
-	}
-	// Return the original value if no substitution occurs
-	return value, nil
-}
-
 func (gc *gitConfigManager) readPolicies(tree *object.Tree, matchingPolicies []string) (map[policyPath]policyData, error) {
 	policiesByPath := make(map[policyPath]policyData)
 	allPolicies := make(map[string]map[string]any)
@@ -355,13 +337,14 @@ func (gc *gitConfigManager) schedule(cfg config.Config, backends map[string]back
 func (gc *gitConfigManager) Start(cfg config.Config, backends map[string]backend.Backend) error {
 	var err error
 	gc.version = 1
+	gc.config = cfg.OrbAgent.ConfigManager.Sources.Git
 
 	if gc.config.URL == "" {
 		return errors.New("URL is required for Git Config Manager")
 	}
 
 	if gc.config.Auth == "basic" {
-		if gc.config.Password, err = resolveEnv(gc.config.Password); err != nil {
+		if gc.config.Password, err = config.ResolveEnv(gc.config.Password); err != nil {
 			return err
 		}
 		gc.authMethod = &http.BasicAuth{
@@ -370,7 +353,7 @@ func (gc *gitConfigManager) Start(cfg config.Config, backends map[string]backend
 		}
 	} else if gc.config.Auth == "ssh" {
 		if gc.config.PrivateKey != "" {
-			if gc.config.Password, err = resolveEnv(gc.config.Password); err != nil {
+			if gc.config.Password, err = config.ResolveEnv(gc.config.Password); err != nil {
 				return err
 			}
 			gc.authMethod, err = ssh.NewPublicKeysFromFile("git", gc.config.PrivateKey, gc.config.Password)