chore(core): decompose config loading (#2639)

### Proposed Changes

* Goal: Support custom loaders of core configuration, to allow
extensibility in downstream integrators.

### Checklist

- [ ] I have added or updated unit tests
- [ ] I have added or updated integration tests (if appropriate)
- [ ] I have added or updated documentation

### Testing Instructions

---------

Co-authored-by: opentdf-automation[bot] <149537512+opentdf-automation[bot]@users.noreply.github.com>

Author: ntrevino-virtru

service/cmd/migrate.go

@@ -121,7 +121,24 @@ func migrateService(cmd *cobra.Command, args []string, migrationFunc func(*db.Cl
 func migrateDBClient(cmd *cobra.Command, opts ...db.OptsFunc) (*db.Client, error) {
 	configFile, _ := cmd.Flags().GetString(configFileFlag)
 	configKey, _ := cmd.Flags().GetString(configKeyFlag)
-	conf, err := config.LoadConfig(cmd.Context(), configKey, configFile)
+	envLoader, err := config.NewEnvironmentValueLoader(configKey, nil)
+	if err != nil {
+		panic(fmt.Errorf("could not load config: %w", err))
+	}
+	configFileLoader, err := config.NewConfigFileLoader(configKey, configFile)
+	if err != nil {
+		panic(fmt.Errorf("could not load config: %w", err))
+	}
+	defaultSettingsLoader, err := config.NewDefaultSettingsLoader()
+	if err != nil {
+		panic(fmt.Errorf("could not load config: %w", err))
+	}
+	conf, err := config.Load(
+		cmd.Context(),
+		envLoader,
+		configFileLoader,
+		defaultSettingsLoader,
+	)
 	if err != nil {
 		panic(fmt.Errorf("could not load config: %w", err))
 	}

service/cmd/policy.go

@@ -35,7 +35,24 @@ var (
 		Run: func(cmd *cobra.Command, _ []string) {
 			configFile, _ := cmd.Flags().GetString(configFileFlag)
 			configKey, _ := cmd.Flags().GetString(configKeyFlag)
-			cfg, err := config.LoadConfig(cmd.Context(), configKey, configFile)
+			envLoader, err := config.NewEnvironmentValueLoader(configKey, nil)
+			if err != nil {
+				panic(fmt.Errorf("could not load config: %w", err))
+			}
+			configFileLoader, err := config.NewConfigFileLoader(configKey, configFile)
+			if err != nil {
+				panic(fmt.Errorf("could not load config: %w", err))
+			}
+			defaultSettingsLoader, err := config.NewDefaultSettingsLoader()
+			if err != nil {
+				panic(fmt.Errorf("could not load config: %w", err))
+			}
+			cfg, err := config.Load(
+				cmd.Context(),
+				envLoader,
+				configFileLoader,
+				defaultSettingsLoader,
+			)
 			if err != nil {
 				panic(fmt.Errorf("could not load config: %w", err))
 			}

service/cmd/provisionFixtures.go

@@ -38,7 +38,24 @@ You can clear/recycle your database with 'docker compose down' and 'docker compo
 		Run: func(cmd *cobra.Command, _ []string) {
 			configFile, _ := cmd.Flags().GetString(configFileFlag)
 			configKey, _ := cmd.Flags().GetString(configKeyFlag)
-			cfg, err := config.LoadConfig(cmd.Context(), configKey, configFile)
+			envLoader, err := config.NewEnvironmentValueLoader(configKey, nil)
+			if err != nil {
+				panic(fmt.Errorf("could not load config: %w", err))
+			}
+			configFileLoader, err := config.NewConfigFileLoader(configKey, configFile)
+			if err != nil {
+				panic(fmt.Errorf("could not load config: %w", err))
+			}
+			defaultSettingsLoader, err := config.NewDefaultSettingsLoader()
+			if err != nil {
+				panic(fmt.Errorf("could not load config: %w", err))
+			}
+			cfg, err := config.Load(
+				cmd.Context(),
+				envLoader,
+				configFileLoader,
+				defaultSettingsLoader,
+			)
 			if err != nil {
 				panic(fmt.Errorf("could not load config: %w", err))
 			}

service/internal/auth/config.go

@@ -10,10 +10,10 @@ import (
 
 // AuthConfig pulls AuthN and AuthZ together
 type Config struct {
-	Enabled      bool     `mapstructure:"enabled" json:"enabled" default:"true" `
-	PublicRoutes []string `mapstructure:"-"`
+	Enabled      bool     `mapstructure:"enabled" json:"enabled" default:"true"`
+	PublicRoutes []string `mapstructure:"-" json:"-"`
 	// Used for re-authentication of IPC connections
-	IPCReauthRoutes []string `mapstructure:"-"`
+	IPCReauthRoutes []string `mapstructure:"-" json:"-"`
 	AuthNConfig     `mapstructure:",squash"`
 }
 
@@ -23,17 +23,17 @@ type AuthNConfig struct { //nolint:revive // AuthNConfig is a valid name
 	Issuer       string        `mapstructure:"issuer" json:"issuer"`
 	Audience     string        `mapstructure:"audience" json:"audience"`
 	Policy       PolicyConfig  `mapstructure:"policy" json:"policy"`
-	CacheRefresh string        `mapstructure:"cache_refresh_interval"`
-	DPoPSkew     time.Duration `mapstructure:"dpopskew" default:"1h"`
-	TokenSkew    time.Duration `mapstructure:"skew" default:"1m"`
+	CacheRefresh string        `mapstructure:"cache_refresh_interval" json:"cache_refresh_interval"`
+	DPoPSkew     time.Duration `mapstructure:"dpopskew" json:"dpopskew" default:"1h"`
+	TokenSkew    time.Duration `mapstructure:"skew" json:"skew" default:"1m"`
 }
 
 type PolicyConfig struct {
 	Builtin string `mapstructure:"-" json:"-"`
 	// Username claim to use for user information
 	UserNameClaim string `mapstructure:"username_claim" json:"username_claim" default:"preferred_username"`
 	// Claim to use for group/role information
-	GroupsClaim string `mapstructure:"groups_claim" json:"group_claim" default:"realm_access.roles"`
+	GroupsClaim string `mapstructure:"groups_claim" json:"groups_claim" default:"realm_access.roles"`
 	// Deprecated: Use GroupClain instead
 	RoleClaim string `mapstructure:"claim" json:"claim" default:"realm_access.roles"`
 	// Deprecated: Use Casbin grouping statements g, <user/group>, <role>

service/internal/server/server.go

@@ -69,7 +69,7 @@ type Config struct {
 	// Enable pprof
 	EnablePprof bool `mapstructure:"enable_pprof" json:"enable_pprof" default:"false"`
 	// Trace is for configuring open telemetry based tracing.
-	Trace tracing.Config `mapstructure:"trace"`
+	Trace tracing.Config `mapstructure:"trace" json:"trace"`
 }
 
 func (c Config) LogValue() slog.Value {
@@ -126,7 +126,7 @@ type CORSConfig struct {
 	AllowedMethods   []string `mapstructure:"allowedmethods" json:"allowedmethods" default:"[\"GET\",\"POST\",\"PATCH\",\"DELETE\",\"OPTIONS\"]"`
 	AllowedHeaders   []string `mapstructure:"allowedheaders" json:"allowedheaders" default:"[\"Accept\",\"Content-Type\",\"Content-Length\",\"Accept-Encoding\",\"X-CSRF-Token\",\"Authorization\",\"X-Requested-With\",\"Dpop\",\"Connect-Protocol-Version\"]"`
 	ExposedHeaders   []string `mapstructure:"exposedheaders" json:"exposedheaders"`
-	AllowCredentials bool     `mapstructure:"allowcredentials" json:"allowedcredentials" default:"true"`
+	AllowCredentials bool     `mapstructure:"allowcredentials" json:"allowcredentials" default:"true"`
 	MaxAge           int      `mapstructure:"maxage" json:"maxage" default:"3600"`
 	Debug            bool     `mapstructure:"debug" json:"debug"`
 }

service/pkg/config/config.go

@@ -3,12 +3,16 @@ package config
 import (
 	"context"
 	"errors"
+	"fmt"
 	"log/slog"
+	"sync"
 
+	"github.com/go-playground/validator/v10"
 	"github.com/opentdf/platform/service/internal/server"
 	"github.com/opentdf/platform/service/logger"
 	"github.com/opentdf/platform/service/pkg/db"
 	"github.com/opentdf/platform/service/tracing"
+	"github.com/spf13/viper"
 )
 
 // ChangeHook is a function invoked when the configuration changes.
@@ -42,15 +46,17 @@ type Config struct {
 	SDKConfig SDKConfig `mapstructure:"sdk_config" json:"sdk_config"`
 
 	// Services represents the configuration settings for the services.
-	Services ServicesMap `mapstructure:"services"`
+	Services ServicesMap `mapstructure:"services" json:"services"`
 
 	// Trace is for configuring open telemetry based tracing.
-	Trace tracing.Config `mapstructure:"trace"`
+	Trace tracing.Config `mapstructure:"trace" json:"trace"`
 
 	// onConfigChangeHooks is a list of functions to call when the configuration changes.
 	onConfigChangeHooks []ChangeHook
 	// loaders is a list of configuration loaders.
 	loaders []Loader
+	// reloadMux ensures that the Reload function is thread-safe.
+	reloadMux *sync.Mutex
 }
 
 // SDKConfig represents the configuration for the SDK.
@@ -115,8 +121,23 @@ func (c *Config) Watch(ctx context.Context) error {
 	if len(c.loaders) == 0 {
 		return nil
 	}
+
 	for _, loader := range c.loaders {
-		if err := loader.Watch(ctx, c, c.OnChange); err != nil {
+		// onChangeCallback is the function that will be called by loaders when a change is detected.
+		// It orchestrates the reloading of the entire configuration and then triggers the registered hooks.
+		loaderName := loader.Name()
+		onChangeCallback := func(ctx context.Context) error {
+			slog.InfoContext(ctx, "configuration change detected, reloading...", slog.String("loader", loaderName))
+			if err := c.Reload(ctx); err != nil {
+				slog.ErrorContext(ctx, "failed to reload configuration", slog.Any("error", err))
+				return err
+			}
+			slog.InfoContext(ctx, "configuration reloaded successfully")
+
+			// Now call the user-provided hooks with the new configuration.
+			return c.OnChange(ctx)
+		}
+		if err := loader.Watch(ctx, c, onChangeCallback); err != nil {
 			return err
 		}
 	}
@@ -138,10 +159,11 @@ func (c *Config) Close(ctx context.Context) error {
 }
 
 // OnChange invokes all registered onConfigChangeHooks after a configuration change.
-func (c *Config) OnChange(_ context.Context) error {
-	if len(c.loaders) == 0 {
+func (c *Config) OnChange(ctx context.Context) error {
+	if len(c.onConfigChangeHooks) == 0 {
 		return nil
 	}
+	slog.DebugContext(ctx, "executing configuration change hooks")
 	for _, hook := range c.onConfigChangeHooks {
 		if err := hook(c.Services); err != nil {
 			return err
@@ -150,6 +172,87 @@ func (c *Config) OnChange(_ context.Context) error {
 	return nil
 }
 
+// Reload re-reads and merges configuration from all registered loaders.
+// It is thread-safe and handles dependencies between loaders by iterating
+// until the configuration stabilizes.
+func (c *Config) Reload(ctx context.Context) error {
+	// Lock to ensure only one reload operation happens at a time.
+	c.reloadMux.Lock()
+	defer c.reloadMux.Unlock()
+
+	// This loop handles dependencies between loaders. It continues to iterate
+	// until a full pass over all loaders adds no new configuration values.
+	// This ensures that a loader can use configuration provided by another
+	// loader that appears later in the priority list.
+	var assigned map[string]struct{}
+	for {
+		lastAssignedCount := len(assigned)
+		assigned = make(map[string]struct{})
+		orderedViper := viper.NewWithOptions(viper.WithLogger(slog.Default()))
+
+		// Loop through loaders in their order of priority.
+		for _, loader := range c.loaders {
+			// The Load call allows the loader to refresh its internal state (e.g., re-read file, re-query DB).
+			// It uses the config `c` from the *previous* iteration to configure itself if needed.
+			if err := loader.Load(*c); err != nil {
+				return fmt.Errorf("loader %s failed to load: %w", loader.Name(), err)
+			}
+
+			// Get all keys this loader knows about.
+			keys, err := loader.GetConfigKeys()
+			if err != nil {
+				slog.WarnContext(
+					ctx,
+					"loader failed to get config keys",
+					slog.String("loader", loader.Name()),
+					slog.Any("error", err),
+				)
+				continue
+			}
+
+			// Merge values from the current loader into Viper.
+			for _, key := range keys {
+				// If a higher-priority loader already set this key, skip.
+				if _, assignedAlready := assigned[key]; assignedAlready {
+					continue
+				}
+				loaderValue, err := loader.Get(key)
+				if err != nil {
+					slog.WarnContext(
+						ctx,
+						"loader.Get failed for a reported key",
+						slog.String("loader", loader.Name()),
+						slog.String("key", key),
+						slog.Any("error", err),
+					)
+					continue
+				}
+				if loaderValue != nil {
+					orderedViper.Set(key, loaderValue)
+					assigned[key] = struct{}{}
+				}
+			}
+		}
+
+		// Unmarshal the merged configuration into the main config struct `c`
+		// so it's available for the next iteration of the dependency loop.
+		if err := orderedViper.Unmarshal(c); err != nil {
+			return errors.Join(err, ErrUnmarshallingConfig)
+		}
+
+		// If no new keys were assigned in this pass, the configuration has stabilized.
+		if len(assigned) == lastAssignedCount {
+			break
+		}
+	}
+
+	// Final validation after the configuration has converged.
+	if err := validator.New().Struct(c); err != nil {
+		return errors.Join(err, ErrUnmarshallingConfig)
+	}
+	return nil
+}
+
 func (c SDKConfig) LogValue() slog.Value {
 	return slog.GroupValue(
 		slog.Group("core",
@@ -165,19 +268,36 @@ func (c SDKConfig) LogValue() slog.Value {
 	)
 }
 
-// LoadConfig loads configuration using the provided loader or creates a default Viper loader
-func LoadConfig(_ context.Context, key, file string) (*Config, error) {
-	config := &Config{}
-
-	// Create default loader if none provided
-	loader, err := NewEnvironmentLoader(key, file)
+// Deprecated: Use the `Load` method with your preferred loaders
+func LoadConfig(ctx context.Context, key, file string) (*Config, error) {
+	envLoader, err := NewEnvironmentValueLoader(key, nil)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("could not load config: %w", err)
+	}
+	configFileLoader, err := NewConfigFileLoader(key, file)
+	if err != nil {
+		return nil, fmt.Errorf("could not load config: %w", err)
+	}
+	defaultSettingsLoader, err := NewDefaultSettingsLoader()
+	if err != nil {
+		return nil, fmt.Errorf("could not load config: %w", err)
+	}
+	return Load(
+		ctx,
+		envLoader,
+		configFileLoader,
+		defaultSettingsLoader,
+	)
+}
+
+// Load loads configuration using the provided loaders
+func Load(ctx context.Context, loaders ...Loader) (*Config, error) {
+	config := &Config{
+		loaders:   loaders,
+		reloadMux: &sync.Mutex{},
 	}
 
-	// Load initial configuration
-	config.AddLoader(loader)
-	if err := loader.Load(config); err != nil {
+	if err := config.Reload(ctx); err != nil {
 		return nil, err
 	}