diff --git a/internal/app/app.go b/internal/app/app.go
index 6762a303..12a09963 100644
--- a/internal/app/app.go
+++ b/internal/app/app.go
@@ -30,6 +30,7 @@ import (
"net/http"
"os"
"os/signal"
+ "sync/atomic"
"time"
"golang.org/x/sync/errgroup"
@@ -91,6 +92,10 @@ type App struct {
// tenants is the registry every tenant-aware path resolves through, and
// the owner of every reload.
tenants *settings.Registry
+ // secrets is the boot secrets as last resolved, replaced whole by the
+ // first AfterAdopt hook (wireSecrets) so every later hook reads the
+ // rotated values.
+ secrets atomic.Pointer[config.Secrets]
// policies is the default tenant's policy, for the ops gate of a flat
// directory.
policies policy.Source
@@ -170,6 +175,9 @@ func New(ctx context.Context, opts Options) (app *App, err error) {
if err := a.wireSettings(); err != nil {
return nil, err
}
+ if err := a.wireSecrets(); err != nil {
+ return nil, err
+ }
a.wireObservability(ctx)
// After observability, so an OTLP log pipeline carries them too.
for _, w := range a.cfg.Warnings() {
diff --git a/internal/app/app_test.go b/internal/app/app_test.go
index 7fbc1c1b..a19aac05 100644
--- a/internal/app/app_test.go
+++ b/internal/app/app_test.go
@@ -268,6 +268,63 @@ func TestReload_DrivesTheRegisteredHooks(t *testing.T) {
assert.False(t, dedup.Open(), "dedupe hook closed the store")
}
+// TestReload_ReReadsMountedSecrets pins the rotation story of #529 through
+// the real wiring: each secret given as a *_file is re-read on every reload,
+// and the hooks that follow apply it — the operator key on the next request,
+// the HMAC verifier rebuilt so the retired secret's tokens stop validating,
+// the ClickHouse pool repointed to the new password. A file that fails to
+// read keeps the previous values, logged, like a rejected reload.
+func TestReload_ReReadsMountedSecrets(t *testing.T) {
+ dir := t.TempDir()
+ write := func(name, value string) string {
+ t.Helper()
+ p := filepath.Join(dir, name)
+ require.NoError(t, os.WriteFile(p, []byte(value+"\n"), 0o600))
+ return p
+ }
+ jwtFile, opFile, pwFile := write("jwt", testutil.TestJWTSecret), write("op", "op-1"), write("pw", "pw-1")
+ cfg := testConfig(t, writeSettings(t, nil))
+ cfg.Auth = config.Auth{JWTSecretFile: jwtFile, OperatorKeyFile: opFile}
+ cfg.ClickHouse.PasswordFile = pwFile
+ a := newApp(t, cfg, Options{})
+
+ ops := func(header, value string) int {
+ req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/v1/ops/pipes", nil)
+ req.Header.Set(header, value)
+ rec := httptest.NewRecorder()
+ a.Handler().ServeHTTP(rec, req)
+ return rec.Code
+ }
+ admin := func(secret string) string {
+ tok := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{"role": "admin", "exp": time.Now().Add(time.Hour).Unix()})
+ signed, err := tok.SignedString([]byte(secret))
+ require.NoError(t, err)
+ return "Bearer " + signed
+ }
+ const rotated = "rotated-secret-at-least-32-chars-long"
+ require.Equal(t, http.StatusOK, ops("X-Operator-Key", "op-1"))
+ require.Equal(t, http.StatusOK, ops("Authorization", admin(testutil.TestJWTSecret)))
+ require.Equal(t, "pw-1", a.pools.Target(tenant.Default).Password)
+
+ write("jwt", rotated)
+ write("op", "op-2")
+ write("pw", "pw-2")
+ _, adopted := a.tenants.Reload("test")
+ require.True(t, adopted)
+ assert.Equal(t, http.StatusOK, ops("X-Operator-Key", "op-2"), "the next request matches the rotated operator key")
+ assert.Equal(t, http.StatusForbidden, ops("X-Operator-Key", "op-1"), "the retired key falls through like any wrong key")
+ assert.Equal(t, http.StatusOK, ops("Authorization", admin(rotated)), "the HMAC verifier was rebuilt with the new secret")
+ assert.Equal(t, http.StatusUnauthorized, ops("Authorization", admin(testutil.TestJWTSecret)), "a token signed with the retired secret stops validating")
+ assert.Equal(t, "pw-2", a.pools.Target(tenant.Default).Password, "the pool was repointed to the new password")
+
+ logs := logtest.Capture(t, slog.LevelError)
+ require.NoError(t, os.Remove(opFile))
+ _, adopted = a.tenants.Reload("test")
+ require.True(t, adopted, "the settings reload itself is unaffected")
+ assert.Contains(t, logs.String(), "secrets re-read failed")
+ assert.Equal(t, http.StatusOK, ops("X-Operator-Key", "op-2"), "an unreadable file keeps the previous values")
+}
+
// writeNestedSettings materializes a nested settings directory: tenant folder
// → the patch writeSettings applies to that tenant's config.json.
func writeNestedSettings(t *testing.T, tenants map[string]map[string]any) string {
diff --git a/internal/app/wire.go b/internal/app/wire.go
index 12b4105e..8b21e77a 100644
--- a/internal/app/wire.go
+++ b/internal/app/wire.go
@@ -13,7 +13,6 @@ import (
"os/signal"
"path/filepath"
"slices"
- "strings"
"sync"
"syscall"
"time"
@@ -76,6 +75,40 @@ func (a *App) wireSettings() error {
return nil
}
+// wireSecrets resolves the boot secrets — the ClickHouse password, the HMAC
+// secret and the operator key, each a literal or a mounted file — and keeps
+// them current: the first AfterAdopt hook re-reads the files after every
+// reload, so the ClickHouse and auth hooks that follow it see a rotated
+// mount and apply it (a repointed pool, rebuilt HMAC verifiers, the next
+// request's operator key) with no restart. A literal never changes. A file
+// that fails to read at reload keeps the previous values, the same posture
+// as a rejected settings reload; at boot config.Load has already refused it.
+func (a *App) wireSecrets() error {
+ boot, err := a.cfg.Secrets()
+ if err != nil {
+ return fmt.Errorf("read secrets: %w", err)
+ }
+ a.secrets.Store(&boot)
+ a.tenants.AfterAdopt(func([]tenant.ID) {
+ next, err := a.cfg.Secrets()
+ if err != nil {
+ slog.Error("secrets re-read failed; the previous values stay in effect", "error", err)
+ return
+ }
+ if changed := next.Changed(*a.secrets.Load()); len(changed) != 0 {
+ slog.Info("secrets rotated", "keys", changed)
+ }
+ a.secrets.Store(&next)
+ })
+ return nil
+}
+
+// authSecrets is the auth half of the current boot secrets.
+func (a *App) authSecrets() auth.Config {
+ s := a.secrets.Load()
+ return auth.Config{JWTSecret: s.JWTSecret, OperatorKey: s.OperatorKey}
+}
+
// defaultPolicy is the default tenant's access-control policy, which the ops
// gate of a flat directory reads its admin role from per request. There
// tenant 0 is the whole directory, always served: a reload that fails keeps
@@ -259,7 +292,9 @@ func (a *App) wireObservability(ctx context.Context) {
// wireClickHouse opens the ClickHouse pools: one per distinct address,
// database, user, password and tls tuple among the served tenants' clickhouse
-// blocks (with the boot-config password), shared by the tenants naming it and
+// blocks (with the boot-config password, re-resolved before this hook runs
+// — a rotated password is a new tuple, so the reconcile repoints every
+// tenant to a pool on it), shared by the tenants naming it and
// sized to their largest ask (#583 story 6). Every reload reconciles them —
// a new tuple opens (no dial), a tenant whose tuple changed is repointed, a
// tuple no tenant names is released after its grace — under the boot
@@ -280,7 +315,7 @@ func (a *App) wireClickHouse() error {
c := store.ClickHouse()
ms = append(ms, chconn.Member{Tenant: id, Params: chconn.Params{
Addr: c.Addr, HTTPPort: c.HTTPPort, HTTPScheme: c.HTTPScheme,
- Database: c.Database, Username: c.Username, Password: a.cfg.ClickHouse.Password,
+ Database: c.Database, Username: c.Username, Password: a.secrets.Load().CHPassword,
QueryTimeout: c.QueryTimeout,
TLS: chconn.TLS(c.TLS),
Headers: c.Headers,
@@ -835,7 +870,9 @@ func (a *App) wireIngestWorker() {
// wireAuth builds the JWT middleware: one verifier per tenant being served,
// from that tenant's auth block (jwks_url, role_claim), with the secrets from
-// boot config shared by all. A tenant's verifier is rebuilt after a reload
+// boot config shared by all — re-resolved before this hook runs, so a
+// rotated mount reaches the verifiers and the operator key through Rotate
+// on the same reload. A tenant's verifier is rebuilt after a reload
// that adopts it with changed wiring, kept when the wiring is unchanged, and
// dropped once the tenant stops being served, removed or rejected alike — no
// work runs for a tenant that is not served, and a folder adopted again is
@@ -852,8 +889,8 @@ func (a *App) wireIngestWorker() {
// default_role (a public tenant). That's a valid posture, so it warns per
// tenant rather than fails.
func (a *App) wireAuth() func(http.Handler) http.Handler {
- cfg := a.cfg
- switch cfg.Auth.JWTSecret {
+ boot := a.secrets.Load()
+ switch boot.JWTSecret {
case "":
// Per tenant: with no boot secret each tenant is as public as its own
// jwks_url leaves it, and one tenant's provider says nothing about
@@ -867,7 +904,7 @@ func (a *App) wireAuth() func(http.Handler) http.Handler {
slog.Warn("WH_AUTH_JWT_SECRET is using the default insecure value")
}
- operatorKey := strings.TrimSpace(cfg.Auth.OperatorKey)
+ operatorKey := boot.OperatorKey
switch {
case operatorKey == "" && a.tenants.Nested():
// Not the recovery concern below: over a nested directory the key is
@@ -897,7 +934,7 @@ func (a *App) wireAuth() func(http.Handler) http.Handler {
}
return store.Tenant(), true
}
- authn := auth.NewAuthenticator(auth.Config{JWTSecret: cfg.Auth.JWTSecret, OperatorKey: operatorKey}, tenantOf, policies)
+ authn := auth.NewAuthenticator(a.authSecrets(), tenantOf, policies)
a.add(component{name: "auth", close: func(context.Context) error {
authn.Close()
return nil
@@ -910,6 +947,7 @@ func (a *App) wireAuth() func(http.Handler) http.Handler {
authn.Reconfigure(id, wiring(store))
}
a.tenants.AfterAdopt(func(adopted []tenant.ID) {
+ authn.Rotate(a.authSecrets())
for _, id := range adopted {
if store, ok := a.tenants.For(id); ok {
authn.Reconfigure(id, wiring(store))
diff --git a/internal/app/wire_ops.go b/internal/app/wire_ops.go
index 2f3c460c..e5fdbf05 100644
--- a/internal/app/wire_ops.go
+++ b/internal/app/wire_ops.go
@@ -6,18 +6,19 @@ package app
import (
"log/slog"
"net/http"
- "strings"
"github.com/Wave-RF/WaveHouse/internal/api"
"github.com/Wave-RF/WaveHouse/internal/auth"
+ "github.com/Wave-RF/WaveHouse/internal/tenant"
)
// wireOpsAuth is the authentication of a process without the api role: the
// operator key and nothing else. Token verifiers — and the JWKS fetches that
// keep them — are per API process, so no token validates here and the reload
-// route admits the operator alone (api.NewOpsRouter).
+// route admits the operator alone (api.NewOpsRouter). The key follows a
+// rotated mount on the same reload, through Rotate.
func (a *App) wireOpsAuth() func(http.Handler) http.Handler {
- operatorKey := strings.TrimSpace(a.cfg.Auth.OperatorKey)
+ operatorKey := a.secrets.Load().OperatorKey
switch {
case operatorKey == "" && a.tenants.Nested():
slog.Warn("no auth.operator_key set: a process without the api role takes only the operator key on POST /v1/ops/settings/reload, and a nested settings directory has no watcher, so its settings can only be reloaded by SIGHUP")
@@ -25,6 +26,9 @@ func (a *App) wireOpsAuth() func(http.Handler) http.Handler {
slog.Warn("no auth.operator_key set: a process without the api role takes only the operator key on POST /v1/ops/settings/reload, so its settings can only be reloaded by SIGHUP or the directory watcher")
}
authn := auth.NewAuthenticator(auth.Config{OperatorKey: operatorKey}, nil, nil)
+ a.tenants.AfterAdopt(func([]tenant.ID) {
+ authn.Rotate(auth.Config{OperatorKey: a.secrets.Load().OperatorKey})
+ })
return authn.Middleware()
}
diff --git a/internal/auth/auth.go b/internal/auth/auth.go
index 1156b089..cb908a1e 100644
--- a/internal/auth/auth.go
+++ b/internal/auth/auth.go
@@ -25,8 +25,10 @@ import (
"golang.org/x/time/rate"
)
-// Config is the boot-config half of authentication: the secrets, fixed for
-// the process lifetime and shared by every tenant.
+// Config is the boot-config half of authentication: the secrets, shared by
+// every tenant. A literal is fixed for the process lifetime; one given as a
+// mounted file is re-read by internal/app after each settings reload and
+// applied through Authenticator.Rotate.
type Config struct {
// JWTSecret is the HMAC secret, the verifier of a tenant whose Wiring
// names no JWKS URL.
@@ -302,10 +304,13 @@ func (b *cappedBody) Read(p []byte) (int, error) {
// builds a new verifier from the tenant's adopted wiring and swaps it in
// when that wiring changed, keeping the one it has when it did not; Prune
// drops the verifiers of tenants that stopped being served, rejected or
-// removed; Close stops every JWKS refresh. The secrets are boot config and
-// never change.
+// removed; Close stops every JWKS refresh. The secrets are boot config,
+// re-resolved from their files after a reload: Rotate swaps the operator key
+// and rebuilds every HMAC verifier when they changed.
type Authenticator struct {
- cfg Config
+ // secrets is replaced whole by Rotate, so a request reads the operator
+ // key with one lock-free load.
+ secrets atomic.Pointer[Config]
tenantOf TenantSource
policies PolicySource
mu sync.Mutex // serializes Reconfigure, Prune, and Close
@@ -320,7 +325,8 @@ type Authenticator struct {
// backs the operator-key path (the admin role it stamps is the request
// tenant's) and may be nil when no operator key is configured.
func NewAuthenticator(cfg Config, tenantOf TenantSource, policies PolicySource) *Authenticator {
- a := &Authenticator{cfg: cfg, tenantOf: tenantOf, policies: policies}
+ a := &Authenticator{tenantOf: tenantOf, policies: policies}
+ a.secrets.Store(&cfg)
a.verifiers.Store(&map[tenant.ID]*verifier{})
return a
}
@@ -338,13 +344,39 @@ func (a *Authenticator) Reconfigure(id tenant.ID, w Wiring) {
return
}
next := maps.Clone(*a.verifiers.Load())
- next[id] = newVerifier(a.cfg, w)
+ next[id] = newVerifier(*a.secrets.Load(), w)
a.verifiers.Store(&next)
if old != nil {
old.stop()
}
}
+// Rotate applies re-resolved secrets: the operator key the next request is
+// matched against, and the HMAC secret every tenant on an HMAC verifier (no
+// JWKS URL) is rebuilt with — its own wiring, the new secret — so a token
+// signed with the retired secret stops validating at once. A JWKS verifier
+// never reads the secret and is kept, refresh and all. Unchanged secrets are
+// a no-op.
+func (a *Authenticator) Rotate(cfg Config) {
+ a.mu.Lock()
+ defer a.mu.Unlock()
+ prev := *a.secrets.Load()
+ if prev == cfg {
+ return
+ }
+ a.secrets.Store(&cfg)
+ if prev.JWTSecret == cfg.JWTSecret {
+ return
+ }
+ next := maps.Clone(*a.verifiers.Load())
+ for id, old := range next {
+ if old.url == "" {
+ next[id] = newVerifier(cfg, Wiring{RoleClaim: old.roleClaim})
+ }
+ }
+ a.verifiers.Store(&next)
+}
+
// Prune drops the verifier of every tenant served does not vouch for — the
// tenants a reload removed or rejected — and stops their JWKS refresh: a
// tenant that is not being served has no work running for it, and one that
@@ -393,9 +425,10 @@ func (a *Authenticator) verifierFor(ctx context.Context) (tenant.ID, *verifier)
}
// Middleware returns the http middleware bound to this Authenticator; it
-// reads the request tenant's current verifier on every request.
+// reads the request tenant's current verifier and the current operator key
+// on every request.
func (a *Authenticator) Middleware() func(http.Handler) http.Handler {
- return middleware(a.verifierFor, a.cfg.OperatorKey, a.policies)
+ return middleware(a.verifierFor, func() string { return a.secrets.Load().OperatorKey }, a.policies)
}
var (
@@ -449,12 +482,13 @@ var operatorKeyFailures, _ = otel.Meter("wavehouse-auth").Int64Counter(
// and counted by wavehouse_auth_operator_key_failures_total (a probing
// signal), then falls through like any unauthenticated request. policies may
// be nil when no operator key is configured.
-func middleware(current func(context.Context) (tenant.ID, *verifier), operatorKeyCfg string, policies PolicySource) func(http.Handler) http.Handler {
+func middleware(current func(context.Context) (tenant.ID, *verifier), currentOperatorKey func() string, policies PolicySource) func(http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- // One verifier per request: the swap is atomic, so a reload lands
- // between requests, never inside one.
+ // One verifier and one operator key per request: each swap is
+ // atomic, so a reload lands between requests, never inside one.
id, v := current(r.Context())
+ operatorKeyCfg := currentOperatorKey()
// Resolved before the operator branch purely for its side effect: it
// strips ?token from r.URL, and an operator-key match returns without
diff --git a/internal/auth/auth_test.go b/internal/auth/auth_test.go
index 9e92cb9a..8a6882ca 100644
--- a/internal/auth/auth_test.go
+++ b/internal/auth/auth_test.go
@@ -979,6 +979,62 @@ func TestAuthenticator_ReconfigureSwapsRoleClaim(t *testing.T) {
assert.Equal(t, "new", got, "the next request sees the reconfigured claim path")
}
+// TestAuthenticator_RotateSwapsOperatorKey pins the rotation contract for
+// the operator key (#529): internal/app re-resolves a *_file secret after
+// each settings reload and passes it through Rotate, so the next request
+// matches the new key and the retired one falls through like any wrong key.
+func TestAuthenticator_RotateSwapsOperatorKey(t *testing.T) {
+ t.Parallel()
+ policies := staticPolicies(&policy.Policy{AdminRole: "admin"})
+ a := newAuth(t, Config{JWTSecret: testutil.TestJWTSecret, OperatorKey: "old-key"}, roleClaim(), policies)
+
+ assert.True(t, serve(t, a, operatorHeader("old-key")).isOperator)
+ assert.False(t, serve(t, a, operatorHeader("new-key")).isOperator)
+
+ a.Rotate(Config{JWTSecret: testutil.TestJWTSecret, OperatorKey: "new-key"})
+ assert.True(t, serve(t, a, operatorHeader("new-key")).isOperator, "the next request matches the rotated key")
+ assert.False(t, serve(t, a, operatorHeader("old-key")).isOperator, "the retired key falls through like any wrong key")
+
+ a.Rotate(Config{JWTSecret: testutil.TestJWTSecret})
+ assert.False(t, serve(t, a, operatorHeader("new-key")).isOperator, "an empty key disables the operator path")
+}
+
+// TestAuthenticator_RotateRebuildsHMACVerifiers pins the HMAC half: a
+// rotated secret rebuilds every HMAC tenant's verifier with its own wiring,
+// so a token signed with the retired secret stops validating at once and one
+// signed with the new secret validates, while a JWKS tenant — which never
+// reads the secret — keeps the verifier it has, fetch and all. Unchanged
+// secrets are a no-op.
+func TestAuthenticator_RotateRebuildsHMACVerifiers(t *testing.T) {
+ t.Parallel()
+ const rotated = "rotated-secret-at-least-32-chars-long"
+ srv, _ := jwksServer(t, "kid-1", 0)
+ a := newAuth(t, cfg(), Wiring{RoleClaim: "app_metadata.role"}, nil)
+ a.Reconfigure("acme", Wiring{JWKSURL: srv.URL})
+ jwksBefore := (*a.verifiers.Load())["acme"]
+ hmacBefore := (*a.verifiers.Load())[tenant.Default]
+
+ sign := func(secret string) string {
+ tok := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{"app_metadata": map[string]any{"role": "analyst"}, "exp": time.Now().Add(time.Hour).Unix()})
+ signed, err := tok.SignedString([]byte(secret))
+ require.NoError(t, err)
+ return signed
+ }
+ oldTok, newTok := sign(testutil.TestJWTSecret), sign(rotated)
+ assert.Equal(t, "analyst", serve(t, a, bearer(oldTok)).role)
+ assert.Empty(t, serve(t, a, bearer(newTok)).role)
+
+ a.Rotate(Config{JWTSecret: rotated})
+ assert.Equal(t, "analyst", serve(t, a, bearer(newTok)).role, "the next request validates against the new secret, under the tenant's own claim path")
+ assert.Empty(t, serve(t, a, bearer(oldTok)).role, "a token signed with the retired secret stops validating")
+ assert.NotSame(t, hmacBefore, (*a.verifiers.Load())[tenant.Default])
+ assert.Same(t, jwksBefore, (*a.verifiers.Load())["acme"], "a JWKS verifier never reads the secret and is kept")
+
+ after := (*a.verifiers.Load())[tenant.Default]
+ a.Rotate(Config{JWTSecret: rotated})
+ assert.Same(t, after, (*a.verifiers.Load())[tenant.Default], "unchanged secrets rebuild nothing")
+}
+
// TestAuthenticator_ReconfigureAppliesUnreachableJWKS pins "settings are the
// authority": a reload pointing at an unreachable JWKS swaps the verifier
// anyway, so the HMAC token stops validating (fail closed) instead of the
diff --git a/internal/config/config.go b/internal/config/config.go
index 1ae02fb0..94cd76b1 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -136,10 +136,14 @@ type Server struct {
// headers, pool size — is the settings directory's `clickhouse` block
// (hot-reloadable: a change swaps the connection). The password stays here
// because secrets don't belong in a tracked JSON file; it is combined with
-// the adopted wiring on every (re)connect. The ceiling stays here because
-// it is capacity, sized once per process, not wiring.
+// the adopted wiring on every (re)connect, either the literal or
+// PasswordFile — a mounted secret file, re-read on every settings reload so
+// a rotated file takes effect without a restart (see Secrets) — never both.
+// The ceiling stays here because it is capacity, sized once per process,
+// not wiring.
type ClickHouse struct {
- Password string `yaml:"password" env:"WH_CH_PASSWORD"`
+ Password string `yaml:"password" env:"WH_CH_PASSWORD"`
+ PasswordFile string `yaml:"password_file" env:"WH_CH_PASSWORD_FILE"`
// MaxTotalConns caps the native connections the process may hold open
// across its pools: the settings directory's clickhouse.max_open_conns
// must not exceed it. 0, the default, is no ceiling.
@@ -161,9 +165,15 @@ type ClickHouse struct {
// operator, independent of the JWT verifier, and keeps working even if the
// policy is missing/deleted (break-glass recovery, e.g. fixing policies.json). Empty (the default) disables
// it. Treat it as an admin secret.
+//
+// Each secret is either its literal key or its *_file key — a mounted
+// secret file, re-read on every settings reload so a rotated file takes
+// effect without a restart (see Secrets) — never both.
type Auth struct {
- JWTSecret string `yaml:"jwt_secret" env:"WH_AUTH_JWT_SECRET"`
- OperatorKey string `yaml:"operator_key" env:"WH_AUTH_OPERATOR_KEY"`
+ JWTSecret string `yaml:"jwt_secret" env:"WH_AUTH_JWT_SECRET"`
+ JWTSecretFile string `yaml:"jwt_secret_file" env:"WH_AUTH_JWT_SECRET_FILE"`
+ OperatorKey string `yaml:"operator_key" env:"WH_AUTH_OPERATOR_KEY"`
+ OperatorKeyFile string `yaml:"operator_key_file" env:"WH_AUTH_OPERATOR_KEY_FILE"`
}
// Role is one part of the work a process can run.
@@ -300,6 +310,19 @@ func (c *Config) Validate() error {
return fmt.Errorf("clickhouse.max_total_conns must be >= 0 (0 is no ceiling), got %d", c.ClickHouse.MaxTotalConns)
}
+ // A secret is the literal key or the *_file key, never both: there is
+ // no precedence rule to learn, and a leftover literal can't silently
+ // shadow a rotated file (or the reverse).
+ for _, k := range []struct{ key, literal, file string }{
+ {"clickhouse.password", c.ClickHouse.Password, c.ClickHouse.PasswordFile},
+ {"auth.jwt_secret", c.Auth.JWTSecret, c.Auth.JWTSecretFile},
+ {"auth.operator_key", c.Auth.OperatorKey, c.Auth.OperatorKeyFile},
+ } {
+ if k.literal != "" && k.file != "" {
+ return fmt.Errorf("%s and %s_file are both set: choose one", k.key, k.key)
+ }
+ }
+
if c.OTel.Enabled {
// Only validate sample rates for signals that are actually enabled —
// an unused field shouldn't block startup.
@@ -392,6 +415,12 @@ func Load(path string) (*Config, error) {
if err := cfg.Validate(); err != nil {
return nil, fmt.Errorf("validate config: %w", err)
}
+ // A *_file secret that can't be read is a boot error, like every other
+ // boot-config fault: a missing mount must refuse to start, not boot with
+ // an empty secret and quietly turn the deployment public.
+ if _, err := cfg.Secrets(); err != nil {
+ return nil, fmt.Errorf("read secrets: %w", err)
+ }
return &cfg, nil
}
diff --git a/internal/config/config_test.go b/internal/config/config_test.go
index ee8b07cc..98b46694 100644
--- a/internal/config/config_test.go
+++ b/internal/config/config_test.go
@@ -1,6 +1,7 @@
package config
import (
+ "fmt"
"os"
"path/filepath"
"reflect"
@@ -417,3 +418,121 @@ func TestValidate_NegativeMaxTotalConns(t *testing.T) {
require.ErrorContains(t, err, "clickhouse.max_total_conns must be >= 0")
require.ErrorContains(t, err, "got -1")
}
+
+// TestSecrets_FromFiles pins the *_file source (#529): each secret can come
+// from a mounted file instead of a literal, a trailing newline (what
+// Docker/Kubernetes secret mounts usually carry) is stripped, the operator
+// key is whitespace-trimmed like its literal form, and Secrets re-reads the
+// files on every call so a rotated mount is seen without a restart. An
+// unreadable or empty file is an error, never an empty secret.
+func TestSecrets_FromFiles(t *testing.T) {
+ t.Parallel()
+ dir := t.TempDir()
+ write := func(name, content string) string {
+ t.Helper()
+ p := filepath.Join(dir, name)
+ require.NoError(t, os.WriteFile(p, []byte(content), 0o600))
+ return p
+ }
+ jwt := write("jwt", "file-jwt-secret\n")
+ op := write("op", " file-operator-key \r\n")
+ pw := write("pw", "file-ch-password")
+ yamlContent := fmt.Sprintf(`
+clickhouse:
+ password_file: %q
+auth:
+ jwt_secret_file: %q
+ operator_key_file: %q
+settings:
+ dir: ./settings
+`, pw, jwt, op)
+ path := filepath.Join(dir, "config.yaml")
+ require.NoError(t, os.WriteFile(path, []byte(yamlContent), 0o600))
+
+ cfg, err := Load(path)
+ require.NoError(t, err)
+ assert.Empty(t, cfg.Auth.JWTSecret, "the literal stays empty; the file is read by Secrets")
+
+ s, err := cfg.Secrets()
+ require.NoError(t, err)
+ assert.Equal(t, "file-jwt-secret", s.JWTSecret, "trailing newline stripped")
+ assert.Equal(t, "file-operator-key", s.OperatorKey, "operator key whitespace-trimmed")
+ assert.Equal(t, "file-ch-password", s.CHPassword)
+
+ write("jwt", "rotated\n")
+ next, err := cfg.Secrets()
+ require.NoError(t, err)
+ assert.Equal(t, "rotated", next.JWTSecret, "each call re-reads the file")
+ assert.Equal(t, []string{"auth.jwt_secret"}, next.Changed(s), "Changed names the rotated key only")
+ assert.Empty(t, next.Changed(next))
+
+ write("jwt", "\n")
+ _, err = cfg.Secrets()
+ require.Error(t, err, "an empty file is a broken mount, never an empty secret")
+ assert.Contains(t, err.Error(), "auth.jwt_secret_file")
+ assert.Contains(t, err.Error(), "is empty")
+
+ require.NoError(t, os.Remove(jwt))
+ _, err = cfg.Secrets()
+ require.Error(t, err, "an unreadable file is an error, never an empty secret")
+ assert.Contains(t, err.Error(), "auth.jwt_secret_file")
+}
+
+// TestSecrets_Literals pins that a literal is returned as-is (the operator key
+// trimmed) with no file involved.
+func TestSecrets_Literals(t *testing.T) {
+ t.Setenv("WH_AUTH_JWT_SECRET", "env-jwt")
+ t.Setenv("WH_AUTH_OPERATOR_KEY", " env-op ")
+ t.Setenv("WH_CH_PASSWORD", "env-pw")
+ cfg, err := Load("nonexistent.yaml")
+ require.NoError(t, err)
+ s, err := cfg.Secrets()
+ require.NoError(t, err)
+ assert.Equal(t, "env-jwt", s.JWTSecret)
+ assert.Equal(t, "env-op", s.OperatorKey)
+ assert.Equal(t, "env-pw", s.CHPassword)
+}
+
+func TestLoad_SecretFileEnvBound(t *testing.T) {
+ p := filepath.Join(t.TempDir(), "pw")
+ require.NoError(t, os.WriteFile(p, []byte("pw"), 0o600))
+ t.Setenv("WH_CH_PASSWORD_FILE", p)
+ cfg, err := Load("nonexistent.yaml")
+ require.NoError(t, err, "the *_file env names are bound, so the unbound-env check accepts them")
+ s, err := cfg.Secrets()
+ require.NoError(t, err)
+ assert.Equal(t, "pw", s.CHPassword)
+}
+
+// A missing secret file refuses boot, like every other boot-config fault.
+func TestLoad_SecretFileMissingRefusesBoot(t *testing.T) {
+ t.Setenv("WH_AUTH_JWT_SECRET_FILE", filepath.Join(t.TempDir(), "absent"))
+ _, err := Load("nonexistent.yaml")
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "read secrets")
+ assert.Contains(t, err.Error(), "auth.jwt_secret_file")
+}
+
+// The literal and the file for one secret are mutually exclusive: no
+// precedence rule, so a leftover literal can't shadow a rotated file.
+func TestValidate_SecretLiteralAndFileBothSet(t *testing.T) {
+ t.Parallel()
+ for _, tt := range []struct {
+ name string
+ mut func(*Config)
+ want string
+ }{
+ {"jwt", func(c *Config) { c.Auth.JWTSecret, c.Auth.JWTSecretFile = "x", "/f" }, "auth.jwt_secret and auth.jwt_secret_file"},
+ {"operator", func(c *Config) { c.Auth.OperatorKey, c.Auth.OperatorKeyFile = "x", "/f" }, "auth.operator_key and auth.operator_key_file"},
+ {"password", func(c *Config) { c.ClickHouse.Password, c.ClickHouse.PasswordFile = "x", "/f" }, "clickhouse.password and clickhouse.password_file"},
+ } {
+ t.Run(tt.name, func(t *testing.T) {
+ t.Parallel()
+ cfg := Config{Server: Server{Port: 8080}, Settings: Settings{Dir: "./settings"}}
+ tt.mut(&cfg)
+ err := cfg.Validate()
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), tt.want)
+ })
+ }
+}
diff --git a/internal/config/secrets.go b/internal/config/secrets.go
new file mode 100644
index 00000000..28be501f
--- /dev/null
+++ b/internal/config/secrets.go
@@ -0,0 +1,78 @@
+package config
+
+import (
+ "errors"
+ "fmt"
+ "os"
+ "strings"
+)
+
+// Secrets is the resolved value of the three boot secrets: what the
+// consumers use, whichever source each came from. The operator key is
+// whitespace-trimmed, so a pasted key with a trailing newline still matches.
+type Secrets struct {
+ JWTSecret string
+ OperatorKey string
+ CHPassword string
+}
+
+// Secrets resolves each secret from its literal key or, when the *_file key
+// is set, from that file — the Docker and Kubernetes secret-mount convention,
+// which is also what every secret-store agent (Vault Agent, External Secrets
+// Operator, the CSI drivers) writes to disk. Files are read on every call:
+// internal/app re-resolves after each settings reload, so rotating a mounted
+// secret is a reload, not a restart. A trailing newline is stripped, since
+// mounted secret files usually carry one. A literal is fixed for the process
+// lifetime, because nothing outside the process can change it.
+//
+// An unreadable or empty file is an error: a path was given, so an empty
+// value is a broken mount, not a choice to run without the secret. Load
+// refuses to boot on it; on reload the caller keeps the previous values.
+func (c *Config) Secrets() (Secrets, error) {
+ var s Secrets
+ var err error
+ if s.JWTSecret, err = secret("auth.jwt_secret", c.Auth.JWTSecret, c.Auth.JWTSecretFile); err != nil {
+ return Secrets{}, err
+ }
+ if s.OperatorKey, err = secret("auth.operator_key", c.Auth.OperatorKey, c.Auth.OperatorKeyFile); err != nil {
+ return Secrets{}, err
+ }
+ s.OperatorKey = strings.TrimSpace(s.OperatorKey)
+ if s.CHPassword, err = secret("clickhouse.password", c.ClickHouse.Password, c.ClickHouse.PasswordFile); err != nil {
+ return Secrets{}, err
+ }
+ return s, nil
+}
+
+// Changed names the secrets whose value differs between s and prev, by
+// config key — what a rotation logs, never the values.
+func (s Secrets) Changed(prev Secrets) []string {
+ var keys []string
+ if s.JWTSecret != prev.JWTSecret {
+ keys = append(keys, "auth.jwt_secret")
+ }
+ if s.OperatorKey != prev.OperatorKey {
+ keys = append(keys, "auth.operator_key")
+ }
+ if s.CHPassword != prev.CHPassword {
+ keys = append(keys, "clickhouse.password")
+ }
+ return keys
+}
+
+// secret returns literal when file is unset, else file's contents minus a
+// trailing newline. Validate has already refused both being set.
+func secret(key, literal, file string) (string, error) {
+ if file == "" {
+ return literal, nil
+ }
+ data, err := os.ReadFile(file) //nolint:gosec // G304: the secret path is operator-controlled by design
+ if err != nil {
+ return "", fmt.Errorf("%s_file: %w", key, err)
+ }
+ value := strings.TrimRight(string(data), "\r\n")
+ if value == "" {
+ return "", fmt.Errorf("%s_file: %w", key, errors.New(file+" is empty"))
+ }
+ return value, nil
+}
Context
#529 settled where the secrets live: a tenant's
clickhouse.passwordandauth.jwt_secretare keys of its settings directoryconfig.jsonand reload with the wiring they belong to;auth.operator_key(andcache.redis.password) stay boot config, restart to rotate. The call there was to keep secrets as ordinary config values and try a mounted-file source later. This issue tracks that later.What a
*_filesource would add*_filesibling (auth.operator_key_file/WH_AUTH_OPERATOR_KEY_FILE, and the same shape for any other boot-level secret) naming a mounted secret — the Docker Composesecrets:and KubernetesSecretvolume convention, and what Vault Agent, External Secrets Operator and the CSI drivers write to disk.mq.natsalready takes its credentials this way (password_file,creds_file,nkey_seed_file), so this is the same shape for the rest.SIGHUP, the watcher,POST /v1/ops/settings/reload) as the firstAfterAdopthook, so the hooks that follow see rotated values; a file that fails to read at reload keeps the previous values, logged, the same posture as a rejected reload. Rotating a mounted secret is then "update the mount, trigger a reload", with no restart. The files themselves are not watched.Authenticatorreads it per request behind an atomic and aRotatecall swaps it.Open questions
config.jsonshould also accept a*_filepath (a path is not a secret, so it could live in the tracked file while the value is mounted) — or whether the cloud fan-out writing the value is enough.*_fileexists;mq.natsis file-path-only.Prior work
A complete implementation of the mechanism for the three secrets as they were before the #529 move (boot-level
clickhouse.password,auth.jwt_secret,auth.operator_key): config keys and validation,Config.Secrets()re-read, the first-hook wiring ininternal/app,Authenticator.Rotaterebuilding every HMAC verifier and swapping the operator key, unit and app-level tests, docs and changelog. Patch against main atafd76695, before the per-tenant move, so it needs rebasing onto the new layout (only the operator key is still boot-level): the Go half (code and tests) is inline below; the docs, sample config and compose half of the same patch is on the author's machine and reads the same as the summary above.file-secrets-go.patch (9 files)