progress save on certreloader + working local tests

Signed-off-by: Kevin Schoonover <me@kschoon.me>

Author: kevinschoonover

Dockerfile

@@ -1,2 +1,2 @@
 FROM squidfunk/mkdocs-material:9.5
-RUN pip install mkdocs-include-markdown-plugin
+RUN pip install mkdocs-include-markdown-plugin

core/pkg/telemetry/builder.go

@@ -11,6 +11,7 @@ import (
 	"connectrpc.com/connect"
 	"connectrpc.com/otelconnect"
 	"github.com/open-feature/flagd/core/pkg/logger"
+	"github.com/open-feature/flagd/flagd/pkg/certreloader"
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc"
 	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
@@ -33,10 +34,11 @@ const (
 )
 
 type CollectorConfig struct {
-	Target   string
-	CertPath string
-	KeyPath  string
-	CAPath   string
+	Target         string
+	CertPath       string
+	KeyPath        string
+	ReloadInterval time.Duration
+	CAPath         string
 }
 
 // Config of the telemetry runtime. These are expected to be mapped to start-up arguments
@@ -132,15 +134,20 @@ func buildTransportCredentials(_ context.Context, cfg CollectorConfig) (credenti
 			}
 		}
 
+		reloader, err := certreloader.NewCertReloader(certreloader.Config{
+			KeyPath:        cfg.KeyPath,
+			CertPath:       cfg.CertPath,
+			ReloadInterval: time.Minute * 5,
+		})
+		if err != nil {
+			return nil, fmt.Errorf("failed to create certreloader: %w", err)
+		}
+
 		tlsConfig := &tls.Config{
-			RootCAs: capool,
+			RootCAs:    capool,
+			MinVersion: tls.VersionTLS13,
 			GetCertificate: func(chi *tls.ClientHelloInfo) (*tls.Certificate, error) {
-				newCert, err := tls.LoadX509KeyPair(cfg.CertPath, cfg.KeyPath)
-				if err != nil {
-					return nil, err
-				}
-
-				return &newCert, err
+				return reloader.GetCertificate()
 			},
 		}
 

flagd/cmd/start.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"log"
 	"strings"
+	"time"
 
 	"github.com/open-feature/flagd/core/pkg/logger"
 	"github.com/open-feature/flagd/core/pkg/sync"
@@ -16,22 +17,23 @@ import (
 )
 
 const (
-	corsFlagName           = "cors-origin"
-	logFormatFlagName      = "log-format"
-	managementPortFlagName = "management-port"
-	metricsExporter        = "metrics-exporter"
-	ofrepPortFlagName      = "ofrep-port"
-	otelCollectorURI       = "otel-collector-uri"
-	otelCertPathFlagName   = "otel-cert-path"
-	otelKeyPathFlagName    = "otel-key-path"
-	otelCAPathFlagName     = "otel-ca-path"
-	portFlagName           = "port"
-	serverCertPathFlagName = "server-cert-path"
-	serverKeyPathFlagName  = "server-key-path"
-	socketPathFlagName     = "socket-path"
-	sourcesFlagName        = "sources"
-	syncPortFlagName       = "sync-port"
-	uriFlagName            = "uri"
+	corsFlagName               = "cors-origin"
+	logFormatFlagName          = "log-format"
+	managementPortFlagName     = "management-port"
+	metricsExporter            = "metrics-exporter"
+	ofrepPortFlagName          = "ofrep-port"
+	otelCollectorURI           = "otel-collector-uri"
+	otelCertPathFlagName       = "otel-cert-path"
+	otelKeyPathFlagName        = "otel-key-path"
+	otelCAPathFlagName         = "otel-ca-path"
+	otelReloadIntervalFlagName = "otel-reload-interval"
+	portFlagName               = "port"
+	serverCertPathFlagName     = "server-cert-path"
+	serverKeyPathFlagName      = "server-key-path"
+	socketPathFlagName         = "socket-path"
+	sourcesFlagName            = "sources"
+	syncPortFlagName           = "sync-port"
+	uriFlagName                = "uri"
 )
 
 func init() {
@@ -73,6 +75,7 @@ func init() {
 	flags.StringP(otelCertPathFlagName, "D", "", "tls certificate path to use with OpenTelemetry collector")
 	flags.StringP(otelKeyPathFlagName, "K", "", "tls key path to use with OpenTelemetry collector")
 	flags.StringP(otelCAPathFlagName, "A", "", "tls certificate authority path to use with OpenTelemetry collector")
+	flags.DurationP(otelReloadIntervalFlagName, "I", time.Hour, "how long between reloading the otel tls certificate from disk")
 
 	_ = viper.BindPFlag(corsFlagName, flags.Lookup(corsFlagName))
 	_ = viper.BindPFlag(logFormatFlagName, flags.Lookup(logFormatFlagName))
@@ -136,20 +139,21 @@ var startCmd = &cobra.Command{
 
 		// Build Runtime -----------------------------------------------------------
 		rt, err := runtime.FromConfig(logger, Version, runtime.Config{
-			CORS:              viper.GetStringSlice(corsFlagName),
-			MetricExporter:    viper.GetString(metricsExporter),
-			ManagementPort:    viper.GetUint16(managementPortFlagName),
-			OfrepServicePort:  viper.GetUint16(ofrepPortFlagName),
-			OtelCollectorURI:  viper.GetString(otelCollectorURI),
-			OtelCertPath:      viper.GetString(otelCertPathFlagName),
-			OtelKeyPath:       viper.GetString(otelKeyPathFlagName),
-			OtelCAPath:        viper.GetString(otelCAPathFlagName),
-			ServiceCertPath:   viper.GetString(serverCertPathFlagName),
-			ServiceKeyPath:    viper.GetString(serverKeyPathFlagName),
-			ServicePort:       viper.GetUint16(portFlagName),
-			ServiceSocketPath: viper.GetString(socketPathFlagName),
-			SyncServicePort:   viper.GetUint16(syncPortFlagName),
-			SyncProviders:     syncProviders,
+			CORS:               viper.GetStringSlice(corsFlagName),
+			MetricExporter:     viper.GetString(metricsExporter),
+			ManagementPort:     viper.GetUint16(managementPortFlagName),
+			OfrepServicePort:   viper.GetUint16(ofrepPortFlagName),
+			OtelCollectorURI:   viper.GetString(otelCollectorURI),
+			OtelCertPath:       viper.GetString(otelCertPathFlagName),
+			OtelKeyPath:        viper.GetString(otelKeyPathFlagName),
+			OtelReloadInterval: viper.GetDuration(otelReloadIntervalFlagName),
+			OtelCAPath:         viper.GetString(otelCAPathFlagName),
+			ServiceCertPath:    viper.GetString(serverCertPathFlagName),
+			ServiceKeyPath:     viper.GetString(serverKeyPathFlagName),
+			ServicePort:        viper.GetUint16(portFlagName),
+			ServiceSocketPath:  viper.GetString(socketPathFlagName),
+			SyncServicePort:    viper.GetUint16(syncPortFlagName),
+			SyncProviders:      syncProviders,
 		})
 		if err != nil {
 			rtLogger.Fatal(err.Error())

flagd/pkg/certreloader/certreloader.go

@@ -0,0 +1,69 @@
+package certreloader
+
+import (
+	"crypto/tls"
+	"fmt"
+	"sync"
+	"time"
+)
+
+type Config struct {
+	KeyPath        string
+	CertPath       string
+	ReloadInterval time.Duration
+}
+
+type certReloader struct {
+	cert       *tls.Certificate
+	mu         sync.RWMutex
+	nextReload time.Time
+	Config
+}
+
+func NewCertReloader(config Config) (*certReloader, error) {
+	reloader := certReloader{
+		Config: config,
+	}
+
+	reloader.mu.Lock()
+	defer reloader.mu.Unlock()
+	cert, err := reloader.loadCertificate()
+	if err != nil {
+		return nil, fmt.Errorf("failed to load initial certificate: %w", err)
+	}
+	reloader.cert = &cert
+
+	return &reloader, nil
+}
+
+func (r *certReloader) GetCertificate() (*tls.Certificate, error) {
+	now := time.Now()
+	// Read locking here before we do the time comparison
+	// If a reload is in progress this will block and we will skip reloading in the current
+	// call once we can continue
+	r.mu.RLock()
+	shouldReload := r.ReloadInterval != 0 && r.nextReload.Before(now)
+	r.mu.RUnlock()
+	if shouldReload {
+		// Need to release the read lock, otherwise we deadlock
+		r.mu.Lock()
+		defer r.mu.Unlock()
+		cert, err := r.loadCertificate()
+		if err != nil {
+			return nil, fmt.Errorf("failed to load TLS cert and key: %w", err)
+		}
+		r.cert = &cert
+		r.nextReload = now.Add(r.ReloadInterval)
+		return r.cert, nil
+	}
+	return r.cert, nil
+}
+
+func (c *certReloader) loadCertificate() (tls.Certificate, error) {
+	newCert, err := tls.LoadX509KeyPair(c.CertPath, c.KeyPath)
+	if err != nil {
+		return tls.Certificate{}, fmt.Errorf("failed to load key pair: %w", err)
+	}
+
+	return newCert, nil
+}