feat: Add TLS to Metrics and Telemetry servers (#7041)

Signed-off-by: Simon Behar <simbeh7@gmail.com>

cmd/argo/commands/server.go

@@ -23,7 +23,6 @@ import (
 	"k8s.io/client-go/kubernetes"
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 	restclient "k8s.io/client-go/rest"
-	"k8s.io/utils/env"
 
 	"github.com/argoproj/argo-workflows/v3"
 	"github.com/argoproj/argo-workflows/v3/cmd/argo/commands/client"
@@ -106,20 +105,10 @@ See %s`, help.ArgoServer),
 			var tlsConfig *tls.Config
 			if secure {
 				log.Infof("Generating Self Signed TLS Certificates for Secure Mode")
-				tlsMinVersion, err := env.GetInt("TLS_MIN_VERSION", tls.VersionTLS12)
+				tlsConfig, err = tlsutils.GenerateTLSConfig()
 				if err != nil {
 					return err
 				}
-				var cer *tls.Certificate
-				cer, err = tlsutils.GenerateX509KeyPair()
-				if err != nil {
-					return err
-				}
-				tlsConfig = &tls.Config{
-					Certificates:       []tls.Certificate{*cer},
-					MinVersion:         uint16(tlsMinVersion),
-					InsecureSkipVerify: true,
-				}
 			} else {
 				log.Warn("You are running in insecure mode. Learn how to enable transport layer security: https://argoproj.github.io/argo-workflows/tls/")
 			}

config/config.go

@@ -253,6 +253,15 @@ type MetricsConfig struct {
 	Port int `json:"port,omitempty"`
 	// IgnoreErrors is a flag that instructs prometheus to ignore metric emission errors
 	IgnoreErrors bool `json:"ignoreErrors,omitempty"`
+	// Secure is a flag that starts the metrics servers using TLS
+	Secure *bool `json:"secure,omitempty"`
+}
+
+func (mc MetricsConfig) GetSecure(defaultValue bool) bool {
+	if mc.Secure != nil {
+		return *mc.Secure
+	}
+	return defaultValue
 }
 
 type WorkflowRestrictions struct {

docs/workflow-controller-configmap.yaml

@@ -224,6 +224,8 @@ data:
     metricsTTL: "10m"
     # IgnoreErrors is a flag that instructs prometheus to ignore metric emission errors. Default is "false"
     ignoreErrors: false
+    # Use a self-signed cert for TLS, default false
+    secure: false
 
     # DEPRECATED: Legacy metrics are now removed, this field is ignored
     disableLegacy: false
@@ -234,6 +236,7 @@ data:
     enabled: true
     path: /telemetry
     port: 8080
+    secure: true  # Use a self-signed cert for TLS, default false
 
   # enable persistence using postgres
   persistence: |

util/tls/tls.go

@@ -15,6 +15,8 @@ import (
 	"net"
 	"os"
 	"time"
+
+	"k8s.io/utils/env"
 )
 
 func pemBlockForKey(priv interface{}) *pem.Block {
@@ -100,3 +102,20 @@ func GenerateX509KeyPair() (*tls.Certificate, error) {
 	}
 	return &cert, nil
 }
+
+func GenerateTLSConfig() (*tls.Config, error) {
+	tlsMinVersion, err := env.GetInt("TLS_MIN_VERSION", tls.VersionTLS12)
+	if err != nil {
+		return nil, err
+	}
+	var cer *tls.Certificate
+	cer, err = GenerateX509KeyPair()
+	if err != nil {
+		return nil, err
+	}
+	return &tls.Config{
+		Certificates:       []tls.Certificate{*cer},
+		MinVersion:         uint16(tlsMinVersion),
+		InsecureSkipVerify: true,
+	}, nil
+}

workflow/controller/controller.go

@@ -1122,12 +1122,15 @@ func (wfc *WorkflowController) getMetricsServerConfig() (metrics.ServerConfig, m
 	if port == 0 {
 		port = metrics.DefaultMetricsServerPort
 	}
+
 	metricsConfig := metrics.ServerConfig{
 		Enabled:      wfc.Config.MetricsConfig.Enabled == nil || *wfc.Config.MetricsConfig.Enabled,
 		Path:         path,
 		Port:         port,
 		TTL:          time.Duration(wfc.Config.MetricsConfig.MetricsTTL),
 		IgnoreErrors: wfc.Config.MetricsConfig.IgnoreErrors,
+		// Default to false until v3.5
+		Secure: wfc.Config.MetricsConfig.GetSecure(false),
 	}
 
 	// Telemetry config
@@ -1140,11 +1143,13 @@ func (wfc *WorkflowController) getMetricsServerConfig() (metrics.ServerConfig, m
 	if wfc.Config.TelemetryConfig.Port > 0 {
 		port = wfc.Config.TelemetryConfig.Port
 	}
+
 	telemetryConfig := metrics.ServerConfig{
 		Enabled:      wfc.Config.TelemetryConfig.Enabled == nil || *wfc.Config.TelemetryConfig.Enabled,
 		Path:         path,
 		Port:         port,
 		IgnoreErrors: wfc.Config.TelemetryConfig.IgnoreErrors,
+		Secure:       wfc.Config.TelemetryConfig.GetSecure(metricsConfig.Secure),
 	}
 	return metricsConfig, telemetryConfig
 }

workflow/metrics/metrics.go

@@ -26,10 +26,11 @@ type ServerConfig struct {
 	Port         int
 	TTL          time.Duration
 	IgnoreErrors bool
+	Secure       bool
 }
 
 func (s ServerConfig) SameServerAs(other ServerConfig) bool {
-	return s.Port == other.Port && s.Path == other.Path && s.Enabled && other.Enabled
+	return s.Port == other.Port && s.Path == other.Path && s.Enabled && other.Enabled && s.Secure == other.Secure
 }
 
 type metric struct {

workflow/metrics/metrics_test.go

@@ -48,6 +48,10 @@ func TestServerConfig_SameServerAs(t *testing.T) {
 	b.Port = DefaultMetricsServerPort
 	b.Path = "/telemetry"
 	assert.False(t, a.SameServerAs(b))
+
+	b.Path = DefaultMetricsServerPath
+	b.Secure = true
+	assert.False(t, a.SameServerAs(b))
 }
 
 func TestMetrics(t *testing.T) {

workflow/metrics/server.go

@@ -11,6 +11,8 @@ import (
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	log "github.com/sirupsen/logrus"
 	runtimeutil "k8s.io/apimachinery/pkg/util/runtime"
+
+	tlsutils "github.com/argoproj/argo-workflows/v3/util/tls"
 )
 
 // RunServer starts a metrics server
@@ -51,12 +53,27 @@ func runServer(config ServerConfig, registry *prometheus.Registry, ctx context.C
 	mux.Handle(config.Path, promhttp.HandlerFor(registry, handlerOpts))
 	srv := &http.Server{Addr: fmt.Sprintf(":%v", config.Port), Handler: mux}
 
-	go func() {
-		log.Infof("Starting prometheus metrics server at localhost:%v%s", config.Port, config.Path)
-		if err := srv.ListenAndServe(); err != nil {
+	if config.Secure {
+		log.Infof("Generating Self Signed TLS Certificates for Telemetry Servers")
+		tlsConfig, err := tlsutils.GenerateTLSConfig()
+		if err != nil {
 			panic(err)
 		}
-	}()
+		srv.TLSConfig = tlsConfig
+		go func() {
+			log.Infof("Starting prometheus metrics server at localhost:%v%s", config.Port, config.Path)
+			if err := srv.ListenAndServeTLS("", ""); err != nil {
+				panic(err)
+			}
+		}()
+	} else {
+		go func() {
+			log.Infof("Starting prometheus metrics server at localhost:%v%s", config.Port, config.Path)
+			if err := srv.ListenAndServe(); err != nil {
+				panic(err)
+			}
+		}()
+	}
 
 	// Waiting for stop signal
 	<-ctx.Done()