operator: Support TLS enabled lokistack-gateway (Kubernetes native) (g…

…rafana#6478)

operator/CHANGELOG.md

@@ -1,5 +1,6 @@
 ## Main
 
+- [6411](https://github.com/grafana/loki/pull/6478) **aminesnow**: Support TLS enabled lokistack-gateway for vanilla kubernetes deployments
 - [6504](https://github.com/grafana/loki/pull/6504) **periklis**: Disable usage report on OpenShift
 - [6411](https://github.com/grafana/loki/pull/6411) **Red-GV**: Extend schema validation in LokiStack webhook
 - [6334](https://github.com/grafana/loki/pull/6433) **periklis**: Move operator cli flags to component config

operator/apis/config/v1/projectconfig_types.go

@@ -9,6 +9,7 @@ import (
 type FeatureFlags struct {
 	EnableCertificateSigningService bool `json:"enableCertSigningService,omitempty"`
 	EnableServiceMonitors           bool `json:"enableServiceMonitors,omitempty"`
+	EnableTLSHTTPServices           bool `json:"enableTlsHttpServices,omitempty"`
 	EnableTLSServiceMonitorConfig   bool `json:"enableTlsServiceMonitorConfig,omitempty"`
 	EnableTLSGRPCServices           bool `json:"enableTlsGrpcServices,omitempty"`
 	EnablePrometheusAlerts          bool `json:"enableLokiStackAlerts,omitempty"`

operator/bundle/manifests/loki-operator-manager-config_v1_configmap.yaml

@@ -18,6 +18,7 @@ data:
       enableCertSigningService: true
       enableServiceMonitors: true
       enableTlsServiceMonitorConfig: true
+      enableTlsHttpServices: true
       enableTlsGRPCServices: true
       enableLokiStackAlerts: true
       enableLokiStackGateway: true

operator/cmd/loki-broker/main.go

@@ -38,6 +38,7 @@ func (c *config) registerFlags(f *flag.FlagSet) {
 	c.featureFlags = manifests.FeatureFlags{}
 	f.BoolVar(&c.featureFlags.EnableCertificateSigningService, "with-cert-signing-service", false, "Enable usage of cert-signing service for scraping prometheus metrics via TLS.")
 	f.BoolVar(&c.featureFlags.EnableServiceMonitors, "with-service-monitors", false, "Enable service monitors for all LokiStack components.")
+	f.BoolVar(&c.featureFlags.EnableTLSHTTPServices, "with-http-tls-services", false, "Enables TLS for lokistack-gateway.")
 	f.BoolVar(&c.featureFlags.EnableTLSServiceMonitorConfig, "with-tls-service-monitors", false, "Enable TLS endpoint for service monitors.")
 	f.BoolVar(&c.featureFlags.EnablePrometheusAlerts, "with-prometheus-alerts", false, "Enables prometheus alerts")
 	f.BoolVar(&c.featureFlags.EnableGateway, "with-lokistack-gateway", false, "Enables the manifest creation for the entire lokistack-gateway.")

operator/config/crd/bases/config.grafana.com_projectconfigs.yaml

@@ -78,6 +78,8 @@ spec:
                 type: boolean
               enableTlsServiceMonitorConfig:
                 type: boolean
+              enableTlsHttpServices:
+                type: boolean
             type: object
           gracefulShutDown:
             description: GracefulShutdownTimeout is the duration given to runnable

operator/config/overlays/openshift/controller_manager_config.yaml

@@ -15,6 +15,7 @@ featureFlags:
   enableCertSigningService: true
   enableServiceMonitors: true
   enableTlsServiceMonitorConfig: true
+  enableTlsHttpServices: true
   enableTlsGRPCServices: true
   enableLokiStackAlerts: true
   enableLokiStackGateway: true

operator/docs/howto_connect_grafana.md

@@ -67,7 +67,7 @@ datasources:
     httpHeaderValue1: ${LOKI_TENANT_ID}
 ```
 
-If the operator was started with the `--with-tls-service-monitors` option, then the protocol used to access the service needs to be set to `https` and, depending on the used certificate another option needs to be added to the `jsonData`: `tlsSkipVerify: true`
+If the operator was started with the `--with-http-tls-services` option, then the protocol used to access the service needs to be set to `https` and, depending on the used certificate another option needs to be added to the `jsonData`: `tlsSkipVerify: true`
 
 The values for the variables used in the configuration file depend on the Lokistack deployment and which Loki tenant needs to be accessed.
 

operator/internal/manifests/build_test.go

@@ -219,6 +219,80 @@ func TestBuildAll_WithFeatureFlags_EnableCertificateSigningService(t *testing.T)
 	}
 }
 
+func TestBuildAll_WithFeatureFlags_EnableTLSHTTPServices(t *testing.T) {
+	opts := Options{
+		Name:      "test",
+		Namespace: "test",
+		Stack: lokiv1beta1.LokiStackSpec{
+			Size: lokiv1beta1.SizeOneXSmall,
+			Rules: &lokiv1beta1.RulesSpec{
+				Enabled: true,
+			},
+		},
+		Flags: FeatureFlags{
+			EnableTLSHTTPServices: true,
+		},
+	}
+
+	err := ApplyDefaultSettings(&opts)
+	require.NoError(t, err)
+	objects, buildErr := BuildAll(opts)
+	require.NoError(t, buildErr)
+
+	for _, obj := range objects {
+		var (
+			name string
+			vs   []corev1.Volume
+			vms  []corev1.VolumeMount
+			args []string
+			rps  corev1.URIScheme
+			lps  corev1.URIScheme
+		)
+
+		switch o := obj.(type) {
+		case *appsv1.Deployment:
+			name = o.Name
+			vs = o.Spec.Template.Spec.Volumes
+			vms = o.Spec.Template.Spec.Containers[0].VolumeMounts
+			args = o.Spec.Template.Spec.Containers[0].Args
+			rps = o.Spec.Template.Spec.Containers[0].ReadinessProbe.ProbeHandler.HTTPGet.Scheme
+			lps = o.Spec.Template.Spec.Containers[0].LivenessProbe.ProbeHandler.HTTPGet.Scheme
+		case *appsv1.StatefulSet:
+			name = o.Name
+			vs = o.Spec.Template.Spec.Volumes
+			vms = o.Spec.Template.Spec.Containers[0].VolumeMounts
+			args = o.Spec.Template.Spec.Containers[0].Args
+			rps = o.Spec.Template.Spec.Containers[0].ReadinessProbe.ProbeHandler.HTTPGet.Scheme
+			lps = o.Spec.Template.Spec.Containers[0].LivenessProbe.ProbeHandler.HTTPGet.Scheme
+		default:
+			continue
+		}
+
+		secretName := fmt.Sprintf("%s-http", name)
+		expVolume := corev1.Volume{
+			Name: secretName,
+			VolumeSource: corev1.VolumeSource{
+				Secret: &corev1.SecretVolumeSource{
+					SecretName: secretName,
+				},
+			},
+		}
+		require.Contains(t, vs, expVolume)
+
+		expVolumeMount := corev1.VolumeMount{
+			Name:      secretName,
+			ReadOnly:  false,
+			MountPath: "/var/run/tls/http",
+		}
+		require.Contains(t, vms, expVolumeMount)
+
+		require.Contains(t, args, "-server.http-tls-cert-path=/var/run/tls/http/tls.crt")
+		require.Contains(t, args, "-server.http-tls-key-path=/var/run/tls/http/tls.key")
+		require.Equal(t, corev1.URISchemeHTTPS, rps)
+		require.Equal(t, corev1.URISchemeHTTPS, lps)
+	}
+}
+
 func TestBuildAll_WithFeatureFlags_EnableTLSServiceMonitorConfig(t *testing.T) {
 	opts := Options{
 		Name:      "test",
@@ -231,6 +305,7 @@ func TestBuildAll_WithFeatureFlags_EnableTLSServiceMonitorConfig(t *testing.T) {
 		},
 		Flags: FeatureFlags{
 			EnableServiceMonitors:         true,
+			EnableTLSHTTPServices:         true,
 			EnableTLSServiceMonitorConfig: true,
 		},
 	}
@@ -480,6 +555,7 @@ func TestBuildAll_WithFeatureFlags_EnableGateway(t *testing.T) {
 				},
 				Flags: FeatureFlags{
 					EnableGateway:                 false,
+					EnableTLSHTTPServices:         true,
 					EnableTLSServiceMonitorConfig: false,
 				},
 			},
@@ -517,6 +593,7 @@ func TestBuildAll_WithFeatureFlags_EnableGateway(t *testing.T) {
 				},
 				Flags: FeatureFlags{
 					EnableGateway:                 true,
+					EnableTLSHTTPServices:         true,
 					EnableTLSServiceMonitorConfig: true,
 				},
 			},

operator/internal/manifests/compactor.go

@@ -20,8 +20,8 @@ import (
 // BuildCompactor builds the k8s objects required to run Loki Compactor.
 func BuildCompactor(opts Options) ([]client.Object, error) {
 	statefulSet := NewCompactorStatefulSet(opts)
-	if opts.Flags.EnableTLSServiceMonitorConfig {
-		if err := configureCompactorServiceMonitorPKI(statefulSet, opts.Name); err != nil {
+	if opts.Flags.EnableTLSHTTPServices {
+		if err := configureCompactorHTTPServicePKI(statefulSet, opts.Name); err != nil {
 			return nil, err
 		}
 	}
@@ -220,9 +220,9 @@ func NewCompactorHTTPService(opts Options) *corev1.Service {
 	}
 }
 
-func configureCompactorServiceMonitorPKI(statefulSet *appsv1.StatefulSet, stackName string) error {
+func configureCompactorHTTPServicePKI(statefulSet *appsv1.StatefulSet, stackName string) error {
 	serviceName := serviceNameCompactorHTTP(stackName)
-	return configureServiceMonitorPKI(&statefulSet.Spec.Template.Spec, serviceName)
+	return configureHTTPServicePKI(&statefulSet.Spec.Template.Spec, serviceName)
 }
 
 func configureCompactorGRPCServicePKI(sts *appsv1.StatefulSet, stackName string) error {

operator/internal/manifests/distributor.go

@@ -19,8 +19,8 @@ import (
 // BuildDistributor returns a list of k8s objects for Loki Distributor
 func BuildDistributor(opts Options) ([]client.Object, error) {
 	deployment := NewDistributorDeployment(opts)
-	if opts.Flags.EnableTLSServiceMonitorConfig {
-		if err := configureDistributorServiceMonitorPKI(deployment, opts.Name); err != nil {
+	if opts.Flags.EnableTLSHTTPServices {
+		if err := configureDistributorHTTPServicePKI(deployment, opts.Name); err != nil {
 			return nil, err
 		}
 	}
@@ -196,9 +196,9 @@ func NewDistributorHTTPService(opts Options) *corev1.Service {
 	}
 }
 
-func configureDistributorServiceMonitorPKI(deployment *appsv1.Deployment, stackName string) error {
+func configureDistributorHTTPServicePKI(deployment *appsv1.Deployment, stackName string) error {
 	serviceName := serviceNameDistributorHTTP(stackName)
-	return configureServiceMonitorPKI(&deployment.Spec.Template.Spec, serviceName)
+	return configureHTTPServicePKI(&deployment.Spec.Template.Spec, serviceName)
 }
 
 func configureDistributorGRPCServicePKI(deployment *appsv1.Deployment, stackName, stackNS string) error {

operator/internal/manifests/gateway.go

@@ -40,7 +40,7 @@ func BuildGateway(opts Options) ([]client.Object, error) {
 
 	objs := []client.Object{cm, dpl, svc, ing}
 
-	if opts.Flags.EnableTLSServiceMonitorConfig {
+	if opts.Flags.EnableTLSHTTPServices {
 		serviceName := serviceNameGatewayHTTP(opts.Name)
 		if err := configureGatewayMetricsPKI(&dpl.Spec.Template.Spec, serviceName); err != nil {
 			return nil, err

operator/internal/manifests/gateway_tenants.go

@@ -72,7 +72,7 @@ func configureDeploymentForMode(d *appsv1.Deployment, mode lokiv1beta1.ModeType,
 			caBundleName,
 			caBundleDir,
 			caFile,
-			flags.EnableTLSServiceMonitorConfig,
+			flags.EnableTLSHTTPServices,
 			flags.EnableCertificateSigningService,
 			secretName,
 			serverName,

operator/internal/manifests/gateway_tenants_test.go

@@ -357,6 +357,7 @@ func TestConfigureDeploymentForMode(t *testing.T) {
 			stackName: "test",
 			stackNs:   "test-ns",
 			flags: FeatureFlags{
+				EnableTLSHTTPServices:         true,
 				EnableTLSServiceMonitorConfig: true,
 			},
 			dpl: &appsv1.Deployment{
@@ -536,6 +537,7 @@ func TestConfigureDeploymentForMode(t *testing.T) {
 			stackName: "test",
 			stackNs:   "test-ns",
 			flags: FeatureFlags{
+				EnableTLSHTTPServices:           true,
 				EnableTLSServiceMonitorConfig:   true,
 				EnableCertificateSigningService: true,
 			},
@@ -822,6 +824,7 @@ func TestConfigureServiceMonitorForMode(t *testing.T) {
 			desc: "openshift-logging mode with-tls-service-monitor-config",
 			mode: lokiv1beta1.OpenshiftLogging,
 			flags: FeatureFlags{
+				EnableTLSHTTPServices:         true,
 				EnableTLSServiceMonitorConfig: true,
 			},
 			sm: &monitoringv1.ServiceMonitor{

operator/internal/manifests/indexgateway.go

@@ -19,8 +19,8 @@ import (
 // BuildIndexGateway returns a list of k8s objects for Loki IndexGateway
 func BuildIndexGateway(opts Options) ([]client.Object, error) {
 	statefulSet := NewIndexGatewayStatefulSet(opts)
-	if opts.Flags.EnableTLSServiceMonitorConfig {
-		if err := configureIndexGatewayServiceMonitorPKI(statefulSet, opts.Name); err != nil {
+	if opts.Flags.EnableTLSHTTPServices {
+		if err := configureIndexGatewayHTTPServicePKI(statefulSet, opts.Name); err != nil {
 			return nil, err
 		}
 	}
@@ -220,9 +220,9 @@ func NewIndexGatewayHTTPService(opts Options) *corev1.Service {
 	}
 }
 
-func configureIndexGatewayServiceMonitorPKI(statefulSet *appsv1.StatefulSet, stackName string) error {
+func configureIndexGatewayHTTPServicePKI(statefulSet *appsv1.StatefulSet, stackName string) error {
 	serviceName := serviceNameIndexGatewayHTTP(stackName)
-	return configureServiceMonitorPKI(&statefulSet.Spec.Template.Spec, serviceName)
+	return configureHTTPServicePKI(&statefulSet.Spec.Template.Spec, serviceName)
 }
 
 func configureIndexGatewayGRPCServicePKI(sts *appsv1.StatefulSet, stackName string) error {

operator/internal/manifests/ingester.go

@@ -23,8 +23,8 @@ import (
 // BuildIngester builds the k8s objects required to run Loki Ingester
 func BuildIngester(opts Options) ([]client.Object, error) {
 	statefulSet := NewIngesterStatefulSet(opts)
-	if opts.Flags.EnableTLSServiceMonitorConfig {
-		if err := configureIngesterServiceMonitorPKI(statefulSet, opts.Name); err != nil {
+	if opts.Flags.EnableTLSHTTPServices {
+		if err := configureIngesterHTTPServicePKI(statefulSet, opts.Name); err != nil {
 			return nil, err
 		}
 	}
@@ -252,9 +252,9 @@ func NewIngesterHTTPService(opts Options) *corev1.Service {
 	}
 }
 
-func configureIngesterServiceMonitorPKI(statefulSet *appsv1.StatefulSet, stackName string) error {
+func configureIngesterHTTPServicePKI(statefulSet *appsv1.StatefulSet, stackName string) error {
 	serviceName := serviceNameIngesterHTTP(stackName)
-	return configureServiceMonitorPKI(&statefulSet.Spec.Template.Spec, serviceName)
+	return configureHTTPServicePKI(&statefulSet.Spec.Template.Spec, serviceName)
 }
 
 func configureIngesterGRPCServicePKI(sts *appsv1.StatefulSet, stackName, stackNS string) error {

operator/internal/manifests/openshift/configure.go

@@ -107,7 +107,7 @@ func ConfigureGatewayDeployment(
 	gwContainer.LivenessProbe.ProbeHandler.HTTPGet.Scheme = corev1.URISchemeHTTPS
 	gwContainer.Args = gwArgs
 
-	// Create and mount TLS secrets volumes if it's not already done by the service monitor config.
+	// Create and mount TLS secrets volumes if not already created.
 	if !withTLS {
 		gwVolumes = append(gwVolumes, corev1.Volume{
 			Name: secretVolumeName,

operator/internal/manifests/options.go

@@ -37,6 +37,7 @@ type Options struct {
 type FeatureFlags struct {
 	EnableCertificateSigningService bool
 	EnableServiceMonitors           bool
+	EnableTLSHTTPServices           bool
 	EnableTLSServiceMonitorConfig   bool
 	EnableTLSGRPCServices           bool
 	EnablePrometheusAlerts          bool

operator/internal/manifests/querier.go

@@ -21,8 +21,8 @@ import (
 // BuildQuerier returns a list of k8s objects for Loki Querier
 func BuildQuerier(opts Options) ([]client.Object, error) {
 	deployment := NewQuerierDeployment(opts)
-	if opts.Flags.EnableTLSServiceMonitorConfig {
-		if err := configureQuerierServiceMonitorPKI(deployment, opts.Name); err != nil {
+	if opts.Flags.EnableTLSHTTPServices {
+		if err := configureQuerierHTTPServicePKI(deployment, opts.Name); err != nil {
 			return nil, err
 		}
 	}
@@ -202,9 +202,9 @@ func NewQuerierHTTPService(opts Options) *corev1.Service {
 	}
 }
 
-func configureQuerierServiceMonitorPKI(deployment *appsv1.Deployment, stackName string) error {
+func configureQuerierHTTPServicePKI(deployment *appsv1.Deployment, stackName string) error {
 	serviceName := serviceNameQuerierHTTP(stackName)
-	return configureServiceMonitorPKI(&deployment.Spec.Template.Spec, serviceName)
+	return configureHTTPServicePKI(&deployment.Spec.Template.Spec, serviceName)
 }
 
 func configureQuerierGRPCServicePKI(deployment *appsv1.Deployment, stackName, stackNS string) error {

operator/internal/manifests/query-frontend.go

@@ -17,8 +17,8 @@ import (
 // BuildQueryFrontend returns a list of k8s objects for Loki QueryFrontend
 func BuildQueryFrontend(opts Options) ([]client.Object, error) {
 	deployment := NewQueryFrontendDeployment(opts)
-	if opts.Flags.EnableTLSServiceMonitorConfig {
-		if err := configureQueryFrontendServiceMonitorPKI(deployment, opts.Name); err != nil {
+	if opts.Flags.EnableTLSHTTPServices {
+		if err := configureQueryFrontendHTTPServicePKI(deployment, opts.Name); err != nil {
 			return nil, err
 		}
 	}
@@ -206,9 +206,9 @@ func NewQueryFrontendHTTPService(opts Options) *corev1.Service {
 	}
 }
 
-func configureQueryFrontendServiceMonitorPKI(deployment *appsv1.Deployment, stackName string) error {
+func configureQueryFrontendHTTPServicePKI(deployment *appsv1.Deployment, stackName string) error {
 	serviceName := serviceNameQueryFrontendHTTP(stackName)
-	return configureServiceMonitorPKI(&deployment.Spec.Template.Spec, serviceName)
+	return configureHTTPServicePKI(&deployment.Spec.Template.Spec, serviceName)
 }
 
 func configureQueryFrontendGRPCServicePKI(deployment *appsv1.Deployment, stackName string) error {

operator/internal/manifests/ruler.go

@@ -20,8 +20,8 @@ import (
 // BuildRuler returns a list of k8s objects for Loki Stack Ruler
 func BuildRuler(opts Options) ([]client.Object, error) {
 	statefulSet := NewRulerStatefulSet(opts)
-	if opts.Flags.EnableTLSServiceMonitorConfig {
-		if err := configureRulerServiceMonitorPKI(statefulSet, opts.Name); err != nil {
+	if opts.Flags.EnableTLSHTTPServices {
+		if err := configureRulerHTTPServicePKI(statefulSet, opts.Name); err != nil {
 			return nil, err
 		}
 	}
@@ -266,9 +266,9 @@ func NewRulerHTTPService(opts Options) *corev1.Service {
 	}
 }
 
-func configureRulerServiceMonitorPKI(statefulSet *appsv1.StatefulSet, stackName string) error {
+func configureRulerHTTPServicePKI(statefulSet *appsv1.StatefulSet, stackName string) error {
 	serviceName := serviceNameRulerHTTP(stackName)
-	return configureServiceMonitorPKI(&statefulSet.Spec.Template.Spec, serviceName)
+	return configureHTTPServicePKI(&statefulSet.Spec.Template.Spec, serviceName)
 }
 
 func configureRulerGRPCServicePKI(sts *appsv1.StatefulSet, stackName string) error {