operator: Add support for custom S3 CA (#6198)

operator/CHANGELOG.md

@@ -1,5 +1,6 @@
 ## Main
 
+- [6198](https://github.com/grafana/loki/pull/6198) **periklis**: Add support for custom S3 CA
 - [6199](https://github.com/grafana/loki/pull/6199) **Red-GV**: Update GCP secret volume path
 - [6125](https://github.com/grafana/loki/pull/6125) **sasagarw**: Add method to get authenticated from GCP
 - [5986](https://github.com/grafana/loki/pull/5986) **periklis**: Add support for Loki Rules reconciliation

operator/api/v1beta1/lokistack_types.go

@@ -350,15 +350,33 @@ type ObjectStorageSecretSpec struct {
 	Name string `json:"name"`
 }
 
+// ObjectStorageTLSSpec is the TLS configuration for reaching the object storage endpoint.
+type ObjectStorageTLSSpec struct {
+	// CA is the name of a ConfigMap containing a CA certificate.
+	// It needs to be in the same namespace as the LokiStack custom resource.
+	//
+	// +optional
+	// +kubebuilder:validation:optional
+	// +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:io.kubernetes:ConfigMap",displayName="CA ConfigMap Name"
+	CA string `json:"caName,omitempty"`
+}
+
 // ObjectStorageSpec defines the requirements to access the object
 // storage bucket to persist logs by the ingester component.
 type ObjectStorageSpec struct {
 	// Secret for object storage authentication.
-	// Name of a secret in the same namespace as the cluster logging operator.
+	// Name of a secret in the same namespace as the LokiStack custom resource.
 	//
 	// +required
 	// +kubebuilder:validation:Required
 	Secret ObjectStorageSecretSpec `json:"secret"`
+
+	// TLS configuration for reaching the object storage endpoint.
+	//
+	// +optional
+	// +kubebuilder:validation:Optional
+	// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="TLS Config"
+	TLS *ObjectStorageTLSSpec `json:"tls,omitempty"`
 }
 
 // QueryLimitSpec defines the limits applies at the query path.
@@ -612,6 +630,11 @@ const (
 	ReasonMissingObjectStorageSecret LokiStackConditionReason = "MissingObjectStorageSecret"
 	// ReasonInvalidObjectStorageSecret when the format of the secret is invalid.
 	ReasonInvalidObjectStorageSecret LokiStackConditionReason = "InvalidObjectStorageSecret"
+	// ReasonMissingObjectStorageCAConfigMap when the required configmap to verify object storage
+	// certificates is missing.
+	ReasonMissingObjectStorageCAConfigMap LokiStackConditionReason = "MissingObjectStorageCAConfigMap"
+	// ReasonInvalidObjectStorageCAConfigMap when the format of the CA configmap is invalid.
+	ReasonInvalidObjectStorageCAConfigMap LokiStackConditionReason = "InvalidObjectStorageCAConfigMap"
 	// ReasonInvalidReplicationConfiguration when the configurated replication factor is not valid
 	// with the select cluster size.
 	ReasonInvalidReplicationConfiguration LokiStackConditionReason = "InvalidReplicationConfiguration"

operator/bundle/manifests/loki-operator.clusterserviceversion.yaml

@@ -394,6 +394,15 @@ spec:
         - urn:alm:descriptor:com.tectonic.ui:select:gcs
         - urn:alm:descriptor:com.tectonic.ui:select:s3
         - urn:alm:descriptor:com.tectonic.ui:select:swift
+      - description: TLS configuration for reaching the object storage endpoint.
+        displayName: TLS Config
+        path: storage.tls
+      - description: CA is the name of a ConfigMap containing a CA certificate. It
+          needs to be in the same namespace as the LokiStack custom resource.
+        displayName: CA ConfigMap Name
+        path: storage.tls.caName
+        x-descriptors:
+        - urn:alm:descriptor:io.kubernetes:ConfigMap
       - description: Storage class name defines the storage class for ingester/querier
           PVCs.
         displayName: Storage Class Name

operator/bundle/manifests/loki.grafana.com_lokistacks.yaml

@@ -316,7 +316,7 @@ spec:
                 properties:
                   secret:
                     description: Secret for object storage authentication. Name of
-                      a secret in the same namespace as the cluster logging operator.
+                      a secret in the same namespace as the LokiStack custom resource.
                     properties:
                       name:
                         description: Name of a secret in the namespace configured
@@ -334,6 +334,16 @@ spec:
                     - name
                     - type
                     type: object
+                  tls:
+                    description: TLS configuration for reaching the object storage
+                      endpoint.
+                    properties:
+                      caName:
+                        description: CA is the name of a ConfigMap containing a CA
+                          certificate. It needs to be in the same namespace as the
+                          LokiStack custom resource.
+                        type: string
+                    type: object
                 required:
                 - secret
                 type: object

operator/config/crd/bases/loki.grafana.com_lokistacks.yaml

@@ -311,7 +311,7 @@ spec:
                 properties:
                   secret:
                     description: Secret for object storage authentication. Name of
-                      a secret in the same namespace as the cluster logging operator.
+                      a secret in the same namespace as the LokiStack custom resource.
                     properties:
                       name:
                         description: Name of a secret in the namespace configured
@@ -329,6 +329,16 @@ spec:
                     - name
                     - type
                     type: object
+                  tls:
+                    description: TLS configuration for reaching the object storage
+                      endpoint.
+                    properties:
+                      caName:
+                        description: CA is the name of a ConfigMap containing a CA
+                          certificate. It needs to be in the same namespace as the
+                          LokiStack custom resource.
+                        type: string
+                    type: object
                 required:
                 - secret
                 type: object

operator/config/manifests/bases/loki-operator.clusterserviceversion.yaml

@@ -308,6 +308,15 @@ spec:
         - urn:alm:descriptor:com.tectonic.ui:select:gcs
         - urn:alm:descriptor:com.tectonic.ui:select:s3
         - urn:alm:descriptor:com.tectonic.ui:select:swift
+      - description: TLS configuration for reaching the object storage endpoint.
+        displayName: TLS Config
+        path: storage.tls
+      - description: CA is the name of a ConfigMap containing a CA certificate. It
+          needs to be in the same namespace as the LokiStack custom resource.
+        displayName: CA ConfigMap Name
+        path: storage.tls.caName
+        x-descriptors:
+        - urn:alm:descriptor:io.kubernetes:ConfigMap
       - description: Storage class name defines the storage class for ingester/querier
           PVCs.
         displayName: Storage Class Name

operator/internal/handlers/internal/gateway/tenant_secrets.go

@@ -8,7 +8,6 @@ import (
 
 	lokiv1beta1 "github.com/grafana/loki/operator/api/v1beta1"
 	"github.com/grafana/loki/operator/internal/external/k8s"
-	"github.com/grafana/loki/operator/internal/handlers/internal/secrets"
 	"github.com/grafana/loki/operator/internal/manifests"
 	"github.com/grafana/loki/operator/internal/status"
 
@@ -48,7 +47,7 @@ func GetTenantSecrets(
 		}
 
 		var ts *manifests.TenantSecrets
-		ts, err := secrets.ExtractGatewaySecret(&gatewaySecret, tenant.TenantName)
+		ts, err := extractSecret(&gatewaySecret, tenant.TenantName)
 		if err != nil {
 			return nil, &status.DegradedError{
 				Message: "Invalid gateway tenant secret contents",
@@ -61,3 +60,27 @@ func GetTenantSecrets(
 
 	return tenantSecrets, nil
 }
+
+// extractSecret reads a k8s secret into a manifest tenant secret struct if valid.
+func extractSecret(s *corev1.Secret, tenantName string) (*manifests.TenantSecrets, error) {
+	// Extract and validate mandatory fields
+	clientID := s.Data["clientID"]
+	if len(clientID) == 0 {
+		return nil, kverrors.New("missing clientID field", "field", "clientID")
+	}
+	clientSecret := s.Data["clientSecret"]
+	if len(clientSecret) == 0 {
+		return nil, kverrors.New("missing clientSecret field", "field", "clientSecret")
+	}
+	issuerCAPath := s.Data["issuerCAPath"]
+	if len(issuerCAPath) == 0 {
+		return nil, kverrors.New("missing issuerCAPath field", "field", "issuerCAPath")
+	}
+
+	return &manifests.TenantSecrets{
+		TenantName:   tenantName,
+		ClientID:     string(clientID),
+		ClientSecret: string(clientSecret),
+		IssuerCAPath: string(issuerCAPath),
+	}, nil
+}

operator/internal/handlers/internal/gateway/tenant_secrets_test.go

@@ -142,3 +142,66 @@ func TestGetTenantSecrets_DynamicMode(t *testing.T) {
 	}
 	require.ElementsMatch(t, ts, expected)
 }
+
+func TestExtractSecret(t *testing.T) {
+	type test struct {
+		name       string
+		tenantName string
+		secret     *corev1.Secret
+		wantErr    bool
+	}
+	table := []test{
+		{
+			name:       "missing clientID",
+			tenantName: "tenant-a",
+			secret:     &corev1.Secret{},
+			wantErr:    true,
+		},
+		{
+			name:       "missing clientSecret",
+			tenantName: "tenant-a",
+			secret: &corev1.Secret{
+				Data: map[string][]byte{
+					"clientID": []byte("test"),
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name:       "missing issuerCAPath",
+			tenantName: "tenant-a",
+			secret: &corev1.Secret{
+				Data: map[string][]byte{
+					"clientID":     []byte("test"),
+					"clientSecret": []byte("test"),
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name:       "all set",
+			tenantName: "tenant-a",
+			secret: &corev1.Secret{
+				Data: map[string][]byte{
+					"clientID":     []byte("test"),
+					"clientSecret": []byte("test"),
+					"issuerCAPath": []byte("/tmp/test"),
+				},
+			},
+		},
+	}
+	for _, tst := range table {
+		tst := tst
+		t.Run(tst.name, func(t *testing.T) {
+			t.Parallel()
+
+			_, err := extractSecret(tst.secret, tst.tenantName)
+			if !tst.wantErr {
+				require.NoError(t, err)
+			}
+			if tst.wantErr {
+				require.NotNil(t, err)
+			}
+		})
+	}
+}

operator/internal/handlers/internal/storage/ca_configmap.go

@@ -0,0 +1,9 @@
+package storage
+
+import corev1 "k8s.io/api/core/v1"
+
+// IsValidCAConfigMap checks if the given CA configMap has an
+// non-empty entry for key `service-ca.crt`
+func IsValidCAConfigMap(cm *corev1.ConfigMap) bool {
+	return cm.Data["service-ca.crt"] != ""
+}

operator/internal/handlers/internal/storage/ca_configmap_test.go

@@ -0,0 +1,49 @@
+package storage_test
+
+import (
+	"testing"
+
+	"github.com/grafana/loki/operator/internal/handlers/internal/storage"
+	"github.com/stretchr/testify/require"
+	corev1 "k8s.io/api/core/v1"
+)
+
+func TestIsValidConfigMap(t *testing.T) {
+	type test struct {
+		name  string
+		cm    *corev1.ConfigMap
+		valid bool
+	}
+	table := []test{
+		{
+			name: "valid CA configmap",
+			cm: &corev1.ConfigMap{
+				Data: map[string]string{
+					"service-ca.crt": "has-some-data",
+				},
+			},
+			valid: true,
+		},
+		{
+			name: "missing `service-ca.crt` key",
+			cm:   &corev1.ConfigMap{},
+		},
+		{
+			name: "missing CA content",
+			cm: &corev1.ConfigMap{
+				Data: map[string]string{
+					"service-ca.crt": "",
+				},
+			},
+		},
+	}
+	for _, tst := range table {
+		tst := tst
+		t.Run(tst.name, func(t *testing.T) {
+			t.Parallel()
+
+			ok := storage.IsValidCAConfigMap(tst.cm)
+			require.Equal(t, tst.valid, ok)
+		})
+	}
+}