[release-4.18] OCPBUGS-49715: CA cert and catalog cache bug fixes

cmd/manager/main.go

@@ -95,12 +95,14 @@ func main() {
 		cachePath                 string
 		operatorControllerVersion bool
 		systemNamespace           string
-		caCertDir                 string
+		catalogdCasDir            string
+		pullCasDir                string
 		globalPullSecret          string
 	)
 	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
-	flag.StringVar(&caCertDir, "ca-certs-dir", "", "The directory of TLS certificate to use for verifying HTTPS connections to the Catalogd and docker-registry web servers.")
+	flag.StringVar(&catalogdCasDir, "catalogd-cas-dir", "", "The directory of TLS certificate authorities to use for verifying HTTPS connections to the Catalogd web service.")
+	flag.StringVar(&pullCasDir, "pull-cas-dir", "", "The directory of TLS certificate authorities to use for verifying HTTPS connections to image registries.")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")
@@ -221,7 +223,7 @@ func main() {
 		os.Exit(1)
 	}
 
-	certPoolWatcher, err := httputil.NewCertPoolWatcher(caCertDir, ctrl.Log.WithName("cert-pool"))
+	certPoolWatcher, err := httputil.NewCertPoolWatcher(catalogdCasDir, ctrl.Log.WithName("cert-pool"))
 	if err != nil {
 		setupLog.Error(err, "unable to create CA certificate pool")
 		os.Exit(1)
@@ -231,8 +233,8 @@ func main() {
 		BaseCachePath: filepath.Join(cachePath, "unpack"),
 		SourceContextFunc: func(logger logr.Logger) (*types.SystemContext, error) {
 			srcContext := &types.SystemContext{
-				DockerCertPath: caCertDir,
-				OCICertPath:    caCertDir,
+				DockerCertPath: pullCasDir,
+				OCICertPath:    pullCasDir,
 			}
 			if _, err := os.Stat(authFilePath); err == nil && globalPullSecretKey != nil {
 				logger.Info("using available authentication information for pulling image")

config/components/tls/patches/manager_deployment_cert.yaml

@@ -6,4 +6,7 @@
   value: {"name":"olmv1-certificate", "readOnly": true, "mountPath":"/var/certs/"}
 - op: add
   path: /spec/template/spec/containers/0/args/-
-  value: "--ca-certs-dir=/var/certs"
+  value: "--catalogd-cas-dir=/var/certs"
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: "--pull-cas-dir=/var/certs"

internal/httputil/certpoolwatcher.go

@@ -4,6 +4,8 @@ import (
 	"crypto/x509"
 	"fmt"
 	"os"
+	"slices"
+	"strings"
 	"sync"
 	"time"
 
@@ -44,8 +46,26 @@ func NewCertPoolWatcher(caDir string, log logr.Logger) (*CertPoolWatcher, error)
 	if err != nil {
 		return nil, err
 	}
-	if err = watcher.Add(caDir); err != nil {
-		return nil, err
+
+	// If the SSL_CERT_DIR or SSL_CERT_FILE environment variables are
+	// specified, this means that we have some control over the system root
+	// location, thus they may change, thus we should watch those locations.
+	watchPaths := strings.Split(os.Getenv("SSL_CERT_DIR"), ":")
+	watchPaths = append(watchPaths, caDir, os.Getenv("SSL_CERT_FILE"))
+	watchPaths = slices.DeleteFunc(watchPaths, func(p string) bool {
+		if p == "" {
+			return true
+		}
+		if _, err := os.Stat(p); err != nil {
+			return true
+		}
+		return false
+	})
+
+	for _, p := range watchPaths {
+		if err := watcher.Add(p); err != nil {
+			return nil, err
+		}
 	}
 
 	cpw := &CertPoolWatcher{

internal/httputil/certpoolwatcher_test.go

@@ -72,6 +72,10 @@ func TestCertPoolWatcher(t *testing.T) {
 	t.Logf("Create cert file at %q\n", certName)
 	createCert(t, certName)
 
+	// Update environment variables for the watcher - some of these should not exist
+	os.Setenv("SSL_CERT_DIR", tmpDir+":/tmp/does-not-exist.dir")
+	os.Setenv("SSL_CERT_FILE", "/tmp/does-not-exist.file")
+
 	// Create the cert pool watcher
 	cpw, err := httputil.NewCertPoolWatcher(tmpDir, log.FromContext(context.Background()))
 	require.NoError(t, err)

openshift/kustomize/overlays/openshift/olmv1-ns/patches/manager_deployment_certs.yaml

@@ -1,15 +1,9 @@
 - op: add
   path: /spec/template/spec/volumes/-
-  value: {"name":"trusted-ca-bundle", "configMap":{"optional":false,"name":"trusted-ca-bundle", "items":[{"key":"ca-bundle.crt","path":"ca-bundle.crt"}]}}
-- op: add
-  path: /spec/template/spec/volumes/-
-  value: {"name":"service-ca", "configMap":{"optional":false,"name":"openshift-service-ca.crt", "items":[{"key":"service-ca.crt","path":"service-ca.crt"}]}}
-- op: add
-  path: /spec/template/spec/containers/0/volumeMounts/-
-  value: {"name":"trusted-ca-bundle", "mountPath":"/var/trusted-cas/ca-bundle.crt", "subPath":"ca-bundle.crt" }
+  value: {"name":"ca-certs", "projected": {"sources":[{"configMap":{"optional":false,"name":"trusted-ca-bundle", "items":[{"key":"ca-bundle.crt","path":"ca-bundle.crt"}]}},{"configMap":{"optional":false,"name":"openshift-service-ca.crt", "items":[{"key":"service-ca.crt","path":"service-ca.crt"}]}}]}}
 - op: add
   path: /spec/template/spec/containers/0/volumeMounts/-
-  value: {"name":"service-ca", "mountPath":"/var/trusted-cas/service-ca.crt", "subPath":"service-ca.crt" }
+  value: {"name":"ca-certs", "mountPath":"/var/ca-certs", "readOnly": true}
 - op: add
-  path: /spec/template/spec/containers/0/args/-
-  value: "--ca-certs-dir=/var/trusted-cas"
+  path: /spec/template/spec/containers/0/env
+  value: [{"name":"SSL_CERT_DIR", "value":"/var/ca-certs"}]

openshift/manifests/20-deployment-openshift-operator-controller-operator-controller-controller-manager.yml

@@ -43,11 +43,13 @@ spec:
             - --health-probe-bind-address=:8081
             - --metrics-bind-address=127.0.0.1:8080
             - --leader-elect
-            - --ca-certs-dir=/var/trusted-cas
             - --v=${LOG_VERBOSITY}
             - --global-pull-secret=openshift-config/pull-secret
           command:
             - /manager
+          env:
+            - name: SSL_CERT_DIR
+              value: /var/ca-certs
           image: ${OPERATOR_CONTROLLER_IMAGE}
           imagePullPolicy: IfNotPresent
           livenessProbe:
@@ -76,12 +78,9 @@ spec:
           volumeMounts:
             - mountPath: /var/cache
               name: cache
-            - mountPath: /var/trusted-cas/ca-bundle.crt
-              name: trusted-ca-bundle
-              subPath: ca-bundle.crt
-            - mountPath: /var/trusted-cas/service-ca.crt
-              name: service-ca
-              subPath: service-ca.crt
+            - mountPath: /var/ca-certs
+              name: ca-certs
+              readOnly: true
             - mountPath: /etc/containers
               name: etc-containers
               readOnly: true
@@ -131,20 +130,21 @@ spec:
       volumes:
         - emptyDir: {}
           name: cache
-        - configMap:
-            items:
-              - key: ca-bundle.crt
-                path: ca-bundle.crt
-            name: operator-controller-trusted-ca-bundle
-            optional: false
-          name: trusted-ca-bundle
-        - configMap:
-            items:
-              - key: service-ca.crt
-                path: service-ca.crt
-            name: openshift-service-ca.crt
-            optional: false
-          name: service-ca
+        - name: ca-certs
+          projected:
+            sources:
+              - configMap:
+                  items:
+                    - key: ca-bundle.crt
+                      path: ca-bundle.crt
+                  name: operator-controller-trusted-ca-bundle
+                  optional: false
+              - configMap:
+                  items:
+                    - key: service-ca.crt
+                      path: service-ca.crt
+                  name: openshift-service-ca.crt
+                  optional: false
         - hostPath:
             path: /etc/containers
             type: Directory