feat(crdvalidator): adding webhook to ensure safety of crd create/upg…

…rades

Signed-off-by: Tyler Slaton <tyslaton@redhat.com>

Dockerfile.crdvalidator

@@ -0,0 +1,23 @@
+FROM golang:1.17-buster AS builder
+
+WORKDIR /workspace
+# Copy the Go Modules manifests
+COPY go.mod go.mod
+COPY go.sum go.sum
+# cache deps before building and copying source so that we don't need to re-download as much
+# and so that source changes don't invalidate our downloaded layer
+RUN go mod download
+
+COPY Makefile Makefile
+COPY cmd/crdvalidator cmd/crdvalidator
+
+RUN make bin/crdvalidator
+
+FROM gcr.io/distroless/static:debug
+
+WORKDIR /
+COPY --from=builder /workspace/bin/crdvalidator .
+EXPOSE 443
+
+ENTRYPOINT ["/crdvalidator"]
+CMD ["run"]

Makefile

@@ -148,7 +148,7 @@ kind-load-bundles:
 	${KIND} load docker-image testdata/bundles/plain-v0:no-manifests --name $(KIND_CLUSTER_NAME)
 	${KIND} load docker-image testdata/bundles/plain-v0:invalid-crds-and-crs --name $(KIND_CLUSTER_NAME)
 
-kind-load: ## Load-image loads the currently constructed image onto the cluster
+kind-load: ## Loads the currently constructed image onto the cluster
 	${KIND} load docker-image $(IMAGE) --name $(KIND_CLUSTER_NAME)
 
 kind-cluster: ## Standup a kind cluster for e2e testing usage
@@ -158,6 +158,26 @@ kind-cluster: ## Standup a kind cluster for e2e testing usage
 e2e: KIND_CLUSTER_NAME=rukpak-e2e
 e2e: build-container kind-cluster kind-load cert-mgr kind-load-bundles deploy test-e2e ## Run e2e tests against a kind cluster
 
+##################################
+# CRDValidator Webhook Targets #
+##################################
+
+##@ crdvalidator:
+
+bin/crdvalidator: 
+	CGO_ENABLED=0 go build $(VERSION_FLAGS) -o $@$(BIN_SUFFIX) ./cmd/crdvalidator
+
+CRD_WEBHOOK_IMAGE_REPO=quay.io/operator-framework/crd-validation-webhook
+CRD_WEBHOOK_IMAGE=$(CRD_WEBHOOK_IMAGE_REPO):main
+build-crdvalidator-image: ## Builds crd-validation-webhook container image locally
+	docker build -f Dockerfile.crdvalidator -t $(CRD_WEBHOOK_IMAGE) .
+
+kind-load-crdvalidator: ## Loads the currently constructed crd-validation-webhook onto the cluster
+	${KIND} load docker-image $(CRD_WEBHOOK_IMAGE) --name $(KIND_CLUSTER_NAME)
+
+deploy-crdvalidator: build-crdvalidator-image kind-load-crdvalidator cert-mgr ## Deploy the crdvalidator manifests to the current cluster. Make sure cert-manager is installed prior.
+	kubectl apply -f cmd/crdvalidator/manifests
+
 ################
 # Hack / Tools #
 ################

cmd/crdvalidator/internal/crd/crd.go

@@ -0,0 +1,172 @@
+package crd
+
+import (
+	"context"
+	"fmt"
+	"reflect"
+
+	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"k8s.io/apiextensions-apiserver/pkg/apiserver/validation"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/validation/field"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func Validate(ctx context.Context, cl client.Client, newCrd *apiextensionsv1.CustomResourceDefinition) error {
+	oldCRD := &apiextensionsv1.CustomResourceDefinition{}
+	if err := cl.Get(ctx, client.ObjectKeyFromObject(newCrd), oldCRD); err != nil && !apierrors.IsNotFound(err) {
+		return err
+	}
+
+	if err := validateCRDCompatibility(ctx, cl, oldCRD, newCrd); err != nil {
+		return fmt.Errorf("error validating existing CRs against new CRD's schema for %q: %w", newCrd.Name, err)
+	}
+
+	// check to see if stored versions changed and whether the upgrade could cause potential data loss
+	safe, err := safeStorageVersionUpgrade(oldCRD, newCrd)
+	if !safe {
+		return fmt.Errorf("risk of data loss updating %q: %w", newCrd.Name, err)
+	}
+	if err != nil {
+		return fmt.Errorf("checking CRD for potential data loss updating %q: %w", newCrd.Name, err)
+	}
+
+	return nil
+}
+
+func keys(m map[string]apiextensionsv1.CustomResourceDefinitionVersion) sets.String {
+	return sets.StringKeySet(m)
+}
+
+func validateCRDCompatibility(ctx context.Context, cl client.Client, oldCRD *apiextensionsv1.CustomResourceDefinition, newCRD *apiextensionsv1.CustomResourceDefinition) error {
+	// Cases to test:
+	//   New CRD removes version that Old CRD had => Must ensure nothing is stored at removed version
+	//   New CRD changes a version that Old CRD has => Must validate existing CRs with new schema
+	//   New CRD adds a version that Old CRD does not have =>
+	//      - If conversion strategy is None, ensure existing CRs validate with new schema.
+	//      - If conversion strategy is Webhook, allow update (assume webhook handles conversion correctly)
+
+	oldVersions := map[string]apiextensionsv1.CustomResourceDefinitionVersion{}
+	newVersions := map[string]apiextensionsv1.CustomResourceDefinitionVersion{}
+
+	for _, v := range oldCRD.Spec.Versions {
+		oldVersions[v.Name] = v
+	}
+	for _, v := range newCRD.Spec.Versions {
+		newVersions[v.Name] = v
+	}
+
+	existingStoredVersions := sets.NewString(oldCRD.Status.StoredVersions...)
+	removedVersions := keys(oldVersions).Difference(keys(newVersions))
+	invalidRemovedVersions := existingStoredVersions.Intersection(removedVersions)
+	if invalidRemovedVersions.Len() > 0 {
+		return fmt.Errorf("cannot remove stored versions %v", invalidRemovedVersions.List())
+	}
+
+	similarVersions := keys(oldVersions).Intersection(keys(newVersions))
+	diffVersions := sets.NewString()
+	for _, v := range similarVersions.List() {
+		if !reflect.DeepEqual(oldVersions[v].Schema, newVersions[v].Schema) {
+			diffVersions.Insert(v)
+		}
+	}
+	convertedCRD := &apiextensions.CustomResourceDefinition{}
+	if err := apiextensionsv1.Convert_v1_CustomResourceDefinition_To_apiextensions_CustomResourceDefinition(newCRD, convertedCRD, nil); err != nil {
+		return err
+	}
+	for _, v := range diffVersions.List() {
+		oldV := oldVersions[v]
+		if oldV.Served {
+			listGVK := schema.GroupVersionKind{Group: oldCRD.Spec.Group, Version: v, Kind: oldCRD.Spec.Names.ListKind}
+			err := validateExistingCRs(ctx, cl, listGVK, newVersions[v])
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	// If the new CRD has no conversion configured or a "None" conversion strategy, we need to check to be sure that the
+	// new schema validates all of the existing CRs of the existing versions.
+	addedVersions := keys(newVersions).Difference(keys(oldVersions))
+	if addedVersions.Len() > 0 && (newCRD.Spec.Conversion == nil || newCRD.Spec.Conversion.Strategy == apiextensionsv1.NoneConverter) {
+		for _, va := range addedVersions.List() {
+			newV := newVersions[va]
+			for _, vs := range similarVersions.List() {
+				oldV := oldVersions[vs]
+				if oldV.Served {
+					listGVK := schema.GroupVersionKind{Group: oldCRD.Spec.Group, Version: oldV.Name, Kind: oldCRD.Spec.Names.ListKind}
+					err := validateExistingCRs(ctx, cl, listGVK, newV)
+					if err != nil {
+						return err
+					}
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+func validateExistingCRs(ctx context.Context, dynamicClient client.Client, listGVK schema.GroupVersionKind, newVersion apiextensionsv1.CustomResourceDefinitionVersion) error {
+	convertedVersion := &apiextensions.CustomResourceDefinitionVersion{}
+	if err := apiextensionsv1.Convert_v1_CustomResourceDefinitionVersion_To_apiextensions_CustomResourceDefinitionVersion(&newVersion, convertedVersion, nil); err != nil {
+		return err
+	}
+
+	crList := &unstructured.UnstructuredList{}
+	crList.SetGroupVersionKind(listGVK)
+	if err := dynamicClient.List(ctx, crList); err != nil {
+		return fmt.Errorf("error listing objects for %s: %w", listGVK, err)
+	}
+	for _, cr := range crList.Items {
+		validator, _, err := validation.NewSchemaValidator(convertedVersion.Schema)
+		if err != nil {
+			return fmt.Errorf("error creating validator for the schema of version %q: %w", newVersion.Name, err)
+		}
+		err = validation.ValidateCustomResource(field.NewPath(""), cr.UnstructuredContent(), validator).ToAggregate()
+		if err != nil {
+			return fmt.Errorf("existing custom object %s/%s failed validation for new schema version %s: %w", cr.GetNamespace(), cr.GetName(), newVersion.Name, err)
+		}
+	}
+	return nil
+}
+
+// safeStorageVersionUpgrade determines whether the new CRD spec includes all the storage versions of the existing on-cluster CRD.
+// For each stored version in the status of the CRD on the cluster (there will be at least one) - each version must exist in the spec of the new CRD that is being installed.
+// See https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definition-versioning/#upgrade-existing-objects-to-a-new-stored-version.
+func safeStorageVersionUpgrade(existingCRD, newCRD *apiextensionsv1.CustomResourceDefinition) (bool, error) {
+	existingStatusVersions, newSpecVersions := getStoredVersions(existingCRD, newCRD)
+	if newSpecVersions == nil {
+		return false, fmt.Errorf("could not find any versions in the new CRD")
+	}
+	if existingStatusVersions == nil {
+		// every on-cluster CRD should have at least one stored version in its status
+		// in the case where there are no existing stored versions, checking against new versions is not relevant
+		return true, nil
+	}
+
+	for name := range existingStatusVersions {
+		if _, ok := newSpecVersions[name]; !ok {
+			// a storage version in the status of the old CRD is not present in the spec of the new CRD
+			// potential data loss of CRs without a storage migration - throw error and block the CRD upgrade
+			return false, fmt.Errorf("new CRD removes version %s that is listed as a stored version on the existing CRD", name)
+		}
+	}
+
+	return true, nil
+}
+
+// getStoredVersions returns the storage versions listed in the status of the old on-cluster CRD
+// and all the versions listed in the spec of the new CRD.
+func getStoredVersions(oldCRD, newCRD *apiextensionsv1.CustomResourceDefinition) (sets.String, sets.String) {
+	newSpecVersions := sets.NewString()
+	for _, version := range newCRD.Spec.Versions {
+		newSpecVersions.Insert(version.Name)
+	}
+
+	return sets.NewString(oldCRD.Status.StoredVersions...), newSpecVersions
+}

cmd/crdvalidator/internal/handlers/crd.go

@@ -0,0 +1,61 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package handlers
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+
+	"github.com/operator-framework/rukpak/cmd/crdvalidator/internal/crd"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
+)
+
+// +kubebuilder:webhook:path=/validate-crd,mutating=false,failurePolicy=fail,groups="",resources=customresourcedefinitions,verbs=create;update,versions=v1,name=crd-validation-webhook.acme.com
+
+// CrdValidator houses a client, decoder and Handle function for ensuring
+// that a CRD create/update request is safe
+type CrdValidator struct {
+	Client  client.Client
+	decoder *admission.Decoder
+}
+
+// Handle takes an incoming CRD create/update request and confirms that it is
+// a safe upgrade based on the crd.Validate() function call
+func (v *CrdValidator) Handle(ctx context.Context, req admission.Request) admission.Response {
+	incomingCrd := &apiextensionsv1.CustomResourceDefinition{}
+
+	err := v.decoder.Decode(req, incomingCrd)
+	if err != nil {
+		return admission.Errored(http.StatusBadRequest, err)
+	}
+
+	err = crd.Validate(ctx, v.Client, incomingCrd)
+	if err != nil {
+		return admission.Denied(fmt.Sprintf("failed to validate safety of CRD %s: %v", req.Operation, err))
+	}
+
+	return admission.Allowed("")
+}
+
+// CrdValidator implements admission.DecoderInjector.
+// A decoder will be automatically injected.
+
+// InjectDecoder injects the decoder.
+func (v *CrdValidator) InjectDecoder(d *admission.Decoder) error {
+	v.decoder = d
+	return nil
+}

cmd/crdvalidator/main.go

@@ -0,0 +1,66 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"os"
+
+	"github.com/operator-framework/rukpak/cmd/crdvalidator/internal/handlers"
+	"sigs.k8s.io/controller-runtime/pkg/client/config"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+	"sigs.k8s.io/controller-runtime/pkg/manager/signals"
+	"sigs.k8s.io/controller-runtime/pkg/webhook"
+
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+)
+
+const defaultCertDir = "/etc/admission-webhook/tls"
+
+func init() {
+	log.SetLogger(zap.New())
+}
+
+func main() {
+	entryLog := log.Log.WithName("crdvalidator")
+
+	// Setup a Manager
+	entryLog.Info("setting up manager")
+	mgr, err := manager.New(config.GetConfigOrDie(), manager.Options{})
+	if err != nil {
+		entryLog.Error(err, "unable to set up overall controller manager")
+		os.Exit(1)
+	}
+
+	entryLog.Info("setting up webhook server")
+	hookServer := mgr.GetWebhookServer()
+
+	// Point to where cert-mgr is placing the cert
+	hookServer.CertDir = defaultCertDir
+
+	// Need manager to have scheme of CRDs
+	utilruntime.Must(apiextensionsv1.AddToScheme(mgr.GetScheme()))
+
+	// Register CRD validation handler
+	entryLog.Info("registering webhooks to the webhook server")
+	hookServer.Register("/validate-crd", &webhook.Admission{Handler: &handlers.CrdValidator{Client: mgr.GetClient()}})
+
+	entryLog.Info("starting manager")
+	if err := mgr.Start(signals.SetupSignalHandler()); err != nil {
+		entryLog.Error(err, "unable to run manager")
+		os.Exit(1)
+	}
+}

cmd/crdvalidator/manifests/00_service_account.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  namespace: default
+  name: crd-validation-webhook

cmd/crdvalidator/manifests/01_cluster_role.yaml

@@ -0,0 +1,8 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: crd-validation-webhook
+rules:
+- apiGroups: ["apiextensions.k8s.io"]
+  resources: ["customresourcedefinitions"]
+  verbs: ["get", "watch", "list"]

cmd/crdvalidator/manifests/02_cluster_role_binding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: crd-validation-webhook
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: crd-validation-webhook
+subjects:
+- kind: ServiceAccount
+  name: crd-validation-webhook
+  namespace: default