allow delete only if annotations meet configured criteria (#1069)

* define annotations for delete protection

* change log level and reduce log lines for e2e tests

* reduce wait_for_pod_start even further

Author: FxKu

charts/postgres-operator/crds/operatorconfigurations.yaml

@@ -117,6 +117,10 @@ spec:
                   type: object
                   additionalProperties:
                     type: string
+                delete_annotation_date_key:
+                  type: string
+                delete_annotation_name_key:
+                  type: string
                 downscaler_annotations:
                   type: array
                   items:

charts/postgres-operator/values-crd.yaml

@@ -67,6 +67,12 @@ configKubernetes:
   #   keya: valuea
   #   keyb: valueb
 
+  # key name for annotation that compares manifest value with current date
+  # delete_annotation_date_key: "delete-date"
+
+  # key name for annotation that compares manifest value with cluster name
+  # delete_annotation_name_key: "delete-clustername"
+
   # list of annotations propagated from cluster manifest to statefulset and deployment
   # downscaler_annotations:
   #   - deployment-time

charts/postgres-operator/values.yaml

@@ -63,6 +63,12 @@ configKubernetes:
   # annotations attached to each database pod
   # custom_pod_annotations: "keya:valuea,keyb:valueb"
 
+  # key name for annotation that compares manifest value with current date
+  # delete_annotation_date_key: "delete-date"
+
+  # key name for annotation that compares manifest value with cluster name
+  # delete_annotation_name_key: "delete-clustername"
+
   # list of annotations propagated from cluster manifest to statefulset and deployment
   # downscaler_annotations: "deployment-time,downscaler/*"
 

docs/administrator.md

@@ -44,7 +44,7 @@ Once the validation is enabled it can only be disabled manually by editing or
 patching the CRD manifest:
 
 ```bash
-zk8 patch crd postgresqls.acid.zalan.do -p '{"spec":{"validation": null}}'
+kubectl patch crd postgresqls.acid.zalan.do -p '{"spec":{"validation": null}}'
 ```
 
 ## Non-default cluster domain
@@ -123,6 +123,68 @@ Every other Postgres cluster which lacks the annotation will be ignored by this
 operator. Conversely, operators without a defined `CONTROLLER_ID` will ignore
 clusters with defined ownership of another operator.
 
+## Delete protection via annotations
+
+To avoid accidental deletes of Postgres clusters the operator can check the
+manifest for two existing annotations containing the cluster name and/or the
+current date (in YYYY-MM-DD format). The name of the annotation keys can be
+defined in the configuration. By default, they are not set which disables the
+delete protection. Thus, one could choose to only go with one annotation.
+
+**postgres-operator ConfigMap**
+
+```yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: postgres-operator
+data:
+  delete_annotation_date_key: "delete-date"
+  delete_annotation_name_key: "delete-clustername"
+```
+
+**OperatorConfiguration**
+
+```yaml
+apiVersion: "acid.zalan.do/v1"
+kind: OperatorConfiguration
+metadata:
+  name: postgresql-operator-configuration
+configuration:
+  kubernetes:
+    delete_annotation_date_key: "delete-date"
+    delete_annotation_name_key: "delete-clustername"
+```
+
+Now, every cluster manifest must contain the configured annotation keys to
+trigger the delete process when running `kubectl delete pg`. Note, that the
+`Postgresql` resource would still get deleted as K8s' API server does not
+block it. Only the operator logs will tell, that the delete criteria wasn't
+met.
+
+**cluster manifest**
+
+```yaml
+apiVersion: "acid.zalan.do/v1"
+kind: postgresql
+metadata:
+  name: demo-cluster
+  annotations:
+    delete-date: "2020-08-31"
+    delete-clustername: "demo-cluster"
+spec:
+  ...
+```
+
+In case, the resource has been deleted accidentally or the annotations were
+simply forgotten, it's safe to recreate the cluster with `kubectl create`.
+Existing Postgres cluster are not replaced by the operator. But, as the
+original cluster still exists the status will show `CreateFailed` at first.
+On the next sync event it should change to `Running`. However, as it is in
+fact a new resource for K8s, the UID will differ which can trigger a rolling
+update of the pods because the UID is used as part of backup path to S3.
+
+
 ## Role-based access control for the operator
 
 The manifest [`operator-service-account-rbac.yaml`](../manifests/operator-service-account-rbac.yaml)
@@ -586,11 +648,11 @@ The configuration paramaters that we will be using are:
 * `gcp_credentials`
 * `wal_gs_bucket`
 
-### Generate a K8 secret resource
+### Generate a K8s secret resource
 
-Generate the K8 secret resource that will contain your service account's 
+Generate the K8s secret resource that will contain your service account's
 credentials. It's highly recommended to use a service account and limit its
-scope to just the WAL-E bucket. 
+scope to just the WAL-E bucket.
 
 ```yaml
 apiVersion: v1
@@ -613,13 +675,13 @@ the operator's configuration is set up like the following:
 ...
 aws_or_gcp:
   additional_secret_mount: "pgsql-wale-creds"
-  additional_secret_mount_path: "/var/secrets/google" # or where ever you want to mount the file
+  additional_secret_mount_path: "/var/secrets/google"  # or where ever you want to mount the file
   # aws_region: eu-central-1
   # kube_iam_role: ""
   # log_s3_bucket: ""
   # wal_s3_bucket: ""
-  wal_gs_bucket: "postgres-backups-bucket-28302F2" # name of bucket on where to save the WAL-E logs
-  gcp_credentials: "/var/secrets/google/key.json" # combination of the mount path & key in the K8 resource. (i.e. key.json)
+  wal_gs_bucket: "postgres-backups-bucket-28302F2"  # name of bucket on where to save the WAL-E logs
+  gcp_credentials: "/var/secrets/google/key.json"  # combination of the mount path & key in the K8s resource. (i.e. key.json)
 ...
 ```
 

docs/reference/operator_parameters.md

@@ -200,6 +200,16 @@ configuration they are grouped under the `kubernetes` key.
   of a database created by the operator. If the annotation key is also provided
   by the database definition, the database definition value is used.
 
+* **delete_annotation_date_key**
+  key name for annotation that compares manifest value with current date in the
+  YYYY-MM-DD format. Allowed pattern: `'([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]'`.
+  The default is empty which also disables this delete protection check.
+
+* **delete_annotation_name_key**
+  key name for annotation that compares manifest value with Postgres cluster name.
+  Allowed pattern: `'([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]'`. The default is
+  empty which also disables this delete protection check.
+
 * **downscaler_annotations**
   An array of annotations that should be passed from Postgres CRD on to the
   statefulset and, if exists, to the connection pooler deployment as well.

e2e/requirements.txt

@@ -1,3 +1,3 @@
-kubernetes==9.0.0
+kubernetes==11.0.0
 timeout_decorator==0.4.1
-pyyaml==5.1
+pyyaml==5.3.1

e2e/tests/test_e2e.py

@@ -7,6 +7,7 @@
 import os
 import yaml
 
+from datetime import datetime
 from kubernetes import client, config
 
 
@@ -614,6 +615,71 @@ def test_infrastructure_roles(self):
             "Origin": 2,
         })
 
+    @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
+    def test_x_cluster_deletion(self):
+        '''
+           Test deletion with configured protection
+        '''
+        k8s = self.k8s
+        cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster'
+
+        # configure delete protection
+        patch_delete_annotations = {
+            "data": {
+                "delete_annotation_date_key": "delete-date",
+                "delete_annotation_name_key": "delete-clustername"
+            }
+        }
+        k8s.update_config(patch_delete_annotations)
+
+        # this delete attempt should be omitted because of missing annotations
+        k8s.api.custom_objects_api.delete_namespaced_custom_object(
+            "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster")
+
+        # check that pods and services are still there
+        k8s.wait_for_running_pods(cluster_label, 2)
+        k8s.wait_for_service(cluster_label)
+
+        # recreate Postgres cluster resource
+        k8s.create_with_kubectl("manifests/minimal-postgres-manifest.yaml")
+
+        # wait a little before proceeding
+        time.sleep(10)
+
+        # add annotations to manifest
+        deleteDate = datetime.today().strftime('%Y-%m-%d')
+        pg_patch_delete_annotations = {
+            "metadata": {
+                "annotations": {
+                    "delete-date": deleteDate,
+                    "delete-clustername": "acid-minimal-cluster",
+                }
+            }
+        }
+        k8s.api.custom_objects_api.patch_namespaced_custom_object(
+            "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster", pg_patch_delete_annotations)
+
+        # wait a little before proceeding
+        time.sleep(10)
+        k8s.wait_for_running_pods(cluster_label, 2)
+        k8s.wait_for_service(cluster_label)
+
+        # now delete process should be triggered
+        k8s.api.custom_objects_api.delete_namespaced_custom_object(
+            "acid.zalan.do", "v1", "default", "postgresqls", "acid-minimal-cluster")
+
+        # wait until cluster is deleted
+        time.sleep(120)
+
+        # check if everything has been deleted
+        self.assertEqual(0, k8s.count_pods_with_label(cluster_label))
+        self.assertEqual(0, k8s.count_services_with_label(cluster_label))
+        self.assertEqual(0, k8s.count_endpoints_with_label(cluster_label))
+        self.assertEqual(0, k8s.count_statefulsets_with_label(cluster_label))
+        self.assertEqual(0, k8s.count_deployments_with_label(cluster_label))
+        self.assertEqual(0, k8s.count_pdbs_with_label(cluster_label))
+        self.assertEqual(0, k8s.count_secrets_with_label(cluster_label))
+
     def get_failover_targets(self, master_node, replica_nodes):
         '''
            If all pods live on the same node, failover will happen to other worker(s)
@@ -700,11 +766,12 @@ def __init__(self):
         self.apps_v1 = client.AppsV1Api()
         self.batch_v1_beta1 = client.BatchV1beta1Api()
         self.custom_objects_api = client.CustomObjectsApi()
+        self.policy_v1_beta1 = client.PolicyV1beta1Api()
 
 
 class K8s:
     '''
-    Wraps around K8 api client and helper methods.
+    Wraps around K8s api client and helper methods.
     '''
 
     RETRY_TIMEOUT_SEC = 10
@@ -755,14 +822,6 @@ def wait_for_pod_start(self, pod_labels, namespace='default'):
             if pods:
                 pod_phase = pods[0].status.phase
 
-            if pods and pod_phase != 'Running':
-                pod_name = pods[0].metadata.name
-                response = self.api.core_v1.read_namespaced_pod(
-                    name=pod_name,
-                    namespace=namespace
-                )
-                print("Pod description {}".format(response))
-
             time.sleep(self.RETRY_TIMEOUT_SEC)
 
     def get_service_type(self, svc_labels, namespace='default'):
@@ -824,6 +883,25 @@ def get_services():
     def count_pods_with_label(self, labels, namespace='default'):
         return len(self.api.core_v1.list_namespaced_pod(namespace, label_selector=labels).items)
 
+    def count_services_with_label(self, labels, namespace='default'):
+        return len(self.api.core_v1.list_namespaced_service(namespace, label_selector=labels).items)
+
+    def count_endpoints_with_label(self, labels, namespace='default'):
+        return len(self.api.core_v1.list_namespaced_endpoints(namespace, label_selector=labels).items)
+
+    def count_secrets_with_label(self, labels, namespace='default'):
+        return len(self.api.core_v1.list_namespaced_secret(namespace, label_selector=labels).items)
+
+    def count_statefulsets_with_label(self, labels, namespace='default'):
+        return len(self.api.apps_v1.list_namespaced_stateful_set(namespace, label_selector=labels).items)
+
+    def count_deployments_with_label(self, labels, namespace='default'):
+        return len(self.api.apps_v1.list_namespaced_deployment(namespace, label_selector=labels).items)
+
+    def count_pdbs_with_label(self, labels, namespace='default'):
+        return len(self.api.policy_v1_beta1.list_namespaced_pod_disruption_budget(
+            namespace, label_selector=labels).items)
+
     def wait_for_pod_failover(self, failover_targets, labels, namespace='default'):
         pod_phase = 'Failing over'
         new_pod_node = ''

manifests/complete-postgres-manifest.yaml

@@ -6,6 +6,8 @@ metadata:
 #    environment: demo
 #  annotations:
 #    "acid.zalan.do/controller": "second-operator"
+#    "delete-date": "2020-08-31"  # can only be deleted on that day if "delete-date "key is configured
+#    "delete-clustername": "acid-test-cluster"  # can only be deleted when name matches if "delete-clustername" key is configured
 spec:
   dockerImage: registry.opensource.zalan.do/acid/spilo-12:1.6-p3
   teamId: "acid"
@@ -34,7 +36,7 @@ spec:
           defaultUsers: false
   postgresql:
     version: "12"
-    parameters: # Expert section
+    parameters:  # Expert section
       shared_buffers: "32MB"
       max_connections: "10"
       log_statement: "all"

manifests/configmap.yaml

@@ -29,6 +29,8 @@ data:
   # default_cpu_request: 100m
   # default_memory_limit: 500Mi
   # default_memory_request: 100Mi
+  # delete_annotation_date_key: delete-date
+  # delete_annotation_name_key: delete-clustername
   docker_image: registry.opensource.zalan.do/acid/spilo-12:1.6-p3
   # downscaler_annotations: "deployment-time,downscaler/*"
   # enable_admin_role_for_users: "true"

manifests/operatorconfiguration.crd.yaml

@@ -113,6 +113,10 @@ spec:
                   type: object
                   additionalProperties:
                     type: string
+                delete_annotation_date_key:
+                  type: string
+                delete_annotation_name_key:
+                  type: string
                 downscaler_annotations:
                   type: array
                   items: