[operator] Manage Virtual Garden ETCD (gardener#7067)

* Move ETCD schedule computation into `timewindow` package This allows to reuse this logic later in the `Garden` controller. * Introduce `etcd.Values` for easier configurability * Extend `Garden` API with virtual cluster specifications * Garden controller reconciles `Etcd` resources * Change role names in ETCD package unit tests * Address PR review feedback * Address PR review feedback * Address PR review feedback * Address PR review feedback * Address PR review feedback
nickytd · Dec 5, 2022 · 219d828 · 219d828
1 parent a60a42f
commit 219d828
Show file tree

Hide file tree

Showing 43 changed files with 2,084 additions and 615 deletions.
diff --git a/Makefile b/Makefile
@@ -305,8 +305,11 @@ kind-ha-multi-zone-down: $(KIND)
 
 kind-operator-up: $(KIND) $(KUBECTL) $(HELM)
 	./hack/kind-up.sh --cluster-name gardener-operator-local --environment $(KIND_ENV) --path-kubeconfig $(REPO_ROOT)/example/gardener-local/kind/operator/kubeconfig --path-cluster-values $(REPO_ROOT)/example/gardener-local/kind/operator/values.yaml
+	mkdir -p $(REPO_ROOT)/dev/local-backupbuckets/gardener-operator
 kind-operator-down: $(KIND)
 	./hack/kind-down.sh --cluster-name gardener-operator-local --path-kubeconfig $(REPO_ROOT)/example/gardener-local/kind/operator/kubeconfig
+	# We need root privileges to clean the backup bucket directory, see https://github.com/gardener/gardener/issues/6752
+	docker run --user root:root -v $(REPO_ROOT)/dev/local-backupbuckets:/dev/local-backupbuckets alpine rm -rf /dev/local-backupbuckets/gardener-operator
 
 # speed-up skaffold deployments by building all images concurrently
 export SKAFFOLD_BUILD_CONCURRENCY = 0

diff --git a/charts/gardener/operator/templates/customresouredefintion.yaml b/charts/gardener/operator/templates/customresouredefintion.yaml
@@ -84,8 +84,129 @@ spec:
                 required:
                 - provider
                 type: object
+              virtualCluster:
+                description: VirtualCluster contains configuration for the virtual
+                  cluster.
+                properties:
+                  etcd:
+                    description: ETCD contains configuration for the etcds of the
+                      virtual garden cluster.
+                    properties:
+                      events:
+                        description: Events contains configuration for the events
+                          etcd.
+                        properties:
+                          storage:
+                            description: Storage contains storage configuration.
+                            properties:
+                              capacity:
+                                anyOf:
+                                - type: integer
+                                - type: string
+                                default: 10Gi
+                                description: Capacity is the storage capacity for
+                                  the volumes.
+                                pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                x-kubernetes-int-or-string: true
+                              className:
+                                description: ClassName is the name of a storage class.
+                                type: string
+                            type: object
+                        type: object
+                      main:
+                        description: Main contains configuration for the main etcd.
+                        properties:
+                          backup:
+                            description: Backup contains the object store configuration
+                              for backups for the virtual garden etcd.
+                            properties:
+                              bucketName:
+                                description: BucketName is the name of the backup
+                                  bucket.
+                                type: string
+                                x-kubernetes-validations:
+                                - message: Value is immutable
+                                  rule: self == oldSelf
+                              provider:
+                                description: Provider is a provider name. This field
+                                  is immutable.
+                                type: string
+                                x-kubernetes-validations:
+                                - message: Value is immutable
+                                  rule: self == oldSelf
+                              secretRef:
+                                description: SecretRef is a reference to a Secret
+                                  object containing the cloud provider credentials
+                                  for the object store where backups should be stored.
+                                  It should have enough privileges to manipulate the
+                                  objects as well as buckets.
+                                properties:
+                                  name:
+                                    description: name is unique within a namespace
+                                      to reference a secret resource.
+                                    type: string
+                                  namespace:
+                                    description: namespace defines the space within
+                                      which the secret name must be unique.
+                                    type: string
+                                type: object
+                                x-kubernetes-map-type: atomic
+                            required:
+                            - bucketName
+                            - provider
+                            - secretRef
+                            type: object
+                          storage:
+                            description: Storage contains storage configuration.
+                            properties:
+                              capacity:
+                                anyOf:
+                                - type: integer
+                                - type: string
+                                default: 10Gi
+                                description: Capacity is the storage capacity for
+                                  the volumes.
+                                pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                x-kubernetes-int-or-string: true
+                              className:
+                                description: ClassName is the name of a storage class.
+                                type: string
+                            type: object
+                        type: object
+                    type: object
+                  maintenance:
+                    description: Maintenance contains information about the time window
+                      for maintenance operations.
+                    properties:
+                      timeWindow:
+                        description: TimeWindow contains information about the time
+                          window for maintenance operations.
+                        properties:
+                          begin:
+                            description: Begin is the beginning of the time window
+                              in the format HHMMSS+ZONE, e.g. "220000+0100". If not
+                              present, a random value will be computed.
+                            pattern: ([0-1][0-9]|2[0-3])[0-5][0-9][0-5][0-9]\+[0-1][0-4]00
+                            type: string
+                          end:
+                            description: End is the end of the time window in the
+                              format HHMMSS+ZONE, e.g. "220000+0100". If not present,
+                              the value will be computed based on the "Begin" value.
+                            pattern: ([0-1][0-9]|2[0-3])[0-5][0-9][0-5][0-9]\+[0-1][0-4]00
+                            type: string
+                        required:
+                        - begin
+                        - end
+                        type: object
+                    required:
+                    - timeWindow
+                    type: object
+                required:
+                - maintenance
+                type: object
             required:
             - runtimeCluster
+            - virtualCluster
             type: object
           status:
             description: Status contains the status of this garden.

diff --git a/charts/gardener/operator/templates/role.yaml b/charts/gardener/operator/templates/role.yaml
@@ -143,3 +143,35 @@ rules:
   - watch
   - patch
   - update
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - networkpolicies
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - watch
+  - patch
+  - update
+- apiGroups:
+  - autoscaling.k8s.io
+  resources:
+  - hvpas
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+- apiGroups:
+  - autoscaling.k8s.io
+  resources:
+  - hvpas
+  resourceNames:
+  - virtual-garden-etcd-events
+  - virtual-garden-etcd-main
+  verbs:
+  - delete
+  - patch
+  - update