Skip to content

Commit

Permalink
Introduce gardener-operator and Garden CRD (gardener#7009)
Browse files Browse the repository at this point in the history
* Basic component config API

* Basic main entrypoint

* Basic operator API

* Basic garden controller

* Deploy CRD for `ManagedResource`s in `resourcemanager` package

* Garden controller reconciles VPA CRD and `gardener-resource-manager`

* Garden controller reconciles VPA components

* Introduce garden-specific `PriorityClass`es similar to seeds and shoots

* Garden controller maintains `Reconciled` condition

* Helm chart

* Skaffold config

* Integration test

* E2E test

* Documentation

* Make `kutil.IsNamespaceInUse` more generic

* `gardenlet` checks if seed cluster is garden cluster at the same time

* Address PR review feedback

* Address PR review feedback

* Address PR review feedback

* Address PR review feedback
  • Loading branch information
rfranzke authored Nov 21, 2022
1 parent 83de074 commit ab9b4f1
Show file tree
Hide file tree
Showing 190 changed files with 6,504 additions and 497 deletions.
8 changes: 8 additions & 0 deletions .ci/resources.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -54,3 +54,11 @@ relation: 'local'
access:
type: 'ociRegistry'
imageReference: 'eu.gcr.io/gardener-project/gardener/seed-admission-controller:${COMPONENT_VERSION}'
---
name: 'operator'
version: '${COMPONENT_VERSION}'
type: 'ociImage'
relation: 'local'
access:
type: 'ociRegistry'
imageReference: 'eu.gcr.io/gardener-project/gardener/operator:${COMPONENT_VERSION}'
10 changes: 10 additions & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,16 @@ WORKDIR /

ENTRYPOINT ["/gardener-resource-manager"]

############# operator #############
FROM distroless-static AS operator

COPY --from=builder /go/bin/gardener-operator /gardener-operator
COPY charts /charts

WORKDIR /

ENTRYPOINT ["/gardener-operator"]

############# gardener-extension-provider-local #############
FROM distroless-static AS gardener-extension-provider-local

Expand Down
43 changes: 33 additions & 10 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ SCHEDULER_IMAGE_REPOSITORY := $(REGISTRY)/scheduler
ADMISSION_IMAGE_REPOSITORY := $(REGISTRY)/admission-controller
SEED_ADMISSION_IMAGE_REPOSITORY := $(REGISTRY)/seed-admission-controller
RESOURCE_MANAGER_IMAGE_REPOSITORY := $(REGISTRY)/resource-manager
OPERATOR_IMAGE_REPOSITORY := $(REGISTRY)/operator
GARDENLET_IMAGE_REPOSITORY := $(REGISTRY)/gardenlet
EXTENSION_PROVIDER_LOCAL_IMAGE_REPOSITORY := $(REGISTRY)/extensions/provider-local
PUSH_LATEST_TAG := false
Expand All @@ -29,6 +30,7 @@ GARDENER_LOCAL_KUBECONFIG := $(REPO_ROOT)/example/gardener-loca
GARDENER_LOCAL2_KUBECONFIG := $(REPO_ROOT)/example/gardener-local/kind/local2/kubeconfig
GARDENER_LOCAL_HA_SINGLE_ZONE_KUBECONFIG := $(REPO_ROOT)/example/gardener-local/kind/ha-single-zone/kubeconfig
GARDENER_LOCAL_HA_MULTI_ZONE_KUBECONFIG := $(REPO_ROOT)/example/gardener-local/kind/ha-multi-zone/kubeconfig
GARDENER_LOCAL_OPERATOR_KUBECONFIG := $(REPO_ROOT)/example/gardener-local/kind/operator/kubeconfig
LOCAL_GARDEN_LABEL := local-garden
REMOTE_GARDEN_LABEL := remote-garden
ACTIVATE_SEEDAUTHORIZER := false
Expand Down Expand Up @@ -105,6 +107,10 @@ start-seed-admission-controller:
start-resource-manager:
@./hack/local-development/start-resource-manager

.PHONY: start-operator
start-operator:
@./hack/local-development/start-operator

.PHONY: start-gardenlet
start-gardenlet: $(HELM) $(YAML2JSON) $(YQ)
@./hack/local-development/start-gardenlet
Expand All @@ -113,7 +119,6 @@ start-gardenlet: $(HELM) $(YAML2JSON) $(YQ)
start-extension-provider-local:
@./hack/local-development/start-extension-provider-local


#################################################################
# Rules related to binary build, Docker image build and release #
#################################################################
Expand All @@ -131,6 +136,7 @@ docker-images:
@docker build --build-arg EFFECTIVE_VERSION=$(EFFECTIVE_VERSION) -t $(ADMISSION_IMAGE_REPOSITORY):$(EFFECTIVE_VERSION) -t $(ADMISSION_IMAGE_REPOSITORY):latest -f Dockerfile --target admission-controller .
@docker build --build-arg EFFECTIVE_VERSION=$(EFFECTIVE_VERSION) -t $(SEED_ADMISSION_IMAGE_REPOSITORY):$(EFFECTIVE_VERSION) -t $(SEED_ADMISSION_IMAGE_REPOSITORY):latest -f Dockerfile --target seed-admission-controller .
@docker build --build-arg EFFECTIVE_VERSION=$(EFFECTIVE_VERSION) -t $(RESOURCE_MANAGER_IMAGE_REPOSITORY):$(EFFECTIVE_VERSION) -t $(RESOURCE_MANAGER_IMAGE_REPOSITORY):latest -f Dockerfile --target resource-manager .
@docker build --build-arg EFFECTIVE_VERSION=$(EFFECTIVE_VERSION) -t $(OPERATOR_IMAGE_REPOSITORY):$(EFFECTIVE_VERSION) -t $(OPERATOR_IMAGE_REPOSITORY):latest -f Dockerfile --target operator .
@docker build --build-arg EFFECTIVE_VERSION=$(EFFECTIVE_VERSION) -t $(GARDENLET_IMAGE_REPOSITORY):$(EFFECTIVE_VERSION) -t $(GARDENLET_IMAGE_REPOSITORY):latest -f Dockerfile --target gardenlet .
@docker build --build-arg EFFECTIVE_VERSION=$(EFFECTIVE_VERSION) -t $(EXTENSION_PROVIDER_LOCAL_IMAGE_REPOSITORY):$(EFFECTIVE_VERSION) -t $(EXTENSION_PROVIDER_LOCAL_IMAGE_REPOSITORY):latest -f Dockerfile --target gardener-extension-provider-local .

Expand All @@ -143,6 +149,7 @@ docker-images-ppc:
@docker build --build-arg EFFECTIVE_VERSION=$(EFFECTIVE_VERSION) -t $(ADMISSION_IMAGE_REPOSITORY):$(EFFECTIVE_VERSION) -t $(ADMISSION_IMAGE_REPOSITORY):latest -f Dockerfile --target admission-controller .
@docker build --build-arg EFFECTIVE_VERSION=$(EFFECTIVE_VERSION) -t $(SEED_ADMISSION_IMAGE_REPOSITORY):$(EFFECTIVE_VERSION) -t $(SEED_ADMISSION_IMAGE_REPOSITORY):latest -f Dockerfile --target seed-admission-controller .
@docker build --build-arg EFFECTIVE_VERSION=$(EFFECTIVE_VERSION) -t $(RESOURCE_MANAGER_IMAGE_REPOSITORY):$(EFFECTIVE_VERSION) -t $(RESOURCE_MANAGER_IMAGE_REPOSITORY):latest -f Dockerfile --target resource-manager .
@docker build --build-arg EFFECTIVE_VERSION=$(EFFECTIVE_VERSION) -t $(OPERATOR_IMAGE_REPOSITORY):$(EFFECTIVE_VERSION) -t $(OPERATOR_IMAGE_REPOSITORY):latest -f Dockerfile --target operator .
@docker build --build-arg EFFECTIVE_VERSION=$(EFFECTIVE_VERSION) -t $(GARDENLET_IMAGE_REPOSITORY):$(EFFECTIVE_VERSION) -t $(GARDENLET_IMAGE_REPOSITORY):latest -f Dockerfile --target gardenlet .
@docker build --build-arg EFFECTIVE_VERSION=$(EFFECTIVE_VERSION) -t $(EXTENSION_PROVIDER_LOCAL_IMAGE_REPOSITORY):$(EFFECTIVE_VERSION) -t $(EXTENSION_PROVIDER_LOCAL_IMAGE_REPOSITORY):latest -f Dockerfile --target gardener-extension-provider-local .

Expand Down Expand Up @@ -284,6 +291,7 @@ kind-up kind-down gardener-up gardener-down register-local-env tear-down-local-e
kind2-up kind2-down gardenlet-kind2-up gardenlet-kind2-down: export KUBECONFIG = $(GARDENER_LOCAL2_KUBECONFIG)
kind-ha-single-zone-up kind-ha-single-zone-down gardener-ha-single-zone-up register-kind-ha-single-zone-env tear-down-kind-ha-single-zone-env ci-e2e-kind-ha-single-zone: export KUBECONFIG = $(GARDENER_LOCAL_HA_SINGLE_ZONE_KUBECONFIG)
kind-ha-multi-zone-up kind-ha-multi-zone-down gardener-ha-multi-zone-up register-kind-ha-multi-zone-env tear-down-kind-ha-multi-zone-env ci-e2e-kind-ha-multi-zone: export KUBECONFIG = $(GARDENER_LOCAL_HA_MULTI_ZONE_KUBECONFIG)
kind-operator-up kind-operator-down operator-up operator-down test-e2e-local-operator ci-e2e-kind-operator: export KUBECONFIG = $(GARDENER_LOCAL_OPERATOR_KUBECONFIG)

kind-up: $(KIND) $(KUBECTL) $(HELM)
./hack/kind-up.sh --cluster-name gardener-local --environment $(KIND_ENV) --path-kubeconfig $(REPO_ROOT)/example/provider-local/seed-kind/base/kubeconfig --path-cluster-values $(REPO_ROOT)/example/gardener-local/kind/local/values.yaml
Expand All @@ -305,12 +313,17 @@ kind-ha-multi-zone-up: $(KIND) $(KUBECTL) $(HELM)
kind-ha-multi-zone-down: $(KIND)
./hack/kind-down.sh --cluster-name gardener-local-ha-multi-zone --path-kubeconfig $(REPO_ROOT)/example/provider-local/seed-kind-ha-multi-zone/base/kubeconfig

kind-operator-up: $(KIND) $(KUBECTL) $(HELM)
./hack/kind-up.sh --cluster-name gardener-operator-local --environment $(KIND_ENV) --path-kubeconfig $(REPO_ROOT)/example/gardener-local/kind/operator/kubeconfig --path-cluster-values $(REPO_ROOT)/example/gardener-local/kind/operator/values.yaml
kind-operator-down: $(KIND)
./hack/kind-down.sh --cluster-name gardener-operator-local --path-kubeconfig $(REPO_ROOT)/example/gardener-local/kind/operator/kubeconfig

# speed-up skaffold deployments by building all images concurrently
export SKAFFOLD_BUILD_CONCURRENCY = 0
# use static label for skaffold to prevent rolling all gardener components on every `skaffold` invocation
gardener-up gardener-down gardener-ha-single-zone-up gardener-ha-single-zone-down gardener-ha-multi-zone-up gardener-ha-multi-zone-down gardenlet-kind2-up gardenlet-kind2-down: export SKAFFOLD_LABEL = skaffold.dev/run-id=gardener-local
# set ldflags for skaffold
gardener-up gardener-ha-single-zone-up gardener-ha-multi-zone-up gardenlet-kind2-up: export LD_FLAGS = $(shell $(REPO_ROOT)/hack/get-build-ld-flags.sh)
gardener-up gardener-ha-single-zone-up gardener-ha-multi-zone-up gardenlet-kind2-up operator-up: export LD_FLAGS = $(shell $(REPO_ROOT)/hack/get-build-ld-flags.sh)

gardener-up: $(SKAFFOLD) $(HELM) $(KUBECTL)
SKAFFOLD_DEFAULT_REPO=localhost:5001 SKAFFOLD_PUSH=true $(SKAFFOLD) run
Expand Down Expand Up @@ -360,22 +373,32 @@ tear-down-kind-ha-multi-zone-env: $(KUBECTL)
$(KUBECTL) delete -k $(REPO_ROOT)/example/provider-local/seed-kind-ha-multi-zone/local
$(KUBECTL) delete -k $(REPO_ROOT)/example/provider-local/garden/local

operator-up: $(SKAFFOLD) $(HELM) $(KUBECTL)
SKAFFOLD_DEFAULT_REPO=localhost:5001 SKAFFOLD_PUSH=true $(SKAFFOLD) run -f skaffold-operator.yaml
operator-down: $(SKAFFOLD) $(HELM) $(KUBECTL)
$(KUBECTL) delete garden --all --ignore-not-found --wait --timeout 5m
$(SKAFFOLD) delete -f skaffold-operator.yaml

test-e2e-local: $(GINKGO)
./hack/test-e2e-local.sh --procs=$(PARALLEL_E2E_TESTS) --label-filter="default" ./test/e2e/gardener/...
test-e2e-local-simple: $(GINKGO)
./hack/test-e2e-local.sh --procs=$(PARALLEL_E2E_TESTS) --label-filter "Shoot && simple"
./hack/test-e2e-local.sh --procs=$(PARALLEL_E2E_TESTS) --label-filter "Shoot && simple" ./test/e2e/gardener/...
test-e2e-local-migration: $(GINKGO)
./hack/test-e2e-local.sh --procs=$(PARALLEL_E2E_TESTS) --label-filter "Shoot && control-plane-migration"
test-e2e-local: $(GINKGO)
./hack/test-e2e-local.sh --procs=$(PARALLEL_E2E_TESTS) --label-filter="default"
./hack/test-e2e-local.sh --procs=$(PARALLEL_E2E_TESTS) --label-filter "Shoot && control-plane-migration" ./test/e2e/gardener/...
test-e2e-local-ha-single-zone: $(GINKGO)
SHOOT_FAILURE_TOLERANCE_TYPE=node ./hack/test-e2e-local.sh --procs=$(PARALLEL_E2E_TESTS) --label-filter "simple || (high-availability && upgrade-to-node)"
SHOOT_FAILURE_TOLERANCE_TYPE=node ./hack/test-e2e-local.sh --procs=$(PARALLEL_E2E_TESTS) --label-filter "simple || (high-availability && upgrade-to-node)" ./test/e2e/gardener/...
test-e2e-local-ha-multi-zone: $(GINKGO)
SHOOT_FAILURE_TOLERANCE_TYPE=zone ./hack/test-e2e-local.sh --procs=$(PARALLEL_E2E_TESTS) --label-filter "simple || (high-availability && upgrade-to-zone)"
SHOOT_FAILURE_TOLERANCE_TYPE=zone ./hack/test-e2e-local.sh --procs=$(PARALLEL_E2E_TESTS) --label-filter "simple || (high-availability && upgrade-to-zone)" ./test/e2e/gardener/...
test-e2e-local-operator: $(GINKGO)
./hack/test-e2e-local.sh operator --procs=$(PARALLEL_E2E_TESTS) --label-filter="default" ./test/e2e/operator/...

ci-e2e-kind: $(KIND) $(YQ)
./hack/ci-e2e-kind.sh
ci-e2e-kind-migration: $(KIND) $(YQ)
./hack/ci-e2e-kind-migration.sh
ci-e2e-kind-ha-single-zone: $(KIND) $(YQ)
./hack/ci-e2e-kind-ha-single-zone.sh
ci-e2e-kind-ha-multi-zone: $(KIND) $(YQ)
./hack/ci-e2e-kind-ha-multi-zone.sh
ci-e2e-kind-migration: $(KIND) $(YQ)
./hack/ci-e2e-kind-migration.sh
ci-e2e-kind-operator: $(KIND) $(YQ)
./hack/ci-e2e-kind-operator.sh
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,7 @@ rules:
- telemetries.telemetry.istio.io
- wasmplugins.extensions.istio.io
- proxyconfigs.networking.istio.io
- managedresources.resources.gardener.cloud
verbs:
- delete
- apiGroups:
Expand Down Expand Up @@ -357,6 +358,7 @@ rules:
- leases
verbs:
- create
- get
- list
- watch
- apiGroups:
Expand All @@ -366,7 +368,6 @@ rules:
resourceNames:
- gardenlet-leader-election
verbs:
- get
- update
- apiGroups:
- coordination.k8s.io
Expand Down
21 changes: 21 additions & 0 deletions charts/gardener/operator/.helmignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*~
# Various IDEs
.project
.idea/
*.tmproj
4 changes: 4 additions & 0 deletions charts/gardener/operator/Chart.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
apiVersion: v1
description: A Helm chart to deploy the Gardener Operator
name: gardener-operator
version: 0.1.0
1 change: 1 addition & 0 deletions charts/gardener/operator/charts/utils-templates
75 changes: 75 additions & 0 deletions charts/gardener/operator/templates/_helpers.tpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
{{- define "operator.kubeconfig.data" -}}
kubeconfig: {{ .Values.config.runtimeClientConnection.kubeconfig | b64enc }}
{{- end -}}

{{- define "operator.kubeconfig.name" -}}
gardener-operator-kubeconfig-{{ include "operator.kubeconfig.data" . | sha256sum | trunc 8 }}
{{- end -}}

{{- define "operator.imagevector-overwrite.data" -}}
images_overwrite.yaml: |
{{ .Values.imageVectorOverwrite | indent 2 }}
{{- end -}}

{{- define "operator.imagevector-overwrite.name" -}}
gardener-operator-imagevector-overwrite-{{ include "operator.imagevector-overwrite.data" . | sha256sum | trunc 8 }}
{{- end -}}

{{- define "operator.imagevector-overwrite-components.data" -}}
components.yaml: |
{{ .Values.componentImageVectorOverwrites | indent 2 }}
{{- end -}}

{{- define "operator.imagevector-overwrite-components.name" -}}
gardener-operator-imagevector-overwrite-components-{{ include "operator.imagevector-overwrite-components.data" . | sha256sum | trunc 8 }}
{{- end -}}

{{- define "operator.config.data" -}}
config.yaml: |
---
apiVersion: operator.config.gardener.cloud/v1alpha1
kind: OperatorConfiguration
runtimeClientConnection:
qps: {{ .Values.config.runtimeClientConnection.qps }}
burst: {{ .Values.config.runtimeClientConnection.burst }}
{{- if .Values.config.runtimeClientConnection.kubeconfig }}
kubeconfig: /etc/gardener-operator/kubeconfig/kubeconfig
{{- end }}
leaderElection:
leaderElect: {{ .Values.config.leaderElection.leaderElect }}
leaseDuration: {{ .Values.config.leaderElection.leaseDuration }}
renewDeadline: {{ .Values.config.leaderElection.renewDeadline }}
retryPeriod: {{ .Values.config.leaderElection.retryPeriod }}
resourceLock: {{ .Values.config.leaderElection.resourceLock }}
resourceName: {{ .Values.config.leaderElection.resourceName }}
resourceNamespace: {{ .Release.Namespace }}
logLevel: {{ .Values.config.logLevel | default "info" }}
logFormat: {{ .Values.config.logFormat | default "json" }}
server:
healthProbes:
bindAddress: {{ .Values.config.server.healthProbes.bindAddress }}
port: {{ .Values.config.server.healthProbes.port }}
metrics:
bindAddress: {{ .Values.config.server.metrics.bindAddress }}
port: {{ .Values.config.server.metrics.port }}
{{- if .Values.config.debugging }}
debugging:
enableProfiling: {{ .Values.config.debugging.enableProfiling }}
enableContentionProfiling: {{ .Values.config.debugging.enableContentionProfiling }}
{{- end }}
featureGates:
{{ toYaml .Values.config.featureGates | indent 4 }}
controllers:
garden:
{{- if .Values.config.controllers.garden.concurrentSyncs }}
concurrentSyncs: {{ .Values.config.controllers.garden.concurrentSyncs }}
{{- end }}
{{- if .Values.config.controllers.garden.syncPeriod }}
syncPeriod: {{ .Values.config.controllers.garden.syncPeriod }}
{{- end }}
{{- end -}}

{{- define "operator.config.name" -}}
gardener-operator-configmap-{{ include "operator.config.data" . | sha256sum | trunc 8 }}
{{- end -}}

98 changes: 98 additions & 0 deletions charts/gardener/operator/templates/clusterrole.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: gardener.cloud:system:gardener-operator
labels:
app: gardener
role: operator
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
release: "{{ .Release.Name }}"
heritage: "{{ .Release.Service }}"
rules:
- apiGroups:
- operator.gardener.cloud
resources:
- gardens
- gardens/status
verbs:
- get
- list
- watch
- patch
- update
- apiGroups:
- ""
resources:
- namespaces
verbs:
- create
- get
- list
- watch
- apiGroups:
- ""
resources:
- namespaces
resourceNames:
- {{ .Release.Namespace }}
verbs:
- update
- patch
- apiGroups:
- ""
resources:
- events
verbs:
- create
- patch
- update
- apiGroups:
- admissionregistration.k8s.io
resources:
- mutatingwebhookconfigurations
- validatingwebhookconfigurations
verbs:
- create
- delete
- deletecollection
- get
- list
- watch
- patch
- update
- apiGroups:
- apiextensions.k8s.io
resources:
- customresourcedefinitions
verbs:
- create
- get
- list
- watch
- patch
- update
- delete
- apiGroups:
- rbac.authorization.k8s.io
resources:
- clusterrolebindings
- clusterroles
verbs:
- create
- delete
- deletecollection
- get
- list
- watch
- patch
- update
# gardener-operator deploys other components like gardener-resource-manager which might have elevated privileges, so
# let's allow both 'bind' and 'escalate' verbs for now.
- apiGroups:
- rbac.authorization.k8s.io
resources:
- clusterroles
verbs:
- bind
- escalate
19 changes: 19 additions & 0 deletions charts/gardener/operator/templates/clusterrolebinding.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: gardener.cloud:system:gardener-operator
labels:
app: gardener
role: operator
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
release: "{{ .Release.Name }}"
heritage: "{{ .Release.Service }}"
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: gardener.cloud:system:gardener-operator
subjects:
- kind: ServiceAccount
name: "{{ required ".Values.serviceAccountName is required" .Values.serviceAccountName }}"
namespace: {{ .Release.Namespace }}
Loading

0 comments on commit ab9b4f1

Please sign in to comment.