diff --git a/charts/gardener/controlplane/charts/application/charts/utils-common b/charts/gardener/controlplane/charts/application/charts/utils-common deleted file mode 120000 index d7636506d63..00000000000 --- a/charts/gardener/controlplane/charts/application/charts/utils-common +++ /dev/null @@ -1 +0,0 @@ -../../utils-common \ No newline at end of file diff --git a/charts/gardener/controlplane/charts/application/charts/utils-templates b/charts/gardener/controlplane/charts/application/charts/utils-templates deleted file mode 120000 index db495912817..00000000000 --- a/charts/gardener/controlplane/charts/application/charts/utils-templates +++ /dev/null @@ -1 +0,0 @@ -../../utils-templates \ No newline at end of file diff --git a/charts/gardener/controlplane/charts/application/templates/apiservice-v1alpha1-operations-gardener-cloud.yaml b/charts/gardener/controlplane/charts/application/templates/apiservice-v1alpha1-operations-gardener-cloud.yaml index 836867118c7..36c1be6b5a3 100644 --- a/charts/gardener/controlplane/charts/application/templates/apiservice-v1alpha1-operations-gardener-cloud.yaml +++ b/charts/gardener/controlplane/charts/application/templates/apiservice-v1alpha1-operations-gardener-cloud.yaml @@ -1,5 +1,5 @@ {{- if .Values.global.apiserver.enabled }} -apiVersion: {{ include "apiserviceversion" . }} +apiVersion: apiregistration.k8s.io/v1 kind: APIService metadata: name: v1alpha1.operations.gardener.cloud diff --git a/charts/gardener/controlplane/charts/application/templates/apiservice-v1alpha1-seedmanagement-gardener-cloud.yaml b/charts/gardener/controlplane/charts/application/templates/apiservice-v1alpha1-seedmanagement-gardener-cloud.yaml index 0d566384266..5a2f592634f 100644 --- a/charts/gardener/controlplane/charts/application/templates/apiservice-v1alpha1-seedmanagement-gardener-cloud.yaml +++ b/charts/gardener/controlplane/charts/application/templates/apiservice-v1alpha1-seedmanagement-gardener-cloud.yaml @@ -1,5 +1,5 @@ {{- if .Values.global.apiserver.enabled }} -apiVersion: {{ include "apiserviceversion" . }} +apiVersion: apiregistration.k8s.io/v1 kind: APIService metadata: name: v1alpha1.seedmanagement.gardener.cloud diff --git a/charts/gardener/controlplane/charts/application/templates/apiservice-v1alpha1-settings-gardener-cloud.yaml b/charts/gardener/controlplane/charts/application/templates/apiservice-v1alpha1-settings-gardener-cloud.yaml index 5f12fe1caa8..1d80a611f0a 100644 --- a/charts/gardener/controlplane/charts/application/templates/apiservice-v1alpha1-settings-gardener-cloud.yaml +++ b/charts/gardener/controlplane/charts/application/templates/apiservice-v1alpha1-settings-gardener-cloud.yaml @@ -1,5 +1,5 @@ {{- if .Values.global.apiserver.enabled }} -apiVersion: {{ include "apiserviceversion" . }} +apiVersion: apiregistration.k8s.io/v1 kind: APIService metadata: name: v1alpha1.settings.gardener.cloud diff --git a/charts/gardener/controlplane/charts/application/templates/apiservice-v1beta1-core-gardener-cloud.yaml b/charts/gardener/controlplane/charts/application/templates/apiservice-v1beta1-core-gardener-cloud.yaml index 259c12d4722..f4eb8f2333f 100644 --- a/charts/gardener/controlplane/charts/application/templates/apiservice-v1beta1-core-gardener-cloud.yaml +++ b/charts/gardener/controlplane/charts/application/templates/apiservice-v1beta1-core-gardener-cloud.yaml @@ -1,5 +1,5 @@ {{- if .Values.global.apiserver.enabled }} -apiVersion: {{ include "apiserviceversion" . }} +apiVersion: apiregistration.k8s.io/v1 kind: APIService metadata: name: v1beta1.core.gardener.cloud diff --git a/charts/gardener/controlplane/charts/application/templates/clusterrole-admission-controller.yaml b/charts/gardener/controlplane/charts/application/templates/clusterrole-admission-controller.yaml index 05f0b48dd7f..85b0bccb849 100644 --- a/charts/gardener/controlplane/charts/application/templates/clusterrole-admission-controller.yaml +++ b/charts/gardener/controlplane/charts/application/templates/clusterrole-admission-controller.yaml @@ -1,6 +1,6 @@ {{- if .Values.global.admission.enabled }} --- -apiVersion: {{ include "rbacversion" . }} +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: gardener.cloud:system:admission-controller diff --git a/charts/gardener/controlplane/charts/application/templates/clusterrole-apiserver.yaml b/charts/gardener/controlplane/charts/application/templates/clusterrole-apiserver.yaml index 9a6ec70d877..e7906b2dbc5 100644 --- a/charts/gardener/controlplane/charts/application/templates/clusterrole-apiserver.yaml +++ b/charts/gardener/controlplane/charts/application/templates/clusterrole-apiserver.yaml @@ -1,6 +1,6 @@ {{- if .Values.global.apiserver.enabled }} --- -apiVersion: {{ include "rbacversion" . }} +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: gardener.cloud:system:apiserver diff --git a/charts/gardener/controlplane/charts/application/templates/clusterrole-controller-manager.yaml b/charts/gardener/controlplane/charts/application/templates/clusterrole-controller-manager.yaml index 073dd138296..d9ffee248f5 100644 --- a/charts/gardener/controlplane/charts/application/templates/clusterrole-controller-manager.yaml +++ b/charts/gardener/controlplane/charts/application/templates/clusterrole-controller-manager.yaml @@ -1,6 +1,6 @@ {{- if .Values.global.controller.enabled }} --- -apiVersion: {{ include "rbacversion" . }} +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: gardener.cloud:system:controller-manager diff --git a/charts/gardener/controlplane/charts/application/templates/clusterrole-scheduler.yaml b/charts/gardener/controlplane/charts/application/templates/clusterrole-scheduler.yaml index d36f5658e46..d1a859f2a79 100644 --- a/charts/gardener/controlplane/charts/application/templates/clusterrole-scheduler.yaml +++ b/charts/gardener/controlplane/charts/application/templates/clusterrole-scheduler.yaml @@ -7,7 +7,7 @@ # Shoots/binding CREATE on binding subresource of shoots - actual scheduling request that leads to setting shoot.Spec.Cloud.Seed # Shoots/status PATCH, UPDATE on status subresource of shoots --- -apiVersion: {{ include "rbacversion" . }} +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: gardener.cloud:system:scheduler diff --git a/charts/gardener/controlplane/charts/application/templates/clusterrole-seed-bootstrapper.yaml b/charts/gardener/controlplane/charts/application/templates/clusterrole-seed-bootstrapper.yaml index 33983b8e4cb..d95d9c0b9c7 100644 --- a/charts/gardener/controlplane/charts/application/templates/clusterrole-seed-bootstrapper.yaml +++ b/charts/gardener/controlplane/charts/application/templates/clusterrole-seed-bootstrapper.yaml @@ -1,5 +1,5 @@ --- -apiVersion: {{ include "rbacversion" . }} +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: gardener.cloud:system:seed-bootstrapper diff --git a/charts/gardener/controlplane/charts/application/templates/clusterrole-seeds.yaml b/charts/gardener/controlplane/charts/application/templates/clusterrole-seeds.yaml index 8a578b0d62f..15d92b09608 100644 --- a/charts/gardener/controlplane/charts/application/templates/clusterrole-seeds.yaml +++ b/charts/gardener/controlplane/charts/application/templates/clusterrole-seeds.yaml @@ -1,6 +1,6 @@ {{- if not .Values.global.rbac.seedAuthorizer.enabled }} --- -apiVersion: {{ include "rbacversion" . }} +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: gardener.cloud:system:seeds diff --git a/charts/gardener/controlplane/charts/application/templates/clusterrolebinding-admission-controller.yaml b/charts/gardener/controlplane/charts/application/templates/clusterrolebinding-admission-controller.yaml index 3d57f5e44a2..43450df7eeb 100644 --- a/charts/gardener/controlplane/charts/application/templates/clusterrolebinding-admission-controller.yaml +++ b/charts/gardener/controlplane/charts/application/templates/clusterrolebinding-admission-controller.yaml @@ -1,6 +1,6 @@ {{- if .Values.global.admission.enabled }} --- -apiVersion: {{ include "rbacversion" . }} +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: gardener.cloud:admission-controller diff --git a/charts/gardener/controlplane/charts/application/templates/clusterrolebinding-apiserver-auth-delegator.yaml b/charts/gardener/controlplane/charts/application/templates/clusterrolebinding-apiserver-auth-delegator.yaml index 2bfe77a1001..950e08bc933 100644 --- a/charts/gardener/controlplane/charts/application/templates/clusterrolebinding-apiserver-auth-delegator.yaml +++ b/charts/gardener/controlplane/charts/application/templates/clusterrolebinding-apiserver-auth-delegator.yaml @@ -1,6 +1,6 @@ {{- if .Values.global.apiserver.enabled }} --- -apiVersion: {{ include "rbacversion" . }} +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: gardener.cloud:apiserver:auth-delegator diff --git a/charts/gardener/controlplane/charts/application/templates/clusterrolebinding-apiserver.yaml b/charts/gardener/controlplane/charts/application/templates/clusterrolebinding-apiserver.yaml index f8bc7c830d7..443f3a28110 100644 --- a/charts/gardener/controlplane/charts/application/templates/clusterrolebinding-apiserver.yaml +++ b/charts/gardener/controlplane/charts/application/templates/clusterrolebinding-apiserver.yaml @@ -1,6 +1,6 @@ {{- if .Values.global.apiserver.enabled }} --- -apiVersion: {{ include "rbacversion" . }} +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: gardener.cloud:system:apiserver diff --git a/charts/gardener/controlplane/charts/application/templates/clusterrolebinding-controller-manager.yaml b/charts/gardener/controlplane/charts/application/templates/clusterrolebinding-controller-manager.yaml index d9f82518f25..7db2c484792 100644 --- a/charts/gardener/controlplane/charts/application/templates/clusterrolebinding-controller-manager.yaml +++ b/charts/gardener/controlplane/charts/application/templates/clusterrolebinding-controller-manager.yaml @@ -1,6 +1,6 @@ {{- if .Values.global.controller.enabled }} --- -apiVersion: {{ include "rbacversion" . }} +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: gardener.cloud:system:controller-manager diff --git a/charts/gardener/controlplane/charts/application/templates/clusterrolebinding-kube-aggregator-admin.yaml b/charts/gardener/controlplane/charts/application/templates/clusterrolebinding-kube-aggregator-admin.yaml index 658028ebb23..000ef8ca1d9 100644 --- a/charts/gardener/controlplane/charts/application/templates/clusterrolebinding-kube-aggregator-admin.yaml +++ b/charts/gardener/controlplane/charts/application/templates/clusterrolebinding-kube-aggregator-admin.yaml @@ -1,6 +1,6 @@ {{- if .Values.global.apiserver.enabled }} --- -apiVersion: {{ include "rbacversion" . }} +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: gardener.cloud:admin diff --git a/charts/gardener/controlplane/charts/application/templates/clusterrolebinding-scheduler.yaml b/charts/gardener/controlplane/charts/application/templates/clusterrolebinding-scheduler.yaml index 212e9fb381d..d052a43bfa8 100644 --- a/charts/gardener/controlplane/charts/application/templates/clusterrolebinding-scheduler.yaml +++ b/charts/gardener/controlplane/charts/application/templates/clusterrolebinding-scheduler.yaml @@ -1,6 +1,6 @@ {{- if .Values.global.scheduler.enabled }} --- -apiVersion: {{ include "rbacversion" . }} +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: gardener.cloud:system:scheduler diff --git a/charts/gardener/controlplane/charts/application/templates/clusterrolebinding-seed-bootstrapper.yaml b/charts/gardener/controlplane/charts/application/templates/clusterrolebinding-seed-bootstrapper.yaml index 3e65ef80c6f..2e4c546e15c 100644 --- a/charts/gardener/controlplane/charts/application/templates/clusterrolebinding-seed-bootstrapper.yaml +++ b/charts/gardener/controlplane/charts/application/templates/clusterrolebinding-seed-bootstrapper.yaml @@ -1,5 +1,5 @@ --- -apiVersion: {{ include "rbacversion" . }} +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: gardener.cloud:system:seed-bootstrapper diff --git a/charts/gardener/controlplane/charts/application/templates/clusterrolebinding-seeds.yaml b/charts/gardener/controlplane/charts/application/templates/clusterrolebinding-seeds.yaml index 1cf8a50fa2b..0618fb4bdbd 100644 --- a/charts/gardener/controlplane/charts/application/templates/clusterrolebinding-seeds.yaml +++ b/charts/gardener/controlplane/charts/application/templates/clusterrolebinding-seeds.yaml @@ -1,6 +1,6 @@ {{- if not .Values.global.rbac.seedAuthorizer.enabled }} --- -apiVersion: {{ include "rbacversion" . }} +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: gardener.cloud:system:seeds diff --git a/charts/gardener/controlplane/charts/application/templates/mutatingwebhook-admission-controller.yaml b/charts/gardener/controlplane/charts/application/templates/mutatingwebhook-admission-controller.yaml index 23be96a95e2..f95c74334f3 100644 --- a/charts/gardener/controlplane/charts/application/templates/mutatingwebhook-admission-controller.yaml +++ b/charts/gardener/controlplane/charts/application/templates/mutatingwebhook-admission-controller.yaml @@ -1,5 +1,5 @@ {{- if .Values.global.admission.enabled }} -apiVersion: {{ include "webhookadmissionregistration" . }} +apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration metadata: name: gardener-admission-controller diff --git a/charts/gardener/controlplane/charts/application/templates/rbac-user.yaml b/charts/gardener/controlplane/charts/application/templates/rbac-user.yaml index 0de4e20929d..fbf78aaf4d2 100644 --- a/charts/gardener/controlplane/charts/application/templates/rbac-user.yaml +++ b/charts/gardener/controlplane/charts/application/templates/rbac-user.yaml @@ -1,6 +1,6 @@ # Cluster role granting full permissions to all resources in the gardener API group. --- -apiVersion: {{ include "rbacversion" . }} +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: gardener.cloud:admin @@ -93,7 +93,7 @@ rules: [] # Cluster role granting viewer permissions for the resources in the gardener API group --- -apiVersion: {{ include "rbacversion" . }} +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: gardener.cloud:viewer @@ -208,7 +208,7 @@ rules: [] # Cluster role with cluster role binding allowing all authenticated users to read some global resources --- -apiVersion: {{ include "rbacversion" . }} +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: gardener.cloud:system:read-global-resources @@ -229,7 +229,7 @@ rules: - list - watch --- -apiVersion: {{ include "rbacversion" . }} +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: gardener.cloud:system:read-global-resources @@ -249,7 +249,7 @@ subjects: # Cluster role with cluster role binding allowing all authenticated users create tokenreviews and selfsubjectaccessreviews --- -apiVersion: {{ include "rbacversion" . }} +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: gardener.cloud:system:user-auth @@ -272,7 +272,7 @@ rules: verbs: - create --- -apiVersion: {{ include "rbacversion" . }} +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: gardener.cloud:system:user-auth @@ -294,7 +294,7 @@ subjects: # IMPORTANT: You need to define a corresponding ClusterRoleBinding binding specific users/ # groups/serviceaccounts to this ClusterRole on your own. --- -apiVersion: {{ include "rbacversion" . }} +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: gardener.cloud:system:project-creation @@ -315,7 +315,7 @@ rules: # in a respective project namespace. # It aggregates all ClusterRoles labeled with rbac.gardener.cloud/aggregate-to-project-member: "true" --- -apiVersion: {{ include "rbacversion" . }} +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: gardener.cloud:system:project-member @@ -330,7 +330,7 @@ aggregationRule: - matchLabels: rbac.gardener.cloud/aggregate-to-project-member: "true" --- -apiVersion: {{ include "rbacversion" . }} +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: gardener.cloud:system:project-member-aggregation @@ -439,7 +439,7 @@ rules: # in a respective project namespace. # It aggregates all ClusterRoles labeled with rbac.gardener.cloud/aggregate-to-project-serviceaccountmanager: "true" --- -apiVersion: {{ include "rbacversion" . }} +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: gardener.cloud:system:project-serviceaccountmanager @@ -454,7 +454,7 @@ aggregationRule: - matchLabels: rbac.gardener.cloud/aggregate-to-project-serviceaccountmanager: "true" --- -apiVersion: {{ include "rbacversion" . }} +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: gardener.cloud:system:project-serviceaccountmanager-aggregation @@ -489,7 +489,7 @@ rules: # in a respective project namespace. # It aggregates all ClusterRoles labeled with rbac.gardener.cloud/aggregate-to-project-viewer: "true" --- -apiVersion: {{ include "rbacversion" . }} +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: gardener.cloud:system:project-viewer @@ -504,7 +504,7 @@ aggregationRule: - matchLabels: rbac.gardener.cloud/aggregate-to-project-viewer: "true" --- -apiVersion: {{ include "rbacversion" . }} +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: gardener.cloud:system:project-viewer-aggregation diff --git a/charts/gardener/controlplane/charts/application/templates/rolebinding-apiserver-auth-reader.yaml b/charts/gardener/controlplane/charts/application/templates/rolebinding-apiserver-auth-reader.yaml index ef99172f056..7c533e04c8e 100644 --- a/charts/gardener/controlplane/charts/application/templates/rolebinding-apiserver-auth-reader.yaml +++ b/charts/gardener/controlplane/charts/application/templates/rolebinding-apiserver-auth-reader.yaml @@ -1,6 +1,6 @@ {{- if .Values.global.apiserver.enabled }} --- -apiVersion: {{ include "rbacversion" . }} +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: gardener.cloud:apiserver:auth-reader diff --git a/charts/gardener/controlplane/charts/application/templates/secret-alerting.yaml b/charts/gardener/controlplane/charts/application/templates/secret-alerting.yaml index 24759cd43af..f46021b4d25 100644 --- a/charts/gardener/controlplane/charts/application/templates/secret-alerting.yaml +++ b/charts/gardener/controlplane/charts/application/templates/secret-alerting.yaml @@ -1 +1,43 @@ +{{- define "gardener.secret-alerting" -}} +{{- range $key, $config := .Values.global.alerting }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: alerting-{{ $key }} + namespace: garden + labels: + app: gardener + chart: "{{ $.Chart.Name }}-{{ $.Chart.Version }}" + release: "{{ $.Release.Name }}" + heritage: "{{ $.Release.Service }}" + gardener.cloud/role: alerting +type: Opaque +data: + auth_type: {{ ( required ".alerting[].auth_type is required" $config.auth_type ) | b64enc }} +{{- if eq $config.auth_type "smtp" }} + to: {{ ( required ".alerting[].to is required" $config.to ) | b64enc }} + from: {{ ( required ".alerting[].from is required" $config.from ) | b64enc }} + smarthost: {{ ( required ".alerting[].smarthost is required" $config.smarthost ) | b64enc }} + auth_username: {{ ( required ".alerting[].auth_username is required" $config.auth_username ) | b64enc }} + auth_identity: {{ ( required ".alerting[].auth_identity is required" $config.auth_identity ) | b64enc }} + auth_password: {{ ( required ".alerting[].auth_password is required" $config.auth_password ) | b64enc }} +{{- end }} +{{- if eq $config.auth_type "none" }} + url: {{ ( required ".alerting[].url is required" $config.url ) | b64enc }} +{{- end }} +{{- if eq $config.auth_type "basic" }} + url: {{ ( required ".alerting[].url is required" $config.url ) | b64enc }} + username: {{ ( required ".alerting[].username is required" $config.username ) | b64enc }} + password: {{ ( required ".alerting[].password is required" $config.password ) | b64enc }} +{{- end }} +{{- if eq $config.auth_type "certificate" }} + url: {{ ( required ".alerting[].url is required" $config.url ) | b64enc }} + ca.crt: {{ ( required ".alerting[].ca_crt is required" $config.ca_crt ) | b64enc }} + tls.crt: {{ ( required ".alerting[].tls_crt is required" $config.tls_cert ) | b64enc }} + tls.key: {{ ( required ".alerting[].tls_key is required" $config.tls_key ) | b64enc }} + insecure_skip_verify: {{ ( required ".alerting[].insecure_skip_verify is required" $config.insecure_skip_verify ) | b64enc }} +{{- end }} +{{- end }} +{{- end -}} {{- include "gardener.secret-alerting" . }} diff --git a/charts/gardener/controlplane/charts/application/templates/secret-default-domain.yaml b/charts/gardener/controlplane/charts/application/templates/secret-default-domain.yaml index e7226539c81..da32d3bf45a 100644 --- a/charts/gardener/controlplane/charts/application/templates/secret-default-domain.yaml +++ b/charts/gardener/controlplane/charts/application/templates/secret-default-domain.yaml @@ -1 +1,26 @@ +{{- define "gardener.secret-default-domain" -}} +{{- range $key, $domain := .Values.global.defaultDomains }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: default-domain-{{ ( required ".defaultDomains[].domain is required" (replace "." "-" $domain.domain) ) }} + namespace: garden + labels: + app: gardener + chart: "{{ $.Chart.Name }}-{{ $.Chart.Version }}" + release: "{{ $.Release.Name }}" + heritage: "{{ $.Release.Service }}" + gardener.cloud/role: default-domain + annotations: + dns.gardener.cloud/provider: {{ ( required ".defaultDomains[].provider is required" $domain.provider ) }} + dns.gardener.cloud/domain: {{ ( required ".defaultDomains[].domain is required" $domain.domain ) }} + {{- if $domain.zone }} + dns.gardener.cloud/zone: {{ $domain.zone }} + {{- end }} +type: Opaque +data: +{{ toYaml $domain.credentials | indent 2 }} +{{- end }} +{{- end -}} {{- include "gardener.secret-default-domain" . }} diff --git a/charts/gardener/controlplane/charts/application/templates/secret-internal-domain.yaml b/charts/gardener/controlplane/charts/application/templates/secret-internal-domain.yaml index 069224b8761..c0f1c811d2c 100644 --- a/charts/gardener/controlplane/charts/application/templates/secret-internal-domain.yaml +++ b/charts/gardener/controlplane/charts/application/templates/secret-internal-domain.yaml @@ -1 +1,24 @@ +{{- define "gardener.secret-internal-domain" -}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: internal-domain-{{ ( required ".internalDomain.domain is required" (replace "." "-" .Values.global.internalDomain.domain) ) }} + namespace: garden + labels: + app: gardener + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" + gardener.cloud/role: internal-domain + annotations: + dns.gardener.cloud/provider: {{ ( required ".internalDomain.provider is required" .Values.global.internalDomain.provider ) }} + dns.gardener.cloud/domain: {{ ( required ".internalDomain.domain is required" .Values.global.internalDomain.domain ) }} + {{- if .Values.global.internalDomain.zone }} + dns.gardener.cloud/zone: {{ .Values.global.internalDomain.zone }} + {{- end }} +type: Opaque +data: +{{ toYaml .Values.global.internalDomain.credentials | indent 2 }} +{{- end -}} {{- include "gardener.secret-internal-domain" . }} diff --git a/charts/gardener/controlplane/charts/application/templates/secret-openvpn-diffie-hellman.yaml b/charts/gardener/controlplane/charts/application/templates/secret-openvpn-diffie-hellman.yaml index 75fdf0421cf..2cdafe11fbe 100644 --- a/charts/gardener/controlplane/charts/application/templates/secret-openvpn-diffie-hellman.yaml +++ b/charts/gardener/controlplane/charts/application/templates/secret-openvpn-diffie-hellman.yaml @@ -1 +1,19 @@ +{{- define "gardener.secret-openvpn-diffie-hellman" -}} +{{- if .Values.global.openVPNDiffieHellmanKey }} +apiVersion: v1 +kind: Secret +metadata: + name: openvpn-diffie-hellman-key + namespace: garden + labels: + app: gardener + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" + gardener.cloud/role: openvpn-diffie-hellman +type: Opaque +data: + dh2048.pem: {{ .Values.global.openVPNDiffieHellmanKey | b64enc }} +{{- end }} +{{- end -}} {{- include "gardener.secret-openvpn-diffie-hellman" . }} diff --git a/charts/gardener/controlplane/charts/application/templates/validatingwebhook-admission-controller.yaml b/charts/gardener/controlplane/charts/application/templates/validatingwebhook-admission-controller.yaml index 55e1df54f1a..6a0f4e47598 100644 --- a/charts/gardener/controlplane/charts/application/templates/validatingwebhook-admission-controller.yaml +++ b/charts/gardener/controlplane/charts/application/templates/validatingwebhook-admission-controller.yaml @@ -1,5 +1,5 @@ {{- if .Values.global.admission.enabled }} -apiVersion: {{ include "webhookadmissionregistration" . }} +apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: name: gardener-admission-controller diff --git a/charts/gardener/controlplane/charts/runtime/charts/utils-common b/charts/gardener/controlplane/charts/runtime/charts/utils-common deleted file mode 120000 index d7636506d63..00000000000 --- a/charts/gardener/controlplane/charts/runtime/charts/utils-common +++ /dev/null @@ -1 +0,0 @@ -../../utils-common \ No newline at end of file diff --git a/charts/gardener/controlplane/charts/runtime/charts/utils-templates b/charts/gardener/controlplane/charts/runtime/charts/utils-templates deleted file mode 120000 index db495912817..00000000000 --- a/charts/gardener/controlplane/charts/runtime/charts/utils-templates +++ /dev/null @@ -1 +0,0 @@ -../../utils-templates \ No newline at end of file diff --git a/charts/utils-templates/templates/_image.tpl b/charts/gardener/controlplane/charts/runtime/templates/_image.tpl similarity index 88% rename from charts/utils-templates/templates/_image.tpl rename to charts/gardener/controlplane/charts/runtime/templates/_image.tpl index fefcf96384d..b075f8cad07 100644 --- a/charts/utils-templates/templates/_image.tpl +++ b/charts/gardener/controlplane/charts/runtime/templates/_image.tpl @@ -1,4 +1,4 @@ -{{- define "utils-templates.image" -}} +{{- define "image" -}} {{- if hasPrefix "sha256:" (required "$.tag is required" $.tag) -}} {{ required "$.repository is required" $.repository }}@{{ required "$.tag is required" $.tag }} {{- else -}} diff --git a/charts/gardener/controlplane/charts/runtime/templates/admission-controller/deployment.yaml b/charts/gardener/controlplane/charts/runtime/templates/admission-controller/deployment.yaml index a34f96bdec6..5dcf796299a 100644 --- a/charts/gardener/controlplane/charts/runtime/templates/admission-controller/deployment.yaml +++ b/charts/gardener/controlplane/charts/runtime/templates/admission-controller/deployment.yaml @@ -1,5 +1,5 @@ {{- if .Values.global.admission.enabled }} -apiVersion: {{ include "deploymentversion" . }} +apiVersion: apps/v1 kind: Deployment metadata: name: gardener-admission-controller @@ -70,7 +70,7 @@ spec: {{- end }} containers: - name: gardener-admission-controller - image: {{ include "utils-templates.image" .Values.global.admission.image }} + image: {{ include "image" .Values.global.admission.image }} imagePullPolicy: {{ .Values.global.admission.image.pullPolicy }} args: - --config=/etc/gardener-admission-controller/config/config.yaml diff --git a/charts/gardener/controlplane/charts/runtime/templates/admission-controller/poddisruptionbudget.yaml b/charts/gardener/controlplane/charts/runtime/templates/admission-controller/poddisruptionbudget.yaml index e3965539f33..3ffb471078a 100644 --- a/charts/gardener/controlplane/charts/runtime/templates/admission-controller/poddisruptionbudget.yaml +++ b/charts/gardener/controlplane/charts/runtime/templates/admission-controller/poddisruptionbudget.yaml @@ -1,5 +1,5 @@ {{- if gt (int .Values.global.admission.replicaCount) 1 }} -apiVersion: {{ include "poddisruptionbudgetversion" .}} +apiVersion: policy/v1 kind: PodDisruptionBudget metadata: name: gardener-admission-controller diff --git a/charts/gardener/controlplane/charts/runtime/templates/admission-controller/vpa.yaml b/charts/gardener/controlplane/charts/runtime/templates/admission-controller/vpa.yaml index 9e6ecdbc75e..8ad9c9dede3 100644 --- a/charts/gardener/controlplane/charts/runtime/templates/admission-controller/vpa.yaml +++ b/charts/gardener/controlplane/charts/runtime/templates/admission-controller/vpa.yaml @@ -6,7 +6,7 @@ metadata: namespace: garden spec: targetRef: - apiVersion: {{ include "deploymentversion" . }} + apiVersion: apps/v1 kind: Deployment name: gardener-admission-controller updatePolicy: @@ -16,4 +16,4 @@ spec: - containerName: '*' minAllowed: memory: 25Mi -{{- end }} \ No newline at end of file +{{- end }} diff --git a/charts/gardener/controlplane/charts/runtime/templates/apiserver/deployment.yaml b/charts/gardener/controlplane/charts/runtime/templates/apiserver/deployment.yaml index 1ca3173aae5..237a897233f 100644 --- a/charts/gardener/controlplane/charts/runtime/templates/apiserver/deployment.yaml +++ b/charts/gardener/controlplane/charts/runtime/templates/apiserver/deployment.yaml @@ -1,5 +1,5 @@ {{- if .Values.global.apiserver.enabled }} -apiVersion: {{ include "deploymentversion" . }} +apiVersion: apps/v1 kind: Deployment metadata: name: gardener-apiserver @@ -89,7 +89,7 @@ spec: {{- end }} containers: - name: gardener-apiserver - image: {{ include "utils-templates.image" .Values.global.apiserver.image }} + image: {{ include "image" .Values.global.apiserver.image }} imagePullPolicy: {{ .Values.global.apiserver.image.pullPolicy }} args: {{- if (include "gardener-apiserver.hasAdmissionPlugins" .) }} diff --git a/charts/gardener/controlplane/charts/runtime/templates/apiserver/hvpa.yaml b/charts/gardener/controlplane/charts/runtime/templates/apiserver/hvpa.yaml index 5da006ec272..061acfe009e 100644 --- a/charts/gardener/controlplane/charts/runtime/templates/apiserver/hvpa.yaml +++ b/charts/gardener/controlplane/charts/runtime/templates/apiserver/hvpa.yaml @@ -89,7 +89,7 @@ spec: startReplicaCount: {{ .Values.global.apiserver.hvpa.maxReplicas }} lastReplicaCount: {{ .Values.global.apiserver.hvpa.maxReplicas }} targetRef: - apiVersion: {{ include "deploymentversion" . }} + apiVersion: apps/v1 kind: Deployment name: gardener-apiserver {{ end }} diff --git a/charts/gardener/controlplane/charts/runtime/templates/apiserver/poddisruptionbudget.yaml b/charts/gardener/controlplane/charts/runtime/templates/apiserver/poddisruptionbudget.yaml index 02d7b3827ed..40384af7ba6 100644 --- a/charts/gardener/controlplane/charts/runtime/templates/apiserver/poddisruptionbudget.yaml +++ b/charts/gardener/controlplane/charts/runtime/templates/apiserver/poddisruptionbudget.yaml @@ -1,5 +1,5 @@ {{- if or .Values.global.apiserver.hvpa.enabled (gt (int .Values.global.apiserver.replicaCount) 1) }} -apiVersion: {{ include "poddisruptionbudgetversion" .}} +apiVersion: policy/v1 kind: PodDisruptionBudget metadata: name: gardener-apiserver diff --git a/charts/gardener/controlplane/charts/runtime/templates/apiserver/vpa.yaml b/charts/gardener/controlplane/charts/runtime/templates/apiserver/vpa.yaml index 016af3911f9..6caa7869b1d 100644 --- a/charts/gardener/controlplane/charts/runtime/templates/apiserver/vpa.yaml +++ b/charts/gardener/controlplane/charts/runtime/templates/apiserver/vpa.yaml @@ -13,7 +13,7 @@ metadata: heritage: "{{ .Release.Service }}" spec: targetRef: - apiVersion: {{ include "deploymentversion" . }} + apiVersion: apps/v1 kind: Deployment name: gardener-apiserver updatePolicy: @@ -24,4 +24,4 @@ spec: minAllowed: memory: 256Mi {{- end }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/charts/gardener/controlplane/charts/runtime/templates/controller-manager/deployment.yaml b/charts/gardener/controlplane/charts/runtime/templates/controller-manager/deployment.yaml index 4f08edc5434..74e3601187e 100644 --- a/charts/gardener/controlplane/charts/runtime/templates/controller-manager/deployment.yaml +++ b/charts/gardener/controlplane/charts/runtime/templates/controller-manager/deployment.yaml @@ -1,5 +1,5 @@ {{- if .Values.global.controller.enabled }} -apiVersion: {{ include "deploymentversion" . }} +apiVersion: apps/v1 kind: Deployment metadata: name: gardener-controller-manager @@ -73,7 +73,7 @@ spec: {{- end }} containers: - name: gardener-controller-manager - image: {{ include "utils-templates.image" .Values.global.controller.image }} + image: {{ include "image" .Values.global.controller.image }} imagePullPolicy: {{ .Values.global.controller.image.pullPolicy }} args: - --config=/etc/gardener-controller-manager/config/config.yaml diff --git a/charts/gardener/controlplane/charts/runtime/templates/controller-manager/poddisruptionbudget.yaml b/charts/gardener/controlplane/charts/runtime/templates/controller-manager/poddisruptionbudget.yaml index dd3c3e2846b..c47edc813a5 100644 --- a/charts/gardener/controlplane/charts/runtime/templates/controller-manager/poddisruptionbudget.yaml +++ b/charts/gardener/controlplane/charts/runtime/templates/controller-manager/poddisruptionbudget.yaml @@ -1,5 +1,5 @@ {{- if gt (int .Values.global.controller.replicaCount) 1 }} -apiVersion: {{ include "poddisruptionbudgetversion" .}} +apiVersion: policy/v1 kind: PodDisruptionBudget metadata: name: gardener-controller-manager diff --git a/charts/gardener/controlplane/charts/runtime/templates/controller-manager/vpa.yaml b/charts/gardener/controlplane/charts/runtime/templates/controller-manager/vpa.yaml index cf6efa33fb5..afd2d6c024c 100644 --- a/charts/gardener/controlplane/charts/runtime/templates/controller-manager/vpa.yaml +++ b/charts/gardener/controlplane/charts/runtime/templates/controller-manager/vpa.yaml @@ -6,7 +6,7 @@ metadata: namespace: garden spec: targetRef: - apiVersion: {{ include "deploymentversion" . }} + apiVersion: apps/v1 kind: Deployment name: gardener-controller-manager updatePolicy: @@ -16,4 +16,4 @@ spec: - containerName: '*' minAllowed: memory: 300Mi -{{- end }} \ No newline at end of file +{{- end }} diff --git a/charts/gardener/controlplane/charts/runtime/templates/priorityclass.yaml b/charts/gardener/controlplane/charts/runtime/templates/priorityclass.yaml index c78c0352791..cc5f5082a11 100644 --- a/charts/gardener/controlplane/charts/runtime/templates/priorityclass.yaml +++ b/charts/gardener/controlplane/charts/runtime/templates/priorityclass.yaml @@ -1,4 +1,4 @@ -apiVersion: {{ include "priorityclassversion" . }} +apiVersion: scheduling.k8s.io/v1 kind: PriorityClass metadata: name: gardener-controlplane diff --git a/charts/gardener/controlplane/charts/runtime/templates/scheduler/deployment.yaml b/charts/gardener/controlplane/charts/runtime/templates/scheduler/deployment.yaml index 935df590e81..35f227807e5 100644 --- a/charts/gardener/controlplane/charts/runtime/templates/scheduler/deployment.yaml +++ b/charts/gardener/controlplane/charts/runtime/templates/scheduler/deployment.yaml @@ -1,5 +1,5 @@ {{- if .Values.global.scheduler.enabled }} -apiVersion: {{ include "deploymentversion" . }} +apiVersion: apps/v1 kind: Deployment metadata: name: gardener-scheduler @@ -69,7 +69,7 @@ spec: {{- end }} containers: - name: gardener-scheduler - image: {{ include "utils-templates.image" .Values.global.scheduler.image }} + image: {{ include "image" .Values.global.scheduler.image }} imagePullPolicy: {{ .Values.global.scheduler.image.pullPolicy }} args: - --config=/etc/gardener-scheduler/config/schedulerconfiguration.yaml diff --git a/charts/gardener/controlplane/charts/runtime/templates/scheduler/poddisruptionbudget.yaml b/charts/gardener/controlplane/charts/runtime/templates/scheduler/poddisruptionbudget.yaml index b9dfbbeb7b7..2a5db226c60 100644 --- a/charts/gardener/controlplane/charts/runtime/templates/scheduler/poddisruptionbudget.yaml +++ b/charts/gardener/controlplane/charts/runtime/templates/scheduler/poddisruptionbudget.yaml @@ -1,5 +1,5 @@ {{- if gt (int .Values.global.scheduler.replicaCount) 1 }} -apiVersion: {{ include "poddisruptionbudgetversion" .}} +apiVersion: policy/v1 kind: PodDisruptionBudget metadata: name: gardener-scheduler diff --git a/charts/gardener/controlplane/charts/runtime/templates/scheduler/vpa.yaml b/charts/gardener/controlplane/charts/runtime/templates/scheduler/vpa.yaml index ea10b3db756..0da66436e37 100644 --- a/charts/gardener/controlplane/charts/runtime/templates/scheduler/vpa.yaml +++ b/charts/gardener/controlplane/charts/runtime/templates/scheduler/vpa.yaml @@ -6,7 +6,7 @@ metadata: namespace: garden spec: targetRef: - apiVersion: {{ include "deploymentversion" . }} + apiVersion: apps/v1 kind: Deployment name: gardener-scheduler updatePolicy: diff --git a/charts/gardener/controlplane/charts/utils-common/Chart.yaml b/charts/gardener/controlplane/charts/utils-common/Chart.yaml deleted file mode 100644 index 25607516b63..00000000000 --- a/charts/gardener/controlplane/charts/utils-common/Chart.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -description: A Helm chart to deploy common resources for the Gardener runtime and application -name: utils-common -version: 0.1.0 diff --git a/charts/gardener/controlplane/charts/utils-common/charts/utils-templates b/charts/gardener/controlplane/charts/utils-common/charts/utils-templates deleted file mode 120000 index fb7e3035e21..00000000000 --- a/charts/gardener/controlplane/charts/utils-common/charts/utils-templates +++ /dev/null @@ -1 +0,0 @@ -../../../../../utils-templates \ No newline at end of file diff --git a/charts/gardener/controlplane/charts/utils-common/templates/_secret-alerting.yaml b/charts/gardener/controlplane/charts/utils-common/templates/_secret-alerting.yaml deleted file mode 100644 index 3155a9326d8..00000000000 --- a/charts/gardener/controlplane/charts/utils-common/templates/_secret-alerting.yaml +++ /dev/null @@ -1,42 +0,0 @@ -{{- define "gardener.secret-alerting" -}} -{{- range $key, $config := .Values.global.alerting }} ---- -apiVersion: v1 -kind: Secret -metadata: - name: alerting-{{ $key }} - namespace: garden - labels: - app: gardener - chart: "{{ $.Chart.Name }}-{{ $.Chart.Version }}" - release: "{{ $.Release.Name }}" - heritage: "{{ $.Release.Service }}" - gardener.cloud/role: alerting -type: Opaque -data: - auth_type: {{ ( required ".alerting[].auth_type is required" $config.auth_type ) | b64enc }} -{{- if eq $config.auth_type "smtp" }} - to: {{ ( required ".alerting[].to is required" $config.to ) | b64enc }} - from: {{ ( required ".alerting[].from is required" $config.from ) | b64enc }} - smarthost: {{ ( required ".alerting[].smarthost is required" $config.smarthost ) | b64enc }} - auth_username: {{ ( required ".alerting[].auth_username is required" $config.auth_username ) | b64enc }} - auth_identity: {{ ( required ".alerting[].auth_identity is required" $config.auth_identity ) | b64enc }} - auth_password: {{ ( required ".alerting[].auth_password is required" $config.auth_password ) | b64enc }} -{{- end }} -{{- if eq $config.auth_type "none" }} - url: {{ ( required ".alerting[].url is required" $config.url ) | b64enc }} -{{- end }} -{{- if eq $config.auth_type "basic" }} - url: {{ ( required ".alerting[].url is required" $config.url ) | b64enc }} - username: {{ ( required ".alerting[].username is required" $config.username ) | b64enc }} - password: {{ ( required ".alerting[].password is required" $config.password ) | b64enc }} -{{- end }} -{{- if eq $config.auth_type "certificate" }} - url: {{ ( required ".alerting[].url is required" $config.url ) | b64enc }} - ca.crt: {{ ( required ".alerting[].ca_crt is required" $config.ca_crt ) | b64enc }} - tls.crt: {{ ( required ".alerting[].tls_crt is required" $config.tls_cert ) | b64enc }} - tls.key: {{ ( required ".alerting[].tls_key is required" $config.tls_key ) | b64enc }} - insecure_skip_verify: {{ ( required ".alerting[].insecure_skip_verify is required" $config.insecure_skip_verify ) | b64enc }} -{{- end }} -{{- end }} -{{- end -}} diff --git a/charts/gardener/controlplane/charts/utils-common/templates/_secret-default-domain.yaml b/charts/gardener/controlplane/charts/utils-common/templates/_secret-default-domain.yaml deleted file mode 100644 index 3f78d960130..00000000000 --- a/charts/gardener/controlplane/charts/utils-common/templates/_secret-default-domain.yaml +++ /dev/null @@ -1,25 +0,0 @@ -{{- define "gardener.secret-default-domain" -}} -{{- range $key, $domain := .Values.global.defaultDomains }} ---- -apiVersion: v1 -kind: Secret -metadata: - name: default-domain-{{ ( required ".defaultDomains[].domain is required" (replace "." "-" $domain.domain) ) }} - namespace: garden - labels: - app: gardener - chart: "{{ $.Chart.Name }}-{{ $.Chart.Version }}" - release: "{{ $.Release.Name }}" - heritage: "{{ $.Release.Service }}" - gardener.cloud/role: default-domain - annotations: - dns.gardener.cloud/provider: {{ ( required ".defaultDomains[].provider is required" $domain.provider ) }} - dns.gardener.cloud/domain: {{ ( required ".defaultDomains[].domain is required" $domain.domain ) }} - {{- if $domain.zone }} - dns.gardener.cloud/zone: {{ $domain.zone }} - {{- end }} -type: Opaque -data: -{{ toYaml $domain.credentials | indent 2 }} -{{- end }} -{{- end -}} diff --git a/charts/gardener/controlplane/charts/utils-common/templates/_secret_internal-domain.yaml b/charts/gardener/controlplane/charts/utils-common/templates/_secret_internal-domain.yaml deleted file mode 100644 index 4fd9ed2e22d..00000000000 --- a/charts/gardener/controlplane/charts/utils-common/templates/_secret_internal-domain.yaml +++ /dev/null @@ -1,23 +0,0 @@ -{{- define "gardener.secret-internal-domain" -}} ---- -apiVersion: v1 -kind: Secret -metadata: - name: internal-domain-{{ ( required ".internalDomain.domain is required" (replace "." "-" .Values.global.internalDomain.domain) ) }} - namespace: garden - labels: - app: gardener - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - gardener.cloud/role: internal-domain - annotations: - dns.gardener.cloud/provider: {{ ( required ".internalDomain.provider is required" .Values.global.internalDomain.provider ) }} - dns.gardener.cloud/domain: {{ ( required ".internalDomain.domain is required" .Values.global.internalDomain.domain ) }} - {{- if .Values.global.internalDomain.zone }} - dns.gardener.cloud/zone: {{ .Values.global.internalDomain.zone }} - {{- end }} -type: Opaque -data: -{{ toYaml .Values.global.internalDomain.credentials | indent 2 }} -{{- end -}} diff --git a/charts/gardener/controlplane/charts/utils-common/templates/_secret_openvpn-diffie-hellman.yaml b/charts/gardener/controlplane/charts/utils-common/templates/_secret_openvpn-diffie-hellman.yaml deleted file mode 100644 index a15e003712f..00000000000 --- a/charts/gardener/controlplane/charts/utils-common/templates/_secret_openvpn-diffie-hellman.yaml +++ /dev/null @@ -1,18 +0,0 @@ -{{- define "gardener.secret-openvpn-diffie-hellman" -}} -{{- if .Values.global.openVPNDiffieHellmanKey }} -apiVersion: v1 -kind: Secret -metadata: - name: openvpn-diffie-hellman-key - namespace: garden - labels: - app: gardener - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - gardener.cloud/role: openvpn-diffie-hellman -type: Opaque -data: - dh2048.pem: {{ .Values.global.openVPNDiffieHellmanKey | b64enc }} -{{- end }} -{{- end -}} diff --git a/charts/gardener/controlplane/charts/utils-common/values.yaml b/charts/gardener/controlplane/charts/utils-common/values.yaml deleted file mode 120000 index ef36be5f38e..00000000000 --- a/charts/gardener/controlplane/charts/utils-common/values.yaml +++ /dev/null @@ -1 +0,0 @@ -../../values.yaml \ No newline at end of file diff --git a/charts/gardener/controlplane/charts/utils-templates b/charts/gardener/controlplane/charts/utils-templates deleted file mode 120000 index 04caed65ef1..00000000000 --- a/charts/gardener/controlplane/charts/utils-templates +++ /dev/null @@ -1 +0,0 @@ -../../../utils-templates \ No newline at end of file diff --git a/charts/gardener/gardenlet/charts/utils-templates b/charts/gardener/gardenlet/charts/utils-templates deleted file mode 120000 index 04caed65ef1..00000000000 --- a/charts/gardener/gardenlet/charts/utils-templates +++ /dev/null @@ -1 +0,0 @@ -../../../utils-templates \ No newline at end of file diff --git a/charts/gardener/gardenlet/templates/_helpers.tpl b/charts/gardener/gardenlet/templates/_helpers.tpl index 1aeca9d978d..3af50eccb73 100644 --- a/charts/gardener/gardenlet/templates/_helpers.tpl +++ b/charts/gardener/gardenlet/templates/_helpers.tpl @@ -1,3 +1,11 @@ +{{- define "image" -}} +{{- if hasPrefix "sha256:" (required "$.tag is required" $.tag) -}} +{{ required "$.repository is required" $.repository }}@{{ required "$.tag is required" $.tag }} +{{- else -}} +{{ required "$.repository is required" $.repository }}:{{ required "$.tag is required" $.tag }} +{{- end -}} +{{- end -}} + {{- define "gardenlet.kubeconfig-garden.data" -}} kubeconfig: {{ .Values.config.gardenClientConnection.kubeconfig | b64enc }} {{- end -}} diff --git a/charts/gardener/gardenlet/templates/deployment.yaml b/charts/gardener/gardenlet/templates/deployment.yaml index 04f52597e53..7ca7c8a6c1e 100644 --- a/charts/gardener/gardenlet/templates/deployment.yaml +++ b/charts/gardener/gardenlet/templates/deployment.yaml @@ -81,7 +81,7 @@ spec: type: RuntimeDefault containers: - name: gardenlet - image: {{ include "utils-templates.image" .Values.image }} + image: {{ include "image" .Values.image }} imagePullPolicy: {{ .Values.image.pullPolicy }} args: - --config=/etc/gardenlet/config/config.yaml diff --git a/charts/gardener/gardenlet/templates/poddisruptionbudget.yaml b/charts/gardener/gardenlet/templates/poddisruptionbudget.yaml index 4eff73a94a1..2df3a2a41cc 100644 --- a/charts/gardener/gardenlet/templates/poddisruptionbudget.yaml +++ b/charts/gardener/gardenlet/templates/poddisruptionbudget.yaml @@ -1,5 +1,5 @@ {{- if gt (int .Values.replicaCount) 1 }} -apiVersion: {{ include "poddisruptionbudgetversion" .}} +apiVersion: policy/v1 kind: PodDisruptionBudget metadata: name: gardenlet diff --git a/charts/gardener/operator/charts/utils-templates b/charts/gardener/operator/charts/utils-templates deleted file mode 120000 index 04caed65ef1..00000000000 --- a/charts/gardener/operator/charts/utils-templates +++ /dev/null @@ -1 +0,0 @@ -../../../utils-templates \ No newline at end of file diff --git a/charts/gardener/operator/templates/_helpers.tpl b/charts/gardener/operator/templates/_helpers.tpl index 29bae6b1fe5..a908c163c5a 100644 --- a/charts/gardener/operator/templates/_helpers.tpl +++ b/charts/gardener/operator/templates/_helpers.tpl @@ -1,3 +1,11 @@ +{{- define "image" -}} +{{- if hasPrefix "sha256:" (required "$.tag is required" $.tag) -}} +{{ required "$.repository is required" $.repository }}@{{ required "$.tag is required" $.tag }} +{{- else -}} +{{ required "$.repository is required" $.repository }}:{{ required "$.tag is required" $.tag }} +{{- end -}} +{{- end -}} + {{- define "operator.kubeconfig.data" -}} kubeconfig: {{ .Values.config.runtimeClientConnection.kubeconfig | b64enc }} {{- end -}} diff --git a/charts/gardener/operator/templates/deployment.yaml b/charts/gardener/operator/templates/deployment.yaml index b20e3152246..76144dba361 100644 --- a/charts/gardener/operator/templates/deployment.yaml +++ b/charts/gardener/operator/templates/deployment.yaml @@ -71,7 +71,7 @@ spec: type: RuntimeDefault containers: - name: gardener-operator - image: {{ include "utils-templates.image" .Values.image }} + image: {{ include "image" .Values.image }} imagePullPolicy: {{ .Values.image.pullPolicy }} args: - --config=/etc/gardener-operator/config/config.yaml diff --git a/charts/gardener/operator/templates/poddisruptionbudget.yaml b/charts/gardener/operator/templates/poddisruptionbudget.yaml index d5fe1810e51..954ca61bae2 100644 --- a/charts/gardener/operator/templates/poddisruptionbudget.yaml +++ b/charts/gardener/operator/templates/poddisruptionbudget.yaml @@ -1,4 +1,4 @@ -apiVersion: {{ include "poddisruptionbudgetversion" .}} +apiVersion: policy/v1 kind: PodDisruptionBudget metadata: name: gardener-operator diff --git a/charts/gardener/provider-local/extension/templates/poddisruptionbudget.yaml b/charts/gardener/provider-local/extension/templates/poddisruptionbudget.yaml index c4ea9081ed9..0e757eb028b 100644 --- a/charts/gardener/provider-local/extension/templates/poddisruptionbudget.yaml +++ b/charts/gardener/provider-local/extension/templates/poddisruptionbudget.yaml @@ -1,5 +1,5 @@ {{- if gt (int .Values.replicaCount) 1 }} -apiVersion: {{ include "poddisruptionbudgetversion" .}} +apiVersion: policy/v1 kind: PodDisruptionBudget metadata: name: {{ include "name" . }} diff --git a/charts/seed-monitoring/charts/alertmanager/charts/utils-templates b/charts/seed-monitoring/charts/alertmanager/charts/utils-templates deleted file mode 120000 index 84502af3c59..00000000000 --- a/charts/seed-monitoring/charts/alertmanager/charts/utils-templates +++ /dev/null @@ -1 +0,0 @@ -../../../../utils-templates \ No newline at end of file diff --git a/charts/seed-monitoring/charts/core/charts/prometheus/charts/utils-templates b/charts/seed-monitoring/charts/core/charts/prometheus/charts/utils-templates deleted file mode 120000 index efc1262a81b..00000000000 --- a/charts/seed-monitoring/charts/core/charts/prometheus/charts/utils-templates +++ /dev/null @@ -1 +0,0 @@ -../../../../../../utils-templates \ No newline at end of file diff --git a/charts/shoot-core/components/charts/monitoring/Chart.yaml b/charts/shoot-core/components/charts/monitoring/Chart.yaml deleted file mode 100644 index 8bfbbb3778b..00000000000 --- a/charts/shoot-core/components/charts/monitoring/Chart.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -description: A Helm chart for monitoring in Shoot clusters -name: monitoring -version: 0.1.0 diff --git a/charts/shoot-core/components/charts/monitoring/charts/prometheus-shoot/Chart.yaml b/charts/shoot-core/components/charts/monitoring/charts/prometheus-shoot/Chart.yaml deleted file mode 100644 index 96da3c49f31..00000000000 --- a/charts/shoot-core/components/charts/monitoring/charts/prometheus-shoot/Chart.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -description: RBAC for prometheus deployed in the shoot cluster -name: prometheus-shoot -version: 0.1.0 diff --git a/charts/shoot-core/components/charts/monitoring/charts/prometheus-shoot/charts/utils-templates b/charts/shoot-core/components/charts/monitoring/charts/prometheus-shoot/charts/utils-templates deleted file mode 120000 index 680a0c961dd..00000000000 --- a/charts/shoot-core/components/charts/monitoring/charts/prometheus-shoot/charts/utils-templates +++ /dev/null @@ -1 +0,0 @@ -../../../../../../../utils-templates \ No newline at end of file diff --git a/charts/shoot-core/components/charts/monitoring/charts/prometheus-shoot/templates/rbac.yaml b/charts/shoot-core/components/charts/monitoring/charts/prometheus-shoot/templates/rbac.yaml deleted file mode 100644 index 1ef62eb0c3f..00000000000 --- a/charts/shoot-core/components/charts/monitoring/charts/prometheus-shoot/templates/rbac.yaml +++ /dev/null @@ -1,43 +0,0 @@ -apiVersion: {{ include "rbacversion" . }} -kind: ClusterRole -metadata: - name: gardener.cloud:monitoring:prometheus -rules: -- apiGroups: [""] - resources: - - nodes - - services - - endpoints - - pods - verbs: - - get - - list - - watch -- apiGroups: [""] - resources: - - nodes/metrics - - pods/log - - nodes/proxy - - services/proxy - - pods/proxy - verbs: - - get -- nonResourceURLs: - - /metrics - verbs: - - get ---- -apiVersion: {{ include "rbacversion" . }} -kind: ClusterRoleBinding -metadata: - name: gardener.cloud:monitoring:prometheus - annotations: - resources.gardener.cloud/delete-on-invalid-update: "true" -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: gardener.cloud:monitoring:prometheus -subjects: -- kind: ServiceAccount - name: prometheus - namespace: kube-system diff --git a/charts/shoot-core/components/charts/monitoring/requirements.yaml b/charts/shoot-core/components/charts/monitoring/requirements.yaml deleted file mode 100644 index df4ca02b688..00000000000 --- a/charts/shoot-core/components/charts/monitoring/requirements.yaml +++ /dev/null @@ -1,4 +0,0 @@ -dependencies: -- name: prometheus-shoot - repository: http://localhost:10191 - version: 0.1.0 diff --git a/charts/shoot-core/components/charts/podsecuritypolicies/charts/utils-templates b/charts/shoot-core/components/charts/podsecuritypolicies/charts/utils-templates deleted file mode 120000 index fb7e3035e21..00000000000 --- a/charts/shoot-core/components/charts/podsecuritypolicies/charts/utils-templates +++ /dev/null @@ -1 +0,0 @@ -../../../../../utils-templates \ No newline at end of file diff --git a/charts/utils-templates/Chart.yaml b/charts/utils-templates/Chart.yaml deleted file mode 100644 index 8cb21eff821..00000000000 --- a/charts/utils-templates/Chart.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -description: Util chart for various templates. -name: utils-templates -version: 0.1.0 diff --git a/charts/utils-templates/README.md b/charts/utils-templates/README.md deleted file mode 100644 index 40abe941a71..00000000000 --- a/charts/utils-templates/README.md +++ /dev/null @@ -1,20 +0,0 @@ -# Important - -To add this chart to another as dependency, execute - -```bash -mkdir -p ./charts/PATH-TO-MY-CHART/charts -ln -sr ./charts/utils-templates ./charts/PATH-TO-MY-CHART/charts/utils-templates -``` - -Then check for broken links with - -``` -find -L charts -type l -``` - -or - -``` -make verify -``` diff --git a/charts/utils-templates/templates/_resources.tpl b/charts/utils-templates/templates/_resources.tpl deleted file mode 100644 index 72c8ee79af5..00000000000 --- a/charts/utils-templates/templates/_resources.tpl +++ /dev/null @@ -1,12 +0,0 @@ -{{/* -util-templates.resource-quantity returns resource quantity based on number of objects (such as nodes, pods etc..), -resource per object, object weight and base resource quantity. -*/}} -{{- define "utils-templates.resource-quantity" -}} -{{- range $resourceKey, $resourceValue := (required "$.resource is required" $.resources) }} -{{ $resourceKey }}: -{{- range $unit, $r := $resourceValue }} - {{ $unit }}: {{ printf "%d%s" ( add $r.base ( mul ( div $.objectCount $r.weight ) $r.perObject $r.weight ) ) $r.unit }} -{{- end -}} -{{- end -}} -{{- end -}} diff --git a/charts/utils-templates/templates/_versions.tpl b/charts/utils-templates/templates/_versions.tpl deleted file mode 100644 index 70564f58210..00000000000 --- a/charts/utils-templates/templates/_versions.tpl +++ /dev/null @@ -1,63 +0,0 @@ -{{- define "apiserverversion" -}} -apiserver.k8s.io/v1alpha1 -{{- end -}} - -{{- define "auditkubernetesversion" -}} -audit.k8s.io/v1 -{{- end -}} - -{{- define "rbacversion" -}} -rbac.authorization.k8s.io/v1 -{{- end -}} - -{{- define "deploymentversion" -}} -apps/v1 -{{- end -}} - -{{- define "daemonsetversion" -}} -apps/v1 -{{- end -}} - -{{- define "statefulsetversion" -}} -apps/v1 -{{- end -}} - -{{- define "apiserviceversion" -}} -apiregistration.k8s.io/v1 -{{- end -}} - -{{- define "networkpolicyversion" -}} -networking.k8s.io/v1 -{{- end -}} - -{{- define "priorityclassversion" -}} -scheduling.k8s.io/v1 -{{- end -}} - -{{- define "cronjobversion" -}} -batch/v1 -{{- end -}} - -{{- define "hpaversion" -}} -{{- if semverCompare ">= 1.23-0" .Capabilities.KubeVersion.GitVersion -}} -autoscaling/v2 -{{- else -}} -autoscaling/v2beta1 -{{- end -}} -{{- end -}} - -{{- define "webhookadmissionregistration" -}} -admissionregistration.k8s.io/v1 -{{- end -}} - -{{- define "poddisruptionbudgetversion" -}} -policy/v1 -{{- end -}} - -{{- define "podsecuritypolicyversion" -}} -policy/v1beta1 -{{- end -}} - -{{- define "ingressversion" -}} -networking.k8s.io/v1 -{{- end -}} diff --git a/example/provider-local/garden/base/controller-registration.yaml b/example/provider-local/garden/base/controller-registration.yaml index 17e5391f6ed..e685706506a 100644 --- a/example/provider-local/garden/base/controller-registration.yaml +++ b/example/provider-local/garden/base/controller-registration.yaml @@ -5,7 +5,7 @@ metadata: name: provider-local type: helm providerConfig: - chart: H4sIAAAAAAAAA+09a3PbtrL9rF+BUe6dPE5IibIkpzyTe8ex3dZzYsdjO8l0ml4PRUISa4rk4UOOmva/n108SPAhUZITJb0Vmo5FAlgsgMViXwAnVuRQn0Ya/ZhQP3YDXwujYO468MoLbMvrfPfg1IV0OBiwv5DKf9lv46Bv9Aa94RDfG8PeYf87Mnh4080pjRMrIuS7KAiSVeWa8v+iadI4/8dTK0r0hTXztm0DJ3jY7y+d/95Bvzj/vW6v1/2OdD9nR5elv/n8W6H7jkY47yaZGy0rDLPHtqF32y2Hxnbkhgl7dUR+ot6M2EgSZBxEJJlS8qMgIfIa6YVcCvIhGUW1fGtGTdJIaq25bLqrQ9utrz02f4fUvP6dwNYnwUPaaFj/vV5/WOL/hwfI//fr/8unToccB+EicifThADfNcj10SW5PiWwuC2fPVjjseu5VkKJHcxCy1/o5MjzCKsSk4jGNJpTRyc3UzcmUJQS+Ou5NpATdUjqIy9APnEUWjb8uQ7Gyb0VUfKaF3lO5jrpAbewaZgQKyZ+kEC9AKpE924M0HxW/fXZ8ekFIIYttDod+Cch1DSSwRbcjPT0LnmCBdoiq/30nwhiEaRkZi2wUZJCY0nWCYEQtI7dhgHwbUru3WTKseFQdITxs4ARjBILiltQIYSnsVqQWIlAmqVpkoRmp3N/f69bDGM9iCYdMWhxR/RVA6xFrbe+R2Mc7X+nbgQ9Hi0I8GqoYI0AV8+6ZxM2iSjkJQFifR+5ietPnpNYDDiCcdw4idxRmhQGTeIIXVcLwLABCbSPrsnZdZu8Oro+u36OQN6f3fz05u0NeX90dXV0cXN2ek3eXJHjNxcnZzdnby7g6QdydPEz+dfZxclzQl2cSRjOMMIeAJouDidQDMK6prSAgtxU4pDa7ti1oWv+JLUmlEwC2B586BEJaTRzY5zWGBB0EIznztzEStirSr/0FhSZBOYEGR3Ssa53sn9Ty77ryBzNDvwkCjwPuF9EJzgWDKgeT4mmhYGjxdROYWAXGvUBVZu+DCN3DuQygfEqMk6ik3lXh/8KzdGPFvSedkpMlrPhzsiKaWcZDiiCkVeAbhq+Su07mpi8Hf7qFCotxJuTi+srageRI56POcRLGEsqXp3548gC0KmdpJF8+SbEUYAhvl7ECZ1BtbE7EXnvg+iORvwBx5NcQrM4L1wYoD4SYkzUYY7TMAyEoCBe4vThzAByEbUTkveVFPraClXoe0ngi6Xm/R8IIfRgOuOtNcGN9T/Yhw57e/1vF2mT+b+dUg84RKwn4Ua6YJP+Nxz0SvPfGxqHe/lvF+nTJ404dOz6IBmhmtYm2p9/tppVNaxHfYeVbqlAPGtEvRikmlC/owsOjj2kI9i+KdCR7gYdbKoAYwmIueWlAqdPn0Cqsb3UyTDViai4ApFq3TKCCMUkS0qI9llL1V64PhAPiIWsun5FPQr7t34ByK3EDOQIELOilCnVo9SZ0ERovhzbMAC5btEBdVwF8WXmf5P1bzN5YGaFmjuDvXkOG3gQaSiWoaxJl9qImtZ/f3hQWv+H/cHBfv3vIiGFuWOiv0NCB6LGiX3HJvaNnFdO+wUz0Z3rOybh8uG5FbZmNLEcK7HMFiHc2FO/WusJR1SKQearWUrsNeJACF+YZs1yRvB/wEug5YT0sbREh7UY3xap1CR/IJCVvS6Ckyvxiy3Er5S2W/9eMJmAKL+mVbjJ/mN0D0vrvz/oD/brfxfpcy1sQREap5HiiuY0pqxftIBk9BbrkgZ12wtSR5BZyjVBkwjI2YIeu14CxKnsxFgBVzSC/eWHs9c3p1e/tohIbDcupdCKYhplRc6txJ6Wiijgn80sewobt2ocmFk+sIzoGVsi2ooCWSP/oovbMi7QtSz/kuFURQImiBnYossi0lfc7nZ7AsMi391EKW19jWFYnlXiJt/0eMwCxx0v1h6PrCAsF8dFYlUKYveY8Hg7QzA0Jok1If+X1/8AsmX0QYcuf0ACRsshrMMP+oqh1P/xv7f4/5Mmqvtj7fl4mnXiyHHK40Fub+XSvJ2lXgK7om/5ya3r3N6SgNmKguifKc7C12Zj+7Rl2mz/j6jjb24H2tz+Yxz2Dvf2n12kbeY/kwM/k/x32CvPf38w2Nt/dpJK+p+YYp2b8x1hvdlQRBRA6oTAFYSmyWoS2DE8oydOina6+X13cEA+ZVvW3KX3TI6MfMtT3hPmbEJ3TBI8eUpeviSPEzt8nOX/mf2iURREcfY4pZaXTAuQQGadUSe178ggrgEQUcvJJQZ3xvwdNtBUMOs801HpjKCHWQFACYZrStOYmN8bg4O85SBO4kLDinqa7+KxjkKOa1MdK5xdguRN3BhEDw0kZHSvTWCh3lsLvfBSj+c2yNaAFcjY3Hy3dju/Bz7trtuQpnUf2paxflvGQ9vqrd9Wb0VbOTWMg+geyBw0og5N7A5UDbw5U0+KNDWzPt7CWzuNIjQwGMAEa6DZzJN9kGd5QRAqDxOFCr3AcpQsyxlZHlomWznIyvLZNfFHlBtWID8sNINcAt1/9CN5oj97+kEXw/6BD/QHmDTqZA+SkXzg6iLxQTv8mM1UPteiYHXiiOXH96BaWGkS1PRq6Rr9Zmf3IVbirfZ/NsXa2mJAk/13MCzH/w27h3v7707SF9z/NU4n24oBBNapH4igDm4zauOaSyObVmxG7sSHam2TtBPQ+duZDGFPLX9CZzTjRlDiEbyl9l2QJiwUwQnsGENdpsE9Rs6AOmuy8JzY7HQmbjJNRzAqs45sMP8x8oJRZ2YhY8EgubiTxqBhy9XBBoANkj5z2jvw5GyXtln/Dg29YDED5raeAtCw/vvDg7L/d4DZ+/W/g7Th+rfCMO5kTOAkI4QCF1DNvFChrBA8WEHA2DCEHlEWABebxGBPcxdr/QSiWxAtXmNEmEl6kBNTj/l2OEbMHvdaQbGCJItCopMFz0ZpAqSLtyF0j8oasMdfp9EE0O4N/jt/99a35pbr4dhxpAhJFiH8vlKBwHu5qARKyuAxfMqMjwkMyLTidJar3wVj/JMwckHa+C/9RoDWX1kxvbRApGsv0dzbT5nXLJ5avcEQIHMnG5MxCsNTGSBC5BSwTJCjZkHqJ9dctj6ybXy6Ce4oUMzY8mIpA+Z2TllXI1Y0URrSCGOayjOTtST+UiVUhE5rkhOULGcautHVu8VSl6nnXTLPuknOxhdBcomhq0C6mWDlzoH+4vgyCkY0RwkEP5jSNKI3U6gwDTwg/IGSi3vFjzRRK6BpPZmapMNl6mIOCMkmedF90S28jmF6cWn8dHNzqWS4vpu4lndCPWtxTWFoHKD2oVo1pJEbOFmeoebFqQ17Zawgbii5iTujsAlmVfNOlVdpjnlhqrLpvGR9QvWiJNWbRAVAuEpuB55J3p5cbg5JAzW+FtrN8SpoRg00WHKRa9fjpkJDHcddkyoO1qSKot4khxaIwnhhrEMUn2/KDaWfQrJSMWZRtXGxDzM6A+4KVQ+7566Sg5HJNC6XtsMUEezOamEUQMjIWgxWhS1AhWN5XnB/KQNtT2PYDIRzUOUurD0rtEauB4uGljBxoiAsvtEQbqEHlvPG9xZXsNP/AC3FLAjWJChR5janwEtn9BwZXGEtMAZ4yedX4VgVuuMcWONgahovNbcUcCcTrkvwVd1szWb8wKHXhT0SUzHUKQBqgR0s/diSgFBuSiRDPfLurYXsLW9V4fG2VBbUCXDo2Eq95DxAobzfU4nYhXEvTJ9G7ujCJBX2j4kvqpqsei62fBo2x3OJuiNTwAK8LK80pysmavdawjbyfya0fR77f7dnlOX//qDf28v/u0jb6v9ZZFaN/r++JP+tasV/n7TV+qfJfRDd8TDVNXhAw/o3uhX/32Bg7OO/dpLUxS3mFQP77l6wXT9f7jyLb/c1S55JaFoSLDfmVePBpA5JmeUef2kkCfjWq+Xlq5JJjQZfFVgkjvqmPElIFShgr9lwRTtWNCVNyPaKSlNVNFYW4ppSBZ8aXJT5K5lHYWJQLebTRB0GD2fyZhFSMfKnbBb2XPhvlrbh/9KHuu6dEE3y37BvlOW/3mB//nsnaVv5T9j6vordl5tUL0AhQwNPxcBbajpjxprAQhqROM8VHLfKlIEsJjQpGaR80ahJDrr8nYSaOiuhSnvXWlB3KBZvsv439fvItHr9G/0DEPaK6//gYK//7SZt7NxZdQqgtd5BHiTvwqEj5ry9khZIDqjiA2nw/WauX+WkzmYnhrCRqTuZasKHg2bEhfThLm2dMyMekl7jmVLCkMTLYzTo8fa2clk1dGXIQW/hYFrzKJgsXvVH1Z/u2sRL1Xy+cLnPincgm/lCnyx4K/slLP9ii7u2I1gC/kStlEcAoSIRY4mcvOoL8VXxuMg684jDR9KKKP0OiGcY4FCw+0XyCx+ANtxZXcAgtsL3lrZCUwIcbh7Qg/byoSh79FbqCgILLTtroWoOm+saa9RAlqOJEiurLl+HgQbVtHgKfDpmp4Ny9NFvpPX7BznkhkX0Qg6c6ucE0g2Yg8Kz4viiqFVyX4H2fRbqFRd8oRermSZbTjW+0QZey5NwgZaPM1aKqP7PcuE8T61Wcc1qwMyRuhlzQlFBzgbS+91LBSoveCrKZbP2GoqpLSBMQf0hXk6izayPWh47B/DxAX0xL5fEc6q19bzm9cK343JLQJYRuxZl82ayqqvaeETe8zt3MI6IOWt4WK+L8UXeAkOLHHYZCg9DdIDDeUGIO+xzzGMX/NxPqU9g/bGLboA3gN6vKw28cn0rWhDb8h8nhLNkaAsKPY7VFq38Iqbg3ufXBaEAQ0ZpgjzFZncJpXh/DgifMI+Qpy8ZK9YKmnYYaA3hviy63bA893SzPWHz0VUqN82hW7i3ZvOmivUbW2OihcbPOiHB5xvn0gZYlTeyxlFWoQz7nl2os3kPeL0mzAN5lQ/nTMLNtHFjtWCa54jH4m4xOTxevQG+4K2bw5eGizXhI8FrbtgELQvFrwWCgeZadw0weaT9cjjGmnCMBji9NeH0qnBG7LapEbuASujnjhstA6aW5tHXJ260EmS2D2rM862he3ct6FnFc+kxL7cDbCZKRtRKtExLebnitoG6ikAz9B4IPIFBgp7HPJBiBV/j9XRW70xUE+EXFYZARyC83EmX8IY4lmrL6UaRUUVPFOPhu3KuhfjYBA9PjzaC41E3la3XjVHaVsLzC4Mmso/zXJDFfgP5mLSft5fBEm3XAXovspZAKWk6RSG60JbI0kYon1qOg+zppblUAF9DFZEV+Z5X1ybPWd5kfc3mFjOBWVw2U243E2ZFvjquq+pu0DL/4cFqHcOqgA34RzQs6ef8uHI+/ef8sHJugqjHFQCJc82w2dizWoQ/S5M1XWTicDABkRikuILoG0xe4zugPan0tUHqCNqkrjqIg6Ddl+r/wF6qAH6L8XIgFQD156pszlWF16dHJ6dXt6evT4/xJsbbi6Pz0+vLo+NTxVvFTqX/AHplMRpq7FLPuaLj4lvxnscfFVxq3Pm/ZMKbbAcS37Pzox9P3wGyb65u37w7vXp/dXZTwdUkHXYLoHKJS6f2VpdV07U0uLM5fPP3YhbTwRuXI6ZNYjqVIL7lIYeNQYU7QTWPE61EhPJZlVuHMIrl4EqBmRvuI2UjfYnmoqL9UI5lFtYIrSXBz3iHZrXGH8QXmr/RLd32IyHVB/5tSffLrkGSSQ36q9C+Um5J1GAtu+Itc5GJSVX17W0nyC0N3gQl8yh+G6MJqQv66wXeT8s0UfUSV1BhbRBvfKa0xuSJDf1KuNYKOq0jBES0NsCAUQLKZ0DmrsUB4f0TT1UdGfbM8lWsUuWFhlidSRSkIT4xexCxWMysvJuUV1E7jNfiRmy28rBdvNzUo7hRWN6PCK6wEORqLoVCbkQu6xFLbcziVndwybQk4LFCVKtICnUiRk/q6l2bvlaL8tIBeCLn5E10zAhmH7ywedrE/7dp3JdMTfFfNfGf3eH+/OdO0ueO/1IN+Wu6A2vcUSXbfvEbFNhULKLGyBgEWbR/xtzDAZsGu6L98drehpfCJSCdNXxTUMzx8v55rISHRe9oJEdK3Hlvw841orl5V9zxzX4z6LizCHwBPALLrkRjLJyfTud9YZefM1GLtyiPrme+xQdHgi3ztdTH4bnhKzTuK+fQXCfC4x893XiB15t3jOHqomOn2zWNrml2hv3WPvLs20qb8P+ae2vX2gUa47+65fP/g338146SkEgnCXmC3ui6WIWnxChfAatKlqtuM9ahHt9ALgPnJCv1ipX63GElm0V7SHZaPrULdeN01DgaDw7R+EYulN1k/UcgpG/zIbCm8/+DbuX+9+5wuF//u0iaphVWNptjK02mQeT+zr/FUZYFj7lEchV4dYd/li7hTRZnlHooHWgEUMuVa235pbGt0glWrai/V990YNqTVM2AxiOX1rxRiwppTDwosQc1b9R6OeKlR7VQ0Tdd+04tXuubXZWlVuZe5MLvPBv490gMI/Jp/Ou5Mf9xj4yO/QqzXym/1uAh05XFG8Slx1qsuIHo4Qguiyaq4sfdDE72tohP+1m7CtwOAH/XVxdRFS7bxzbu3QPbku/Z9snzSiHXpUCfIn7LRvPBSNRsRpkLtZ4sl6HSbrM/GNEqFnRGFFyFZbO/jNfxuo6wTxY+TKQWCF2FwJWMih5dHYVMnolLj50xDKDn/i7XJp0DF+I/QSuMJC/LQiVFKRFNKBZP5sOqPHfw7DwV4JjvNy48qCwCo9KVX2qWKG7x6LYCawT2Rysv0JeJ3IgPe1aikvVbMIrFLDn5jw6/DQ0WYZowria8BYWLwkWb7FS1HGz2wQ03z1UNWG7GYJn6SbPmysLsOktd3Lesx1bI+EftnGPNZlB4mwpO0irqARgJyKMeYCuLc3opA3/Q5v6Kz8wX2+OhCeFvlEOwAkMoVZU+GvABOf434F9MkCicWhFRmZ9H7fjaEtzD0iby/6bnvmRqkP97RuX7L73h/vu/u0lrH+76LAp6xdC7TmQ3mkUxWtJz2AFj5nMmj3/51Jbu4LbZvjm+bD9vY17bXM+t/OevjzfDgEWXU+po/ECAxk9yxZowoBYQK+NRjBJ6XsZ8K1ykl30VHrsaoGz2NWkVERgoZhFot73iiDbgyIG2YYAqFUvt4d4BFViwf3sjTPEbnmxHyjDNA+1gCF08s6Moa2sdVdrB2aTi0UexB55hqEZuhlrvYJByHLISH9IYHLj6lOSa8Rxfm+Xtk5K22P+F1L++GNBk/z/ols9/9o1DY7//7yKt2v+lmPxV7fTr3C75tQfxL5w2Wf/z0NrG/N8Y/2EMy+vfOOzvv/+xk1SKSMMpFnc/lFx+7aphojM32oJjvBMWicvAOcosEut/PQ6aXZN5SEGoBm8phKkHGYvvuM6hRKFyO5AMdcgy+HHNx89kTMjM9Y+4dK3cBSTusARM8epLN6IOaS9HR89h6LwqHs6TNdsrOlJTk3WtIEtyMSwL36471Y/vKyf7m4IFoQA3saoDyN/w+EBF5kPU1cJ6Xu4bcnjuUyE18/85n91tGL9IDfz/oN8r83/2SeA9/99BEqfHaapP7Ah5ekYQQAZoPc2/d6A4LYskYjLxIGmtvma7pUZR4NXDLXlSxkQvZ9DKDr6YBI+7YAVp9Saf/mwBe2G3xfHdKYt4r+PwVX4tufWwz24cXs3T2riFtVstJT4ZC6nO3WwnUc+R8qMRmetyRZmiV3dFQe6aXVGg1tO7suUsym9JCaHlrSjBw7vPLmGkjN4hxgDqBr96QpxhVTO6eY5RzFHq9Io5PcxR3fUcGXnsAIp2HDrvcFdlwa3fLu7w5/m5hjY7Hp8R85K6ykn0FQOQOSXlzTTVw50m/7TP0tPgUnVpVY8+muSXX1ulg4zsXcn4lIF4ROoOEuBn0x4R+QFsk/2WIfuhlcY0P2rL8mDmGdwrGgaxm7Dlonz/JDfdqT/Vb6CMUtdzOgx054TFybILmTnsSIUqOE0QTDx6m0fa8rqaNXOGfVEtsSYwdQc6UBF/MZdijQEEpH/8a/fKqPSq/T8vsWc9nqHreqtVMKWZ+QLlNrd+/0C8kqeqDPx8SqulmJzF5wby01/sBTLgwi0dpsJN8aIOU/rqW48eQQsJ4zIsqFnAfk6oPtFJLK/QGS0IM5Dnl9ZATYkIQgc4nH1n1+DIuiwz4+7yUh7B5Fvysw4V9t+S69lsqYOIKzk798nXqHr4k20nhOFqSrMm285MzmL28uk+7dM+7dM+7dM+7dM+/X9N/wFHzkvIAKAAAA== + chart: H4sIAAAAAAAAA+09a3Pbtpb9rF+BUXYnjw0pUZbklHeyO47ttp4bP8Z2kuk0dz0UCUmsKZKXDzlq2v++5+BBgg+Jkpwo7VZoOhYJ4OAAODg4L4ATK3KoTyONfkqoH7uBr4VRMHcdeOUFtuV1vnt06kI6HAzYX0jlv+y3cdA3eoPecIjvjWHvsP8dGTy+6eaUxokVEfJdFATJqnJN+X/RNGmc/+OpFSX6wpp527aBEzzs95fOf++gX5z/XrfX635Hul+yo8vS33z+rdB9TyOcd5PMjZYVhtlj29C77ZZDYztyw4S9OiI/UW9GbCQJMg4ikkwp+VGQEHmL9EKuBPmQjKJavjWjJmkktdZcNt3Voe3Wtx6bv0NqXv9OYOuT4DFtNKz/Xq8/LPH/wwPk//v1//VTp0OOg3ARuZNpQoDvGuTm6IrcnBJY3JbPHqzx2PVcK6HEDmah5S90cuR5hFWJSURjGs2po5PbqRsTKEoJ/PVcG8iJOiT1kRcgnzgKLRv+3ATj5MGKKHnLi7wkc530gFvYNEyIFRM/SKBeAFWiBzcGaD6r/vbs+PQCEMMWWp0O/JMQahrJYAtuRnp6lzzDAm2R1X7+DwSxCFIysxbYKEmhsSTrhEAIWsduwwD4NiUPbjLl2HAoOsL4WcAIRokFxS2oEMLTWC1IrEQgzdI0SUKz03l4eNAthrEeRJOOGLS4I/qqAdai1jvfozGO9r9TN4IejxYEeDVUsEaAq2c9sAmbRBTykgCxfojcxPUnL0ksBhzBOG6cRO4oTQqDJnGErqsFYNiABNpHN+Tspk3eHN2c3bxEIB/Obn+6fHdLPhxdXx9d3J6d3pDLa3J8eXFydnt2eQFPP5Cji5/JP88uTl4S6uJMwnCGEfYA0HRxOIFiENYNpQUU5KYSh9R2x64NXfMnqTWhZBLA9uBDj0hIo5kb47TGgKCDYDx35iZWwl5V+qW3oMgkMCfI6JCOdb2T/Zta9n1H5mh24CdR4HnA/SI6wbFgQPV4SjQtDBwtpnYKA7vQqA+o2vR1GLlzIJcJjFeRcRKdzLs6/Fdojn6yoPe0U2KynA13RlZMO8twQBGMvAF00/BNat/TxOTt8FenUGkh3pxc3FxTO4gc8XzMIV7BWFLx6swfRxaATu0kjeTLyxBHAYb4ZhEndAbVxu5E5H0Ionsa8QccT3IFzeK8cGGA+kiIMVGHOU7DMBCCgniJ04czA8hF1E5I3ldS6GsrVKHvJYGvlpr3fyCE0IPpjLfWBDfW/2AfOuzt9b9dpE3m/25KPeAQsZ6EG+mCTfrfcNArzX9vaBzu5b9dpM+fNeLQseuDZIRqWptof/zRalbVsB71HVa6pQLxrBH1YpBqQv2eLjg49pCOYPumQEe6G3SwqQKMJSDmlpcKnD5/BqnG9lInw1QnouIKRKp1ywgiFJMsKSHaZy1Ve+H6QDwgFrLq+jX1KOzf+gUgtxIzkCNAzIpSplSPUmdCE6H5cmzDAOS6RQfUcRXE15n/Tda/zeSBmRVq7gz25jls4EGkoViGsiZdaiNqWv/94UFp/R/2Bwf79b+LhBTmjon+HgkdiBon9j2b2Es5r5z2C2aie9d3TMLlw3MrbM1oYjlWYpktQrixp3611hOOqBSDzFezlNhrxIEQvjDNmuWM4H+Hl0DLCeljaYkOazG+K1KpSX5HICt7XQQnV+JXW4jfKG23/r1gMgFRfk2rcJP9x+geltZ/f9Af7Nf/LtKXWtiCIjROI8UVzWlMWb9oAcnoLdYlDeq2F6SOILOUa4ImEZCzBT12vQSIU9mJsQKuaAT7yw9nb29Pr//VIiKx3biUQiuKaZQVObcSe1oqooB/MbPsKWzcqnFgZvnAMqIXbIloKwpkjfyTLu7KuEDXsvwrhlMVCZggZmCLropIX3O7290JDIt8dxultPUthmF5Vomb/KnHYxY47nix9nhkBWG5OC4Sq1IQu8eEx7sZgqExSawJ+d+8/keQLaOPOnT5IxIwWg5hHX7UVwyl/l//c4f/P2uiut/Xno/nWSeOHKc8HuTuTi7Nu1nqJbAr+paf3LnO3R0JmK0oiP6R4ix8aza2T1umzfb/iDr+5nagze0/xmHvcG//2UXaZv4zOfALyX+HvfL89weDvf1nJ6mk/4kp1rk53xHWmw1FRAGkTghcQWiarCaBHcMzeuKkaKeb33cHB+RztmXNXfrA5MjItzzlPWHOJnTHJMGz5+T1a/I0scOnWf4f2S8aRUEUZ49TannJtAAJZNYZdVL7ngziGgARtZxcYnBnzN9hA00Fs84LHZXOCHqYFQCUYLimNI2J+b0xOMhbDuIkLjSsqKf5Lh7rKOS4NtWxwtkVSN7EjUH00EBCRvfaBBbqg7XQCy/1eG6DbA1YgYzNzXdrt/Nb4NPuug1pWvexbRnrt2U8tq3e+m31VrSVU8M4iB6AzEEj6tDE7kDVwJsz9aRIUzPr0x28tdMoQgODAUywBprNPNkHeZYXBKHyMFGo0AssR8mynJHloWWylYOsLJ9dE39EuWEF8sNCM8gl0P1HP5Fn+ovnH3Ux7B/5QH+ESaNO9iAZyUeuLhIftMNP2Uzlcy0KVieOWH78AKqFlSZBTa+WrtE/7ew+xkq81f7PplhbWwxosv8OhuX4v2H3cG//3Un6ivu/xulkWzGAwDr1AxHUwW1GbVxzaWTTis3InfhQrW2SdgI6fzuTIeyp5U/ojGbcCEo8gbfUvg/ShIUiOIEdY6jLNHjAyBlQZ00WnhObnc7ETabpCEZl1pEN5j9GXjDqzCxkLBgkF3fSGDRsuTrYALBB0mdOeweenO3SNuvfoaEXLGbA3NZTABrWf394UPb/DjB7v/53kDZc/1YYxp2MCZxkhFDgAqqZFyqUFYJHKwgYG4bQI8oC4GKTGOxp7mKtn0B0C6LFW4wIM0kPcmLqMd8Ox4jZ494qKFaQZFFIdLLg2ShNgHTxLoTuUVkD9vibNJoA2r3Bf+bv3vnW3HI9HDuOFCHJIoTf1yoQeC8XlUBJGTyGT5nxMYEBmVacznL1u2CMfxZGLkgb/6HfCtD6GyumVxaIdO0lmnv7OfOaxVOrNxgCZO5kYzJGYXgqA0SInAKWCXLULEj95IbL1ke2jU+3wT0FihlbXixlwNzOKetqxIomSkMaYUxTeWaylsRfqoSK0GlNcoKS5UxDN7p6t1jqKvW8K+ZZN8nZ+CJIrjB0FUg3E6zcOdBfHF9FwYjmKIHgB1OaRvR2ChWmgQeEP1Byca/4kSZqBTStJ1OTdLhMXcwBIdkkr7qvuoXXMUwvLo2fbm+vlAzXdxPX8k6oZy1uKAyNA9Q+VKuGNHIDJ8sz1Lw4tWGvjBXEDSU3cWcUNsGsat6p8irNMS9MVTadV6xPqF6UpHqTqAAIV8ntwDPJu5OrzSFpoMbXQrs9XgXNqIEGSy5y7XrcVGio47hrUsXBmlRR1Jvk0AJRGK+MdYjiy025ofRTSFYqxiyqNi72YUZnwF2h6mH33FVyMDKZxuXSdpgigt1ZLYwCCBlZi8GqsAWocCzPCx6uZKDtaQybgXAOqtyFtWeF1sj1YNHQEiZOFITFNxrCLfTAci59b3ENO/0P0FLMgmBNghJlbnMKvHRGz5HBFdYCY4BXfH4VjlWhO86BNQ6mpvFSc0sBdzLhugRf1c3WbMYPHHpT2CMxFUOdAqAW2MHSTy0JCOWmRDLUI+/BWsje8lYVHm9LZUGdAIeOrdRLzgMUyvs9lYhdGPfC9Gnkni5MUmH/mPiiqsmq52LLp2FzPJeoOzIFLMDL8kpzumKidq8lbCP/Z0Lbl7H/d3tGWf7vD/q9vfy/i7St/p9FZtXo/+tL8n9Wrfjvk7Za/zR5CKJ7Hqa6Bg9oWP9Gt+L/GwyMffzXTpK6uMW8YmDf/Su26+fLnWfx7b5myTMJTUuC5ca8ajyY1CEps9zjL40kAd96tbx8VTKp0eCrAovEUd+UJwmpAgXsNRuuaMeKpqQJ2V5RaaqKxspCXFOq4FODizJ/JfMoTAyqxXyaqMPg4UzeLkIqRv6UzcKeC//N0jb8X/pQ170Tokn+G/aNsvzXG+zPf+8kbSv/CVvfN7H7cpPqBShkaOCpGHhLTWfMWBNYSCMS57mC41aZMpDFhCYlg5QvGjXJQZe/k1BTZyVUae9aC+oOxeJN1v+mfh+ZVq9/o38Awl5x/R8c7PW/3aSNnTurTgG01jvIg+RdOHTEnLfX0gLJAVV8IA2+38z1q5zU2ezEEDYydSdTTfhw0Iy4kD7cpa1zZsRD0ms8U0oYknh5jAY93t5WLquGrgw56C0cTGseBZPFq/6o+tNdm3ipms8XLvdZ8Q5kM1/okwVvZb+E5V9scTd2BEvAn6iV8gggVCRiLJGTV30hviqeFllnHnH4RFoRpd8B8QwDHAp2v0h+4QPQhjurCxjEVvje0lZoSoDDzQN60F4+FGWP3kpdQWChZWctVM1hc11jjRrIcjRRYmXV5esw0KCaFk+BT8fsdFCOPvqNtH7/IIfcsIheyYFT/ZxAugFzUHhWHF8UtUruK9C+z0K94oIv9GI102TLqcY32sBreRIu0PJxxkoR1f9ZLpznqdUqrlkNmDlSN2NOKCrI2UB6v3+tQOUFT0W5bNbeQjG1BYQpqD/Ey0m0mfVJy2PnAD4+oC/m9ZJ4TrW2nte8Wfh2XG4JyDJi16Js3kxWdVUbT8gHfucOxhExZw0P63UxvshbYGiRwy5D4WGIDnA4Lwhxh32JeeyCn4cp9QmsP3bRDfAG0Pt1pYE3rm9FC2Jb/tOEcJYMbUGhp7HaopVfxBQ8+Py6IBRgyChNkKfY7C6hFO/PAeET5hHy9CVjxVpB0w4DrSHc10W3G5bnnm62J2w+ukrlpjl0C/fWbN5UsX5ja0y00PhZJyT4fONc2gCrcilrHGUVyrAf2IU6m/eA12vCPJBX+XDOJNxMGzdWC6Z5jngs7haTw+PVG+AL3ro5fGm4WBM+Erzmhk3QslD8WiAYaK511wCTR9ovh2OsCcdogNNbE06vCmfEbpsasQuohH7uuNEyYGppHn194kYrQWb7oMY83xq6d9eCnlU8lx7zcjvAZqJkRK1Ey7SU1ytuG6irCDRDH4DAExgk6HnMAylW8DVeT2f1zkQ1EX5RYQh0BMLLvXQJb4hjqbacbhQZVfREMR6+K+daiI9N8PD0aCM4HnVT2XrdGKVtJTy/MGgi+zjPBVnsV5CPSftlexks0XYdoA8iawmUkqZTFKILbYksbYTyqeU4yJ5em0sF8DVUEVmR73l1bfKc5U3W12xuMROYxWUz5XYzYVbkq+O6qu4GLfMfHqzWMawK2IB/RMOSfs6PK+fTf84PK+cmiHpcAZA41wybjT2rRfiLNFnTRSYOBxMQiUGKK4i+weQtvgPak0pfG6SOoE3qqoM4CNp9qf4P7KUK4NcYLwdSAVB/rsrmXFV4e3p0cnp9d/r29BhvYry7ODo/vbk6Oj5VvFXsVPoPoFcWo6HGLvWcazouvhXvefxRwaXGnf9LJrzJdiDxPTs/+vH0PSB7eX13+f70+sP12W0FV5N02C2AyiUundpbXVZN19Lgzubwzd+KWUwHb1yOmDaJ6VSC+JaHHDYGFe4E1TxOtBIRymdVbh3CKJaDKwVmbriPlI30JZqLivZDOZZZWCO0lgQ/4x2a1Rq/E19o/ka3dNuPhFQf+Lcl3S+7BkkmNeivQvtKuSVRg7XsirfMRSYmVdW3t50gtzR4E5TMo/hdjCakLuivF3g/LdNE1UtcQYW1QbzxmdIak2c29CvhWivotI4QENHaAANGCSifAZm7FgeE9088V3Vk2DPLV7FKlRcaYnUmUZCG+MTsQcRiMbPyblJeRe0wXosbsdnKw3bxclOP4kZheT8iuMJCkKu5FAq5EbmsRyy1MYtb3cEl05KAxwpRrSIp1IkYPamrd236Wi3KSwfgiZyTy+iYEcw+eGHztIn/b9O4L5ma4r9q4j+7w/35z52kLx3/pRry13QH1rijSrb94jcosKlYRI2RMQiyaP+MuYcDNg12RfvTtb0Nr4VLQDpr+KagmOPl/fNYCQ+L3tNIjpS4896GnWtEc/OuuOOb/WbQcWcR+AJ4BJZdicZYOD+dzvvCLj9nohZvUR5dz3yLj44EW+ZrqY/Dc8M3aNxXzqG5ToTHP3q68QqvN+8Yw9VFx063axpd0+wM+6195NmfK23C/2vurV1rF2g8/9+vnP/tDfbxvztJQiKdJOQZeqPrYhWeE6N8BWx+QzHfHq4C5ySjjDeMMr500MhmsRySWZbP5ELdOB019vXRARh/metiN1n/EQjp23wIrOn8/6Bbuf+9Oxzu1/8ukqZphZXN5thKk2kQub/xb3GUZcFjLpFcB17d4Z+li3yT5RulHkoHGgHUcuVaW35pbKt0glUr6u/VNx2Y9iRVM6DxyKU1b9SiQhoTD0rsQc0btV6OeOlRLVT0Tde+U4vX+mZXZamVuRe58DvPntNoJIYROTn+9dyY/3hAVsh+hdmvlF9r8JjpyuIN4tJjLVbcQPR4BJdFE1Xx424GJ3tbxKf9ol0FbgeAv+uri6gKl+10G/fukW3J92yD5XmlkOtSoE8Rv2Wj+WgkajajzIVaT5bLUGm32R+MaBULOiMKrsKy2V/G63hdR9gnCx8mUguErkLgSkZFj66OQibxxKXHzhgG0HN/k2uTzoEL8Z+gFUaSl2WhkqKUiCYUiyfzYVWeO3h2ngpwzPcbFx5UFoFR6covNUsUt3h0W4E1AvujlRfoy0RuxIc9K1HJ+jUYxWKWnPxHh9+GBoswTRhXE96CwkXhok12qloONvvghpvnqgYsN2OwTP2kWXNlFWedpS7uW9ZjK2T8o3bOsWYzKLxNBSdpFfUAjAQkVg+wlcU5vZSBP2pzf8Nn5qvt8dCE8DfKIViBIZSqSh8N+ICk/yvwLyZIFE6tiKjML6OYfGsJ7nFpE/l/03NfMjXI/z2j8v2X3nD//d/dpLUPd30RFb5i6F0nshvNohgt6TnsgDHzOZOnv3xuS3dw22zfHl+1X7Yxr22u51b+419PN8OARZdT6mj8QIDGT3LFmjCgFhAr41GMEnpZxnwrXKSXfRUeuxqgbPY1aTcRGCiGE2i3veKINuDIgbZhgCoVS+3h3gEVWLB/eyNM8RuebEfKMM0D7WAIXTyzoyhrax1V2sHZpOLRR7EHnmGoRm6oWu9gkHIcshIf0hgcuPqU5JrxHN+a5e2TkrbY/4XUv74Y0GT/P+iWz3/2jUNjv//vIq3a/6WY/E0t+evcLvmtB/EvnDZZ//PQ2sb83xj/YQzL69847O+//7GTVIpIwykWdz+UXH7tqmGiMzfagmO8FxaJq8A5yiwS6389Dppdk3lIQagGbymEqQcZi++4zqFEoXI7kAx1yDL4cc2nL2RMyMz1j7h0rdwFJO6wBEzx6ks3og5pL0dHz2HovCoezpM12ys6UlOTda0gS3IxLAvfrjvVj+8rJ/ubggWhADexqgPI3/D4QEXmQ9TVwnpe7i/lEv1bpWb+P+ezuw3jF6mB/x/0e2X+zz4JvOf/O0ji9DhN9YkdIU/PCALIAK2n+fcOFKdlkURMJh4krdXXbLfUOAu8erglT8qY6OUMWtnBF5PgcResIK3e5PMfLWAv7LY4vjtlEe91HL7KryW3HvbZjcOreVobt7B2q6XEJ2Mh1bmb7STqOVJ+NCJzXa4oU/TqrijIXbMrCtR6ele2nEX5LSkhtLwVJXh499kVjJTRO8QYQN3gV0+IM6xqRjfPMYo5Sp1eMaeHOaq7niMjjx1A0Y5D5x3uqiy49dvFHf48P9fQZsfjM2JeUlc5ib5iADKnpLyZpnq40+Sf9ll6GlyqLq3q0UeT/PKvVukgI3tXMj5lIJ6QuoME+Nm0J0R+ANtkv2XIfmilMc2P2rI8mHkG95qGQewmbLko3z/JTXfqT/UbKKPU9ZwOA905YXGy7EJmDjtSoQpOEwQTj97lkba8rmbNnGFfVEusCUzdgQ5UxF/MpVhjAAHpn/7avTIqvWr/92vsWY9n6LreahVMaWa+QLnNrd8/EK/kqSoDP5/SaikmZ/G5gfz0F3uBDLhwS4epcFO8qMOUvvrWkyfQQsK4DAtqFrBfEqpPdBLLK3RGC8IM5PmlNVBTIoLQAQ5n39k1OLIuy8y4u7yURzD5lvysQ4X9t+R6NlvqIOJKzs598jWqHv5k2wlhuJrSrMm2M5OzmL18uk/7tE/7tE/7tE/7tE//X9P/AaeBOd4AoAAA values: image: eu.gcr.io/gardener-project/gardener/extensions/provider-local:v0.0.0 --- diff --git a/hack/generate-monitoring-docs.sh b/hack/generate-monitoring-docs.sh index 201a7524be1..86a30420dd0 100755 --- a/hack/generate-monitoring-docs.sh +++ b/hack/generate-monitoring-docs.sh @@ -40,7 +40,7 @@ cat < $PROJECT_ROOT/docs/monitoring/operator_alerts.md |---|---|---|---| EOF -pushd $PROJECT_ROOT/charts/seed-monitoring/charts/core/charts/prometheus > /dev/null +pushd $PROJECT_ROOT/pkg/component/monitoring/charts/seed-monitoring/charts/core/charts/prometheus > /dev/null for file in rules/worker/*.yaml rules/*.yaml; do cat $file | yaml2json | jq -r ' .groups | diff --git a/hack/test-prometheus.sh b/hack/test-prometheus.sh index a8aead2f819..2874c2ba224 100755 --- a/hack/test-prometheus.sh +++ b/hack/test-prometheus.sh @@ -21,7 +21,7 @@ set -o pipefail echo "> Test Prometheus" echo "Executing Prometheus alert tests" -pushd "$(dirname $0)/../charts/seed-monitoring/charts/core/charts/prometheus" > /dev/null +pushd "$(dirname $0)/../pkg/component/monitoring/charts/seed-monitoring/charts/core/charts/prometheus" > /dev/null promtool test rules rules-tests/*test.yaml popd > /dev/null diff --git a/pkg/apis/core/v1beta1/constants/types_constants.go b/pkg/apis/core/v1beta1/constants/types_constants.go index 891eaa8c4a3..6d16c77b566 100644 --- a/pkg/apis/core/v1beta1/constants/types_constants.go +++ b/pkg/apis/core/v1beta1/constants/types_constants.go @@ -753,6 +753,8 @@ const ( // IngressTLSCertificateValidity is the default validity for ingress TLS certificates. IngressTLSCertificateValidity = 730 * 24 * time.Hour // ~2 years, see https://support.apple.com/en-us/HT210176 + // VPNTunnel dictates that VPN is used as a tunnel between seed and shoot networks. + VPNTunnel string = "vpn-shoot" ) var ( diff --git a/pkg/component/monitoring/alertmanager.go b/pkg/component/monitoring/alertmanager.go new file mode 100644 index 00000000000..dfde1f6c35c --- /dev/null +++ b/pkg/component/monitoring/alertmanager.go @@ -0,0 +1,77 @@ +// Copyright 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package monitoring + +import ( + "context" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes" +) + +func deleteAlertmanager(ctx context.Context, k8sClient client.Client, namespace string) error { + objs := []client.Object{ + &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: v1beta1constants.StatefulSetNameAlertManager, + Namespace: namespace, + }, + }, + &networkingv1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: "alertmanager", + Namespace: namespace, + }, + }, + &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "alertmanager-client", + Namespace: namespace, + }, + }, + &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "alertmanager", + Namespace: namespace, + }, + }, + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "alertmanager-basic-auth", + Namespace: namespace, + }, + }, + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "alertmanager-config", + Namespace: namespace, + }, + }, + &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "alertmanager-db-alertmanager-0", + Namespace: namespace, + }, + }, + } + + return kubernetesutils.DeleteObjects(ctx, k8sClient, objs...) +} diff --git a/pkg/component/monitoring/bootstrap.go b/pkg/component/monitoring/bootstrap.go index 3ea61bf17b7..a0d92079072 100644 --- a/pkg/component/monitoring/bootstrap.go +++ b/pkg/component/monitoring/bootstrap.go @@ -35,7 +35,6 @@ import ( "github.com/gardener/gardener/pkg/component/hvpa" "github.com/gardener/gardener/pkg/component/istio" "github.com/gardener/gardener/pkg/component/kubestatemetrics" - "github.com/gardener/gardener/pkg/operation/common" "github.com/gardener/gardener/pkg/utils" kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes" secretsutils "github.com/gardener/gardener/pkg/utils/secrets" @@ -44,12 +43,12 @@ import ( var ( //go:embed charts/bootstrap - chart embed.FS - chartPath = filepath.Join("charts", "bootstrap") + chartBootstrap embed.FS + chartPathBootstrap = filepath.Join("charts", "bootstrap") ) -// Values is a set of configuration values for the monitoring components. -type Values struct { +// ValuesBootstrap is a set of configuration values for the monitoring components. +type ValuesBootstrap struct { // AlertingSMTPSecret is the alerting SMTP secret.. AlertingSMTPSecret *corev1.Secret // GlobalMonitoringSecret is the global monitoring secret for the garden cluster. @@ -78,13 +77,13 @@ type Values struct { WildcardCertName *string } -// New creates a new instance of Deployer for the monitoring components. -func New( +// NewBootstrap creates a new instance of Deployer for the monitoring components. +func NewBootstrap( client client.Client, chartApplier kubernetes.ChartApplier, secretsManager secretsmanager.Interface, namespace string, - values Values, + values ValuesBootstrap, ) component.Deployer { return &bootstrapper{ client: client, @@ -100,7 +99,7 @@ type bootstrapper struct { chartApplier kubernetes.ChartApplier namespace string secretsManager secretsmanager.Interface - values Values + values ValuesBootstrap } func (b *bootstrapper) Deploy(ctx context.Context) error { @@ -192,7 +191,7 @@ func (b *bootstrapper) Deploy(ctx context.Context) error { alertManagerConfig["emailConfigs"] = []map[string]interface{}{emailConfig} } else { alertManagerConfig["enabled"] = false - if err := common.DeleteAlertmanager(ctx, b.client, b.namespace); err != nil { + if err := deleteAlertmanager(ctx, b.client, b.namespace); err != nil { return err } } @@ -268,7 +267,7 @@ func (b *bootstrapper) Deploy(ctx context.Context) error { }, }) - return b.chartApplier.ApplyFromEmbeddedFS(ctx, chart, chartPath, b.namespace, "monitoring", values, applierOptions) + return b.chartApplier.ApplyFromEmbeddedFS(ctx, chartBootstrap, chartPathBootstrap, b.namespace, "monitoring", values, applierOptions) } func (b *bootstrapper) Destroy(ctx context.Context) error { diff --git a/charts/seed-monitoring/charts/alertmanager/Chart.yaml b/pkg/component/monitoring/charts/seed-monitoring/charts/alertmanager/Chart.yaml similarity index 100% rename from charts/seed-monitoring/charts/alertmanager/Chart.yaml rename to pkg/component/monitoring/charts/seed-monitoring/charts/alertmanager/Chart.yaml diff --git a/charts/seed-monitoring/charts/alertmanager/templates/alertmanager.yaml b/pkg/component/monitoring/charts/seed-monitoring/charts/alertmanager/templates/alertmanager.yaml similarity index 97% rename from charts/seed-monitoring/charts/alertmanager/templates/alertmanager.yaml rename to pkg/component/monitoring/charts/seed-monitoring/charts/alertmanager/templates/alertmanager.yaml index 55371ec375a..aa981a00a80 100644 --- a/charts/seed-monitoring/charts/alertmanager/templates/alertmanager.yaml +++ b/pkg/component/monitoring/charts/seed-monitoring/charts/alertmanager/templates/alertmanager.yaml @@ -36,7 +36,7 @@ spec: component: alertmanager role: monitoring --- -apiVersion: {{ include "statefulsetversion" . }} +apiVersion: apps/v1 kind: StatefulSet metadata: name: alertmanager @@ -164,7 +164,7 @@ metadata: namespace: {{ .Release.Namespace }} spec: targetRef: - apiVersion: {{ include "deploymentversion" . }} + apiVersion: apps/v1 kind: StatefulSet name: alertmanager updatePolicy: diff --git a/charts/seed-monitoring/charts/alertmanager/templates/_config.tpl b/pkg/component/monitoring/charts/seed-monitoring/charts/alertmanager/templates/config.tpl similarity index 100% rename from charts/seed-monitoring/charts/alertmanager/templates/_config.tpl rename to pkg/component/monitoring/charts/seed-monitoring/charts/alertmanager/templates/config.tpl diff --git a/charts/seed-monitoring/charts/alertmanager/templates/config.yaml b/pkg/component/monitoring/charts/seed-monitoring/charts/alertmanager/templates/config.yaml similarity index 100% rename from charts/seed-monitoring/charts/alertmanager/templates/config.yaml rename to pkg/component/monitoring/charts/seed-monitoring/charts/alertmanager/templates/config.yaml diff --git a/charts/seed-monitoring/charts/alertmanager/templates/ingress.yaml b/pkg/component/monitoring/charts/seed-monitoring/charts/alertmanager/templates/ingress.yaml similarity index 95% rename from charts/seed-monitoring/charts/alertmanager/templates/ingress.yaml rename to pkg/component/monitoring/charts/seed-monitoring/charts/alertmanager/templates/ingress.yaml index e2393897bab..b7680059baa 100644 --- a/charts/seed-monitoring/charts/alertmanager/templates/ingress.yaml +++ b/pkg/component/monitoring/charts/seed-monitoring/charts/alertmanager/templates/ingress.yaml @@ -1,4 +1,4 @@ -apiVersion: {{ include "ingressversion" . }} +apiVersion: networking.k8s.io/v1 kind: Ingress metadata: annotations: diff --git a/charts/seed-monitoring/charts/alertmanager/values.yaml b/pkg/component/monitoring/charts/seed-monitoring/charts/alertmanager/values.yaml similarity index 100% rename from charts/seed-monitoring/charts/alertmanager/values.yaml rename to pkg/component/monitoring/charts/seed-monitoring/charts/alertmanager/values.yaml diff --git a/charts/seed-monitoring/charts/core/Chart.yaml b/pkg/component/monitoring/charts/seed-monitoring/charts/core/Chart.yaml similarity index 100% rename from charts/seed-monitoring/charts/core/Chart.yaml rename to pkg/component/monitoring/charts/seed-monitoring/charts/core/Chart.yaml diff --git a/charts/seed-monitoring/charts/core/charts/prometheus/Chart.yaml b/pkg/component/monitoring/charts/seed-monitoring/charts/core/charts/prometheus/Chart.yaml similarity index 100% rename from charts/seed-monitoring/charts/core/charts/prometheus/Chart.yaml rename to pkg/component/monitoring/charts/seed-monitoring/charts/core/charts/prometheus/Chart.yaml diff --git a/charts/seed-monitoring/charts/core/charts/prometheus/optional-rules/alertmanager.rules.yaml b/pkg/component/monitoring/charts/seed-monitoring/charts/core/charts/prometheus/optional-rules/alertmanager.rules.yaml similarity index 100% rename from charts/seed-monitoring/charts/core/charts/prometheus/optional-rules/alertmanager.rules.yaml rename to pkg/component/monitoring/charts/seed-monitoring/charts/core/charts/prometheus/optional-rules/alertmanager.rules.yaml diff --git a/charts/seed-monitoring/charts/core/charts/prometheus/rules-tests/kube-kubelet.rules.test.yaml b/pkg/component/monitoring/charts/seed-monitoring/charts/core/charts/prometheus/rules-tests/kube-kubelet.rules.test.yaml similarity index 100% rename from charts/seed-monitoring/charts/core/charts/prometheus/rules-tests/kube-kubelet.rules.test.yaml rename to pkg/component/monitoring/charts/seed-monitoring/charts/core/charts/prometheus/rules-tests/kube-kubelet.rules.test.yaml diff --git a/charts/seed-monitoring/charts/core/charts/prometheus/rules-tests/kube-pods.rules.test.yaml b/pkg/component/monitoring/charts/seed-monitoring/charts/core/charts/prometheus/rules-tests/kube-pods.rules.test.yaml similarity index 100% rename from charts/seed-monitoring/charts/core/charts/prometheus/rules-tests/kube-pods.rules.test.yaml rename to pkg/component/monitoring/charts/seed-monitoring/charts/core/charts/prometheus/rules-tests/kube-pods.rules.test.yaml diff --git a/charts/seed-monitoring/charts/core/charts/prometheus/rules-tests/prometheus.rules.test.yaml b/pkg/component/monitoring/charts/seed-monitoring/charts/core/charts/prometheus/rules-tests/prometheus.rules.test.yaml similarity index 100% rename from charts/seed-monitoring/charts/core/charts/prometheus/rules-tests/prometheus.rules.test.yaml rename to pkg/component/monitoring/charts/seed-monitoring/charts/core/charts/prometheus/rules-tests/prometheus.rules.test.yaml diff --git a/charts/seed-monitoring/charts/core/charts/prometheus/rules/prometheus.rules.yaml b/pkg/component/monitoring/charts/seed-monitoring/charts/core/charts/prometheus/rules/prometheus.rules.yaml similarity index 100% rename from charts/seed-monitoring/charts/core/charts/prometheus/rules/prometheus.rules.yaml rename to pkg/component/monitoring/charts/seed-monitoring/charts/core/charts/prometheus/rules/prometheus.rules.yaml diff --git a/charts/seed-monitoring/charts/core/charts/prometheus/rules/worker/kube-kubelet.rules.yaml b/pkg/component/monitoring/charts/seed-monitoring/charts/core/charts/prometheus/rules/worker/kube-kubelet.rules.yaml similarity index 100% rename from charts/seed-monitoring/charts/core/charts/prometheus/rules/worker/kube-kubelet.rules.yaml rename to pkg/component/monitoring/charts/seed-monitoring/charts/core/charts/prometheus/rules/worker/kube-kubelet.rules.yaml diff --git a/charts/seed-monitoring/charts/core/charts/prometheus/rules/worker/kube-pods.rules.yaml b/pkg/component/monitoring/charts/seed-monitoring/charts/core/charts/prometheus/rules/worker/kube-pods.rules.yaml similarity index 100% rename from charts/seed-monitoring/charts/core/charts/prometheus/rules/worker/kube-pods.rules.yaml rename to pkg/component/monitoring/charts/seed-monitoring/charts/core/charts/prometheus/rules/worker/kube-pods.rules.yaml diff --git a/charts/seed-monitoring/charts/core/charts/prometheus/rules/worker/networking.rules.yaml b/pkg/component/monitoring/charts/seed-monitoring/charts/core/charts/prometheus/rules/worker/networking.rules.yaml similarity index 100% rename from charts/seed-monitoring/charts/core/charts/prometheus/rules/worker/networking.rules.yaml rename to pkg/component/monitoring/charts/seed-monitoring/charts/core/charts/prometheus/rules/worker/networking.rules.yaml diff --git a/charts/seed-monitoring/charts/core/charts/prometheus/rules/workerless/kube-pods.rules.yaml b/pkg/component/monitoring/charts/seed-monitoring/charts/core/charts/prometheus/rules/workerless/kube-pods.rules.yaml similarity index 100% rename from charts/seed-monitoring/charts/core/charts/prometheus/rules/workerless/kube-pods.rules.yaml rename to pkg/component/monitoring/charts/seed-monitoring/charts/core/charts/prometheus/rules/workerless/kube-pods.rules.yaml diff --git a/charts/seed-monitoring/charts/core/charts/prometheus/rules/workerless/networking.rules.yaml b/pkg/component/monitoring/charts/seed-monitoring/charts/core/charts/prometheus/rules/workerless/networking.rules.yaml similarity index 100% rename from charts/seed-monitoring/charts/core/charts/prometheus/rules/workerless/networking.rules.yaml rename to pkg/component/monitoring/charts/seed-monitoring/charts/core/charts/prometheus/rules/workerless/networking.rules.yaml diff --git a/charts/seed-monitoring/charts/core/charts/prometheus/templates/blackbox-exporter-config.yaml b/pkg/component/monitoring/charts/seed-monitoring/charts/core/charts/prometheus/templates/blackbox-exporter-config.yaml similarity index 100% rename from charts/seed-monitoring/charts/core/charts/prometheus/templates/blackbox-exporter-config.yaml rename to pkg/component/monitoring/charts/seed-monitoring/charts/core/charts/prometheus/templates/blackbox-exporter-config.yaml diff --git a/charts/seed-monitoring/charts/core/charts/prometheus/templates/config.yaml b/pkg/component/monitoring/charts/seed-monitoring/charts/core/charts/prometheus/templates/config.yaml similarity index 100% rename from charts/seed-monitoring/charts/core/charts/prometheus/templates/config.yaml rename to pkg/component/monitoring/charts/seed-monitoring/charts/core/charts/prometheus/templates/config.yaml diff --git a/charts/seed-monitoring/charts/core/charts/prometheus/templates/_helpers.tpl b/pkg/component/monitoring/charts/seed-monitoring/charts/core/charts/prometheus/templates/helpers.tpl similarity index 100% rename from charts/seed-monitoring/charts/core/charts/prometheus/templates/_helpers.tpl rename to pkg/component/monitoring/charts/seed-monitoring/charts/core/charts/prometheus/templates/helpers.tpl diff --git a/charts/seed-monitoring/charts/core/charts/prometheus/templates/ingress.yaml b/pkg/component/monitoring/charts/seed-monitoring/charts/core/charts/prometheus/templates/ingress.yaml similarity index 95% rename from charts/seed-monitoring/charts/core/charts/prometheus/templates/ingress.yaml rename to pkg/component/monitoring/charts/seed-monitoring/charts/core/charts/prometheus/templates/ingress.yaml index ca1c4addf42..25563e9e187 100644 --- a/charts/seed-monitoring/charts/core/charts/prometheus/templates/ingress.yaml +++ b/pkg/component/monitoring/charts/seed-monitoring/charts/core/charts/prometheus/templates/ingress.yaml @@ -1,4 +1,4 @@ -apiVersion: {{ include "ingressversion" . }} +apiVersion: networking.k8s.io/v1 kind: Ingress metadata: annotations: diff --git a/charts/seed-monitoring/charts/core/charts/prometheus/templates/prometheus-vpa.yaml b/pkg/component/monitoring/charts/seed-monitoring/charts/core/charts/prometheus/templates/prometheus-vpa.yaml similarity index 94% rename from charts/seed-monitoring/charts/core/charts/prometheus/templates/prometheus-vpa.yaml rename to pkg/component/monitoring/charts/seed-monitoring/charts/core/charts/prometheus/templates/prometheus-vpa.yaml index fcafcbabc96..b6885832163 100644 --- a/charts/seed-monitoring/charts/core/charts/prometheus/templates/prometheus-vpa.yaml +++ b/pkg/component/monitoring/charts/seed-monitoring/charts/core/charts/prometheus/templates/prometheus-vpa.yaml @@ -5,7 +5,7 @@ metadata: namespace: {{ .Release.Namespace }} spec: targetRef: - apiVersion: {{ include "statefulsetversion" . }} + apiVersion: apps/v1 kind: StatefulSet name: prometheus updatePolicy: diff --git a/charts/seed-monitoring/charts/core/charts/prometheus/templates/prometheus.yaml b/pkg/component/monitoring/charts/seed-monitoring/charts/core/charts/prometheus/templates/prometheus.yaml similarity index 99% rename from charts/seed-monitoring/charts/core/charts/prometheus/templates/prometheus.yaml rename to pkg/component/monitoring/charts/seed-monitoring/charts/core/charts/prometheus/templates/prometheus.yaml index 96eec7cf854..08f45c090ae 100644 --- a/charts/seed-monitoring/charts/core/charts/prometheus/templates/prometheus.yaml +++ b/pkg/component/monitoring/charts/seed-monitoring/charts/core/charts/prometheus/templates/prometheus.yaml @@ -31,7 +31,7 @@ spec: sessionAffinity: None type: ClusterIP --- -apiVersion: {{include "statefulsetversion" .}} +apiVersion: apps/v1 kind: StatefulSet metadata: name: prometheus diff --git a/charts/seed-monitoring/charts/core/charts/prometheus/templates/rbac.yaml b/pkg/component/monitoring/charts/seed-monitoring/charts/core/charts/prometheus/templates/rbac.yaml similarity index 83% rename from charts/seed-monitoring/charts/core/charts/prometheus/templates/rbac.yaml rename to pkg/component/monitoring/charts/seed-monitoring/charts/core/charts/prometheus/templates/rbac.yaml index eb71760f820..d4792a875ed 100644 --- a/charts/seed-monitoring/charts/core/charts/prometheus/templates/rbac.yaml +++ b/pkg/component/monitoring/charts/seed-monitoring/charts/core/charts/prometheus/templates/rbac.yaml @@ -1,4 +1,4 @@ -apiVersion: {{ include "rbacversion" . }} +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: prometheus-{{ .Release.Namespace }} @@ -16,4 +16,4 @@ roleRef: subjects: - kind: ServiceAccount name: prometheus - namespace: {{ .Release.Namespace }} \ No newline at end of file + namespace: {{ .Release.Namespace }} diff --git a/charts/seed-monitoring/charts/core/charts/prometheus/templates/rules.yaml b/pkg/component/monitoring/charts/seed-monitoring/charts/core/charts/prometheus/templates/rules.yaml similarity index 100% rename from charts/seed-monitoring/charts/core/charts/prometheus/templates/rules.yaml rename to pkg/component/monitoring/charts/seed-monitoring/charts/core/charts/prometheus/templates/rules.yaml diff --git a/charts/seed-monitoring/charts/core/charts/prometheus/values.yaml b/pkg/component/monitoring/charts/seed-monitoring/charts/core/charts/prometheus/values.yaml similarity index 100% rename from charts/seed-monitoring/charts/core/charts/prometheus/values.yaml rename to pkg/component/monitoring/charts/seed-monitoring/charts/core/charts/prometheus/values.yaml diff --git a/charts/seed-monitoring/charts/core/values.yaml b/pkg/component/monitoring/charts/seed-monitoring/charts/core/values.yaml similarity index 100% rename from charts/seed-monitoring/charts/core/values.yaml rename to pkg/component/monitoring/charts/seed-monitoring/charts/core/values.yaml diff --git a/pkg/component/monitoring/component.go b/pkg/component/monitoring/component.go new file mode 100644 index 00000000000..8292736976a --- /dev/null +++ b/pkg/component/monitoring/component.go @@ -0,0 +1,677 @@ +// Copyright 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package monitoring + +import ( + "context" + "embed" + "fmt" + "path/filepath" + "strings" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + vpaautoscalingv1 "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" + + gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + resourcesv1alpha1 "github.com/gardener/gardener/pkg/apis/resources/v1alpha1" + "github.com/gardener/gardener/pkg/client/kubernetes" + "github.com/gardener/gardener/pkg/component" + "github.com/gardener/gardener/pkg/component/etcd" + "github.com/gardener/gardener/pkg/controllerutils" + gardenletconfig "github.com/gardener/gardener/pkg/gardenlet/apis/config" + "github.com/gardener/gardener/pkg/utils" + gardenerutils "github.com/gardener/gardener/pkg/utils/gardener" + kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes" + "github.com/gardener/gardener/pkg/utils/managedresources" + secretsutils "github.com/gardener/gardener/pkg/utils/secrets" + secretsmanager "github.com/gardener/gardener/pkg/utils/secrets/manager" +) + +var ( + //go:embed charts/seed-monitoring/charts/alertmanager + chartAlertmanager embed.FS + chartPathAlertmanager = filepath.Join("charts", "seed-monitoring", "charts", "alertmanager") + + //go:embed charts/seed-monitoring/charts/core + chartCore embed.FS + chartPathCore = filepath.Join("charts", "seed-monitoring", "charts", "core") + + managedResourceNamePrometheus = "shoot-core-prometheus" +) + +// Interface contains functions for a monitoring deployer. +type Interface interface { + component.Deployer + // SetNamespaceUID sets the UID of the namespace into which the monitoring components shall be deployed. + SetNamespaceUID(types.UID) + // SetComponents sets the monitoring components. + SetComponents([]component.MonitoringComponent) +} + +// Values is a set of configuration values for the monitoring components. +type Values struct { + // AlertingSecrets is a list of alerting secrets. + AlertingSecrets []*corev1.Secret + // AlertmanagerEnabled specifies whether Alertmanager is enabled. + AlertmanagerEnabled bool + // APIServerDomain is the domain of the API server. + APIServerDomain string + // APIServerHost is the host of the API server. + APIServerHost string + // APIServerServiceIP is the service IP of the API server. + APIServerServiceIP *string + // Components is a list of monitoring components. + Components []component.MonitoringComponent + // Config is the monitoring config. + Config *gardenletconfig.MonitoringConfig + // GardenletManagesMCM specifies whether MCM is managed by gardenlet. + GardenletManagesMCM bool + // GlobalShootRemoteWriteSecret is the global secret for remote write config. + GlobalShootRemoteWriteSecret *corev1.Secret + // IgnoreAlerts specifies whether alerts should be ignored. + IgnoreAlerts bool + // ImageAlertmanager is the image of Alertmanager. + ImageAlertmanager string + // ImageBlackboxExporter is the image of BlackboxExporter. + ImageBlackboxExporter string + // ImageConfigmapReloader is the image of ConfigmapReloader. + ImageConfigmapReloader string + // ImagePrometheus is the image of Prometheus. + ImagePrometheus string + // IngressHostAlertmanager is the host name of Alertmanager. + IngressHostAlertmanager string + // IngressHostPrometheus is the host name of Prometheus. + IngressHostPrometheus string + // IsWorkerless specifies whether the cluster is workerless. + IsWorkerless bool + // KubernetesVersion is the Kubernetes version of the target cluster. + KubernetesVersion string + // MonitoringConfig is the monitoring config. + MonitoringConfig *gardencorev1beta1.Monitoring + // NamespaceUID is the UID of the namespace in the runtime cluster. + NamespaceUID types.UID + // NodeLocalDNSEnabled specifies whether node-local-dns is enabled. + NodeLocalDNSEnabled bool + // ProjectName is the name of the project. + ProjectName string + // PodNetworkCIDR is the CIDR of the pod network. + PodNetworkCIDR *string + // ServiceNetworkCIDR is the CIDR of the service network. + ServiceNetworkCIDR *string + // NodeNetworkCIDR is the CIDR of the node network. + NodeNetworkCIDR *string + // Replicas is the number of replicas. + Replicas int32 + // RuntimeProviderType is the provider type of the runtime cluster. + RuntimeProviderType string + // RuntimeRegion is the region of the runtime cluster. + RuntimeRegion string + // StorageCapacityAlertmanager is the storage capacity of Alertmanager. + StorageCapacityAlertmanager string + // TargetName is the name of the target cluster. + TargetName string + // TargetProviderType is the provider type of the target cluster. + TargetProviderType string + // WildcardCertName is name of wildcard tls certificate which is issued for the seed's ingress domain. + WildcardCertName *string +} + +// New creates a new instance of Interface for the monitoring components. +func New( + client client.Client, + chartApplier kubernetes.ChartApplier, + secretsManager secretsmanager.Interface, + namespace string, + values Values, +) Interface { + return &monitoring{ + client: client, + chartApplier: chartApplier, + namespace: namespace, + secretsManager: secretsManager, + values: values, + } +} + +type monitoring struct { + client client.Client + chartApplier kubernetes.ChartApplier + namespace string + secretsManager secretsmanager.Interface + values Values +} + +func (m *monitoring) Deploy(ctx context.Context) error { + credentialsSecret, found := m.secretsManager.Get(v1beta1constants.SecretNameObservabilityIngressUsers) + if !found { + return fmt.Errorf("secret %q not found", v1beta1constants.SecretNameObservabilityIngressUsers) + } + + alerting, err := m.getCustomAlertingConfigs(ctx) + if err != nil { + return err + } + + alertingRules, scrapeConfigs, err := m.getAlertingRulesAndScrapeConfigs(ctx) + if err != nil { + return err + } + + // Create shoot token secret for prometheus component + shootAccessSecret := m.newShootAccessSecret() + if err := shootAccessSecret.Reconcile(ctx, m.client); err != nil { + return err + } + + var ingressTLSSecretName string + if m.values.WildcardCertName != nil { + ingressTLSSecretName = *m.values.WildcardCertName + } else { + ingressTLSSecret, err := m.secretsManager.Generate(ctx, &secretsutils.CertificateSecretConfig{ + Name: "prometheus-tls", + CommonName: "prometheus", + Organization: []string{"gardener.cloud:monitoring:ingress"}, + DNSNames: []string{m.values.IngressHostPrometheus}, + CertType: secretsutils.ServerCert, + Validity: pointer.Duration(v1beta1constants.IngressTLSCertificateValidity), + SkipPublishingCACertificate: true, + }, secretsmanager.SignedByCA(v1beta1constants.SecretNameCACluster)) + if err != nil { + return err + } + ingressTLSSecretName = ingressTLSSecret.Name + } + + clusterCASecret, found := m.secretsManager.Get(v1beta1constants.SecretNameCACluster) + if !found { + return fmt.Errorf("secret %q not found", v1beta1constants.SecretNameCACluster) + } + + etcdCASecret, found := m.secretsManager.Get(v1beta1constants.SecretNameCAETCD) + if !found { + return fmt.Errorf("secret %q not found", v1beta1constants.SecretNameCAETCD) + } + + etcdClientSecret, found := m.secretsManager.Get(etcd.SecretNameClient) + if !found { + return fmt.Errorf("secret %q not found", etcd.SecretNameClient) + } + + var ( + networks = map[string]interface{}{} + prometheusConfig = map[string]interface{}{ + "images": map[string]string{ + "blackbox-exporter": m.values.ImageBlackboxExporter, + "configmap-reloader": m.values.ImageConfigmapReloader, + "prometheus": m.values.ImagePrometheus, + }, + "secretNameClusterCA": clusterCASecret.Name, + "secretNameEtcdCA": etcdCASecret.Name, + "secretNameEtcdClientCert": etcdClientSecret.Name, + "kubernetesVersion": m.values.KubernetesVersion, + "nodeLocalDNS": map[string]interface{}{ + "enabled": m.values.NodeLocalDNSEnabled, + }, + "gardenletManagesMCM": m.values.GardenletManagesMCM, + "ingress": map[string]interface{}{ + "class": v1beta1constants.SeedNginxIngressClass, + "authSecretName": credentialsSecret.Name, + "hosts": []map[string]interface{}{ + { + "hostName": m.values.IngressHostPrometheus, + "secretName": ingressTLSSecretName, + }, + }, + }, + "namespace": map[string]interface{}{ + "uid": m.values.NamespaceUID, + }, + "replicas": m.values.Replicas, + "seed": map[string]interface{}{ + "apiserver": m.values.APIServerHost, + "region": m.values.RuntimeRegion, + "provider": m.values.RuntimeProviderType, + }, + "rules": map[string]interface{}{ + "optional": map[string]interface{}{ + "alertmanager": map[string]interface{}{ + "enabled": m.values.AlertmanagerEnabled, + }, + }, + }, + "shoot": map[string]interface{}{ + "apiserver": fmt.Sprintf("https://%s", m.values.APIServerDomain), + "apiserverServerName": m.values.APIServerDomain, + "provider": m.values.TargetProviderType, + "name": m.values.TargetName, + "project": m.values.ProjectName, + "workerless": m.values.IsWorkerless, + }, + "ignoreAlerts": m.values.IgnoreAlerts, + "alerting": alerting, + "additionalRules": alertingRules.String(), + "additionalScrapeConfigs": scrapeConfigs.String(), + } + ) + + if services := m.values.ServiceNetworkCIDR; services != nil { + networks["services"] = services + } + if pods := m.values.PodNetworkCIDR; pods != nil { + networks["pods"] = pods + } + if apiServer := m.values.APIServerServiceIP; apiServer != nil { + prometheusConfig["apiserverServiceIP"] = apiServer + } + if m.values.NodeNetworkCIDR != nil { + networks["nodes"] = *m.values.NodeNetworkCIDR + } + + prometheusConfig["networks"] = networks + + // Add remotewrite to prometheus when enabled + if m.values.Config != nil && + m.values.Config.Shoot != nil && + m.values.Config.Shoot.RemoteWrite != nil && + m.values.Config.Shoot.RemoteWrite.URL != "" { + // if remoteWrite Url is set add config into values + remoteWriteConfig := map[string]interface{}{ + "url": m.values.Config.Shoot.RemoteWrite.URL, + } + // get secret for basic_auth in remote write + if remoteWriteBasicAuth := m.values.GlobalShootRemoteWriteSecret; remoteWriteBasicAuth != nil { + remoteWriteUsername := string(remoteWriteBasicAuth.Data["username"]) + remoteWritePassword := string(remoteWriteBasicAuth.Data["password"]) + if remoteWriteUsername != "" && + remoteWritePassword != "" { + remoteWriteConfig["basic_auth"] = map[string]interface{}{ + "username": remoteWriteUsername, + "password": remoteWritePassword, + } + } + } + // add list with keep metrics if set + if len(m.values.Config.Shoot.RemoteWrite.Keep) != 0 { + remoteWriteConfig["keep"] = m.values.Config.Shoot.RemoteWrite.Keep + } + // add queue_config if set + if m.values.Config.Shoot.RemoteWrite.QueueConfig != nil && + len(*m.values.Config.Shoot.RemoteWrite.QueueConfig) != 0 { + remoteWriteConfig["queue_config"] = m.values.Config.Shoot.RemoteWrite.QueueConfig + } + prometheusConfig["remoteWrite"] = remoteWriteConfig + } + + // set externalLabels + if m.values.Config != nil && m.values.Config.Shoot != nil && len(m.values.Config.Shoot.ExternalLabels) != 0 { + prometheusConfig["externalLabels"] = m.values.Config.Shoot.ExternalLabels + } + + coreValues := map[string]interface{}{ + "global": map[string]interface{}{ + "shootKubeVersion": map[string]interface{}{ + "gitVersion": m.values.KubernetesVersion, + }, + }, + "prometheus": prometheusConfig, + } + + if err := m.chartApplier.ApplyFromEmbeddedFS(ctx, chartCore, chartPathCore, m.namespace, "core", kubernetes.Values(coreValues)); err != nil { + return err + } + + if err := m.reconcilePrometheusShootResources(ctx, shootAccessSecret.ServiceAccountName); err != nil { + return err + } + + // Check if we want to deploy an alertmanager into the shoot namespace. + if m.values.AlertmanagerEnabled { + var emailConfigs []map[string]interface{} + if m.values.MonitoringConfig != nil && m.values.MonitoringConfig.Alerting != nil { + for _, email := range m.values.MonitoringConfig.Alerting.EmailReceivers { + for _, secret := range m.values.AlertingSecrets { + if string(secret.Data["auth_type"]) != "smtp" { + continue + } + emailConfigs = append(emailConfigs, map[string]interface{}{ + "to": email, + "from": string(secret.Data["from"]), + "smarthost": string(secret.Data["smarthost"]), + "auth_username": string(secret.Data["auth_username"]), + "auth_identity": string(secret.Data["auth_identity"]), + "auth_password": string(secret.Data["auth_password"]), + }) + } + } + } + + var alertManagerIngressTLSSecretName string + if m.values.WildcardCertName != nil { + alertManagerIngressTLSSecretName = *m.values.WildcardCertName + } else { + ingressTLSSecret, err := m.secretsManager.Generate(ctx, &secretsutils.CertificateSecretConfig{ + Name: "alertmanager-tls", + CommonName: "alertmanager", + Organization: []string{"gardener.cloud:monitoring:ingress"}, + DNSNames: []string{m.values.IngressHostAlertmanager}, + CertType: secretsutils.ServerCert, + Validity: pointer.Duration(v1beta1constants.IngressTLSCertificateValidity), + SkipPublishingCACertificate: true, + }, secretsmanager.SignedByCA(v1beta1constants.SecretNameCACluster)) + if err != nil { + return err + } + alertManagerIngressTLSSecretName = ingressTLSSecret.Name + } + + alertManagerValues := map[string]interface{}{ + "images": map[string]string{ + "alertmanager": m.values.ImageAlertmanager, + "configmap-reloader": m.values.ImageConfigmapReloader, + }, + "ingress": map[string]interface{}{ + "class": v1beta1constants.SeedNginxIngressClass, + "authSecretName": credentialsSecret.Name, + "hosts": []map[string]interface{}{ + { + "hostName": m.values.IngressHostAlertmanager, + "secretName": alertManagerIngressTLSSecretName, + }, + }, + }, + "replicas": m.values.Replicas, + "storage": m.values.StorageCapacityAlertmanager, + "emailConfigs": emailConfigs, + } + + return m.chartApplier.ApplyFromEmbeddedFS(ctx, chartAlertmanager, chartPathAlertmanager, m.namespace, "alertmanager", kubernetes.Values(alertManagerValues)) + } + + return deleteAlertmanager(ctx, m.client, m.namespace) +} + +func (m *monitoring) Destroy(ctx context.Context) error { + if err := deleteAlertmanager(ctx, m.client, m.namespace); err != nil { + return err + } + + if err := managedresources.DeleteForShoot(ctx, m.client, m.namespace, managedResourceNamePrometheus); err != nil { + return err + } + + objects := []client.Object{ + &networkingv1.NetworkPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: m.namespace, + Name: "allow-from-prometheus", + }, + }, + &networkingv1.NetworkPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: m.namespace, + Name: "allow-prometheus", + }, + }, + m.newShootAccessSecret().Secret, + &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: m.namespace, + Name: "prometheus-config", + }, + }, + &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: m.namespace, + Name: "prometheus-rules", + }, + }, + &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: m.namespace, + Name: "blackbox-exporter-config-prometheus", + }, + }, + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: m.namespace, + Name: "prometheus-basic-auth", + }, + }, + &networkingv1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: m.namespace, + Name: "prometheus", + }, + }, + &vpaautoscalingv1.VerticalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: m.namespace, + Name: "prometheus-vpa", + }, + }, + &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: m.namespace, + Name: "prometheus", + }, + }, + &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: m.namespace, + Name: "prometheus-web", + }, + }, + &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: m.namespace, + Name: "prometheus", + }, + }, + &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: m.namespace, + Name: "prometheus-" + m.namespace, + }, + }, + &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: m.namespace, + Name: "prometheus-db-prometheus-0", + }, + }, + } + + return kubernetesutils.DeleteObjects(ctx, m.client, objects...) +} + +func (m *monitoring) SetNamespaceUID(uid types.UID) { m.values.NamespaceUID = uid } +func (m *monitoring) SetComponents(c []component.MonitoringComponent) { m.values.Components = c } + +func (m *monitoring) newShootAccessSecret() *gardenerutils.AccessSecret { + return gardenerutils.NewShootAccessSecret(v1beta1constants.StatefulSetNamePrometheus, m.namespace) +} + +func (m *monitoring) reconcilePrometheusShootResources(ctx context.Context, serviceAccountName string) error { + var ( + registry = managedresources.NewRegistry(kubernetes.ShootScheme, kubernetes.ShootCodec, kubernetes.ShootSerializer) + + clusterRole = &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gardener.cloud:monitoring:prometheus-target", + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{corev1.GroupName}, + Resources: []string{"nodes", "services", "endpoints", "pods"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{corev1.GroupName}, + Resources: []string{"nodes/metrics", "pods/log", "nodes/proxy", "services/proxy", "pods/proxy"}, + Verbs: []string{"get"}, + }, + { + NonResourceURLs: []string{"/metrics"}, + Verbs: []string{"get"}, + }, + }, + } + clusterRoleBinding = &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gardener.cloud:monitoring:prometheus-target", + Annotations: map[string]string{resourcesv1alpha1.DeleteOnInvalidUpdate: "true"}, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: rbacv1.GroupName, + Kind: "ClusterRole", + Name: clusterRole.Name, + }, + Subjects: []rbacv1.Subject{{ + Kind: rbacv1.ServiceAccountKind, + Name: serviceAccountName, + Namespace: metav1.NamespaceSystem, + }}, + } + ) + + data, err := registry.AddAllAndSerialize(clusterRole, clusterRoleBinding) + if err != nil { + return err + } + + return managedresources.CreateForShoot(ctx, m.client, m.namespace, managedResourceNamePrometheus, managedresources.LabelValueGardener, false, data) +} + +func (m *monitoring) getCustomAlertingConfigs(ctx context.Context) (map[string]interface{}, error) { + configs := map[string]interface{}{ + "auth_type": map[string]interface{}{}, + } + + for _, secret := range m.values.AlertingSecrets { + if string(secret.Data["auth_type"]) == "none" { + + if url, ok := secret.Data["url"]; ok { + configs["auth_type"] = map[string]interface{}{ + "none": map[string]interface{}{ + "url": string(url), + }, + } + } + break + } + + if string(secret.Data["auth_type"]) == "basic" { + url, urlOk := secret.Data["url"] + username, usernameOk := secret.Data["username"] + password, passwordOk := secret.Data["password"] + + if urlOk && usernameOk && passwordOk { + configs["auth_type"] = map[string]interface{}{ + "basic": map[string]interface{}{ + "url": string(url), + "username": string(username), + "password": string(password), + }, + } + } + break + } + + if string(secret.Data["auth_type"]) == "certificate" { + data := map[string][]byte{} + url, urlOk := secret.Data["url"] + ca, caOk := secret.Data["ca.crt"] + cert, certOk := secret.Data["tls.crt"] + key, keyOk := secret.Data["tls.key"] + insecure, insecureOk := secret.Data["insecure_skip_verify"] + + if urlOk && caOk && certOk && keyOk && insecureOk { + configs["auth_type"] = map[string]interface{}{ + "certificate": map[string]interface{}{ + "url": string(url), + "insecure_skip_verify": string(insecure), + }, + } + data["ca.crt"] = ca + data["tls.crt"] = cert + data["tls.key"] = key + amSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "prometheus-remote-am-tls", + Namespace: m.namespace, + }, + } + + if _, err := controllerutils.GetAndCreateOrMergePatch(ctx, m.client, amSecret, func() error { + amSecret.Data = data + amSecret.Type = corev1.SecretTypeOpaque + return nil + }); err != nil { + return nil, err + } + } + break + } + } + + return configs, nil +} + +func (m *monitoring) getAlertingRulesAndScrapeConfigs(ctx context.Context) (alertingRules, scrapeConfigs strings.Builder, err error) { + for _, component := range m.values.Components { + componentsScrapeConfigs, err := component.ScrapeConfigs() + if err != nil { + return alertingRules, scrapeConfigs, err + } + for _, config := range componentsScrapeConfigs { + scrapeConfigs.WriteString(fmt.Sprintf("- %s\n", utils.Indent(config, 2))) + } + + componentsAlertingRules, err := component.AlertingRules() + if err != nil { + return alertingRules, scrapeConfigs, err + } + for filename, rule := range componentsAlertingRules { + alertingRules.WriteString(fmt.Sprintf("%s: |\n %s\n", filename, utils.Indent(rule, 2))) + } + } + + // Fetch extensions provider-specific monitoring configuration + existingConfigMaps := &corev1.ConfigMapList{} + if err := m.client.List(ctx, existingConfigMaps, + client.InNamespace(m.namespace), + client.MatchingLabels{v1beta1constants.LabelExtensionConfiguration: v1beta1constants.LabelMonitoring}); err != nil { + return alertingRules, scrapeConfigs, err + } + + // Need stable order before passing the dashboards to Prometheus config to avoid unnecessary changes + kubernetesutils.ByName().Sort(existingConfigMaps) + + // Read extension monitoring configurations + for _, cm := range existingConfigMaps.Items { + alertingRules.WriteString(fmt.Sprintln(cm.Data[v1beta1constants.PrometheusConfigMapAlertingRules])) + scrapeConfigs.WriteString(fmt.Sprintln(cm.Data[v1beta1constants.PrometheusConfigMapScrapeConfig])) + } + + return +} diff --git a/pkg/controller/networkpolicy/reconciler.go b/pkg/controller/networkpolicy/reconciler.go index d63f2af6383..f7382904672 100644 --- a/pkg/controller/networkpolicy/reconciler.go +++ b/pkg/controller/networkpolicy/reconciler.go @@ -42,7 +42,6 @@ import ( "github.com/gardener/gardener/pkg/controller/networkpolicy/hostnameresolver" "github.com/gardener/gardener/pkg/controllerutils" "github.com/gardener/gardener/pkg/extensions" - "github.com/gardener/gardener/pkg/operation/common" "github.com/gardener/gardener/pkg/utils" kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes" ) @@ -429,7 +428,7 @@ func (r *Reconciler) reconcileNetworkPolicyAllowToDNS(ctx context.Context, log l return err } - runtimeDNSServerAddress, err := common.ComputeOffsetIP(runtimeServiceCIDR, 10) + runtimeDNSServerAddress, err := utils.ComputeOffsetIP(runtimeServiceCIDR, 10) if err != nil { return fmt.Errorf("cannot calculate CoreDNS ClusterIP: %w", err) } diff --git a/pkg/gardenlet/controller/seed/seed/components.go b/pkg/gardenlet/controller/seed/seed/components.go index 73520de4fb1..2bef57dee41 100644 --- a/pkg/gardenlet/controller/seed/seed/components.go +++ b/pkg/gardenlet/controller/seed/seed/components.go @@ -447,12 +447,12 @@ func defaultMonitoring( return nil, err } - return monitoring.New( + return monitoring.NewBootstrap( c, chartApplier, secretsManager, namespace, - monitoring.Values{ + monitoring.ValuesBootstrap{ AlertingSMTPSecret: alertingSMTPSecret, GlobalMonitoringSecret: globalMonitoringSecret, HVPAEnabled: hvpaEnabled, diff --git a/pkg/gardenlet/controller/shoot/shoot/reconciler_delete.go b/pkg/gardenlet/controller/shoot/shoot/reconciler_delete.go index 22e3303ef7c..9b767ecb375 100644 --- a/pkg/gardenlet/controller/shoot/shoot/reconciler_delete.go +++ b/pkg/gardenlet/controller/shoot/shoot/reconciler_delete.go @@ -315,7 +315,7 @@ func (r *Reconciler) runDeleteShootFlow(ctx context.Context, o *operation.Operat }) deleteSeedMonitoring = g.Add(flow.Task{ Name: "Deleting shoot monitoring stack in Seed", - Fn: flow.TaskFn(botanist.DeleteSeedMonitoring).RetryUntilTimeout(defaultInterval, defaultTimeout), + Fn: flow.TaskFn(botanist.Shoot.Components.Monitoring.Monitoring.Destroy).RetryUntilTimeout(defaultInterval, defaultTimeout), Dependencies: flow.NewTaskIDs(initializeShootClients), }) deleteClusterAutoscaler = g.Add(flow.Task{ diff --git a/pkg/gardenlet/controller/shoot/shoot/reconciler_reconcile.go b/pkg/gardenlet/controller/shoot/shoot/reconciler_reconcile.go index 169d1cd35f0..6f71ad7e66a 100644 --- a/pkg/gardenlet/controller/shoot/shoot/reconciler_reconcile.go +++ b/pkg/gardenlet/controller/shoot/shoot/reconciler_reconcile.go @@ -327,7 +327,7 @@ func (r *Reconciler) runReconcileShootFlow(ctx context.Context, o *operation.Ope }) deploySeedLogging = g.Add(flow.Task{ Name: "Deploying shoot logging stack in Seed", - Fn: flow.TaskFn(botanist.DeploySeedLogging).RetryUntilTimeout(defaultInterval, defaultTimeout), + Fn: flow.TaskFn(botanist.DeployLogging).RetryUntilTimeout(defaultInterval, defaultTimeout), Dependencies: flow.NewTaskIDs(deployNamespace, initializeSecretsManagement).InsertIf(shootControlPlaneLoggingEnabled, waitUntilGardenerResourceManagerReady), }) deployShootNamespaces = g.Add(flow.Task{ @@ -675,7 +675,7 @@ func (r *Reconciler) runReconcileShootFlow(ctx context.Context, o *operation.Ope _ = g.Add(flow.Task{ Name: "Reconciling Plutono for Shoot in Seed for the logging stack", - Fn: flow.TaskFn(botanist.DeploySeedPlutono).RetryUntilTimeout(defaultInterval, 2*time.Minute), + Fn: flow.TaskFn(botanist.DeployPlutono).RetryUntilTimeout(defaultInterval, 2*time.Minute), Dependencies: flow.NewTaskIDs(deploySeedLogging), }) nginxLBReady = g.Add(flow.Task{ @@ -705,7 +705,7 @@ func (r *Reconciler) runReconcileShootFlow(ctx context.Context, o *operation.Ope }) deploySeedMonitoring = g.Add(flow.Task{ Name: "Deploying Shoot monitoring stack in Seed", - Fn: flow.TaskFn(botanist.DeploySeedMonitoring).RetryUntilTimeout(defaultInterval, 2*time.Minute), + Fn: flow.TaskFn(botanist.DeployMonitoring).RetryUntilTimeout(defaultInterval, 2*time.Minute), Dependencies: flow.NewTaskIDs(initializeShootClients, waitUntilTunnelConnectionExists, waitUntilWorkerReady).InsertIf(!staticNodesCIDR, waitUntilInfrastructureReady), }) _ = g.Add(flow.Task{ @@ -715,7 +715,7 @@ func (r *Reconciler) runReconcileShootFlow(ctx context.Context, o *operation.Ope }) _ = g.Add(flow.Task{ Name: "Reconciling Plutono for Shoot in Seed for the monitoring stack", - Fn: flow.TaskFn(botanist.DeploySeedPlutono).RetryUntilTimeout(defaultInterval, 2*time.Minute), + Fn: flow.TaskFn(botanist.DeployPlutono).RetryUntilTimeout(defaultInterval, 2*time.Minute), Dependencies: flow.NewTaskIDs(deploySeedMonitoring), }) diff --git a/pkg/operation/botanist/addons.go b/pkg/operation/botanist/addons.go index 464ac686e31..cd55055adef 100644 --- a/pkg/operation/botanist/addons.go +++ b/pkg/operation/botanist/addons.go @@ -16,39 +16,34 @@ package botanist import ( "context" - "fmt" + "embed" "path/filepath" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/pointer" - "github.com/gardener/gardener/charts" - "github.com/gardener/gardener/pkg/chartrenderer" - "github.com/gardener/gardener/pkg/operation/common" "github.com/gardener/gardener/pkg/utils/managedresources" ) +var ( + //go:embed charts/shoot-core/components + chartPSPs embed.FS + chartPathPSPs = filepath.Join("charts", "shoot-core", "components") +) + // DeployManagedResourceForAddons deploys all the ManagedResource CRDs for the gardener-resource-manager. func (b *Botanist) DeployManagedResourceForAddons(ctx context.Context) error { - renderedChart, err := b.generateCoreAddonsChart() - if err != nil { - return fmt.Errorf("error rendering shoot-core chart: %w", err) - } - - return managedresources.CreateForShoot(ctx, b.SeedClientSet.Client(), b.Shoot.SeedNamespace, "shoot-core", managedresources.LabelValueGardener, false, renderedChart.AsSecretData()) -} - -// generateCoreAddonsChart renders the gardener-resource-manager configuration for the core addons. After that it -// creates a ManagedResource CRD that references the rendered manifests and creates it. -func (b *Botanist) generateCoreAddonsChart() (*chartrenderer.RenderedChart, error) { - podSecurityPolicies := map[string]interface{}{ - "allowPrivilegedContainers": pointer.BoolDeref(b.Shoot.GetInfo().Spec.Kubernetes.AllowPrivilegedContainers, false), + values := map[string]interface{}{ + "podsecuritypolicies": map[string]interface{}{ + "enabled": !b.Shoot.PSPDisabled && !b.Shoot.IsWorkerless, + "allowPrivilegedContainers": pointer.BoolDeref(b.Shoot.GetInfo().Spec.Kubernetes.AllowPrivilegedContainers, false), + }, } - values := map[string]interface{}{ - "monitoring": common.GenerateAddonConfig(map[string]interface{}{}, b.Operation.IsShootMonitoringEnabled()), - "podsecuritypolicies": common.GenerateAddonConfig(podSecurityPolicies, !b.Shoot.PSPDisabled && !b.Shoot.IsWorkerless), + renderedChart, err := b.ShootClientSet.ChartRenderer().RenderEmbeddedFS(chartPSPs, chartPathPSPs, "shoot-core", metav1.NamespaceSystem, values) + if err != nil { + return err } - return b.ShootClientSet.ChartRenderer().Render(filepath.Join(charts.Path, "shoot-core", "components"), "shoot-core", metav1.NamespaceSystem, values) + return managedresources.CreateForShoot(ctx, b.SeedClientSet.Client(), b.Shoot.SeedNamespace, "shoot-core", managedresources.LabelValueGardener, false, renderedChart.AsSecretData()) } diff --git a/pkg/operation/botanist/botanist.go b/pkg/operation/botanist/botanist.go index dca8f81b962..c68e133a472 100644 --- a/pkg/operation/botanist/botanist.go +++ b/pkg/operation/botanist/botanist.go @@ -216,6 +216,12 @@ func New(ctx context.Context, o *operation.Operation) (*Botanist, error) { o.Shoot.Components.DependencyWatchdogAccess = b.DefaultDependencyWatchdogAccess() o.Shoot.Components.GardenerAccess = b.DefaultGardenerAccess() + // Monitoring + o.Shoot.Components.Monitoring.Monitoring, err = b.DefaultMonitoring() + if err != nil { + return nil, err + } + // Logging o.Shoot.Components.Logging.ShootRBACProxy, err = kuberbacproxy.New(b.SeedClientSet.Client(), b.Shoot.SeedNamespace) if err != nil { diff --git a/charts/shoot-core/components/Chart.yaml b/pkg/operation/botanist/charts/shoot-core/components/Chart.yaml similarity index 100% rename from charts/shoot-core/components/Chart.yaml rename to pkg/operation/botanist/charts/shoot-core/components/Chart.yaml diff --git a/charts/shoot-core/components/charts/podsecuritypolicies/Chart.yaml b/pkg/operation/botanist/charts/shoot-core/components/charts/podsecuritypolicies/Chart.yaml similarity index 100% rename from charts/shoot-core/components/charts/podsecuritypolicies/Chart.yaml rename to pkg/operation/botanist/charts/shoot-core/components/charts/podsecuritypolicies/Chart.yaml diff --git a/charts/shoot-core/components/charts/podsecuritypolicies/templates/_defaults.tpl b/pkg/operation/botanist/charts/shoot-core/components/charts/podsecuritypolicies/templates/defaults.tpl similarity index 100% rename from charts/shoot-core/components/charts/podsecuritypolicies/templates/_defaults.tpl rename to pkg/operation/botanist/charts/shoot-core/components/charts/podsecuritypolicies/templates/defaults.tpl diff --git a/charts/shoot-core/components/charts/podsecuritypolicies/templates/privileged-clusterrole.yaml b/pkg/operation/botanist/charts/shoot-core/components/charts/podsecuritypolicies/templates/privileged-clusterrole.yaml similarity index 83% rename from charts/shoot-core/components/charts/podsecuritypolicies/templates/privileged-clusterrole.yaml rename to pkg/operation/botanist/charts/shoot-core/components/charts/podsecuritypolicies/templates/privileged-clusterrole.yaml index ae462d096cd..b9b21804b8d 100644 --- a/charts/shoot-core/components/charts/podsecuritypolicies/templates/privileged-clusterrole.yaml +++ b/pkg/operation/botanist/charts/shoot-core/components/charts/podsecuritypolicies/templates/privileged-clusterrole.yaml @@ -1,4 +1,4 @@ -apiVersion: {{ include "rbacversion" . }} +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: gardener.cloud:psp:privileged diff --git a/charts/shoot-core/components/charts/podsecuritypolicies/templates/privileged-clusterrolebinding.yaml b/pkg/operation/botanist/charts/shoot-core/components/charts/podsecuritypolicies/templates/privileged-clusterrolebinding.yaml similarity index 94% rename from charts/shoot-core/components/charts/podsecuritypolicies/templates/privileged-clusterrolebinding.yaml rename to pkg/operation/botanist/charts/shoot-core/components/charts/podsecuritypolicies/templates/privileged-clusterrolebinding.yaml index e0e5b20b5ed..b60fa878d42 100644 --- a/charts/shoot-core/components/charts/podsecuritypolicies/templates/privileged-clusterrolebinding.yaml +++ b/pkg/operation/botanist/charts/shoot-core/components/charts/podsecuritypolicies/templates/privileged-clusterrolebinding.yaml @@ -1,4 +1,4 @@ -apiVersion: {{ include "rbacversion" . }} +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: gardener.cloud:psp:privileged diff --git a/charts/shoot-core/components/charts/podsecuritypolicies/templates/privileged-psp.yaml b/pkg/operation/botanist/charts/shoot-core/components/charts/podsecuritypolicies/templates/privileged-psp.yaml similarity index 92% rename from charts/shoot-core/components/charts/podsecuritypolicies/templates/privileged-psp.yaml rename to pkg/operation/botanist/charts/shoot-core/components/charts/podsecuritypolicies/templates/privileged-psp.yaml index e959c502694..8e24f43b83a 100644 --- a/charts/shoot-core/components/charts/podsecuritypolicies/templates/privileged-psp.yaml +++ b/pkg/operation/botanist/charts/shoot-core/components/charts/podsecuritypolicies/templates/privileged-psp.yaml @@ -1,4 +1,4 @@ -apiVersion: {{ include "podsecuritypolicyversion" .}} +apiVersion: policy/v1beta1 kind: PodSecurityPolicy metadata: name: gardener.privileged diff --git a/charts/shoot-core/components/charts/podsecuritypolicies/templates/unprivileged-clusterrole.yaml b/pkg/operation/botanist/charts/shoot-core/components/charts/podsecuritypolicies/templates/unprivileged-clusterrole.yaml similarity index 83% rename from charts/shoot-core/components/charts/podsecuritypolicies/templates/unprivileged-clusterrole.yaml rename to pkg/operation/botanist/charts/shoot-core/components/charts/podsecuritypolicies/templates/unprivileged-clusterrole.yaml index 7c6f197a5f1..895bcf25bcd 100644 --- a/charts/shoot-core/components/charts/podsecuritypolicies/templates/unprivileged-clusterrole.yaml +++ b/pkg/operation/botanist/charts/shoot-core/components/charts/podsecuritypolicies/templates/unprivileged-clusterrole.yaml @@ -1,4 +1,4 @@ -apiVersion: {{ include "rbacversion" . }} +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: gardener.cloud:psp:unprivileged diff --git a/charts/shoot-core/components/charts/podsecuritypolicies/templates/unprivileged-clusterrolebinding.yaml b/pkg/operation/botanist/charts/shoot-core/components/charts/podsecuritypolicies/templates/unprivileged-clusterrolebinding.yaml similarity index 91% rename from charts/shoot-core/components/charts/podsecuritypolicies/templates/unprivileged-clusterrolebinding.yaml rename to pkg/operation/botanist/charts/shoot-core/components/charts/podsecuritypolicies/templates/unprivileged-clusterrolebinding.yaml index 20de7df605a..9c787d9f3de 100644 --- a/charts/shoot-core/components/charts/podsecuritypolicies/templates/unprivileged-clusterrolebinding.yaml +++ b/pkg/operation/botanist/charts/shoot-core/components/charts/podsecuritypolicies/templates/unprivileged-clusterrolebinding.yaml @@ -1,4 +1,4 @@ -apiVersion: {{ include "rbacversion" .}} +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: gardener.cloud:psp:unprivileged diff --git a/charts/shoot-core/components/charts/podsecuritypolicies/templates/unprivileged-kube-system-rolebinding.yaml b/pkg/operation/botanist/charts/shoot-core/components/charts/podsecuritypolicies/templates/unprivileged-kube-system-rolebinding.yaml similarity index 91% rename from charts/shoot-core/components/charts/podsecuritypolicies/templates/unprivileged-kube-system-rolebinding.yaml rename to pkg/operation/botanist/charts/shoot-core/components/charts/podsecuritypolicies/templates/unprivileged-kube-system-rolebinding.yaml index 9db807853d4..25cf0ee0120 100644 --- a/charts/shoot-core/components/charts/podsecuritypolicies/templates/unprivileged-kube-system-rolebinding.yaml +++ b/pkg/operation/botanist/charts/shoot-core/components/charts/podsecuritypolicies/templates/unprivileged-kube-system-rolebinding.yaml @@ -1,4 +1,4 @@ -apiVersion: {{ include "rbacversion" . }} +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: gardener.cloud:psp:unprivileged diff --git a/charts/shoot-core/components/charts/podsecuritypolicies/templates/unprivileged-psp.yaml b/pkg/operation/botanist/charts/shoot-core/components/charts/podsecuritypolicies/templates/unprivileged-psp.yaml similarity index 95% rename from charts/shoot-core/components/charts/podsecuritypolicies/templates/unprivileged-psp.yaml rename to pkg/operation/botanist/charts/shoot-core/components/charts/podsecuritypolicies/templates/unprivileged-psp.yaml index c177d25ba36..b9a02ae192a 100644 --- a/charts/shoot-core/components/charts/podsecuritypolicies/templates/unprivileged-psp.yaml +++ b/pkg/operation/botanist/charts/shoot-core/components/charts/podsecuritypolicies/templates/unprivileged-psp.yaml @@ -1,4 +1,4 @@ -apiVersion: {{ include "podsecuritypolicyversion" .}} +apiVersion: policy/v1beta1 kind: PodSecurityPolicy metadata: name: gardener.unprivileged diff --git a/charts/shoot-core/components/charts/podsecuritypolicies/values.yaml b/pkg/operation/botanist/charts/shoot-core/components/charts/podsecuritypolicies/values.yaml similarity index 100% rename from charts/shoot-core/components/charts/podsecuritypolicies/values.yaml rename to pkg/operation/botanist/charts/shoot-core/components/charts/podsecuritypolicies/values.yaml diff --git a/charts/shoot-core/components/requirements.yaml b/pkg/operation/botanist/charts/shoot-core/components/requirements.yaml similarity index 56% rename from charts/shoot-core/components/requirements.yaml rename to pkg/operation/botanist/charts/shoot-core/components/requirements.yaml index 2bbfc5d4ecf..5de1fe64be5 100644 --- a/charts/shoot-core/components/requirements.yaml +++ b/pkg/operation/botanist/charts/shoot-core/components/requirements.yaml @@ -1,8 +1,4 @@ dependencies: -- name: monitoring - repository: http://localhost:10191 - version: 0.1.0 - condition: monitoring.enabled - name: podsecuritypolicies repository: http://localhost:10191 version: 0.1.0 diff --git a/charts/shoot-core/components/values.yaml b/pkg/operation/botanist/charts/shoot-core/components/values.yaml similarity index 72% rename from charts/shoot-core/components/values.yaml rename to pkg/operation/botanist/charts/shoot-core/components/values.yaml index f5176df9571..453d01bb7ee 100644 --- a/charts/shoot-core/components/values.yaml +++ b/pkg/operation/botanist/charts/shoot-core/components/values.yaml @@ -1,5 +1,3 @@ -monitoring: - enabled: true podsecuritypolicies: enabled: true allowPrivilegedContainers: false diff --git a/pkg/operation/botanist/logging.go b/pkg/operation/botanist/logging.go index f60b7dd3fca..6c582b4b0bf 100644 --- a/pkg/operation/botanist/logging.go +++ b/pkg/operation/botanist/logging.go @@ -35,8 +35,8 @@ import ( kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes" ) -// DeploySeedLogging will install the logging stack for the Shoot in the Seed clusters. -func (b *Botanist) DeploySeedLogging(ctx context.Context) error { +// DeployLogging will install the logging stack for the Shoot in the Seed clusters. +func (b *Botanist) DeployLogging(ctx context.Context) error { if !b.Shoot.IsShootControlPlaneLoggingEnabled(b.Config) { return b.DestroySeedLogging(ctx) } diff --git a/pkg/operation/botanist/logging_test.go b/pkg/operation/botanist/logging_test.go index f3220e3c8f9..9995320ef13 100644 --- a/pkg/operation/botanist/logging_test.go +++ b/pkg/operation/botanist/logging_test.go @@ -159,7 +159,7 @@ var _ = Describe("Logging", func() { ctrl.Finish() }) - Describe("#DeploySeedLogging", func() { + Describe("#DeployLogging", func() { It("should successfully delete the logging stack when shoot is with testing purpose", func() { botanist.Shoot.Purpose = shootPurposeTesting gomock.InOrder( @@ -171,7 +171,7 @@ var _ = Describe("Logging", func() { valiDeployer.EXPECT().Destroy(ctx), ) - Expect(botanist.DeploySeedLogging(ctx)).To(Succeed()) + Expect(botanist.DeployLogging(ctx)).To(Succeed()) }) It("should successfully delete the logging stack when it is disabled", func() { @@ -185,7 +185,7 @@ var _ = Describe("Logging", func() { valiDeployer.EXPECT().Destroy(ctx), ) - Expect(botanist.DeploySeedLogging(ctx)).To(Succeed()) + Expect(botanist.DeployLogging(ctx)).To(Succeed()) }) It("should successfully clean up the existing Loki based deployment and deploy all of the components in the logging stack when it is enabled", func() { @@ -276,7 +276,7 @@ var _ = Describe("Logging", func() { valiDeployer.EXPECT().Deploy(ctx), ) - Expect(botanist.DeploySeedLogging(ctx)).To(Succeed()) + Expect(botanist.DeployLogging(ctx)).To(Succeed()) }) It("should successfully deploy all of the components in the logging stack when it is enabled", func() { @@ -289,7 +289,7 @@ var _ = Describe("Logging", func() { valiDeployer.EXPECT().Deploy(ctx), ) - Expect(botanist.DeploySeedLogging(ctx)).To(Succeed()) + Expect(botanist.DeployLogging(ctx)).To(Succeed()) }) It("should not deploy event logger when it is disabled", func() { @@ -304,7 +304,7 @@ var _ = Describe("Logging", func() { valiDeployer.EXPECT().Deploy(ctx), ) - Expect(botanist.DeploySeedLogging(ctx)).To(Succeed()) + Expect(botanist.DeployLogging(ctx)).To(Succeed()) }) It("should not deploy shoot node logging for workerless shoot", func() { @@ -319,7 +319,7 @@ var _ = Describe("Logging", func() { valiDeployer.EXPECT().Deploy(ctx), ) - Expect(botanist.DeploySeedLogging(ctx)).To(Succeed()) + Expect(botanist.DeployLogging(ctx)).To(Succeed()) }) It("should not deploy shoot node logging when it is disabled", func() { @@ -334,7 +334,7 @@ var _ = Describe("Logging", func() { valiDeployer.EXPECT().Deploy(ctx), ) - Expect(botanist.DeploySeedLogging(ctx)).To(Succeed()) + Expect(botanist.DeployLogging(ctx)).To(Succeed()) }) It("should not deploy shoot node logging and Vali when Vali is disabled", func() { @@ -349,14 +349,14 @@ var _ = Describe("Logging", func() { valiDeployer.EXPECT().Destroy(ctx), ) - Expect(botanist.DeploySeedLogging(ctx)).To(Succeed()) + Expect(botanist.DeployLogging(ctx)).To(Succeed()) }) Context("Tests expecting a failure", func() { It("should fail to delete the logging stack when ShootRBACProxyDeployer Destroy returns error", func() { *botanist.Config.Logging.Enabled = false shootRBACProxyDeployer.EXPECT().Destroy(ctx).Return(fakeErr) - Expect(botanist.DeploySeedLogging(ctx)).ToNot(Succeed()) + Expect(botanist.DeployLogging(ctx)).ToNot(Succeed()) }) It("should fail to delete the logging stack when ShootEventLoggerDeployer Destroy return error", func() { @@ -368,7 +368,7 @@ var _ = Describe("Logging", func() { eventLoggerDeployer.EXPECT().Destroy(ctx).Return(fakeErr), ) - Expect(botanist.DeploySeedLogging(ctx)).ToNot(Succeed()) + Expect(botanist.DeployLogging(ctx)).ToNot(Succeed()) }) It("should fail to delete the logging stack when logging is disbaled and ShootValiDeployer Destroy return error", func() { @@ -382,7 +382,7 @@ var _ = Describe("Logging", func() { valiDeployer.EXPECT().Destroy(ctx).Return(fakeErr), ) - Expect(botanist.DeploySeedLogging(ctx)).ToNot(Succeed()) + Expect(botanist.DeployLogging(ctx)).ToNot(Succeed()) }) It("should fail to deploy the logging stack when ShootEventLoggerDeployer Deploy returns an error", func() { @@ -391,7 +391,7 @@ var _ = Describe("Logging", func() { eventLoggerDeployer.EXPECT().Deploy(ctx).Return(fakeErr), ) - Expect(botanist.DeploySeedLogging(ctx)).ToNot(Succeed()) + Expect(botanist.DeployLogging(ctx)).ToNot(Succeed()) }) It("should fail to deploy the logging stack when deploying of the shoot event logging fails", func() { @@ -401,7 +401,7 @@ var _ = Describe("Logging", func() { eventLoggerDeployer.EXPECT().Deploy(ctx).Return(fakeErr), ) - Expect(botanist.DeploySeedLogging(ctx)).ToNot(Succeed()) + Expect(botanist.DeployLogging(ctx)).ToNot(Succeed()) }) It("should fail to deploy the logging stack when KubeRBACProxyDeployer Deploy returns an error", func() { @@ -411,7 +411,7 @@ var _ = Describe("Logging", func() { shootRBACProxyDeployer.EXPECT().Deploy(ctx).Return(fakeErr), ) - Expect(botanist.DeploySeedLogging(ctx)).ToNot(Succeed()) + Expect(botanist.DeployLogging(ctx)).ToNot(Succeed()) }) It("should fail to deploy the logging stack when ValiDeployer Deploy returns error", func() { @@ -422,7 +422,7 @@ var _ = Describe("Logging", func() { valiDeployer.EXPECT().Deploy(ctx).Return(fakeErr), ) - Expect(botanist.DeploySeedLogging(ctx)).ToNot(Succeed()) + Expect(botanist.DeployLogging(ctx)).ToNot(Succeed()) }) }) }) diff --git a/pkg/operation/botanist/monitoring.go b/pkg/operation/botanist/monitoring.go index f2f572789ef..84d078212ed 100644 --- a/pkg/operation/botanist/monitoring.go +++ b/pkg/operation/botanist/monitoring.go @@ -16,58 +16,115 @@ package botanist import ( "context" - "fmt" - "path/filepath" - "strings" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - networkingv1 "k8s.io/api/networking/v1" - rbacv1 "k8s.io/api/rbac/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - vpaautoscalingv1 "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" "k8s.io/utils/pointer" - "sigs.k8s.io/controller-runtime/pkg/client" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" - "github.com/gardener/gardener/pkg/client/kubernetes" "github.com/gardener/gardener/pkg/component" - "github.com/gardener/gardener/pkg/component/etcd" - "github.com/gardener/gardener/pkg/component/plutono" - "github.com/gardener/gardener/pkg/component/shared" - "github.com/gardener/gardener/pkg/controllerutils" + "github.com/gardener/gardener/pkg/component/monitoring" "github.com/gardener/gardener/pkg/features" gardenlethelper "github.com/gardener/gardener/pkg/gardenlet/apis/config/helper" - "github.com/gardener/gardener/pkg/operation/common" - "github.com/gardener/gardener/pkg/utils" gardenerutils "github.com/gardener/gardener/pkg/utils/gardener" "github.com/gardener/gardener/pkg/utils/images" - kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes" - "github.com/gardener/gardener/pkg/utils/secrets" - secretsmanager "github.com/gardener/gardener/pkg/utils/secrets/manager" ) -const ( - secretNameIngress = v1beta1constants.SecretNameObservabilityIngressUsers -) +// DefaultMonitoring creates a new monitoring component. +func (b *Botanist) DefaultMonitoring() (monitoring.Interface, error) { + imageAlertmanager, err := b.ImageVector.FindImage(images.ImageNameAlertmanager) + if err != nil { + return nil, err + } + imageBlackboxExporter, err := b.ImageVector.FindImage(images.ImageNameBlackboxExporter) + if err != nil { + return nil, err + } + imageConfigmapReloader, err := b.ImageVector.FindImage(images.ImageNameConfigmapReloader) + if err != nil { + return nil, err + } + imagePrometheus, err := b.ImageVector.FindImage(images.ImageNamePrometheus) + if err != nil { + return nil, err + } -// DeploySeedMonitoring installs the Helm release "seed-monitoring" in the Seed clusters. It comprises components -// to monitor the Shoot cluster whose control plane runs in the Seed cluster. -func (b *Botanist) DeploySeedMonitoring(ctx context.Context) error { - if !b.IsShootMonitoringEnabled() { - return b.DeleteSeedMonitoring(ctx) + var alertingSecrets []*corev1.Secret + for _, key := range b.GetSecretKeysOfRole(v1beta1constants.GardenRoleAlerting) { + alertingSecrets = append(alertingSecrets, b.LoadSecret(key)) + } + + var wildcardSecretName *string + if b.ControlPlaneWildcardCert != nil { + wildcardSecretName = &b.ControlPlaneWildcardCert.Name + } + + values := monitoring.Values{ + AlertingSecrets: alertingSecrets, + AlertmanagerEnabled: b.Shoot.WantsAlertmanager, + APIServerDomain: gardenerutils.GetAPIServerDomain(b.Shoot.InternalClusterDomain), + APIServerHost: b.SeedClientSet.RESTConfig().Host, + Config: b.Config.Monitoring, + GardenletManagesMCM: features.DefaultFeatureGate.Enabled(features.MachineControllerManagerDeployment), + GlobalShootRemoteWriteSecret: b.LoadSecret(v1beta1constants.GardenRoleGlobalShootRemoteWriteMonitoring), + IgnoreAlerts: b.Shoot.IgnoreAlerts, + ImageAlertmanager: imageAlertmanager.String(), + ImageBlackboxExporter: imageBlackboxExporter.String(), + ImageConfigmapReloader: imageConfigmapReloader.String(), + ImagePrometheus: imagePrometheus.String(), + IngressHostAlertmanager: b.ComputeAlertManagerHost(), + IngressHostPrometheus: b.ComputePrometheusHost(), + IsWorkerless: b.Shoot.IsWorkerless, + KubernetesVersion: b.Shoot.GetInfo().Spec.Kubernetes.Version, + MonitoringConfig: b.Shoot.GetInfo().Spec.Monitoring, + NodeLocalDNSEnabled: b.Shoot.NodeLocalDNSEnabled, + ProjectName: b.Garden.Project.Name, + Replicas: b.Shoot.GetReplicas(1), + RuntimeProviderType: b.Seed.GetInfo().Spec.Provider.Type, + RuntimeRegion: b.Seed.GetInfo().Spec.Provider.Region, + StorageCapacityAlertmanager: b.Seed.GetValidVolumeSize("1Gi"), + TargetName: b.Shoot.GetInfo().Name, + TargetProviderType: b.Shoot.GetInfo().Spec.Provider.Type, + WildcardCertName: wildcardSecretName, + } + + if b.Shoot.Networks != nil { + if services := b.Shoot.Networks.Services; services != nil { + values.ServiceNetworkCIDR = pointer.String(services.String()) + } + if pods := b.Shoot.Networks.Pods; pods != nil { + values.PodNetworkCIDR = pointer.String(pods.String()) + } + if apiServer := b.Shoot.Networks.APIServer; apiServer != nil { + values.APIServerServiceIP = pointer.String(apiServer.String()) + } } - credentialsSecret, found := b.SecretsManager.Get(secretNameIngress) - if !found { - return fmt.Errorf("secret %q not found", secretNameIngress) + if b.Shoot.GetInfo().Spec.Networking != nil { + values.NodeNetworkCIDR = b.Shoot.GetInfo().Spec.Networking.Nodes + } + + return monitoring.New( + b.SeedClientSet.Client(), + b.SeedClientSet.ChartApplier(), + b.SecretsManager, + b.Shoot.SeedNamespace, + values, + ), nil +} + +// DeployMonitoring installs the Helm release "seed-monitoring" in the Seed clusters. It comprises components +// to monitor the Shoot cluster whose control plane runs in the Seed cluster. +func (b *Botanist) DeployMonitoring(ctx context.Context) error { + if !b.IsShootMonitoringEnabled() { + return b.Shoot.Components.Monitoring.Monitoring.Destroy(ctx) } - var ( - alertingRules = strings.Builder{} - scrapeConfigs = strings.Builder{} - ) + b.Shoot.Components.Monitoring.Monitoring.SetNamespaceUID(b.SeedNamespaceObject.UID) + b.Shoot.Components.Monitoring.Monitoring.SetComponents(b.getMonitoringComponents()) + return b.Shoot.Components.Monitoring.Monitoring.Deploy(ctx) +} +func (b *Botanist) getMonitoringComponents() []component.MonitoringComponent { // Fetch component-specific monitoring configuration monitoringComponents := []component.MonitoringComponent{ b.Shoot.Components.ControlPlane.EtcdMain, @@ -110,517 +167,5 @@ func (b *Botanist) DeploySeedMonitoring(ctx context.Context) error { } } - for _, component := range monitoringComponents { - componentsScrapeConfigs, err := component.ScrapeConfigs() - if err != nil { - return err - } - for _, config := range componentsScrapeConfigs { - scrapeConfigs.WriteString(fmt.Sprintf("- %s\n", utils.Indent(config, 2))) - } - - componentsAlertingRules, err := component.AlertingRules() - if err != nil { - return err - } - for filename, rule := range componentsAlertingRules { - alertingRules.WriteString(fmt.Sprintf("%s: |\n %s\n", filename, utils.Indent(rule, 2))) - } - } - - // Fetch extensions provider-specific monitoring configuration - existingConfigMaps := &corev1.ConfigMapList{} - if err := b.SeedClientSet.Client().List(ctx, existingConfigMaps, - client.InNamespace(b.Shoot.SeedNamespace), - client.MatchingLabels{v1beta1constants.LabelExtensionConfiguration: v1beta1constants.LabelMonitoring}); err != nil { - return err - } - - // Need stable order before passing the dashboards to Plutono config to avoid unnecessary changes - kubernetesutils.ByName().Sort(existingConfigMaps) - - // Read extension monitoring configurations - for _, cm := range existingConfigMaps.Items { - alertingRules.WriteString(fmt.Sprintln(cm.Data[v1beta1constants.PrometheusConfigMapAlertingRules])) - scrapeConfigs.WriteString(fmt.Sprintln(cm.Data[v1beta1constants.PrometheusConfigMapScrapeConfig])) - } - - // Create shoot token secret for prometheus component - if err := gardenerutils.NewShootAccessSecret(v1beta1constants.StatefulSetNamePrometheus, b.Shoot.SeedNamespace).Reconcile(ctx, b.SeedClientSet.Client()); err != nil { - return err - } - - alerting, err := b.getCustomAlertingConfigs(ctx, b.GetSecretKeysOfRole(v1beta1constants.GardenRoleAlerting)) - if err != nil { - return err - } - - var prometheusIngressTLSSecretName string - if b.ControlPlaneWildcardCert != nil { - prometheusIngressTLSSecretName = b.ControlPlaneWildcardCert.GetName() - } else { - ingressTLSSecret, err := b.SecretsManager.Generate(ctx, &secrets.CertificateSecretConfig{ - Name: "prometheus-tls", - CommonName: "prometheus", - Organization: []string{"gardener.cloud:monitoring:ingress"}, - DNSNames: b.ComputePrometheusHosts(), - CertType: secrets.ServerCert, - Validity: pointer.Duration(v1beta1constants.IngressTLSCertificateValidity), - SkipPublishingCACertificate: true, - }, secretsmanager.SignedByCA(v1beta1constants.SecretNameCACluster)) - if err != nil { - return err - } - prometheusIngressTLSSecretName = ingressTLSSecret.Name - } - - clusterCASecret, found := b.SecretsManager.Get(v1beta1constants.SecretNameCACluster) - if !found { - return fmt.Errorf("secret %q not found", v1beta1constants.SecretNameCACluster) - } - - etcdCASecret, found := b.SecretsManager.Get(v1beta1constants.SecretNameCAETCD) - if !found { - return fmt.Errorf("secret %q not found", v1beta1constants.SecretNameCAETCD) - } - - etcdClientSecret, found := b.SecretsManager.Get(etcd.SecretNameClient) - if !found { - return fmt.Errorf("secret %q not found", etcd.SecretNameClient) - } - - var ( - networks = map[string]interface{}{} - prometheusConfig = map[string]interface{}{ - "secretNameClusterCA": clusterCASecret.Name, - "secretNameEtcdCA": etcdCASecret.Name, - "secretNameEtcdClientCert": etcdClientSecret.Name, - "kubernetesVersion": b.Shoot.GetInfo().Spec.Kubernetes.Version, - "nodeLocalDNS": map[string]interface{}{ - "enabled": b.Shoot.NodeLocalDNSEnabled, - }, - "gardenletManagesMCM": features.DefaultFeatureGate.Enabled(features.MachineControllerManagerDeployment), - "ingress": map[string]interface{}{ - "class": v1beta1constants.SeedNginxIngressClass, - "authSecretName": credentialsSecret.Name, - "hosts": []map[string]interface{}{ - { - "hostName": b.ComputePrometheusHost(), - "secretName": prometheusIngressTLSSecretName, - }, - }, - }, - "namespace": map[string]interface{}{ - "uid": b.SeedNamespaceObject.UID, - }, - "replicas": b.Shoot.GetReplicas(1), - "seed": map[string]interface{}{ - "apiserver": b.SeedClientSet.RESTConfig().Host, - "region": b.Seed.GetInfo().Spec.Provider.Region, - "provider": b.Seed.GetInfo().Spec.Provider.Type, - }, - "rules": map[string]interface{}{ - "optional": map[string]interface{}{ - "alertmanager": map[string]interface{}{ - "enabled": b.Shoot.WantsAlertmanager, - }, - }, - }, - "shoot": map[string]interface{}{ - "apiserver": fmt.Sprintf("https://%s", gardenerutils.GetAPIServerDomain(b.Shoot.InternalClusterDomain)), - "apiserverServerName": gardenerutils.GetAPIServerDomain(b.Shoot.InternalClusterDomain), - "provider": b.Shoot.GetInfo().Spec.Provider.Type, - "name": b.Shoot.GetInfo().Name, - "project": b.Garden.Project.Name, - "workerless": b.Shoot.IsWorkerless, - }, - "ignoreAlerts": b.Shoot.IgnoreAlerts, - "alerting": alerting, - "additionalRules": alertingRules.String(), - "additionalScrapeConfigs": scrapeConfigs.String(), - } - ) - - if b.Shoot.Networks != nil { - if services := b.Shoot.Networks.Services; services != nil { - networks["services"] = services.String() - } - if pods := b.Shoot.Networks.Pods; pods != nil { - networks["pods"] = pods.String() - } - if apiServer := b.Shoot.Networks.APIServer; apiServer != nil { - prometheusConfig["apiserverServiceIP"] = apiServer.String() - } - } - if b.Shoot.GetInfo().Spec.Networking != nil && b.Shoot.GetInfo().Spec.Networking.Nodes != nil { - networks["nodes"] = *b.Shoot.GetInfo().Spec.Networking.Nodes - } - - prometheusConfig["networks"] = networks - - // Add remotewrite to prometheus when enabled - if b.Config.Monitoring != nil && - b.Config.Monitoring.Shoot != nil && - b.Config.Monitoring.Shoot.RemoteWrite != nil && - b.Config.Monitoring.Shoot.RemoteWrite.URL != "" { - // if remoteWrite Url is set add config into values - remoteWriteConfig := map[string]interface{}{ - "url": b.Config.Monitoring.Shoot.RemoteWrite.URL, - } - // get secret for basic_auth in remote write - remoteWriteBasicAuth := b.LoadSecret(v1beta1constants.GardenRoleGlobalShootRemoteWriteMonitoring) - if remoteWriteBasicAuth != nil { - remoteWriteUsername := string(remoteWriteBasicAuth.Data["username"]) - remoteWritePassword := string(remoteWriteBasicAuth.Data["password"]) - if remoteWriteUsername != "" && - remoteWritePassword != "" { - remoteWriteConfig["basic_auth"] = map[string]interface{}{ - "username": remoteWriteUsername, - "password": remoteWritePassword, - } - } - } - // add list with keep metrics if set - if len(b.Config.Monitoring.Shoot.RemoteWrite.Keep) != 0 { - remoteWriteConfig["keep"] = b.Config.Monitoring.Shoot.RemoteWrite.Keep - } - // add queue_config if set - if b.Config.Monitoring.Shoot.RemoteWrite.QueueConfig != nil && - len(*b.Config.Monitoring.Shoot.RemoteWrite.QueueConfig) != 0 { - remoteWriteConfig["queue_config"] = b.Config.Monitoring.Shoot.RemoteWrite.QueueConfig - } - prometheusConfig["remoteWrite"] = remoteWriteConfig - } - - // set externalLabels - if b.Config.Monitoring != nil && - b.Config.Monitoring.Shoot != nil && - len(b.Config.Monitoring.Shoot.ExternalLabels) != 0 { - prometheusConfig["externalLabels"] = b.Config.Monitoring.Shoot.ExternalLabels - } - - prometheus, err := b.InjectSeedShootImages(prometheusConfig, images.ImageNamePrometheus, images.ImageNameConfigmapReloader, images.ImageNameBlackboxExporter) - if err != nil { - return err - } - - coreValues := map[string]interface{}{ - "global": map[string]interface{}{ - "shootKubeVersion": map[string]interface{}{ - "gitVersion": b.Shoot.GetInfo().Spec.Kubernetes.Version, - }, - }, - "prometheus": prometheus, - } - - if err := b.SeedClientSet.ChartApplier().Apply(ctx, filepath.Join(ChartsPath, "seed-monitoring", "charts", "core"), b.Shoot.SeedNamespace, fmt.Sprintf("%s-monitoring", b.Shoot.SeedNamespace), kubernetes.Values(coreValues)); err != nil { - return err - } - - // Check if we want to deploy an alertmanager into the shoot namespace. - if b.Shoot.WantsAlertmanager { - var ( - alertingSMTPKeys = b.GetSecretKeysOfRole(v1beta1constants.GardenRoleAlerting) - emailConfigs = []map[string]interface{}{} - ) - - if b.Shoot.GetInfo().Spec.Monitoring != nil && b.Shoot.GetInfo().Spec.Monitoring.Alerting != nil { - for _, email := range b.Shoot.GetInfo().Spec.Monitoring.Alerting.EmailReceivers { - for _, key := range alertingSMTPKeys { - secret := b.LoadSecret(key) - - if string(secret.Data["auth_type"]) != "smtp" { - continue - } - emailConfigs = append(emailConfigs, map[string]interface{}{ - "to": email, - "from": string(secret.Data["from"]), - "smarthost": string(secret.Data["smarthost"]), - "auth_username": string(secret.Data["auth_username"]), - "auth_identity": string(secret.Data["auth_identity"]), - "auth_password": string(secret.Data["auth_password"]), - }) - } - } - } - - var alertManagerIngressTLSSecretName string - if b.ControlPlaneWildcardCert != nil { - alertManagerIngressTLSSecretName = b.ControlPlaneWildcardCert.GetName() - } else { - ingressTLSSecret, err := b.SecretsManager.Generate(ctx, &secrets.CertificateSecretConfig{ - Name: "alertmanager-tls", - CommonName: "alertmanager", - Organization: []string{"gardener.cloud:monitoring:ingress"}, - DNSNames: b.ComputeAlertManagerHosts(), - CertType: secrets.ServerCert, - Validity: pointer.Duration(v1beta1constants.IngressTLSCertificateValidity), - SkipPublishingCACertificate: true, - }, secretsmanager.SignedByCA(v1beta1constants.SecretNameCACluster)) - if err != nil { - return err - } - alertManagerIngressTLSSecretName = ingressTLSSecret.Name - } - - alertManagerValues, err := b.InjectSeedShootImages(map[string]interface{}{ - "ingress": map[string]interface{}{ - "class": v1beta1constants.SeedNginxIngressClass, - "authSecretName": credentialsSecret.Name, - "hosts": []map[string]interface{}{ - { - "hostName": b.ComputeAlertManagerHost(), - "secretName": alertManagerIngressTLSSecretName, - }, - }, - }, - "replicas": b.Shoot.GetReplicas(1), - "storage": b.Seed.GetValidVolumeSize("1Gi"), - "emailConfigs": emailConfigs, - }, images.ImageNameAlertmanager, images.ImageNameConfigmapReloader) - if err != nil { - return err - } - - return b.SeedClientSet.ChartApplier().Apply(ctx, filepath.Join(ChartsPath, "seed-monitoring", "charts", "alertmanager"), b.Shoot.SeedNamespace, fmt.Sprintf("%s-monitoring", b.Shoot.SeedNamespace), kubernetes.Values(alertManagerValues)) - } - - return common.DeleteAlertmanager(ctx, b.SeedClientSet.Client(), b.Shoot.SeedNamespace) -} - -// DefaultPlutono returns a deployer for Plutono. -func (b *Botanist) DefaultPlutono() (plutono.Interface, error) { - var wildcardCertName *string - if b.ControlPlaneWildcardCert != nil { - wildcardCertName = pointer.String(b.ControlPlaneWildcardCert.GetName()) - } - - return shared.NewPlutono( - b.SeedClientSet.Client(), - b.Shoot.SeedNamespace, - b.ImageVector, - b.SecretsManager, - "", - component.ClusterTypeShoot, - b.ComputePlutonoHost(), - b.ShootUsesDNS(), - features.DefaultFeatureGate.Enabled(features.MachineControllerManagerDeployment), - b.Shoot.NodeLocalDNSEnabled, - b.Shoot.IsWorkerless, - b.Shoot.VPNHighAvailabilityEnabled, - v1beta1constants.PriorityClassNameShootControlPlane100, - b.Shoot.GetReplicas(1), - wildcardCertName, - b.Shoot.WantsVerticalPodAutoscaler, - ) -} - -// DeploySeedPlutono deploys the plutono in the Seed cluster. -func (b *Botanist) DeploySeedPlutono(ctx context.Context) error { - // disable monitoring if shoot has purpose testing or monitoring and vali is disabled - if !b.Operation.WantsPlutono() { - if err := b.Shoot.Components.ControlPlane.Plutono.Destroy(ctx); err != nil { - return err - } - - secretName := gardenerutils.ComputeShootProjectSecretName(b.Shoot.GetInfo().Name, gardenerutils.ShootProjectSecretSuffixMonitoring) - return kubernetesutils.DeleteObject(ctx, b.GardenClient, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: secretName, Namespace: b.Shoot.GetInfo().Namespace}}) - } - - // TODO(rickardsjp, istvanballok): Remove in release v1.77 once the Grafana to Plutono migration is complete. - if err := b.DeleteGrafana(ctx); err != nil { - return err - } - - if err := b.Shoot.Components.ControlPlane.Plutono.Deploy(ctx); err != nil { - return err - } - - credentialsSecret, found := b.SecretsManager.Get(secretNameIngress) - if !found { - return fmt.Errorf("secret %q not found", secretNameIngress) - } - - return b.syncShootCredentialToGarden( - ctx, - gardenerutils.ShootProjectSecretSuffixMonitoring, - map[string]string{v1beta1constants.GardenRole: v1beta1constants.GardenRoleMonitoring}, - map[string]string{"url": "https://" + b.ComputePlutonoHost()}, - credentialsSecret.Data, - ) -} - -func (b *Botanist) getCustomAlertingConfigs(ctx context.Context, alertingSecretKeys []string) (map[string]interface{}, error) { - configs := map[string]interface{}{ - "auth_type": map[string]interface{}{}, - } - - for _, key := range alertingSecretKeys { - secret := b.LoadSecret(key) - - if string(secret.Data["auth_type"]) == "none" { - - if url, ok := secret.Data["url"]; ok { - configs["auth_type"] = map[string]interface{}{ - "none": map[string]interface{}{ - "url": string(url), - }, - } - } - break - } - - if string(secret.Data["auth_type"]) == "basic" { - url, urlOk := secret.Data["url"] - username, usernameOk := secret.Data["username"] - password, passwordOk := secret.Data["password"] - - if urlOk && usernameOk && passwordOk { - configs["auth_type"] = map[string]interface{}{ - "basic": map[string]interface{}{ - "url": string(url), - "username": string(username), - "password": string(password), - }, - } - } - break - } - - if string(secret.Data["auth_type"]) == "certificate" { - data := map[string][]byte{} - url, urlOk := secret.Data["url"] - ca, caOk := secret.Data["ca.crt"] - cert, certOk := secret.Data["tls.crt"] - key, keyOk := secret.Data["tls.key"] - insecure, insecureOk := secret.Data["insecure_skip_verify"] - - if urlOk && caOk && certOk && keyOk && insecureOk { - configs["auth_type"] = map[string]interface{}{ - "certificate": map[string]interface{}{ - "url": string(url), - "insecure_skip_verify": string(insecure), - }, - } - data["ca.crt"] = ca - data["tls.crt"] = cert - data["tls.key"] = key - amSecret := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "prometheus-remote-am-tls", - Namespace: b.Shoot.SeedNamespace, - }, - } - - if _, err := controllerutils.GetAndCreateOrMergePatch(ctx, b.SeedClientSet.Client(), amSecret, func() error { - amSecret.Data = data - amSecret.Type = corev1.SecretTypeOpaque - return nil - }); err != nil { - return nil, err - } - } - break - } - } - - return configs, nil -} - -// DeleteGrafana will delete all Grafana resources from the seed cluster. -func (b *Botanist) DeleteGrafana(ctx context.Context) error { - return common.DeleteGrafana(ctx, b.SeedClientSet, b.Shoot.SeedNamespace) -} - -// DeleteSeedMonitoring will delete the monitoring stack from the Seed cluster to avoid phantom alerts -// during the deletion process. More precisely, the Alertmanager and Prometheus StatefulSets will be -// deleted. -func (b *Botanist) DeleteSeedMonitoring(ctx context.Context) error { - if err := common.DeleteAlertmanager(ctx, b.SeedClientSet.Client(), b.Shoot.SeedNamespace); err != nil { - return err - } - - objects := []client.Object{ - &networkingv1.NetworkPolicy{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: b.Shoot.SeedNamespace, - Name: "allow-from-prometheus", - }, - }, - &networkingv1.NetworkPolicy{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: b.Shoot.SeedNamespace, - Name: "allow-prometheus", - }, - }, - gardenerutils.NewShootAccessSecret(v1beta1constants.StatefulSetNamePrometheus, b.Shoot.SeedNamespace).Secret, - &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: b.Shoot.SeedNamespace, - Name: "prometheus-config", - }, - }, - &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: b.Shoot.SeedNamespace, - Name: "prometheus-rules", - }, - }, - &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: b.Shoot.SeedNamespace, - Name: "blackbox-exporter-config-prometheus", - }, - }, - &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: b.Shoot.SeedNamespace, - Name: "prometheus-basic-auth", - }, - }, - &networkingv1.Ingress{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: b.Shoot.SeedNamespace, - Name: "prometheus", - }, - }, - &vpaautoscalingv1.VerticalPodAutoscaler{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: b.Shoot.SeedNamespace, - Name: "prometheus-vpa", - }, - }, - &corev1.ServiceAccount{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: b.Shoot.SeedNamespace, - Name: "prometheus", - }, - }, - &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: b.Shoot.SeedNamespace, - Name: "prometheus-web", - }, - }, - &appsv1.StatefulSet{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: b.Shoot.SeedNamespace, - Name: "prometheus", - }, - }, - &rbacv1.ClusterRoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: b.Shoot.SeedNamespace, - Name: "prometheus-" + b.Shoot.SeedNamespace, - }, - }, - &corev1.PersistentVolumeClaim{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: b.Shoot.SeedNamespace, - Name: "prometheus-db-prometheus-0", - }, - }, - } - - return kubernetesutils.DeleteObjects(ctx, b.SeedClientSet.Client(), objects...) + return monitoringComponents } diff --git a/pkg/operation/botanist/nginxingress.go b/pkg/operation/botanist/nginxingress.go index 498df53b43a..ddc7686eac1 100644 --- a/pkg/operation/botanist/nginxingress.go +++ b/pkg/operation/botanist/nginxingress.go @@ -27,7 +27,6 @@ import ( extensionsdnsrecord "github.com/gardener/gardener/pkg/component/extensions/dnsrecord" sharedcomponent "github.com/gardener/gardener/pkg/component/shared" "github.com/gardener/gardener/pkg/controllerutils" - "github.com/gardener/gardener/pkg/operation/common" "github.com/gardener/gardener/pkg/utils" ) @@ -99,7 +98,7 @@ func (b *Botanist) NeedsIngressDNS() bool { // DefaultIngressDNSRecord creates the default deployer for the ingress DNSRecord resource. func (b *Botanist) DefaultIngressDNSRecord() extensionsdnsrecord.Interface { values := &extensionsdnsrecord.Values{ - Name: b.Shoot.GetInfo().Name + "-" + common.ShootDNSIngressName, + Name: b.Shoot.GetInfo().Name + "-ingress", SecretName: DNSRecordSecretPrefix + "-" + b.Shoot.GetInfo().Name + "-" + v1beta1constants.DNSRecordExternalName, Namespace: b.Shoot.SeedNamespace, TTL: b.Config.Controllers.Shoot.DNSEntryTTLSeconds, diff --git a/pkg/operation/botanist/nginxingress_test.go b/pkg/operation/botanist/nginxingress_test.go index 40ffacc1dd5..d847e610315 100644 --- a/pkg/operation/botanist/nginxingress_test.go +++ b/pkg/operation/botanist/nginxingress_test.go @@ -47,7 +47,6 @@ import ( "github.com/gardener/gardener/pkg/gardenlet/apis/config" "github.com/gardener/gardener/pkg/operation" . "github.com/gardener/gardener/pkg/operation/botanist" - "github.com/gardener/gardener/pkg/operation/common" "github.com/gardener/gardener/pkg/operation/garden" shootpkg "github.com/gardener/gardener/pkg/operation/shoot" gardenerutils "github.com/gardener/gardener/pkg/utils/gardener" @@ -241,7 +240,7 @@ var _ = Describe("NginxIngress", func() { actual := c.GetValues() Expect(actual).To(DeepEqual(&dnsrecord.Values{ - Name: b.Shoot.GetInfo().Name + "-" + common.ShootDNSIngressName, + Name: b.Shoot.GetInfo().Name + "-ingress", SecretName: DNSRecordSecretPrefix + "-" + b.Shoot.GetInfo().Name + "-" + v1beta1constants.DNSRecordExternalName, Namespace: seedNamespace, TTL: pointer.Int64(ttl), @@ -287,7 +286,7 @@ var _ = Describe("NginxIngress", func() { actual := c.GetValues() Expect(actual).To(DeepEqual(&dnsrecord.Values{ - Name: b.Shoot.GetInfo().Name + "-" + common.ShootDNSIngressName, + Name: b.Shoot.GetInfo().Name + "-ingress", SecretName: DNSRecordSecretPrefix + "-" + b.Shoot.GetInfo().Name + "-" + v1beta1constants.DNSRecordExternalName, Namespace: seedNamespace, TTL: pointer.Int64(ttl), @@ -315,7 +314,7 @@ var _ = Describe("NginxIngress", func() { Expect(c.Deploy(ctx)).ToNot(HaveOccurred()) dnsRecord := &extensionsv1alpha1.DNSRecord{} - err := client.Get(ctx, types.NamespacedName{Name: shootName + "-" + common.ShootDNSIngressName, Namespace: seedNamespace}, dnsRecord) + err := client.Get(ctx, types.NamespacedName{Name: shootName + "-ingress", Namespace: seedNamespace}, dnsRecord) Expect(err).ToNot(HaveOccurred()) Expect(dnsRecord).To(DeepDerivativeEqual(&extensionsv1alpha1.DNSRecord{ TypeMeta: metav1.TypeMeta{ @@ -323,7 +322,7 @@ var _ = Describe("NginxIngress", func() { APIVersion: "extensions.gardener.cloud/v1alpha1", }, ObjectMeta: metav1.ObjectMeta{ - Name: shootName + "-" + common.ShootDNSIngressName, + Name: shootName + "-ingress", Namespace: seedNamespace, ResourceVersion: "1", Annotations: map[string]string{ diff --git a/pkg/operation/botanist/plutono.go b/pkg/operation/botanist/plutono.go new file mode 100644 index 00000000000..34f66c9326c --- /dev/null +++ b/pkg/operation/botanist/plutono.go @@ -0,0 +1,95 @@ +// Copyright 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package botanist + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + "github.com/gardener/gardener/pkg/component" + "github.com/gardener/gardener/pkg/component/plutono" + "github.com/gardener/gardener/pkg/component/shared" + "github.com/gardener/gardener/pkg/features" + "github.com/gardener/gardener/pkg/operation/common" + gardenerutils "github.com/gardener/gardener/pkg/utils/gardener" + kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes" +) + +// DefaultPlutono returns a deployer for Plutono. +func (b *Botanist) DefaultPlutono() (plutono.Interface, error) { + var wildcardCertName *string + if b.ControlPlaneWildcardCert != nil { + wildcardCertName = pointer.String(b.ControlPlaneWildcardCert.GetName()) + } + + return shared.NewPlutono( + b.SeedClientSet.Client(), + b.Shoot.SeedNamespace, + b.ImageVector, + b.SecretsManager, + "", + component.ClusterTypeShoot, + b.ComputePlutonoHost(), + b.ShootUsesDNS(), + features.DefaultFeatureGate.Enabled(features.MachineControllerManagerDeployment), + b.Shoot.NodeLocalDNSEnabled, + b.Shoot.IsWorkerless, + b.Shoot.VPNHighAvailabilityEnabled, + v1beta1constants.PriorityClassNameShootControlPlane100, + b.Shoot.GetReplicas(1), + wildcardCertName, + b.Shoot.WantsVerticalPodAutoscaler, + ) +} + +// DeployPlutono deploys the plutono in the Seed cluster. +func (b *Botanist) DeployPlutono(ctx context.Context) error { + // disable monitoring if shoot has purpose testing or monitoring and vali is disabled + if !b.Operation.WantsPlutono() { + if err := b.Shoot.Components.ControlPlane.Plutono.Destroy(ctx); err != nil { + return err + } + + secretName := gardenerutils.ComputeShootProjectSecretName(b.Shoot.GetInfo().Name, gardenerutils.ShootProjectSecretSuffixMonitoring) + return kubernetesutils.DeleteObject(ctx, b.GardenClient, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: secretName, Namespace: b.Shoot.GetInfo().Namespace}}) + } + + // TODO(rickardsjp, istvanballok): Remove in release v1.77 once the Grafana to Plutono migration is complete. + if err := common.DeleteGrafana(ctx, b.SeedClientSet, b.Shoot.SeedNamespace); err != nil { + return err + } + + if err := b.Shoot.Components.ControlPlane.Plutono.Deploy(ctx); err != nil { + return err + } + + credentialsSecret, found := b.SecretsManager.Get(v1beta1constants.SecretNameObservabilityIngressUsers) + if !found { + return fmt.Errorf("secret %q not found", v1beta1constants.SecretNameObservabilityIngressUsers) + } + + return b.syncShootCredentialToGarden( + ctx, + gardenerutils.ShootProjectSecretSuffixMonitoring, + map[string]string{v1beta1constants.GardenRole: v1beta1constants.GardenRoleMonitoring}, + map[string]string{"url": "https://" + b.ComputePlutonoHost()}, + credentialsSecret.Data, + ) +} diff --git a/pkg/operation/botanist/monitoring_test.go b/pkg/operation/botanist/plutono_test.go similarity index 60% rename from pkg/operation/botanist/monitoring_test.go rename to pkg/operation/botanist/plutono_test.go index 88c0dc8a233..92095b082f3 100644 --- a/pkg/operation/botanist/monitoring_test.go +++ b/pkg/operation/botanist/plutono_test.go @@ -1,4 +1,4 @@ -// Copyright 2022 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// Copyright 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -21,33 +21,17 @@ import ( "github.com/golang/mock/gomock" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/version" "k8s.io/client-go/rest" "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" - "github.com/gardener/gardener/pkg/chartrenderer" "github.com/gardener/gardener/pkg/client/kubernetes" "github.com/gardener/gardener/pkg/client/kubernetes/fake" - mockcoredns "github.com/gardener/gardener/pkg/component/coredns/mock" - mocketcd "github.com/gardener/gardener/pkg/component/etcd/mock" - mockkubeapiserver "github.com/gardener/gardener/pkg/component/kubeapiserver/mock" - mockkubecontrollermanager "github.com/gardener/gardener/pkg/component/kubecontrollermanager/mock" - mockkubeproxy "github.com/gardener/gardener/pkg/component/kubeproxy/mock" - mockkubescheduler "github.com/gardener/gardener/pkg/component/kubescheduler/mock" - mockkubestatemetrics "github.com/gardener/gardener/pkg/component/kubestatemetrics/mock" mockcomponent "github.com/gardener/gardener/pkg/component/mock" - mockresourcemanager "github.com/gardener/gardener/pkg/component/resourcemanager/mock" - mockvpa "github.com/gardener/gardener/pkg/component/vpa/mock" - mockvpnseedserver "github.com/gardener/gardener/pkg/component/vpnseedserver/mock" - mockvpnshoot "github.com/gardener/gardener/pkg/component/vpnshoot/mock" "github.com/gardener/gardener/pkg/gardenlet/apis/config" "github.com/gardener/gardener/pkg/operation" . "github.com/gardener/gardener/pkg/operation/botanist" @@ -61,7 +45,7 @@ import ( . "github.com/gardener/gardener/pkg/utils/test/matchers" ) -var _ = Describe("Monitoring", func() { +var _ = Describe("Plutono", func() { var ( ctrl *gomock.Controller @@ -69,22 +53,9 @@ var _ = Describe("Monitoring", func() { seedClient client.Client seedClientSet kubernetes.Interface - chartApplier kubernetes.ChartApplier - sm secretsmanager.Interface - - mockEtcdMain *mocketcd.MockInterface - mockEtcdEvents *mocketcd.MockInterface - mockKubeAPIServer *mockkubeapiserver.MockInterface - mockKubeScheduler *mockkubescheduler.MockInterface - mockKubeControllerManager *mockkubecontrollermanager.MockInterface - mockKubeStateMetrics *mockkubestatemetrics.MockInterface - mockCoreDNS *mockcoredns.MockInterface - mockKubeProxy *mockkubeproxy.MockInterface - mockPlutono *mockcomponent.MockDeployWaiter - mockVPNSeedServer *mockvpnseedserver.MockInterface - mockVPNShoot *mockvpnshoot.MockInterface - mockResourceManager *mockresourcemanager.MockInterface - mockVPA *mockvpa.MockInterface + sm secretsmanager.Interface + + mockPlutono *mockcomponent.MockDeployWaiter botanist *Botanist @@ -103,13 +74,8 @@ var _ = Describe("Monitoring", func() { gardenClient = fakeclient.NewClientBuilder().WithScheme(kubernetes.GardenScheme).Build() seedClient = fakeclient.NewClientBuilder().WithScheme(kubernetes.SeedScheme).Build() - mapper := meta.NewDefaultRESTMapper([]schema.GroupVersion{corev1.SchemeGroupVersion, appsv1.SchemeGroupVersion}) - renderer := chartrenderer.NewWithServerVersion(&version.Info{GitVersion: "1.2.3"}) - chartApplier = kubernetes.NewChartApplier(renderer, kubernetes.NewApplier(seedClient, mapper)) - seedClientSet = fake.NewClientSetBuilder(). WithClient(seedClient). - WithChartApplier(chartApplier). WithRESTConfig(&rest.Config{}). Build() sm = fakesecretsmanager.New(seedClient, seedNamespace) @@ -125,19 +91,7 @@ var _ = Describe("Monitoring", func() { Data: map[string][]byte{"username": {}, "password": {}, "auth": {}}, })).To(Succeed()) - mockEtcdMain = mocketcd.NewMockInterface(ctrl) - mockEtcdEvents = mocketcd.NewMockInterface(ctrl) - mockKubeAPIServer = mockkubeapiserver.NewMockInterface(ctrl) - mockKubeScheduler = mockkubescheduler.NewMockInterface(ctrl) - mockKubeControllerManager = mockkubecontrollermanager.NewMockInterface(ctrl) - mockKubeStateMetrics = mockkubestatemetrics.NewMockInterface(ctrl) - mockCoreDNS = mockcoredns.NewMockInterface(ctrl) - mockKubeProxy = mockkubeproxy.NewMockInterface(ctrl) mockPlutono = mockcomponent.NewMockDeployWaiter(ctrl) - mockVPNSeedServer = mockvpnseedserver.NewMockInterface(ctrl) - mockVPNShoot = mockvpnshoot.NewMockInterface(ctrl) - mockResourceManager = mockresourcemanager.NewMockInterface(ctrl) - mockVPA = mockvpa.NewMockInterface(ctrl) botanist = &Botanist{ Operation: &operation.Operation{ @@ -158,30 +112,12 @@ var _ = Describe("Monitoring", func() { }, Components: &shootpkg.Components{ ControlPlane: &shootpkg.ControlPlane{ - EtcdMain: mockEtcdMain, - EtcdEvents: mockEtcdEvents, - KubeAPIServer: mockKubeAPIServer, - KubeScheduler: mockKubeScheduler, - KubeControllerManager: mockKubeControllerManager, - KubeStateMetrics: mockKubeStateMetrics, - Plutono: mockPlutono, - ResourceManager: mockResourceManager, - VerticalPodAutoscaler: mockVPA, - VPNSeedServer: mockVPNSeedServer, - }, - SystemComponents: &shootpkg.SystemComponents{ - CoreDNS: mockCoreDNS, - KubeProxy: mockKubeProxy, - VPNShoot: mockVPNShoot, + Plutono: mockPlutono, }, }, }, ImageVector: imagevector.ImageVector{ {Name: "plutono"}, - {Name: "prometheus"}, - {Name: "configmap-reloader"}, - {Name: "blackbox-exporter"}, - {Name: "kube-state-metrics"}, }, }, } @@ -210,11 +146,11 @@ var _ = Describe("Monitoring", func() { ctrl.Finish() }) - Describe("#DeploySeedPlutono", func() { + Describe("#DeployPlutono", func() { It("should successfully deploy plutono sync the ingress credentials for the users observability to the garden project namespace", func() { Expect(gardenClient.Get(ctx, kubernetesutils.Key(projectNamespace, shootName+".monitoring"), &corev1.Secret{})).To(BeNotFoundError()) mockPlutono.EXPECT().Deploy(ctx) - Expect(botanist.DeploySeedPlutono(ctx)).To(Succeed()) + Expect(botanist.DeployPlutono(ctx)).To(Succeed()) secret := &corev1.Secret{} Expect(gardenClient.Get(ctx, kubernetesutils.Key(projectNamespace, shootName+".monitoring"), secret)).To(Succeed()) @@ -226,13 +162,13 @@ var _ = Describe("Monitoring", func() { It("should cleanup the secrets when shoot purpose is changed", func() { Expect(gardenClient.Get(ctx, kubernetesutils.Key(projectNamespace, shootName+".monitoring"), &corev1.Secret{})).To(BeNotFoundError()) mockPlutono.EXPECT().Deploy(ctx) - Expect(botanist.DeploySeedPlutono(ctx)).To(Succeed()) + Expect(botanist.DeployPlutono(ctx)).To(Succeed()) Expect(gardenClient.Get(ctx, kubernetesutils.Key(projectNamespace, shootName+".monitoring"), &corev1.Secret{})).To(Succeed()) Expect(*botanist.Shoot.GetInfo().Spec.Purpose == shootPurposeEvaluation).To(BeTrue()) botanist.Shoot.Purpose = shootPurposeTesting mockPlutono.EXPECT().Destroy(ctx) - Expect(botanist.DeploySeedPlutono(ctx)).To(Succeed()) + Expect(botanist.DeployPlutono(ctx)).To(Succeed()) Expect(gardenClient.Get(ctx, kubernetesutils.Key(projectNamespace, shootName+".monitoring"), &corev1.Secret{})).To(BeNotFoundError()) }) diff --git a/pkg/operation/botanist/tunnel_test.go b/pkg/operation/botanist/tunnel_test.go index ee7a15b58fe..8f74f1331d9 100644 --- a/pkg/operation/botanist/tunnel_test.go +++ b/pkg/operation/botanist/tunnel_test.go @@ -32,7 +32,6 @@ import ( "github.com/gardener/gardener/pkg/client/kubernetes/fake" mockclient "github.com/gardener/gardener/pkg/mock/controller-runtime/client" "github.com/gardener/gardener/pkg/operation/botanist" - "github.com/gardener/gardener/pkg/operation/common" "github.com/gardener/gardener/pkg/utils/test" ) @@ -55,7 +54,7 @@ var _ = Describe("Tunnel", func() { ctx = context.Background() cl = mockclient.NewMockClient(ctrl) log = logr.Discard() - tunnelName = common.VPNTunnel + tunnelName = "vpn-shoot" tunnelPod = corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: metav1.NamespaceSystem, diff --git a/pkg/operation/botanist/waiter.go b/pkg/operation/botanist/waiter.go index 5d3e3c8d891..4287f7bba0f 100644 --- a/pkg/operation/botanist/waiter.go +++ b/pkg/operation/botanist/waiter.go @@ -25,7 +25,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/gardener/gardener/pkg/operation/common" + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes" "github.com/gardener/gardener/pkg/utils/retry" ) @@ -49,7 +49,7 @@ func (b *Botanist) WaitUntilTunnelConnectionExists(ctx context.Context) error { const timeout = 15 * time.Minute return retry.UntilTimeout(ctx, 5*time.Second, timeout, func(ctx context.Context) (bool, error) { - return CheckTunnelConnection(ctx, b.Logger, b.ShootClientSet, common.VPNTunnel) + return CheckTunnelConnection(ctx, b.Logger, b.ShootClientSet, v1beta1constants.VPNTunnel) }) } diff --git a/pkg/operation/care/health.go b/pkg/operation/care/health.go index 530c73a2918..02964667a30 100644 --- a/pkg/operation/care/health.go +++ b/pkg/operation/care/health.go @@ -33,6 +33,7 @@ import ( apiextensions "github.com/gardener/gardener/pkg/api/extensions" "github.com/gardener/gardener/pkg/apis/core" gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" v1beta1helper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper" extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" resourcesv1alpha1 "github.com/gardener/gardener/pkg/apis/resources/v1alpha1" @@ -42,7 +43,6 @@ import ( gardenlethelper "github.com/gardener/gardener/pkg/gardenlet/apis/config/helper" "github.com/gardener/gardener/pkg/operation" "github.com/gardener/gardener/pkg/operation/botanist" - "github.com/gardener/gardener/pkg/operation/common" "github.com/gardener/gardener/pkg/operation/shoot" "github.com/gardener/gardener/pkg/utils/flow" gardenerutils "github.com/gardener/gardener/pkg/utils/gardener" @@ -454,7 +454,7 @@ func (h *Health) checkSystemComponents( return &c, nil } - if established, err := botanist.CheckTunnelConnection(ctx, logr.Discard(), h.shootClient, common.VPNTunnel); err != nil || !established { + if established, err := botanist.CheckTunnelConnection(ctx, logr.Discard(), h.shootClient, v1beta1constants.VPNTunnel); err != nil || !established { msg := "Tunnel connection has not been established" if err != nil { msg += fmt.Sprintf(" (%+v)", err) diff --git a/pkg/operation/common/common_suite_test.go b/pkg/operation/common/common_suite_test.go deleted file mode 100644 index 2ed4e1ed5ad..00000000000 --- a/pkg/operation/common/common_suite_test.go +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2018 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package common_test - -import ( - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func TestCommon(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Operation Common Suite") -} diff --git a/pkg/operation/common/types.go b/pkg/operation/common/types.go deleted file mode 100644 index a5e7a4459f1..00000000000 --- a/pkg/operation/common/types.go +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2018 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package common - -const ( - // VPNTunnel dictates that VPN is used as a tunnel between seed and shoot networks. - VPNTunnel string = "vpn-shoot" - - // KubeAPIServerPrefix is a constant for a prefix used for the KubeAPIServer instance which uses a TLS certificate from a trusted origin. - KubeAPIServerPrefix = "api" - - // PlutonoUsersPrefix is a constant for a prefix used for the users Plutono instance. - PlutonoUsersPrefix = "gu" - - // PrometheusPrefix is a constant for a prefix used for the Prometheus instance. - PrometheusPrefix = "p" - - // AlertManagerPrefix is a constant for a prefix used for the AlertManager instance. - AlertManagerPrefix = "au" - - // ValiPrefix is a constant for a prefix used for the Vali instance. - ValiPrefix = "v" - - // ManagedResourceShootCoreName is the name of the shoot core managed resource. - ManagedResourceShootCoreName = "shoot-core" - // ManagedResourceAddonsName is the name of the addons managed resource. - ManagedResourceAddonsName = "addons" - - // ShootDNSIngressName is a constant for the DNS resources used for the shoot ingress addon. - ShootDNSIngressName = "ingress" -) diff --git a/pkg/operation/common/utils.go b/pkg/operation/common/utils.go index c7493a47f17..20b100c2329 100644 --- a/pkg/operation/common/utils.go +++ b/pkg/operation/common/utils.go @@ -17,9 +17,6 @@ package common import ( "context" "fmt" - "math/big" - "net" - "strings" "time" hvpav1alpha1 "github.com/gardener/hvpa-controller/api/v1alpha1" @@ -37,67 +34,7 @@ import ( kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes" ) -// FilterEntriesByPrefix returns a list of strings which begin with the given prefix. -func FilterEntriesByPrefix(prefix string, entries []string) []string { - var result []string - for _, entry := range entries { - if strings.HasPrefix(entry, prefix) { - result = append(result, entry) - } - } - return result -} - -// ComputeOffsetIP parses the provided and offsets with the value of . -// For example, = 100.64.0.0/11 and = 10 the result would be 100.64.0.10 -// IPv6 and IPv4 is supported. -func ComputeOffsetIP(subnet *net.IPNet, offset int64) (net.IP, error) { - if subnet == nil { - return nil, fmt.Errorf("subnet is nil") - } - - isIPv6 := false - - bytes := subnet.IP.To4() - if bytes == nil { - isIPv6 = true - bytes = subnet.IP.To16() - } - - ip := net.IP(big.NewInt(0).Add(big.NewInt(0).SetBytes(bytes), big.NewInt(offset)).Bytes()) - - if !subnet.Contains(ip) { - return nil, fmt.Errorf("cannot compute IP with offset %d - subnet %q too small", offset, subnet) - } - - // there is no broadcast address on IPv6 - if isIPv6 { - return ip, nil - } - - for i := range ip { - // IP address is not the same, so it's not the broadcast ip. - if ip[i] != ip[i]|^subnet.Mask[i] { - return ip.To4(), nil - } - } - - return nil, fmt.Errorf("computed IPv4 address %q is broadcast for subnet %q", ip, subnet) -} - -// GenerateAddonConfig returns the provided in case is true. Otherwise, nil is -// being returned. -func GenerateAddonConfig(values map[string]interface{}, enabled bool) map[string]interface{} { - v := map[string]interface{}{ - "enabled": enabled, - } - if enabled { - for key, value := range values { - v[key] = value - } - } - return v -} +// TODO(rickardsjp, istvanballok): remove this package in release v1.77 // LokiPvcExists checks if the loki-loki-0 PVC exists in the given namespace. func LokiPvcExists(ctx context.Context, k8sClient client.Client, namespace string, log logr.Logger) (bool, error) { @@ -130,7 +67,7 @@ func DeleteLokiRetainPvc(ctx context.Context, k8sClient client.Client, namespace &networkingv1.Ingress{ObjectMeta: metav1.ObjectMeta{Name: "loki", Namespace: namespace}}, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "shoot-access-promtail", Namespace: namespace}}, // We retain the PVC and reuse it with Vali. - //&corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: "loki-loki-0", Namespace: namespace}}, + // &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: "loki-loki-0", Namespace: namespace}}, } // Loki currently needs 30s to terminate because after 1s of graceful shutdown preparation it waits for 30s until it is eventually @@ -511,56 +448,6 @@ func RenameLokiPvcToValiPvc(ctx context.Context, k8sClient client.Client, namesp return nil } -// DeleteAlertmanager deletes all resources of the Alertmanager in a given namespace. -func DeleteAlertmanager(ctx context.Context, k8sClient client.Client, namespace string) error { - objs := []client.Object{ - &appsv1.StatefulSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: v1beta1constants.StatefulSetNameAlertManager, - Namespace: namespace, - }, - }, - &networkingv1.Ingress{ - ObjectMeta: metav1.ObjectMeta{ - Name: "alertmanager", - Namespace: namespace, - }, - }, - &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "alertmanager-client", - Namespace: namespace, - }, - }, - &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "alertmanager", - Namespace: namespace, - }, - }, - &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "alertmanager-basic-auth", - Namespace: namespace, - }, - }, - &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "alertmanager-config", - Namespace: namespace, - }, - }, - &corev1.PersistentVolumeClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: "alertmanager-db-alertmanager-0", - Namespace: namespace, - }, - }, - } - - return kubernetesutils.DeleteObjects(ctx, k8sClient, objs...) -} - // DeleteGrafana deletes the Grafana resources that are no longer necessary due to the migration to Plutono. func DeleteGrafana(ctx context.Context, k8sClient kubernetes.Interface, namespace string) error { if k8sClient == nil { diff --git a/pkg/operation/common/utils_test.go b/pkg/operation/common/utils_test.go deleted file mode 100644 index e64068c1207..00000000000 --- a/pkg/operation/common/utils_test.go +++ /dev/null @@ -1,226 +0,0 @@ -// Copyright 2018 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package common_test - -import ( - "fmt" - "net" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - . "github.com/gardener/gardener/pkg/operation/common" -) - -var _ = Describe("common", func() { - Describe("utils", func() { - Describe("#ComputeOffsetIP", func() { - Context("IPv4", func() { - It("should return a cluster IPv4 IP", func() { - _, subnet, _ := net.ParseCIDR("100.64.0.0/13") - result, err := ComputeOffsetIP(subnet, 10) - - Expect(err).NotTo(HaveOccurred()) - - Expect(result).To(HaveLen(net.IPv4len)) - Expect(result).To(Equal(net.ParseIP("100.64.0.10").To4())) - }) - - It("should return error if subnet nil is passed", func() { - result, err := ComputeOffsetIP(nil, 10) - - Expect(err).To(HaveOccurred()) - Expect(result).To(BeNil()) - }) - - It("should return error if subnet is not big enough is passed", func() { - _, subnet, _ := net.ParseCIDR("100.64.0.0/32") - result, err := ComputeOffsetIP(subnet, 10) - - Expect(err).To(HaveOccurred()) - Expect(result).To(BeNil()) - }) - - It("should return error if ip address is broadcast ip", func() { - _, subnet, _ := net.ParseCIDR("10.0.0.0/24") - result, err := ComputeOffsetIP(subnet, 255) - - Expect(err).To(HaveOccurred()) - Expect(result).To(BeNil()) - }) - }) - - Context("IPv6", func() { - It("should return a cluster IPv6 IP", func() { - _, subnet, _ := net.ParseCIDR("fc00::/8") - result, err := ComputeOffsetIP(subnet, 10) - - Expect(err).NotTo(HaveOccurred()) - Expect(result).To(HaveLen(net.IPv6len)) - Expect(result).To(Equal(net.ParseIP("fc00::a"))) - }) - - It("should return error if subnet nil is passed", func() { - result, err := ComputeOffsetIP(nil, 10) - - Expect(err).To(HaveOccurred()) - Expect(result).To(BeNil()) - }) - - It("should return error if subnet is not big enough is passed", func() { - _, subnet, _ := net.ParseCIDR("fc00::/128") - result, err := ComputeOffsetIP(subnet, 10) - - Expect(err).To(HaveOccurred()) - Expect(result).To(BeNil()) - }) - }) - }) - - Describe("#GenerateAddonConfig", func() { - Context("values=nil and enabled=false", func() { - It("should return a map with key enabled=false", func() { - var ( - values map[string]interface{} - enabled = false - ) - - result := GenerateAddonConfig(values, enabled) - - Expect(result).To(SatisfyAll( - HaveKeyWithValue("enabled", enabled), - HaveLen(1), - )) - }) - }) - - Context("values=nil and enabled=true", func() { - It("should return a map with key enabled=true", func() { - var ( - values map[string]interface{} - enabled = true - ) - - result := GenerateAddonConfig(values, enabled) - - Expect(result).To(SatisfyAll( - HaveKeyWithValue("enabled", enabled), - HaveLen(1), - )) - }) - }) - - Context("values= and enabled=true", func() { - It("should return a map with key enabled=true", func() { - var ( - values = map[string]interface{}{} - enabled = true - ) - - result := GenerateAddonConfig(values, enabled) - - Expect(result).To(SatisfyAll( - HaveKeyWithValue("enabled", enabled), - HaveLen(1), - )) - }) - }) - - Context("values= and enabled=true", func() { - It("should return a map with the values and key enabled=true", func() { - var ( - values = map[string]interface{}{ - "foo": "bar", - } - enabled = true - ) - - result := GenerateAddonConfig(values, enabled) - - for key := range values { - _, ok := result[key] - Expect(ok).To(BeTrue()) - } - Expect(result).To(SatisfyAll( - HaveKeyWithValue("enabled", enabled), - HaveLen(1+len(values)), - )) - }) - }) - - Context("values= and enabled=false", func() { - It("should return a map with key enabled=false", func() { - var ( - values = map[string]interface{}{ - "foo": "bar", - } - enabled = false - ) - - result := GenerateAddonConfig(values, enabled) - - Expect(result).To(SatisfyAll( - HaveKeyWithValue("enabled", enabled), - HaveLen(1), - )) - }) - }) - }) - }) - - Describe("#FilterEntriesByPrefix", func() { - var ( - prefix string - entries []string - ) - - BeforeEach(func() { - prefix = "role" - entries = []string{ - "foo", - "bar", - } - }) - - It("should only return entries with prefix", func() { - expectedEntries := []string{ - fmt.Sprintf("%s-%s", prefix, "foo"), - fmt.Sprintf("%s-%s", prefix, "bar"), - } - - entries = append(entries, expectedEntries...) - - result := FilterEntriesByPrefix(prefix, entries) - Expect(result).To(ContainElements(expectedEntries)) - }) - - It("should return all entries", func() { - expectedEntries := []string{ - fmt.Sprintf("%s-%s", prefix, "foo"), - fmt.Sprintf("%s-%s", prefix, "bar"), - } - - entries = expectedEntries - - result := FilterEntriesByPrefix(prefix, entries) - Expect(result).To(ContainElements(expectedEntries)) - }) - - It("should return no entries", func() { - result := FilterEntriesByPrefix(prefix, entries) - Expect(result).To(BeEmpty()) - }) - }) -}) diff --git a/pkg/operation/operation.go b/pkg/operation/operation.go index 85fdabb1f2e..f205f7b2f13 100644 --- a/pkg/operation/operation.go +++ b/pkg/operation/operation.go @@ -36,11 +36,10 @@ import ( "github.com/gardener/gardener/pkg/client/kubernetes/clientmap/keys" "github.com/gardener/gardener/pkg/gardenlet/apis/config" "github.com/gardener/gardener/pkg/gardenlet/apis/config/helper" - "github.com/gardener/gardener/pkg/operation/common" "github.com/gardener/gardener/pkg/operation/garden" "github.com/gardener/gardener/pkg/operation/seed" shootpkg "github.com/gardener/gardener/pkg/operation/shoot" - "github.com/gardener/gardener/pkg/utils/chart" + "github.com/gardener/gardener/pkg/utils" "github.com/gardener/gardener/pkg/utils/flow" gardenerutils "github.com/gardener/gardener/pkg/utils/gardener" "github.com/gardener/gardener/pkg/utils/imagevector" @@ -367,7 +366,7 @@ func (o *Operation) IsAPIServerRunning(ctx context.Context) (bool, error) { // GetSecretKeysOfRole returns a list of keys which are present in the Garden Secrets map and which // are prefixed with . func (o *Operation) GetSecretKeysOfRole(kind string) []string { - return common.FilterEntriesByPrefix(kind, o.AllSecretKeys()) + return utils.FilterEntriesByPrefix(kind, o.AllSecretKeys()) } // ReportShootProgress will update the last operation object in the Shoot manifest `status` section @@ -418,40 +417,11 @@ func (o *Operation) ShootVersion() string { return o.Shoot.GetInfo().Spec.Kubernetes.Version } -// InjectSeedSeedImages injects images that shall run on the Seed and target the Seed's Kubernetes version. -func (o *Operation) InjectSeedSeedImages(values map[string]interface{}, names ...string) (map[string]interface{}, error) { - return chart.InjectImages(values, o.ImageVector, names, imagevector.RuntimeVersion(o.SeedVersion()), imagevector.TargetVersion(o.SeedVersion())) -} - -// InjectSeedShootImages injects images that shall run on the Seed but target the Shoot's Kubernetes version. -func (o *Operation) InjectSeedShootImages(values map[string]interface{}, names ...string) (map[string]interface{}, error) { - return chart.InjectImages(values, o.ImageVector, names, imagevector.RuntimeVersion(o.SeedVersion()), imagevector.TargetVersion(o.ShootVersion())) -} - -// InjectShootShootImages injects images that shall run on the Shoot and target the Shoot's Kubernetes version. -func (o *Operation) InjectShootShootImages(values map[string]interface{}, names ...string) (map[string]interface{}, error) { - return chart.InjectImages(values, o.ImageVector, names, imagevector.RuntimeVersion(o.ShootVersion()), imagevector.TargetVersion(o.ShootVersion())) -} - // DeleteClusterResourceFromSeed deletes the `Cluster` extension resource for the shoot in the seed cluster. func (o *Operation) DeleteClusterResourceFromSeed(ctx context.Context) error { return client.IgnoreNotFound(o.SeedClientSet.Client().Delete(ctx, &extensionsv1alpha1.Cluster{ObjectMeta: metav1.ObjectMeta{Name: o.Shoot.SeedNamespace}})) } -// ComputePrometheusHosts computes the hosts for prometheus. -func (o *Operation) ComputePrometheusHosts() []string { - return []string{ - o.ComputePrometheusHost(), - } -} - -// ComputeAlertManagerHosts computes the host for alert manager. -func (o *Operation) ComputeAlertManagerHosts() []string { - return []string{ - o.ComputeAlertManagerHost(), - } -} - // IsShootMonitoringEnabled returns true if shoot monitoring is enabled and shoot is not of purpose testing. func (o *Operation) IsShootMonitoringEnabled() bool { return helper.IsMonitoringEnabled(o.Config) && o.Shoot.Purpose != gardencorev1beta1.ShootPurposeTesting @@ -464,27 +434,27 @@ func (o *Operation) WantsPlutono() bool { // ComputeKubeAPIServerHost computes the host with a TLS certificate from a trusted origin for KubeAPIServer. func (o *Operation) ComputeKubeAPIServerHost() string { - return o.ComputeIngressHost(common.KubeAPIServerPrefix) + return o.ComputeIngressHost("api") } // ComputePlutonoHost computes the host for Plutono. func (o *Operation) ComputePlutonoHost() string { - return o.ComputeIngressHost(common.PlutonoUsersPrefix) + return o.ComputeIngressHost("gu") } // ComputeAlertManagerHost computes the host for alert manager. func (o *Operation) ComputeAlertManagerHost() string { - return o.ComputeIngressHost(common.AlertManagerPrefix) + return o.ComputeIngressHost("au") } // ComputePrometheusHost computes the host for prometheus. func (o *Operation) ComputePrometheusHost() string { - return o.ComputeIngressHost(common.PrometheusPrefix) + return o.ComputeIngressHost("p") } // ComputeValiHost computes the host for vali. func (o *Operation) ComputeValiHost() string { - return o.ComputeIngressHost(common.ValiPrefix) + return o.ComputeIngressHost("v") } // technicalIDPattern addresses the ambiguity that one or two dashes could follow the prefix "shoot" in the technical ID of the shoot. diff --git a/pkg/operation/shoot/shoot.go b/pkg/operation/shoot/shoot.go index 191413703b2..2bbde778c8f 100644 --- a/pkg/operation/shoot/shoot.go +++ b/pkg/operation/shoot/shoot.go @@ -34,7 +34,7 @@ import ( gardenerextensions "github.com/gardener/gardener/pkg/extensions" "github.com/gardener/gardener/pkg/gardenlet/apis/config" gardenlethelper "github.com/gardener/gardener/pkg/gardenlet/apis/config/helper" - "github.com/gardener/gardener/pkg/operation/common" + "github.com/gardener/gardener/pkg/utils" gardenerutils "github.com/gardener/gardener/pkg/utils/gardener" kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes" ) @@ -184,6 +184,7 @@ func (b *Builder) Build(ctx context.Context, c client.Reader) (*Shoot, error) { ControlPlane: &ControlPlane{}, SystemComponents: &SystemComponents{}, Logging: &Logging{}, + Monitoring: &Monitoring{}, Addons: &Addons{}, } @@ -512,12 +513,12 @@ func ToNetworks(s *gardencorev1beta1.Shoot, workerless bool) (*Networks, error) return nil, fmt.Errorf("shoot's service cidr is empty") } - apiserver, err := common.ComputeOffsetIP(svc, 1) + apiserver, err := utils.ComputeOffsetIP(svc, 1) if err != nil { return nil, fmt.Errorf("cannot calculate default/kubernetes ClusterIP: %w", err) } - coreDNS, err := common.ComputeOffsetIP(svc, 10) + coreDNS, err := utils.ComputeOffsetIP(svc, 10) if err != nil { return nil, fmt.Errorf("cannot calculate CoreDNS ClusterIP: %w", err) } diff --git a/pkg/operation/shoot/types.go b/pkg/operation/shoot/types.go index 1269322785d..7e0b9c651f3 100644 --- a/pkg/operation/shoot/types.go +++ b/pkg/operation/shoot/types.go @@ -48,6 +48,7 @@ import ( "github.com/gardener/gardener/pkg/component/kubestatemetrics" "github.com/gardener/gardener/pkg/component/logging/vali" "github.com/gardener/gardener/pkg/component/machinecontrollermanager" + "github.com/gardener/gardener/pkg/component/monitoring" "github.com/gardener/gardener/pkg/component/nodeexporter" "github.com/gardener/gardener/pkg/component/nodelocaldns" "github.com/gardener/gardener/pkg/component/plutono" @@ -116,9 +117,10 @@ type Components struct { Extensions *Extensions SystemComponents *SystemComponents Logging *Logging + Monitoring *Monitoring + Addons *Addons GardenerAccess component.Deployer DependencyWatchdogAccess component.Deployer - Addons *Addons } // ControlPlane contains references to K8S control plane components. @@ -172,7 +174,12 @@ type SystemComponents struct { VPNShoot vpnshoot.Interface } -// Logging contains references to logging deployers +// Monitoring contains references to monitoring deployers. +type Monitoring struct { + Monitoring monitoring.Interface +} + +// Logging contains references to logging deployers. type Logging struct { // TODO(rfranzke): Drop the `ShootRBACProxy` field after gardener/gardener@v1.78 has been released. ShootRBACProxy component.Deployer diff --git a/pkg/utils/miscellaneous.go b/pkg/utils/miscellaneous.go index bed0e8bb118..963303b541c 100644 --- a/pkg/utils/miscellaneous.go +++ b/pkg/utils/miscellaneous.go @@ -16,6 +16,7 @@ package utils import ( "fmt" + "math/big" "net" "regexp" "strings" @@ -199,3 +200,51 @@ func InterfaceMapToStringMap(in map[string]interface{}) map[string]string { } return m } + +// FilterEntriesByPrefix returns a list of strings which begin with the given prefix. +func FilterEntriesByPrefix(prefix string, entries []string) []string { + var result []string + for _, entry := range entries { + if strings.HasPrefix(entry, prefix) { + result = append(result, entry) + } + } + return result +} + +// ComputeOffsetIP parses the provided and offsets with the value of . +// For example, = 100.64.0.0/11 and = 10 the result would be 100.64.0.10 +// IPv6 and IPv4 is supported. +func ComputeOffsetIP(subnet *net.IPNet, offset int64) (net.IP, error) { + if subnet == nil { + return nil, fmt.Errorf("subnet is nil") + } + + isIPv6 := false + + bytes := subnet.IP.To4() + if bytes == nil { + isIPv6 = true + bytes = subnet.IP.To16() + } + + ip := net.IP(big.NewInt(0).Add(big.NewInt(0).SetBytes(bytes), big.NewInt(offset)).Bytes()) + + if !subnet.Contains(ip) { + return nil, fmt.Errorf("cannot compute IP with offset %d - subnet %q too small", offset, subnet) + } + + // there is no broadcast address on IPv6 + if isIPv6 { + return ip, nil + } + + for i := range ip { + // IP address is not the same, so it's not the broadcast ip. + if ip[i] != ip[i]|^subnet.Mask[i] { + return ip.To4(), nil + } + } + + return nil, fmt.Errorf("computed IPv4 address %q is broadcast for subnet %q", ip, subnet) +} diff --git a/pkg/utils/miscellaneous_test.go b/pkg/utils/miscellaneous_test.go index eed1d7fa8e7..e2c81181217 100644 --- a/pkg/utils/miscellaneous_test.go +++ b/pkg/utils/miscellaneous_test.go @@ -15,6 +15,8 @@ package utils_test import ( + "fmt" + "net" "time" . "github.com/onsi/ginkgo/v2" @@ -192,4 +194,111 @@ baz`, spaces)).To(Equal(`foo Expect(InterfaceMapToStringMap(input)).Should(Equal(output)) }) }) + + Describe("#ComputeOffsetIP", func() { + Context("IPv4", func() { + It("should return a cluster IPv4 IP", func() { + _, subnet, _ := net.ParseCIDR("100.64.0.0/13") + result, err := ComputeOffsetIP(subnet, 10) + + Expect(err).NotTo(HaveOccurred()) + + Expect(result).To(HaveLen(net.IPv4len)) + Expect(result).To(Equal(net.ParseIP("100.64.0.10").To4())) + }) + + It("should return error if subnet nil is passed", func() { + result, err := ComputeOffsetIP(nil, 10) + + Expect(err).To(HaveOccurred()) + Expect(result).To(BeNil()) + }) + + It("should return error if subnet is not big enough is passed", func() { + _, subnet, _ := net.ParseCIDR("100.64.0.0/32") + result, err := ComputeOffsetIP(subnet, 10) + + Expect(err).To(HaveOccurred()) + Expect(result).To(BeNil()) + }) + + It("should return error if ip address is broadcast ip", func() { + _, subnet, _ := net.ParseCIDR("10.0.0.0/24") + result, err := ComputeOffsetIP(subnet, 255) + + Expect(err).To(HaveOccurred()) + Expect(result).To(BeNil()) + }) + }) + + Context("IPv6", func() { + It("should return a cluster IPv6 IP", func() { + _, subnet, _ := net.ParseCIDR("fc00::/8") + result, err := ComputeOffsetIP(subnet, 10) + + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(HaveLen(net.IPv6len)) + Expect(result).To(Equal(net.ParseIP("fc00::a"))) + }) + + It("should return error if subnet nil is passed", func() { + result, err := ComputeOffsetIP(nil, 10) + + Expect(err).To(HaveOccurred()) + Expect(result).To(BeNil()) + }) + + It("should return error if subnet is not big enough is passed", func() { + _, subnet, _ := net.ParseCIDR("fc00::/128") + result, err := ComputeOffsetIP(subnet, 10) + + Expect(err).To(HaveOccurred()) + Expect(result).To(BeNil()) + }) + }) + }) + + Describe("#FilterEntriesByPrefix", func() { + var ( + prefix string + entries []string + ) + + BeforeEach(func() { + prefix = "role" + entries = []string{ + "foo", + "bar", + } + }) + + It("should only return entries with prefix", func() { + expectedEntries := []string{ + fmt.Sprintf("%s-%s", prefix, "foo"), + fmt.Sprintf("%s-%s", prefix, "bar"), + } + + entries = append(entries, expectedEntries...) + + result := FilterEntriesByPrefix(prefix, entries) + Expect(result).To(ContainElements(expectedEntries)) + }) + + It("should return all entries", func() { + expectedEntries := []string{ + fmt.Sprintf("%s-%s", prefix, "foo"), + fmt.Sprintf("%s-%s", prefix, "bar"), + } + + entries = expectedEntries + + result := FilterEntriesByPrefix(prefix, entries) + Expect(result).To(ContainElements(expectedEntries)) + }) + + It("should return no entries", func() { + result := FilterEntriesByPrefix(prefix, entries) + Expect(result).To(BeEmpty()) + }) + }) }) diff --git a/skaffold-operator.yaml b/skaffold-operator.yaml index 9ac214bffac..8cb94a2da85 100644 --- a/skaffold-operator.yaml +++ b/skaffold-operator.yaml @@ -107,6 +107,7 @@ build: - pkg/component/logging/kuberbacproxy - pkg/component/machinecontrollermanager - pkg/component/metricsserver + - pkg/component/monitoring - pkg/component/namespaces - pkg/component/nginxingress - pkg/component/nodeexporter