Add tolerations for Project and Shoot resources

charts/gardener/controlplane/charts/runtime/templates/apiserver/configmap-admission-config.yaml

@@ -0,0 +1,16 @@
+{{- if and .Values.global.apiserver.enabled .Values.global.apiserver.admissionConfig }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: gardener-apiserver-admission-config
+  namespace: garden
+  labels:
+    app: gardener
+    role: apiserver
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    release: "{{ .Release.Name }}"
+    heritage: "{{ .Release.Service }}"
+data:
+  configuration.yaml: |-
+{{ .Values.global.apiserver.admissionConfig | indent 4 }}
+{{- end }}

charts/gardener/controlplane/charts/runtime/templates/apiserver/deployment.yaml

@@ -30,6 +30,9 @@ spec:
         checksum/secret-gardener-apiserver-encryption-config: {{ include (print $.Template.BasePath "/apiserver/secret-gardener-apiserver-encryption-config.yaml") . | sha256sum }}
         {{- end }}
         checksum/secret-gardener-apiserver-kubeconfig: {{ include (print $.Template.BasePath "/apiserver/secret-kubeconfig.yaml") . | sha256sum }}
+        {{- if .Values.global.apiserver.admissionConfig }}
+        checksum/configmap-gardener-apiserver-admission-config: {{ include (print $.Template.BasePath "/apiserver/configmap-admission-config.yaml") . | sha256sum }}
+        {{- end }}
         {{- if .Values.global.apiserver.podAnnotations }}
 {{ toYaml .Values.global.apiserver.podAnnotations | indent 8 }}
         {{- end }}
@@ -71,6 +74,9 @@ spec:
         imagePullPolicy: {{ .Values.global.apiserver.image.pullPolicy }}
         command:
         - /gardener-apiserver
+        {{- if .Values.global.apiserver.admissionConfig }}
+        - --admission-control-config-file=/etc/gardener-apiserver/admission/configuration.yaml
+        {{- end }}
         {{- if .Values.global.apiserver.audit.dynamicConfiguration }}
         - --audit-dynamic-configuration={{ .Values.global.apiserver.audit.dynamicConfiguration }}
         {{- end }}
@@ -234,6 +240,10 @@ spec:
         - name: gardener-audit-webhook-config
           mountPath: /etc/gardener-apiserver/auditwebhook
         {{- end }}
+        {{- if .Values.global.apiserver.admissionConfig }}
+        - name: gardener-apiserver-admission-config
+          mountPath: /etc/gardener-apiserver/admission
+        {{- end }}
       {{- if .Values.global.apiserver.etcd.useSidecar }}
       - name: etcd
         image: quay.io/coreos/etcd:v3.3.12
@@ -295,6 +305,11 @@ spec:
         secret:
           secretName: gardener-audit-webhook-config
       {{- end }}
+      {{- if .Values.global.apiserver.admissionConfig }}
+      - name: gardener-apiserver-admission-config
+        configMap:
+          name: gardener-apiserver-admission-config
+      {{- end }}
 {{- if .Values.global.apiserver.etcd.useSidecar }}
 ---
 apiVersion: v1

charts/gardener/controlplane/values.yaml

@@ -61,6 +61,10 @@ global:
         ...
         -----END RSA PRIVATE KEY-----
     featureGates: {}
+  # enableAdmissionPlugins: [] # List of admission plugins to be enabled in addition to default enabled ones.
+  # disableAdmissionPlugins: [] # List of admission plugins that should be disabled although they are in the default enabled plugins list.
+  # admissionConfig: |
+  #   configuration for the admission plugins. See example/20-admissionconfiguration.yaml
     vpa: false
     hvpa:
       enabled: false
@@ -93,10 +97,6 @@ global:
         memory:
           value: "5G"
           percentage: 80
-    # List of admission plugins that should be disabled although they are in the default enabled plugins list.
-  # disableAdmissionPlugins: []
-    # List of admission plugins to be enabled in addition to default enabled ones.
-  # enableAdmissionPlugins: []
 
     audit:
  #    dynamicConfiguration: false                             Enables dynamic audit configuration. This feature also requires the DynamicAuditing feature flag

cmd/gardener-apiserver/app/gardener_apiserver.go

@@ -44,6 +44,7 @@ import (
 	"github.com/gardener/gardener/plugin/pkg/shoot/oidc/clusteropenidconnectpreset"
 	"github.com/gardener/gardener/plugin/pkg/shoot/oidc/openidconnectpreset"
 	shootquotavalidator "github.com/gardener/gardener/plugin/pkg/shoot/quotavalidator"
+	shoottolerationrestriction "github.com/gardener/gardener/plugin/pkg/shoot/tolerationrestriction"
 	shootvalidator "github.com/gardener/gardener/plugin/pkg/shoot/validator"
 	shootstatedeletionvalidator "github.com/gardener/gardener/plugin/pkg/shootstate/validator"
 
@@ -152,6 +153,7 @@ func (o *Options) complete() error {
 	resourcereferencemanager.Register(o.Recommended.Admission.Plugins)
 	deletionconfirmation.Register(o.Recommended.Admission.Plugins)
 	extensionvalidation.Register(o.Recommended.Admission.Plugins)
+	shoottolerationrestriction.Register(o.Recommended.Admission.Plugins)
 	shootquotavalidator.Register(o.Recommended.Admission.Plugins)
 	shootdns.Register(o.Recommended.Admission.Plugins)
 	shootvalidator.Register(o.Recommended.Admission.Plugins)
@@ -164,6 +166,7 @@ func (o *Options) complete() error {
 	allOrderedPlugins := []string{
 		resourcereferencemanager.PluginName,
 		extensionvalidation.PluginName,
+		shoottolerationrestriction.PluginName,
 		shootdns.PluginName,
 		shootquotavalidator.PluginName,
 		shootvalidator.PluginName,
@@ -174,7 +177,6 @@ func (o *Options) complete() error {
 		clusteropenidconnectpreset.PluginName,
 		shootstatedeletionvalidator.PluginName,
 	}
-
 	o.Recommended.Admission.RecommendedPluginOrder = append(o.Recommended.Admission.RecommendedPluginOrder, allOrderedPlugins...)
 
 	return nil

docs/README.md

@@ -17,17 +17,18 @@
 ## Usage
 
 * [Audit a Kubernetes cluster](usage/shoot_auditpolicy.md)
-* [Shoot cluster purposes](usage/shoot_purposes.md)
 * [Custom `CoreDNS` configuration](usage/custom-dns.md)
-* [Trusted TLS certificate for shoot control planes](usage/trusted-tls-for-control-planes.md)
 * [Gardener configuration and usage](usage/configuration.md)
+* [`ManagedIstio` feature](usage/istio.md)
 * [OpenIDConnect presets](usage/openidconnect-presets.md)
-* [Supported Kubernetes versions](usage/supported_k8s_versions.md)
-* [Shoot Maintenance](usage/shoot_maintenance.md)
+* [Shoot cluster purposes](usage/shoot_purposes.md)
 * [Shoot Kubernetes and Operating System versioning](usage/shoot_versions.md)
+* [Shoot Maintenance](usage/shoot_maintenance.md)
+* [Supported Kubernetes versions](usage/supported_k8s_versions.md)
+* [Tolerations](usage/tolerations.md)
 * [Trigger shoot operations](usage/shoot_operations.md)
 * [Troubleshooting guide](usage/trouble_shooting_guide.md)
-* [ManagedIstio Feature Gate](usage/istio.md)
+* [Trusted TLS certificate for shoot control planes](usage/trusted-tls-for-control-planes.md)
 
 ## Proposals
 

docs/usage/tolerations.md

@@ -0,0 +1,32 @@
+# Taints and Tolerations for `Seed`s and `Shoot`s
+
+Similar to [taints and tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) for `Node`s and `Pod`s in Kubernetes, the `Seed` resource supports specifying taints (`.spec.taints`) while the `Shoot` resource supports specifying tolerations (`.spec.tolerations`).
+The feature is used to control scheduling to seeds as well as decisions whether a shoot can use a certain seed.
+
+Compared to Kubernetes, Gardener's taints and tolerations are very much down-stripped right now and have some behavioral differences.
+Please read the following explanations carefully if you plan to use it.
+
+## Scheduling
+
+When scheduling a new shoot then the gardener-scheduler will filter all seed candidates whose taints are not tolerated by the shoot.
+As Gardener's taints/tolerations don't support `effect`s yet you can compare this behaviour with using a `NoSchedule` effect taint in Kubernetes.
+
+Be reminded that taints/tolerations are no means to define any affinity or selection for seeds - please use `.spec.seedSelector` in the `Shoot` to state such desires.
+
+⚠️ Please note that - unlike how it's implemented in Kubernetes - a certain seed cluster **may** only be used when the shoot tolerates **all** the seed's taints.
+This means that specifying `.spec.seedName` for a seed whose taints are not tolerated will make the gardener-apiserver rejecting the request.
+
+Consequently, the taints/tolerations feature can be used as means to restrict usage of certain seeds.
+
+## Toleration Defaults and Whitelist
+
+The `Project` resource features a `.spec.tolerations` object that may carry `defaults` and a `whitelist`.
+The corresponding `ShootTolerationRestriction` admission plugin (cf. Kubernetes' `PodTolerationRestriction` admission plugin) is responsible for evaluating these settings during creation/update of `Shoot`s.
+
+### Whitelist
+
+If a shoot gets created or updated with tolerations then it is validated that only those tolerations may be used which were added to either a) the `Project`'s `.spec.tolerations.whitelist`, or b) to the global whitelist in the `ShootTolerationRestriction`'s admission config (see [this example](https://github.com/rfranzke/gardener/blob/feature/tolerations/example/20-admissionconfig.yaml#L7-L14)).  
+
+### Defaults
+
+If a shoot gets created then the default tolerations specified in both the `Project`'s `.spec.tolerations.defaults` and global default list in the `ShootTolerationRestriction` admission plugin's configuration will be added to the `.spec.tolerations` of the `Shoot` (unless it already specifies a certain key).

example/05-project-dev.yaml

@@ -30,3 +30,8 @@ spec:
   # If the namespace is set then the namespace must be labelled with `gardener.cloud/role: project`
   # and `project.gardener.cloud/name: <project-name>` (<project-name>=dev in this case).
   namespace: garden-dev
+# tolerations:
+#   defaults:
+#   - key: <some-key>
+#   whitelist:
+#   - key: <some-key>

example/20-admissionconfig.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: apiserver.k8s.io/v1alpha1
+kind: AdmissionConfiguration
+plugins:
+- name: ShootTolerationRestriction
+  configuration:
+    apiVersion: shoottolerationrestriction.admission.gardener.cloud/v1alpha1
+    kind: Configuration
+    defaults:
+    - key: foo
+    whitelist:
+    - key: foo
+    - key: bar
+      value: baz

example/90-shoot.yaml

@@ -264,7 +264,9 @@ spec:
     kubernetesDashboard:
       enabled: true
     # authenticationMode: basic # allowed values: basic,token
-# Explicitly specify the seed that will run the shoot control plane. 
+# tolerations:
+# - key: <some-key>
+# Explicitly specify the seed that will run the shoot control plane.
 # seedName: my-seed
 # Provide labels of seeds that are eligible to run this shoot - will be evaluated by the scheduler
 # seedSelector:

hack/api-reference/core.md

@@ -896,6 +896,20 @@ string
 A nil value means that Gardener will determine the name of the namespace.</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>tolerations</code></br>
+<em>
+<a href="#core.gardener.cloud/v1beta1.ProjectTolerations">
+ProjectTolerations
+</a>
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>Tolerations contains the tolerations for taints on seed clusters.</p>
+</td>
+</tr>
 </table>
 </td>
 </tr>
@@ -1554,6 +1568,20 @@ Kubernetes meta/v1.LabelSelector
 <p>Resources holds a list of named resource references that can be referred to in extension configs by their names.</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>tolerations</code></br>
+<em>
+<a href="#core.gardener.cloud/v1beta1.Toleration">
+[]Toleration
+</a>
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>Tolerations contains the tolerations for taints on seed clusters.</p>
+</td>
+</tr>
 </table>
 </td>
 </tr>
@@ -5902,6 +5930,20 @@ string
 A nil value means that Gardener will determine the name of the namespace.</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>tolerations</code></br>
+<em>
+<a href="#core.gardener.cloud/v1beta1.ProjectTolerations">
+ProjectTolerations
+</a>
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>Tolerations contains the tolerations for taints on seed clusters.</p>
+</td>
+</tr>
 </tbody>
 </table>
 <h3 id="core.gardener.cloud/v1beta1.ProjectStatus">ProjectStatus
@@ -5948,6 +5990,54 @@ ProjectPhase
 </tr>
 </tbody>
 </table>
+<h3 id="core.gardener.cloud/v1beta1.ProjectTolerations">ProjectTolerations
+</h3>
+<p>
+(<em>Appears on:</em>
+<a href="#core.gardener.cloud/v1beta1.ProjectSpec">ProjectSpec</a>)
+</p>
+<p>
+<p>ProjectTolerations contains the tolerations for taints on seed clusters.</p>
+</p>
+<table>
+<thead>
+<tr>
+<th>Field</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>
+<code>defaults</code></br>
+<em>
+<a href="#core.gardener.cloud/v1beta1.Toleration">
+[]Toleration
+</a>
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>Defaults contains a list of tolerations that are added to the shoots in this project by default.</p>
+</td>
+</tr>
+<tr>
+<td>
+<code>whitelist</code></br>
+<em>
+<a href="#core.gardener.cloud/v1beta1.Toleration">
+[]Toleration
+</a>
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>Whitelist contains a list of tolerations that are allowed to be added to the shoots in this project. Please note
+that this list may only be added by users having the <code>spec-tolerations-whitelist</code> verb for project resources.</p>
+</td>
+</tr>
+</tbody>
+</table>
 <h3 id="core.gardener.cloud/v1beta1.Provider">Provider
 </h3>
 <p>
@@ -7362,6 +7452,20 @@ Kubernetes meta/v1.LabelSelector
 <p>Resources holds a list of named resource references that can be referred to in extension configs by their names.</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>tolerations</code></br>
+<em>
+<a href="#core.gardener.cloud/v1beta1.Toleration">
+[]Toleration
+</a>
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>Tolerations contains the tolerations for taints on seed clusters.</p>
+</td>
+</tr>
 </tbody>
 </table>
 <h3 id="core.gardener.cloud/v1beta1.ShootStatus">ShootStatus
@@ -7530,6 +7634,49 @@ It is used to compute unique hashes.</p>
 </tr>
 </tbody>
 </table>
+<h3 id="core.gardener.cloud/v1beta1.Toleration">Toleration
+</h3>
+<p>
+(<em>Appears on:</em>
+<a href="#core.gardener.cloud/v1beta1.ProjectTolerations">ProjectTolerations</a>, 
+<a href="#core.gardener.cloud/v1beta1.ShootSpec">ShootSpec</a>)
+</p>
+<p>
+<p>Toleration is a toleration for a seed taint.</p>
+</p>
+<table>
+<thead>
+<tr>
+<th>Field</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>
+<code>key</code></br>
+<em>
+string
+</em>
+</td>
+<td>
+<p>Key is the toleration key to be applied to a project or shoot.</p>
+</td>
+</tr>
+<tr>
+<td>
+<code>value</code></br>
+<em>
+string
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>Value is the toleration value corresponding to the toleration key.</p>
+</td>
+</tr>
+</tbody>
+</table>
 <h3 id="core.gardener.cloud/v1beta1.VersionClassification">VersionClassification
 (<code>string</code> alias)</p></h3>
 <p>