From 964280712d0a16d66189b65d3b498e141297f480 Mon Sep 17 00:00:00 2001 From: drewhagen Date: Thu, 25 Apr 2024 15:31:03 -0500 Subject: [PATCH 01/82] Tracking commit for v1.31 docs From d0d907c351d8e324fbbea298027811b44ffc5df4 Mon Sep 17 00:00:00 2001 From: carlory Date: Mon, 6 May 2024 15:30:30 +0800 Subject: [PATCH 02/82] Remove ConsistentHTTPGetHandlers feature-gate --- .../feature-gates/consistent-http-get-handlers.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/consistent-http-get-handlers.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/consistent-http-get-handlers.md index 702bebe15e999..fed8e417c633e 100644 --- a/content/en/docs/reference/command-line-tools-reference/feature-gates/consistent-http-get-handlers.md +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/consistent-http-get-handlers.md @@ -9,6 +9,9 @@ stages: - stage: stable defaultValue: true fromVersion: "1.25" + toVersion: "1.30" + +removed: true --- Normalize HTTP get URL and Header passing for lifecycle handlers with probers. From d84a36be9e2e722ab45b784f1549b329e14c3a4f Mon Sep 17 00:00:00 2001 From: Chris Henzie Date: Wed, 1 May 2024 14:33:58 -0700 Subject: [PATCH 03/82] Remove ReadWriteOncePod feature gate This feature is GA and its feature gates have been removed in v1.31. --- .../feature-gates/read-write-once-pod.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/read-write-once-pod.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/read-write-once-pod.md index 10dfb7f7b2d75..fcc1cdbe817d8 100644 --- a/content/en/docs/reference/command-line-tools-reference/feature-gates/read-write-once-pod.md +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/read-write-once-pod.md @@ -17,6 +17,9 @@ stages: - stage: stable defaultValue: true fromVersion: "1.29" + toVersion: "1.30" + +removed: true --- Enables the usage of `ReadWriteOncePod` PersistentVolume access mode. From 8c8dc5a40541a6faf41e88c3b7aae9ac6c9c6c47 Mon Sep 17 00:00:00 2001 From: xuzhenglun Date: Wed, 8 May 2024 15:32:31 +0800 Subject: [PATCH 04/82] Remove GA ServiceNodePortStaticSubrange feature gate --- content/en/docs/concepts/services-networking/service.md | 2 -- .../feature-gates/service-nodeport-static-subrange.md | 3 +++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/content/en/docs/concepts/services-networking/service.md b/content/en/docs/concepts/services-networking/service.md index bd09e179f2aa9..3402aa12df415 100644 --- a/content/en/docs/concepts/services-networking/service.md +++ b/content/en/docs/concepts/services-networking/service.md @@ -523,8 +523,6 @@ spec: #### Reserve Nodeport ranges to avoid collisions {#avoid-nodeport-collisions} -{{< feature-state for_k8s_version="v1.29" state="stable" >}} - The policy for assigning ports to NodePort services applies to both the auto-assignment and the manual assignment scenarios. When a user wants to create a NodePort service that uses a specific port, the target port may conflict with another port that has already been assigned. diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/service-nodeport-static-subrange.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/service-nodeport-static-subrange.md index 9226d1c903690..43dc8eb6e88f7 100644 --- a/content/en/docs/reference/command-line-tools-reference/feature-gates/service-nodeport-static-subrange.md +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/service-nodeport-static-subrange.md @@ -17,6 +17,9 @@ stages: - stage: stable defaultValue: true fromVersion: "1.29" + toVersion: "1.30" + +removed: true --- Enables the use of different port allocation strategies for NodePort Services. For more details, see From 72f8423adeeadd7e8fa415dcf8b928c0b69b446f Mon Sep 17 00:00:00 2001 From: carlory Date: Thu, 9 May 2024 10:32:34 +0800 Subject: [PATCH 05/82] remove feature-gate CSINodeExpandSecret --- content/en/docs/concepts/storage/volumes.md | 10 +++++----- .../feature-gates/csi-node-expand-secret.md | 5 ++++- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/content/en/docs/concepts/storage/volumes.md b/content/en/docs/concepts/storage/volumes.md index a673ecf27b9cc..2ab53fb80d9b9 100644 --- a/content/en/docs/concepts/storage/volumes.md +++ b/content/en/docs/concepts/storage/volumes.md @@ -1058,12 +1058,12 @@ persistent volume: secret is required. If the object contains more than one secret, all secrets are passed. When you have configured secret data for node-initiated volume expansion, the kubelet passes that data via the `NodeExpandVolume()` - call to the CSI driver. In order to use the `nodeExpandSecretRef` field, your - cluster should be running Kubernetes version 1.25 or later. -* If you are running Kubernetes Version 1.25 or 1.26, you must enable - the [feature gate](/docs/reference/command-line-tools-reference/feature-gates/) + call to the CSI driver. All supported versions of Kubernetes offer the + `nodeExpandSecretRef` field, and have it available by default. Kubernetes releases + prior to v1.25 did not include this support. +* Enable the [feature gate](/docs/reference/command-line-tools-reference/feature-gates-removed/) named `CSINodeExpandSecret` for each kube-apiserver and for the kubelet on every - node. In Kubernetes version 1.27 this feature has been enabled by default + node. Since Kubernetes version 1.27 this feature has been enabled by default and no explicit enablement of the feature gate is required. You must also be using a CSI driver that supports or requires secret data during node-initiated storage resize operations. diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/csi-node-expand-secret.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/csi-node-expand-secret.md index f1d22919158d2..b3e352b2b161a 100644 --- a/content/en/docs/reference/command-line-tools-reference/feature-gates/csi-node-expand-secret.md +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/csi-node-expand-secret.md @@ -16,7 +16,10 @@ stages: toVersion: "1.28" - stage: stable defaultValue: true - fromVersion: "1.29" + fromVersion: "1.29" + toVersion: "1.30" + +removed: true --- Enable passing secret authentication data to a CSI driver for use during a `NodeExpandVolume` CSI operation. From 8c773b2e31fa4578377059735a9fda9457d608ad Mon Sep 17 00:00:00 2001 From: carlory Date: Fri, 10 May 2024 11:26:09 +0800 Subject: [PATCH 06/82] update PersistentVolumeLabel admission plugin --- .../access-authn-authz/admission-controllers.md | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/content/en/docs/reference/access-authn-authz/admission-controllers.md b/content/en/docs/reference/access-authn-authz/admission-controllers.md index 22d3b647f2fc6..7e67e94cf092d 100644 --- a/content/en/docs/reference/access-authn-authz/admission-controllers.md +++ b/content/en/docs/reference/access-authn-authz/admission-controllers.md @@ -622,23 +622,6 @@ allowVolumeExpansion: true For more information about persistent volume claims, see [PersistentVolumeClaims](/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims). -### PersistentVolumeLabel {#persistentvolumelabel} - -{{< feature-state for_k8s_version="v1.13" state="deprecated" >}} - -**Type**: Mutating. - -This admission controller automatically attaches region or zone labels to PersistentVolumes -as defined by the cloud provider (for example, Azure or GCP). -It helps ensure the Pods and the PersistentVolumes mounted are in the same -region and/or zone. -If the admission controller doesn't support automatic labelling your PersistentVolumes, you -may need to add the labels manually to prevent pods from mounting volumes from -a different zone. PersistentVolumeLabel is **deprecated** as labeling for persistent volumes has been taken over by -the {{< glossary_tooltip text="cloud-controller-manager" term_id="cloud-controller-manager" >}}. - -This admission controller is disabled by default. - ### PodNodeSelector {#podnodeselector} {{< feature-state for_k8s_version="v1.5" state="alpha" >}} From 292b0bfc9a683bfeeb42eb54bbd31b24a23e8483 Mon Sep 17 00:00:00 2001 From: carlory Date: Fri, 10 May 2024 14:31:36 +0800 Subject: [PATCH 07/82] update rbd volume --- .../concepts/storage/persistent-volumes.md | 4 +- content/en/docs/concepts/storage/volumes.md | 62 ++----------------- 2 files changed, 8 insertions(+), 58 deletions(-) diff --git a/content/en/docs/concepts/storage/persistent-volumes.md b/content/en/docs/concepts/storage/persistent-volumes.md index 65041f5587d3a..5b0743229329c 100644 --- a/content/en/docs/concepts/storage/persistent-volumes.md +++ b/content/en/docs/concepts/storage/persistent-volumes.md @@ -529,8 +529,6 @@ please install corresponding CSI drivers. (**migration on by default** starting v1.23) * [`portworxVolume`](/docs/concepts/storage/volumes/#portworxvolume) - Portworx volume (**deprecated** starting v1.25) -* [`rbd`](/docs/concepts/storage/volumes/#rbd) - Rados Block Device (RBD) volume - (**deprecated** starting v1.28, no migration plan, support will be removed in a future release) * [`vsphereVolume`](/docs/concepts/storage/volumes/#vspherevolume) - vSphere VMDK volume (**migration on by default** starting v1.25) @@ -546,6 +544,8 @@ Older versions of Kubernetes also supported the following in-tree PersistentVolu (**not available** starting v1.25) * `storageos` - StorageOS volume. (**not available** starting v1.25) +* [`rbd`](/docs/concepts/storage/volumes/#rbd) - Rados Block Device (RBD) volume + (**not available** starting v1.31) ## Persistent Volumes diff --git a/content/en/docs/concepts/storage/volumes.md b/content/en/docs/concepts/storage/volumes.md index a673ecf27b9cc..2b1ab8a633911 100644 --- a/content/en/docs/concepts/storage/volumes.md +++ b/content/en/docs/concepts/storage/volumes.md @@ -745,66 +745,16 @@ To enable the feature, set `CSIMigrationPortworx=true` in kube-controller-manage A projected volume maps several existing volume sources into the same directory. For more details, see [projected volumes](/docs/concepts/storage/projected-volumes/). -### rbd -{{< feature-state for_k8s_version="v1.28" state="deprecated" >}} - -{{< note >}} -The Kubernetes project suggests that you use the [Ceph CSI](https://github.com/ceph/ceph-csi) -third party storage driver instead, in RBD mode. -{{< /note >}} - -An `rbd` volume allows a -[Rados Block Device](https://docs.ceph.com/en/latest/rbd/) (RBD) volume to mount -into your Pod. Unlike `emptyDir`, which is erased when a pod is removed, the -contents of an `rbd` volume are preserved and the volume is unmounted. This -means that a RBD volume can be pre-populated with data, and that data can be -shared between pods. +### rbd (removed) {#rbd} -{{< note >}} -You must have a Ceph installation running before you can use RBD. -{{< /note >}} - -A feature of RBD is that it can be mounted as read-only by multiple consumers -simultaneously. This means that you can pre-populate a volume with your dataset -and then serve it in parallel from as many pods as you need. Unfortunately, -RBD volumes can only be mounted by a single consumer in read-write mode. -Simultaneous writers are not allowed. - -See the [RBD example](https://github.com/kubernetes/examples/tree/master/volumes/rbd) -for more details. - -#### RBD CSI migration {#rbd-csi-migration} - -{{< feature-state for_k8s_version="v1.28" state="deprecated" >}} + -The `CSIMigration` feature for `RBD`, when enabled, redirects all plugin -operations from the existing in-tree plugin to the `rbd.csi.ceph.com` {{< -glossary_tooltip text="CSI" term_id="csi" >}} driver. In order to use this -feature, the -[Ceph CSI driver](https://github.com/ceph/ceph-csi) -must be installed on the cluster and the `CSIMigrationRBD` -[feature gate](/docs/reference/command-line-tools-reference/feature-gates/) -must be enabled. (Note that the `csiMigrationRBD` flag has been removed and -replaced with `CSIMigrationRBD` in release v1.24) +Kubernetes {{< skew currentVersion >}} does not include a `rbd` volume type. -{{< note >}} +The [Rados Block Device](https://docs.ceph.com/en/latest/rbd/) (RBD) in-tree storage driver and its csi migration support were deprecated in the Kubernetes v1.28 release +and then removed entirely in the v1.31 release. -As a Kubernetes cluster operator that administers storage, here are the -prerequisites that you must complete before you attempt migration to the -RBD CSI driver: - -* You must install the Ceph CSI driver (`rbd.csi.ceph.com`), v3.5.0 or above, - into your Kubernetes cluster. -* considering the `clusterID` field is a required parameter for CSI driver for - its operations, but in-tree StorageClass has `monitors` field as a required - parameter, a Kubernetes storage admin has to create a clusterID based on the - monitors hash ( ex:`#echo -n - '' | md5sum`) in the CSI config map and keep the monitors - under this clusterID configuration. -* Also, if the value of `adminId` in the in-tree Storageclass is different from - `admin`, the `adminSecretName` mentioned in the in-tree Storageclass has to be - patched with the base64 value of the `adminId` parameter value, otherwise this - step can be skipped. {{< /note >}} ### secret From 6cc56263f25ad923460fb5a01b73e8862d630dd1 Mon Sep 17 00:00:00 2001 From: carlory Date: Tue, 14 May 2024 10:22:31 +0800 Subject: [PATCH 08/82] update cephfs volume --- .../concepts/storage/persistent-volumes.md | 16 +++++++------- content/en/docs/concepts/storage/volumes.md | 22 +++++-------------- 2 files changed, 13 insertions(+), 25 deletions(-) diff --git a/content/en/docs/concepts/storage/persistent-volumes.md b/content/en/docs/concepts/storage/persistent-volumes.md index 5b0743229329c..093966f1b54ec 100644 --- a/content/en/docs/concepts/storage/persistent-volumes.md +++ b/content/en/docs/concepts/storage/persistent-volumes.md @@ -519,8 +519,6 @@ please install corresponding CSI drivers. (**migration on by default** starting v1.23) * [`azureFile`](/docs/concepts/storage/volumes/#azurefile) - Azure File (**migration on by default** starting v1.24) -* [`cephfs`](/docs/concepts/storage/volumes/#cephfs) - CephFS volume - (**deprecated** starting v1.28, no migration plan, support will be removed in a future release) * [`cinder`](/docs/concepts/storage/volumes/#cinder) - Cinder (OpenStack block storage) (**migration on by default** starting v1.21) * [`flexVolume`](/docs/concepts/storage/volumes/#flexvolume) - FlexVolume @@ -534,18 +532,20 @@ please install corresponding CSI drivers. Older versions of Kubernetes also supported the following in-tree PersistentVolume types: -* `photonPersistentDisk` - Photon controller persistent disk. - (**not available** starting v1.15) -* `scaleIO` - ScaleIO volume. - (**not available** starting v1.21) +* [`cephfs`](/docs/concepts/storage/volumes/#cephfs) + (**not available** starting v1.31) * `flocker` - Flocker storage. (**not available** starting v1.25) +* `photonPersistentDisk` - Photon controller persistent disk. + (**not available** starting v1.15) * `quobyte` - Quobyte volume. (**not available** starting v1.25) -* `storageos` - StorageOS volume. - (**not available** starting v1.25) * [`rbd`](/docs/concepts/storage/volumes/#rbd) - Rados Block Device (RBD) volume (**not available** starting v1.31) +* `scaleIO` - ScaleIO volume. + (**not available** starting v1.21) +* `storageos` - StorageOS volume. + (**not available** starting v1.25) ## Persistent Volumes diff --git a/content/en/docs/concepts/storage/volumes.md b/content/en/docs/concepts/storage/volumes.md index 20d9b067d5197..05b5f72db9978 100644 --- a/content/en/docs/concepts/storage/volumes.md +++ b/content/en/docs/concepts/storage/volumes.md @@ -124,26 +124,14 @@ Azure File CSI driver does not support using same volume with different fsgroups To disable the `azureFile` storage plugin from being loaded by the controller manager and the kubelet, set the `InTreePluginAzureFileUnregister` flag to `true`. -### cephfs (deprecated) {#cephfs} -{{< feature-state for_k8s_version="v1.28" state="deprecated" >}} +### cephfs (removed) {#cephfs} -{{< note >}} -The Kubernetes project suggests that you use the [CephFS CSI](https://github.com/ceph/ceph-csi) third party -storage driver instead. -{{< /note >}} - -A `cephfs` volume allows an existing CephFS volume to be -mounted into your Pod. Unlike `emptyDir`, which is erased when a pod is -removed, the contents of a `cephfs` volume are preserved and the volume is merely -unmounted. This means that a `cephfs` volume can be pre-populated with data, and -that data can be shared between pods. The `cephfs` volume can be mounted by multiple -writers simultaneously. + -{{< note >}} -You must have your own Ceph server running with the share exported before you can use it. -{{< /note >}} +Kubernetes {{< skew currentVersion >}} does not include a `cephfs` volume type. -See the [CephFS example](https://github.com/kubernetes/examples/tree/master/volumes/cephfs/) for more details. +The `cephfs` in-tree storage driver was deprecated in the Kubernetes v1.28 release and then removed entirely in the v1.31 release. ### cinder (deprecated) {#cinder} From 82348a64ea0ffa0acc9df6e56fb6deb43e45732f Mon Sep 17 00:00:00 2001 From: SataQiu Date: Wed, 8 May 2024 18:26:22 +0800 Subject: [PATCH 09/82] kubeadm: update docs for removed UpgradeAddonsBeforeControlPlane feature gate Co-authored-by: Lubomir I. Ivanov --- .../setup-tools/kubeadm/kubeadm-init.md | 28 +++---------------- .../kubeadm/kubeadm-upgrade.md | 6 +--- 2 files changed, 5 insertions(+), 29 deletions(-) diff --git a/content/en/docs/reference/setup-tools/kubeadm/kubeadm-init.md b/content/en/docs/reference/setup-tools/kubeadm/kubeadm-init.md index 2caa94d6764a1..b07a26931720c 100644 --- a/content/en/docs/reference/setup-tools/kubeadm/kubeadm-init.md +++ b/content/en/docs/reference/setup-tools/kubeadm/kubeadm-init.md @@ -192,30 +192,6 @@ on a control plane node to become ready. The wait process starts right after the is started by kubeadm. You are advised to enable this feature gate in case you wish to observe a ready state from all control plane components during the `kubeadm init` or `kubeadm join` command execution. -List of deprecated feature gates: - -{{< table caption="kubeadm deprecated feature gates" >}} -Feature | Default -:-------|:-------- -`UpgradeAddonsBeforeControlPlane` | `false` -{{< /table >}} - -Feature gate descriptions: - -`UpgradeAddonsBeforeControlPlane` -: This is as a **disabled** feature gate that was introduced for Kubernetes v1.28, in order to allow reactivating a legacy -and deprecated behavior during cluster upgrade. For kubeadm versions prior to v1.28, kubeadm upgrades cluster addons (including -CoreDNS and kube-proxy) immediately during `kubeadm upgrade apply`, regardless of whether there are other control plane -instances that have not been upgraded. This may cause compatibility problems. Since v1.28, kubeadm defaults to a mode that -always checks whether all the control plane instances have been upgraded before starting to upgrade the addons. This behavior -is applied to both `kubeadm upgrade apply` and `kubeadm upgrade node`. kubeadm determines whether a control plane instance -has been upgraded by checking whether the image of the kube-apiserver Pod has been upgraded. You must perform control plane -instances upgrade sequentially or at least ensure that the last control plane instance upgrade is not started until all the -other control plane instances have been upgraded completely, and the addons upgrade will be performed after the last control plane -instance is upgraded. The deprecated `UpgradeAddonsBeforeControlPlane` feature gate gives you a chance to keep the old upgrade -behavior. You should not need this old behavior; if you do, you should consider changing your cluster or upgrade processes, as this -feature gate will be removed in a future release. - List of removed feature gates: {{< table caption="kubeadm removed feature gates" >}} @@ -223,6 +199,7 @@ Feature | Alpha | Beta | GA | Removed :-------|:------|:-----|:---|:------- `IPv6DualStack` | 1.16 | 1.21 | 1.23 | 1.24 `UnversionedKubeletConfigMap` | 1.22 | 1.23 | 1.25 | 1.26 +`UpgradeAddonsBeforeControlPlane` | 1.28 | - | - | 1.31 {{< /table >}} Feature gate descriptions: @@ -241,6 +218,9 @@ or `kubeadm upgrade apply`), kubeadm respects the value of `UnversionedKubeletCo (during `kubeadm join`, `kubeadm reset`, `kubeadm upgrade ...`), kubeadm attempts to use unversioned ConfigMap name first; if that does not succeed, kubeadm falls back to using the legacy (versioned) name for that ConfigMap. +`UpgradeAddonsBeforeControlPlane` +: This feature gate has been removed. It was introduced in v1.28 as a deprecated feature and then removed in v1.31. For documentation on older versions, please switch to the corresponding website version. + ### Adding kube-proxy parameters {#kube-proxy} For information about kube-proxy parameters in the kubeadm configuration see: diff --git a/content/en/docs/tasks/administer-cluster/kubeadm/kubeadm-upgrade.md b/content/en/docs/tasks/administer-cluster/kubeadm/kubeadm-upgrade.md index daa7a141e6e72..f9363b2792098 100644 --- a/content/en/docs/tasks/administer-cluster/kubeadm/kubeadm-upgrade.md +++ b/content/en/docs/tasks/administer-cluster/kubeadm/kubeadm-upgrade.md @@ -189,11 +189,7 @@ Pick a control plane node that you wish to upgrade first. It must have the `/etc the control plane instances have been upgraded before starting to upgrade the addons. You must perform control plane instances upgrade sequentially or at least ensure that the last control plane instance upgrade is not started until all the other control plane instances have been upgraded completely, and the addons upgrade will be performed after the last - control plane instance is upgraded. If you want to keep the old upgrade behavior, please enable the `UpgradeAddonsBeforeControlPlane` - feature gate by `kubeadm upgrade apply --feature-gates=UpgradeAddonsBeforeControlPlane=true`. The Kubernetes project does - not in general recommend enabling this feature gate, you should instead change your upgrade process or cluster addons so - that you do not need to enable the legacy behavior. The `UpgradeAddonsBeforeControlPlane` feature gate will be removed in - a future release. + control plane instance is upgraded. {{}} 1. Manually upgrade your CNI provider plugin. From b8ec088c475f70a288c94e7ad30231d9e6b216a9 Mon Sep 17 00:00:00 2001 From: SataQiu Date: Sun, 19 May 2024 16:42:42 +0800 Subject: [PATCH 10/82] kubeadm: add corednsdeployment as a patch target --- .../tools/kubeadm/control-plane-flags.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/content/en/docs/setup/production-environment/tools/kubeadm/control-plane-flags.md b/content/en/docs/setup/production-environment/tools/kubeadm/control-plane-flags.md index 6b3724c8f7781..3cfdd6e4b0918 100644 --- a/content/en/docs/setup/production-environment/tools/kubeadm/control-plane-flags.md +++ b/content/en/docs/setup/production-environment/tools/kubeadm/control-plane-flags.md @@ -168,8 +168,7 @@ patches: The directory must contain files named `target[suffix][+patchtype].extension`. For example, `kube-apiserver0+merge.yaml` or just `etcd.json`. -- `target` can be one of `kube-apiserver`, `kube-controller-manager`, `kube-scheduler`, `etcd` -and `kubeletconfiguration`. +- `target` can be one of `kube-apiserver`, `kube-controller-manager`, `kube-scheduler`, `etcd`, `kubeletconfiguration` and `corednsdeployment`. - `patchtype` can be one of `strategic`, `merge` or `json` and these must match the patching formats [supported by kubectl](/docs/tasks/manage-kubernetes-objects/update-api-object-kubectl-patch). The default `patchtype` is `strategic`. From 7f892d60db8dc186a6f349a776b41ce21df67d19 Mon Sep 17 00:00:00 2001 From: "Lubomir I. Ivanov" Date: Mon, 20 May 2024 17:07:37 +0300 Subject: [PATCH 11/82] kubeadm: move the RootlessControlPlane FG to deprecated --- .../setup-tools/kubeadm/kubeadm-init.md | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/content/en/docs/reference/setup-tools/kubeadm/kubeadm-init.md b/content/en/docs/reference/setup-tools/kubeadm/kubeadm-init.md index b07a26931720c..3781f9045a510 100644 --- a/content/en/docs/reference/setup-tools/kubeadm/kubeadm-init.md +++ b/content/en/docs/reference/setup-tools/kubeadm/kubeadm-init.md @@ -157,7 +157,6 @@ Feature | Default | Alpha | Beta | GA :-------|:--------|:------|:-----|:---- `EtcdLearnerMode` | `true` | 1.27 | 1.29 | - `PublicKeysECDSA` | `false` | 1.19 | - | - -`RootlessControlPlane` | `false` | 1.22 | - | - `WaitForAllControlPlaneComponents` | `false` | 1.30 | - | - {{< /table >}} @@ -176,12 +175,6 @@ as a learner and promoted to a voting member only after the etcd data are fully Renewal of existing ECDSA certificates is also supported using `kubeadm certs renew`, but you cannot switch between the RSA and ECDSA algorithms on the fly or during upgrades. -`RootlessControlPlane` -: Setting this flag configures the kubeadm deployed control plane component static Pod containers -for `kube-apiserver`, `kube-controller-manager`, `kube-scheduler` and `etcd` to run as non-root users. -If the flag is not set, those components run as root. You can change the value of this feature gate before -you upgrade to a newer version of Kubernetes. - `WaitForAllControlPlaneComponents` : With this feature gate enabled kubeadm will wait for all control plane components (kube-apiserver, kube-controller-manager, kube-scheduler) on a control plane node to report status 200 on their `/healthz` @@ -192,6 +185,22 @@ on a control plane node to become ready. The wait process starts right after the is started by kubeadm. You are advised to enable this feature gate in case you wish to observe a ready state from all control plane components during the `kubeadm init` or `kubeadm join` command execution. +List of deprecated feature gates: + +{{< table caption="kubeadm deprecated feature gates" >}} +Feature | Default | Alpha | Beta | GA | Deprecated +:-------|:--------|:------|:-----|:---|:---------- +`RootlessControlPlane` | `false` | 1.22 | - | - | 1.31 +{{< /table >}} + +Feature gate descriptions: + +`RootlessControlPlane` +: Setting this flag configures the kubeadm deployed control plane component static Pod containers +for `kube-apiserver`, `kube-controller-manager`, `kube-scheduler` and `etcd` to run as non-root users. +If the flag is not set, those components run as root. You can change the value of this feature gate before +you upgrade to a newer version of Kubernetes. + List of removed feature gates: {{< table caption="kubeadm removed feature gates" >}} From e6de84dacd1616ebb6b7d0b751d15f2b677b6bb7 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Sat, 25 May 2024 09:43:15 -0400 Subject: [PATCH 12/82] Update nftables kube-proxy docs for 1.31 beta --- .../feature-gates/nftables-proxy-mode.md | 6 +- .../docs/reference/networking/virtual-ips.md | 69 +++++++++++++++++-- 2 files changed, 67 insertions(+), 8 deletions(-) diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/nftables-proxy-mode.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/nftables-proxy-mode.md index de2cf6bf7b2a5..efbf6d8da5a58 100644 --- a/content/en/docs/reference/command-line-tools-reference/feature-gates/nftables-proxy-mode.md +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/nftables-proxy-mode.md @@ -9,5 +9,9 @@ stages: - stage: alpha defaultValue: false fromVersion: "1.29" + toVersion: "1.30" + - stage: beta + defaultValue: true + fromVersion: "1.31" --- -Allow running kube-proxy with in [nftables mode](/docs/reference/networking/virtual-ips/#proxy-mode-nftables). +Allow running kube-proxy in [nftables mode](/docs/reference/networking/virtual-ips/#proxy-mode-nftables). diff --git a/content/en/docs/reference/networking/virtual-ips.md b/content/en/docs/reference/networking/virtual-ips.md index 269c41654133f..2592bdb3ab348 100644 --- a/content/en/docs/reference/networking/virtual-ips.md +++ b/content/en/docs/reference/networking/virtual-ips.md @@ -262,20 +262,75 @@ exits with an error. ### `nftables` proxy mode {#proxy-mode-nftables} -{{< feature-state for_k8s_version="v1.29" state="alpha" >}} +{{< feature-state feature_gate_name="NFTablesProxyMode" >}} -_This proxy mode is only available on Linux nodes._ +_This proxy mode is only available on Linux nodes, and requires kernel +5.13 or later._ In this mode, kube-proxy configures packet forwarding rules using the nftables API of the kernel netfilter subsystem. For each endpoint, it installs nftables rules which, by default, select a backend Pod at random. -The nftables API is the successor to the iptables API, and although it -is designed to provide better performance and scalability than -iptables, the kube-proxy nftables mode is still under heavy -development as of {{< skew currentVersion >}} and is not necessarily -expected to outperform the other Linux modes at this time. +The nftables API is the successor to the iptables API and is designed +to provide better performance and scalability than iptables. The +`nftables` proxy mode is able to process changes to service endpoints +faster and more efficiently than the `iptables` mode, and is also able +to more efficiently process packets in the kernel (though this only +becomes noticeable in clusters with tens of thousands of services). + +As of Kubernetes {{< skew currentVersion >}}, the `nftables` mode is +still relatively new, and may not be compatible with all network +plugins; consult the documentation for your network plugin. + +#### Migrating from `iptables` mode to `nftables` + +Users who want to switch from the default `iptables` mode to the +`nftables` mode should be aware that some features work slightly +differently the `nftables` mode: + + - **NodePort interfaces**: In `iptables` mode, by default, + [NodePort services](/docs/concepts/services-networking/service/#type-nodeport) + are reachable on all local IP addresses. This is usually not what + users want, so the `nftables` mode defaults to + `--nodeport-addresses primary`, meaning NodePort services are only + reachable on the node's primary IPv4 and/or IPv6 addresses. You can + override this by specifying an explicit value for that option: + e.g., `--nodeport-addresses 0.0.0.0/0` to listen on all (local) + IPv4 IPs. + + - **NodePort services on `127.0.0.1`**: In `iptables` mode, if the + `--nodeport-addresses` range includes `127.0.0.1` (and the option + `--iptables-localhost-nodeports false` option is not passed), then + NodePort services are reachable even on "localhost" (`127.0.0.1`). + In `nftables` mode (and `ipvs` mode), this will not work. If you + are not sure if you are depending on this functionality, you can + check kube-proxy's + `iptables_localhost_nodeports_accepted_packets_total` metric; if it + is non-0, that means that some client has connected to a NodePort + service via `127.0.0.1`. + + - **NodePort interaction with firewalls**: The `iptables` mode of + kube-proxy tries to be compatible with overly-agressive firewalls; + for each NodePort service, it will add rules to accept inbound + traffic on that port, in case that traffic would otherwise be + blocked by a firewall. This approach will not work with firewalls + based on nftables, so kube-proxy's `nftables` mode does not do + anything here; if you have a local firewall, you must ensure that + it is properly configured to allow Kubernetes traffic through + (e.g., by allowing inbound traffic on the entire NodePort range). + + - **Conntrack bug workarounds**: Linux kernels prior to 6.1 have a + bug that can result in long-lived TCP connections to service IPs + being closed with the error "Connection reset by peer". The + `iptables` mode of kube-proxy installs a workaround for this bug, + but this workaround was later found to cause other problems in some + clusters. The `nftables` mode does not install any workaround by + default, but you can check kube-proxy's + `iptables_ct_state_invalid_dropped_packets_total` metric to see if + your cluster is depending on the workaround, and if so, you can run + kube-proxy with the option `--conntrack-tcp-be-liberal` to work + around the problem in `nftables` mode. ### `kernelspace` proxy mode {#proxy-mode-kernelspace} From 9eccaf39d29d1d16ab7ad5166bf45ccf2ae95476 Mon Sep 17 00:00:00 2001 From: Sascha Grunert Date: Wed, 29 May 2024 10:38:52 +0200 Subject: [PATCH 13/82] Make cri-tools optional for kubeadm Signed-off-by: Sascha Grunert --- .../production-environment/tools/kubeadm/install-kubeadm.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/content/en/docs/setup/production-environment/tools/kubeadm/install-kubeadm.md b/content/en/docs/setup/production-environment/tools/kubeadm/install-kubeadm.md index a6b98847e47d3..6a0246ecb3187 100644 --- a/content/en/docs/setup/production-environment/tools/kubeadm/install-kubeadm.md +++ b/content/en/docs/setup/production-environment/tools/kubeadm/install-kubeadm.md @@ -297,10 +297,10 @@ DOWNLOAD_DIR="/usr/local/bin" sudo mkdir -p "$DOWNLOAD_DIR" ``` -Install crictl (required for kubeadm / Kubelet Container Runtime Interface (CRI)): +Optionally install crictl (required for interaction with the Container Runtime Interface (CRI), optional for kubeadm): ```bash -CRICTL_VERSION="v1.28.0" +CRICTL_VERSION="v1.31.0" ARCH="amd64" curl -L "https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-${ARCH}.tar.gz" | sudo tar -C $DOWNLOAD_DIR -xz ``` From 878f27cf6175fac9333ed7eddd6ba487790abdf8 Mon Sep 17 00:00:00 2001 From: Ed Bartosh Date: Thu, 15 Feb 2024 14:05:35 +0200 Subject: [PATCH 14/82] Device Plugins: add info about GA graduation --- .../extend-kubernetes/compute-storage-net/device-plugins.md | 2 +- .../feature-gates/device-plugin-cdi-devices.md | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/content/en/docs/concepts/extend-kubernetes/compute-storage-net/device-plugins.md b/content/en/docs/concepts/extend-kubernetes/compute-storage-net/device-plugins.md index 6752c43a0d88d..5d6355844f4dd 100644 --- a/content/en/docs/concepts/extend-kubernetes/compute-storage-net/device-plugins.md +++ b/content/en/docs/concepts/extend-kubernetes/compute-storage-net/device-plugins.md @@ -160,7 +160,7 @@ The general workflow of a device plugin includes the following steps: The processing of the fully-qualified CDI device names by the Device Manager requires that the `DevicePluginCDIDevices` [feature gate](/docs/reference/command-line-tools-reference/feature-gates/) is enabled for both the kubelet and the kube-apiserver. This was added as an alpha feature in Kubernetes - v1.28 and graduated to beta in v1.29. + v1.28, graduated to beta in v1.29 and to GA in v1.31. {{< /note >}} ### Handling kubelet restarts diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/device-plugin-cdi-devices.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/device-plugin-cdi-devices.md index 4af3ec4c84bae..e424602bdc7bc 100644 --- a/content/en/docs/reference/command-line-tools-reference/feature-gates/device-plugin-cdi-devices.md +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/device-plugin-cdi-devices.md @@ -13,6 +13,10 @@ stages: - stage: beta defaultValue: true fromVersion: "1.29" + toVersion: "1.30" + - stage: stable + defaultValue: true + fromVersion: "1.31" --- Enable support to CDI device IDs in the From c07a10864bef5ac4c7f2f203bb3ab96d90754a59 Mon Sep 17 00:00:00 2001 From: Roman Bednar Date: Mon, 20 May 2024 13:40:38 +0200 Subject: [PATCH 15/82] graduate PersistentVolumeLastPhaseTransitionTime to GA in v1.31 --- content/en/docs/concepts/storage/persistent-volumes.md | 7 +------ .../persistent-volume-last-phase-transition-time.md | 4 ++++ 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/content/en/docs/concepts/storage/persistent-volumes.md b/content/en/docs/concepts/storage/persistent-volumes.md index e143a060a34a2..48900678ba463 100644 --- a/content/en/docs/concepts/storage/persistent-volumes.md +++ b/content/en/docs/concepts/storage/persistent-volumes.md @@ -766,18 +766,13 @@ You can see the name of the PVC bound to the PV using `kubectl describe persiste #### Phase transition timestamp -{{< feature-state for_k8s_version="v1.29" state="beta" >}} +{{< feature-state feature_gate_name="PersistentVolumeLastPhaseTransitionTime" >}} The `.status` field for a PersistentVolume can include an alpha `lastPhaseTransitionTime` field. This field records the timestamp of when the volume last transitioned its phase. For newly created volumes the phase is set to `Pending` and `lastPhaseTransitionTime` is set to the current time. -{{< note >}} -You need to enable the `PersistentVolumeLastPhaseTransitionTime` [feature gate](/docs/reference/command-line-tools-reference/feature-gates/) -to use or see the `lastPhaseTransitionTime` field. -{{< /note >}} - ## PersistentVolumeClaims Each PVC contains a spec and status, which is the specification and status of the claim. diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/persistent-volume-last-phase-transition-time.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/persistent-volume-last-phase-transition-time.md index 4c2900c978c18..d489173d4f810 100644 --- a/content/en/docs/reference/command-line-tools-reference/feature-gates/persistent-volume-last-phase-transition-time.md +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/persistent-volume-last-phase-transition-time.md @@ -13,6 +13,10 @@ stages: - stage: beta defaultValue: true fromVersion: "1.29" + toVersion: "1.30" + - stage: stable + defaultValue: true + fromVersion: "1.31" --- Adds a new field to PersistentVolume which holds a timestamp of when the volume last transitioned its phase. From 24b51c287f8d653d8a7c6d0c6b3008bda58bc8ce Mon Sep 17 00:00:00 2001 From: carlory Date: Wed, 12 Jun 2024 14:44:32 +0800 Subject: [PATCH 16/82] promote HonorPVReclaimPolicy to beta --- content/en/docs/concepts/storage/persistent-volumes.md | 2 +- .../feature-gates/honor-pv-reclaim-policy.md | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/content/en/docs/concepts/storage/persistent-volumes.md b/content/en/docs/concepts/storage/persistent-volumes.md index e143a060a34a2..a333857e28a3d 100644 --- a/content/en/docs/concepts/storage/persistent-volumes.md +++ b/content/en/docs/concepts/storage/persistent-volumes.md @@ -248,7 +248,7 @@ However, the particular path specified in the custom recycler Pod template in th `volumes` part is replaced with the particular path of the volume that is being recycled. ### PersistentVolume deletion protection finalizer -{{< feature-state for_k8s_version="v1.23" state="alpha" >}} +{{< feature-state feature_gate_name="HonorPVReclaimPolicy" >}} Finalizers can be added on a PersistentVolume to ensure that PersistentVolumes having `Delete` reclaim policy are deleted only after the backing storage are deleted. diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/honor-pv-reclaim-policy.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/honor-pv-reclaim-policy.md index 88d599eca6808..a02110cf6856b 100644 --- a/content/en/docs/reference/command-line-tools-reference/feature-gates/honor-pv-reclaim-policy.md +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/honor-pv-reclaim-policy.md @@ -9,6 +9,10 @@ stages: - stage: alpha defaultValue: false fromVersion: "1.23" + toVersion: "1.30" + - stage: beta + defaultValue: true + fromVersion: "1.31" --- Honor persistent volume reclaim policy when it is `Delete` irrespective of PV-PVC deletion ordering. For more details, check the From 76ec4489c14d716e663f3b9a3e02af2e8d679264 Mon Sep 17 00:00:00 2001 From: MinpengJin Date: Tue, 18 Jun 2024 18:53:12 +0800 Subject: [PATCH 17/82] Add documentation for LogarithmicScaleDown featuregate --- content/en/docs/concepts/workloads/controllers/replicaset.md | 3 +-- .../feature-gates/logarithmic-scale-down.md | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/content/en/docs/concepts/workloads/controllers/replicaset.md b/content/en/docs/concepts/workloads/controllers/replicaset.md index 28aa6423636c4..5f27e1d48b74f 100644 --- a/content/en/docs/concepts/workloads/controllers/replicaset.md +++ b/content/en/docs/concepts/workloads/controllers/replicaset.md @@ -339,8 +339,7 @@ prioritize scaling down pods based on the following general algorithm: the pod with the lower value will come first. 1. Pods on nodes with more replicas come before pods on nodes with fewer replicas. 1. If the pods' creation times differ, the pod that was created more recently - comes before the older pod (the creation times are bucketed on an integer log scale - when the `LogarithmicScaleDown` [feature gate](/docs/reference/command-line-tools-reference/feature-gates/) is enabled) + comes before the older pod (the creation times are bucketed on an integer log scale). If all of the above match, then selection is random. diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/logarithmic-scale-down.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/logarithmic-scale-down.md index 434233e84a066..0911f2a7e36a9 100644 --- a/content/en/docs/reference/command-line-tools-reference/feature-gates/logarithmic-scale-down.md +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/logarithmic-scale-down.md @@ -13,6 +13,10 @@ stages: - stage: beta defaultValue: true fromVersion: "1.22" + toVersion: "1.30" + - stage: stable + defaultValue: true + fromVersion: "1.31" --- Enable semi-random selection of pods to evict on controller scaledown based on logarithmic bucketing of pod timestamps. From 0fc04db4def6fcffa894b673b5792ff570bf5b5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=A8=E6=9C=B1=20=C2=B7=20Kiki?= Date: Fri, 21 Jun 2024 07:17:21 +0800 Subject: [PATCH 18/82] update portworxVolume doc (#46506) * update portworxVolume doc Co-authored-by: Tim Bannister * Update content/en/docs/concepts/storage/volumes.md Co-authored-by: Qiming Teng * Update content/en/docs/concepts/storage/volumes.md Co-authored-by: Qiming Teng --------- Co-authored-by: Tim Bannister Co-authored-by: Qiming Teng --- .../en/docs/concepts/storage/persistent-volumes.md | 2 +- content/en/docs/concepts/storage/volumes.md | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/content/en/docs/concepts/storage/persistent-volumes.md b/content/en/docs/concepts/storage/persistent-volumes.md index a333857e28a3d..c55d1deed8740 100644 --- a/content/en/docs/concepts/storage/persistent-volumes.md +++ b/content/en/docs/concepts/storage/persistent-volumes.md @@ -526,7 +526,7 @@ please install corresponding CSI drivers. * [`gcePersistentDisk`](/docs/concepts/storage/volumes/#gcePersistentDisk) - GCE Persistent Disk (**migration on by default** starting v1.23) * [`portworxVolume`](/docs/concepts/storage/volumes/#portworxvolume) - Portworx volume - (**deprecated** starting v1.25) + (**migration on by default** starting v1.31) * [`vsphereVolume`](/docs/concepts/storage/volumes/#vspherevolume) - vSphere VMDK volume (**migration on by default** starting v1.25) diff --git a/content/en/docs/concepts/storage/volumes.md b/content/en/docs/concepts/storage/volumes.md index f0a7540fd5ac9..2abda301501ff 100644 --- a/content/en/docs/concepts/storage/volumes.md +++ b/content/en/docs/concepts/storage/volumes.md @@ -720,13 +720,18 @@ For more details, see the [Portworx volume](https://github.com/kubernetes/exampl #### Portworx CSI migration {{< feature-state for_k8s_version="v1.25" state="beta" >}} -The `CSIMigration` feature for Portworx has been added but disabled by default in Kubernetes 1.23 since it's in alpha state. -It has been beta now since v1.25 but it is still turned off by default. +By default, Kubernetes {{% skew currentVersion %}} attempts to migrate legacy +Portworx volumes to use CSI. (CSI migration for Portworx has been available since +Kubernetes v1.23, but was only turned on by default since the v1.31 release). +If you want to disable automatic migration, you can set the `CSIMigrationPortworx` +[feature gate](/docs/reference/command-line-tools-reference/feature-gates/) +to `false`; you need to make that change for the kube-controller-manager **and** on +every relevant kubelet. + It redirects all plugin operations from the existing in-tree plugin to the `pxd.portworx.com` Container Storage Interface (CSI) Driver. [Portworx CSI Driver](https://docs.portworx.com/portworx-enterprise/operations/operate-kubernetes/storage-operations/csi) must be installed on the cluster. -To enable the feature, set `CSIMigrationPortworx=true` in kube-controller-manager and kubelet. ### projected From a49b8a546926a3ae5e73ed30a63ede83270e1cb9 Mon Sep 17 00:00:00 2001 From: Jun <56640554+SequoiaGod@users.noreply.github.com> Date: Sun, 23 Jun 2024 15:56:56 +0100 Subject: [PATCH 19/82] kubeadm: add note about a bug in the PublicKeysECDSA feature gate (#46852) * kubeadm: add note about a bug in the PublicKeysECDSA feature gate xref #46799 * kubeadm: add note about a bug in the PublicKeysECDSA feature gate xref #46799 --- content/en/docs/reference/setup-tools/kubeadm/kubeadm-init.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/content/en/docs/reference/setup-tools/kubeadm/kubeadm-init.md b/content/en/docs/reference/setup-tools/kubeadm/kubeadm-init.md index 701ce650fdbd5..cc17055e456e5 100644 --- a/content/en/docs/reference/setup-tools/kubeadm/kubeadm-init.md +++ b/content/en/docs/reference/setup-tools/kubeadm/kubeadm-init.md @@ -175,7 +175,8 @@ as a learner and promoted to a voting member only after the etcd data are fully Renewal of existing ECDSA certificates is also supported using `kubeadm certs renew`, but you cannot switch between the RSA and ECDSA algorithms on the fly or during upgrades. Kubernetes {{< skew currentVersion >}} has a bug where keys in generated kubeconfig files are set use RSA -despite the feature gate being enabled. +despite the feature gate being enabled. Kubernetes versions before v1.31 had a bug where keys in generated kubeconfig files +were set use RSA, even when you had enabled the `PublicKeysECDSA` feature gate. `WaitForAllControlPlaneComponents` : With this feature gate enabled kubeadm will wait for all control plane components (kube-apiserver, From 84cd8d400527509ae25a869ca18793f329ce8fc6 Mon Sep 17 00:00:00 2001 From: HirazawaUi <695097494plus@gmail.com> Date: Sun, 31 Mar 2024 23:44:03 +0800 Subject: [PATCH 20/82] promote DisableNodeKubeProxyVersion feature gate to beta --- .../feature-gates/disable-node-kube-proxy-version.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/disable-node-kube-proxy-version.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/disable-node-kube-proxy-version.md index e532d107e4d0e..48b138224311b 100644 --- a/content/en/docs/reference/command-line-tools-reference/feature-gates/disable-node-kube-proxy-version.md +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/disable-node-kube-proxy-version.md @@ -9,5 +9,9 @@ stages: - stage: alpha defaultValue: false fromVersion: "1.29" + toVersion: "1.30" + - stage: beta + defaultValue: true + fromVersion: "1.31" --- Disable setting the `kubeProxyVersion` field of the Node. From 1cc31fc270b5c9620e4a22d4b9508909438b8b4b Mon Sep 17 00:00:00 2001 From: Christian Schlotter Date: Mon, 24 Jun 2024 10:32:02 +0200 Subject: [PATCH 21/82] kubeadm: add ControlPlaneKubeletLocalMode FG descriptions --- .../en/docs/reference/setup-tools/kubeadm/kubeadm-init.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/content/en/docs/reference/setup-tools/kubeadm/kubeadm-init.md b/content/en/docs/reference/setup-tools/kubeadm/kubeadm-init.md index cc17055e456e5..fa170017f78be 100644 --- a/content/en/docs/reference/setup-tools/kubeadm/kubeadm-init.md +++ b/content/en/docs/reference/setup-tools/kubeadm/kubeadm-init.md @@ -155,6 +155,7 @@ List of feature gates: {{< table caption="kubeadm feature gates" >}} Feature | Default | Alpha | Beta | GA :-------|:--------|:------|:-----|:---- +`ControlPlaneKubeletLocalMode` | `false` | 1.31 | - | - `EtcdLearnerMode` | `true` | 1.27 | 1.29 | - `PublicKeysECDSA` | `false` | 1.19 | - | - `WaitForAllControlPlaneComponents` | `false` | 1.30 | - | - @@ -166,6 +167,11 @@ Once a feature gate goes GA its value becomes locked to `true` by default. Feature gate descriptions: +`ControlPlaneKubeletLocalMode` +: With this feature gate enabled, when joining a new control plane node, kubeadm will configure the kubelet +to connect to the local kube-apiserver. This ensures that there will not be a violation of the version skew +policy during rolling upgrades. + `EtcdLearnerMode` : With this feature gate enabled, when joining a new control plane node, a new etcd member will be created as a learner and promoted to a voting member only after the etcd data are fully aligned. From 9a39692ec8a976fa864da315da38a6c68cdbaf26 Mon Sep 17 00:00:00 2001 From: Sean Sullivan Date: Mon, 24 Jun 2024 16:00:48 -0700 Subject: [PATCH 22/82] PortForward over WebSockets update feature gates --- .../feature-gates/port-forward-websockets.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/port-forward-websockets.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/port-forward-websockets.md index fb541f9f0ae15..003b8d868637d 100644 --- a/content/en/docs/reference/command-line-tools-reference/feature-gates/port-forward-websockets.md +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/port-forward-websockets.md @@ -9,6 +9,10 @@ stages: - stage: alpha defaultValue: false fromVersion: "1.30" + toVersion: "1.30" + - stage: beta + defaultValue: true + fromVersion: "1.31" --- Allow WebSocket streaming of the portforward sub-protocol (`port-forward`) from clients requesting From ef8238501c572cad6d656d884cf2fcbf2da369ea Mon Sep 17 00:00:00 2001 From: Joe Betz Date: Wed, 26 Jun 2024 10:45:09 -0400 Subject: [PATCH 23/82] KEP-4420: Retry Generate Name: Promote to beta --- .../feature-gates/name-generation-retries.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/name-generation-retries.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/name-generation-retries.md index a2d0018e7fd3f..1ea932bbb4d05 100644 --- a/content/en/docs/reference/command-line-tools-reference/feature-gates/name-generation-retries.md +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/name-generation-retries.md @@ -10,7 +10,10 @@ stages: - stage: alpha defaultValue: false fromVersion: "1.30" - + toVersion: "1.30" + - stage: beta + defaultValue: true + fromVersion: "1.31" --- Enables retrying of object creation when the {{< glossary_tooltip text="API server" term_id="kube-apiserver" >}} From c03a912e5490f5d195a9b5e0efb645a41e78fc20 Mon Sep 17 00:00:00 2001 From: Qiming Teng Date: Tue, 25 Jun 2024 15:35:25 +0800 Subject: [PATCH 24/82] [zh-cn] Fix indentation in sample policy --- .../services-networking/network-policies.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/content/zh-cn/docs/concepts/services-networking/network-policies.md b/content/zh-cn/docs/concepts/services-networking/network-policies.md index eb441af8a88ea..278e7ecd03394 100644 --- a/content/zh-cn/docs/concepts/services-networking/network-policies.md +++ b/content/zh-cn/docs/concepts/services-networking/network-policies.md @@ -609,14 +609,14 @@ spec: matchLabels: app: myapp policyTypes: - - Egress + - Egress egress: - - to: - - namespaceSelector: - matchExpressions: - - key: namespace - operator: In - values: ["frontend", "backend"] + - to: + - namespaceSelector: + matchExpressions: + - key: namespace + operator: In + values: ["frontend", "backend"] ``` {{< note >}} From cbea62f4c1a7d3b26e83031105764a196170b0f9 Mon Sep 17 00:00:00 2001 From: ahg-g Date: Thu, 27 Jun 2024 06:42:30 +0000 Subject: [PATCH 25/82] Graduate ElasticIndexedJob to GA --- content/en/docs/concepts/workloads/controllers/job.md | 6 ++---- .../feature-gates/elastic-indexed-job.md | 4 ++++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/content/en/docs/concepts/workloads/controllers/job.md b/content/en/docs/concepts/workloads/controllers/job.md index 37b839a148d16..67830dd1305d7 100644 --- a/content/en/docs/concepts/workloads/controllers/job.md +++ b/content/en/docs/concepts/workloads/controllers/job.md @@ -1004,13 +1004,11 @@ observe that pods from a Job are stuck with the tracking finalizer. ### Elastic Indexed Jobs -{{< feature-state for_k8s_version="v1.27" state="beta" >}} +{{< feature-state feature_gate_name="ElasticIndexedJob" >}} You can scale Indexed Jobs up or down by mutating both `.spec.parallelism` and `.spec.completions` together such that `.spec.parallelism == .spec.completions`. -When the `ElasticIndexedJob`[feature gate](/docs/reference/command-line-tools-reference/feature-gates/) -on the [API server](/docs/reference/command-line-tools-reference/kube-apiserver/) -is disabled, `.spec.completions` is immutable. +When scaling down, Kubernetes removes the Pods with higher indexes. Use cases for elastic Indexed Jobs include batch workloads which require scaling an indexed Job, such as MPI, Horovord, Ray, and PyTorch training jobs. diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/elastic-indexed-job.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/elastic-indexed-job.md index b1238b3e99a57..000dabce6faf6 100644 --- a/content/en/docs/reference/command-line-tools-reference/feature-gates/elastic-indexed-job.md +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/elastic-indexed-job.md @@ -9,6 +9,10 @@ stages: - stage: beta defaultValue: true fromVersion: "1.27" + toVersion: "1.30" + - stage: stable + defaultValue: true + fromVersion: "1.31" --- Enables Indexed Jobs to be scaled up or down by mutating both `spec.completions` and `spec.parallelism` together such that `spec.completions == spec.parallelism`. From 4b2940eb26d42ac0ef1b267eb9b373156eb83f76 Mon Sep 17 00:00:00 2001 From: Vinayak Goyal Date: Wed, 26 Jun 2024 21:32:55 +0000 Subject: [PATCH 26/82] KEP-4633: Add documentation for Configurable Endpoints for Anonymous Auth. Signed-off-by: Vinayak Goyal --- .../access-authn-authz/authentication.md | 35 +++++++++++++++++++ .../anonymous-auth-configurable-endpoints.md | 14 ++++++++ .../services-networking/network-policies.md | 14 ++++---- 3 files changed, 56 insertions(+), 7 deletions(-) create mode 100644 content/en/docs/reference/command-line-tools-reference/feature-gates/anonymous-auth-configurable-endpoints.md diff --git a/content/en/docs/reference/access-authn-authz/authentication.md b/content/en/docs/reference/access-authn-authz/authentication.md index 342d7502d9191..f6371199b4510 100644 --- a/content/en/docs/reference/access-authn-authz/authentication.md +++ b/content/en/docs/reference/access-authn-authz/authentication.md @@ -1082,6 +1082,41 @@ Starting in 1.6, the ABAC and RBAC authorizers require explicit authorization of `system:anonymous` user or the `system:unauthenticated` group, so legacy policy rules that grant access to the `*` user or `*` group do not include anonymous users. +### Anonymous Authenticator Configuration + +{{< feature-state feature_gate_name="AnonymousAuthConfigurableEndpoints" >}} + +The `AuthenticationConfiguration` can be used to configure the anonymous +authenticator. To enable configuring anonymous auth via the config file you need +enable the `AnonymousAuthConfigurableEndpoints` feature gate. When this feature +gate is enabled you cannot set the `--anonymous-auth` flag. + +The main advantage of configuring anonymous authenticator using the authentication +configuration file is that in addition to enabling and disabling anonymous authentication +you can also configure which endpoints support anonymous authentication. + +A sample authentication configuration file is below: + +```yaml +--- +# +# CAUTION: this is an example configuration. +# Do not use this for your own cluster! +# +apiVersion: apiserver.config.k8s.io/v1beta1 +kind: AuthenticationConfiguration +anonymous: + enabled: true + conditions: + - path: /livez + - path: /readyz + - path: /healthz +``` + +In the configuration above only the `/livez`, `/readyz` and `/healthz` endpoints +are reachable by anonymous requests. Any other endpoints will not be reachable +even if it is allowed by RBAC configuration. + ## User impersonation A user can act as another user through impersonation headers. These let requests diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/anonymous-auth-configurable-endpoints.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/anonymous-auth-configurable-endpoints.md new file mode 100644 index 0000000000000..38538a15d2137 --- /dev/null +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/anonymous-auth-configurable-endpoints.md @@ -0,0 +1,14 @@ +--- +title: AnonymousAuthConfigurableEndpoints +content_type: feature_gate +_build: + list: never + render: false + +stages: + - stage: alpha + defaultValue: false + fromVersion: "1.31" +--- +Enable [configurable endpoints for anonymous auth](/docs/reference/access-authn-authz/authentication/#anonymous-authenticator-onfiguration) +for the API server. diff --git a/content/zh-cn/docs/concepts/services-networking/network-policies.md b/content/zh-cn/docs/concepts/services-networking/network-policies.md index 278e7ecd03394..eb441af8a88ea 100644 --- a/content/zh-cn/docs/concepts/services-networking/network-policies.md +++ b/content/zh-cn/docs/concepts/services-networking/network-policies.md @@ -609,14 +609,14 @@ spec: matchLabels: app: myapp policyTypes: - - Egress + - Egress egress: - - to: - - namespaceSelector: - matchExpressions: - - key: namespace - operator: In - values: ["frontend", "backend"] + - to: + - namespaceSelector: + matchExpressions: + - key: namespace + operator: In + values: ["frontend", "backend"] ``` {{< note >}} From ab961e64dce10d6d290e1c48e0a2035359f57537 Mon Sep 17 00:00:00 2001 From: Alexander Constantinescu Date: Mon, 1 Jul 2024 13:36:02 +0200 Subject: [PATCH 27/82] KEP-3836 - v1.31 doc update --- .../feature-gates/kube-proxy-draining-terminating-nodes.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/kube-proxy-draining-terminating-nodes.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/kube-proxy-draining-terminating-nodes.md index d0c1b00f8f385..4e6a0650f8518 100644 --- a/content/en/docs/reference/command-line-tools-reference/feature-gates/kube-proxy-draining-terminating-nodes.md +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/kube-proxy-draining-terminating-nodes.md @@ -13,6 +13,10 @@ stages: - stage: beta defaultValue: true fromVersion: "1.30" + toVersion: "1.31" + - stage: stable + defaultValue: true + fromVersion: "1.31" --- Implement connection draining for terminating nodes for `externalTrafficPolicy: Cluster` services. From 47920ec05991711253ae536a4c49b3609b83adaa Mon Sep 17 00:00:00 2001 From: Alexander Constantinescu Date: Mon, 1 Jul 2024 14:43:20 +0200 Subject: [PATCH 28/82] Update content/en/docs/reference/command-line-tools-reference/feature-gates/kube-proxy-draining-terminating-nodes.md Co-authored-by: Dipesh Rawat --- .../feature-gates/kube-proxy-draining-terminating-nodes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/kube-proxy-draining-terminating-nodes.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/kube-proxy-draining-terminating-nodes.md index 4e6a0650f8518..7048999005a92 100644 --- a/content/en/docs/reference/command-line-tools-reference/feature-gates/kube-proxy-draining-terminating-nodes.md +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/kube-proxy-draining-terminating-nodes.md @@ -13,7 +13,7 @@ stages: - stage: beta defaultValue: true fromVersion: "1.30" - toVersion: "1.31" + toVersion: "1.30" - stage: stable defaultValue: true fromVersion: "1.31" From ca7f124629474371b91e586f1d9157564a8d9ea3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20Tyczy=C5=84ski?= Date: Tue, 2 Jul 2024 11:36:49 +0200 Subject: [PATCH 29/82] ResilientWatchCacheInitialization documentation --- .../resilient-watch-cache-initialization.md | 15 +++++++++++++++ .../watch-cache-initialization-post-start-hook.md | 15 +++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 content/en/docs/reference/command-line-tools-reference/feature-gates/resilient-watch-cache-initialization.md create mode 100644 content/en/docs/reference/command-line-tools-reference/feature-gates/watch-cache-initialization-post-start-hook.md diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/resilient-watch-cache-initialization.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/resilient-watch-cache-initialization.md new file mode 100644 index 0000000000000..933d280243c74 --- /dev/null +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/resilient-watch-cache-initialization.md @@ -0,0 +1,15 @@ +--- +title: ResilientWatchCacheInitialization +content_type: feature_gate + +_build: + list: never + render: false + +stages: + - stage: beta + defaultValue: true + fromVersion: "1.31" + +--- +Enables resilient watchcache initialization to avoid controlplane overload. diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/watch-cache-initialization-post-start-hook.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/watch-cache-initialization-post-start-hook.md new file mode 100644 index 0000000000000..f838d4bfea740 --- /dev/null +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/watch-cache-initialization-post-start-hook.md @@ -0,0 +1,15 @@ +--- +title: WatchCacheInitializationPostStartHook +content_type: feature_gate + +_build: + list: never + render: false + +stages: + - stage: beta + defaultValue: false + fromVersion: "1.31" + +--- +Enables post-start-hook for watchcache initialization to be part of readyz (with timeout). From 45aa96c9e1a39910859666663f94fae0e48cde5d Mon Sep 17 00:00:00 2001 From: "Lubomir I. Ivanov" Date: Fri, 5 Jul 2024 15:55:09 +0300 Subject: [PATCH 30/82] kubeadm: update notes about upgrade and --config --- .../kubeadm/kubeadm-upgrade.md | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/content/en/docs/tasks/administer-cluster/kubeadm/kubeadm-upgrade.md b/content/en/docs/tasks/administer-cluster/kubeadm/kubeadm-upgrade.md index 1494e79b33b38..b96169a808dae 100644 --- a/content/en/docs/tasks/administer-cluster/kubeadm/kubeadm-upgrade.md +++ b/content/en/docs/tasks/administer-cluster/kubeadm/kubeadm-upgrade.md @@ -49,9 +49,10 @@ The upgrade workflow at high level is the following: - All containers are restarted after upgrade, because the container spec hash value is changed. - To verify that the kubelet service has successfully restarted after the kubelet has been upgraded, you can execute `systemctl status kubelet` or view the service logs with `journalctl -xeu kubelet`. -- Usage of the `--config` flag of `kubeadm upgrade` with - [kubeadm configuration API types](/docs/reference/config-api/kubeadm-config.v1beta3) - with the purpose of reconfiguring the cluster is not recommended and can have unexpected results. Follow the steps in +- `kubeadm upgrade` supports `--config` with a +[`UpgradeConfiguration` API type](/docs/reference/config-api/kubeadm-config.v1beta4) which can +be used to configure the upgrade process. +- `kubeadm upgrade` does not support reconfiguration of an existing cluster. Follow the steps in [Reconfiguring a kubeadm cluster](/docs/tasks/administer-cluster/kubeadm/kubeadm-reconfigure) instead. ### Considerations when upgrading etcd @@ -161,13 +162,7 @@ Pick a control plane node that you wish to upgrade first. It must have the `/etc For more information see the [certificate management guide](/docs/tasks/administer-cluster/kubeadm/kubeadm-certs). {{}} - {{< note >}} - If `kubeadm upgrade plan` shows any component configs that require manual upgrade, users must provide - a config file with replacement configs to `kubeadm upgrade apply` via the `--config` command line flag. - Failing to do so will cause `kubeadm upgrade apply` to exit with an error and not perform an upgrade. - {{}} - -1. Choose a version to upgrade to, and run the appropriate command. For example: + 1. Choose a version to upgrade to, and run the appropriate command. For example: ```shell # replace x with the patch version you picked for this upgrade @@ -309,7 +304,7 @@ manually restored in `/etc/kubernetes/manifests`. If for some reason there is no and post-upgrade manifest file for a certain component, a backup file for it will not be written. {{< note >}} -After the cluster upgrade using kubeadm, the backup directory `/etc/kubernetes/tmp` will remain and +After the cluster upgrade using kubeadm, the backup directory `/etc/kubernetes/tmp` will remain and these backup files will need to be cleared manually. {{}} From efc1133fa4536b2af8a5f5cc5e6ad6ecda773a9a Mon Sep 17 00:00:00 2001 From: "Lubomir I. Ivanov" Date: Fri, 5 Jul 2024 16:06:06 +0300 Subject: [PATCH 31/82] kubeadm: use v1beta4 in all docs examples --- .../setup-tools/kubeadm/kubeadm-init-phase.md | 4 +- .../setup-tools/kubeadm/kubeadm-init.md | 6 +-- .../setup-tools/kubeadm/kubeadm-join.md | 4 +- .../tools/kubeadm/control-plane-flags.md | 37 +++++++++++-------- .../tools/kubeadm/create-cluster-kubeadm.md | 8 ++-- .../tools/kubeadm/dual-stack-support.md | 30 ++++++++------- .../tools/kubeadm/high-availability.md | 4 +- .../kubeadm/setup-ha-etcd-with-kubeadm.md | 4 +- .../tools/kubeadm/troubleshooting-kubeadm.md | 21 ++++++----- .../kubeadm/configure-cgroup-driver.md | 2 +- .../kubeadm/kubeadm-certs.md | 18 +++++---- .../kubeadm/kubeadm-reconfigure.md | 2 +- .../kubeadm/kubeadm-upgrade.md | 2 +- 13 files changed, 77 insertions(+), 65 deletions(-) diff --git a/content/en/docs/reference/setup-tools/kubeadm/kubeadm-init-phase.md b/content/en/docs/reference/setup-tools/kubeadm/kubeadm-init-phase.md index c08427d4b675e..f9783e1fc398f 100644 --- a/content/en/docs/reference/setup-tools/kubeadm/kubeadm-init-phase.md +++ b/content/en/docs/reference/setup-tools/kubeadm/kubeadm-init-phase.md @@ -144,8 +144,8 @@ install them selectively. {{< tab name="kube-proxy" include="generated/kubeadm_init_phase_addon_kube-proxy.md" />}} {{< /tabs >}} -For more details on each field in the `v1beta3` configuration you can navigate to our -[API reference pages.](/docs/reference/config-api/kubeadm-config.v1beta3/) +For more details on each field in the `v1beta4` configuration you can navigate to our +[API reference pages.](/docs/reference/config-api/kubeadm-config.v1beta4/) ## {{% heading "whatsnext" %}} diff --git a/content/en/docs/reference/setup-tools/kubeadm/kubeadm-init.md b/content/en/docs/reference/setup-tools/kubeadm/kubeadm-init.md index fa170017f78be..dba238e9a9da9 100644 --- a/content/en/docs/reference/setup-tools/kubeadm/kubeadm-init.md +++ b/content/en/docs/reference/setup-tools/kubeadm/kubeadm-init.md @@ -133,7 +133,7 @@ If your configuration is not using the latest version it is **recommended** that the [kubeadm config migrate](/docs/reference/setup-tools/kubeadm/kubeadm-config/) command. For more information on the fields and usage of the configuration you can navigate to our -[API reference page](/docs/reference/config-api/kubeadm-config.v1beta3/). +[API reference page](/docs/reference/config-api/kubeadm-config.v1beta4/). ### Using kubeadm init with feature gates {#feature-gates} @@ -143,7 +143,7 @@ of the cluster. Feature gates are removed after a feature graduates to GA. To pass a feature gate you can either use the `--feature-gates` flag for `kubeadm init`, or you can add items into the `featureGates` field when you pass -a [configuration file](/docs/reference/config-api/kubeadm-config.v1beta3/#kubeadm-k8s-io-v1beta3-ClusterConfiguration) +a [configuration file](/docs/reference/config-api/kubeadm-config.v1beta4/#kubeadm-k8s-io-v1beta4-ClusterConfiguration) using `--config`. Passing [feature gates for core Kubernetes components](/docs/reference/command-line-tools-reference/feature-gates) @@ -321,7 +321,7 @@ kubeadm init phase upload-certs --upload-certs --config=SOME_YAML_FILE ``` {{< note >}} A predefined `certificateKey` can be provided in `InitConfiguration` when passing the -[configuration file](/docs/reference/config-api/kubeadm-config.v1beta3/) with `--config`. +[configuration file](/docs/reference/config-api/kubeadm-config.v1beta4/) with `--config`. {{< /note >}} If a predefined certificate key is not passed to `kubeadm init` and diff --git a/content/en/docs/reference/setup-tools/kubeadm/kubeadm-join.md b/content/en/docs/reference/setup-tools/kubeadm/kubeadm-join.md index a2ca3e2050564..8b8275b1c382d 100644 --- a/content/en/docs/reference/setup-tools/kubeadm/kubeadm-join.md +++ b/content/en/docs/reference/setup-tools/kubeadm/kubeadm-join.md @@ -182,7 +182,7 @@ In case the discovery file does not contain credentials, the TLS discovery token #### Use of custom kubelet credentials with `kubeadm join` -To allow `kubeadm join` to use predefined kubelet credentials and skip client TLS bootstrap +To allow `kubeadm join` to use predefined kubelet credentials and skip client TLS bootstrap and CSR approval for a new node: 1. From a working control plane node in the cluster that has `/etc/kubernetes/pki/ca.key` @@ -317,7 +317,7 @@ If your configuration is not using the latest version it is **recommended** that the [kubeadm config migrate](/docs/reference/setup-tools/kubeadm/kubeadm-config/#cmd-config-migrate) command. For more information on the fields and usage of the configuration you can navigate to our -[API reference](/docs/reference/config-api/kubeadm-config.v1beta3/). +[API reference](/docs/reference/config-api/kubeadm-config.v1beta4/). ## {{% heading "whatsnext" %}} diff --git a/content/en/docs/setup/production-environment/tools/kubeadm/control-plane-flags.md b/content/en/docs/setup/production-environment/tools/kubeadm/control-plane-flags.md index 046a727f76fc7..b30355eb63131 100644 --- a/content/en/docs/setup/production-environment/tools/kubeadm/control-plane-flags.md +++ b/content/en/docs/setup/production-environment/tools/kubeadm/control-plane-flags.md @@ -14,7 +14,7 @@ and kube-proxy you can use `KubeletConfiguration` and `KubeProxyConfiguration`, All of these options are possible via the kubeadm configuration API. For more details on each field in the configuration you can navigate to our -[API reference pages](/docs/reference/config-api/kubeadm-config.v1beta3/). +[API reference pages](/docs/reference/config-api/kubeadm-config.v1beta4/). {{< note >}} Customizing the CoreDNS deployment of kubeadm is currently not supported. You must manually @@ -42,7 +42,7 @@ The components are defined using the following structures: - `scheduler` - `etcd` -These structures contain a common `extraArgs` field, that consists of `key: value` pairs. +These structures contain a common `extraArgs` field, that consists of `name` / `value` pairs. To override a flag for a control plane component: 1. Add the appropriate `extraArgs` to your configuration. @@ -72,14 +72,15 @@ For details, see the [reference documentation for kube-apiserver](/docs/referenc Example usage: ```yaml -apiVersion: kubeadm.k8s.io/v1beta3 +apiVersion: kubeadm.k8s.io/v1beta4 kind: ClusterConfiguration kubernetesVersion: v1.16.0 apiServer: extraArgs: - anonymous-auth: "false" - enable-admission-plugins: AlwaysPullImages,DefaultStorageClass - audit-log-path: /home/johndoe/audit.log + - name: "enable-admission-plugins" + value: "AlwaysPullImages,DefaultStorageClass" + - name: "audit-log-path" + value: "/home/johndoe/audit.log" ``` ### ControllerManager flags @@ -89,13 +90,15 @@ For details, see the [reference documentation for kube-controller-manager](/docs Example usage: ```yaml -apiVersion: kubeadm.k8s.io/v1beta3 +apiVersion: kubeadm.k8s.io/v1beta4 kind: ClusterConfiguration kubernetesVersion: v1.16.0 controllerManager: extraArgs: - cluster-signing-key-file: /home/johndoe/keys/ca.key - deployment-controller-sync-period: "50" + - name: "cluster-signing-key-file" + value: "/home/johndoe/keys/ca.key" + - name: "deployment-controller-sync-period" + value: "50" ``` ### Scheduler flags @@ -105,12 +108,13 @@ For details, see the [reference documentation for kube-scheduler](/docs/referenc Example usage: ```yaml -apiVersion: kubeadm.k8s.io/v1beta3 +apiVersion: kubeadm.k8s.io/v1beta4 kind: ClusterConfiguration kubernetesVersion: v1.16.0 scheduler: extraArgs: - config: /etc/kubernetes/scheduler-config.yaml + - name: "config" + value: "/etc/kubernetes/scheduler-config.yaml" extraVolumes: - name: schedulerconfig hostPath: /home/johndoe/schedconfig.yaml @@ -126,12 +130,13 @@ For details, see the [etcd server documentation](https://etcd.io/docs/). Example usage: ```yaml -apiVersion: kubeadm.k8s.io/v1beta3 +apiVersion: kubeadm.k8s.io/v1beta4 kind: ClusterConfiguration etcd: local: extraArgs: - election-timeout: 1000 + - name: "election-timeout" + value: 1000 ``` ## Customizing with patches {#patches} @@ -145,7 +150,7 @@ is written to disk. You can pass this file to `kubeadm init` with `--config `: ```yaml -apiVersion: kubeadm.k8s.io/v1beta3 +apiVersion: kubeadm.k8s.io/v1beta4 kind: InitConfiguration patches: directory: /home/user/somedir @@ -159,7 +164,7 @@ separated by `---`. You can pass this file to `kubeadm join` with `--config `: ```yaml -apiVersion: kubeadm.k8s.io/v1beta3 +apiVersion: kubeadm.k8s.io/v1beta4 kind: JoinConfiguration patches: directory: /home/user/somedir @@ -206,7 +211,7 @@ For additional details see [Configuring each kubelet in your cluster using kubea To customize kube-proxy you can pass a `KubeProxyConfiguration` next your `ClusterConfiguration` or `InitConfiguration` to `kubeadm init` separated by `---`. -For more details you can navigate to our [API reference pages](/docs/reference/config-api/kubeadm-config.v1beta3/). +For more details you can navigate to our [API reference pages](/docs/reference/config-api/kubeadm-config.v1beta4/). {{< note >}} kubeadm deploys kube-proxy as a {{< glossary_tooltip text="DaemonSet" term_id="daemonset" >}}, which means diff --git a/content/en/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm.md b/content/en/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm.md index a2078c77fc4b6..1f16ac79bd89d 100644 --- a/content/en/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm.md +++ b/content/en/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm.md @@ -116,7 +116,7 @@ to a Kubernetes component, the component may exit with an error. To configure the API server advertise address for control plane nodes created with both `init` and `join`, the flag `--apiserver-advertise-address` can be used. -Preferably, this option can be set in the [kubeadm API](/docs/reference/config-api/kubeadm-config.v1beta3) +Preferably, this option can be set in the [kubeadm API](/docs/reference/config-api/kubeadm-config.v1beta4) as `InitConfiguration.localAPIEndpoint` and `JoinConfiguration.controlPlane.localAPIEndpoint`. For kubelets on all nodes, the `--node-ip` option can be passed in @@ -335,8 +335,8 @@ See a list of add-ons that implement the Please refer to the [Installing Addons](/docs/concepts/cluster-administration/addons/#networking-and-network-policy) -page for a non-exhaustive list of networking addons supported by Kubernetes. -You can install a Pod network add-on with the following command on the +page for a non-exhaustive list of networking addons supported by Kubernetes. +You can install a Pod network add-on with the following command on the control-plane node or a node that has the kubeconfig credentials: ```bash @@ -578,7 +578,7 @@ match the kubeadm version with the versions of the control plane components, kub kubeadm can be used with Kubernetes components that are the same version as kubeadm or one version older. The Kubernetes version can be specified to kubeadm by using the `--kubernetes-version` flag of `kubeadm init` or the -[`ClusterConfiguration.kubernetesVersion`](/docs/reference/config-api/kubeadm-config.v1beta3/) +[`ClusterConfiguration.kubernetesVersion`](/docs/reference/config-api/kubeadm-config.v1beta4/) field when using `--config`. This option will control the versions of kube-apiserver, kube-controller-manager, kube-scheduler and kube-proxy. diff --git a/content/en/docs/setup/production-environment/tools/kubeadm/dual-stack-support.md b/content/en/docs/setup/production-environment/tools/kubeadm/dual-stack-support.md index fabd97d3d1b39..c29272a7b4ce5 100644 --- a/content/en/docs/setup/production-environment/tools/kubeadm/dual-stack-support.md +++ b/content/en/docs/setup/production-environment/tools/kubeadm/dual-stack-support.md @@ -50,25 +50,26 @@ kubeadm init --pod-network-cidr=10.244.0.0/16,2001:db8:42:0::/56 --service-cidr= ``` To make things clearer, here is an example kubeadm -[configuration file](/docs/reference/config-api/kubeadm-config.v1beta3/) +[configuration file](/docs/reference/config-api/kubeadm-config.v1beta4/) `kubeadm-config.yaml` for the primary dual-stack control plane node. ```yaml --- -apiVersion: kubeadm.k8s.io/v1beta3 +apiVersion: kubeadm.k8s.io/v1beta4 kind: ClusterConfiguration networking: podSubnet: 10.244.0.0/16,2001:db8:42:0::/56 serviceSubnet: 10.96.0.0/16,2001:db8:42:1::/112 --- -apiVersion: kubeadm.k8s.io/v1beta3 +apiVersion: kubeadm.k8s.io/v1beta4 kind: InitConfiguration localAPIEndpoint: advertiseAddress: "10.100.0.1" bindPort: 6443 nodeRegistration: kubeletExtraArgs: - node-ip: 10.100.0.2,fd00:1:2:3::2 + - name: "node-ip" + value: "10.100.0.2,fd00:1:2:3::2" ``` `advertiseAddress` in InitConfiguration specifies the IP address that the API Server @@ -92,11 +93,11 @@ The `--apiserver-advertise-address` flag does not support dual-stack. Before joining a node, make sure that the node has IPv6 routable network interface and allows IPv6 forwarding. -Here is an example kubeadm [configuration file](/docs/reference/config-api/kubeadm-config.v1beta3/) +Here is an example kubeadm [configuration file](/docs/reference/config-api/kubeadm-config.v1beta4/) `kubeadm-config.yaml` for joining a worker node to the cluster. ```yaml -apiVersion: kubeadm.k8s.io/v1beta3 +apiVersion: kubeadm.k8s.io/v1beta4 kind: JoinConfiguration discovery: bootstrapToken: @@ -107,14 +108,15 @@ discovery: # change auth info above to match the actual token and CA certificate hash for your cluster nodeRegistration: kubeletExtraArgs: - node-ip: 10.100.0.3,fd00:1:2:3::3 + - name: "node-ip" + value: "10.100.0.2,fd00:1:2:3::3" ``` -Also, here is an example kubeadm [configuration file](/docs/reference/config-api/kubeadm-config.v1beta3/) +Also, here is an example kubeadm [configuration file](/docs/reference/config-api/kubeadm-config.v1beta4/) `kubeadm-config.yaml` for joining another control plane node to the cluster. ```yaml -apiVersion: kubeadm.k8s.io/v1beta3 +apiVersion: kubeadm.k8s.io/v1beta4 kind: JoinConfiguration controlPlane: localAPIEndpoint: @@ -129,8 +131,8 @@ discovery: # change auth info above to match the actual token and CA certificate hash for your cluster nodeRegistration: kubeletExtraArgs: - node-ip: 10.100.0.4,fd00:1:2:3::4 - + - name: "node-ip" + value: "10.100.0.2,fd00:1:2:3::4" ``` `advertiseAddress` in JoinConfiguration.controlPlane specifies the IP address that the @@ -149,11 +151,11 @@ You can deploy a single-stack cluster that has the dual-stack networking feature {{< /note >}} To make things more clear, here is an example kubeadm -[configuration file](/docs/reference/config-api/kubeadm-config.v1beta3/) +[configuration file](/docs/reference/config-api/kubeadm-config.v1beta4/) `kubeadm-config.yaml` for the single-stack control plane node. ```yaml -apiVersion: kubeadm.k8s.io/v1beta3 +apiVersion: kubeadm.k8s.io/v1beta4 kind: ClusterConfiguration networking: podSubnet: 10.244.0.0/16 @@ -164,4 +166,4 @@ networking: * [Validate IPv4/IPv6 dual-stack](/docs/tasks/network/validate-dual-stack) networking * Read about [Dual-stack](/docs/concepts/services-networking/dual-stack/) cluster networking -* Learn more about the kubeadm [configuration format](/docs/reference/config-api/kubeadm-config.v1beta3/) +* Learn more about the kubeadm [configuration format](/docs/reference/config-api/kubeadm-config.v1beta4/) diff --git a/content/en/docs/setup/production-environment/tools/kubeadm/high-availability.md b/content/en/docs/setup/production-environment/tools/kubeadm/high-availability.md index 98ba9069ea324..354322e8a9a5d 100644 --- a/content/en/docs/setup/production-environment/tools/kubeadm/high-availability.md +++ b/content/en/docs/setup/production-environment/tools/kubeadm/high-availability.md @@ -173,7 +173,7 @@ option. Your cluster requirements may need a different configuration. {{< note >}} The `kubeadm init` flags `--config` and `--certificate-key` cannot be mixed, therefore if you want - to use the [kubeadm configuration](/docs/reference/config-api/kubeadm-config.v1beta3/) + to use the [kubeadm configuration](/docs/reference/config-api/kubeadm-config.v1beta4/) you must add the `certificateKey` field in the appropriate config locations (under `InitConfiguration` and `JoinConfiguration: controlPlane`). {{< /note >}} @@ -291,7 +291,7 @@ in the kubeadm config file. ```yaml --- - apiVersion: kubeadm.k8s.io/v1beta3 + apiVersion: kubeadm.k8s.io/v1beta4 kind: ClusterConfiguration kubernetesVersion: stable controlPlaneEndpoint: "LOAD_BALANCER_DNS:LOAD_BALANCER_PORT" # change this (see below) diff --git a/content/en/docs/setup/production-environment/tools/kubeadm/setup-ha-etcd-with-kubeadm.md b/content/en/docs/setup/production-environment/tools/kubeadm/setup-ha-etcd-with-kubeadm.md index 988925739e760..5296c819ba700 100644 --- a/content/en/docs/setup/production-environment/tools/kubeadm/setup-ha-etcd-with-kubeadm.md +++ b/content/en/docs/setup/production-environment/tools/kubeadm/setup-ha-etcd-with-kubeadm.md @@ -126,14 +126,14 @@ on Kubernetes dual-stack support see [Dual-stack support with kubeadm](/docs/set NAME=${NAMES[$i]} cat << EOF > /tmp/${HOST}/kubeadmcfg.yaml --- - apiVersion: "kubeadm.k8s.io/v1beta3" + apiVersion: "kubeadm.k8s.io/v1beta4" kind: InitConfiguration nodeRegistration: name: ${NAME} localAPIEndpoint: advertiseAddress: ${HOST} --- - apiVersion: "kubeadm.k8s.io/v1beta3" + apiVersion: "kubeadm.k8s.io/v1beta4" kind: ClusterConfiguration etcd: local: diff --git a/content/en/docs/setup/production-environment/tools/kubeadm/troubleshooting-kubeadm.md b/content/en/docs/setup/production-environment/tools/kubeadm/troubleshooting-kubeadm.md index 64a4ce2286897..4f1bb912f340d 100644 --- a/content/en/docs/setup/production-environment/tools/kubeadm/troubleshooting-kubeadm.md +++ b/content/en/docs/setup/production-environment/tools/kubeadm/troubleshooting-kubeadm.md @@ -274,7 +274,7 @@ Error from server: Get https://10.19.0.41:10250/containerLogs/default/mysql-ddc6 When using DigitalOcean, it can be the public one (assigned to `eth0`) or the private one (assigned to `eth1`) should you want to use the optional private network. The `kubeletExtraArgs` section of the kubeadm - [`NodeRegistrationOptions` structure](/docs/reference/config-api/kubeadm-config.v1beta3/#kubeadm-k8s-io-v1beta3-NodeRegistrationOptions) + [`NodeRegistrationOptions` structure](/docs/reference/config-api/kubeadm-config.v1beta4/#kubeadm-k8s-io-v1beta4-NodeRegistrationOptions) can be used for this. Then restart `kubelet`: @@ -352,7 +352,7 @@ Alternatively, you can try separating the `key=value` pairs like so: `--apiserver-extra-args "enable-admission-plugins=LimitRanger,enable-admission-plugins=NamespaceExists"` but this will result in the key `enable-admission-plugins` only having the value of `NamespaceExists`. -A known workaround is to use the kubeadm [configuration file](/docs/reference/config-api/kubeadm-config.v1beta3/). +A known workaround is to use the kubeadm [configuration file](/docs/reference/config-api/kubeadm-config.v1beta4/). ## kube-proxy scheduled before node is initialized by cloud-controller-manager @@ -408,33 +408,36 @@ FlexVolume was deprecated in the Kubernetes v1.23 release. {{< /note >}} To workaround this issue, you can configure the flex-volume directory using the kubeadm -[configuration file](/docs/reference/config-api/kubeadm-config.v1beta3/). +[configuration file](/docs/reference/config-api/kubeadm-config.v1beta4/). On the primary control-plane Node (created using `kubeadm init`), pass the following file using `--config`: ```yaml -apiVersion: kubeadm.k8s.io/v1beta3 +apiVersion: kubeadm.k8s.io/v1beta4 kind: InitConfiguration nodeRegistration: kubeletExtraArgs: - volume-plugin-dir: "/opt/libexec/kubernetes/kubelet-plugins/volume/exec/" + - name: "volume-plugin-dir" + value: "/opt/libexec/kubernetes/kubelet-plugins/volume/exec/" --- -apiVersion: kubeadm.k8s.io/v1beta3 +apiVersion: kubeadm.k8s.io/v1beta4 kind: ClusterConfiguration controllerManager: extraArgs: - flex-volume-plugin-dir: "/opt/libexec/kubernetes/kubelet-plugins/volume/exec/" + - name: "flex-volume-plugin-dir" + value: "/opt/libexec/kubernetes/kubelet-plugins/volume/exec/" ``` On joining Nodes: ```yaml -apiVersion: kubeadm.k8s.io/v1beta3 +apiVersion: kubeadm.k8s.io/v1beta4 kind: JoinConfiguration nodeRegistration: kubeletExtraArgs: - volume-plugin-dir: "/opt/libexec/kubernetes/kubelet-plugins/volume/exec/" + - name: "volume-plugin-dir" + value: "/opt/libexec/kubernetes/kubelet-plugins/volume/exec/" ``` Alternatively, you can modify `/etc/fstab` to make the `/usr` mount writeable, but please diff --git a/content/en/docs/tasks/administer-cluster/kubeadm/configure-cgroup-driver.md b/content/en/docs/tasks/administer-cluster/kubeadm/configure-cgroup-driver.md index b811985819c00..c570fed973e36 100644 --- a/content/en/docs/tasks/administer-cluster/kubeadm/configure-cgroup-driver.md +++ b/content/en/docs/tasks/administer-cluster/kubeadm/configure-cgroup-driver.md @@ -48,7 +48,7 @@ A minimal example of configuring the field explicitly: ```yaml # kubeadm-config.yaml kind: ClusterConfiguration -apiVersion: kubeadm.k8s.io/v1beta3 +apiVersion: kubeadm.k8s.io/v1beta4 kubernetesVersion: v1.21.0 --- kind: KubeletConfiguration diff --git a/content/en/docs/tasks/administer-cluster/kubeadm/kubeadm-certs.md b/content/en/docs/tasks/administer-cluster/kubeadm/kubeadm-certs.md index 623e528aa80d6..be7a1e0df88de 100644 --- a/content/en/docs/tasks/administer-cluster/kubeadm/kubeadm-certs.md +++ b/content/en/docs/tasks/administer-cluster/kubeadm/kubeadm-certs.md @@ -238,15 +238,17 @@ To activate the built-in signer, you must pass the `--cluster-signing-cert-file` `--cluster-signing-key-file` flags. If you're creating a new cluster, you can use a kubeadm -[configuration file](/docs/reference/config-api/kubeadm-config.v1beta3/): +[configuration file](/docs/reference/config-api/kubeadm-config.v1beta4/): ```yaml -apiVersion: kubeadm.k8s.io/v1beta3 +apiVersion: kubeadm.k8s.io/v1beta4 kind: ClusterConfiguration controllerManager: extraArgs: - cluster-signing-cert-file: /etc/kubernetes/pki/ca.crt - cluster-signing-key-file: /etc/kubernetes/pki/ca.key + - name: "cluster-signing-cert-file" + value: "/etc/kubernetes/pki/ca.crt" + - name: "cluster-signing-key-file" + value: "/etc/kubernetes/pki/ca.key" ``` ### Create certificate signing requests (CSR) @@ -286,7 +288,7 @@ To configure the kubelets in a new kubeadm cluster to obtain properly signed ser certificates you must pass the following minimal configuration to `kubeadm init`: ```yaml -apiVersion: kubeadm.k8s.io/v1beta3 +apiVersion: kubeadm.k8s.io/v1beta4 kind: ClusterConfiguration --- apiVersion: kubelet.config.k8s.io/v1beta1 @@ -357,7 +359,7 @@ Sharing the `admin.conf` with additional users is **not recommended**! Instead, you can use the [`kubeadm kubeconfig user`](/docs/reference/setup-tools/kubeadm/kubeadm-kubeconfig) command to generate kubeconfig files for additional users. The command accepts a mixture of command line flags and -[kubeadm configuration](/docs/reference/config-api/kubeadm-config.v1beta3/) options. +[kubeadm configuration](/docs/reference/config-api/kubeadm-config.v1beta4/) options. The generated kubeconfig will be written to stdout and can be piped to a file using `kubeadm kubeconfig user ... > somefile.conf`. @@ -365,7 +367,7 @@ Example configuration file that can be used with `--config`: ```yaml # example.yaml -apiVersion: kubeadm.k8s.io/v1beta3 +apiVersion: kubeadm.k8s.io/v1beta4 kind: ClusterConfiguration # Will be used as the target "cluster" in the kubeconfig clusterName: "kubernetes" @@ -412,7 +414,7 @@ directory for kubeconfig files is `/etc/kubernetes`. These defaults can be overridden with the flags `--cert-dir` and `--kubeconfig-dir`, respectively. To pass custom options to `kubeadm certs generate-csr` use the `--config` flag, -which accepts a [kubeadm configuration](/docs/reference/config-api/kubeadm-config.v1beta3/) +which accepts a [kubeadm configuration](/docs/reference/config-api/kubeadm-config.v1beta4/) file, similarly to commands such as `kubeadm init`. Any specification such as extra SANs and custom IP addresses must be stored in the same configuration file and used for all relevant kubeadm commands by passing it as `--config`. diff --git a/content/en/docs/tasks/administer-cluster/kubeadm/kubeadm-reconfigure.md b/content/en/docs/tasks/administer-cluster/kubeadm/kubeadm-reconfigure.md index d2d6a83d3565a..568838e3ca7fa 100644 --- a/content/en/docs/tasks/administer-cluster/kubeadm/kubeadm-reconfigure.md +++ b/content/en/docs/tasks/administer-cluster/kubeadm/kubeadm-reconfigure.md @@ -60,7 +60,7 @@ component configuration and avoid introducing typos and YAML indentation mistake #### Updating the `ClusterConfiguration` During cluster creation and upgrade, kubeadm writes its -[`ClusterConfiguration`](/docs/reference/config-api/kubeadm-config.v1beta3/) +[`ClusterConfiguration`](/docs/reference/config-api/kubeadm-config.v1beta4/) in a ConfigMap called `kubeadm-config` in the `kube-system` namespace. To change a particular option in the `ClusterConfiguration` you can edit the ConfigMap with this command: diff --git a/content/en/docs/tasks/administer-cluster/kubeadm/kubeadm-upgrade.md b/content/en/docs/tasks/administer-cluster/kubeadm/kubeadm-upgrade.md index b96169a808dae..911ab5807346b 100644 --- a/content/en/docs/tasks/administer-cluster/kubeadm/kubeadm-upgrade.md +++ b/content/en/docs/tasks/administer-cluster/kubeadm/kubeadm-upgrade.md @@ -162,7 +162,7 @@ Pick a control plane node that you wish to upgrade first. It must have the `/etc For more information see the [certificate management guide](/docs/tasks/administer-cluster/kubeadm/kubeadm-certs). {{}} - 1. Choose a version to upgrade to, and run the appropriate command. For example: +1. Choose a version to upgrade to, and run the appropriate command. For example: ```shell # replace x with the patch version you picked for this upgrade From c5205da71763e157065179dd21bc5946bd139897 Mon Sep 17 00:00:00 2001 From: Peter Schuurman Date: Mon, 8 Jul 2024 18:23:20 -0700 Subject: [PATCH 32/82] Remove StatefulSetStartOrdinal feature gate to target stable in 1.31 --- .../en/docs/concepts/workloads/controllers/statefulset.md | 8 +++----- .../feature-gates/stateful-set-start-ordinal.md | 4 ++++ 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/content/en/docs/concepts/workloads/controllers/statefulset.md b/content/en/docs/concepts/workloads/controllers/statefulset.md index 516cb4f60c736..177dd3b371abd 100644 --- a/content/en/docs/concepts/workloads/controllers/statefulset.md +++ b/content/en/docs/concepts/workloads/controllers/statefulset.md @@ -178,13 +178,11 @@ will also add a pod label with this index: `apps.kubernetes.io/pod-index`. ### Start ordinal -{{< feature-state for_k8s_version="v1.27" state="beta" >}} +{{< feature-state feature_gate_name="StatefulSetStartOrdinal" >}} `.spec.ordinals` is an optional field that allows you to configure the integer -ordinals assigned to each Pod. It defaults to nil. You must enable the -`StatefulSetStartOrdinal` -[feature gate](/docs/reference/command-line-tools-reference/feature-gates/) to -use this field. Once enabled, you can configure the following options: +ordinals assigned to each Pod. It defaults to nil. Within the field, you can +configure the following options: * `.spec.ordinals.start`: If the `.spec.ordinals.start` field is set, Pods will be assigned ordinals from `.spec.ordinals.start` up through diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/stateful-set-start-ordinal.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/stateful-set-start-ordinal.md index 4bd6b41061841..1309dc1b86399 100644 --- a/content/en/docs/reference/command-line-tools-reference/feature-gates/stateful-set-start-ordinal.md +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/stateful-set-start-ordinal.md @@ -13,6 +13,10 @@ stages: - stage: beta defaultValue: true fromVersion: "1.27" + toVersion: "1.30" + - stage: stable + defaultValue: true + fromVersion: "1.31" --- Allow configuration of the start ordinal in a StatefulSet. See From c5f020c0e43c7e704350d77137e3666e0019ee81 Mon Sep 17 00:00:00 2001 From: Shingo Omura Date: Wed, 10 Jul 2024 22:15:54 +0900 Subject: [PATCH 33/82] KEP-3619: Fine-grained SupplementalGroups control --- .../supplemental-groups-policy.md | 14 ++ .../security-context.md | 175 +++++++++++++++++- .../pods/security/security-context-5.yaml | 15 ++ .../pods/security/security-context-6.yaml | 16 ++ .../pods/security/security-context.yaml | 1 + 5 files changed, 219 insertions(+), 2 deletions(-) create mode 100644 content/en/docs/reference/command-line-tools-reference/feature-gates/supplemental-groups-policy.md create mode 100644 content/en/examples/pods/security/security-context-5.yaml create mode 100644 content/en/examples/pods/security/security-context-6.yaml diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/supplemental-groups-policy.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/supplemental-groups-policy.md new file mode 100644 index 0000000000000..d48f78c898e10 --- /dev/null +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/supplemental-groups-policy.md @@ -0,0 +1,14 @@ +--- +title: SupplementalGroupsPolicy +content_type: feature_gate +_build: + list: never + render: false + +stages: + - stage: alpha + defaultValue: false + fromVersion: "1.31" +--- +Enables support for fine-grained SupplementalGroups control. +For more details, see [Configure fine-grained SupplementalGroups control for a Pod](/content/en/docs/tasks/configure-pod-container/security-context/#supplementalgroupspolicy). diff --git a/content/en/docs/tasks/configure-pod-container/security-context.md b/content/en/docs/tasks/configure-pod-container/security-context.md index b176d20df51c5..21c6f1abd8a8f 100644 --- a/content/en/docs/tasks/configure-pod-container/security-context.md +++ b/content/en/docs/tasks/configure-pod-container/security-context.md @@ -66,6 +66,8 @@ all processes within any containers of the Pod. If this field is omitted, the pr will be root(0). Any files created will also be owned by user 1000 and group 3000 when `runAsGroup` is specified. Since `fsGroup` field is specified, all processes of the container are also part of the supplementary group ID 2000. The owner for volume `/data/demo` and any files created in that volume will be Group ID 2000. +Additionally, since `supplementalGroups` field is specified, all processes of the container are also part of the +specified group IDs. If this field is omitted, it means empty. Create the Pod: @@ -142,13 +144,15 @@ id The output is similar to this: ```none -uid=1000 gid=3000 groups=2000 +uid=1000 gid=3000 groups=2000,3000,4000 ``` From the output, you can see that `gid` is 3000 which is same as the `runAsGroup` field. If the `runAsGroup` was omitted, the `gid` would remain as 0 (root) and the process will be able to interact with files that are owned by the root(0) group and groups that have -the required group permissions for the root (0) group. +the required group permissions for the root (0) group. You can also see `groups` +contains the group IDs which are specified by `fsGroup` and `supplementalGroups` other +than `gid`. Exit your shell: @@ -156,6 +160,173 @@ Exit your shell: exit ``` +### Implicit group memberships defined in `/etc/group` in the container image + +By default, kubernetes merges group information for the container's primary user defined in +`/etc/group` in the container image. + +{{% code_sample file="pods/security/security-context-5.yaml" %}} + +In this configuration, it just specifies `runAsUser`, `runAsGroup` and `supplementalGroups`. +However, you can see that the actual supplementary groups attached to the container process +will include group IDs which come from `/etc/group` in the container image. + +Create the Pod: + +```shell +kubectl apply -f https://k8s.io/examples/pods/security/security-context-5.yaml +``` + +Verify that the Pod's Container is running: + +```shell +kubectl get pod security-context-demo +``` + +Get a shell to the running Container: + +```shell +kubectl exec -it security-context-demo -- sh +``` + +Check the process identity: + +```shell +$ id +``` + +The output is similar to this: + +```none +uid=1000(user-defined-in-image) gid=3000 groups=3000,4000,50000(group-defined-in-image) +``` + +You can see `groups` includes group ID `50000`. This is because the user (`uid=1000(user-defined-in-image)`) +belongs to the group `group-defined-in-image(gid=50000)` which is defined in `/etc/group` in the container image. + +Check the `/etc/group` in the container image: + +```shell +$ cat /etc/group +``` + +You can see the group entry that `user-defined-in-image(uid=1000)` belongs to `group-defined-in-image(gid=50000)`. + +```none +... +group-defined-in-image:x:50000:user-defined-in-image +``` + +Exit your shell: + +```shell +exit +``` + +{{}} +Implicitly _merged_ supplementary groups may cause security concerns particularly in accessing +the volumes (see [kubernetes/kubernetes#112879](https://issue.k8s.io/112879) for details). +If you want to avoid this. Please see the below section. +{{}} + +## Configure fine-grained SupplementalGroups control for a Pod {#supplementalgroupspolicy} + +{{< feature-state feature_gate_name="SupplementalGroupsPolicy" >}} + +This feature can be enabled by setting the `SupplementalGroupsPolicy` +[feature gate](/docs/reference/command-line-tools-reference/feature-gates/) for kubelet and +kube-apiserver, and setting the `.spec.securityContext.supplementalGroupsPolicy` field for a pod. + +**supplementalGroupsPolicy** - `supplementalGroupsPolicy` defines behavior for calculating +supplementary groups for the container processes in a pod. + +* _Merge_: The group membership defined in `/etc/group` for the container's primary user will be merged. If not specified, this policy will be applied. + +* _Strict_: it only attaches group IDs in `fsGroup`, `supplementalGroups`, or `runAsGroup` fields as the supplementary groups of the container processess. This means no group membership defined in `/etc/group` for the container's primary user will be merged. + +When the feature is enabled, it also exposes the process identity attached to the first container process of the container +in `.status.containerStatuses[].user.linux` field. It would be helpful to detect if implicit group ID's are attached. + +{{% code_sample file="pods/security/security-context-6.yaml" %}} + +This pod manifest defines `supplementalGroupsPolicy=Strict`. You can see no group membership defined in `/etc/group` will be merged to the supplementary groups for container processes. + +Create the Pod: + +```shell +kubectl apply -f https://k8s.io/examples/pods/security/security-context-6.yaml +``` + +Verify that the Pod's Container is running: + +```shell +kubectl get pod security-context-demo +``` + +Check the process identity: + +```shell +kubectl exec -it security-context-demo -- id +``` + +The output is similar to this: + +```none +uid=1000(user-defined-in-image) gid=3000 groups=3000,4000 +``` + +See the Pod's status: + +```shell +kubectl get pod security-context-demo -o yaml +``` + +You can see `status.containerStatuses[].user.linux` field exposes the process identitiy +attached to the first container process. + +```none +... +status: + containerStatuses: + - name: sec-ctx-demo + user: + linux: + gid: 3000 + supplementalGroups: + - 3000 + - 4000 + uid: 1000 +... +``` + +{{}} +Please note that the values in `status.containerStatuses[].user.linux` field is _the firstly attached_ +process identity to the first container process in the container. If the container has sufficient privilege +to call system calls related to process identity (e.g. [`setuid(2)`](https://man7.org/linux/man-pages/man2/setuid.2.html), [`setgid(2)`](https://man7.org/linux/man-pages/man2/setgid.2.html) or [`setgroups(2)`](https://man7.org/linux/man-pages/man2/setgroups.2.html), etc.), +the container process can change its identity. Thus, the _actual_ process identity will be dynamic. +{{}} + +### Implementations {#implementations-supplementalgroupspolicy} + +{{% thirdparty-content %}} + +The following container runtimes are known to support fine-grained SupplementalGroups control. + +CRI-level: +- [containerd](https://containerd.io/), since v2.0 +- [CRI-O](https://cri-o.io/), since v1.31 + +You can see if the feature is supported in the node status. + +```yaml +apiVersion: v1 +kind: Node +... +status: + features: + supplementalGroupsPolicy: true +``` + ## Configure volume permission and ownership change policy for Pods {{< feature-state for_k8s_version="v1.23" state="stable" >}} diff --git a/content/en/examples/pods/security/security-context-5.yaml b/content/en/examples/pods/security/security-context-5.yaml new file mode 100644 index 0000000000000..26e057150df61 --- /dev/null +++ b/content/en/examples/pods/security/security-context-5.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: security-context-demo +spec: + securityContext: + runAsUser: 1000 + runAsGroup: 3000 + supplementalGroups: [4000] + containers: + - name: sec-ctx-demo + image: registry.k8s.io/e2e-test-images/agnhost:2.45 + command: [ "sh", "-c", "sleep 1h" ] + securityContext: + allowPrivilegeEscalation: false diff --git a/content/en/examples/pods/security/security-context-6.yaml b/content/en/examples/pods/security/security-context-6.yaml new file mode 100644 index 0000000000000..6bbd1288ec45a --- /dev/null +++ b/content/en/examples/pods/security/security-context-6.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Pod +metadata: + name: security-context-demo +spec: + securityContext: + runAsUser: 1000 + runAsGroup: 3000 + supplementalGroups: [4000] + supplementalGroupsPolicy: Strict + containers: + - name: sec-ctx-demo + image: registry.k8s.io/e2e-test-images/agnhost:2.45 + command: [ "sh", "-c", "sleep 1h" ] + securityContext: + allowPrivilegeEscalation: false diff --git a/content/en/examples/pods/security/security-context.yaml b/content/en/examples/pods/security/security-context.yaml index 7903c39c6467c..aeeaebb92d57b 100644 --- a/content/en/examples/pods/security/security-context.yaml +++ b/content/en/examples/pods/security/security-context.yaml @@ -7,6 +7,7 @@ spec: runAsUser: 1000 runAsGroup: 3000 fsGroup: 2000 + supplementalGroups: [4000] volumes: - name: sec-ctx-vol emptyDir: {} From 9f6bbb67a6dc5d962f7fded73cf7527f0f60d5c4 Mon Sep 17 00:00:00 2001 From: Gaurav Ghildiyal Date: Tue, 25 Jun 2024 10:54:24 -0700 Subject: [PATCH 34/82] Docs update for k8s 1.31 regarding Service trafficDistribution --- .../feature-gates/service-traffic-distribution.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/service-traffic-distribution.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/service-traffic-distribution.md index 4c1e6d6c17933..cafc44adc5b4f 100644 --- a/content/en/docs/reference/command-line-tools-reference/feature-gates/service-traffic-distribution.md +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/service-traffic-distribution.md @@ -10,6 +10,10 @@ stages: - stage: alpha defaultValue: false fromVersion: "1.30" + toVersion: "1.30" +- stage: beta + defaultValue: true + fromVersion: "1.31" --- Allows usage of the optional `spec.trafficDistribution` field in Services. The field offers a way to express preferences for how traffic is distributed to From d4fd8912887ce81bf34c593d4b0f0859dba9daa7 Mon Sep 17 00:00:00 2001 From: Jiaxin Shan Date: Fri, 28 Jun 2024 01:04:25 -0700 Subject: [PATCH 35/82] Add docs for KEP-4176:Add a new static policy DistributeCPUsAcrossCores --- .../cpu-management-policies.md | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/content/en/docs/tasks/administer-cluster/cpu-management-policies.md b/content/en/docs/tasks/administer-cluster/cpu-management-policies.md index 5b1ddfe5d248f..1c3750c869ab5 100644 --- a/content/en/docs/tasks/administer-cluster/cpu-management-policies.md +++ b/content/en/docs/tasks/administer-cluster/cpu-management-policies.md @@ -267,6 +267,7 @@ The following policy options exist for the static `CPUManager` policy: * `full-pcpus-only` (beta, visible by default) (1.22 or higher) * `distribute-cpus-across-numa` (alpha, hidden by default) (1.23 or higher) * `align-by-socket` (alpha, hidden by default) (1.25 or higher) +* `distribute-cpus-across-cores` (alpha, hidden by default) (1.31 or higher) If the `full-pcpus-only` policy option is specified, the static policy will always allocate full physical cores. By default, without this option, the static policy allocates CPUs using a topology-aware best-fit allocation. @@ -303,6 +304,21 @@ policy option is not compatible with `TopologyManager` `single-numa-node` policy and does not apply to hardware where the number of sockets is greater than number of NUMA nodes. + +If the `distribute-cpus-across-cores` policy option is specified, the static policy +will attempt to allocate virtual cores (hardware threads) across different physical cores. +By default, the `CPUManager` tends to pack cpus onto as few physical cores as possible, +which can lead to contention among cpus on the same physical core and result +in performance bottlenecks. By enabling the `distribute-cpus-across-cores` policy, +the static policy ensures that cpus are distributed across as many physical cores +as possible, reducing the contention on the same physical core and thereby +improving overall performance. However, it is important to note that this strategy +might be less effective when the system is heavily loaded. Under such conditions, +the benefit of reducing contention diminishes. Conversely, default behavior +can help in reducing inter-core communication overhead, potentially providing +better performance under high load conditions. + + The `full-pcpus-only` option can be enabled by adding `full-pcpus-only=true` to the CPUManager policy options. Likewise, the `distribute-cpus-across-numa` option can be enabled by adding @@ -313,3 +329,8 @@ cores. The `align-by-socket` policy option can be enabled by adding `align-by-socket=true` to the `CPUManager` policy options. It is also additive to the `full-pcpus-only` and `distribute-cpus-across-numa` policy options. + +The `distribute-cpus-across-cores` option can be enabled by adding +`distribute-cpus-across-cores=true` to the `CPUManager` policy options. +It cannot be used with `full-pcpus-only` or `distribute-cpus-across-numa` policy +options together at this moment. From 5b40c51031ec381145ec9d118e491aaf82dc5bd7 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Fri, 14 Jun 2024 10:03:30 +0200 Subject: [PATCH 36/82] DRA: update for 1.31 The concepts are mostly the same as before. Support for parameters in a CRD was removed in favor of storing selection and configuration parameters directly in ResourceClaim and DeviceClass. --- .../dynamic-resource-allocation.md | 155 +++++++++--------- .../dra-control-plane-controller.md | 15 ++ .../dynamic-resource-allocation.md | 5 +- 3 files changed, 96 insertions(+), 79 deletions(-) create mode 100644 content/en/docs/reference/command-line-tools-reference/feature-gates/dra-control-plane-controller.md diff --git a/content/en/docs/concepts/scheduling-eviction/dynamic-resource-allocation.md b/content/en/docs/concepts/scheduling-eviction/dynamic-resource-allocation.md index b32dbc334f371..7c590653d9790 100644 --- a/content/en/docs/concepts/scheduling-eviction/dynamic-resource-allocation.md +++ b/content/en/docs/concepts/scheduling-eviction/dynamic-resource-allocation.md @@ -9,18 +9,28 @@ weight: 65 +Core Dynamic Resource Allocation with structured parameters: + {{< feature-state feature_gate_name="DynamicResourceAllocation" >}} +Dynamic Resource Allocation with control plane controller: + +{{< feature-state feature_gate_name="DRAControlPlaneController" >}} + Dynamic resource allocation is an API for requesting and sharing resources between pods and containers inside a pod. It is a generalization of the -persistent volumes API for generic resources. Third-party resource drivers are -responsible for tracking and allocating resources, with additional support -provided by Kubernetes via _structured parameters_ (introduced in Kubernetes 1.30). -When a driver uses structured parameters, Kubernetes handles scheduling -and resource allocation without having to communicate with the driver. +persistent volumes API for generic resources. Typically those resources +are devices like GPUs. + +Third-party resource drivers are +responsible for tracking and preparing resources, with allocation of +resources handled by Kubernetes via _structured parameters_ (introduced in Kubernetes 1.30). Different kinds of resources support arbitrary parameters for defining requirements and initialization. +When a driver provides a _control plane controller_, the driver itself +handles allocation in cooperation with the Kubernetes scheduler. + ## {{% heading "prerequisites" %}} Kubernetes v{{< skew currentVersion >}} includes cluster-level API support for @@ -34,63 +44,47 @@ check the documentation for that version of Kubernetes. ## API -The `resource.k8s.io/v1alpha2` {{< glossary_tooltip text="API group" +The `resource.k8s.io/v1alpha3` {{< glossary_tooltip text="API group" term_id="api-group" >}} provides these types: -ResourceClass -: Defines which resource driver handles a certain kind of - resource and provides common parameters for it. ResourceClasses - are created by a cluster administrator when installing a resource - driver. - ResourceClaim -: Defines a particular resource instance that is required by a - workload. Created by a user (lifecycle managed manually, can be shared - between different Pods) or for individual Pods by the control plane based on - a ResourceClaimTemplate (automatic lifecycle, typically used by just one - Pod). +: Describes a request for access to resources in the cluster, + for use by workloads. For example, if a workload needs an accelerator device + with specific properties, this is how that request is expressed. The status + stanza tracks whether this claim has been satisfied and what specific + resources have been allocated. ResourceClaimTemplate : Defines the spec and some metadata for creating ResourceClaims. Created by a user when deploying a workload. + The per-Pod ResourceClaims are then created and removed by Kubernetes + automatically. + +DeviceClass +: Contains pre-defined selection criteria for certain devices and + configuration for them. DeviceClasses are created by a cluster administrator + when installing a resource driver. Each request to allocate a device + in a ResourceClaim must reference exactly one DeviceClass. PodSchedulingContext : Used internally by the control plane and resource drivers to coordinate pod scheduling when ResourceClaims need to be allocated - for a Pod. + for a Pod and those ResourceClaims use a control plane controller. ResourceSlice : Used with structured parameters to publish information about resources that are available in the cluster. -ResourceClaimParameters -: Contain the parameters for a ResourceClaim which influence scheduling, - in a format that is understood by Kubernetes (the "structured parameter - model"). Additional parameters may be embedded in an opaque - extension, for use by the vendor driver when setting up the underlying - resource. - -ResourceClassParameters -: Similar to ResourceClaimParameters, the ResourceClassParameters provides - a type for ResourceClass parameters which is understood by Kubernetes. - -Parameters for ResourceClass and ResourceClaim are stored in separate objects, -typically using the type defined by a {{< glossary_tooltip -term_id="CustomResourceDefinition" text="CRD" >}} that was created when -installing a resource driver. - -The developer of a resource driver decides whether they want to handle these -parameters in their own external controller or instead rely on Kubernetes to -handle them through the use of structured parameters. A +The developer of a resource driver decides whether they want to handle +allocation themselves with a control plane controller or instead rely on allocation +through Kubernetes with structured parameters. A custom controller provides more flexibility, but cluster autoscaling is not going to work reliably for node-local resources. Structured parameters enable cluster autoscaling, but might not satisfy all use-cases. -When a driver uses structured parameters, it is still possible to let the -end-user specify parameters with vendor-specific CRDs. When doing so, the -driver needs to translate those -custom parameters into the in-tree types. Alternatively, a driver may also -document how to use the in-tree types directly. +When a driver uses structured parameters, all parameters that select devices +are defined in the ResourceClaim and DeviceClass with in-tree types. Configuration +parameters can be embedded there as arbitrary JSON objects. The `core/v1` `PodSpec` defines ResourceClaims that are needed for a Pod in a `resourceClaims` field. Entries in that list reference either a ResourceClaim @@ -107,17 +101,13 @@ Here is an example for a fictional resource driver. Two ResourceClaim objects will get created for this Pod and each container gets access to one of them. ```yaml -apiVersion: resource.k8s.io/v1alpha2 -kind: ResourceClass +apiVersion: resource.k8s.io/v1alpha3 +kind: DeviceClass name: resource.example.com -driverName: resource-driver.example.com ---- -apiVersion: cats.resource.example.com/v1 -kind: ClaimParameters -name: large-black-cat-claim-parameters spec: - color: black - size: large + selectors: + - cel: + expression: device.driver == "resource-driver.example.com" --- apiVersion: resource.k8s.io/v1alpha2 kind: ResourceClaimTemplate @@ -125,11 +115,15 @@ metadata: name: large-black-cat-claim-template spec: spec: - resourceClassName: resource.example.com - parametersRef: - apiGroup: cats.resource.example.com - kind: ClaimParameters - name: large-black-cat-claim-parameters + devices: + requests: + - name: req-0 + deviceClassName: resource.example.com + selectors: + - cel: + expression: |- + device.attributes["resource-driver.example.com"].color == "black" && + device.attributes["resource-driver.example.com"].size == "large" –-- apiVersion: v1 kind: Pod @@ -151,16 +145,14 @@ spec: - name: cat-1 resourceClaims: - name: cat-0 - source: - resourceClaimTemplateName: large-black-cat-claim-template + resourceClaimTemplateName: large-black-cat-claim-template - name: cat-1 - source: - resourceClaimTemplateName: large-black-cat-claim-template + resourceClaimTemplateName: large-black-cat-claim-template ``` ## Scheduling -### Without structured parameters +### With control plane controller In contrast to native resources (CPU, RAM) and extended resources (managed by a device plugin, advertised by kubelet), without structured parameters @@ -171,12 +163,7 @@ responsible for that. They mark ResourceClaims as "allocated" once resources for it are reserved. This also then tells the scheduler where in the cluster a ResourceClaim is available. -ResourceClaims can get allocated as soon as they are created ("immediate -allocation"), without considering which Pods will use them. The default is to -delay allocation until a Pod gets scheduled which needs the ResourceClaim -(i.e. "wait for first consumer"). - -In that mode, the scheduler checks all ResourceClaims needed by a Pod and +When a pod gets scheduled, the scheduler checks all ResourceClaims needed by a Pod and creates a PodScheduling object where it informs the resource drivers responsible for those ResourceClaims about nodes that the scheduler considers suitable for the Pod. The resource drivers respond by excluding nodes that @@ -213,12 +200,16 @@ responsibility of allocating resources to a ResourceClaim whenever a pod needs them. It does so by retrieving the full list of available resources from ResourceSlice objects, tracking which of those resources have already been allocated to existing ResourceClaims, and then selecting from those resources -that remain. The exact resources selected are subject to the constraints -provided in any ResourceClaimParameters or ResourceClassParameters associated -with the ResourceClaim. +that remain. + +The only kind of supported resources at the moment are devices. A device +instance has a name and several attributes and capacities. Devices get selected +through CEL expressions which check those attributes and capacities. In +addition, the set of selected devices also can be restricted to sets which meet +certain constraints. The chosen resource is recorded in the ResourceClaim status together with any -vendor-specific parameters, so when a pod is about to start on a node, the +vendor-specific configuration, so when a pod is about to start on a node, the resource driver on the node has all the information it needs to prepare the resource. @@ -279,21 +270,25 @@ the `.spec.nodeName` field and to use a node selector instead. Dynamic resource allocation is an *alpha feature* and only enabled when the `DynamicResourceAllocation` [feature gate](/docs/reference/command-line-tools-reference/feature-gates/) and the -`resource.k8s.io/v1alpha2` {{< glossary_tooltip text="API group" +`resource.k8s.io/v1alpha3` {{< glossary_tooltip text="API group" term_id="api-group" >}} are enabled. For details on that, see the `--feature-gates` and `--runtime-config` [kube-apiserver parameters](/docs/reference/command-line-tools-reference/kube-apiserver/). kube-scheduler, kube-controller-manager and kubelet also need the feature gate. +When a resource driver uses a control plane controller, then the +`DRAControlPlaneController` feature gate has to be enabled in addition to +`DynamicResourceAllocation`. + A quick check whether a Kubernetes cluster supports the feature is to list ResourceClass objects with: ```shell -kubectl get resourceclasses +kubectl get deviceclasses ``` If your cluster supports dynamic resource allocation, the response is either a -list of ResourceClass objects or: +list of DeviceClass objects or: ``` No resources found @@ -302,9 +297,14 @@ No resources found If not supported, this error is printed instead: ``` -error: the server doesn't have a resource type "resourceclasses" +error: the server doesn't have a resource type "deviceclasses" ``` +A control plane controller is supported when it is possible to create a +ResourceClaim where the `spec.controller` field is set. When the +`DRAControlPlaneController` feature is disabled, that field automatically +gets cleared when storing the ResourceClaim. + The default configuration of kube-scheduler enables the "DynamicResources" plugin if and only if the feature gate is enabled and when using the v1 configuration API. Custom configurations may have to be modified to @@ -316,5 +316,6 @@ be installed. Please refer to the driver's documentation for details. ## {{% heading "whatsnext" %}} - For more information on the design, see the -[Dynamic Resource Allocation KEP](https://github.com/kubernetes/enhancements/blob/master/keps/sig-node/3063-dynamic-resource-allocation/README.md) - and the [Structured Parameters KEP](https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/4381-dra-structured-parameters). + [Structured Parameters with Structured Parameters](https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/4381-dra-structured-parameters) + and the + [Dynamic Resource Allocation with Control Plane Controller](https://github.com/kubernetes/enhancements/blob/master/keps/sig-node/3063-dynamic-resource-allocation/README.md) KEPs. diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/dra-control-plane-controller.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/dra-control-plane-controller.md new file mode 100644 index 0000000000000..b5af438f14578 --- /dev/null +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/dra-control-plane-controller.md @@ -0,0 +1,15 @@ +--- +title: DRAControlPlaneController +content_type: feature_gate +_build: + list: never + render: false + +stages: + - stage: alpha + defaultValue: false + fromVersion: "1.26" +--- +Enables support for resources with custom parameters and a lifecycle +that is independent of a Pod. Allocation of resources is handled +by a resource driver's control plane controller. diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/dynamic-resource-allocation.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/dynamic-resource-allocation.md index d0e3d0ea0630b..60a00583db0d0 100644 --- a/content/en/docs/reference/command-line-tools-reference/feature-gates/dynamic-resource-allocation.md +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/dynamic-resource-allocation.md @@ -8,7 +8,8 @@ _build: stages: - stage: alpha defaultValue: false - fromVersion: "1.26" + fromVersion: "1.30" --- Enables support for resources with custom parameters and a lifecycle -that is independent of a Pod. +that is independent of a Pod. Allocation of resources is handled +by the Kubernetes scheduler based on "structured parameters". From 9c209a8ac6be1d593fbd2ecb07e47f94ac1cdc05 Mon Sep 17 00:00:00 2001 From: Antonio Ojea Date: Mon, 24 Jun 2024 09:54:33 +0000 Subject: [PATCH 37/82] KEP1880: graduate to beta Signed-off-by: Antonio Ojea --- .../feature-gates/multi-cidr-service-allocator.md | 4 ++++ content/en/docs/reference/networking/virtual-ips.md | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/multi-cidr-service-allocator.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/multi-cidr-service-allocator.md index 278ef255911e3..bc73f7a0ca394 100644 --- a/content/en/docs/reference/command-line-tools-reference/feature-gates/multi-cidr-service-allocator.md +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/multi-cidr-service-allocator.md @@ -9,5 +9,9 @@ stages: - stage: alpha defaultValue: false fromVersion: "1.27" + toVersion: "1.30" + - stage: beta + defaultValue: false + fromVersion: "1.31" --- Track IP address allocations for Service cluster IPs using IPAddress objects. diff --git a/content/en/docs/reference/networking/virtual-ips.md b/content/en/docs/reference/networking/virtual-ips.md index 2592bdb3ab348..38ef2ff7bd38a 100644 --- a/content/en/docs/reference/networking/virtual-ips.md +++ b/content/en/docs/reference/networking/virtual-ips.md @@ -424,7 +424,7 @@ IP addresses that are no longer used by any Services. #### IP address allocation tracking using the Kubernetes API {#ip-address-objects} -{{< feature-state for_k8s_version="v1.27" state="alpha" >}} +{{< feature-state feature_gate_name="MultiCIDRServiceAllocator" >}} If you enable the `MultiCIDRServiceAllocator` [feature gate](/docs/reference/command-line-tools-reference/feature-gates/) and the @@ -483,7 +483,7 @@ Users can create or delete new ServiceCIDR objects to manage the available IP ra ```shell cat <<'EOF' | kubectl apply -f - -apiVersion: networking.k8s.io/v1alpha1 +apiVersion: networking.k8s.io/v1beta1 kind: ServiceCIDR metadata: name: newservicecidr From f364b4c247567cc30bb2a2ee1ac47124e86a110a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filip=20K=C5=99epinsk=C3=BD?= Date: Wed, 21 Feb 2024 23:53:04 +0100 Subject: [PATCH 38/82] promote PDBUnhealthyPodEvictionPolicy to GA --- .../feature-gates/pdb-unhealthy-pod-eviction-policy.md | 4 ++++ content/en/docs/tasks/run-application/configure-pdb.md | 6 ------ 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/pdb-unhealthy-pod-eviction-policy.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/pdb-unhealthy-pod-eviction-policy.md index d48bace0b1af4..8d82c5bff40c7 100644 --- a/content/en/docs/reference/command-line-tools-reference/feature-gates/pdb-unhealthy-pod-eviction-policy.md +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/pdb-unhealthy-pod-eviction-policy.md @@ -13,6 +13,10 @@ stages: - stage: beta defaultValue: true fromVersion: "1.27" + toVersion: "1.30" + - stage: stable + defaultValue: true + fromVersion: "1.31" --- Enables the `unhealthyPodEvictionPolicy` field of a `PodDisruptionBudget`. This specifies when unhealthy pods should be considered for eviction. Please see [Unhealthy Pod Eviction Policy](/docs/tasks/run-application/configure-pdb/#unhealthy-pod-eviction-policy) diff --git a/content/en/docs/tasks/run-application/configure-pdb.md b/content/en/docs/tasks/run-application/configure-pdb.md index c57e6b867d333..0016cbfe06d00 100644 --- a/content/en/docs/tasks/run-application/configure-pdb.md +++ b/content/en/docs/tasks/run-application/configure-pdb.md @@ -243,12 +243,6 @@ These pods are tracked via `.status.currentHealthy` field in the PDB status. {{< feature-state feature_gate_name="PDBUnhealthyPodEvictionPolicy" >}} -{{< note >}} -This feature is enabled by default. You can disable it by disabling the `PDBUnhealthyPodEvictionPolicy` -[feature gate](/docs/reference/command-line-tools-reference/feature-gates/) -on the [API server](/docs/reference/command-line-tools-reference/kube-apiserver/). -{{< /note >}} - PodDisruptionBudget guarding an application ensures that `.status.currentHealthy` number of pods does not fall below the number specified in `.status.desiredHealthy` by disallowing eviction of healthy pods. By using `.spec.unhealthyPodEvictionPolicy`, you can also define the criteria when unhealthy pods From 97ab1080ce388f64be0c8dd31e4ca08223920e95 Mon Sep 17 00:00:00 2001 From: Cici Huang <8658046+cici37@users.noreply.github.com> Date: Mon, 22 Jul 2024 16:28:45 -0700 Subject: [PATCH 39/82] Remove the fg CustomResourceValidationExpressions (#47237) * Remove the fg CustomResourceValidationExpressions * Fix the version --- .../feature-gates/custom-resource-validation-expressions.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/custom-resource-validation-expressions.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/custom-resource-validation-expressions.md index a7546388c932a..406efbccf7203 100644 --- a/content/en/docs/reference/command-line-tools-reference/feature-gates/custom-resource-validation-expressions.md +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/custom-resource-validation-expressions.md @@ -16,7 +16,10 @@ stages: toVersion: "1.28" - stage: stable defaultValue: true - fromVersion: "1.29" + fromVersion: "1.29" + toVersion: "1.30" + +removed: true --- Enable expression language validation in CRD which will validate customer resource based on validation rules written in From edd8ab30be1134ddd5f06729818315fcdf23f442 Mon Sep 17 00:00:00 2001 From: Shingo Omura Date: Tue, 23 Jul 2024 09:51:57 +0900 Subject: [PATCH 40/82] KEP-3619: addresses review feedbakcks --- .../configure-pod-container/security-context.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/content/en/docs/tasks/configure-pod-container/security-context.md b/content/en/docs/tasks/configure-pod-container/security-context.md index 21c6f1abd8a8f..f3c3bc2536985 100644 --- a/content/en/docs/tasks/configure-pod-container/security-context.md +++ b/content/en/docs/tasks/configure-pod-container/security-context.md @@ -162,8 +162,7 @@ exit ### Implicit group memberships defined in `/etc/group` in the container image -By default, kubernetes merges group information for the container's primary user defined in -`/etc/group` in the container image. +By default, kubernetes merges group information from the Pod with information defined in `/etc/group` in the container image. {{% code_sample file="pods/security/security-context-5.yaml" %}} @@ -198,11 +197,10 @@ $ id The output is similar to this: ```none -uid=1000(user-defined-in-image) gid=3000 groups=3000,4000,50000(group-defined-in-image) +uid=1000 gid=3000 groups=3000,4000,50000 ``` -You can see `groups` includes group ID `50000`. This is because the user (`uid=1000(user-defined-in-image)`) -belongs to the group `group-defined-in-image(gid=50000)` which is defined in `/etc/group` in the container image. +You can see `groups` includes group ID `50000`. This is because the user (`uid=1000`), which is defined in the image, belongs to the group (`gid=50000`), which is defined in `/etc/group` inside the container image. Check the `/etc/group` in the container image: @@ -210,10 +208,11 @@ Check the `/etc/group` in the container image: $ cat /etc/group ``` -You can see the group entry that `user-defined-in-image(uid=1000)` belongs to `group-defined-in-image(gid=50000)`. +You can see that uid `1000` belongs to group `50000`. ```none ... +user-defined-in-image:x:1000: group-defined-in-image:x:50000:user-defined-in-image ``` @@ -272,7 +271,7 @@ kubectl exec -it security-context-demo -- id The output is similar to this: ```none -uid=1000(user-defined-in-image) gid=3000 groups=3000,4000 +uid=1000 gid=3000 groups=3000,4000 ``` See the Pod's status: From e72fb3cd190c1f3c6515aea959fb2ce157e25c72 Mon Sep 17 00:00:00 2001 From: carlory Date: Tue, 23 Jul 2024 10:25:50 +0800 Subject: [PATCH 41/82] remove some unneeded feature-gates --- .../feature-gates/in-tree-plugin-aws-unregister.md | 3 +++ .../feature-gates/in-tree-plugin-azure-disk-unregister.md | 3 +++ .../feature-gates/in-tree-plugin-azure-file-unregister.md | 3 +++ .../feature-gates/in-tree-plugin-gce-unregister.md | 3 +++ .../feature-gates/in-tree-plugin-openstack-unregister.md | 3 +++ .../feature-gates/in-tree-plugin-vsphere-unregister.md | 3 +++ 6 files changed, 18 insertions(+) diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/in-tree-plugin-aws-unregister.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/in-tree-plugin-aws-unregister.md index 99308e36f5357..d1eb32968302e 100644 --- a/content/en/docs/reference/command-line-tools-reference/feature-gates/in-tree-plugin-aws-unregister.md +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/in-tree-plugin-aws-unregister.md @@ -9,6 +9,9 @@ stages: - stage: alpha defaultValue: false fromVersion: "1.21" + toVersion: "1.30" + +removed: true --- Stops registering the aws-ebs in-tree plugin in kubelet and volume controllers. diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/in-tree-plugin-azure-disk-unregister.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/in-tree-plugin-azure-disk-unregister.md index e408a4a183dd7..9d4732b0db57a 100644 --- a/content/en/docs/reference/command-line-tools-reference/feature-gates/in-tree-plugin-azure-disk-unregister.md +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/in-tree-plugin-azure-disk-unregister.md @@ -9,6 +9,9 @@ stages: - stage: alpha defaultValue: false fromVersion: "1.21" + toVersion: "1.30" + +removed: true --- Stops registering the azuredisk in-tree plugin in kubelet and volume controllers. diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/in-tree-plugin-azure-file-unregister.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/in-tree-plugin-azure-file-unregister.md index 48734f545acd5..eb532ba48b8ad 100644 --- a/content/en/docs/reference/command-line-tools-reference/feature-gates/in-tree-plugin-azure-file-unregister.md +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/in-tree-plugin-azure-file-unregister.md @@ -9,6 +9,9 @@ stages: - stage: alpha defaultValue: false fromVersion: "1.21" + toVersion: "1.30" + +removed: true --- Stops registering the azurefile in-tree plugin in kubelet and volume controllers. diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/in-tree-plugin-gce-unregister.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/in-tree-plugin-gce-unregister.md index 358582ffd0351..6f5df7b6d466e 100644 --- a/content/en/docs/reference/command-line-tools-reference/feature-gates/in-tree-plugin-gce-unregister.md +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/in-tree-plugin-gce-unregister.md @@ -9,6 +9,9 @@ stages: - stage: alpha defaultValue: false fromVersion: "1.21" + toVersion: "1.30" + +removed: true --- Stops registering the gce-pd in-tree plugin in kubelet and volume controllers. diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/in-tree-plugin-openstack-unregister.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/in-tree-plugin-openstack-unregister.md index d4c6d008de7bb..24d81e19c0530 100644 --- a/content/en/docs/reference/command-line-tools-reference/feature-gates/in-tree-plugin-openstack-unregister.md +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/in-tree-plugin-openstack-unregister.md @@ -9,6 +9,9 @@ stages: - stage: alpha defaultValue: false fromVersion: "1.21" + toVersion: "1.30" + +removed: true --- Stops registering the OpenStack cinder in-tree plugin in kubelet and volume controllers. diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/in-tree-plugin-vsphere-unregister.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/in-tree-plugin-vsphere-unregister.md index cd4fde558d59e..ba88fbcf33947 100644 --- a/content/en/docs/reference/command-line-tools-reference/feature-gates/in-tree-plugin-vsphere-unregister.md +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/in-tree-plugin-vsphere-unregister.md @@ -9,6 +9,9 @@ stages: - stage: alpha defaultValue: false fromVersion: "1.21" + toVersion: "1.30" + +removed: true --- Stops registering the vSphere in-tree plugin in kubelet and volume controllers. From 9e9a008587aab5eeb011b63f7f3a8bbc593a8719 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krzysztof=20Wilczy=C5=84ski?= Date: Mon, 22 Jul 2024 14:28:09 +0900 Subject: [PATCH 42/82] KEP-4191: Split Image Filesystem promotion to Beta MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Krzysztof Wilczyński --- .../feature-gates/kubelet-separate-disk-gc.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/kubelet-separate-disk-gc.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/kubelet-separate-disk-gc.md index e484ac227bcde..9d7d784e01886 100644 --- a/content/en/docs/reference/command-line-tools-reference/feature-gates/kubelet-separate-disk-gc.md +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/kubelet-separate-disk-gc.md @@ -9,6 +9,11 @@ stages: - stage: alpha defaultValue: false fromVersion: "1.29" + toVersion: "1.30" + - stage: beta + defaultValue: true + fromVersion: "1.31" --- -Enable kubelet to garbage collect container images and containers -even when those are on a separate filesystem. +The split image filesystem feature enables kubelet to perform garbage collection +of images (read-only layers) and/or containers (writeable layers) deployed on +separate filesystems. From 06aff012a2adae8d18e48a3318f839e4e641e356 Mon Sep 17 00:00:00 2001 From: Peter Hunt Date: Tue, 23 Jul 2024 12:19:14 -0400 Subject: [PATCH 43/82] PSS: add container_engine_t to allowed list of selinux types Signed-off-by: Peter Hunt --- content/en/docs/concepts/security/pod-security-standards.md | 1 + .../access-authn-authz/psp-to-pod-security-standards.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/content/en/docs/concepts/security/pod-security-standards.md b/content/en/docs/concepts/security/pod-security-standards.md index fb9cab9d15c8c..6f3a48a139391 100644 --- a/content/en/docs/concepts/security/pod-security-standards.md +++ b/content/en/docs/concepts/security/pod-security-standards.md @@ -213,6 +213,7 @@ fail validation.
  • container_t
  • container_init_t
  • container_kvm_t
    • +
  • container_engine_t (since Kubernetes 1.31)

    • Restricted Fields

    diff --git a/content/en/docs/reference/access-authn-authz/psp-to-pod-security-standards.md b/content/en/docs/reference/access-authn-authz/psp-to-pod-security-standards.md index 34e40fca43611..b1eb2d7467585 100644 --- a/content/en/docs/reference/access-authn-authz/psp-to-pod-security-standards.md +++ b/content/en/docs/reference/access-authn-authz/psp-to-pod-security-standards.md @@ -130,7 +130,7 @@ under the `.spec` field path.
    • user is unset ("" / undefined / nil)
    • role is unset ("" / undefined / nil)
      • -
    • type is unset or one of: container_t, container_init_t, container_kvm_t
      • +
    • type is unset or one of: container_t, container_init_t, container_kvm_t, container_engine_t
    • level is anything
    From 87a705579f4303462e76c781b3c63f90613858a2 Mon Sep 17 00:00:00 2001 From: Vinayak Goyal Date: Fri, 28 Jun 2024 18:26:43 +0000 Subject: [PATCH 44/82] KEP-24: Graduate Kubernetes' support for AppArmor to GA. Signed-off-by: Vinayak Goyal --- .../feature-gates/apparmor.md | 4 ++ .../security-context.md | 50 +++++++++++++++++++ content/en/docs/tutorials/_index.md | 2 +- 3 files changed, 55 insertions(+), 1 deletion(-) diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/apparmor.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/apparmor.md index 005105630bf0e..404ea8181ca6c 100644 --- a/content/en/docs/reference/command-line-tools-reference/feature-gates/apparmor.md +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/apparmor.md @@ -9,6 +9,10 @@ stages: - stage: beta defaultValue: true fromVersion: "1.4" + toVersion: "1.30" + - stage: stable + defaultValue: true + fromVersion: "1.31" --- Enable use of AppArmor mandatory access control for Pods running on Linux nodes. See [AppArmor Tutorial](/docs/tutorials/security/apparmor/) for more details. diff --git a/content/en/docs/tasks/configure-pod-container/security-context.md b/content/en/docs/tasks/configure-pod-container/security-context.md index b176d20df51c5..8e36720c65641 100644 --- a/content/en/docs/tasks/configure-pod-container/security-context.md +++ b/content/en/docs/tasks/configure-pod-container/security-context.md @@ -419,6 +419,56 @@ securityContext: localhostProfile: my-profiles/profile-allow.json ``` +## Set the AppArmor Profile for a Container + +To set the AppArmor profile for a Container, include the `appArmorProfile` field +in the `securityContext` section of your Container. The `appArmorProfile` field +is a +[AppArmorProfile](/docs/reference/generated/kubernetes-api/{{< param "version" +>}}/#apparmorprofile-v1-core) object consisting of `type` and `localhostProfile`. +Valid options for `type` include `RuntimeDefault`(default), `Unconfined`, and +`Localhost`. `localhostProfile` must only be set if `type` is `Localhost`. It +indicates the name of the pre-configured profile on the node. The profile needs +to be loaded onto all nodes suitable for the Pod, since you don't know where the +pod will be scheduled. +Approaches for setting up custom profiles are discussed in +[Setting up nodes with profiles](/docs/tutorials/security/apparmor/#setting-up-nodes-with-profiles). + +Note: If `containers[*].securityContext.appArmorProfile.type` is explicitly set +to `RuntimeDefault`, then the Pod will not be admitted if AppArmor is not +enabled on the Node. However if `containers[*].securityContext.appArmorProfile.type` +is not specified, then the default (which is also `RuntimeDefault`) will only +be applied if the node has AppArmor enabled. If the node has AppArmor disabled +the Pod will be admitted but the Container will not be restricted by the +`RuntimeDefault` profile. + +Here is an example that sets the AppArmor profile to the node's container runtime +default profile: + +```yaml +... +containers: +- name: container-1 + securityContext: + appArmorProfile: + type: RuntimeDefault +``` + +Here is an example that sets the AppArmor profile to a pre-configured profile +named `k8s-apparmor-example-deny-write`: + +```yaml +... +containers: +- name: container-1 + securityContext: + appArmorProfile: + type: Localhost + localhostProfile: k8s-apparmor-example-deny-write +``` + +For more details please see, [Restrict a Container's Access to Resources with AppArmor](/docs/tutorials/security/apparmor/). + ## Assign SELinux labels to a Container To assign SELinux labels to a Container, include the `seLinuxOptions` field in diff --git a/content/en/docs/tutorials/_index.md b/content/en/docs/tutorials/_index.md index 97a3bacbdf6a5..d9c865c5f7c6c 100644 --- a/content/en/docs/tutorials/_index.md +++ b/content/en/docs/tutorials/_index.md @@ -49,7 +49,7 @@ Before walking through each tutorial, you may want to bookmark the * [Apply Pod Security Standards at Cluster level](/docs/tutorials/security/cluster-level-pss/) * [Apply Pod Security Standards at Namespace level](/docs/tutorials/security/ns-level-pss/) -* [AppArmor](/docs/tutorials/security/apparmor/) +* [Restrict a Container's Access to Resources with AppArmor](/docs/tutorials/security/apparmor/) * [Seccomp](/docs/tutorials/security/seccomp/) ## {{% heading "whatsnext" %}} From a12454f2dc41ce3507c473f4082b4c16c61c2efa Mon Sep 17 00:00:00 2001 From: Sascha Grunert Date: Mon, 24 Jun 2024 10:58:13 +0200 Subject: [PATCH 45/82] Add ImageVolume documentation Add a basic task how to use image volumes in pods. Signed-off-by: Sascha Grunert --- content/en/docs/concepts/storage/volumes.md | 65 +++++++++++++++++ .../feature-gates/image-volume.md | 14 ++++ .../configure-pod-container/image-volumes.md | 72 +++++++++++++++++++ content/en/examples/pods/image-volumes.yaml | 17 +++++ 4 files changed, 168 insertions(+) create mode 100644 content/en/docs/reference/command-line-tools-reference/feature-gates/image-volume.md create mode 100644 content/en/docs/tasks/configure-pod-container/image-volumes.md create mode 100644 content/en/examples/pods/image-volumes.yaml diff --git a/content/en/docs/concepts/storage/volumes.md b/content/en/docs/concepts/storage/volumes.md index 2abda301501ff..be8cbc64390b7 100644 --- a/content/en/docs/concepts/storage/volumes.md +++ b/content/en/docs/concepts/storage/volumes.md @@ -534,6 +534,71 @@ spec: type: FileOrCreate ``` +### image + +{{< feature-state feature_gate_name="ImageVolume" >}} + +An `image` volume source represents an OCI object (a container image or +artifact) which is available on the kubelet's host machine. + +One example to use the `image` volume source is: + +{{% code_sample file="pods/image-volumes.yaml" %}} + +The volume is resolved at pod startup depending on which `pullPolicy` value is +provided: + +`Always` +: the kubelet always attempts to pull the reference. If the pull fails, the kubelet sets the Pod to `Failed`. + +`Never` +: the kubelet never pulls the reference and only uses a local image or artifact. The Pod becomes `Failed` if any layers of the image aren't already present locally, or if the manifest for that image isn't already cached. + +`IfNotPresent` +: the kubelet pulls if the reference isn't already present on disk. The Pod becomes `Failed` if the reference isn't present and the pull fails. + +The volume gets re-resolved if the pod gets deleted and recreated, which means +that new remote content will become available on pod recreation. A failure to +resolve or pull the image during pod startup will block containers from starting +and may add significant latency. Failures will be retried using normal volume +backoff and will be reported on the pod reason and message. + +The types of objects that may be mounted by this volume are defined by the +container runtime implementation on a host machine and at minimum must include +all valid types supported by the container image field. The OCI object gets +mounted in a single directory (`spec.containers[*].volumeMounts.mountPath`) by +will be mounted read-only. On Linux, the container runtime typically also mounts the +volume with file execution blocked (`noexec`). + +Beside that: +- Sub path mounts for containers are not supported + (`spec.containers[*].volumeMounts.subpath`). +- The field `spec.securityContext.fsGroupChangePolicy` has no effect on this + volume type. +- The [`AlwaysPullImages` Admission Controller](/docs/reference/access-authn-authz/admission-controllers/#alwayspullimages) + does also work for this volume source like for container images. + +The following fields are available for the `image` type: + +`reference` +: Artifact reference to be used. For example, you could specify +`registry.k8s.io/conformance:v{{< skew currentPatchVersion >}}` to load the +files from the Kubernetes conformance test image. Behaves in the same way as +`pod.spec.containers[*].image`. Pull secrets will be assembled in the same way +as for the container image by looking up node credentials, service account image +pull secrets, and pod spec image pull secrets. This field is optional to allow +higher level config management to default or override container images in +workload controllers like Deployments and StatefulSets. +[More info about container images](/docs/concepts/containers/images) + +`pullPolicy` +: Policy for pulling OCI objects. Possible values are: `Always`, `Never` or +`IfNotPresent`. Defaults to `Always` if `:latest` tag is specified, or +`IfNotPresent` otherwise. + +See the [_Use an Image Volume With a Pod_](/docs/tasks/configure-pod-container/image-volumes) +example for more details on how to use the volume source. + ### iscsi An `iscsi` volume allows an existing iSCSI (SCSI over IP) volume to be mounted diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/image-volume.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/image-volume.md new file mode 100644 index 0000000000000..cb92837bdef3b --- /dev/null +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/image-volume.md @@ -0,0 +1,14 @@ +--- +title: ImageVolume +content_type: feature_gate +_build: + list: never + render: false + +stages: + - stage: alpha + defaultValue: false + fromVersion: "1.31" +--- +Allow using the [`image`](/docs/concepts/storage/volumes/) volume source in a Pod. +This volume source lets you mount a container image as a read-only volume. diff --git a/content/en/docs/tasks/configure-pod-container/image-volumes.md b/content/en/docs/tasks/configure-pod-container/image-volumes.md new file mode 100644 index 0000000000000..08da62beafe10 --- /dev/null +++ b/content/en/docs/tasks/configure-pod-container/image-volumes.md @@ -0,0 +1,72 @@ +--- +title: Use an Image Volume With a Pod +reviewers: +content_type: task +weight: 210 +min-kubernetes-server-version: v1.31 +--- + + + +{{< feature-state feature_gate_name="ImageVolume" >}} + +This page shows how to configure a pod using image volumes. This allows you to +mount content from OCI registries inside containers. + +## {{% heading "prerequisites" %}} + +{{< include "task-tutorial-prereqs.md" >}} {{< version-check >}} + +- The container runtime needs to support the image volumes feature +- You need to exec commands in the host +- You need to be able to exec into pods +- You need to enable the `ImageVolume` [feature gate](/docs/reference/command-line-tools-reference/feature-gates/) + + + +## Run a Pod that uses an image volume {#create-pod} + +An image volume for a pod is enabled setting the `volumes.[*].image` field of `.spec` +to a valid reference and consuming it in the `volumeMounts` of the container. For example: + +{{% code_sample file="pods/image-volumes.yaml" %}} + +1. Create the pod on your cluster: + + ```shell + kubectl apply -f https://k8s.io/examples/pods/image-volumes.yaml + ``` + +1. Attach to the container: + + ```shell + kubectl attach -it image-volume bash + ``` + +Run this command: + +```shell +cat /volume/dir/file +``` + +The output is similar to: + +```shell +1 +``` + +Also run: + +```shell +cat /volume/file +``` + +The output is similar to: + +```shell +2 +``` + +## Further reading + +- [`image` volumes](/docs/concepts/storage/volumes/#image) diff --git a/content/en/examples/pods/image-volumes.yaml b/content/en/examples/pods/image-volumes.yaml new file mode 100644 index 0000000000000..3a3cc79a4a3d3 --- /dev/null +++ b/content/en/examples/pods/image-volumes.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Pod +metadata: + name: image-volume +spec: + containers: + - name: shell + command: ["sleep", "infinity"] + image: debian + volumeMounts: + - name: volume + mountPath: /volume + volumes: + - name: volume + image: + reference: quay.io/crio/artifact:v1 + pullPolicy: IfNotPresent From f6be58192159088e0e9642c37c9472552d96f7f2 Mon Sep 17 00:00:00 2001 From: Yuki Iwai Date: Wed, 26 Jun 2024 02:15:38 +0900 Subject: [PATCH 46/82] Update JobSuccessPolicy documentations for the beta graduation Signed-off-by: Yuki Iwai --- content/en/docs/concepts/workloads/controllers/job.md | 2 +- .../feature-gates/job-success-policy.md | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/content/en/docs/concepts/workloads/controllers/job.md b/content/en/docs/concepts/workloads/controllers/job.md index 37b839a148d16..48d898fd2ae95 100644 --- a/content/en/docs/concepts/workloads/controllers/job.md +++ b/content/en/docs/concepts/workloads/controllers/job.md @@ -598,7 +598,7 @@ Here is a manifest for a Job with `successPolicy`: In the example above, both `succeededIndexes` and `succeededCount` have been specified. Therefore, the job controller will mark the Job as succeeded and terminate the lingering Pods when either of the specified indexes, 0, 2, or 3, succeed. -The Job that meets the success policy gets the `SuccessCriteriaMet` condition. +The Job that meets the success policy gets the `SuccessCriteriaMet` condition with a `SuccessPolicy` reason. After the removal of the lingering Pods is issued, the Job gets the `Complete` condition. Note that the `succeededIndexes` is represented as intervals separated by a hyphen. diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/job-success-policy.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/job-success-policy.md index 601680357ccc9..69c89b138a55e 100644 --- a/content/en/docs/reference/command-line-tools-reference/feature-gates/job-success-policy.md +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/job-success-policy.md @@ -10,5 +10,9 @@ stages: - stage: alpha defaultValue: false fromVersion: "1.30" + toVersion: "1.30" + - stage: beta + defaultValue: true + fromVersion: "1.31" --- Allow users to specify when a Job can be declared as succeeded based on the set of succeeded pods. From 67fe8ed862b5d4e9a0b42dd8f5a8f886e6787181 Mon Sep 17 00:00:00 2001 From: Michal Wozniak Date: Thu, 13 Jun 2024 14:19:55 +0200 Subject: [PATCH 47/82] Update the docs for JobManagedBy and JobPodReplacementPolicy related to pod termination --- .../concepts/workloads/controllers/job.md | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/content/en/docs/concepts/workloads/controllers/job.md b/content/en/docs/concepts/workloads/controllers/job.md index 67830dd1305d7..88379f0a64357 100644 --- a/content/en/docs/concepts/workloads/controllers/job.md +++ b/content/en/docs/concepts/workloads/controllers/job.md @@ -444,6 +444,10 @@ kubectl get -o yaml job job-backoff-limit-per-index-example type: Failed ``` +Note that, since v1.31, you will also observe in the status the `FailureTarget` +Job condition, with the same `reason` and `message` as for the the `Failed` +condition (see also [Job termination and cleanup](#job-termination-and-cleanup)). + Additionally, you may want to use the per-index backoff along with a [pod failure policy](#pod-failure-policy). When using per-index backoff, there is a new `FailIndex` action available which allows you to @@ -553,6 +557,11 @@ terminating Pods only once these Pods reach the terminal `Failed` phase. This be to `podReplacementPolicy: Failed`. For more information, see [Pod replacement policy](#pod-replacement-policy). {{< /note >}} +When you use the `podFailurePolicy`, and the Job fails due to the pod +matching the rule with the `FailJob` action, then the Job controller triggers +the Job termination process by adding the `FailureTarget` condition. +See [Job termination and cleanup](#job-termination-and-cleanup) for more details. + ## Success policy {#success-policy} {{< feature-state feature_gate_name="JobSuccessPolicy" >}} @@ -659,6 +668,45 @@ there is no automatic Job restart once the Job status is `type: Failed`. That is, the Job termination mechanisms activated with `.spec.activeDeadlineSeconds` and `.spec.backoffLimit` result in a permanent Job failure that requires manual intervention to resolve. +### Terminal Job conditions + +A Job has two possible terminal states, it ends up either succeeded, or failed, +and these states are reflected by the presence of the Job conditions `Complete` +or `Failed`, respectively. + +The failure scenarios encompass: +- the `.spec.backoffLimit` +- the `.spec.activeDeadlineSeconds` is exceeded +- the `.spec.backoffLimitPerIndex` is exceeded (see [Backoff limit per index](#backoff-limit-per-index)) +- the Pod matches the Job Pod Failure Policy rule with the `FailJob` action (see more [Pod failure policy](#pod-failure-policy)) + +The success scenarios encompass: +- the `.spec.completions` is reached +- the criteria specified by the Job Success Policy are met (see more [Success policy](#success-policy)) + +### Termination of Job pods + +Prior to v1.31 the Job terminal conditions are added when the Job termination +process is triggered, and all Pod finalizers are removed, but some pods may +still remain running at that point in time. + +Since v1.31, when you enable either the `JobManagedBy` or +`JobPodReplacementPolicy` (enabled by default) +[feature gate](/docs/reference/command-line-tools-reference/feature-gates/), the +Job controller awaits for termination of all pods before adding a condition +indicating that the Job is finished (either `Complete` or `Failed`). + +Note that, the process of terminating all pods may take a substantial amount +of time, depending on a Pod's `terminationGracePeriodSeconds` (see +[Pod termination](#docs/concepts/workloads/pods/pod-lifecycle/#pod-termination)), +and thus adding the terminal Job condition, even if the fate of the Job is +already determined. + +If you want to know the fate of the Job as soon as determined you can use, +since v1.31, the `FailureTarget` and `SuccessCriteriaMet` conditions, which +cover all scenarios in which Job controller triggers the Job termination process +(see [Terminal Job conditions](#terminal-job-conditions)). + ## Clean up finished jobs automatically Finished Jobs are usually no longer needed in the system. Keeping them around in @@ -1063,6 +1111,13 @@ status: terminating: 3 # three Pods are terminating and have not yet reached the Failed phase ``` +{{< note >}} +Since v1.31, when you enable the `JobPodReplacementPolicy` +[feature gate](/docs/reference/command-line-tools-reference/feature-gates/) +(enabled by default), the Job controller awaits for termination of all pods +before marking a Job as terminal (see [Termination of Job Pods](#termination-of-job-pods)). +{{< /note >}} + ### Delegation of managing a Job object to external controller {{< feature-state feature_gate_name="JobManagedBy" >}} @@ -1107,6 +1162,13 @@ after the operation: the built-in Job controller and the external controller indicated by the field value. {{< /warning >}} +{{< note >}} +Since v1.31, when you enable the `JobManagedBy` +[feature gate](/docs/reference/command-line-tools-reference/feature-gates/), +the Job controller awaits for termination of all pods before marking a Job as +terminal (see [Termination of Job Pods](#termination-of-job-pods)). +{{< /note >}} + ## Alternatives ### Bare Pods From 108e2a9f0f6431a50b0704267e18d7218d3c1bf7 Mon Sep 17 00:00:00 2001 From: Kensei Nakada Date: Fri, 26 Jul 2024 05:03:11 +0900 Subject: [PATCH 48/82] graduate MatchLabelKeysInPodAffinity to beta (#45181) * graduate MatchLabelKeysInPodAffinity to beta * update feature-state * Correct the grammar Co-authored-by: Tim Bannister * add comments * Update version appropriately --------- Co-authored-by: Tim Bannister --- .../concepts/scheduling-eviction/assign-pod-node.md | 12 ++++++------ .../match-label-keys-in-pod-affinity.md | 6 +++++- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/content/en/docs/concepts/scheduling-eviction/assign-pod-node.md b/content/en/docs/concepts/scheduling-eviction/assign-pod-node.md index 50e1bbafa9eda..5a6d18baf6f1a 100644 --- a/content/en/docs/concepts/scheduling-eviction/assign-pod-node.md +++ b/content/en/docs/concepts/scheduling-eviction/assign-pod-node.md @@ -363,10 +363,10 @@ null `namespaceSelector` matches the namespace of the Pod where the rule is defi {{< feature-state feature_gate_name="MatchLabelKeysInPodAffinity" >}} {{< note >}} - -The `matchLabelKeys` field is an alpha-level field and is disabled by default in + +The `matchLabelKeys` field is a beta-level field and is enabled by default in Kubernetes {{< skew currentVersion >}}. -When you want to use it, you have to enable it via the +When you want to disable it, you have to disable it explicitly via the `MatchLabelKeysInPodAffinity` [feature gate](/docs/reference/command-line-tools-reference/feature-gates/). {{< /note >}} @@ -414,10 +414,10 @@ spec: {{< feature-state feature_gate_name="MatchLabelKeysInPodAffinity" >}} {{< note >}} - -The `mismatchLabelKeys` field is an alpha-level field and is disabled by default in + +The `mismatchLabelKeys` field is a beta-level field and is enabled by default in Kubernetes {{< skew currentVersion >}}. -When you want to use it, you have to enable it via the +When you want to disable it, you have to disable it explicitly via the `MatchLabelKeysInPodAffinity` [feature gate](/docs/reference/command-line-tools-reference/feature-gates/). {{< /note >}} diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/match-label-keys-in-pod-affinity.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/match-label-keys-in-pod-affinity.md index 196c48551efc0..0a8cb00265ef9 100644 --- a/content/en/docs/reference/command-line-tools-reference/feature-gates/match-label-keys-in-pod-affinity.md +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/match-label-keys-in-pod-affinity.md @@ -9,6 +9,10 @@ stages: - stage: alpha defaultValue: false fromVersion: "1.29" + toVersion: "1.30" + - stage: beta + defaultValue: true + fromVersion: "1.31" --- -Enable the `matchLabelKeys` and `mismatchLabelKeys` field for +Enable the `matchLabelKeys` and `mismatchLabelKeys` fields for [pod (anti)affinity](/docs/concepts/scheduling-eviction/assign-pod-node/). From 3fb2925b56ed8d7ca7adc66c0111901b27f89b43 Mon Sep 17 00:00:00 2001 From: Shingo Omura Date: Fri, 26 Jul 2024 10:07:58 +0900 Subject: [PATCH 49/82] Apply suggestions for improvements from code review Co-authored-by: Qiming Teng --- .../security-context.md | 49 ++++++++++++------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/content/en/docs/tasks/configure-pod-container/security-context.md b/content/en/docs/tasks/configure-pod-container/security-context.md index f3c3bc2536985..30ac3779cfbb6 100644 --- a/content/en/docs/tasks/configure-pod-container/security-context.md +++ b/content/en/docs/tasks/configure-pod-container/security-context.md @@ -66,8 +66,8 @@ all processes within any containers of the Pod. If this field is omitted, the pr will be root(0). Any files created will also be owned by user 1000 and group 3000 when `runAsGroup` is specified. Since `fsGroup` field is specified, all processes of the container are also part of the supplementary group ID 2000. The owner for volume `/data/demo` and any files created in that volume will be Group ID 2000. -Additionally, since `supplementalGroups` field is specified, all processes of the container are also part of the -specified group IDs. If this field is omitted, it means empty. +Additionally, when the `supplementalGroups` field is specified, all processes of the container are also part of the +specified groups. If this field is omitted, it means empty. Create the Pod: @@ -150,9 +150,9 @@ uid=1000 gid=3000 groups=2000,3000,4000 From the output, you can see that `gid` is 3000 which is same as the `runAsGroup` field. If the `runAsGroup` was omitted, the `gid` would remain as 0 (root) and the process will be able to interact with files that are owned by the root(0) group and groups that have -the required group permissions for the root (0) group. You can also see `groups` -contains the group IDs which are specified by `fsGroup` and `supplementalGroups` other -than `gid`. +the required group permissions for the root (0) group. You can also see that `groups` +contains the group IDs which are specified by `fsGroup` and `supplementalGroups`, +in addition to `gid`. Exit your shell: @@ -166,7 +166,7 @@ By default, kubernetes merges group information from the Pod with information de {{% code_sample file="pods/security/security-context-5.yaml" %}} -In this configuration, it just specifies `runAsUser`, `runAsGroup` and `supplementalGroups`. +This Pod security context contains `runAsUser`, `runAsGroup` and `supplementalGroups`. However, you can see that the actual supplementary groups attached to the container process will include group IDs which come from `/etc/group` in the container image. @@ -200,7 +200,9 @@ The output is similar to this: uid=1000 gid=3000 groups=3000,4000,50000 ``` -You can see `groups` includes group ID `50000`. This is because the user (`uid=1000`), which is defined in the image, belongs to the group (`gid=50000`), which is defined in `/etc/group` inside the container image. +You can see that `groups` includes group ID `50000`. This is because the user (`uid=1000`), +which is defined in the image, belongs to the group (`gid=50000`), which is defined in `/etc/group` +inside the container image. Check the `/etc/group` in the container image: @@ -223,7 +225,7 @@ exit ``` {{}} -Implicitly _merged_ supplementary groups may cause security concerns particularly in accessing +_Implicitly merged_ supplementary groups may cause security problems particularly when accessing the volumes (see [kubernetes/kubernetes#112879](https://issue.k8s.io/112879) for details). If you want to avoid this. Please see the below section. {{}} @@ -236,19 +238,25 @@ This feature can be enabled by setting the `SupplementalGroupsPolicy` [feature gate](/docs/reference/command-line-tools-reference/feature-gates/) for kubelet and kube-apiserver, and setting the `.spec.securityContext.supplementalGroupsPolicy` field for a pod. -**supplementalGroupsPolicy** - `supplementalGroupsPolicy` defines behavior for calculating -supplementary groups for the container processes in a pod. +The `supplementalGroupsPolicy` field defines the policy for calculating the +supplementary groups for the container processes in a pod. There are two valid +values for this field: -* _Merge_: The group membership defined in `/etc/group` for the container's primary user will be merged. If not specified, this policy will be applied. +* `Merge`: The group membership defined in `/etc/group` for the container's primary user will be merged. + This is the default policy if not specified. -* _Strict_: it only attaches group IDs in `fsGroup`, `supplementalGroups`, or `runAsGroup` fields as the supplementary groups of the container processess. This means no group membership defined in `/etc/group` for the container's primary user will be merged. +* `Strict`: Only group IDs in `fsGroup`, `supplementalGroups`, or `runAsGroup` fields + are attached as the supplementary groups of the container processes. + This means no group membership from `/etc/group` for the container's primary user will be merged. -When the feature is enabled, it also exposes the process identity attached to the first container process of the container -in `.status.containerStatuses[].user.linux` field. It would be helpful to detect if implicit group ID's are attached. +When the feature is enabled, it also exposes the process identity attached to the first container process +in `.status.containerStatuses[].user.linux` field. It would be useful for detecting if +implicit group ID's are attached. {{% code_sample file="pods/security/security-context-6.yaml" %}} -This pod manifest defines `supplementalGroupsPolicy=Strict`. You can see no group membership defined in `/etc/group` will be merged to the supplementary groups for container processes. +This pod manifest defines `supplementalGroupsPolicy=Strict`. You can see that no group memberships +defined in `/etc/group` are merged to the supplementary groups for container processes. Create the Pod: @@ -280,7 +288,7 @@ See the Pod's status: kubectl get pod security-context-demo -o yaml ``` -You can see `status.containerStatuses[].user.linux` field exposes the process identitiy +You can see that the `status.containerStatuses[].user.linux` field exposes the process identitiy attached to the first container process. ```none @@ -299,9 +307,12 @@ status: ``` {{}} -Please note that the values in `status.containerStatuses[].user.linux` field is _the firstly attached_ +Please note that the values in the `status.containerStatuses[].user.linux` field is _the first attached_ process identity to the first container process in the container. If the container has sufficient privilege -to call system calls related to process identity (e.g. [`setuid(2)`](https://man7.org/linux/man-pages/man2/setuid.2.html), [`setgid(2)`](https://man7.org/linux/man-pages/man2/setgid.2.html) or [`setgroups(2)`](https://man7.org/linux/man-pages/man2/setgroups.2.html), etc.), +to make system calls related to process identity +(e.g. [`setuid(2)`](https://man7.org/linux/man-pages/man2/setuid.2.html), +[`setgid(2)`](https://man7.org/linux/man-pages/man2/setgid.2.html) or +[`setgroups(2)`](https://man7.org/linux/man-pages/man2/setgroups.2.html), etc.), the container process can change its identity. Thus, the _actual_ process identity will be dynamic. {{}} @@ -315,7 +326,7 @@ CRI-level: - [containerd](https://containerd.io/), since v2.0 - [CRI-O](https://cri-o.io/), since v1.31 -You can see if the feature is supported in the node status. +You can see if the feature is supported in the Node status. ```yaml apiVersion: v1 From 086bbc8a373e911a4daaff45746f62121c89a103 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krzysztof=20Wilczy=C5=84ski?= Date: Mon, 24 Jun 2024 23:10:38 +0900 Subject: [PATCH 50/82] KEP-4191: Split Image Filesystem add documentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Krzysztof Wilczyński --- .../node-pressure-eviction.md | 163 +++++++++++++----- 1 file changed, 123 insertions(+), 40 deletions(-) diff --git a/content/en/docs/concepts/scheduling-eviction/node-pressure-eviction.md b/content/en/docs/concepts/scheduling-eviction/node-pressure-eviction.md index 76f930721ef6e..5d56811d9cf28 100644 --- a/content/en/docs/concepts/scheduling-eviction/node-pressure-eviction.md +++ b/content/en/docs/concepts/scheduling-eviction/node-pressure-eviction.md @@ -6,6 +6,17 @@ weight: 100 {{}}
    +{{< feature-state feature_gate_name="KubeletSeparateDiskGC" >}} + +{{}} +The _split image filesystem_ feature, which enables support for the `containerfs` +filesystem, adds several new eviction signals, thresholds and metrics. To use +`containerfs`, the Kubernetes release v{{< skew currentVersion >}} requires the +`KubeletSeparateDiskGC` [feature gate](/docs/reference/command-line-tools-reference/feature-gates/) +to be enabled. Currently, only CRI-O (v1.29 or higher) offers the `containerfs` +filesystem support. +{{}} + The {{}} monitors resources like memory, disk space, and filesystem inodes on your cluster's nodes. When one or more of these resources reach specific consumption levels, the @@ -61,23 +72,25 @@ The kubelet uses various parameters to make eviction decisions, like the followi ### Eviction signals {#eviction-signals} Eviction signals are the current state of a particular resource at a specific -point in time. Kubelet uses eviction signals to make eviction decisions by +point in time. The kubelet uses eviction signals to make eviction decisions by comparing the signals to eviction thresholds, which are the minimum amount of the resource that should be available on the node. On Linux, the kubelet uses the following eviction signals: -| Eviction Signal | Description | -|----------------------|---------------------------------------------------------------------------------------| -| `memory.available` | `memory.available` := `node.status.capacity[memory]` - `node.stats.memory.workingSet` | -| `nodefs.available` | `nodefs.available` := `node.stats.fs.available` | -| `nodefs.inodesFree` | `nodefs.inodesFree` := `node.stats.fs.inodesFree` | -| `imagefs.available` | `imagefs.available` := `node.stats.runtime.imagefs.available` | -| `imagefs.inodesFree` | `imagefs.inodesFree` := `node.stats.runtime.imagefs.inodesFree` | -| `pid.available` | `pid.available` := `node.stats.rlimit.maxpid` - `node.stats.rlimit.curproc` | +| Eviction Signal | Description | +|--------------------------|---------------------------------------------------------------------------------------| +| `memory.available` | `memory.available` := `node.status.capacity[memory]` - `node.stats.memory.workingSet` | +| `nodefs.available` | `nodefs.available` := `node.stats.fs.available` | +| `nodefs.inodesFree` | `nodefs.inodesFree` := `node.stats.fs.inodesFree` | +| `imagefs.available` | `imagefs.available` := `node.stats.runtime.imagefs.available` | +| `imagefs.inodesFree` | `imagefs.inodesFree` := `node.stats.runtime.imagefs.inodesFree` | +| `containerfs.available` | `containerfs.available` := `node.stats.runtime.containerfs.available` | +| `containerfs.inodesFree` | `containerfs.inodesFree` := `node.stats.runtime.containerfs.inodesFree` | +| `pid.available` | `pid.available` := `node.stats.rlimit.maxpid` - `node.stats.rlimit.curproc` | In this table, the **Description** column shows how kubelet gets the value of the -signal. Each signal supports either a percentage or a literal value. Kubelet +signal. Each signal supports either a percentage or a literal value. The kubelet calculates the percentage value relative to the total capacity associated with the signal. @@ -93,16 +106,43 @@ reproduces the same set of steps that the kubelet performs to calculate file-backed memory on the inactive LRU list) from its calculation, as it assumes that memory is reclaimable under pressure. -The kubelet recognizes two specific filesystem identifiers: +The kubelet recognizes three specific filesystem identifiers: + +1. `nodefs`: The node's main filesystem, used for local disk volumes, + emptyDir volumes not backed by memory, log storage, ephemeral storage, + and more. For example, `nodefs` contains `/var/lib/kubelet`. + +1. `imagefs`: An optional filesystem that container runtimes can use to store + container images (which are the read-only layers) and container writable + layers. + +1. `containerfs`: An optional filesystem that container runtime can use to + store the writeable layers. Similar to the main filesystem (see `nodefs`), + it's used to store local disk volumes, emptyDir volumes not backed by memory, + log storage, and ephemeral storage, except for the container images. When + `containerfs` is used, the `imagefs` filesystem can be split to only store + images (read-only layers) and nothing else. + +As such, kubelet generally allows three options for container filesystems: + +- Everything is on the single `nodefs`, also referred to as "rootfs" or + simply "root", and there is no dedicated image filesystem. -1. `nodefs`: The node's main filesystem, used for local disk volumes, emptyDir - volumes not backed by memory, log storage, and more. - For example, `nodefs` contains `/var/lib/kubelet/`. -1. `imagefs`: An optional filesystem that container runtimes use to store container - images and container writable layers. +- Container storage (see `nodefs`) is on a dedicated disk, and `imagefs` + (writable and read-only layers) is separate from the root filesystem. + This is often referred to as "split disk" (or "separate disk") filesystem. -Kubelet auto-discovers these filesystems and ignores other node local filesystems. Kubelet -does not support other configurations. +- Container filesystem `containerfs` (same as `nodefs` plus writable + layers) is on root and the container images (read-only layers) are + stored on separate `imagefs`. This is often referred to as "split image" + filesystem. + +The kubelet will attempt to auto-discover these filesystems with their current +configuration directly from the underlying container runtime and will ignore +other local node filesystems. + +The kubelet does not support other container filesystems or storage configurations, +and it does not currently support multiple filesystems for images and containers. Some kubelet garbage collection features are deprecated in favor of eviction: @@ -177,6 +217,19 @@ then the values of other parameters will not be inherited as the default values and will be set to zero. In order to provide custom values, you should provide all the thresholds respectively. +The `containerfs.available` and `containerfs.inodesFree` (Linux nodes) default +eviction thresholds will be set as follows: + +- If a single filesystem is used for everything, then `containerfs` thresholds + are set the same as `nodefs`. + +- If separate filesystems are configured for both images and containers, + then `containerfs` thresholds are set the same as `imagefs`. + +Setting custom overrides for thresholds related to `containersfs` is currently +not supported, and a warning will be issued if an attempt to do so is made; any +provided custom values will, as such, be ignored. + ## Eviction monitoring interval The kubelet evaluates eviction thresholds based on its configured `housekeeping-interval`, @@ -190,11 +243,11 @@ threshold is met, independent of configured grace periods. The kubelet maps eviction signals to node conditions as follows: -| Node Condition | Eviction Signal | Description | -|-------------------|---------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------| -| `MemoryPressure` | `memory.available` | Available memory on the node has satisfied an eviction threshold | -| `DiskPressure` | `nodefs.available`, `nodefs.inodesFree`, `imagefs.available`, or `imagefs.inodesFree` | Available disk space and inodes on either the node's root filesystem or image filesystem has satisfied an eviction threshold | -| `PIDPressure` | `pid.available` | Available processes identifiers on the (Linux) node has fallen below an eviction threshold | +| Node Condition | Eviction Signal | Description | +|-------------------|---------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------| +| `MemoryPressure` | `memory.available` | Available memory on the node has satisfied an eviction threshold | +| `DiskPressure` | `nodefs.available`, `nodefs.inodesFree`, `imagefs.available`, `imagefs.inodesFree`, `containerfs.available`, or `containerfs.inodesFree` | Available disk space and inodes on either the node's root filesystem, image filesystem, or container filesystem has satisfied an eviction threshold | +| `PIDPressure` | `pid.available` | Available processes identifiers on the (Linux) node has fallen below an eviction threshold | The control plane also [maps](/docs/concepts/scheduling-eviction/taint-and-toleration/#taint-nodes-by-condition) these node conditions to taints. @@ -219,23 +272,36 @@ The kubelet tries to reclaim node-level resources before it evicts end-user pods When a `DiskPressure` node condition is reported, the kubelet reclaims node-level resources based on the filesystems on the node. +#### Without `imagefs` or `containerfs` + +If the node only has a `nodefs` filesystem that meets eviction thresholds, +the kubelet frees up disk space in the following order: + +1. Garbage collect dead pods and containers. +1. Delete unused images. + #### With `imagefs` If the node has a dedicated `imagefs` filesystem for container runtimes to use, the kubelet does the following: -- If the `nodefs` filesystem meets the eviction thresholds, the kubelet garbage collects - dead pods and containers. +- If the `nodefs` filesystem meets the eviction thresholds, the kubelet garbage + collects dead pods and containers. + - If the `imagefs` filesystem meets the eviction thresholds, the kubelet deletes all unused images. -#### Without `imagefs` +#### With `imagefs` and `containerfs` -If the node only has a `nodefs` filesystem that meets eviction thresholds, -the kubelet frees up disk space in the following order: +If the node has a dedicated `containerfs` alongside the `imagefs` filesystem +configured for the container runtimes to use, then kubelet will attempt to +reclaim resources as follows: + +- If the `containerfs` filesystem meets the eviction thresholds, the kubelet + garbage collects dead pods and containers. -1. Garbage collect dead pods and containers -1. Delete unused images +- If the `imagefs` filesystem meets the eviction thresholds, the kubelet + deletes all unused images. ### Pod selection for kubelet eviction @@ -253,6 +319,7 @@ As a result, kubelet ranks and evicts pods in the following order: 1. `BestEffort` or `Burstable` pods where the usage exceeds requests. These pods are evicted based on their Priority and then by how much their usage level exceeds the request. + 1. `Guaranteed` pods and `Burstable` pods where the usage is less than requests are evicted last, based on their Priority. @@ -283,23 +350,38 @@ the Pods' relative priority to determine the eviction order, because inodes and requests. The kubelet sorts pods differently based on whether the node has a dedicated -`imagefs` filesystem: +`imagefs` or `containerfs` filesystem: -#### With `imagefs` +#### Without `imagefs` or `containerfs` (`nodefs` and `imagefs` use the same filesystem) {#without-imagefs} + +- If `nodefs` triggers evictions, the kubelet sorts pods based on their + total disk usage (`local volumes + logs and a writable layer of all containers`). + +#### With `imagefs` (`nodefs` and `imagefs` filesystems are separate) {#with-imagefs} -If `nodefs` is triggering evictions, the kubelet sorts pods based on `nodefs` -usage (`local volumes + logs of all containers`). +- If `nodefs` triggers evictions, the kubelet sorts pods based on `nodefs` + usage (`local volumes + logs of all containers`). -If `imagefs` is triggering evictions, the kubelet sorts pods based on the -writable layer usage of all containers. +- If `imagefs` triggers evictions, the kubelet sorts pods based on the + writable layer usage of all containers. -#### Without `imagefs` +#### With `imagesfs` and `containerfs` (`imagefs` and `containerfs` have been split) {#with-containersfs} -If `nodefs` is triggering evictions, the kubelet sorts pods based on their total -disk usage (`local volumes + logs & writable layer of all containers`) +- If `containerfs` triggers evictions, the kubelet sorts pods based on + `containerfs` usage (`local volumes + logs and a writable layer of all containers`). + +- If `imagefs` triggers evictions, the kubelet sorts pods based on the + `storage of images` rank, which represents the disk usage of a given image. ### Minimum eviction reclaim +{{}} +As of Kubernetes v{{< skew currentVersion >}}, you cannot set a custom value +for the `containerfs.available` metric. The configuration for this specific +metric will be set automatically to reflect values set for either the `nodefs` +or `imagefs`, depending on the configuration. +{{}} + In some cases, pod eviction only reclaims a small amount of the starved resource. This can lead to the kubelet repeatedly hitting the configured eviction thresholds and triggering multiple evictions. @@ -326,7 +408,8 @@ evictionMinimumReclaim: In this example, if the `nodefs.available` signal meets the eviction threshold, the kubelet reclaims the resource until the signal reaches the threshold of 1GiB, -and then continues to reclaim the minimum amount of 500MiB, until the available nodefs storage value reaches 1.5GiB. +and then continues to reclaim the minimum amount of 500MiB, until the available +nodefs storage value reaches 1.5GiB. Similarly, the kubelet tries to reclaim the `imagefs` resource until the `imagefs.available` value reaches `102Gi`, representing 102 GiB of available container image storage. If the amount From 8f1e1488642deb9905034d7ff8f0e82132b5f675 Mon Sep 17 00:00:00 2001 From: Joe Betz Date: Fri, 26 Jul 2024 19:53:58 -0400 Subject: [PATCH 51/82] KEP-4358: Custom Resource Field Selectors: Promote to Beta (#46980) * KEP-4358: Custom Resource Field Selectors: Promote to Beta * Update content/en/docs/reference/command-line-tools-reference/feature-gates/custom-resource-field-selectors.md Co-authored-by: Dipesh Rawat * Remove alpha level notes about enabling the feature * Update content/en/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions.md Co-authored-by: Tim Bannister * Update content/en/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions.md Co-authored-by: lakshmi prasuna <56723673+T-Lakshmi@users.noreply.github.com> * Update content/en/docs/concepts/extend-kubernetes/api-extension/custom-resources.md Co-authored-by: lakshmi prasuna <56723673+T-Lakshmi@users.noreply.github.com> --------- Co-authored-by: Dipesh Rawat Co-authored-by: Tim Bannister Co-authored-by: lakshmi prasuna <56723673+T-Lakshmi@users.noreply.github.com> --- .../api-extension/custom-resources.md | 9 +++------ .../custom-resource-field-selectors.md | 4 ++++ .../custom-resources/custom-resource-definitions.md | 13 +++++++------ 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/content/en/docs/concepts/extend-kubernetes/api-extension/custom-resources.md b/content/en/docs/concepts/extend-kubernetes/api-extension/custom-resources.md index 819dbc306b65f..7e9f85443e752 100644 --- a/content/en/docs/concepts/extend-kubernetes/api-extension/custom-resources.md +++ b/content/en/docs/concepts/extend-kubernetes/api-extension/custom-resources.md @@ -313,13 +313,10 @@ may also be used with field selectors when included in the `spec.versions[*].sel {{< feature-state feature_gate_name="CustomResourceFieldSelectors" >}} -You need to enable the `CustomResourceFieldSelectors` -[feature gate](/docs/reference/command-line-tools-reference/feature-gates/) to -use this behavior, which then applies to all CustomResourceDefinitions in your -cluster. - The `spec.versions[*].selectableFields` field of a {{< glossary_tooltip term_id="CustomResourceDefinition" text="CustomResourceDefinition" >}} may be used to -declare which other fields in a custom resource may be used in field selectors. +declare which other fields in a custom resource may be used in field selectors +with the feature of `CustomResourceFieldSelectors` +[feature gate](/docs/reference/command-line-tools-reference/feature-gates/) (This feature gate is enabled by default since Kubernetes v1.31). The following example adds the `.spec.color` and `.spec.size` fields as selectable fields. diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/custom-resource-field-selectors.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/custom-resource-field-selectors.md index 5ef021173f8de..77c579c748951 100644 --- a/content/en/docs/reference/command-line-tools-reference/feature-gates/custom-resource-field-selectors.md +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/custom-resource-field-selectors.md @@ -9,6 +9,10 @@ stages: - stage: alpha defaultValue: false fromVersion: "1.30" + toVersion: "1.30" + - stage: beta + defaultValue: true + fromVersion: "1.31" --- Enable `selectableFields` in the diff --git a/content/en/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions.md b/content/en/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions.md index 578843c0f60bf..b3c35664cc3f9 100644 --- a/content/en/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions.md +++ b/content/en/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions.md @@ -1686,13 +1686,14 @@ may also be used with field selectors when included in the `spec.versions[*].sel {{< feature-state feature_gate_name="CustomResourceFieldSelectors" >}} -You need to enable the `CustomResourceFieldSelectors` -[feature gate](/docs/reference/command-line-tools-reference/feature-gates/) to -use this behavior, which then applies to all CustomResourceDefinitions in your -cluster. - +For Kubernetes {{< skew currentVersion >}} the ability to define field selectors for +custom resources is available by default (enabled by default since Kubernetes v1.31); +you can disable it for your cluster by turning off the `CustomResourceFieldSelectors` +[feature gate](/docs/reference/command-line-tools-reference/feature-gates/). The `spec.versions[*].selectableFields` field of a {{< glossary_tooltip term_id="CustomResourceDefinition" text="CustomResourceDefinition" >}} may be used to -declare which other fields in a custom resource may be used in field selectors. +declare which other fields in a custom resource may be used in field selectors +with the feature of `CustomResourceFieldSelectors` +[feature gate](/docs/reference/command-line-tools-reference/feature-gates/) (This feature gate is enabled by default since Kubernetes v1.31). The following example adds the `.spec.color` and `.spec.size` fields as selectable fields. From 3b9ca420db5f9c8af94a13f6d6081d7865b6a28b Mon Sep 17 00:00:00 2001 From: Deepak Kinni Date: Fri, 26 Jul 2024 17:01:29 -0700 Subject: [PATCH 52/82] Doc update for Persistent Volumes Signed-off-by: Deepak Kinni --- .../concepts/storage/persistent-volumes.md | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/content/en/docs/concepts/storage/persistent-volumes.md b/content/en/docs/concepts/storage/persistent-volumes.md index da2bac2f1ba03..982fc30fffd19 100644 --- a/content/en/docs/concepts/storage/persistent-volumes.md +++ b/content/en/docs/concepts/storage/persistent-volumes.md @@ -253,12 +253,15 @@ However, the particular path specified in the custom recycler Pod template in th Finalizers can be added on a PersistentVolume to ensure that PersistentVolumes having `Delete` reclaim policy are deleted only after the backing storage are deleted. -The newly introduced finalizers `kubernetes.io/pv-controller` and -`external-provisioner.volume.kubernetes.io/finalizer` -are only added to dynamically provisioned volumes. +The finalizer `external-provisioner.volume.kubernetes.io/finalizer`(introduced +in v1.31) is added to both dynamically provisioned and statically provisioned +CSI volumes. -The finalizer `kubernetes.io/pv-controller` is added to in-tree plugin volumes. -The following is an example +The finalizer `kubernetes.io/pv-controller`(introduced in v1.31) is added to +dynamically provisioned in-tree plugin volumes and skipped for statically +provisioned in-tree plugin volumes. + +The following is an example of dynamically provisioned in-tree plugin volume: ```shell kubectl describe pv pvc-74a498d6-3929-47e8-8c02-078c1ece4d78 @@ -317,6 +320,11 @@ When the `CSIMigration{provider}` feature flag is enabled for a specific in-tree the `kubernetes.io/pv-controller` finalizer is replaced by the `external-provisioner.volume.kubernetes.io/finalizer` finalizer. +The finalizers ensure that the PV object is removed only after the volume is deleted +from the storage backend provided the reclaim policy of the PV is `Delete`. This +also ensures that the volume is deleted from storage backend irrespective of the +order of deletion of PV and PVC. + ### Reserving a PersistentVolume The control plane can [bind PersistentVolumeClaims to matching PersistentVolumes](#binding) From 5dab30d474bf12aaeb515bb857b49eba60c95392 Mon Sep 17 00:00:00 2001 From: Jordan Liggitt Date: Wed, 26 Jun 2024 14:12:51 -0400 Subject: [PATCH 53/82] KEP-4601: alpha docs --- .../docs/reference/access-authn-authz/node.md | 7 +++ .../validating-admission-policy.md | 1 + .../reference/access-authn-authz/webhook.md | 46 ++++++++++++++++++- .../authorize-node-with-selectors.md | 14 ++++++ .../feature-gates/authorize-with-selectors.md | 17 +++++++ content/en/docs/reference/using-api/cel.md | 12 +++++ 6 files changed, 95 insertions(+), 2 deletions(-) create mode 100644 content/en/docs/reference/command-line-tools-reference/feature-gates/authorize-node-with-selectors.md create mode 100644 content/en/docs/reference/command-line-tools-reference/feature-gates/authorize-with-selectors.md diff --git a/content/en/docs/reference/access-authn-authz/node.md b/content/en/docs/reference/access-authn-authz/node.md index f47a481170897..d39b404c0c80b 100644 --- a/content/en/docs/reference/access-authn-authz/node.md +++ b/content/en/docs/reference/access-authn-authz/node.md @@ -27,6 +27,13 @@ Read operations: * secrets, configmaps, persistent volume claims and persistent volumes related to pods bound to the kubelet's node +{{< feature-state feature_gate_name="AuthorizeNodeWithSelectors" >}} + +When the `AuthorizeNodeWithSelectors` feature is enabled +(along with the pre-requisite `AuthorizeWithSelectors` feature), +kubelets are only allowed to read their own Node objects, +and are only allowed to read pods bound to their node. + Write operations: * nodes and node status (enable the `NodeRestriction` admission plugin to limit diff --git a/content/en/docs/reference/access-authn-authz/validating-admission-policy.md b/content/en/docs/reference/access-authn-authz/validating-admission-policy.md index 2d0ae273442ad..925d0e5c0c91e 100644 --- a/content/en/docs/reference/access-authn-authz/validating-admission-policy.md +++ b/content/en/docs/reference/access-authn-authz/validating-admission-policy.md @@ -283,6 +283,7 @@ variables as well as some other useful variables: The value is null if the incoming object is cluster-scoped. - `authorizer` - A CEL Authorizer. May be used to perform authorization checks for the principal (authenticated user) of the request. See + [AuthzSelectors](https://pkg.go.dev/k8s.io/apiserver/pkg/cel/library#AuthzSelectors) and [Authz](https://pkg.go.dev/k8s.io/apiserver/pkg/cel/library#Authz) in the Kubernetes CEL library documentation for more details. - `authorizer.requestResource` - A shortcut for an authorization check configured with the request diff --git a/content/en/docs/reference/access-authn-authz/webhook.md b/content/en/docs/reference/access-authn-authz/webhook.md index d5362e0acf5ba..cb91ac34c60a3 100644 --- a/content/en/docs/reference/access-authn-authz/webhook.md +++ b/content/en/docs/reference/access-authn-authz/webhook.md @@ -164,6 +164,46 @@ Access to non-resource paths are sent as: } ``` +{{< feature-state feature_gate_name="AuthorizeWithSelectors" >}} + +With the `AuthorizeWithSelectors` feature enabled, field and label selectors in the request +are passed to the authorization webhook. The webhook can make authorization decisions +informed by the scoped field and label selectors, if it wishes. + +The [SubjectAccessReview API documentation](/docs/reference/kubernetes-api/authorization-resources/subject-access-review-v1/) +gives guidelines for how these fields should be interpreted and handled by authorization webhooks, +specifically using the parsed requirements rather than the raw selector strings, +and how to handle unrecognized operators safely. + +```json +{ + "apiVersion": "authorization.k8s.io/v1beta1", + "kind": "SubjectAccessReview", + "spec": { + "resourceAttributes": { + "verb": "list", + "group": "", + "resource": "pods", + "fieldSelector": { + "requirements": [ + {"key":"spec.nodeName", "operator":"In", "values":["mynode"]} + ] + }, + "labelSelector": { + "requirements": [ + {"key":"example.com/mykey", "operator":"In", "values":["myvalue"]} + ] + } + }, + "user": "jane", + "group": [ + "group1", + "group2" + ] + } +} +``` + Non-resource paths include: `/api`, `/apis`, `/metrics`, `/logs`, `/debug`, `/healthz`, `/livez`, `/openapi/v2`, `/readyz`, and `/version.` Clients require access to `/api`, `/api/*`, `/apis`, `/apis/*`, @@ -171,6 +211,8 @@ and `/version` to discover what resources and versions are present on the server Access to other non-resource paths can be disallowed without restricting access to the REST api. -For further documentation refer to the authorization.v1beta1 API objects and -[webhook.go](https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/webhook.go). +For further information, refer to the +[SubjectAccessReview API documentation](/docs/reference/kubernetes-api/authorization-resources/subject-access-review-v1/) +and +[webhook.go implementation](https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/webhook.go). diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/authorize-node-with-selectors.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/authorize-node-with-selectors.md new file mode 100644 index 0000000000000..9c09c59f97290 --- /dev/null +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/authorize-node-with-selectors.md @@ -0,0 +1,14 @@ +--- +title: AuthorizeNodeWithSelectors +content_type: feature_gate +_build: + list: never + render: false + +stages: + - stage: alpha + defaultValue: false + fromVersion: "1.31" +--- +Make the [Node authorizer](/docs/reference/access-authn-authz/node/) use fine-grained selector authorization. +Requires `AuthorizeWithSelectors` to be enabled. diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/authorize-with-selectors.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/authorize-with-selectors.md new file mode 100644 index 0000000000000..4626d486b174c --- /dev/null +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/authorize-with-selectors.md @@ -0,0 +1,17 @@ +--- +title: AuthorizeWithSelectors +content_type: feature_gate +_build: + list: never + render: false + +stages: + - stage: alpha + defaultValue: false + fromVersion: "1.31" +--- +Allows authorization to use field and label selectors. +Enables `fieldSelector` and `labelSelector` fields in the [SubjectAccessReview API](/docs/reference/kubernetes-api/authorization-resources/subject-access-review-v1/), +passes field and label selector information to [authorization webhooks](/docs/reference/access-authn-authz/webhook/), +enables `fieldSelector` and `labelSelector` functions in the [authorizer CEL library](https://pkg.go.dev/k8s.io/apiserver/pkg/cel/library#AuthzSelectors), +and enables checking `fieldSelector` and `labelSelector` fields in [authorization webhook `matchConditions`](/docs/reference/access-authn-authz/authorization/#using-configuration-file-for-authorization). \ No newline at end of file diff --git a/content/en/docs/reference/using-api/cel.md b/content/en/docs/reference/using-api/cel.md index 28adf0bb07912..f8c4ed4bcf039 100644 --- a/content/en/docs/reference/using-api/cel.md +++ b/content/en/docs/reference/using-api/cel.md @@ -200,7 +200,19 @@ To perform an authorization check for a service account: | `authorizer.serviceAccount('default', 'myserviceaccount').resource('deployments').check('delete').allowed()` | Checks if the service account is authorized to delete deployments. | {{< /table >}} +{{< feature-state state="alpha" for_k8s_version="v1.31" >}} + +With the alpha `AuthorizeWithSelectors` feature enabled, field and label selectors can be added to authorization checks. + +{{< table caption="Examples of CEL expressions using selector authorization functions" >}} +| CEL Expression | Purpose | +|--------------------------------------------------------------------------------------------------------------|------------------------------------------------| +| `authorizer.group('').resource('pods').fieldSelector('spec.nodeName=mynode').check('list').allowed()` | Returns true if the principal (user or service account) is allowed to list pods with the field selector `spec.nodeName=mynode`. | +| `authorizer.group('').resource('pods').labelSelector('example.com/mylabel=myvalue').check('list').allowed()` | Returns true if the principal (user or service account) is allowed to list pods with the label selector `example.com/mylabel=myvalue`. | +{{< /table >}} + See the [Kubernetes Authz library](https://pkg.go.dev/k8s.io/apiserver/pkg/cel/library#Authz) +and [Kubernetes AuthzSelectors library](https://pkg.go.dev/k8s.io/apiserver/pkg/cel/library#AuthzSelectors) godoc for more information. ### Kubernetes quantity library From 7bf346a1f6c6a68ac46622b2f5319fa8b1c0ed76 Mon Sep 17 00:00:00 2001 From: Michal Wozniak Date: Fri, 26 Jul 2024 10:36:55 +0200 Subject: [PATCH 54/82] Address review remarks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Filip Křepinský Co-authored-by: Shannon Kularathna Co-authored-by: Tim Bannister --- .../concepts/workloads/controllers/job.md | 106 ++++++++++-------- .../en/docs/tasks/job/pod-failure-policy.md | 13 ++- 2 files changed, 69 insertions(+), 50 deletions(-) diff --git a/content/en/docs/concepts/workloads/controllers/job.md b/content/en/docs/concepts/workloads/controllers/job.md index 88379f0a64357..7c314663f0b7c 100644 --- a/content/en/docs/concepts/workloads/controllers/job.md +++ b/content/en/docs/concepts/workloads/controllers/job.md @@ -438,15 +438,21 @@ kubectl get -o yaml job job-backoff-limit-per-index-example succeeded: 5 # 1 succeeded pod for each of 5 succeeded indexes failed: 10 # 2 failed pods (1 retry) for each of 5 failed indexes conditions: + - message: Job has failed indexes + reason: FailedIndexes + status: "True" + type: FailureTarget - message: Job has failed indexes reason: FailedIndexes status: "True" type: Failed ``` -Note that, since v1.31, you will also observe in the status the `FailureTarget` -Job condition, with the same `reason` and `message` as for the the `Failed` -condition (see also [Job termination and cleanup](#job-termination-and-cleanup)). +The Job controller adds the `FailureTarget` Job condition to trigger +[Job termination and cleanup](#job-termination-and-cleanup). The +`Failed` condition has the same values for `reason` and `message` as the +`FailureTarget` Job condition, but is added to the Job at the moment all Pods +are terminated; for details see [Termination of Job pods](#termination-of-job-pods). Additionally, you may want to use the per-index backoff along with a [pod failure policy](#pod-failure-policy). When using @@ -560,7 +566,7 @@ to `podReplacementPolicy: Failed`. For more information, see [Pod replacement po When you use the `podFailurePolicy`, and the Job fails due to the pod matching the rule with the `FailJob` action, then the Job controller triggers the Job termination process by adding the `FailureTarget` condition. -See [Job termination and cleanup](#job-termination-and-cleanup) for more details. +For more details, see [Job termination and cleanup](#job-termination-and-cleanup). ## Success policy {#success-policy} @@ -670,42 +676,64 @@ and `.spec.backoffLimit` result in a permanent Job failure that requires manual ### Terminal Job conditions -A Job has two possible terminal states, it ends up either succeeded, or failed, -and these states are reflected by the presence of the Job conditions `Complete` -or `Failed`, respectively. +A Job has two possible terminal states, each of which has a corresponding Job +condition: +* Succeeded: Job condition `Complete` +* Failed: Job condition `Failed`. + +The possible reasons for a Job failure: +- The number of Pod failures exceeded the specified `.spec.backoffLimit` in the Job + specification. For details, see [Pod backoff failure policy](#pod-backoff-failure-policy). +- The Job runtime exceeded the specified `.spec.activeDeadlineSeconds` +- An indexed Job that used `.spec.backoffLimitPerIndex` has failed indexes. + For details, see [Backoff limit per index](#backoff-limit-per-index). +- The number of failed indexes in the Job exceeded the specified + `spec.maxFailedIndexes`. For details, see [Backoff limit per index](#backoff-limit-per-index) +- A failed Pod matches a rule in `.spec.podFailurePolicy` that has the `FailJob` + action. For details about how Pod failure policy rules might affect failure + evaluation, see [Pod failure policy](#pod-failure-policy). + +The possible reasons for a Job success: +- The number of succeeded Pods reached the specified `.spec.completions` +- The criteria specified in `.spec.successPolicy` are met. For details, see + [Success policy](#success-policy). + +In Kubernetes v1.31 and later the Job controller delays the addition of the +terminal conditions,`Failed` or `Succeeded`, until all pods are terminated. -The failure scenarios encompass: -- the `.spec.backoffLimit` -- the `.spec.activeDeadlineSeconds` is exceeded -- the `.spec.backoffLimitPerIndex` is exceeded (see [Backoff limit per index](#backoff-limit-per-index)) -- the Pod matches the Job Pod Failure Policy rule with the `FailJob` action (see more [Pod failure policy](#pod-failure-policy)) +{{< note >}} +In Kubernetes v1.30 and earlier, Job terminal conditions were added when the Job +termination process is triggered, and all Pod finalizers are removed, but some +pods may still remain running/terminating at that point in time. -The success scenarios encompass: -- the `.spec.completions` is reached -- the criteria specified by the Job Success Policy are met (see more [Success policy](#success-policy)) +The change of the behavior is activated by enablement of the `JobManagedBy` or +`JobPodReplacementPolicy` (enabled by default) +[feature gates](/docs/reference/command-line-tools-reference/feature-gates/). +{{< /note >}} ### Termination of Job pods -Prior to v1.31 the Job terminal conditions are added when the Job termination -process is triggered, and all Pod finalizers are removed, but some pods may -still remain running at that point in time. +The Job controller adds the `FailureTarget` condition or the `SuccessCriteriaMet` +condition to the Job to trigger Pod termination after a Job meets either the +success or failure criteria. -Since v1.31, when you enable either the `JobManagedBy` or -`JobPodReplacementPolicy` (enabled by default) -[feature gate](/docs/reference/command-line-tools-reference/feature-gates/), the -Job controller awaits for termination of all pods before adding a condition -indicating that the Job is finished (either `Complete` or `Failed`). +Factors like `terminationGracePeriodSeconds` might increase the amount of time +from the moment that the Job controller adds the `FailureTarget` condition or the +`SuccessCriteriaMet` condition to the moment that all of the Job Pods terminate +and the Job controller adds a [terminal condition](#terminal-job-conditions) +(`Failed` or `Complete`). -Note that, the process of terminating all pods may take a substantial amount -of time, depending on a Pod's `terminationGracePeriodSeconds` (see -[Pod termination](#docs/concepts/workloads/pods/pod-lifecycle/#pod-termination)), -and thus adding the terminal Job condition, even if the fate of the Job is -already determined. +You can use the `FailureTarget` or the `SuccessCriteriaMet` condition to evaluate +whether the Job has failed or succeeded without having to wait for the controller +to add a terminal condition. -If you want to know the fate of the Job as soon as determined you can use, -since v1.31, the `FailureTarget` and `SuccessCriteriaMet` conditions, which -cover all scenarios in which Job controller triggers the Job termination process -(see [Terminal Job conditions](#terminal-job-conditions)). +{{< note >}} +For example, you can use the `FailureTarget` condition to quickly decide whether +to create a replacement Job, but it could result in Pods from the failing and +replacement Jobs running at the same time for a while. Thus, if your cluster +capacity is limited, you may prefer to wait for the `Failed` condition before +creating the replacement Job. +{{< /note >}} ## Clean up finished jobs automatically @@ -1111,13 +1139,6 @@ status: terminating: 3 # three Pods are terminating and have not yet reached the Failed phase ``` -{{< note >}} -Since v1.31, when you enable the `JobPodReplacementPolicy` -[feature gate](/docs/reference/command-line-tools-reference/feature-gates/) -(enabled by default), the Job controller awaits for termination of all pods -before marking a Job as terminal (see [Termination of Job Pods](#termination-of-job-pods)). -{{< /note >}} - ### Delegation of managing a Job object to external controller {{< feature-state feature_gate_name="JobManagedBy" >}} @@ -1162,13 +1183,6 @@ after the operation: the built-in Job controller and the external controller indicated by the field value. {{< /warning >}} -{{< note >}} -Since v1.31, when you enable the `JobManagedBy` -[feature gate](/docs/reference/command-line-tools-reference/feature-gates/), -the Job controller awaits for termination of all pods before marking a Job as -terminal (see [Termination of Job Pods](#termination-of-job-pods)). -{{< /note >}} - ## Alternatives ### Bare Pods diff --git a/content/en/docs/tasks/job/pod-failure-policy.md b/content/en/docs/tasks/job/pod-failure-policy.md index ee1bbb3ac8929..d9dc1833472fb 100644 --- a/content/en/docs/tasks/job/pod-failure-policy.md +++ b/content/en/docs/tasks/job/pod-failure-policy.md @@ -53,10 +53,15 @@ After around 30s the entire Job should be terminated. Inspect the status of the kubectl get jobs -l job-name=job-pod-failure-policy-failjob -o yaml ``` -In the Job status, see a job `Failed` condition with the field `reason` -equal `PodFailurePolicy`. Additionally, the `message` field contains a -more detailed information about the Job termination, such as: -`Container main for pod default/job-pod-failure-policy-failjob-8ckj8 failed with exit code 42 matching FailJob rule at index 0`. +In the Job status, the following conditions display: +- `FailureTarget` condition: has a `reason` field set to `PodFailurePolicy` and + a `message` field with more information about the termination, like + `Container main for pod default/job-pod-failure-policy-failjob-8ckj8 failed with exit code 42 matching FailJob rule at index 0`. + The Job controller adds this condition as soon as the Job is considered a failure. + For details, see [Termination of Job Pods](/docs/concepts/workloads/controllers/job/#termination-of-job-pods). +- `Failed` condition: same `reason` and `message` as the `FailureTarget` + condition. The Job controller adds this condition after all of the Job's Pods + are terminated. For comparison, if the Pod failure policy was disabled it would take 6 retries of the Pod, taking at least 2 minutes. From ffeb6beab01c29029a10253c1375e71e3492a628 Mon Sep 17 00:00:00 2001 From: Drew Sirenko <68304519+AndrewSirenko@users.noreply.github.com> Date: Tue, 11 Jun 2024 15:11:51 -0400 Subject: [PATCH 55/82] KEP-3751 Beta promotion for VolumeAttributeClass --- .../storage/volume-attributes-classes.md | 16 ++++++++-------- .../feature-gates/volume-attributes-class.md | 6 +++++- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/content/en/docs/concepts/storage/volume-attributes-classes.md b/content/en/docs/concepts/storage/volume-attributes-classes.md index 69b4e41289237..29da23d372b81 100644 --- a/content/en/docs/concepts/storage/volume-attributes-classes.md +++ b/content/en/docs/concepts/storage/volume-attributes-classes.md @@ -8,7 +8,7 @@ weight: 40 --- -{{< feature-state for_k8s_version="v1.29" state="alpha" >}} +{{< feature-state feature_gate_name="VolumeAttributesClass" >}} This page assumes that you are familiar with [StorageClasses](/docs/concepts/storage/storage-classes/), [volumes](/docs/concepts/storage/volumes/) and [PersistentVolumes](/docs/concepts/storage/persistent-volumes/) @@ -18,11 +18,11 @@ in Kubernetes. A VolumeAttributesClass provides a way for administrators to describe the mutable "classes" of storage they offer. Different classes might map to different quality-of-service levels. -Kubernetes itself is unopinionated about what these classes represent. +Kubernetes itself is un-opinionated about what these classes represent. -This is an alpha feature and disabled by default. +This is a beta feature and disabled by default. -If you want to test the feature whilst it's alpha, you need to enable the `VolumeAttributesClass` +If you want to test the feature whilst it's beta, you need to enable the `VolumeAttributesClass` [feature gate](/docs/reference/command-line-tools-reference/feature-gates/) for the kube-controller-manager and the kube-apiserver. You use the `--feature-gates` command line argument: ``` @@ -45,7 +45,7 @@ While the name of a VolumeAttributesClass object in a `PersistentVolumeClaim` is ```yaml -apiVersion: storage.k8s.io/v1alpha1 +apiVersion: storage.k8s.io/v1beta1 kind: VolumeAttributesClass metadata: name: silver @@ -74,7 +74,7 @@ Each VolumeAttributesClass has a resizer that determines what volume plugin is u The modifying volume feature support for VolumeAttributesClass is implemented in [kubernetes-csi/external-resizer](https://github.com/kubernetes-csi/external-resizer). -For example, a existing PersistentVolumeClaim is using a VolumeAttributesClass named silver: +For example, an existing PersistentVolumeClaim is using a VolumeAttributesClass named silver: ```yaml apiVersion: v1 @@ -91,7 +91,7 @@ A new VolumeAttributesClass gold is available in the cluster: ```yaml -apiVersion: storage.k8s.io/v1alpha1 +apiVersion: storage.k8s.io/v1beta1 kind: VolumeAttributesClass metadata: name: gold @@ -128,4 +128,4 @@ the parameters may be used depends on the CSI driver implementation. Please refer to the related CSI driver documentation for more details. There can be at most 512 parameters defined for a VolumeAttributesClass. -The total length of the parameters object including its keys and values cannot exceed 256 KiB. \ No newline at end of file +The total length of the parameters object including its keys and values cannot exceed 256 KiB. diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/volume-attributes-class.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/volume-attributes-class.md index b4d5f49946918..ffef10bc8284e 100644 --- a/content/en/docs/reference/command-line-tools-reference/feature-gates/volume-attributes-class.md +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/volume-attributes-class.md @@ -6,9 +6,13 @@ _build: render: false stages: - - stage: alpha + - stage: alpha defaultValue: false fromVersion: "1.29" + toVersion: "1.30" + - stage: beta + defaultValue: false + fromVersion: "1.31" --- Enable support for VolumeAttributesClasses. See [Volume Attributes Classes](/docs/concepts/storage/volume-attributes-classes/) From 8c25433782953a16afe14184d55ae47d7d00bc8c Mon Sep 17 00:00:00 2001 From: Drew Sirenko <68304519+AndrewSirenko@users.noreply.github.com> Date: Thu, 25 Jul 2024 10:40:13 -0400 Subject: [PATCH 56/82] Add instructions to enable API group to VolumeAttributesClass --- .../en/docs/concepts/storage/volume-attributes-classes.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/content/en/docs/concepts/storage/volume-attributes-classes.md b/content/en/docs/concepts/storage/volume-attributes-classes.md index 29da23d372b81..7b8db09fc3e8a 100644 --- a/content/en/docs/concepts/storage/volume-attributes-classes.md +++ b/content/en/docs/concepts/storage/volume-attributes-classes.md @@ -29,6 +29,12 @@ If you want to test the feature whilst it's beta, you need to enable the `Volume --feature-gates="...,VolumeAttributesClass=true" ``` +You will also have to enable the `storage.k8s.io/v1beta1` API group through the `kube-apiserver` [runtime-config](https://kubernetes.io/docs/tasks/administer-cluster/enable-disable-api/). You use the following command line argument: + +``` +--runtime-config=storage.k8s.io/v1beta1=true +``` + You can also only use VolumeAttributesClasses with storage backed by {{< glossary_tooltip text="Container Storage Interface" term_id="csi" >}}, and only where the relevant CSI driver implements the `ModifyVolume` API. From 42fda96d073450f0362b2785466f4ca700e95ed1 Mon Sep 17 00:00:00 2001 From: Sohan Kunkerkar Date: Mon, 29 Jul 2024 10:49:54 -0400 Subject: [PATCH 57/82] KEP-4033: small beta updates Signed-off-by: Peter Hunt --- .../feature-gates/kubelet-cgroup-driver-from-cri.md | 4 ++++ .../production-environment/container-runtimes.md | 12 ++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/kubelet-cgroup-driver-from-cri.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/kubelet-cgroup-driver-from-cri.md index 8de9e1a118b0c..e374828a268cd 100644 --- a/content/en/docs/reference/command-line-tools-reference/feature-gates/kubelet-cgroup-driver-from-cri.md +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/kubelet-cgroup-driver-from-cri.md @@ -9,6 +9,10 @@ stages: - stage: alpha defaultValue: false fromVersion: "1.28" + toVersion: "1.30" + - stage: beta + defaultValue: true + fromVersion: "1.31" --- Enable detection of the kubelet cgroup driver configuration option from the {{}}. diff --git a/content/en/docs/setup/production-environment/container-runtimes.md b/content/en/docs/setup/production-environment/container-runtimes.md index 93c5f92e3da64..0018dd8f0b15e 100644 --- a/content/en/docs/setup/production-environment/container-runtimes.md +++ b/content/en/docs/setup/production-environment/container-runtimes.md @@ -140,12 +140,6 @@ Starting with v1.22 and later, when creating a cluster with kubeadm, if the user the `cgroupDriver` field under `KubeletConfiguration`, kubeadm defaults it to `systemd`. {{< /note >}} -In Kubernetes v1.28, with the `KubeletCgroupDriverFromCRI` -[feature gate](/docs/reference/command-line-tools-reference/feature-gates/) -enabled and a container runtime that supports the `RuntimeConfig` CRI RPC, -the kubelet automatically detects the appropriate cgroup driver from the runtime, -and ignores the `cgroupDriver` setting within the kubelet configuration. - If you configure `systemd` as the cgroup driver for the kubelet, you must also configure `systemd` as the cgroup driver for the container runtime. Refer to the documentation for your container runtime for instructions. For example: @@ -153,6 +147,12 @@ the documentation for your container runtime for instructions. For example: * [containerd](#containerd-systemd) * [CRI-O](#cri-o) +In Kubernetes {{< skew currentVersion >}}, with the `KubeletCgroupDriverFromCRI` +[feature gate](/docs/reference/command-line-tools-reference/feature-gates/) +enabled and a container runtime that supports the `RuntimeConfig` CRI RPC, +the kubelet automatically detects the appropriate cgroup driver from the runtime, +and ignores the `cgroupDriver` setting within the kubelet configuration. + {{< caution >}} Changing the cgroup driver of a Node that has joined a cluster is a sensitive operation. If the kubelet has created Pods using the semantics of one cgroup driver, changing the container From f2b97990618911f7e12ed6ac815eaccd0350c0ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Wo=C5=BAniak?= Date: Mon, 29 Jul 2024 18:06:58 +0200 Subject: [PATCH 58/82] Review remarks 2 --- .../concepts/workloads/controllers/job.md | 47 ++++++++++--------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/content/en/docs/concepts/workloads/controllers/job.md b/content/en/docs/concepts/workloads/controllers/job.md index 7c314663f0b7c..9ee63e2cc8b7f 100644 --- a/content/en/docs/concepts/workloads/controllers/job.md +++ b/content/en/docs/concepts/workloads/controllers/job.md @@ -449,10 +449,10 @@ kubectl get -o yaml job job-backoff-limit-per-index-example ``` The Job controller adds the `FailureTarget` Job condition to trigger -[Job termination and cleanup](#job-termination-and-cleanup). The -`Failed` condition has the same values for `reason` and `message` as the -`FailureTarget` Job condition, but is added to the Job at the moment all Pods -are terminated; for details see [Termination of Job pods](#termination-of-job-pods). +[Job termination and cleanup](#job-termination-and-cleanup). When all of the +Job Pods are terminated, the Job controller adds the `Failed` condition +with the same values for `reason` and `message` as the `FailureTarget` Job +condition. For details, see [Termination of Job Pods](#termination-of-job-pods). Additionally, you may want to use the per-index backoff along with a [pod failure policy](#pod-failure-policy). When using @@ -679,9 +679,9 @@ and `.spec.backoffLimit` result in a permanent Job failure that requires manual A Job has two possible terminal states, each of which has a corresponding Job condition: * Succeeded: Job condition `Complete` -* Failed: Job condition `Failed`. +* Failed: Job condition `Failed` -The possible reasons for a Job failure: +Jobs fail for the following reasons: - The number of Pod failures exceeded the specified `.spec.backoffLimit` in the Job specification. For details, see [Pod backoff failure policy](#pod-backoff-failure-policy). - The Job runtime exceeded the specified `.spec.activeDeadlineSeconds` @@ -693,23 +693,23 @@ The possible reasons for a Job failure: action. For details about how Pod failure policy rules might affect failure evaluation, see [Pod failure policy](#pod-failure-policy). -The possible reasons for a Job success: +Jobs succeed for the following reasons: - The number of succeeded Pods reached the specified `.spec.completions` - The criteria specified in `.spec.successPolicy` are met. For details, see [Success policy](#success-policy). In Kubernetes v1.31 and later the Job controller delays the addition of the -terminal conditions,`Failed` or `Succeeded`, until all pods are terminated. +terminal conditions,`Failed` or `Complete`, until all of the Job Pods are terminated. -{{< note >}} -In Kubernetes v1.30 and earlier, Job terminal conditions were added when the Job -termination process is triggered, and all Pod finalizers are removed, but some -pods may still remain running/terminating at that point in time. +In Kubernetes v1.30 and earlier, the Job controller added the `Complete` or the +`Failed` Job terminal conditions as soon as the Job termination process was +triggered and all Pod finalizers were removed. However, some Pods would still +be running or terminating at the moment that the terminal condition was added. -The change of the behavior is activated by enablement of the `JobManagedBy` or -`JobPodReplacementPolicy` (enabled by default) +In Kubernetes v1.31 and later, the controller only adds the Job terminal conditions +_after_ all of the Pods are terminated. You can enable this behavior by using the +`JobManagedBy` or the `JobPodReplacementPolicy` (enabled by default) [feature gates](/docs/reference/command-line-tools-reference/feature-gates/). -{{< /note >}} ### Termination of Job pods @@ -727,13 +727,16 @@ You can use the `FailureTarget` or the `SuccessCriteriaMet` condition to evaluat whether the Job has failed or succeeded without having to wait for the controller to add a terminal condition. -{{< note >}} -For example, you can use the `FailureTarget` condition to quickly decide whether -to create a replacement Job, but it could result in Pods from the failing and -replacement Jobs running at the same time for a while. Thus, if your cluster -capacity is limited, you may prefer to wait for the `Failed` condition before -creating the replacement Job. -{{< /note >}} +For example, you might want to decide when to create a replacement Job +that replaces a failed Job. If you replace the failed Job when the `FailureTarget` +condition appears, your replacement Job runs sooner, but could result in Pods +from the failed and the replacement Job running at the same time, using +extra compute resources. + +Alternatively, if your cluster has limited resource capacity, you could choose to +wait until the `Failed` condition appears on the Job, which would delay your +replacement Job but would ensure that you conserve resources by waiting +until all of the failed Pods are removed. ## Clean up finished jobs automatically From 97899e05117e37783d53e13b87dd9023c2995105 Mon Sep 17 00:00:00 2001 From: PannagaRamamanohara Date: Wed, 17 Jul 2024 16:00:51 -0400 Subject: [PATCH 59/82] update LocalStorageCapacityIsolationFSQuotaMonitoring to beta Signed-off-by: PannagaRamamanohara --- .../manage-resources-containers.md | 21 ++++++++++++++++--- ...-capacity-isolation-fs-quota-monitoring.md | 17 ++++++++------- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/content/en/docs/concepts/configuration/manage-resources-containers.md b/content/en/docs/concepts/configuration/manage-resources-containers.md index 4d21ec995051f..de5babde9c3c0 100644 --- a/content/en/docs/concepts/configuration/manage-resources-containers.md +++ b/content/en/docs/concepts/configuration/manage-resources-containers.md @@ -463,7 +463,7 @@ that file but the kubelet does not categorize the space as in use. {{% /tab %}} {{% tab name="Filesystem project quota" %}} -{{< feature-state for_k8s_version="v1.15" state="alpha" >}} +{{< feature-state feature_gate_name="LocalStorageCapacityIsolationFSQuotaMonitoring" >}} Project quotas are an operating-system level feature for managing storage use on filesystems. With Kubernetes, you can enable project @@ -489,13 +489,21 @@ If a file is created and deleted, but has an open file descriptor, it continues to consume space. Quota tracking records that space accurately whereas directory scans overlook the storage used by deleted files. +To use quotas to track a pod's resource usage, the pod must be in +a user namespace. Within user namespaces, the kernel restricts changes +to projectIDs on the filesystem, ensuring the reliability of storage +metrics calculated by quotas. + If you want to use project quotas, you should: * Enable the `LocalStorageCapacityIsolationFSQuotaMonitoring=true` [feature gate](/docs/reference/command-line-tools-reference/feature-gates/) using the `featureGates` field in the - [kubelet configuration](/docs/reference/config-api/kubelet-config.v1beta1/) - or the `--feature-gates` command line flag. + [kubelet configuration](/docs/reference/config-api/kubelet-config.v1beta1/). + +* Ensure the `UserNamespacesSupport` + [feature gate](/docs/reference/command-line-tools-reference/feature-gates/) + is enabled, and that the kernel, CRI implementation and OCI runtime support user namespaces. * Ensure that the root filesystem (or optional runtime filesystem) has project quotas enabled. All XFS filesystems support project quotas. @@ -511,6 +519,13 @@ If you want to use project quotas, you should: mounted with project quotas enabled. For both XFS and ext4fs, the mount option is named `prjquota`. + +If you don't want to use project quotas, you should: + +* Disable the `LocalStorageCapacityIsolationFSQuotaMonitoring` + [feature gate](/docs/reference/command-line-tools-reference/feature-gates/) + using the `featureGates` field in the + [kubelet configuration](/docs/reference/config-api/kubelet-config.v1beta1/). {{% /tab %}} {{< /tabs >}} diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/local-storage-capacity-isolation-fs-quota-monitoring.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/local-storage-capacity-isolation-fs-quota-monitoring.md index ee25d6ed424e6..3449033dc7b18 100644 --- a/content/en/docs/reference/command-line-tools-reference/feature-gates/local-storage-capacity-isolation-fs-quota-monitoring.md +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/local-storage-capacity-isolation-fs-quota-monitoring.md @@ -9,11 +9,14 @@ stages: - stage: alpha defaultValue: false fromVersion: "1.15" + toVersion: "1.30" + - stage: beta + defaultValue: false + fromVersion: "1.31" --- -When `LocalStorageCapacityIsolation` -is enabled for -[local ephemeral storage](/docs/concepts/configuration/manage-resources-containers/) -and the backing filesystem for [emptyDir volumes](/docs/concepts/storage/volumes/#emptydir) -supports project quotas and they are enabled, use project quotas to monitor -[emptyDir volume](/docs/concepts/storage/volumes/#emptydir) storage consumption rather than -filesystem walk for better performance and accuracy. +When `LocalStorageCapacityIsolation` +is enabled for +[local ephemeral storage](/docs/concepts/configuration/manage-resources-containers/), +the backing filesystem for [emptyDir volumes](/docs/concepts/storage/volumes/#emptydir) supports project quotas, +and `UserNamespacesSupport` is enabled, +project quotas are used to monitor `emptyDir` volume storage consumption rather than using filesystem walk, ensuring better performance and accuracy. \ No newline at end of file From 45a47d170f226adadf07aae7927ace68b844d73e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Wo=C5=BAniak?= Date: Mon, 29 Jul 2024 20:56:08 +0200 Subject: [PATCH 60/82] Graduate Job Pod Failure Policy to stable (#46807) * Graduate Job Pod Failure Policy to stable * Update content/en/docs/concepts/workloads/controllers/job.md Co-authored-by: Shannon Kularathna * Break the line after link --------- Co-authored-by: Shannon Kularathna --- .../docs/concepts/workloads/controllers/job.md | 16 ++-------------- .../docs/concepts/workloads/pods/disruptions.md | 14 ++++---------- .../concepts/workloads/pods/pod-lifecycle.md | 3 +-- .../feature-gates/job-pod-failure-policy.md | 4 ++++ .../feature-gates/pod-disruption-conditions.md | 4 ++++ content/en/docs/tasks/job/pod-failure-policy.md | 3 --- 6 files changed, 15 insertions(+), 29 deletions(-) diff --git a/content/en/docs/concepts/workloads/controllers/job.md b/content/en/docs/concepts/workloads/controllers/job.md index 2fb703da4822d..d24e91d2feeff 100644 --- a/content/en/docs/concepts/workloads/controllers/job.md +++ b/content/en/docs/concepts/workloads/controllers/job.md @@ -344,9 +344,7 @@ sometimes be started twice. If you do specify `.spec.parallelism` and `.spec.completions` both greater than 1, then there may be multiple pods running at once. Therefore, your pods must also be tolerant of concurrency. -When the [feature gates](/docs/reference/command-line-tools-reference/feature-gates/) -`PodDisruptionConditions` and `JobPodFailurePolicy` are both enabled, -and the `.spec.podFailurePolicy` field is set, the Job controller does not consider a terminating +If you specify the `.spec.podFailurePolicy` field, the Job controller does not consider a terminating Pod (a pod that has a `.metadata.deletionTimestamp` field set) as a failure until that Pod is terminal (its `.status.phase` is `Failed` or `Succeeded`). However, the Job controller creates a replacement Pod as soon as the termination becomes apparent. Once the @@ -451,17 +449,7 @@ avoid unnecessary retries within an index. ### Pod failure policy {#pod-failure-policy} -{{< feature-state for_k8s_version="v1.26" state="beta" >}} - -{{< note >}} -You can only configure a Pod failure policy for a Job if you have the -`JobPodFailurePolicy` [feature gate](/docs/reference/command-line-tools-reference/feature-gates/) -enabled in your cluster. Additionally, it is recommended -to enable the `PodDisruptionConditions` feature gate in order to be able to detect and handle -Pod disruption conditions in the Pod failure policy (see also: -[Pod disruption conditions](/docs/concepts/workloads/pods/disruptions#pod-disruption-conditions)). -Both feature gates are available in Kubernetes {{< skew currentVersion >}}. -{{< /note >}} +{{< feature-state feature_gate_name="JobPodFailurePolicy" >}} A Pod failure policy, defined with the `.spec.podFailurePolicy` field, enables your cluster to handle Pod failures based on the container exit codes and the diff --git a/content/en/docs/concepts/workloads/pods/disruptions.md b/content/en/docs/concepts/workloads/pods/disruptions.md index e2dd6e973d070..4319165c4936c 100644 --- a/content/en/docs/concepts/workloads/pods/disruptions.md +++ b/content/en/docs/concepts/workloads/pods/disruptions.md @@ -233,15 +233,10 @@ can happen, according to: ## Pod disruption conditions {#pod-disruption-conditions} -{{< feature-state for_k8s_version="v1.26" state="beta" >}} +{{< feature-state feature_gate_name="PodDisruptionConditions" >}} -{{< note >}} -In order to use this behavior, you must have the `PodDisruptionConditions` -[feature gate](/docs/reference/command-line-tools-reference/feature-gates/) -enabled in your cluster. -{{< /note >}} - -When enabled, a dedicated Pod `DisruptionTarget` [condition](/docs/concepts/workloads/pods/pod-lifecycle/#pod-conditions) is added to indicate +A dedicated Pod `DisruptionTarget` [condition](/docs/concepts/workloads/pods/pod-lifecycle/#pod-conditions) +is added to indicate that the Pod is about to be deleted due to a {{}}. The `reason` field of the condition additionally indicates one of the following reasons for the Pod termination: @@ -269,8 +264,7 @@ deleted. In such a situation, after some time, the Pod disruption condition will be cleared. {{< /note >}} -When the `PodDisruptionConditions` feature gate is enabled, -along with cleaning up the pods, the Pod garbage collector (PodGC) will also mark them as failed if they are in a non-terminal +Along with cleaning up the pods, the Pod garbage collector (PodGC) will also mark them as failed if they are in a non-terminal phase (see also [Pod garbage collection](/docs/concepts/workloads/pods/pod-lifecycle/#pod-garbage-collection)). When using a Job (or CronJob), you may want to use these Pod disruption conditions as part of your Job's diff --git a/content/en/docs/concepts/workloads/pods/pod-lifecycle.md b/content/en/docs/concepts/workloads/pods/pod-lifecycle.md index cbee1a8cce7d0..36c6b2b423292 100644 --- a/content/en/docs/concepts/workloads/pods/pod-lifecycle.md +++ b/content/en/docs/concepts/workloads/pods/pod-lifecycle.md @@ -668,8 +668,7 @@ Additionally, PodGC cleans up any Pods which satisfy any of the following condit [`node.kubernetes.io/out-of-service`](/docs/reference/labels-annotations-taints/#node-kubernetes-io-out-of-service), when the `NodeOutOfServiceVolumeDetach` feature gate is enabled. -When the `PodDisruptionConditions` feature gate is enabled, along with -cleaning up the Pods, PodGC will also mark them as failed if they are in a non-terminal +Along with cleaning up the Pods, PodGC will also mark them as failed if they are in a non-terminal phase. Also, PodGC adds a Pod disruption condition when cleaning up an orphan Pod. See [Pod disruption conditions](/docs/concepts/workloads/pods/disruptions#pod-disruption-conditions) for more details. diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/job-pod-failure-policy.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/job-pod-failure-policy.md index 4200f4b349548..7812e21cd2e30 100644 --- a/content/en/docs/reference/command-line-tools-reference/feature-gates/job-pod-failure-policy.md +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/job-pod-failure-policy.md @@ -13,6 +13,10 @@ stages: - stage: beta defaultValue: true fromVersion: "1.26" + toVersion: "1.30" + - stage: stable + defaultValue: true + fromVersion: "1.31" --- Allow users to specify handling of pod failures based on container exit codes and pod conditions. diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/pod-disruption-conditions.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/pod-disruption-conditions.md index 3b259ffd86b0e..bd2fc3a40ae2d 100644 --- a/content/en/docs/reference/command-line-tools-reference/feature-gates/pod-disruption-conditions.md +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/pod-disruption-conditions.md @@ -13,5 +13,9 @@ stages: - stage: beta defaultValue: true fromVersion: "1.26" + toVersion: "1.30" + - stage: stable + defaultValue: true + fromVersion: "1.31" --- Enables support for appending a dedicated pod condition indicating that the pod is being deleted due to a disruption. diff --git a/content/en/docs/tasks/job/pod-failure-policy.md b/content/en/docs/tasks/job/pod-failure-policy.md index ee1bbb3ac8929..0fcccfd1ad8ef 100644 --- a/content/en/docs/tasks/job/pod-failure-policy.md +++ b/content/en/docs/tasks/job/pod-failure-policy.md @@ -28,9 +28,6 @@ You should already be familiar with the basic use of [Job](/docs/concepts/worklo {{< include "task-tutorial-prereqs.md" >}} {{< version-check >}} -Ensure that the [feature gates](/docs/reference/command-line-tools-reference/feature-gates/) -`PodDisruptionConditions` and `JobPodFailurePolicy` are both enabled in your cluster. - ## Using Pod failure policy to avoid unnecessary Pod retries With the following example, you can learn how to use Pod failure policy to From 1daf72a6f7aa45be1c9ec7c46bcf85653d88ef8b Mon Sep 17 00:00:00 2001 From: Jordan Liggitt Date: Fri, 26 Jul 2024 23:36:49 -0400 Subject: [PATCH 61/82] KEP-4193: Update for beta ServiceAccountTokenNodeBinding graduation --- .../feature-gates/service-account-token-node-binding.md | 6 +++++- .../configure-pod-container/configure-service-account.md | 7 ++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/service-account-token-node-binding.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/service-account-token-node-binding.md index 28e2ce6a7ed4c..247951b96dbaf 100644 --- a/content/en/docs/reference/command-line-tools-reference/feature-gates/service-account-token-node-binding.md +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/service-account-token-node-binding.md @@ -9,5 +9,9 @@ stages: - stage: alpha defaultValue: false fromVersion: "1.29" + toVersion: "1.30" + - stage: beta + defaultValue: true + fromVersion: "1.31" --- -Controls whether the apiserver allows binding service account tokens to Node objects. +Controls whether the API server allows binding service account tokens to Node objects. diff --git a/content/en/docs/tasks/configure-pod-container/configure-service-account.md b/content/en/docs/tasks/configure-pod-container/configure-service-account.md index a9928edc8de8c..d56b18a675732 100644 --- a/content/en/docs/tasks/configure-pod-container/configure-service-account.md +++ b/content/en/docs/tasks/configure-pod-container/configure-service-account.md @@ -184,12 +184,13 @@ ServiceAccount. You can request a specific token duration using the `--duration` command line argument to `kubectl create token` (the actual duration of the issued token might be shorter, or could even be longer). +{{< feature-state feature_gate_name="ServiceAccountTokenNodeBinding" >}} + When the `ServiceAccountTokenNodeBinding` and `ServiceAccountTokenNodeBindingValidation` -features are enabled and the `KUBECTL_NODE_BOUND_TOKENS` environment variable is set to `true`, -it is possible to create a service account token that is directly bound to a `Node`: +features are enabled, it is possible to create a service account token that is directly bound to a `Node`: ```shell -KUBECTL_NODE_BOUND_TOKENS=true kubectl create token build-robot --bound-object-kind Node --bound-object-name node-001 --bound-object-uid 123...456 +kubectl create token build-robot --bound-object-kind Node --bound-object-name node-001 --bound-object-uid 123...456 ``` The token will be valid until it expires or either the associated `Node` or service account are deleted. From 0b16a923ca69c25bf87eafbe2ebb375d802765eb Mon Sep 17 00:00:00 2001 From: Jefftree Date: Tue, 16 Jul 2024 21:46:34 +0000 Subject: [PATCH 62/82] Add CLE documentation --- .../coordinated-leader-election.md | 56 +++++++++++++++++++ .../coordinated-leader-election.md | 14 +++++ 2 files changed, 70 insertions(+) create mode 100644 content/en/docs/concepts/architecture/coordinated-leader-election.md create mode 100644 content/en/docs/reference/command-line-tools-reference/feature-gates/coordinated-leader-election.md diff --git a/content/en/docs/concepts/architecture/coordinated-leader-election.md b/content/en/docs/concepts/architecture/coordinated-leader-election.md new file mode 100644 index 0000000000000..8ed93ac5a3e5c --- /dev/null +++ b/content/en/docs/concepts/architecture/coordinated-leader-election.md @@ -0,0 +1,56 @@ +--- +reviewers: +- jpbetz +title: Coordinated Leader Election +content_type: concept +weight: 200 +--- + + + +{{< feature-state feature_gate_name="CoordinatedLeaderElection" >}} + +Kubernetes {{< skew currentVersion >}} includes an alpha feature that allows +components to deterministically select a leader via Coordinated Leader Election. +This is useful to satisfy Kubernetes version skew during cluster upgrades. +Currently, the only supported selection strategy is `OldestEmulationVersion`, +preferring the leader with the lowest emulation version, followed by binary +version, followed by creation timestamp. + +## Enabling Coordinated Leader Election + +Ensure that `CoordinatedLeaderElection` [feature +gate](/docs/reference/command-line-tools-reference/feature-gates/) is enabled +when you start the {{< glossary_tooltip text="API Server" +term_id="kube-apiserver" >}}: and that the `coordination.k8s.io/v1alpha1` API is +enabled. + +This can be done by setting `FEATURE_GATES="CoordinatedLeaderElection=true"` and +`RUNTIME_CONFIG="coordination.k8s.io/v1alpha1=true"`. + +## Component Configuration + +With Coordinated Leader Election, components need to both run a LeaseCandidate +and Lease goroutine (both found in client-go/pkg/leaderelection). Two components +(kube-controller-manager and kube-scheduler) will automatically use coordinated +leader election if enabled. Please refer to the example found in +`k8s.io/cmd/kube-scheduler/app/server.go` on set up. + +The created LeaseCandidate object looks similar to below: + +``` +apiVersion: coordination.k8s.io/v1alpha1 +kind: LeaseCandidate +metadata: + name: hostname_uuid + namespace: kube-system +spec: + binaryVersion: 1.31.0 + emulationVersion: 1.31.0 + leaseName: kube-scheduler + preferredStrategies: + - OldestEmulationVersion + renewTime: "2024-07-30T03:45:18.325483Z" +``` + +Please refer to the documentation for LeaseCandidate for the full API details. \ No newline at end of file diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/coordinated-leader-election.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/coordinated-leader-election.md new file mode 100644 index 0000000000000..bf33b3449c25a --- /dev/null +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/coordinated-leader-election.md @@ -0,0 +1,14 @@ +--- +title: CoordinatedLeaderElection +content_type: feature_gate +_build: + list: never + render: false + +stages: + - stage: alpha + defaultValue: false + fromVersion: "1.31" +--- +Enables coordinated leader election for leases, deterministically +managing the leader in leader elected components. \ No newline at end of file From 89632ae9756835e8002fd115b64616d758ac04fc Mon Sep 17 00:00:00 2001 From: Jefftree Date: Tue, 30 Jul 2024 13:04:47 +0000 Subject: [PATCH 63/82] address comments --- .../architecture/coordinated-leader-election.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/content/en/docs/concepts/architecture/coordinated-leader-election.md b/content/en/docs/concepts/architecture/coordinated-leader-election.md index 8ed93ac5a3e5c..f66ec916e3a26 100644 --- a/content/en/docs/concepts/architecture/coordinated-leader-election.md +++ b/content/en/docs/concepts/architecture/coordinated-leader-election.md @@ -12,8 +12,8 @@ weight: 200 Kubernetes {{< skew currentVersion >}} includes an alpha feature that allows components to deterministically select a leader via Coordinated Leader Election. -This is useful to satisfy Kubernetes version skew during cluster upgrades. -Currently, the only supported selection strategy is `OldestEmulationVersion`, +This is useful to satisfy Kubernetes version skew constraints during cluster upgrades. +Currently, the only builtin selection strategy is `OldestEmulationVersion`, preferring the leader with the lowest emulation version, followed by binary version, followed by creation timestamp. @@ -25,8 +25,8 @@ when you start the {{< glossary_tooltip text="API Server" term_id="kube-apiserver" >}}: and that the `coordination.k8s.io/v1alpha1` API is enabled. -This can be done by setting `FEATURE_GATES="CoordinatedLeaderElection=true"` and -`RUNTIME_CONFIG="coordination.k8s.io/v1alpha1=true"`. +This can be done by setting flags `--feature-gates="CoordinatedLeaderElection=true"` and +`--runtime-config="coordination.k8s.io/v1alpha1=true"`. ## Component Configuration @@ -34,7 +34,7 @@ With Coordinated Leader Election, components need to both run a LeaseCandidate and Lease goroutine (both found in client-go/pkg/leaderelection). Two components (kube-controller-manager and kube-scheduler) will automatically use coordinated leader election if enabled. Please refer to the example found in -`k8s.io/cmd/kube-scheduler/app/server.go` on set up. +[k8s.io/kubernetes/cmd/kube-scheduler/app/server.go](https://github.com/kubernetes/kubernetes/blob/master/cmd/kube-scheduler/app/server.go) on set up. The created LeaseCandidate object looks similar to below: @@ -53,4 +53,4 @@ spec: renewTime: "2024-07-30T03:45:18.325483Z" ``` -Please refer to the documentation for LeaseCandidate for the full API details. \ No newline at end of file +Please refer to the [documentation](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#leasecandidate-v1alpha1-coordination-k8s-io) for LeaseCandidate for the full API details. \ No newline at end of file From 0cdbbb512d96b6928124d8c82ce7eb7d3b85f86d Mon Sep 17 00:00:00 2001 From: Jordan Liggitt Date: Tue, 30 Jul 2024 08:40:29 -0400 Subject: [PATCH 64/82] KEP-4193: clarify required kubectl version --- .../configure-pod-container/configure-service-account.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/content/en/docs/tasks/configure-pod-container/configure-service-account.md b/content/en/docs/tasks/configure-pod-container/configure-service-account.md index d56b18a675732..0a5c7002c9a76 100644 --- a/content/en/docs/tasks/configure-pod-container/configure-service-account.md +++ b/content/en/docs/tasks/configure-pod-container/configure-service-account.md @@ -187,13 +187,14 @@ token might be shorter, or could even be longer). {{< feature-state feature_gate_name="ServiceAccountTokenNodeBinding" >}} When the `ServiceAccountTokenNodeBinding` and `ServiceAccountTokenNodeBindingValidation` -features are enabled, it is possible to create a service account token that is directly bound to a `Node`: +features are enabled, and using `kubectl` v1.31 or later, it is possible to create a service +account token that is directly bound to a Node: ```shell kubectl create token build-robot --bound-object-kind Node --bound-object-name node-001 --bound-object-uid 123...456 ``` -The token will be valid until it expires or either the associated `Node` or service account are deleted. +The token will be valid until it expires or either the associated Node or service account are deleted. {{< note >}} Versions of Kubernetes before v1.22 automatically created long term credentials for From 729762108799a26e047e58f3ae3e200ad090224a Mon Sep 17 00:00:00 2001 From: Jefftree Date: Tue, 30 Jul 2024 13:07:06 +0000 Subject: [PATCH 65/82] use convention for links to docs --- .../docs/concepts/architecture/coordinated-leader-election.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/en/docs/concepts/architecture/coordinated-leader-election.md b/content/en/docs/concepts/architecture/coordinated-leader-election.md index f66ec916e3a26..2a7592a69e3e8 100644 --- a/content/en/docs/concepts/architecture/coordinated-leader-election.md +++ b/content/en/docs/concepts/architecture/coordinated-leader-election.md @@ -53,4 +53,4 @@ spec: renewTime: "2024-07-30T03:45:18.325483Z" ``` -Please refer to the [documentation](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#leasecandidate-v1alpha1-coordination-k8s-io) for LeaseCandidate for the full API details. \ No newline at end of file +Please refer to the [documentation](/docs/reference/generated/kubernetes-api/{{< param "version" >}}/#leasecandidate-v1alpha1-coordination-k8s-io) for LeaseCandidate for the full API details. \ No newline at end of file From fdc8e3ed7fe3e9ec8c24c9de1cdc7903043e9e1b Mon Sep 17 00:00:00 2001 From: Laurent Goderre Date: Tue, 18 Jun 2024 13:25:01 -0400 Subject: [PATCH 66/82] KEP-4622: Add docs for TopologyManager policy option for MaxAllowableNUMANodes --- .../administer-cluster/topology-manager.md | 36 +++++++++++++------ 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/content/en/docs/tasks/administer-cluster/topology-manager.md b/content/en/docs/tasks/administer-cluster/topology-manager.md index 76200482ca5b0..fa2b26a69b6dd 100644 --- a/content/en/docs/tasks/administer-cluster/topology-manager.md +++ b/content/en/docs/tasks/administer-cluster/topology-manager.md @@ -222,17 +222,31 @@ You will still have to enable each option using the `TopologyManagerPolicyOption The following policy options exists: * `prefer-closest-numa-nodes` (beta, visible by default; `TopologyManagerPolicyOptions` and `TopologyManagerPolicyBetaOptions` feature gates have to be enabled). -The `prefer-closest-numa-nodes` policy option is beta in Kubernetes {{< skew currentVersion >}}. - -If the `prefer-closest-numa-nodes` policy option is specified, the `best-effort` and `restricted` -policies will favor sets of NUMA nodes with shorter distance between them when making admission decisions. -You can enable this option by adding `prefer-closest-numa-nodes=true` to the Topology Manager policy options. -By default, without this option, Topology Manager aligns resources on either a single NUMA node or -the minimum number of NUMA nodes (in cases where more than one NUMA node is required). However, -the `TopologyManager` is not aware of NUMA distances and does not take them into account when making admission decisions. -This limitation surfaces in multi-socket, as well as single-socket multi NUMA systems, -and can cause significant performance degradation in latency-critical execution and high-throughput applications if the -Topology Manager decides to align resources on non-adjacent NUMA nodes. + The `prefer-closest-numa-nodes` policy option is beta in Kubernetes {{< skew currentVersion >}}. + + If the `prefer-closest-numa-nodes` policy option is specified, the `best-effort` and `restricted` + policies will favor sets of NUMA nodes with shorter distance between them when making admission decisions. + You can enable this option by adding `prefer-closest-numa-nodes=true` to the Topology Manager policy options. + By default, without this option, Topology Manager aligns resources on either a single NUMA node or + the minimum number of NUMA nodes (in cases where more than one NUMA node is required). However, + the `TopologyManager` is not aware of NUMA distances and does not take them into account when making admission decisions. + This limitation surfaces in multi-socket, as well as single-socket multi NUMA systems, + and can cause significant performance degradation in latency-critical execution and high-throughput applications if the + Topology Manager decides to align resources on non-adjacent NUMA nodes. + +* `max-allowable-numa-nodes` (beta, visible by default). + The `max-allowable-numa-nodes` policy option is beta in Kubernetes {{< skew currentVersion >}}. + + The time to admit a pod is tied to the number of NUMA nodes on the physical machine. + By default, Kubernetes does not run a kubelet with the topology manager enabled, on any (Kubernetes) node where more than 8 NUMA nodes are detected. + If you select the the `max-allowable-numa-nodes` policy option, nodes with more than 8 NUMA nodes can + be allowed to run with the topology manager enabled. The Kubernetes project only has limited data on the impact + of using the topology manager on (Kubernetes) nodes with more than 8 NUMA nodes. Because of that + lack of data, using this policy option is **not** recommended and is at your own risk. + Setting a value of `max-allowable-numa-nodes` does not (in and of itself) affect the + latency of pod admission, but binding a Pod to a (Kubernetes) node with many NUMA does does have an impact. + Future, potential improvements to Kubernetes may improve Pod admission performance and the high + latency that happens as the number of NUMA nodes increases. ### Pod Interactions with Topology Manager Policies From 41487af5336ef54afb16f3a36ceb9f9844d89b6b Mon Sep 17 00:00:00 2001 From: Tim Bannister Date: Tue, 30 Jul 2024 15:52:04 +0100 Subject: [PATCH 67/82] Drop mention of kubelet command line arguments Configuring the topology manager via the kubelet command line is deprecated. Explain the recommended approach. --- .../docs/tasks/administer-cluster/topology-manager.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/content/en/docs/tasks/administer-cluster/topology-manager.md b/content/en/docs/tasks/administer-cluster/topology-manager.md index fa2b26a69b6dd..9bc1e7605d58d 100644 --- a/content/en/docs/tasks/administer-cluster/topology-manager.md +++ b/content/en/docs/tasks/administer-cluster/topology-manager.md @@ -96,12 +96,15 @@ The Topology Manager can deal with the alignment of resources in a couple of dis * `container` (default) * `pod` -Either option can be selected at a time of the kubelet startup, with `--topology-manager-scope` -flag. +Either option can be selected at a time of the kubelet startup, by setting the +`topologyManagerScope` in the +[kubelet configuration file](/docs/tasks/administer-cluster/kubelet-config-file/). ### container scope -The `container` scope is used by default. +The `container` scope is used by default. You can also explicitly set the +`topologyManagerScope` to `container` in the +[kubelet configuration file](/docs/tasks/administer-cluster/kubelet-config-file/). Within this scope, the Topology Manager performs a number of sequential resource alignments, i.e., for each container (in a pod) a separate alignment is computed. In other words, there is no notion @@ -113,7 +116,7 @@ scope, for example the `pod` scope. ### pod scope -To select the `pod` scope, start the kubelet with the command line option `--topology-manager-scope=pod`. +To select the `pod` scope, set `topologyManagerScope` in the [kubelet configuration file](/docs/tasks/administer-cluster/kubelet-config-file/) to `pod`.` This scope allows for grouping all containers in a pod to a common set of NUMA nodes. That is, the Topology Manager treats a pod as a whole and attempts to allocate the entire pod (all containers) From 2b71e63c053378d01f0884cce817af8774730f68 Mon Sep 17 00:00:00 2001 From: Tim Bannister Date: Tue, 30 Jul 2024 16:00:39 +0100 Subject: [PATCH 68/82] Write headings in sentence case --- .../administer-cluster/topology-manager.md | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/content/en/docs/tasks/administer-cluster/topology-manager.md b/content/en/docs/tasks/administer-cluster/topology-manager.md index 9bc1e7605d58d..ee80db843bba8 100644 --- a/content/en/docs/tasks/administer-cluster/topology-manager.md +++ b/content/en/docs/tasks/administer-cluster/topology-manager.md @@ -35,7 +35,7 @@ responsible for these optimizations. -## How Topology Manager Works +## How topology manager works Prior to the introduction of Topology Manager, the CPU and Device Manager in Kubernetes make resource allocation decisions independently of each other. This can result in undesirable @@ -60,7 +60,7 @@ the pod can be accepted or rejected from the node based on the selected hint. The hint is then stored in the Topology Manager for use by the *Hint Providers* when making the resource allocation decisions. -## Topology Manager Scopes and Policies +## Topology manager scopes and policies The Topology Manager currently: @@ -89,7 +89,7 @@ Manager should be enabled and proper Memory Manager policy should be configured [Memory Manager](/docs/tasks/administer-cluster/memory-manager/) documentation. {{< /note >}} -### Topology Manager Scopes +## Topology manager scopes The Topology Manager can deal with the alignment of resources in a couple of distinct scopes: @@ -100,7 +100,7 @@ Either option can be selected at a time of the kubelet startup, by setting the `topologyManagerScope` in the [kubelet configuration file](/docs/tasks/administer-cluster/kubelet-config-file/). -### container scope +### `container` scope The `container` scope is used by default. You can also explicitly set the `topologyManagerScope` to `container` in the @@ -114,7 +114,7 @@ the Topology Manager performs an arbitrary alignment of individual containers to The notion of grouping the containers was endorsed and implemented on purpose in the following scope, for example the `pod` scope. -### pod scope +### `pod` scope To select the `pod` scope, set `topologyManagerScope` in the [kubelet configuration file](/docs/tasks/administer-cluster/kubelet-config-file/) to `pod`.` @@ -150,7 +150,7 @@ is present among possible allocations. Reconsider the example above: To recap, Topology Manager first computes a set of NUMA nodes and then tests it against Topology Manager policy, which either leads to the rejection or admission of the pod. -### Topology Manager Policies +## Topology manager policies Topology Manager supports four allocation policies. You can set a policy via a Kubelet flag, `--topology-manager-policy`. There are four supported policies: @@ -166,11 +166,11 @@ the policy, is reflecting requirements of the entire pod, and thus each containe will result with **the same** topology alignment decision. {{< /note >}} -### none policy {#policy-none} +### `none` policy {#policy-none} This is the default policy and does not perform any topology alignment. -### best-effort policy {#policy-best-effort} +### `best-effort` policy {#policy-best-effort} For each container in a Pod, the kubelet, with `best-effort` topology management policy, calls each Hint Provider to discover their resource availability. Using this information, the Topology @@ -180,7 +180,7 @@ preferred, Topology Manager will store this and admit the pod to the node anyway The *Hint Providers* can then use this information when making the resource allocation decision. -### restricted policy {#policy-restricted} +### `restricted` policy {#policy-restricted} For each container in a Pod, the kubelet, with `restricted` topology management policy, calls each Hint Provider to discover their resource availability. Using this information, the Topology @@ -196,7 +196,7 @@ have the `Topology Affinity` error. If the pod is admitted, the *Hint Providers* can then use this information when making the resource allocation decision. -### single-numa-node policy {#policy-single-numa-node} +### `single-numa-node` policy {#policy-single-numa-node} For each container in a Pod, the kubelet, with `single-numa-node` topology management policy, calls each Hint Provider to discover their resource availability. Using this information, the @@ -211,7 +211,7 @@ reschedule the pod. It is recommended to use a Deployment with replicas to trigg the Pod.An external control loop could be also implemented to trigger a redeployment of pods that have the `Topology Affinity` error. -### Topology manager policy options +## Topology manager policy options Support for the Topology Manager policy options requires `TopologyManagerPolicyOptions` [feature gate](/docs/reference/command-line-tools-reference/feature-gates/) to be enabled @@ -251,7 +251,7 @@ The following policy options exists: Future, potential improvements to Kubernetes may improve Pod admission performance and the high latency that happens as the number of NUMA nodes increases. -### Pod Interactions with Topology Manager Policies +## Pod interactions with topology manager policies Consider the containers in the following pod specs: @@ -360,7 +360,7 @@ Using this information the Topology Manager calculates the optimal hint for the this information, which will be used by the Hint Providers when they are making their resource assignments. -### Known Limitations +## Known limitations 1. The maximum number of NUMA nodes that Topology Manager allows is 8. With more than 8 NUMA nodes there will be a state explosion when trying to enumerate the possible NUMA affinities and From 2deb95c701dc4da9dff43b4486168d1038e71e31 Mon Sep 17 00:00:00 2001 From: Tim Bannister Date: Tue, 30 Jul 2024 16:04:14 +0100 Subject: [PATCH 69/82] Tweak capitalization --- content/en/docs/tasks/administer-cluster/topology-manager.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/content/en/docs/tasks/administer-cluster/topology-manager.md b/content/en/docs/tasks/administer-cluster/topology-manager.md index ee80db843bba8..fe442e6fd5f6e 100644 --- a/content/en/docs/tasks/administer-cluster/topology-manager.md +++ b/content/en/docs/tasks/administer-cluster/topology-manager.md @@ -78,13 +78,13 @@ carry out the alignment (e.g. `best-effort`, `restricted`, `single-numa-node`, e Details on the various `scopes` and `policies` available today can be found below. {{< note >}} -To align CPU resources with other requested resources in a Pod Spec, the CPU Manager should be +To align CPU resources with other requested resources in a Pod spec, the CPU Manager should be enabled and proper CPU Manager policy should be configured on a Node. See [control CPU Management Policies](/docs/tasks/administer-cluster/cpu-management-policies/). {{< /note >}} {{< note >}} -To align memory (and hugepages) resources with other requested resources in a Pod Spec, the Memory +To align memory (and hugepages) resources with other requested resources in a Pod spec, the Memory Manager should be enabled and proper Memory Manager policy should be configured on a Node. Examine [Memory Manager](/docs/tasks/administer-cluster/memory-manager/) documentation. {{< /note >}} From cca6aaf6aafe712f5d8b9092a88281360fe9b6fb Mon Sep 17 00:00:00 2001 From: Tim Bannister Date: Tue, 30 Jul 2024 16:10:35 +0100 Subject: [PATCH 70/82] Reword guidance about topology manager policy options --- .../administer-cluster/topology-manager.md | 77 ++++++++++++------- 1 file changed, 48 insertions(+), 29 deletions(-) diff --git a/content/en/docs/tasks/administer-cluster/topology-manager.md b/content/en/docs/tasks/administer-cluster/topology-manager.md index fe442e6fd5f6e..ccaa870cc0078 100644 --- a/content/en/docs/tasks/administer-cluster/topology-manager.md +++ b/content/en/docs/tasks/administer-cluster/topology-manager.md @@ -223,37 +223,55 @@ You can toggle groups of options on and off based upon their maturity level usin You will still have to enable each option using the `TopologyManagerPolicyOptions` kubelet option. -The following policy options exists: -* `prefer-closest-numa-nodes` (beta, visible by default; `TopologyManagerPolicyOptions` and `TopologyManagerPolicyBetaOptions` feature gates have to be enabled). - The `prefer-closest-numa-nodes` policy option is beta in Kubernetes {{< skew currentVersion >}}. - - If the `prefer-closest-numa-nodes` policy option is specified, the `best-effort` and `restricted` - policies will favor sets of NUMA nodes with shorter distance between them when making admission decisions. - You can enable this option by adding `prefer-closest-numa-nodes=true` to the Topology Manager policy options. - By default, without this option, Topology Manager aligns resources on either a single NUMA node or - the minimum number of NUMA nodes (in cases where more than one NUMA node is required). However, - the `TopologyManager` is not aware of NUMA distances and does not take them into account when making admission decisions. - This limitation surfaces in multi-socket, as well as single-socket multi NUMA systems, - and can cause significant performance degradation in latency-critical execution and high-throughput applications if the - Topology Manager decides to align resources on non-adjacent NUMA nodes. - -* `max-allowable-numa-nodes` (beta, visible by default). - The `max-allowable-numa-nodes` policy option is beta in Kubernetes {{< skew currentVersion >}}. - - The time to admit a pod is tied to the number of NUMA nodes on the physical machine. - By default, Kubernetes does not run a kubelet with the topology manager enabled, on any (Kubernetes) node where more than 8 NUMA nodes are detected. - If you select the the `max-allowable-numa-nodes` policy option, nodes with more than 8 NUMA nodes can - be allowed to run with the topology manager enabled. The Kubernetes project only has limited data on the impact - of using the topology manager on (Kubernetes) nodes with more than 8 NUMA nodes. Because of that - lack of data, using this policy option is **not** recommended and is at your own risk. - Setting a value of `max-allowable-numa-nodes` does not (in and of itself) affect the - latency of pod admission, but binding a Pod to a (Kubernetes) node with many NUMA does does have an impact. - Future, potential improvements to Kubernetes may improve Pod admission performance and the high - latency that happens as the number of NUMA nodes increases. +### `prefer-closest-numa-nodes` (beta) {#policy-option-prefer-closest-numa-nodes} + +The `prefer-closest-numa-nodes` option is beta since Kubernetes 1.28. In Kubernetes {{< skew currentVersion >}} +this policy option is visible by default provided that the `TopologyManagerPolicyOptions` and +`TopologyManagerPolicyBetaOptions` [feature gates](/docs/reference/command-line-tools-reference/feature-gates/) +are enabled. + +The topology manager is not aware by default of NUMA distances, and does not take them into account when making +Pod admission decisions. This limitation surfaces in multi-socket, as well as single-socket multi NUMA systems, +and can cause significant performance degradation in latency-critical execution and high-throughput applications +if the topology manager decides to align resources on non-adjacent NUMA nodes. + +If you specify the `prefer-closest-numa-nodes` policy option, the `best-effort` and `restricted` +policies favor sets of NUMA nodes with shorter distance between them when making admission decisions. + +You can enable this option by adding `prefer-closest-numa-nodes=true` to the Topology Manager policy options. + +By default (without this option), Topology Manager aligns resources on either a single NUMA node or, +in the case where more than one NUMA node is required, using the minimum number of NUMA nodes. + +### `max-allowable-numa-nodes` (beta) {#policy-option-max-allowable-numa-nodes} + +The `max-allowable-numa-nodes` option is beta since Kubernetes 1.31. In Kubernetes {{< skew currentVersion >}} +this policy option is visible by default provided that the `TopologyManagerPolicyOptions` and +`TopologyManagerPolicyBetaOptions` [feature gates](/docs/reference/command-line-tools-reference/feature-gates/) +are enabled. + +The time to admit a pod is tied to the number of NUMA nodes on the physical machine. +By default, Kubernetes does not run a kubelet with the topology manager enabled, on any (Kubernetes) node where +more than 8 NUMA nodes are detected. + +{{< note >}} +If you select the the `max-allowable-numa-nodes` policy option, nodes with more than 8 NUMA nodes can +be allowed to run with the topology manager enabled. The Kubernetes project only has limited data on the impact +of using the topology manager on (Kubernetes) nodes with more than 8 NUMA nodes. Because of that +lack of data, using this policy option with Kubernetes {{< skew currentVersion >}} is **not** recommended and is +at your own risk. +{{< /note >}} + +You can enable this option by adding `max-allowable-numa-nodes=true` to the Topology Manager policy options. + +Setting a value of `max-allowable-numa-nodes` does not (in and of itself) affect the +latency of pod admission, but binding a Pod to a (Kubernetes) node with many NUMA does does have an impact. +Future, potential improvements to Kubernetes may improve Pod admission performance and the high +latency that happens as the number of NUMA nodes increases. ## Pod interactions with topology manager policies -Consider the containers in the following pod specs: +Consider the containers in the following Pod manifest: ```yaml spec: @@ -364,7 +382,8 @@ assignments. 1. The maximum number of NUMA nodes that Topology Manager allows is 8. With more than 8 NUMA nodes there will be a state explosion when trying to enumerate the possible NUMA affinities and - generating their hints. + generating their hints. See [`max-allowable-numa-nodes`](#policy-option-max-allowable-numa-nodes) + (beta) for more options. 2. The scheduler is not topology-aware, so it is possible to be scheduled on a node and then fail on the node due to the Topology Manager. \ No newline at end of file From bf0deb058fb9391991bebe3c991ca725509c83c4 Mon Sep 17 00:00:00 2001 From: Tim Bannister Date: Tue, 30 Jul 2024 17:21:10 +0100 Subject: [PATCH 71/82] Mark v1.31 release as current --- hugo.toml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/hugo.toml b/hugo.toml index 85ca4c8be7666..2475786a6efaf 100644 --- a/hugo.toml +++ b/hugo.toml @@ -142,9 +142,9 @@ time_format_default = "January 02, 2006 at 3:04 PM PST" description = "Production-Grade Container Orchestration" showedit = true -latest = "v1.30" +latest = "v1.31" -version = "v1.30" +version = "v1.31" githubbranch = "main" docsbranch = "main" deprecated = false @@ -184,11 +184,17 @@ js = [ ] [[params.versions]] -version = "v1.30" -githubbranch = "v1.30.0" +version = "v1.31" +githubbranch = "v1.31.0" docsbranch = "main" url = "https://kubernetes.io" +[[params.versions]] +version = "v1.30" +githubbranch = "v1.30.0" +docsbranch = "release-1.30" +url = "https://v1-30.docs.kubernetes.io" + [[params.versions]] version = "v1.29" githubbranch = "v1.29.3" @@ -207,12 +213,6 @@ githubbranch = "v1.27.12" docsbranch = "release-1.27" url = "https://v1-27.docs.kubernetes.io" -[[params.versions]] -version = "v1.26" -githubbranch = "v1.26.15" -docsbranch = "release-1.26" -url = "https://v1-26.docs.kubernetes.io" - # User interface configuration [params.ui] # Enable to show the side bar menu in its compact state. From cdd0f612b34558fda068424ae52ca69bb77b6085 Mon Sep 17 00:00:00 2001 From: Sergey Kanzhelev Date: Tue, 30 Jul 2024 10:37:14 -0700 Subject: [PATCH 72/82] device plugin failures (#47029) --- .../compute-storage-net/device-plugins.md | 29 +++++++++++++++++++ .../feature-gates/resource-health-status.md | 16 ++++++++++ 2 files changed, 45 insertions(+) create mode 100644 content/en/docs/reference/command-line-tools-reference/feature-gates/resource-health-status.md diff --git a/content/en/docs/concepts/extend-kubernetes/compute-storage-net/device-plugins.md b/content/en/docs/concepts/extend-kubernetes/compute-storage-net/device-plugins.md index 5d6355844f4dd..ef0a0ca050d34 100644 --- a/content/en/docs/concepts/extend-kubernetes/compute-storage-net/device-plugins.md +++ b/content/en/docs/concepts/extend-kubernetes/compute-storage-net/device-plugins.md @@ -170,6 +170,35 @@ kubelet instance. A new kubelet instance deletes all the existing Unix sockets u `/var/lib/kubelet/device-plugins` when it starts. A device plugin can monitor the deletion of its Unix socket and re-register itself upon such an event. +### Device plugin and unhealthy devices + +There are cases when devices fail or are shut down. The responsibility of the Device Plugin +in this case is to notify the kubelet about the situation using the `ListAndWatchResponse` API. + +Once a device is marked as unhealthy, the kubelet will decrease the allocatable count +for this resource on the Node to reflect how many devices can be used for scheduling new pods. +Capacity count for the resource will not change. + +Pods that were assigned to the failed devices will continue be assigned to this device. +It is typical that code relying on the device will start failing and Pod may get +into Failed phase if `restartPolicy` for the Pod was not `Always` or enter the crash loop +otherwise. + +Before Kubernetes v1.31, the way to know whether or not a Pod is associated with the +failed device is to use the [PodResources API](#monitoring-device-plugin-resources). + +{{< feature-state feature_gate_name="ResourceHealthStatus" >}} + +By enabling the feature gate `ResourceHealthStatus`, the field `allocatedResourcesStatus` +will be added to each container status, within the `.status` for each Pod. The `allocatedResourcesStatus` +field +reports health information for each device assigned to the container. + +For a failed Pod, or or where you suspect a fault, you can use this status to understand whether +the Pod behavior may be associated with device failure. For example, if an accelerator is reporting +an over-temperature event, the `allocatedResourcesStatus` field may be able to report this. + + ## Device plugin deployment You can deploy a device plugin as a DaemonSet, as a package for your node's operating system, diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/resource-health-status.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/resource-health-status.md new file mode 100644 index 0000000000000..eb2186de40e1a --- /dev/null +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/resource-health-status.md @@ -0,0 +1,16 @@ +--- +title: ResourceHealthStatus +content_type: feature_gate +_build: + list: never + render: false + +stages: + - stage: alpha + defaultValue: false + fromVersion: "1.31" +--- +Enable the `allocatedResourcesStatus` field within the `.status` for a Pod. The field +reports additional details for each container in the Pod, +with the health information for each device assigned to the Pod. +See [Device plugin and unhealthy devices](/docs/concepts/extend-kubernetes/compute-storage-net/device-plugins/#device-plugin-and-unhealthy-devices) for more details. From dfde96c9b6c08098f01c8c933e743da6d6ec6299 Mon Sep 17 00:00:00 2001 From: Jefftree Date: Tue, 30 Jul 2024 18:08:28 +0000 Subject: [PATCH 73/82] Address comments --- .../coordinated-leader-election.md | 4 ++-- .../feature-gates/coordinated-leader-election.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) rename content/en/docs/concepts/{architecture => cluster-administration}/coordinated-leader-election.md (91%) diff --git a/content/en/docs/concepts/architecture/coordinated-leader-election.md b/content/en/docs/concepts/cluster-administration/coordinated-leader-election.md similarity index 91% rename from content/en/docs/concepts/architecture/coordinated-leader-election.md rename to content/en/docs/concepts/cluster-administration/coordinated-leader-election.md index 2a7592a69e3e8..47dc1e0f6b300 100644 --- a/content/en/docs/concepts/architecture/coordinated-leader-election.md +++ b/content/en/docs/concepts/cluster-administration/coordinated-leader-election.md @@ -11,13 +11,13 @@ weight: 200 {{< feature-state feature_gate_name="CoordinatedLeaderElection" >}} Kubernetes {{< skew currentVersion >}} includes an alpha feature that allows -components to deterministically select a leader via Coordinated Leader Election. +{{< glossary_tooltip text="control plane" term_id="control-plane" >}} components to deterministically select a leader via _coordinated leader election. This is useful to satisfy Kubernetes version skew constraints during cluster upgrades. Currently, the only builtin selection strategy is `OldestEmulationVersion`, preferring the leader with the lowest emulation version, followed by binary version, followed by creation timestamp. -## Enabling Coordinated Leader Election +## Enabling coordinated leader election Ensure that `CoordinatedLeaderElection` [feature gate](/docs/reference/command-line-tools-reference/feature-gates/) is enabled diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/coordinated-leader-election.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/coordinated-leader-election.md index bf33b3449c25a..aa5763e69a2b3 100644 --- a/content/en/docs/reference/command-line-tools-reference/feature-gates/coordinated-leader-election.md +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/coordinated-leader-election.md @@ -10,5 +10,5 @@ stages: defaultValue: false fromVersion: "1.31" --- -Enables coordinated leader election for leases, deterministically -managing the leader in leader elected components. \ No newline at end of file +Enables the behaviors supporting the LeaseCandidate API, and also enables +coordinated leader election for the Kubernetes control plane, deterministically. \ No newline at end of file From 8ed531de12df8b52b5c8299419b474b141a99a31 Mon Sep 17 00:00:00 2001 From: Jefftree Date: Tue, 30 Jul 2024 18:12:36 +0000 Subject: [PATCH 74/82] add placeholder lease candidate page --- .../cluster-resources/lease-candidate-v1alpha1.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 content/en/docs/reference/kubernetes-api/cluster-resources/lease-candidate-v1alpha1.md diff --git a/content/en/docs/reference/kubernetes-api/cluster-resources/lease-candidate-v1alpha1.md b/content/en/docs/reference/kubernetes-api/cluster-resources/lease-candidate-v1alpha1.md new file mode 100644 index 0000000000000..2a2255dda8a8c --- /dev/null +++ b/content/en/docs/reference/kubernetes-api/cluster-resources/lease-candidate-v1alpha1.md @@ -0,0 +1,12 @@ +--- +api_metadata: + apiVersion: "coordination.k8s.io/v1alpha1" + import: "k8s.io/api/coordination/v1alpha1" + kind: "LeaseCandidate" +content_type: "api_reference" +description: "This is a placeholder page to be overwritten when the docs are generated for 1.31" +title: "Lease Candidate" +weight: 5 +--- + + From 80e240983f1e56ea0298876074d72b369c18af04 Mon Sep 17 00:00:00 2001 From: Jefftree Date: Tue, 30 Jul 2024 19:44:37 +0000 Subject: [PATCH 75/82] Comments --- .../coordinated-leader-election.md | 42 ++++++------------- .../lease-candidate-v1alpha1.md | 2 +- 2 files changed, 14 insertions(+), 30 deletions(-) diff --git a/content/en/docs/concepts/cluster-administration/coordinated-leader-election.md b/content/en/docs/concepts/cluster-administration/coordinated-leader-election.md index 47dc1e0f6b300..2bd831324ae62 100644 --- a/content/en/docs/concepts/cluster-administration/coordinated-leader-election.md +++ b/content/en/docs/concepts/cluster-administration/coordinated-leader-election.md @@ -10,8 +10,9 @@ weight: 200 {{< feature-state feature_gate_name="CoordinatedLeaderElection" >}} -Kubernetes {{< skew currentVersion >}} includes an alpha feature that allows -{{< glossary_tooltip text="control plane" term_id="control-plane" >}} components to deterministically select a leader via _coordinated leader election. +Kubernetes {{< skew currentVersion >}} includes an alpha feature that allows {{< +glossary_tooltip text="control plane" term_id="control-plane" >}} components to +deterministically select a leader via _coordinated leader election_. This is useful to satisfy Kubernetes version skew constraints during cluster upgrades. Currently, the only builtin selection strategy is `OldestEmulationVersion`, preferring the leader with the lowest emulation version, followed by binary @@ -22,35 +23,18 @@ version, followed by creation timestamp. Ensure that `CoordinatedLeaderElection` [feature gate](/docs/reference/command-line-tools-reference/feature-gates/) is enabled when you start the {{< glossary_tooltip text="API Server" -term_id="kube-apiserver" >}}: and that the `coordination.k8s.io/v1alpha1` API is +term_id="kube-apiserver" >}}: and that the `coordination.k8s.io/v1alpha1` API group is enabled. This can be done by setting flags `--feature-gates="CoordinatedLeaderElection=true"` and `--runtime-config="coordination.k8s.io/v1alpha1=true"`. -## Component Configuration - -With Coordinated Leader Election, components need to both run a LeaseCandidate -and Lease goroutine (both found in client-go/pkg/leaderelection). Two components -(kube-controller-manager and kube-scheduler) will automatically use coordinated -leader election if enabled. Please refer to the example found in -[k8s.io/kubernetes/cmd/kube-scheduler/app/server.go](https://github.com/kubernetes/kubernetes/blob/master/cmd/kube-scheduler/app/server.go) on set up. - -The created LeaseCandidate object looks similar to below: - -``` -apiVersion: coordination.k8s.io/v1alpha1 -kind: LeaseCandidate -metadata: - name: hostname_uuid - namespace: kube-system -spec: - binaryVersion: 1.31.0 - emulationVersion: 1.31.0 - leaseName: kube-scheduler - preferredStrategies: - - OldestEmulationVersion - renewTime: "2024-07-30T03:45:18.325483Z" -``` - -Please refer to the [documentation](/docs/reference/generated/kubernetes-api/{{< param "version" >}}/#leasecandidate-v1alpha1-coordination-k8s-io) for LeaseCandidate for the full API details. \ No newline at end of file +## Component configuration +Provided that you have enabled the `CoordinatedLeaderElection` feature gate _and_ +have the `coordination.k8s.io/v1alpha1` API group enabled, compatible control plane +components automatically use the LeaseCandidate and Lease APIs to elect a leader +as needed. + +For Kubernetes {{< skew currentVersion >}}, two control plane components +(kube-controller-manager and kube-scheduler) automatically use coordinated +leader election when the feature gate and API group are enabled. \ No newline at end of file diff --git a/content/en/docs/reference/kubernetes-api/cluster-resources/lease-candidate-v1alpha1.md b/content/en/docs/reference/kubernetes-api/cluster-resources/lease-candidate-v1alpha1.md index 2a2255dda8a8c..85397cccc42e6 100644 --- a/content/en/docs/reference/kubernetes-api/cluster-resources/lease-candidate-v1alpha1.md +++ b/content/en/docs/reference/kubernetes-api/cluster-resources/lease-candidate-v1alpha1.md @@ -5,7 +5,7 @@ api_metadata: kind: "LeaseCandidate" content_type: "api_reference" description: "This is a placeholder page to be overwritten when the docs are generated for 1.31" -title: "Lease Candidate" +title: "LeaseCandidate" weight: 5 --- From 44d591d7824ed187d318dea7730d4e10d8cc716d Mon Sep 17 00:00:00 2001 From: Tim Bannister Date: Tue, 30 Jul 2024 21:14:28 +0100 Subject: [PATCH 76/82] Link to docs about LeaseCandidate --- content/en/docs/concepts/architecture/leases.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/content/en/docs/concepts/architecture/leases.md b/content/en/docs/concepts/architecture/leases.md index 2b4e94db91c0d..0dcf2759269a3 100644 --- a/content/en/docs/concepts/architecture/leases.md +++ b/content/en/docs/concepts/architecture/leases.md @@ -34,6 +34,10 @@ This is used by control plane components like `kube-controller-manager` and `kub HA configurations, where only one instance of the component should be actively running while the other instances are on stand-by. +Read [coordinated leader election](/docs/concepts/cluster-administration/coordinated-leader-election) +to learn about how Kubernetes builds on the Lease API to select which component instance +acts as leader. + ## API server identity {{< feature-state feature_gate_name="APIServerIdentity" >}} From 4b71738c736a982fe1d7bbcccac3cc278f3a05fa Mon Sep 17 00:00:00 2001 From: Marek Siarkowicz Date: Fri, 21 Jun 2024 11:49:39 +0200 Subject: [PATCH 77/82] Beta documentation for Consistent Reads from Cache --- .../feature-gates/consistent-list-from-cache.md | 14 +++++++++++++- .../en/docs/reference/using-api/api-concepts.md | 5 +++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/content/en/docs/reference/command-line-tools-reference/feature-gates/consistent-list-from-cache.md b/content/en/docs/reference/command-line-tools-reference/feature-gates/consistent-list-from-cache.md index d08f57ee91bb9..a2162e02ce021 100644 --- a/content/en/docs/reference/command-line-tools-reference/feature-gates/consistent-list-from-cache.md +++ b/content/en/docs/reference/command-line-tools-reference/feature-gates/consistent-list-from-cache.md @@ -9,6 +9,18 @@ stages: - stage: alpha defaultValue: false fromVersion: "1.28" + toVersion: "1.30" + - stage: beta + defaultValue: true + fromVersion: "1.31" --- -Allow the API server to serve consistent lists from cache. +Enhance Kubernetes API server performance by serving consistent **list** requests +directly from its watch cache, improving scalability and response times. +To consistent list from cache Kubernetes requires a newer etcd version (v3.4.31+ or v3.5.13+), +that includes fixes to watch progress request feature. +If older etcd version is provided Kubernetes will automatically detect it and fallback to serving consistent reads from etcd. +Progress notifications ensure watch cache is consistent with etcd while reducing +the need for resource-intensive quorum reads from etcd. + +See the Kubernetes documentation on [Semantics for **get** and **list**](/docs/reference/using-api/api-concepts/#semantics-for-get-and-list) for more details. diff --git a/content/en/docs/reference/using-api/api-concepts.md b/content/en/docs/reference/using-api/api-concepts.md index acd54e8030fd2..41bc720d60004 100644 --- a/content/en/docs/reference/using-api/api-concepts.md +++ b/content/en/docs/reference/using-api/api-concepts.md @@ -1168,6 +1168,11 @@ Any Most recent : Return data at the most recent resource version. The returned data must be consistent (in detail: served from etcd via a quorum read). + For etcd v3.4.31+ and v3.5.13+ Kubernetes {{< skew currentVersion >}} serves “most recent” reads from the _watch cache_: + an internal, in-memory store within the API server that caches and mirrors the state of data + persisted into etcd. Kubernetes requests progress notification to maintain cache consistency against + the etcd persistence layer. Kubernetes versions v1.28 through to v1.30 also supported this + feature, although as Alpha it was not recommended for production nor enabled by default until the v1.31 release. Not older than : Return data at least as new as the provided `resourceVersion`. The newest From b3ce13edaf008bc7bcc2c82cc6af094bab9ef277 Mon Sep 17 00:00:00 2001 From: Mark Rossett Date: Mon, 29 Jul 2024 15:56:15 -0700 Subject: [PATCH 78/82] Updating Windows node eviction signals docs --- .../node-pressure-eviction.md | 42 ++++++++++++------- content/en/docs/concepts/windows/intro.md | 3 +- 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/content/en/docs/concepts/scheduling-eviction/node-pressure-eviction.md b/content/en/docs/concepts/scheduling-eviction/node-pressure-eviction.md index 5d56811d9cf28..ef7a448e4eab9 100644 --- a/content/en/docs/concepts/scheduling-eviction/node-pressure-eviction.md +++ b/content/en/docs/concepts/scheduling-eviction/node-pressure-eviction.md @@ -76,25 +76,27 @@ point in time. The kubelet uses eviction signals to make eviction decisions by comparing the signals to eviction thresholds, which are the minimum amount of the resource that should be available on the node. -On Linux, the kubelet uses the following eviction signals: - -| Eviction Signal | Description | -|--------------------------|---------------------------------------------------------------------------------------| -| `memory.available` | `memory.available` := `node.status.capacity[memory]` - `node.stats.memory.workingSet` | -| `nodefs.available` | `nodefs.available` := `node.stats.fs.available` | -| `nodefs.inodesFree` | `nodefs.inodesFree` := `node.stats.fs.inodesFree` | -| `imagefs.available` | `imagefs.available` := `node.stats.runtime.imagefs.available` | -| `imagefs.inodesFree` | `imagefs.inodesFree` := `node.stats.runtime.imagefs.inodesFree` | -| `containerfs.available` | `containerfs.available` := `node.stats.runtime.containerfs.available` | -| `containerfs.inodesFree` | `containerfs.inodesFree` := `node.stats.runtime.containerfs.inodesFree` | -| `pid.available` | `pid.available` := `node.stats.rlimit.maxpid` - `node.stats.rlimit.curproc` | +The kubelet uses the following eviction signals: + +| Eviction Signal | Description | Linux Only | +|--------------------------|---------------------------------------------------------------------------------------|------------| +| `memory.available` | `memory.available` := `node.status.capacity[memory]` - `node.stats.memory.workingSet` | | +| `nodefs.available` | `nodefs.available` := `node.stats.fs.available` | | +| `nodefs.inodesFree` | `nodefs.inodesFree` := `node.stats.fs.inodesFree` | • | +| `imagefs.available` | `imagefs.available` := `node.stats.runtime.imagefs.available` | | +| `imagefs.inodesFree` | `imagefs.inodesFree` := `node.stats.runtime.imagefs.inodesFree` | • | +| `containerfs.available` | `containerfs.available` := `node.stats.runtime.containerfs.available` | | +| `containerfs.inodesFree` | `containerfs.inodesFree` := `node.stats.runtime.containerfs.inodesFree` | • | +| `pid.available` | `pid.available` := `node.stats.rlimit.maxpid` - `node.stats.rlimit.curproc` | • | In this table, the **Description** column shows how kubelet gets the value of the signal. Each signal supports either a percentage or a literal value. The kubelet calculates the percentage value relative to the total capacity associated with the signal. -The value for `memory.available` is derived from the cgroupfs instead of tools +#### Memory signals + +On Linux nodes, the value for `memory.available` is derived from the cgroupfs instead of tools like `free -m`. This is important because `free -m` does not work in a container, and if users use the [node allocatable](/docs/tasks/administer-cluster/reserve-compute-resources/#node-allocatable) feature, out of resource decisions @@ -106,7 +108,14 @@ reproduces the same set of steps that the kubelet performs to calculate file-backed memory on the inactive LRU list) from its calculation, as it assumes that memory is reclaimable under pressure. -The kubelet recognizes three specific filesystem identifiers: +On Windows nodes, the value for `memory.available` is derived from the node's global +memory commit levels (queried through the [`GetPerformanceInfo()`](https://learn.microsoft.com/windows/win32/api/psapi/nf-psapi-getperformanceinfo) +system call) by subtracting the node's global [`CommitTotal`](https://learn.microsoft.com/windows/win32/api/psapi/ns-psapi-performance_information) from the node's [`CommitLimit`](https://learn.microsoft.com/windows/win32/api/psapi/ns-psapi-performance_information). Please note that `CommitLimit` can change if the node's page-file size changes! + +#### Filesystem signals + +The kubelet recognizes three specific filesystem identifiers that can be used with +eviction signals (`.inodesFree` or `.available`): 1. `nodefs`: The node's main filesystem, used for local disk volumes, emptyDir volumes not backed by memory, log storage, ephemeral storage, @@ -144,6 +153,8 @@ other local node filesystems. The kubelet does not support other container filesystems or storage configurations, and it does not currently support multiple filesystems for images and containers. +#### Deprecated kubelet garbage collection features + Some kubelet garbage collection features are deprecated in favor of eviction: | Existing Flag | Rationale | @@ -205,7 +216,8 @@ thresholds like `memory.available<1Gi`. The kubelet has the following default hard eviction thresholds: -- `memory.available<100Mi` +- `memory.available<100Mi` (Linux nodes) +- `memory.available<500Mi` (Windows nodes) - `nodefs.available<10%` - `imagefs.available<15%` - `nodefs.inodesFree<5%` (Linux nodes) diff --git a/content/en/docs/concepts/windows/intro.md b/content/en/docs/concepts/windows/intro.md index dcf4db95b9910..0fe3ad520052c 100644 --- a/content/en/docs/concepts/windows/intro.md +++ b/content/en/docs/concepts/windows/intro.md @@ -149,13 +149,12 @@ Some kubelet command line options behave differently on Windows, as described be * The `--kube-reserved`, `--system-reserved` , and `--eviction-hard` flags update [NodeAllocatable](/docs/tasks/administer-cluster/reserve-compute-resources/#node-allocatable) * Eviction by using `--enforce-node-allocable` is not implemented -* Eviction by using `--eviction-hard` and `--eviction-soft` are not implemented * When running on a Windows node the kubelet does not have memory or CPU restrictions. `--kube-reserved` and `--system-reserved` only subtract from `NodeAllocatable` and do not guarantee resource provided for workloads. See [Resource Management for Windows nodes](/docs/concepts/configuration/windows-resource-management/#resource-reservation) for more information. -* The `MemoryPressure` Condition is not implemented +* The `PIDPressure` Condition is not implemented * The kubelet does not take OOM eviction actions ### API compatibility {#api} From 370622fbbbdbd59c8f2ce1726450ca6616de0612 Mon Sep 17 00:00:00 2001 From: Michal Wozniak Date: Thu, 1 Aug 2024 12:07:06 +0200 Subject: [PATCH 79/82] Update the documentation for Pod Disruption Conditions --- content/en/docs/concepts/workloads/pods/disruptions.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/content/en/docs/concepts/workloads/pods/disruptions.md b/content/en/docs/concepts/workloads/pods/disruptions.md index 4319165c4936c..3f94f2dcc93d6 100644 --- a/content/en/docs/concepts/workloads/pods/disruptions.md +++ b/content/en/docs/concepts/workloads/pods/disruptions.md @@ -254,7 +254,9 @@ indicates one of the following reasons for the Pod termination: : Pod, that is bound to a no longer existing Node, is due to be deleted by [Pod garbage collection](/docs/concepts/workloads/pods/pod-lifecycle/#pod-garbage-collection). `TerminationByKubelet` -: Pod has been terminated by the kubelet, because of either {{}} or the [graceful node shutdown](/docs/concepts/architecture/nodes/#graceful-node-shutdown). +: Pod has been terminated by the kubelet, because of either {{}}, + the [graceful node shutdown](/docs/concepts/architecture/nodes/#graceful-node-shutdown), + or preemption for [system critical pods](/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/). {{< note >}} A Pod disruption might be interrupted. The control plane might re-attempt to From 58fadced10b10d1154c087d47f4d5938b000c787 Mon Sep 17 00:00:00 2001 From: Oluebube Princes Egbuna Date: Thu, 1 Aug 2024 12:01:14 +0100 Subject: [PATCH 80/82] Update release schedule for v1.31 --- data/releases/schedule.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/data/releases/schedule.yaml b/data/releases/schedule.yaml index e095045b3c59b..97d415f5dfdb8 100644 --- a/data/releases/schedule.yaml +++ b/data/releases/schedule.yaml @@ -4,6 +4,14 @@ # schedule-builder -uc data/releases/schedule.yaml -e data/releases/eol.yaml --- schedules: +- endOfLifeDate: "2025-10-28" + maintenanceModeStartDate: "2025-08-28" + next: + cherryPickDeadline: "2024-09-06" + release: 1.31.1 + targetDate: "2024-09-11" + release: "1.31" + releaseDate: "2024-08-13" - endOfLifeDate: "2025-06-28" maintenanceModeStartDate: "2025-04-28" next: From 6fbfd77f3b6d2e56e96a2f6cecef2050d288e381 Mon Sep 17 00:00:00 2001 From: Mark Rossetti Date: Thu, 1 Aug 2024 08:55:50 -0700 Subject: [PATCH 81/82] Update content/en/docs/concepts/scheduling-eviction/node-pressure-eviction.md Co-authored-by: Tim Bannister --- .../docs/concepts/scheduling-eviction/node-pressure-eviction.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/en/docs/concepts/scheduling-eviction/node-pressure-eviction.md b/content/en/docs/concepts/scheduling-eviction/node-pressure-eviction.md index ef7a448e4eab9..70a932bef90aa 100644 --- a/content/en/docs/concepts/scheduling-eviction/node-pressure-eviction.md +++ b/content/en/docs/concepts/scheduling-eviction/node-pressure-eviction.md @@ -153,7 +153,7 @@ other local node filesystems. The kubelet does not support other container filesystems or storage configurations, and it does not currently support multiple filesystems for images and containers. -#### Deprecated kubelet garbage collection features +### Deprecated kubelet garbage collection features Some kubelet garbage collection features are deprecated in favor of eviction: From a85dab08d64e8b254d2e71b285ed914a600c1055 Mon Sep 17 00:00:00 2001 From: drewhagen Date: Wed, 7 Aug 2024 15:44:41 -0500 Subject: [PATCH 82/82] chore: Updates v1.30 hugo.toml to include latest patches ahead of release --- hugo.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hugo.toml b/hugo.toml index 2475786a6efaf..c603dcb143888 100644 --- a/hugo.toml +++ b/hugo.toml @@ -191,25 +191,25 @@ url = "https://kubernetes.io" [[params.versions]] version = "v1.30" -githubbranch = "v1.30.0" +githubbranch = "v1.30.3" docsbranch = "release-1.30" url = "https://v1-30.docs.kubernetes.io" [[params.versions]] version = "v1.29" -githubbranch = "v1.29.3" +githubbranch = "v1.29.7" docsbranch = "release-1.29" url = "https://v1-29.docs.kubernetes.io" [[params.versions]] version = "v1.28" -githubbranch = "v1.28.8" +githubbranch = "v1.28.12" docsbranch = "release-1.28" url = "https://v1-28.docs.kubernetes.io" [[params.versions]] version = "v1.27" -githubbranch = "v1.27.12" +githubbranch = "v1.27.16" docsbranch = "release-1.27" url = "https://v1-27.docs.kubernetes.io"