feat: AKS node pool KubeletConfig

azure/converters/managedagentpool.go

@@ -23,7 +23,7 @@ import (
 // AgentPoolToManagedClusterAgentPoolProfile converts a AgentPoolSpec to an Azure SDK ManagedClusterAgentPoolProfile used in managedcluster reconcile.
 func AgentPoolToManagedClusterAgentPoolProfile(pool containerservice.AgentPool) containerservice.ManagedClusterAgentPoolProfile {
 	properties := pool.ManagedClusterAgentPoolProfileProperties
-	return containerservice.ManagedClusterAgentPoolProfile{
+	agentPool := containerservice.ManagedClusterAgentPoolProfile{
 		Name:                 pool.Name, // Note: if converting from agentPoolSpec.Parameters(), this field will not be set
 		VMSize:               properties.VMSize,
 		OsType:               properties.OsType,
@@ -47,4 +47,8 @@ func AgentPoolToManagedClusterAgentPoolProfile(pool containerservice.AgentPool)
 		ScaleSetPriority:     properties.ScaleSetPriority,
 		Tags:                 properties.Tags,
 	}
+	if properties.KubeletConfig != nil {
+		agentPool.KubeletConfig = properties.KubeletConfig
+	}
+	return agentPool
 }

azure/scope/managedmachinepool.go

@@ -206,6 +206,24 @@ func buildAgentPoolSpec(managedControlPlane *infrav1exp.AzureManagedControlPlane
 		}
 	}
 
+	if managedMachinePool.Spec.KubeletConfig != nil {
+		agentPoolSpec.KubeletConfig = &agentpools.KubeletConfig{
+			CPUManagerPolicy:      (*string)(managedMachinePool.Spec.KubeletConfig.CPUManagerPolicy),
+			CPUCfsQuota:           managedMachinePool.Spec.KubeletConfig.CPUCfsQuota,
+			CPUCfsQuotaPeriod:     managedMachinePool.Spec.KubeletConfig.CPUCfsQuotaPeriod,
+			ImageGcHighThreshold:  managedMachinePool.Spec.KubeletConfig.ImageGcHighThreshold,
+			ImageGcLowThreshold:   managedMachinePool.Spec.KubeletConfig.ImageGcLowThreshold,
+			TopologyManagerPolicy: (*string)(managedMachinePool.Spec.KubeletConfig.TopologyManagerPolicy),
+			FailSwapOn:            managedMachinePool.Spec.KubeletConfig.FailSwapOn,
+			ContainerLogMaxSizeMB: managedMachinePool.Spec.KubeletConfig.ContainerLogMaxSizeMB,
+			ContainerLogMaxFiles:  managedMachinePool.Spec.KubeletConfig.ContainerLogMaxFiles,
+			PodMaxPids:            managedMachinePool.Spec.KubeletConfig.PodMaxPids,
+		}
+		if len(managedMachinePool.Spec.KubeletConfig.AllowedUnsafeSysctls) > 0 {
+			agentPoolSpec.KubeletConfig.AllowedUnsafeSysctls = &managedMachinePool.Spec.KubeletConfig.AllowedUnsafeSysctls
+		}
+	}
+
 	return agentPoolSpec
 }
 

azure/services/agentpools/spec.go

@@ -30,6 +30,32 @@ import (
 	azureutil "sigs.k8s.io/cluster-api-provider-azure/util/azure"
 )
 
+// KubeletConfig defines the set of kubelet configurations for nodes in pools.
+type KubeletConfig struct {
+	// CPUManagerPolicy - CPU Manager policy to use.
+	CPUManagerPolicy *string
+	// CPUCfsQuota - Enable CPU CFS quota enforcement for containers that specify CPU limits.
+	CPUCfsQuota *bool
+	// CPUCfsQuotaPeriod - Sets CPU CFS quota period value.
+	CPUCfsQuotaPeriod *string
+	// ImageGcHighThreshold - The percent of disk usage after which image garbage collection is always run.
+	ImageGcHighThreshold *int32
+	// ImageGcLowThreshold - The percent of disk usage before which image garbage collection is never run.
+	ImageGcLowThreshold *int32
+	// TopologyManagerPolicy - Topology Manager policy to use.
+	TopologyManagerPolicy *string
+	// AllowedUnsafeSysctls - Allowlist of unsafe sysctls or unsafe sysctl patterns (ending in `*`).
+	AllowedUnsafeSysctls *[]string
+	// FailSwapOn - If set to true it will make the Kubelet fail to start if swap is enabled on the node.
+	FailSwapOn *bool
+	// ContainerLogMaxSizeMB - The maximum size (e.g. 10Mi) of container log file before it is rotated.
+	ContainerLogMaxSizeMB *int32
+	// ContainerLogMaxFiles - The maximum number of container log files that can be present for a container. The number must be ≥ 2.
+	ContainerLogMaxFiles *int32
+	// PodMaxPids - The maximum number of processes per pod.
+	PodMaxPids *int32
+}
+
 // AgentPoolSpec contains agent pool specification details.
 type AgentPoolSpec struct {
 	// Name is the name of agent pool.
@@ -101,6 +127,9 @@ type AgentPoolSpec struct {
 	// ScaleSetPriority specifies the ScaleSetPriority for the node pool. Allowed values are 'Spot' and 'Regular'
 	ScaleSetPriority *string `json:"scaleSetPriority,omitempty"`
 
+	// KubeletConfig specifies the kubelet configurations for nodes.
+	KubeletConfig *KubeletConfig `json:"kubeletConfig,omitempty"`
+
 	// AdditionalTags is an optional set of tags to add to Azure resources managed by the Azure provider, in addition to the ones added by default.
 	AdditionalTags infrav1.Tags
 }
@@ -153,6 +182,7 @@ func (s *AgentPoolSpec) Parameters(existing interface{}) (params interface{}, er
 				NodeLabels:          existingPool.NodeLabels,
 				NodeTaints:          existingPool.NodeTaints,
 				Tags:                existingPool.Tags,
+				KubeletConfig:       existingPool.KubeletConfig,
 			},
 		}
 
@@ -170,6 +200,22 @@ func (s *AgentPoolSpec) Parameters(existing interface{}) (params interface{}, er
 			},
 		}
 
+		if s.KubeletConfig != nil {
+			normalizedProfile.KubeletConfig = &containerservice.KubeletConfig{
+				CPUManagerPolicy:      s.KubeletConfig.CPUManagerPolicy,
+				CPUCfsQuota:           s.KubeletConfig.CPUCfsQuota,
+				CPUCfsQuotaPeriod:     s.KubeletConfig.CPUCfsQuotaPeriod,
+				ImageGcHighThreshold:  s.KubeletConfig.ImageGcHighThreshold,
+				ImageGcLowThreshold:   s.KubeletConfig.ImageGcLowThreshold,
+				TopologyManagerPolicy: s.KubeletConfig.TopologyManagerPolicy,
+				FailSwapOn:            s.KubeletConfig.FailSwapOn,
+				ContainerLogMaxSizeMB: s.KubeletConfig.ContainerLogMaxSizeMB,
+				ContainerLogMaxFiles:  s.KubeletConfig.ContainerLogMaxFiles,
+				PodMaxPids:            s.KubeletConfig.PodMaxPids,
+				AllowedUnsafeSysctls:  s.KubeletConfig.AllowedUnsafeSysctls,
+			}
+		}
+
 		// When autoscaling is set, the count of the nodes differ based on the autoscaler and should not depend on the
 		// count present in MachinePool or AzureManagedMachinePool, hence we should not make an update API call based
 		// on difference in count.
@@ -208,12 +254,30 @@ func (s *AgentPoolSpec) Parameters(existing interface{}) (params interface{}, er
 		vnetSubnetID = &s.VnetSubnetID
 	}
 
-	return containerservice.AgentPool{
+	var kubeletConfig *containerservice.KubeletConfig
+	if s.KubeletConfig != nil {
+		kubeletConfig = &containerservice.KubeletConfig{
+			CPUManagerPolicy:      s.KubeletConfig.CPUManagerPolicy,
+			CPUCfsQuota:           s.KubeletConfig.CPUCfsQuota,
+			CPUCfsQuotaPeriod:     s.KubeletConfig.CPUCfsQuotaPeriod,
+			ImageGcHighThreshold:  s.KubeletConfig.ImageGcHighThreshold,
+			ImageGcLowThreshold:   s.KubeletConfig.ImageGcLowThreshold,
+			TopologyManagerPolicy: s.KubeletConfig.TopologyManagerPolicy,
+			FailSwapOn:            s.KubeletConfig.FailSwapOn,
+			ContainerLogMaxSizeMB: s.KubeletConfig.ContainerLogMaxSizeMB,
+			ContainerLogMaxFiles:  s.KubeletConfig.ContainerLogMaxFiles,
+			PodMaxPids:            s.KubeletConfig.PodMaxPids,
+			AllowedUnsafeSysctls:  s.KubeletConfig.AllowedUnsafeSysctls,
+		}
+	}
+
+	agentPool := containerservice.AgentPool{
 		ManagedClusterAgentPoolProfileProperties: &containerservice.ManagedClusterAgentPoolProfileProperties{
 			AvailabilityZones:    availabilityZones,
 			Count:                &s.Replicas,
 			EnableAutoScaling:    s.EnableAutoScaling,
 			EnableUltraSSD:       s.EnableUltraSSD,
+			KubeletConfig:        kubeletConfig,
 			MaxCount:             s.MaxCount,
 			MaxPods:              s.MaxPods,
 			MinCount:             s.MinCount,
@@ -232,7 +296,9 @@ func (s *AgentPoolSpec) Parameters(existing interface{}) (params interface{}, er
 			NodePublicIPPrefixID: s.NodePublicIPPrefixID,
 			Tags:                 converters.TagsToMap(s.AdditionalTags),
 		},
-	}, nil
+	}
+
+	return agentPool, nil
 }
 
 // mergeSystemNodeLabels appends any kubernetes.azure.com-prefixed labels from the AKS label set

config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedmachinepools.yaml

@@ -220,6 +220,75 @@ spec:
                 description: EnableUltraSSD enables the storage type UltraSSD_LRS
                   for the agent pool.
                 type: boolean
+              kubeletConfig:
+                description: KubeletConfig specifies the kubelet configurations for
+                  nodes.
+                properties:
+                  allowedUnsafeSysctls:
+                    description: AllowedUnsafeSysctls - Allowlist of unsafe sysctls
+                      or unsafe sysctl patterns (ending in `*`).
+                    items:
+                      type: string
+                    type: array
+                  containerLogMaxFiles:
+                    description: ContainerLogMaxFiles - The maximum number of container
+                      log files that can be present for a container. The number must
+                      be ≥ 2.
+                    format: int32
+                    minimum: 2
+                    type: integer
+                  containerLogMaxSizeMB:
+                    description: ContainerLogMaxSizeMB - The maximum size (e.g. 10Mi)
+                      of container log file before it is rotated.
+                    format: int32
+                    type: integer
+                  cpuCfsQuota:
+                    description: CPUCfsQuota - Enable CPU CFS quota enforcement for
+                      containers that specify CPU limits.
+                    type: boolean
+                  cpuCfsQuotaPeriod:
+                    description: CPUCfsQuotaPeriod - Sets CPU CFS quota period value.
+                    type: string
+                  cpuManagerPolicy:
+                    description: CPUManagerPolicy - CPU Manager policy to use.
+                    enum:
+                    - none
+                    - static
+                    type: string
+                  failSwapOn:
+                    description: FailSwapOn - If set to true it will make the Kubelet
+                      fail to start if swap is enabled on the node.
+                    type: boolean
+                  imageGcHighThreshold:
+                    description: ImageGcHighThreshold - The percent of disk usage
+                      after which image garbage collection is always run.
+                    format: int32
+                    maximum: 100
+                    minimum: 0
+                    type: integer
+                  imageGcLowThreshold:
+                    description: ImageGcLowThreshold - The percent of disk usage before
+                      which image garbage collection is never run.
+                    format: int32
+                    maximum: 100
+                    minimum: 0
+                    type: integer
+                  podMaxPids:
+                    description: PodMaxPids - The maximum number of processes per
+                      pod.
+                    format: int32
+                    minimum: -1
+                    type: integer
+                  topologyManagerPolicy:
+                    description: TopologyManagerPolicy - Topology Manager policy to
+                      use.
+                    enum:
+                    - none
+                    - best-effort
+                    - restricted
+                    - single-numa-node
+                    type: string
+                type: object
               maxPods:
                 description: MaxPods specifies the kubelet --max-pods configuration
                   for the node pool.

docs/book/src/topics/managedcluster.md

@@ -401,6 +401,59 @@ spec:
   osType: Windows
 ```
 
+### AKS Node Pool Kubelet Custom Configuration
+
+Reference:
+
+- https://learn.microsoft.com/en-us/azure/aks/custom-node-configuration
+
+When you create your node pool (`AzureManagedMachinePool`), you may specify various kubelet configuration which tunes the kubelet runtime on all nodes in that pool. For example:
+
+```yaml
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: AzureManagedMachinePool
+metadata:
+  name: pool1
+spec:
+  mode: User
+  kubeletConfig:
+    cpuManagerPolicy: "static"
+    cpuCfsQuota: true
+    cpuCfsQuotaPeriod: "110ms"
+    imageGcHighThreshold: 70
+    imageGcLowThreshold: 50
+    topologyManagerPolicy: "best-effort"
+    allowedUnsafeSysctls:
+      - "net.*"
+      - "kernel.msg*"
+    failSwapOn: false
+    containerLogMaxSizeMB: 500
+    containerLogMaxFiles: 50
+    podMaxPids: 2048
+```
+
+Below are the full set of AKS-supported kubeletConfig configurations. All properties are children of the `spec.kubeletConfig` configuration in an `AzureManagedMachinePool` resource:
+
+| Configuration               | Property Type     | Allowed Value(s)                                                                         |
+|-----------------------------|-------------------|------------------------------------------------------------------------------------------|
+| `cpuManagerPolicy`          | string            | `"none"`, `"static"`                                                                     |
+| `cpuCfsQuota`               | boolean           | `true`, `false`                                                                          |
+| `cpuCfsQuotaPeriod`         | string            | value in milliseconds, must end in `"ms"`, e.g., `"100ms"`                               |
+| `failSwapOn`                | boolean           | `true`, `false`                                                                          |
+| `imageGcHighThreshold`      | integer           | integer values in the range 0-100 (inclusive)                                            |
+| `imageGcLowThreshold`       | integer           | integer values in the range 0-100 (inclusive), must be lower than `imageGcHighThreshold` |
+| `topologyManagerPolicy`     | string            | `"none"`, `"best-effort"`, `"restricted"`, `"single-numa-node"`                          |
+| `allowedUnsafeSysctls`      | string            | `"kernel.shm*"`, `"kernel.msg*"`, `"kernel.sem"`, `"fs.mqueue.*"`, `"net.*"`             |
+| `containerLogMaxSizeMB`     | integer           | any integer                                                                              |
+| `containerLogMaxFiles`      | integer           | any integer >= `2`                                                                       |
+| `podMaxPids`                | integer           | any integer >= `-1`, note that this must not be higher than kernel PID limit             |
+
+For more detailed information on the behaviors of the above configurations, see [the official Kubernetes documentation](https://kubernetes.io/docs/reference/config-api/kubelet-config.v1beta1/). Note that not all possible Kubernetes Kubelet Configuration options are available to use on your AKS node pool, only those specified above.
+
+CAPZ will not assign any default values for any excluded configuration properties. It is also not required to include the `spec.kubeletConfig` configuration in an `AzureManagedMachinePool` resource spec. In cases where no CAPZ configuration is declared, AKS will apply its own opinionated default configurations when the node pool is created.
+
+Note: these configurations can not be updated after a node pool is created.
+
 ### Enable AKS features with custom headers (--aks-custom-headers)
 
 To enable some AKS cluster / node pool features you need to pass special headers to the cluster / node pool create request.
@@ -508,6 +561,7 @@ Following is the list of immutable fields for managed clusters:
 | AzureManagedMachinePool   | .spec.osType                 |                           |
 | AzureManagedMachinePool   | .spec.enableNodePublicIP     |                           |
 | AzureManagedMachinePool   | .spec.nodePublicIPPrefixID   |                           |
+| AzureManagedMachinePool   | .spec.kubeletConfig          |                           |
 
 ## Features
 

exp/api/v1alpha3/azuremanagedmachinepool_conversion.go

@@ -49,6 +49,9 @@ func (src *AzureManagedMachinePool) ConvertTo(dstRaw conversion.Hub) error {
 	dst.Spec.NodePublicIPPrefixID = restored.Spec.NodePublicIPPrefixID
 	dst.Spec.ScaleSetPriority = restored.Spec.ScaleSetPriority
 	dst.Spec.AdditionalTags = restored.Spec.AdditionalTags
+	if restored.Spec.KubeletConfig != nil {
+		dst.Spec.KubeletConfig = restored.Spec.KubeletConfig
+	}
 
 	dst.Status.LongRunningOperationStates = restored.Status.LongRunningOperationStates
 	dst.Status.Conditions = restored.Status.Conditions

exp/api/v1alpha4/azuremanagedmachinepool_conversion.go

@@ -49,6 +49,9 @@ func (src *AzureManagedMachinePool) ConvertTo(dstRaw conversion.Hub) error {
 	dst.Spec.NodePublicIPPrefixID = restored.Spec.NodePublicIPPrefixID
 	dst.Spec.ScaleSetPriority = restored.Spec.ScaleSetPriority
 	dst.Spec.AdditionalTags = restored.Spec.AdditionalTags
+	if restored.Spec.KubeletConfig != nil {
+		dst.Spec.KubeletConfig = restored.Spec.KubeletConfig
+	}
 
 	dst.Status.LongRunningOperationStates = restored.Status.LongRunningOperationStates
 	dst.Status.Conditions = restored.Status.Conditions