Merge master

cluster-autoscaler/OWNERS

@@ -1,14 +1,14 @@
 approvers:
+- aleksandra-malinowska
 - BigDarkClown
 - feiskyer
 - towca
 - x13n
 reviewers:
+- aleksandra-malinowska
 - BigDarkClown
 - feiskyer
 - vadasambar
 - x13n
-emeritus_approvers:
-- aleksandra-malinowska # 2022-09-30
 labels:
 - area/cluster-autoscaler

cluster-autoscaler/cloudprovider/azure/test/Makefile

@@ -0,0 +1,53 @@
+REPO_ROOT:=$(shell git rev-parse --show-toplevel)
+CAS_ROOT:=$(REPO_ROOT)/cluster-autoscaler
+CAS_CHART:=$(REPO_ROOT)/charts/cluster-autoscaler
+
+include $(CAS_ROOT)/Makefile
+
+TOOLS_BIN_DIR := $(abspath bin)
+
+export PATH := $(TOOLS_BIN_DIR):$(PATH)
+
+HELM_VER := v3.15.2
+HELM_BIN := helm
+HELM := $(TOOLS_BIN_DIR)/$(HELM_BIN)-$(HELM_VER)
+
+helm: $(HELM)
+
+$(HELM):
+	mkdir -p $(TOOLS_BIN_DIR)
+	rm -f "$(TOOLS_BIN_DIR)/$(HELM_BIN)*"
+	curl --retry 3 -fsSL -o $(TOOLS_BIN_DIR)/get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
+	chmod 700 $(TOOLS_BIN_DIR)/get_helm.sh
+	USE_SUDO=false HELM_INSTALL_DIR=$(TOOLS_BIN_DIR) DESIRED_VERSION=$(HELM_VER) BINARY_NAME=$(HELM_BIN)-$(HELM_VER) $(TOOLS_BIN_DIR)/get_helm.sh
+	ln -sf $(HELM) $(TOOLS_BIN_DIR)/$(HELM_BIN)
+	rm -f $(TOOLS_BIN_DIR)/get_helm.sh
+
+CLUSTER_AUTOSCALER_NAMESPACE?=default
+CLUSTER_AUTOSCALER_SERVICEACCOUNT_NAME?=cluster-autoscaler
+
+.PHONY: install-e2e
+install-e2e: $(HELM)
+	$(MAKE) -C $(CAS_ROOT) build-arch-$(GOARCH) make-image-arch-$(GOARCH)
+	docker push $(IMAGE)-$(GOARCH):$(TAG)
+	$(HELM) install cluster-autoscaler $(CAS_CHART) \
+		--namespace $(CLUSTER_AUTOSCALER_NAMESPACE) --create-namespace \
+		--set cloudProvider=azure \
+		--set azureTenantID=$(AZURE_TENANT_ID) \
+		--set azureSubscriptionID=$(AZURE_SUBSCRIPTION_ID) \
+		--set azureUseWorkloadIdentityExtension=true \
+		--set-string podLabels."azure\.workload\.identity/use"=true \
+		--set rbac.serviceAccount.name=$(CLUSTER_AUTOSCALER_SERVICEACCOUNT_NAME) \
+		--set rbac.serviceAccount.annotations."azure\.workload\.identity/tenant-id"=$(AZURE_TENANT_ID) \
+		--set rbac.serviceAccount.annotations."azure\.workload\.identity/client-id"="$$(KUBECONFIG= kubectl get userassignedidentities -o jsonpath='{.items[0].status.clientId}')" \
+		--set autoDiscovery.clusterName="$$(KUBECONFIG= kubectl get cluster -o jsonpath='{.items[0].metadata.name}')" \
+		--set azureResourceGroup="$$(KUBECONFIG= kubectl get managedclusters -o jsonpath='{.items[0].status.nodeResourceGroup}')" \
+		--set nodeSelector."kubernetes\.io/os"=linux \
+		--set image.repository=$(IMAGE)-$(GOARCH) \
+		--set image.tag=$(TAG) \
+		--set image.pullPolicy=Always \
+		--wait
+
+.PHONY: test-e2e
+test-e2e: install-e2e
+	# TODO

cluster-autoscaler/cloudprovider/azure/test/templates/cluster-template-prow-aks-aso-cluster-autoscaler.yaml

@@ -0,0 +1,271 @@
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster
+metadata:
+  name: ${CLUSTER_NAME}
+  namespace: default
+spec:
+  controlPlaneRef:
+    apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+    kind: AzureASOManagedControlPlane
+    name: ${CLUSTER_NAME}
+  infrastructureRef:
+    apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+    kind: AzureASOManagedCluster
+    name: ${CLUSTER_NAME}
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+kind: AzureASOManagedControlPlane
+metadata:
+  name: ${CLUSTER_NAME}
+  namespace: default
+spec:
+  resources:
+  - apiVersion: containerservice.azure.com/v1api20231001
+    kind: ManagedCluster
+    metadata:
+      annotations:
+        serviceoperator.azure.com/credential-from: ${ASO_CREDENTIAL_SECRET_NAME}
+      name: ${CLUSTER_NAME}
+    spec:
+      dnsPrefix: ${CLUSTER_NAME}
+      identity:
+        type: SystemAssigned
+      location: ${AZURE_LOCATION}
+      networkProfile:
+        networkPlugin: azure
+      nodeResourceGroup: MC_${CLUSTER_NAME}_${CLUSTER_NAME}_${AZURE_LOCATION}
+      oidcIssuerProfile:
+        enabled: true
+      operatorSpec:
+        configMaps:
+          oidcIssuerProfile:
+            key: issuer
+            name: ${CLUSTER_NAME}-oidc
+      owner:
+        name: ${CLUSTER_NAME}
+      securityProfile:
+        workloadIdentity:
+          enabled: true
+      servicePrincipalProfile:
+        clientId: msi
+      tags:
+        buildProvenance: ${BUILD_PROVENANCE}
+        creationTimestamp: ${TIMESTAMP}
+        jobName: ${JOB_NAME}
+  version: ${KUBERNETES_VERSION}
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+kind: AzureASOManagedCluster
+metadata:
+  name: ${CLUSTER_NAME}
+  namespace: default
+spec:
+  resources:
+  - apiVersion: resources.azure.com/v1api20200601
+    kind: ResourceGroup
+    metadata:
+      annotations:
+        serviceoperator.azure.com/credential-from: ${ASO_CREDENTIAL_SECRET_NAME}
+      name: ${CLUSTER_NAME}
+    spec:
+      location: ${AZURE_LOCATION}
+      tags:
+        buildProvenance: ${BUILD_PROVENANCE}
+        creationTimestamp: ${TIMESTAMP}
+        jobName: ${JOB_NAME}
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: MachinePool
+metadata:
+  annotations:
+    cluster.x-k8s.io/replicas-managed-by: cluster-autoscaler
+  name: ${CLUSTER_NAME}-pool0
+  namespace: default
+spec:
+  clusterName: ${CLUSTER_NAME}
+  replicas: ${WORKER_MACHINE_COUNT:=2}
+  template:
+    metadata: {}
+    spec:
+      bootstrap:
+        dataSecretName: ""
+      clusterName: ${CLUSTER_NAME}
+      infrastructureRef:
+        apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+        kind: AzureASOManagedMachinePool
+        name: ${CLUSTER_NAME}-pool0
+      version: ${KUBERNETES_VERSION}
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+kind: AzureASOManagedMachinePool
+metadata:
+  name: ${CLUSTER_NAME}-pool0
+  namespace: default
+spec:
+  resources:
+  - apiVersion: containerservice.azure.com/v1api20231001
+    kind: ManagedClustersAgentPool
+    metadata:
+      annotations:
+        serviceoperator.azure.com/credential-from: ${ASO_CREDENTIAL_SECRET_NAME}
+      name: ${CLUSTER_NAME}-pool0
+    spec:
+      azureName: pool0
+      mode: System
+      owner:
+        name: ${CLUSTER_NAME}
+      tags:
+        cluster-autoscaler-enabled: "true"
+        cluster-autoscaler-name: ${CLUSTER_NAME}
+        max: "5"
+        min: "1"
+      type: VirtualMachineScaleSets
+      vmSize: ${AZURE_AKS_NODE_MACHINE_TYPE:=Standard_D2s_v3}
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: MachinePool
+metadata:
+  annotations:
+    cluster.x-k8s.io/replicas-managed-by: cluster-autoscaler
+  name: ${CLUSTER_NAME}-pool1
+  namespace: default
+spec:
+  clusterName: ${CLUSTER_NAME}
+  replicas: ${WORKER_MACHINE_COUNT:=2}
+  template:
+    metadata: {}
+    spec:
+      bootstrap:
+        dataSecretName: ""
+      clusterName: ${CLUSTER_NAME}
+      infrastructureRef:
+        apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+        kind: AzureASOManagedMachinePool
+        name: ${CLUSTER_NAME}-pool1
+      version: ${KUBERNETES_VERSION}
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+kind: AzureASOManagedMachinePool
+metadata:
+  name: ${CLUSTER_NAME}-pool1
+  namespace: default
+spec:
+  resources:
+  - apiVersion: containerservice.azure.com/v1api20231001
+    kind: ManagedClustersAgentPool
+    metadata:
+      annotations:
+        serviceoperator.azure.com/credential-from: ${ASO_CREDENTIAL_SECRET_NAME}
+      name: ${CLUSTER_NAME}-pool1
+    spec:
+      azureName: pool1
+      mode: User
+      owner:
+        name: ${CLUSTER_NAME}
+      tags:
+        cluster-autoscaler-enabled: "true"
+        cluster-autoscaler-name: ${CLUSTER_NAME}
+        max: "5"
+        min: "1"
+      type: VirtualMachineScaleSets
+      vmSize: ${AZURE_AKS_NODE_MACHINE_TYPE:=Standard_D2s_v3}
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: MachinePool
+metadata:
+  annotations:
+    cluster.x-k8s.io/replicas-managed-by: cluster-autoscaler
+  name: ${CLUSTER_NAME}-pool2
+  namespace: default
+spec:
+  clusterName: ${CLUSTER_NAME}
+  replicas: 1
+  template:
+    metadata: {}
+    spec:
+      bootstrap:
+        dataSecretName: ""
+      clusterName: ${CLUSTER_NAME}
+      infrastructureRef:
+        apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+        kind: AzureASOManagedMachinePool
+        name: ${CLUSTER_NAME}-pool2
+      version: ${KUBERNETES_VERSION}
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+kind: AzureASOManagedMachinePool
+metadata:
+  name: ${CLUSTER_NAME}-pool2
+  namespace: default
+spec:
+  resources:
+  - apiVersion: containerservice.azure.com/v1api20231001
+    kind: ManagedClustersAgentPool
+    metadata:
+      annotations:
+        serviceoperator.azure.com/credential-from: ${ASO_CREDENTIAL_SECRET_NAME}
+      name: ${CLUSTER_NAME}-pool2
+    spec:
+      azureName: pool2
+      mode: User
+      osType: Windows
+      owner:
+        name: ${CLUSTER_NAME}
+      tags:
+        cluster-autoscaler-enabled: "true"
+        cluster-autoscaler-name: ${CLUSTER_NAME}
+        max: "5"
+        min: "1"
+      type: VirtualMachineScaleSets
+      vmSize: ${AZURE_AKS_NODE_MACHINE_TYPE:=Standard_D2s_v3}
+---
+apiVersion: managedidentity.azure.com/v1api20230131
+kind: UserAssignedIdentity
+metadata:
+  annotations:
+    serviceoperator.azure.com/credential-from: ${ASO_CREDENTIAL_SECRET_NAME}
+  name: ${CLUSTER_NAME}
+  namespace: default
+spec:
+  location: ${AZURE_LOCATION}
+  operatorSpec:
+    configMaps:
+      principalId:
+        key: principal-id
+        name: ${CLUSTER_NAME}-identity
+  owner:
+    name: ${CLUSTER_NAME}
+---
+apiVersion: managedidentity.azure.com/v1api20230131
+kind: FederatedIdentityCredential
+metadata:
+  annotations:
+    serviceoperator.azure.com/credential-from: ${ASO_CREDENTIAL_SECRET_NAME}
+  name: ${CLUSTER_NAME}
+  namespace: default
+spec:
+  audiences:
+  - api://AzureADTokenExchange
+  issuerFromConfig:
+    key: issuer
+    name: ${CLUSTER_NAME}-oidc
+  owner:
+    name: ${CLUSTER_NAME}
+  subject: system:serviceaccount:${CLUSTER_AUTOSCALER_NAMESPACE:=default}:${CLUSTER_AUTOSCALER_SERVICEACCOUNT_NAME:=cluster-autoscaler}
+---
+apiVersion: authorization.azure.com/v1api20220401
+kind: RoleAssignment
+metadata:
+  annotations:
+    serviceoperator.azure.com/credential-from: ${ASO_CREDENTIAL_SECRET_NAME}
+  name: ${CLUSTER_NAME}
+  namespace: default
+spec:
+  owner:
+    armId: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/MC_${CLUSTER_NAME}_${CLUSTER_NAME}_${AZURE_LOCATION}
+  principalIdFromConfig:
+    key: principal-id
+    name: ${CLUSTER_NAME}-identity
+  roleDefinitionReference:
+    # Contributor
+    armId: /subscriptions/${AZURE_SUBSCRIPTION_ID}/providers/Microsoft.Authorization/roleDefinitions/b24988ac-6180-42a0-ab88-20f7382dd24c

cluster-autoscaler/cloudprovider/gce/gce_manager.go

@@ -37,7 +37,7 @@ import (
 	"k8s.io/client-go/util/workqueue"
 
 	apiv1 "k8s.io/api/core/v1"
-	provider_gce "k8s.io/legacy-cloud-providers/gce"
+	provider_gce "k8s.io/cloud-provider-gcp/providers/gce"
 
 	"cloud.google.com/go/compute/metadata"
 	"golang.org/x/oauth2"

cluster-autoscaler/cloudprovider/oci/examples/oci-ip-cluster-autoscaler-w-config.yaml

@@ -34,6 +34,9 @@ rules:
     resources: ["endpoints"]
     resourceNames: ["cluster-autoscaler"]
     verbs: ["get", "update"]
+  - apiGroups: [""]
+    resources: ["namespaces"]
+    verbs: ["watch", "list", "get"]
   - apiGroups: [""]
     resources: ["nodes"]
     verbs: ["watch", "list", "get", "patch", "update"]
@@ -55,7 +58,7 @@ rules:
     resources: ["statefulsets", "replicasets", "daemonsets"]
     verbs: ["watch", "list", "get"]
   - apiGroups: ["storage.k8s.io"]
-    resources: ["storageclasses", "csinodes"]
+    resources: ["storageclasses", "csinodes", "csistoragecapacities", "csidrivers"]
     verbs: ["watch", "list", "get"]
   - apiGroups: ["batch", "extensions"]
     resources: ["jobs"]
@@ -158,6 +161,8 @@ spec:
           env:
             - name: OCI_USE_INSTANCE_PRINCIPAL
               value: "false"
+            - name: OCI_SDK_APPEND_USER_AGENT
+              value: "oci-oke-cluster-autoscaler"
           volumeMounts:
             - name: oci-config-vol
               mountPath: "/root/.oci"