feat: support workload identity setting in static PV mount

charts/latest/blob-csi-driver/templates/csi-blob-driver.yaml

@@ -12,3 +12,5 @@ spec:
   volumeLifecycleModes:
     - Persistent
     - Ephemeral
+  tokenRequests:
+    - audience: api://AzureADTokenExchange

docs/workload-identity-static-pv-mount.md

@@ -0,0 +1,178 @@
+# Example of static PV mount with workload identity
+
+> Note: 
+> - Available kubernetes version >= v1.20
+
+## prerequisite
+
+
+### 1. Create a cluster with oidc-issuer enabled and get the credential
+
+Following the [documentation](https://learn.microsoft.com/en-us/azure/aks/use-oidc-issuer#create-an-aks-cluster-with-oidc-issuer) to create an AKS cluster with the `--enable-oidc-issuer` parameter and get the AKS credentials. And export following environment variables:
+```
+export RESOURCE_GROUP=<your resource group name>
+export CLUSTER_NAME=<your cluster name>
+export REGION=<your region>
+```
+
+
+### 2. Create a new storage account and fileshare
+
+Following the [documentation](https://learn.microsoft.com/en-us/azure/storage/blobs/storage-quickstart-blobs-cli) to create a new storage account and container or use your own. And export following environment variables:
+```
+export STORAGE_RESOURCE_GROUP=<your storage account resource group>
+export ACCOUNT=<your storage account name>
+export CONTAINER=<your container name>
+```
+
+### 3. Create managed identity and role assignment
+```
+export UAMI=<your managed identity name>
+az identity create --name $UAMI --resource-group $RESOURCE_GROUP
+
+export USER_ASSIGNED_CLIENT_ID="$(az identity show -g $RESOURCE_GROUP --name $UAMI --query 'clientId' -o tsv)"
+export IDENTITY_TENANT=$(az aks show --name $CLUSTER_NAME --resource-group $RESOURCE_GROUP --query identity.tenantId -o tsv)
+export ACCOUNT_SCOPE=$(az storage account show --name $ACCOUNT --query id -o tsv)
+
+# please retry if you meet `Cannot find user or service principal in graph database` error, it may take a while for the identity to propagate
+az role assignment create --role "Storage Account Contributor" --assignee $USER_ASSIGNED_CLIENT_ID --scope $ACCOUNT_SCOPE
+```
+
+### 4. Create service account on AKS
+```
+export SERVICE_ACCOUNT_NAME=<your sa name>
+export SERVICE_ACCOUNT_NAMESPACE=<your sa namespace>
+
+cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: ${SERVICE_ACCOUNT_NAME}
+  namespace: ${SERVICE_ACCOUNT_NAMESPACE}
+EOF
+```
+
+### 5. Create the federated identity credential between the managed identity, service account issuer, and subject using the `az identity federated-credential create` command.
+```
+export FEDERATED_IDENTITY_NAME=<your federated identity name>
+export AKS_OIDC_ISSUER="$(az aks show --resource-group $RESOURCE_GROUP --name $CLUSTER_NAME --query "oidcIssuerProfile.issuerUrl" -o tsv)"
+
+az identity federated-credential create --name $FEDERATED_IDENTITY_NAME \
+--identity-name $UAMI \
+--resource-group $RESOURCE_GROUP \
+--issuer $AKS_OIDC_ISSUER \
+--subject system:serviceaccount:${SERVICE_ACCOUNT_NAMESPACE}:${SERVICE_ACCOUNT_NAME}
+```
+
+## option#1: static provision with PV
+```
+cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  annotations:
+    pv.kubernetes.io/provisioned-by: blob.csi.azure.com
+  name: pv-blob
+spec:
+  capacity:
+    storage: 10Gi
+  accessModes:
+    - ReadWriteMany
+  persistentVolumeReclaimPolicy: Retain
+  storageClassName: blob-fuse
+  mountOptions:
+    - -o allow_other
+    - --file-cache-timeout-in-seconds=120
+  csi:
+    driver: blob.csi.azure.com
+    # make sure volumeid is unique for every storage blob container in the cluster
+    # the # character is reserved for internal use, the / character is not allowed
+    volumeHandle: unique_volume_id
+    volumeAttributes:
+      storageaccount: $ACCOUNT # required
+      containerName: $CONTAINER  # required
+      clientID: $USER_ASSIGNED_CLIENT_ID # required
+      resourcegroup: $STORAGE_RESOURCE_GROUP # optional, specified when the storage account is not under AKS node resource group(which is prefixed with "MC_")
+      # tenantID: $IDENTITY_TENANT  #optional, only specified when workload identity and AKS cluster are in different tenant
+      # subscriptionid: $SUBSCRIPTION #optional, only specified when workload identity and AKS cluster are in different subscription
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: statefulset-blob
+  labels:
+    app: nginx
+spec:
+  serviceName: statefulset-blob
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      serviceAccountName: $SERVICE_ACCOUNT_NAME  #required, Pod does not use this service account has no permission to mount the volume
+      nodeSelector:
+        "kubernetes.io/os": linux
+      containers:
+        - name: statefulset-blob
+          image: mcr.microsoft.com/oss/nginx/nginx:1.19.5
+          command:
+            - "/bin/bash"
+            - "-c"
+            - set -euo pipefail; while true; do echo $(date) >> /mnt/blob/outfile; sleep 1; done
+          volumeMounts:
+            - name: persistent-storage
+              mountPath: /mnt/blob
+              readOnly: false
+  updateStrategy:
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      app: nginx
+  volumeClaimTemplates:
+    - metadata:
+        name: persistent-storage
+      spec:
+        storageClassName: blob-fuse
+        accessModes: ["ReadWriteMany"]
+        resources:
+          requests:
+            storage: 10Gi
+EOF
+```
+
+## option#2: Pod with ephemeral inline volume
+```
+cat <<EOF | kubectl apply -f -
+kind: Pod
+apiVersion: v1
+metadata:
+  name: nginx-blobfuse-inline-volume
+spec:
+  serviceAccountName: $SERVICE_ACCOUNT_NAME  #required, Pod does not use this service account has no permission to mount the volume
+  nodeSelector:
+    "kubernetes.io/os": linux
+  containers:
+    - image: mcr.microsoft.com/oss/nginx/nginx:1.19.5
+      name: nginx-blobfuse
+      command:
+        - "/bin/bash"
+        - "-c"
+        - set -euo pipefail; while true; do echo $(date) >> /mnt/blobfuse/outfile; sleep 1; done
+      volumeMounts:
+        - name: persistent-storage
+          mountPath: "/mnt/blobfuse"
+          readOnly: false
+  volumes:
+    - name: persistent-storage
+      csi:
+        driver: blob.csi.azure.com
+        volumeAttributes:
+          storageaccount: $ACCOUNT # required
+          containerName: $CONTAINER  # required
+          clientID: $USER_ASSIGNED_CLIENT_ID # required
+          resourcegroup: $STORAGE_RESOURCE_GROUP # optional, specified when the storage account is not under AKS node resource group(which is prefixed with "MC_")
+          # tenantID: $IDENTITY_TENANT  # optional, only specified when workload identity and AKS cluster are in different tenant
+          # subscriptionid: $SUBSCRIPTION # optional, only specified when workload identity and AKS cluster are in different subscription
+EOF
+```

go.mod

@@ -31,7 +31,7 @@ require (
 	k8s.io/kubernetes v1.28.5
 	k8s.io/mount-utils v0.28.4
 	k8s.io/utils v0.0.0-20231127182322-b307cd553661
-	sigs.k8s.io/cloud-provider-azure v1.27.1-0.20231208022044-b9ede3fc98e9
+	sigs.k8s.io/cloud-provider-azure v1.27.1-0.20231213062409-f1ce7de3fdcb
 	sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader v0.0.0-20231208022044-b9ede3fc98e9
 	sigs.k8s.io/yaml v1.4.0
 )

go.sum

@@ -473,8 +473,8 @@ k8s.io/utils v0.0.0-20231127182322-b307cd553661 h1:FepOBzJ0GXm8t0su67ln2wAZjbQ6R
 k8s.io/utils v0.0.0-20231127182322-b307cd553661/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2 h1:trsWhjU5jZrx6UvFu4WzQDrN7Pga4a7Qg+zcfcj64PA=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2/go.mod h1:+qG7ISXqCDVVcyO8hLn12AKVYYUjM7ftlqsqmrhMZE0=
-sigs.k8s.io/cloud-provider-azure v1.27.1-0.20231208022044-b9ede3fc98e9 h1:UybRilKUwfcg3CZh51++O/e6ppBRdT/UY0TjGJfWkPw=
-sigs.k8s.io/cloud-provider-azure v1.27.1-0.20231208022044-b9ede3fc98e9/go.mod h1:BsbV0DptIzi3NdbPXIxruq9TRI4RSp49eV4CFXBssy4=
+sigs.k8s.io/cloud-provider-azure v1.27.1-0.20231213062409-f1ce7de3fdcb h1:YApm24ngCVkpQTUxu0/wYV/oiccfqWEPZnX1BbpadKY=
+sigs.k8s.io/cloud-provider-azure v1.27.1-0.20231213062409-f1ce7de3fdcb/go.mod h1:UkVMiNELbKLa07K/ubQ+vg8AK3XFyd2FMr5vCIYk0Pg=
 sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.0-20231205023417-1ba5a224ab0e h1:U001A7jnOOi8eiYceYeCLK2S3rTX4K2atR8uNDw+SL8=
 sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.0-20231205023417-1ba5a224ab0e/go.mod h1:dckGAqm0wUQNqqvCEeWhfXKL7DB/r9zchDq9xdcF/Qk=
 sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader v0.0.0-20231208022044-b9ede3fc98e9 h1:0XsdZlKjVI0UZYhvg3VbXCPFRRQS5VL1idrTKgzJjnc=

pkg/blob/blob.go

@@ -96,6 +96,9 @@ const (
 	requireInfraEncryptionField    = "requireinfraencryption"
 	ephemeralField                 = "csi.storage.k8s.io/ephemeral"
 	podNamespaceField              = "csi.storage.k8s.io/pod.namespace"
+	serviceAccountTokenField       = "csi.storage.k8s.io/serviceAccount.tokens"
+	clientIDField                  = "clientID"
+	tenantIDField                  = "tenantID"
 	mountOptionsField              = "mountoptions"
 	falseValue                     = "false"
 	trueValue                      = "true"
@@ -446,6 +449,9 @@ func (d *Driver) GetAuthEnv(ctx context.Context, volumeID, protocol string, attr
 		authEnv                 []string
 		getAccountKeyFromSecret bool
 		getLatestAccountKey     bool
+		clientID                string
+		tenantID                string
+		serviceAccountToken     string
 	)
 
 	for k, v := range attrib {
@@ -495,6 +501,12 @@ func (d *Driver) GetAuthEnv(ctx context.Context, volumeID, protocol string, attr
 			if getLatestAccountKey, err = strconv.ParseBool(v); err != nil {
 				return rgName, accountName, accountKey, containerName, authEnv, fmt.Errorf("invalid %s: %s in volume context", getLatestAccountKeyField, v)
 			}
+		case strings.ToLower(clientIDField):
+			clientID = v
+		case strings.ToLower(tenantIDField):
+			tenantID = v
+		case strings.ToLower(serviceAccountTokenField):
+			serviceAccountToken = v
 		}
 	}
 	klog.V(2).Infof("volumeID(%s) authEnv: %s", volumeID, authEnv)
@@ -516,6 +528,21 @@ func (d *Driver) GetAuthEnv(ctx context.Context, volumeID, protocol string, attr
 		rgName = d.cloud.ResourceGroup
 	}
 
+	if tenantID == "" {
+		tenantID = d.cloud.TenantID
+	}
+
+	// if client id is specified, we only use service account token to get account key
+	if clientID != "" {
+		klog.V(2).Infof("clientID(%s) is specified, use service account token to get account key", clientID)
+		if subsID == "" {
+			subsID = d.cloud.SubscriptionID
+		}
+		accountKey, err := d.cloud.GetStorageAccesskeyFromServiceAccountToken(ctx, subsID, accountName, rgName, clientID, tenantID, serviceAccountToken)
+		authEnv = append(authEnv, "AZURE_STORAGE_ACCESS_KEY="+accountKey)
+		return rgName, accountName, accountKey, containerName, authEnv, err
+	}
+
 	// 1. If keyVaultURL is not nil, preferentially use the key stored in key vault.
 	// 2. Then if secrets map is not nil, use the key stored in the secrets map.
 	// 3. Finally if both keyVaultURL and secrets map are nil, get the key from Azure.

pkg/blob/nodeserver.go

@@ -81,6 +81,19 @@ func (d *Driver) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolu
 	mountPermissions := d.mountPermissions
 	context := req.GetVolumeContext()
 	if context != nil {
+		// token request
+		if context[serviceAccountTokenField] != "" && getClientID(context) != "" {
+			klog.V(2).Infof("NodePublishVolume: volume(%s) mount on %s with service account token, clientID: %s", volumeID, target, getClientID(context))
+			_, err := d.NodeStageVolume(ctx, &csi.NodeStageVolumeRequest{
+				StagingTargetPath: target,
+				VolumeContext:     context,
+				VolumeCapability:  volCap,
+				VolumeId:          volumeID,
+			})
+			return &csi.NodePublishVolumeResponse{}, err
+		}
+
+		// ephemeral volume
 		if strings.EqualFold(context[ephemeralField], trueValue) {
 			setKeyValueInMap(context, secretNamespaceField, context[podNamespaceField])
 			if !d.allowInlineVolumeKeyAccessWithIdentity {
@@ -241,6 +254,11 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe
 	attrib := req.GetVolumeContext()
 	secrets := req.GetSecrets()
 
+	if getClientID(attrib) != "" && attrib[serviceAccountTokenField] == "" {
+		klog.V(2).Infof("Skip NodeStageVolume for volume(%s) since clientID %s is provided but service account token is empty", volumeID, getClientID(attrib))
+		return &csi.NodeStageVolumeResponse{}, nil
+	}
+
 	mc := metrics.NewMetricContext(blobCSIDriverName, "node_stage_volume", d.cloud.ResourceGroup, "", d.Name)
 	isOperationSucceeded := false
 	defer func() {
@@ -676,6 +694,15 @@ func waitForMount(path string, intervel, timeout time.Duration) error {
 	}
 }
 
+func getClientID(context map[string]string) string {
+	for k, v := range context {
+		if strings.EqualFold(k, clientIDField) && v != "" {
+			return v
+		}
+	}
+	return ""
+}
+
 func checkGidPresentInMountFlags(mountFlags []string) bool {
 	for _, mountFlag := range mountFlags {
 		if strings.Contains(mountFlag, "gid=") {

pkg/blob/nodeserver_test.go

@@ -832,6 +832,59 @@ func Test_waitForMount(t *testing.T) {
 	}
 }
 
+func Test_getClientID(t *testing.T) {
+	type args struct {
+		context map[string]string
+	}
+	tests := []struct {
+		name string
+		args args
+		want string
+	}{
+		{
+			name: "get client id",
+			args: args{
+				context: map[string]string{
+					clientIDField: "test-client-id",
+				},
+			},
+			want: "test-client-id",
+		},
+		{
+			name: "case not sensitive client id",
+			args: args{
+				context: map[string]string{
+					"ClientId": "test-client-id",
+				},
+			},
+			want: "test-client-id",
+		},
+		{
+			name: "no client id",
+			args: args{
+				context: map[string]string{},
+			},
+			want: "",
+		},
+		{
+			name: "client id empty",
+			args: args{
+				context: map[string]string{
+					clientIDField: "",
+				},
+			},
+			want: "",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := getClientID(tt.args.context); got != tt.want {
+				t.Errorf("getClientID() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
 func TestCheckGidPresentInMountFlags(t *testing.T) {
 	tests := []struct {
 		desc       string

vendor/modules.txt

@@ -1541,7 +1541,7 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/metrics
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/common/metrics
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client
-# sigs.k8s.io/cloud-provider-azure v1.27.1-0.20231208022044-b9ede3fc98e9
+# sigs.k8s.io/cloud-provider-azure v1.27.1-0.20231213062409-f1ce7de3fdcb
 ## explicit; go 1.21
 sigs.k8s.io/cloud-provider-azure/pkg/azureclients
 sigs.k8s.io/cloud-provider-azure/pkg/azureclients/armclient