fixes for azure-workload-identity - use helm in minikube + fix aws no…

… roles after last commit (#205)

aws/README.md

@@ -49,43 +49,20 @@ aws sts get-caller-identity
 aws eks --region AWS_REGION update-kubeconfig --name CLUSTER_NAME
 ```
 
-4. Provision EFS CSI Driver (optional)
-
-The `cluster.yml` file that is included in this folder uses the EBS drivers but also deploys the EFS IAM policies ie you still need to install the EFS CSI drivers. This can be done following the [AWS Docs ](https://docs.aws.amazon.com/eks/latest/userguide/efs-csi.html)
-
-5. [Provision Secrets Drivers](https://github.com/aws/secrets-store-csi-driver-provider-aws)
+4. [Provision Secrets Drivers](https://github.com/aws/secrets-store-csi-driver-provider-aws)
 
 Once the deployment has completed, please provision the Secrets Manager identity and the CSI drivers
 
-Use `quorum` (or equivalent) for `NAMESPACE` below and update `AWS_REGION` and `CLUSTER_NAME` to match your settings from step 2.
-
 ```bash
-
-helm repo add secrets-store-csi-driver https://kubernetes-sigs.github.io/secrets-store-csi-driver/charts
-helm install --namespace kube-system --create-namespace csi-secrets-store secrets-store-csi-driver/secrets-store-csi-driver
-kubectl apply -f https://raw.githubusercontent.com/aws/secrets-store-csi-driver-provider-aws/main/deployment/aws-provider-installer.yaml 
-
-POLICY_ARN=$(aws --region AWS_REGION --query Policy.Arn --output text iam create-policy --policy-name quorum-node-secrets-mgr-policy --policy-document '{
-    "Version": "2012-10-17",
-    "Statement": [ {
-        "Effect": "Allow",
-        "Action": ["secretsmanager:CreateSecret","secretsmanager:UpdateSecret","secretsmanager:DescribeSecret","secretsmanager:GetSecretValue","secretsmanager:PutSecretValue","secretsmanager:ReplicateSecretToRegions","secretsmanager:TagResource"],
-        "Resource": ["arn:aws:secretsmanager:AWS_REGION:AWS_ACCOUNT:secret:goquorum-node-*", "arn:aws:secretsmanager:AWS_REGION:AWS_ACCOUNT:secret:besu-node-*"]
-    } ]
-}')
-
-
-If you have deployed the above policy before, you can acquire its ARN:
-POLICY_ARN=$(aws iam list-policies --scope Local \
---query 'Policies[?PolicyName==`quorum-node-secrets-mgr-policy`].Arn' \
---output text)
-
-eksctl create iamserviceaccount --name quorum-node-secrets-sa --namespace NAMESPACE --region=AWS_REGION --cluster CLUSTER_NAME --attach-policy-arn "$POLICY_ARN" --approve --override-existing-serviceaccounts
+./scripts/bootstrap.sh  "AWS_REGION" "AWS_ACCOUNT" "CLUSTER_NAME" "AKS_NAMESPACE"
 ```
 
-| ⚠️ **Note**: The above command creates a service account called `quorum-node-secrets-sa`. Please use the same in the values.yml files under the `aws` map. If you would like to change the name of the service account, please remember to do it in both places |
+| ⚠️ **Note**: The above command creates a service account called `quorum-sa`. Please use the same in the values.yml files under the `aws` map. If you would like to change the name of the service account, please remember to do it in both places |
 | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 
+5. Provision EFS CSI Driver (optional)
+
+The `cluster.yml` file that is included in this folder uses the EBS drivers but also deploys the EFS IAM policies ie you still need to install the EFS CSI drivers. This can be done following the [AWS Docs ](https://docs.aws.amazon.com/eks/latest/userguide/efs-csi.html)
 
 6. Deploy the charts as per the `helm` folder readme files
 

aws/scripts/bootstrap.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+#
+# Run as:
+# ./bootstrap.sh "AWS_REGION" "AWS_ACCOUNT" "CLUSTER_NAME" "AKS_NAMESPACE"
+#
+
+set -eux
+
+AWS_REGION=${1:-rg}
+AWS_ACCOUNT=${2:-account}
+CLUSTER_NAME=${3:-cluster}
+# quourum
+AKS_NAMESPACE=${4:-quorum}
+
+echo "aws get-credentials ..."
+aws sts get-caller-identity
+aws eks --region "${AWS_REGION}" update-kubeconfig --name "${CLUSTER_NAME}"
+
+helm repo add secrets-store-csi-driver https://kubernetes-sigs.github.io/secrets-store-csi-driver/charts
+helm install --namespace kube-system --create-namespace csi-secrets-store secrets-store-csi-driver/secrets-store-csi-driver
+kubectl apply -f https://raw.githubusercontent.com/aws/secrets-store-csi-driver-provider-aws/main/deployment/aws-provider-installer.yaml 
+
+# If you have deployed the above policy before, acquire its ARN:
+POLICY_ARN=$(aws iam list-policies --scope Local --query 'Policies[?PolicyName==`quorum-node-secrets-mgr-policy`].Arn' --output text)
+if [ $? -eq 1 ] 
+then
+  echo "Deploy the policy"
+  POLICY_ARN=$(aws --region $AWS_REGION --query Policy.Arn --output text iam create-policy --policy-name quorum-node-secrets-mgr-policy --policy-document '{
+      "Version": "2012-10-17",
+      "Statement": [ {
+          "Effect": "Allow",
+          "Action": ["secretsmanager:CreateSecret","secretsmanager:UpdateSecret","secretsmanager:DescribeSecret","secretsmanager:GetSecretValue","secretsmanager:PutSecretValue","secretsmanager:ReplicateSecretToRegions","secretsmanager:TagResource"],
+          "Resource": ["arn:aws:secretsmanager:$AWS_REGION:$AWS_ACCOUNT:secret:goquorum-node-*", "arn:aws:secretsmanager:$AWS_REGION:$AWS_ACCOUNT:secret:besu-node-*"]
+      } ]
+  }')
+fi
+
+eksctl create iamserviceaccount --name quorum-sa --namespace "${NAMESPACE}" --region="${AWS_REGION}" --cluster "${CLUSTER_NAME}" --attach-policy-arn "$POLICY_ARN" --approve --override-existing-serviceaccounts
+echo "Done... "

azure/arm/azuredeploy.json

@@ -41,7 +41,7 @@
   "variables": {
     "apiVersionVirtualNetworks": "2020-05-01",
     "apiVersionNetworkSecurityGroups": "2018-11-01",
-    "apiVersionManangedClusters": "2020-09-01",
+    "apiVersionManangedClusters": "2023-06-01",
     "apiVersionUserManangedIdentity": "2018-11-30",
     "apiVersionRoleAssignments": "2018-09-01-preview",
     "apiVersionKeyVault": "2018-02-14",
@@ -73,9 +73,7 @@
     "monitoringMetricsPublisherRoleId": "3913510d-42f4-4e42-8a64-420c390055eb",
     "readerRoleId": "acdd72a7-3385-48ef-bd42-f606fba81ae7",
     "aksClusterAdminRoleId": "0ab0b1a8-8aac-4efd-b8c2-3ee1fb270be8",
-    "networkContributorRoleId": "4d97b98b-1d4f-4787-a291-c67834d212e7",
-    "serviceBusReceiverRoleId": "4f6d3b9b-027b-4f4c-9142-0e5a2a2247e0",
-    "serviceBusSenderRoleId": "69a216fc-b8fb-44d8-bc22-1f3c2cd27a39"
+    "networkContributorRoleId": "4d97b98b-1d4f-4787-a291-c67834d212e7"
   },
   "resources": [
     {
@@ -222,34 +220,6 @@
         "principalType": "ServicePrincipal"
       }
     },
-    {
-      "type": "Microsoft.Authorization/roleAssignments",
-      "apiVersion": "[variables('apiVersionRoleAssignments')]",
-      "name": "[guid(resourceGroup().id, deployment().name, variables('opsManagedIdentity'), '-sb-receiver-ra')]",
-      "dependsOn": [
-        "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('opsManagedIdentity'))]"
-      ],
-      "properties": {
-        "roleDefinitionId": "[resourceId('Microsoft.Authorization/roleDefinitions', variables('serviceBusReceiverRoleId'))]",
-        "principalId": "[reference(variables('opsManagedIdentity')).principalId]",
-        "scope": "[resourceGroup().id]",
-        "principalType": "ServicePrincipal"
-      }
-    },
-    {
-      "type": "Microsoft.Authorization/roleAssignments",
-      "apiVersion": "[variables('apiVersionRoleAssignments')]",
-      "name": "[guid(resourceGroup().id, deployment().name, variables('opsManagedIdentity'), '-sb-sender-ra')]",
-      "dependsOn": [
-        "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('opsManagedIdentity'))]"
-      ],
-      "properties": {
-        "roleDefinitionId": "[resourceId('Microsoft.Authorization/roleDefinitions', variables('serviceBusSenderRoleId'))]",
-        "principalId": "[reference(variables('opsManagedIdentity')).principalId]",
-        "scope": "[resourceGroup().id]",
-        "principalType": "ServicePrincipal"
-      }
-    },
     {
       "type": "Microsoft.Authorization/roleAssignments",
       "apiVersion": "[variables('apiVersionRoleAssignments')]",
@@ -360,19 +330,13 @@
         "servicePrincipalProfile": {
           "clientId": "msi"
         },
-        "podIdentityProfile": {
-          "enabled": true,
-          "userAssignedIdentities": [
-            {
-              "name": "quorum-pod-identity",
-              "namespace": "default",
-              "identity": {
-                "resourceId": "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('opsManagedIdentity'))]",
-                "clientId": "[reference(variables('opsManagedIdentity')).clientId]",
-                "objectId": "[reference(variables('opsManagedIdentity')).principalId]"
-              }
-            }
-          ]
+        "oidcIssuerProfile": {
+          "enabled": true
+        },
+        "securityProfile": {
+          "workloadIdentity": {
+            "enabled": true
+          }
         },
         "networkProfile": {
           "networkPlugin": "azure",

azure/scripts/bootstrap.sh

@@ -1,8 +1,5 @@
 #!/bin/bash
 #
-# This bootstraps the ops vm to run helm charts on the aks cluster.
-# This is for dev only at present, and this functionality will be moved to a lambda function
-#
 # Run as:
 # ./bootstrap.sh "AKS_RESOURCE_GROUP" "AKS_CLUSTER_NAME" "AKS_MANAGED_IDENTITY" "AKS_NAMESPACE"
 #
@@ -13,7 +10,7 @@ AKS_CLUSTER_NAME=${2:-cluster}
 AKS_MANAGED_IDENTITY=${3:-identity}
 # quourum
 AKS_NAMESPACE=${4:-quorum}
-SA_NAME=${5:-quorum}
+SA_NAME=${5:-quorum-sa}
 
 echo "az get-credentials ..."
 # if running this on a VM/Function/etc use a managed identity
@@ -22,61 +19,26 @@ echo "az get-credentials ..."
 az login
 
 # https://learn.microsoft.com/en-us/azure/aks/use-oidc-issuer
-echo "Update the cluster to use oidc issuer and workload identity ... "
-az aks update -g myResourceGroup -n myAKSCluster --enable-oidc-issuer --enable-workload-identity
-
-echo "Provisioning AAD pod-identity... "
+echo "Get the oidc issuer and workload identity ID from the cluster... "
 AKS_MANAGED_IDENTITY_RESOURCE_ID=$(az identity show --name "$AKS_MANAGED_IDENTITY"  --resource-group "$AKS_RESOURCE_GROUP" | jq -r '.id')
-AKS_OIDC_ISSUER=$(az aks show --name "$AKS_MANAGED_IDENTITY" --resource-group "$AKS_RESOURCE_GROUP"     --query "oidcIssuerProfile.issuerUrl" -otsv)
+AKS_OIDC_ISSUER=$(az aks show -n $AKS_CLUSTER_NAME -g "$AKS_RESOURCE_GROUP" --query "oidcIssuerProfile.issuerUrl" -otsv)
 
-# https://learn.microsoft.com/en-us/azure/aks/workload-identity-deploy-cluster
+# https://learn.microsoft.com/en-gb/azure/aks/workload-identity-deploy-cluster#create-kubernetes-service-account
 cat <<EOF | kubectl apply -f -
 apiVersion: v1
 kind: ServiceAccount
 metadata:
   annotations:
     azure.workload.identity/client-id: "${AKS_MANAGED_IDENTITY_RESOURCE_ID}"
-  name: "${SA_NAME}"
-  namespace: "${AKS_NAMESPACE}"
+  name: "$SA_NAME"
+  namespace: "$AKS_NAMESPACE"
 EOF
 
-cat <<EOF | kubectl apply -f -
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  namespace: "${AKS_NAMESPACE}"
-  name: "${SA_NAME}"
-rules:
-- apiGroups: [""]
-  resources: ["secrets", "configmaps"]
-  verbs: ["create", "get", "list", "update", "delete", "patch"]
-- apiGroups: [""]
-  resources: ["services"]
-  verbs: ["get", "list"]
-EOF  
-
-cat <<EOF | kubectl apply -f -
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: "${SA_NAME}"
-  namespace: "${AKS_NAMESPACE}"
-subjects:
-- kind: ServiceAccount
-  name: "${SA_NAME}"
-  namespace: "${AKS_NAMESPACE}"
-roleRef:
-  kind: Role
-  name: "${SA_NAME}"
-  apiGroup: rbac.authorization.k8s.io
-EOF   
-
-az identity federated-credential create --name aks-federated-credential --identity-name "${AKS_MANAGED_IDENTITY}" --resource-group "${RESOURCE_GROUP}" --issuer "${AKS_OIDC_ISSUER}" --subject system:serviceaccount:"${AKS_NAMESPACE}":"${SA_NAME}" --audience api://AzureADTokenExchange
-
-
+# Create the federated identity credential between the managed identity, the service account issuer, and the subject.
+az identity federated-credential create --name "${AKS_MANAGED_IDENTITY}-fc" --identity-name "${AKS_MANAGED_IDENTITY}" --resource-group "${AKS_RESOURCE_GROUP}" --issuer "${AKS_OIDC_ISSUER}" --subject system:serviceaccount:"${AKS_NAMESPACE}":"${SA_NAME}" --audience api://AzureADTokenExchange
 
 echo "Provisioning CSI drivers... "
-az aks get-credentials --resource-group "$AKS_RESOURCE_GROUP" --name "$AKS_CLUSTER_NAME" --admin
+az aks get-credentials --resource-group "${AKS_RESOURCE_GROUP}" --name "${AKS_CLUSTER_NAME}" --admin
 # Helm charts for KeyVault drivers
 helm repo add stable https://charts.helm.sh/stable
 helm repo add csi-secrets-store-provider-azure https://raw.githubusercontent.com/Azure/secrets-store-csi-driver-provider-azure/master/charts

helm/README.md

@@ -9,8 +9,8 @@ cluster:
   cloudNativeServices: false # set to true to use Cloud Native Services (SecretsManager and IAM for AWS; KeyVault & Managed Identities for Azure)
 
 aws:
-  # the aws cli commands uses the name 'quorum-node-secrets-sa' so only change this if you altered the name
-  serviceAccountName: quorum-node-secrets-sa
+  # the aws cli commands uses the name 'quorum-sa' so only change this if you altered the name
+  serviceAccountName: quorum-sa
   # the region you are deploying to
   region: ap-southeast-2
 

helm/charts/besu-genesis/templates/genesis-job-cleanup.yaml

@@ -32,11 +32,13 @@ spec:
         app.kubernetes.io/namespace: {{ .Release.Namespace }}
         app.kubernetes.io/managed-by: helm
     spec:
-{{- if and (eq .Values.cluster.provider "aws") (.Values.cluster.cloudNativeServices) }}
+{{- if and (eq .Values.cluster.provider "azure") (.Values.cluster.cloudNativeServices) }}
+      serviceAccountName: {{ .Values.azure.serviceAccountName }}
+{{- else if and (eq .Values.cluster.provider "aws") (.Values.cluster.cloudNativeServices) }}
       serviceAccountName: {{ .Values.aws.serviceAccountName }}
 {{- else }}
-      serviceAccountName: {{ .Values.azure.serviceAccountName}}
-{{- end }}        
+      serviceAccountName: {{ .Values.azure.serviceAccountName}}-sa
+{{- end }}       
       restartPolicy: "Never"
       containers:
         - name: delete-genesis

helm/charts/besu-genesis/templates/genesis-job-init.yaml

@@ -31,10 +31,12 @@ spec:
         app.kubernetes.io/namespace: {{ .Release.Namespace }}
         app.kubernetes.io/managed-by: helm
     spec:
-{{- if and (eq .Values.cluster.provider "aws") (.Values.cluster.cloudNativeServices) }}
+{{- if and (eq .Values.cluster.provider "azure") (.Values.cluster.cloudNativeServices) }}
+      serviceAccountName: {{ .Values.azure.serviceAccountName }}
+{{- else if and (eq .Values.cluster.provider "aws") (.Values.cluster.cloudNativeServices) }}
       serviceAccountName: {{ .Values.aws.serviceAccountName }}
 {{- else }}
-      serviceAccountName: {{ .Values.azure.serviceAccountName}}
+      serviceAccountName: {{ .Values.azure.serviceAccountName}}-sa
 {{- end }}
       restartPolicy: "Never"
       containers:

helm/charts/besu-genesis/templates/genesis-service-account.yaml

@@ -0,0 +1,45 @@
+
+{{- if not .Values.cluster.cloudNativeServices }}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "besu-genesis.name" . }}-sa
+  namespace: {{ .Release.Namespace }}
+
+{{- end }}
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ include "besu-genesis.name" . }}-role
+  namespace: {{ .Release.Namespace }}
+rules:
+  - apiGroups: [""]
+    resources: ["secrets", "configmaps"]
+    verbs: ["create", "get", "list", "update", "delete" ]
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list", "watch" ]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ include "besu-genesis.name" . }}-rb
+  namespace: {{ .Release.Namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ include "besu-genesis.name" . }}-role
+subjects:
+  - kind: ServiceAccount
+    namespace: {{ .Release.Namespace }}
+{{- if and (eq .Values.cluster.provider "azure") (.Values.cluster.cloudNativeServices) }}
+    name: {{ .Values.azure.serviceAccountName }}
+{{- else if and (eq .Values.cluster.provider "aws") (.Values.cluster.cloudNativeServices) }}
+    name: {{ .Values.aws.serviceAccountName }}
+{{- else }}
+    name: {{ include "besu-genesis.name" . }}-sa
+{{- end}}

helm/charts/besu-genesis/values.yaml

@@ -8,13 +8,13 @@ cluster:
   cloudNativeServices: false # set to true to use Cloud Native Services (SecretsManager and IAM for AWS; KeyVault & Managed Identities for Azure)
 
 aws:
-  # the aws cli commands uses the name 'quorum-node-secrets-sa' so only change this if you altered the name
-  serviceAccountName: quorum-node-secrets-sa
+  # the aws cli commands uses the name 'quorum-sa' so only change this if you altered the name
+  serviceAccountName: quorum-sa
   # the region you are deploying to
   region: ap-southeast-2
 
 azure:
-  serviceAccountName: quorum
+  serviceAccountName: quorum-sa
   # the clientId of the user assigned managed identity created in the template
   identityClientId: azure-clientId
   keyvaultName: azure-keyvault

helm/charts/besu-node/templates/node-hooks-pre-delete.yaml

@@ -30,11 +30,13 @@ spec:
         app.kubernetes.io/name: pre-delete-hook
         app.kubernetes.io/instance: {{ .Release.Name }} 
     spec:
+{{- if and (eq .Values.cluster.provider "azure") (.Values.cluster.cloudNativeServices) }}
+      serviceAccountName: {{ .Values.azure.serviceAccountName }}
 {{- if and (eq .Values.cluster.provider "aws") (.Values.cluster.cloudNativeServices) }}
       serviceAccountName: {{ .Values.aws.serviceAccountName }}
 {{- else }}
-      serviceAccountName: {{ .Values.azure.serviceAccountName}}
-{{- end }}    
+      serviceAccountName: {{ include "besu-node.fullname" . }}-sa
+{{- end }}
       restartPolicy: "OnFailure"
       containers:
         - name: {{ template "besu-node.fullname" . }}-node-pre-delete-hook

helm/charts/besu-node/templates/node-hooks-pre-install.yaml

@@ -30,10 +30,12 @@ spec:
         app.kubernetes.io/name: pre-install-hook
         app.kubernetes.io/instance: {{ .Release.Name }} 
     spec:
+{{- if and (eq .Values.cluster.provider "azure") (.Values.cluster.cloudNativeServices) }}
+      serviceAccountName: {{ .Values.azure.serviceAccountName }}
 {{- if and (eq .Values.cluster.provider "aws") (.Values.cluster.cloudNativeServices) }}
       serviceAccountName: {{ .Values.aws.serviceAccountName }}
 {{- else }}
-      serviceAccountName: {{ .Values.azure.serviceAccountName}}
+      serviceAccountName: {{ include "besu-node.fullname" . }}-sa
 {{- end }}
       restartPolicy: "OnFailure"
       containers: