From 2ee2acfd31be2465b1ccd2aa17cefca9b4b52252 Mon Sep 17 00:00:00 2001 From: Fabio Sforza Date: Mon, 4 Nov 2024 14:37:06 +0100 Subject: [PATCH] feat: support Istio add-on in AKS (#3584) ## Description This Pull Request enable Istio add-on on AKS. Following features are enabled: - Istio Revisions Selection - Internal Ingress Gateway - External Ingress Gateway - Plug your own CA from Key Vault Closes #3363 ## Pipeline Reference | Pipeline | | -------- | | [![avm.res.container-service.managed-cluster](https://github.com/fsforza/bicep-registry-modules/actions/workflows/avm.res.container-service.managed-cluster.yml/badge.svg)](https://github.com/fsforza/bicep-registry-modules/actions/workflows/avm.res.container-service.managed-cluster.yml) | ## Type of Change - [ ] Update to CI Environment or utilities (Non-module affecting changes) - [ ] Azure Verified Module updates: - [ ] Bugfix containing backwards-compatible bug fixes, and I have NOT bumped the MAJOR or MINOR version in `version.json`: - [ ] Someone has opened a bug report issue, and I have included "Closes #{bug_report_issue_number}" in the PR description. - [ ] The bug was found by the module author, and no one has opened an issue to report it yet. - [x] Feature update backwards compatible feature updates, and I have bumped the MINOR version in `version.json`. - [ ] Breaking changes and I have bumped the MAJOR version in `version.json`. - [ ] Update to documentation ## Checklist - [x] I'm sure there are no other open Pull Requests for the same update/change - [x] I have run `Set-AVMModule` locally to generate the supporting module files. - [x] My corresponding pipelines / checks run clean and green without any errors or warnings --- .../managed-cluster/README.md | 256 +++++++++++++++++- .../managed-cluster/main.bicep | 65 ++++- .../managed-cluster/main.json | 84 +++++- .../tests/e2e/istio/dependencies.bicep | 96 +++++++ .../tests/e2e/istio/main.rbac.bicep | 22 ++ .../tests/e2e/istio/main.test.bicep | 91 +++++++ .../Set-CertificateAuthorityInKeyVault.ps1 | 168 ++++++++++++ .../managed-cluster/version.json | 4 +- 8 files changed, 771 insertions(+), 15 deletions(-) create mode 100644 avm/res/container-service/managed-cluster/tests/e2e/istio/dependencies.bicep create mode 100644 avm/res/container-service/managed-cluster/tests/e2e/istio/main.rbac.bicep create mode 100644 avm/res/container-service/managed-cluster/tests/e2e/istio/main.test.bicep create mode 100644 avm/res/container-service/managed-cluster/tests/e2e/istio/scripts/Set-CertificateAuthorityInKeyVault.ps1 diff --git a/avm/res/container-service/managed-cluster/README.md b/avm/res/container-service/managed-cluster/README.md index 8fc9f67ba1..a5f83980ea 100644 --- a/avm/res/container-service/managed-cluster/README.md +++ b/avm/res/container-service/managed-cluster/README.md @@ -35,9 +35,10 @@ The following section provides usage examples for the module, which were used to - [Using only defaults and use AKS Automatic mode](#example-1-using-only-defaults-and-use-aks-automatic-mode) - [Using Azure CNI Network Plugin.](#example-2-using-azure-cni-network-plugin) - [Using only defaults](#example-3-using-only-defaults) -- [Using Kubenet Network Plugin.](#example-4-using-kubenet-network-plugin) -- [Using Private Cluster.](#example-5-using-private-cluster) -- [WAF-aligned](#example-6-waf-aligned) +- [Using Istio Service Mesh add-on](#example-4-using-istio-service-mesh-add-on) +- [Using Kubenet Network Plugin.](#example-5-using-kubenet-network-plugin) +- [Using Private Cluster.](#example-6-using-private-cluster) +- [WAF-aligned](#example-7-waf-aligned) ### Example 1: _Using only defaults and use AKS Automatic mode_ @@ -1203,7 +1204,162 @@ param managedIdentities = {

-### Example 4: _Using Kubenet Network Plugin._ +### Example 4: _Using Istio Service Mesh add-on_ + +This instance deploys the module with Istio Service Mesh add-on and plug a Certificate Authority from Key Vault. + + +

+ +via Bicep module + +```bicep +module managedCluster 'br/public:avm/res/container-service/managed-cluster:' = { + name: 'managedClusterDeployment' + params: { + // Required parameters + name: 'csist001' + primaryAgentPoolProfiles: [ + { + count: 3 + mode: 'System' + name: 'systempool' + vmSize: 'Standard_DS2_v2' + } + ] + // Non-required parameters + enableKeyvaultSecretsProvider: true + enableSecretRotation: true + istioServiceMeshCertificateAuthority: { + certChainObjectName: '' + certObjectName: '' + keyObjectName: '' + keyVaultResourceId: '' + rootCertObjectName: '' + } + istioServiceMeshEnabled: true + istioServiceMeshInternalIngressGatewayEnabled: true + istioServiceMeshRevisions: [ + 'asm-1-22' + ] + location: '' + managedIdentities: { + systemAssigned: true + } + } +} +``` + +
+

+ +

+ +via JSON parameters file + +```json +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + // Required parameters + "name": { + "value": "csist001" + }, + "primaryAgentPoolProfiles": { + "value": [ + { + "count": 3, + "mode": "System", + "name": "systempool", + "vmSize": "Standard_DS2_v2" + } + ] + }, + // Non-required parameters + "enableKeyvaultSecretsProvider": { + "value": true + }, + "enableSecretRotation": { + "value": true + }, + "istioServiceMeshCertificateAuthority": { + "value": { + "certChainObjectName": "", + "certObjectName": "", + "keyObjectName": "", + "keyVaultResourceId": "", + "rootCertObjectName": "" + } + }, + "istioServiceMeshEnabled": { + "value": true + }, + "istioServiceMeshInternalIngressGatewayEnabled": { + "value": true + }, + "istioServiceMeshRevisions": { + "value": [ + "asm-1-22" + ] + }, + "location": { + "value": "" + }, + "managedIdentities": { + "value": { + "systemAssigned": true + } + } + } +} +``` + +
+

+ +

+ +via Bicep parameters file + +```bicep-params +using 'br/public:avm/res/container-service/managed-cluster:' + +// Required parameters +param name = 'csist001' +param primaryAgentPoolProfiles = [ + { + count: 3 + mode: 'System' + name: 'systempool' + vmSize: 'Standard_DS2_v2' + } +] +// Non-required parameters +param enableKeyvaultSecretsProvider = true +param enableSecretRotation = true +param istioServiceMeshCertificateAuthority = { + certChainObjectName: '' + certObjectName: '' + keyObjectName: '' + keyVaultResourceId: '' + rootCertObjectName: '' +} +param istioServiceMeshEnabled = true +param istioServiceMeshInternalIngressGatewayEnabled = true +param istioServiceMeshRevisions = [ + 'asm-1-22' +] +param location = '' +param managedIdentities = { + systemAssigned: true +} +``` + +
+

+ +### Example 5: _Using Kubenet Network Plugin._ This instance deploys the module with Kubenet network plugin . @@ -1602,7 +1758,7 @@ param tags = {

-### Example 5: _Using Private Cluster._ +### Example 6: _Using Private Cluster._ This instance deploys the module with a private cluster instance. @@ -1912,7 +2068,7 @@ param skuTier = 'Standard'

-### Example 6: _WAF-aligned_ +### Example 7: _WAF-aligned_ This instance deploys the module in alignment with the best-practices of the Well-Architected Framework. @@ -2552,6 +2708,11 @@ param tags = { | [`identityProfile`](#parameter-identityprofile) | object | Identities associated with the cluster. | | [`imageCleanerIntervalHours`](#parameter-imagecleanerintervalhours) | int | The interval in hours Image Cleaner will run. The maximum value is three months. | | [`ingressApplicationGatewayEnabled`](#parameter-ingressapplicationgatewayenabled) | bool | Specifies whether the ingressApplicationGateway (AGIC) add-on is enabled or not. | +| [`istioServiceMeshCertificateAuthority`](#parameter-istioservicemeshcertificateauthority) | object | The Istio Certificate Authority definition. | +| [`istioServiceMeshEnabled`](#parameter-istioservicemeshenabled) | bool | Specifies whether the Istio ServiceMesh add-on is enabled or not. | +| [`istioServiceMeshExternalIngressGatewayEnabled`](#parameter-istioservicemeshexternalingressgatewayenabled) | bool | Specifies whether the External Istio Ingress Gateway is enabled or not. | +| [`istioServiceMeshInternalIngressGatewayEnabled`](#parameter-istioservicemeshinternalingressgatewayenabled) | bool | Specifies whether the Internal Istio Ingress Gateway is enabled or not. | +| [`istioServiceMeshRevisions`](#parameter-istioservicemeshrevisions) | array | The list of revisions of the Istio control plane. When an upgrade is not in progress, this holds one value. When canary upgrade is in progress, this can only hold two consecutive values. | | [`kedaAddon`](#parameter-kedaaddon) | bool | Enables Kubernetes Event-driven Autoscaling (KEDA). | | [`kubeDashboardEnabled`](#parameter-kubedashboardenabled) | bool | Specifies whether the kubeDashboard add-on is enabled or not. | | [`kubernetesVersion`](#parameter-kubernetesversion) | string | Version of Kubernetes specified when creating the managed cluster. | @@ -4173,6 +4334,89 @@ Specifies whether the ingressApplicationGateway (AGIC) add-on is enabled or not. - Type: bool - Default: `False` +### Parameter: `istioServiceMeshCertificateAuthority` + +The Istio Certificate Authority definition. + +- Required: No +- Type: object + +**Required parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`certChainObjectName`](#parameter-istioservicemeshcertificateauthoritycertchainobjectname) | string | The Certificate chain object name in Azure Key Vault. | +| [`certObjectName`](#parameter-istioservicemeshcertificateauthoritycertobjectname) | string | The Intermediate certificate object name in Azure Key Vault. | +| [`keyObjectName`](#parameter-istioservicemeshcertificateauthoritykeyobjectname) | string | The Intermediate certificate private key object name in Azure Key Vault. | +| [`keyVaultResourceId`](#parameter-istioservicemeshcertificateauthoritykeyvaultresourceid) | string | The resource ID of a key vault to reference a Certificate Authority from. | +| [`rootCertObjectName`](#parameter-istioservicemeshcertificateauthorityrootcertobjectname) | string | Root certificate object name in Azure Key Vault. | + +### Parameter: `istioServiceMeshCertificateAuthority.certChainObjectName` + +The Certificate chain object name in Azure Key Vault. + +- Required: Yes +- Type: string + +### Parameter: `istioServiceMeshCertificateAuthority.certObjectName` + +The Intermediate certificate object name in Azure Key Vault. + +- Required: Yes +- Type: string + +### Parameter: `istioServiceMeshCertificateAuthority.keyObjectName` + +The Intermediate certificate private key object name in Azure Key Vault. + +- Required: Yes +- Type: string + +### Parameter: `istioServiceMeshCertificateAuthority.keyVaultResourceId` + +The resource ID of a key vault to reference a Certificate Authority from. + +- Required: Yes +- Type: string + +### Parameter: `istioServiceMeshCertificateAuthority.rootCertObjectName` + +Root certificate object name in Azure Key Vault. + +- Required: Yes +- Type: string + +### Parameter: `istioServiceMeshEnabled` + +Specifies whether the Istio ServiceMesh add-on is enabled or not. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `istioServiceMeshExternalIngressGatewayEnabled` + +Specifies whether the External Istio Ingress Gateway is enabled or not. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `istioServiceMeshInternalIngressGatewayEnabled` + +Specifies whether the Internal Istio Ingress Gateway is enabled or not. + +- Required: No +- Type: bool +- Default: `False` + +### Parameter: `istioServiceMeshRevisions` + +The list of revisions of the Istio control plane. When an upgrade is not in progress, this holds one value. When canary upgrade is in progress, this can only hold two consecutive values. + +- Required: No +- Type: array + ### Parameter: `kedaAddon` Enables Kubernetes Event-driven Autoscaling (KEDA). diff --git a/avm/res/container-service/managed-cluster/main.bicep b/avm/res/container-service/managed-cluster/main.bicep index 4cd9c21330..ea126b9dea 100644 --- a/avm/res/container-service/managed-cluster/main.bicep +++ b/avm/res/container-service/managed-cluster/main.bicep @@ -405,6 +405,21 @@ param metricLabelsAllowlist string = '' @description('Optional. A comma-separated list of Kubernetes cluster metrics annotations.') param metricAnnotationsAllowList string = '' +@description('Optional. Specifies whether the Istio ServiceMesh add-on is enabled or not.') +param istioServiceMeshEnabled bool = false + +@description('Optional. The list of revisions of the Istio control plane. When an upgrade is not in progress, this holds one value. When canary upgrade is in progress, this can only hold two consecutive values.') +param istioServiceMeshRevisions array? + +@description('Optional. Specifies whether the Internal Istio Ingress Gateway is enabled or not.') +param istioServiceMeshInternalIngressGatewayEnabled bool = false + +@description('Optional. Specifies whether the External Istio Ingress Gateway is enabled or not.') +param istioServiceMeshExternalIngressGatewayEnabled bool = false + +@description('Optional. The Istio Certificate Authority definition.') +param istioServiceMeshCertificateAuthority istioServiceMeshCertificateAuthorityType + // =========== // // Variables // // =========== // @@ -826,6 +841,37 @@ resource managedCluster 'Microsoft.ContainerService/managedClusters@2024-03-02-p } } supportPlan: supportPlan + serviceMeshProfile: istioServiceMeshEnabled + ? { + istio: { + revisions: !empty(istioServiceMeshRevisions) ? istioServiceMeshRevisions : null + components: { + ingressGateways: [ + { + enabled: istioServiceMeshInternalIngressGatewayEnabled + mode: 'Internal' + } + { + enabled: istioServiceMeshExternalIngressGatewayEnabled + mode: 'External' + } + ] + } + certificateAuthority: !empty(istioServiceMeshCertificateAuthority) + ? { + plugin: { + certChainObjectName: istioServiceMeshCertificateAuthority.?certChainObjectName + certObjectName: istioServiceMeshCertificateAuthority.?certObjectName + keyObjectName: istioServiceMeshCertificateAuthority.?keyObjectName + keyVaultId: istioServiceMeshCertificateAuthority.?keyVaultResourceId + rootCertObjectName: istioServiceMeshCertificateAuthority.?rootCertObjectName + } + } + : null + } + mode: 'Istio' + } + : null } } @@ -1291,4 +1337,21 @@ type maintenanceConfigurationType = { @description('Required. Maintenance window for the maintenance configuration.') maintenanceWindow: object -} +}? + +type istioServiceMeshCertificateAuthorityType = { + @description('Required. The resource ID of a key vault to reference a Certificate Authority from.') + keyVaultResourceId: string + + @description('Required. The Certificate chain object name in Azure Key Vault.') + certChainObjectName: string + + @description('Required. The Intermediate certificate object name in Azure Key Vault.') + certObjectName: string + + @description('Required. The Intermediate certificate private key object name in Azure Key Vault.') + keyObjectName: string + + @description('Required. Root certificate object name in Azure Key Vault.') + rootCertObjectName: string +}? diff --git a/avm/res/container-service/managed-cluster/main.json b/avm/res/container-service/managed-cluster/main.json index 314963ebd2..40b0d76ed4 100644 --- a/avm/res/container-service/managed-cluster/main.json +++ b/avm/res/container-service/managed-cluster/main.json @@ -6,7 +6,7 @@ "_generator": { "name": "bicep", "version": "0.30.23.60470", - "templateHash": "17374623663141250391" + "templateHash": "543007463534644066" }, "name": "Azure Kubernetes Service (AKS) Managed Clusters", "description": "This module deploys an Azure Kubernetes Service (AKS) Managed Cluster.", @@ -705,9 +705,46 @@ } } }, + "nullable": true, "metadata": { "__bicep_export!": true } + }, + "istioServiceMeshCertificateAuthorityType": { + "type": "object", + "properties": { + "keyVaultResourceId": { + "type": "string", + "metadata": { + "description": "Required. The resource ID of a key vault to reference a Certificate Authority from." + } + }, + "certChainObjectName": { + "type": "string", + "metadata": { + "description": "Required. The Certificate chain object name in Azure Key Vault." + } + }, + "certObjectName": { + "type": "string", + "metadata": { + "description": "Required. The Intermediate certificate object name in Azure Key Vault." + } + }, + "keyObjectName": { + "type": "string", + "metadata": { + "description": "Required. The Intermediate certificate private key object name in Azure Key Vault." + } + }, + "rootCertObjectName": { + "type": "string", + "metadata": { + "description": "Required. Root certificate object name in Azure Key Vault." + } + } + }, + "nullable": true } }, "parameters": { @@ -1564,6 +1601,40 @@ "metadata": { "description": "Optional. A comma-separated list of Kubernetes cluster metrics annotations." } + }, + "istioServiceMeshEnabled": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Specifies whether the Istio ServiceMesh add-on is enabled or not." + } + }, + "istioServiceMeshRevisions": { + "type": "array", + "nullable": true, + "metadata": { + "description": "Optional. The list of revisions of the Istio control plane. When an upgrade is not in progress, this holds one value. When canary upgrade is in progress, this can only hold two consecutive values." + } + }, + "istioServiceMeshInternalIngressGatewayEnabled": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Specifies whether the Internal Istio Ingress Gateway is enabled or not." + } + }, + "istioServiceMeshExternalIngressGatewayEnabled": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Specifies whether the External Istio Ingress Gateway is enabled or not." + } + }, + "istioServiceMeshCertificateAuthority": { + "$ref": "#/definitions/istioServiceMeshCertificateAuthorityType", + "metadata": { + "description": "Optional. The Istio Certificate Authority definition." + } } }, "variables": { @@ -1802,7 +1873,8 @@ "enabled": "[parameters('enableStorageProfileSnapshotController')]" } }, - "supportPlan": "[parameters('supportPlan')]" + "supportPlan": "[parameters('supportPlan')]", + "serviceMeshProfile": "[if(parameters('istioServiceMeshEnabled'), createObject('istio', createObject('revisions', if(not(empty(parameters('istioServiceMeshRevisions'))), parameters('istioServiceMeshRevisions'), null()), 'components', createObject('ingressGateways', createArray(createObject('enabled', parameters('istioServiceMeshInternalIngressGatewayEnabled'), 'mode', 'Internal'), createObject('enabled', parameters('istioServiceMeshExternalIngressGatewayEnabled'), 'mode', 'External'))), 'certificateAuthority', if(not(empty(parameters('istioServiceMeshCertificateAuthority'))), createObject('plugin', createObject('certChainObjectName', tryGet(parameters('istioServiceMeshCertificateAuthority'), 'certChainObjectName'), 'certObjectName', tryGet(parameters('istioServiceMeshCertificateAuthority'), 'certObjectName'), 'keyObjectName', tryGet(parameters('istioServiceMeshCertificateAuthority'), 'keyObjectName'), 'keyVaultId', tryGet(parameters('istioServiceMeshCertificateAuthority'), 'keyVaultResourceId'), 'rootCertObjectName', tryGet(parameters('istioServiceMeshCertificateAuthority'), 'rootCertObjectName'))), null())), 'mode', 'Istio'), null())]" } }, "managedCluster_lock": { @@ -1935,8 +2007,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.28.1.47646", - "templateHash": "14041521425406296021" + "version": "0.30.23.60470", + "templateHash": "2505380725266419010" }, "name": "Azure Kubernetes Service (AKS) Managed Cluster Maintenance Configurations", "description": "This module deploys an Azure Kubernetes Service (AKS) Managed Cluster Maintenance Configurations.", @@ -2132,8 +2204,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.28.1.47646", - "templateHash": "9210340338669366768" + "version": "0.30.23.60470", + "templateHash": "13856766172443517827" }, "name": "Azure Kubernetes Service (AKS) Managed Cluster Agent Pools", "description": "This module deploys an Azure Kubernetes Service (AKS) Managed Cluster Agent Pool.", diff --git a/avm/res/container-service/managed-cluster/tests/e2e/istio/dependencies.bicep b/avm/res/container-service/managed-cluster/tests/e2e/istio/dependencies.bicep new file mode 100644 index 0000000000..ff0496b6e1 --- /dev/null +++ b/avm/res/container-service/managed-cluster/tests/e2e/istio/dependencies.bicep @@ -0,0 +1,96 @@ +@description('Optional. The location to deploy to.') +param location string = resourceGroup().location + +@description('Required. The name of the Key Vault to create.') +param keyVaultName string + +@description('Required. The name of the organization to which the Root Certificate is issued. It helps identify the legal entity that owns the root certificate') +param rootOrganization string + +@description('Required. The name of the organization to which the Certificate Authority is issued. It helps identify the legal entity that owns the certificate authority') +param caOrganization string + +@description('Required. The subject distinguished name is the name of the user of the certificate authority. The distinguished name for the certificate is a textual representation of the subject or issuer of the certificate') +param caSubjectName string + +@description('Required. The name of the Managed Identity to create.') +param managedIdentityName string + +@description('Required. The name of the Deployment Script to create for the Certificate generation.') +param cacertDeploymentScriptName string + +@description('Optional. Do not provide a value. Used to force the deployment script to rerun on every redeployment.') +param utcValue string = utcNow() + +resource keyVault 'Microsoft.KeyVault/vaults@2022-07-01' = { + name: keyVaultName + location: location + properties: { + sku: { + family: 'A' + name: 'standard' + } + tenantId: tenant().tenantId + enablePurgeProtection: null + enabledForTemplateDeployment: true + enabledForDiskEncryption: true + enabledForDeployment: true + enableRbacAuthorization: true + accessPolicies: [] + } +} + +resource managedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2018-11-30' = { + name: managedIdentityName + location: location +} + +resource keyPermissions 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid('msi-${managedIdentity.name}-KeyVault-Admin-RoleAssignment') + scope: keyVault + properties: { + principalId: managedIdentity.properties.principalId + roleDefinitionId: subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + '00482a5a-887f-4fb3-b363-3b7fe8e74483' + ) // Key Vault Administrator + principalType: 'ServicePrincipal' + } +} + +resource cacertDeploymentScript 'Microsoft.Resources/deploymentScripts@2020-10-01' = { + name: cacertDeploymentScriptName + location: location + kind: 'AzurePowerShell' + identity: { + type: 'UserAssigned' + userAssignedIdentities: { + '${managedIdentity.id}': {} + } + } + properties: { + azPowerShellVersion: '11.0' + retentionInterval: 'P1D' + forceUpdateTag: utcValue + arguments: ' -KeyVaultName "${keyVault.name}" -RootOrganization "${rootOrganization}" -CAOrganization "${caOrganization}" -CertSubjectName "${caSubjectName}"' + scriptContent: loadTextContent('scripts/Set-CertificateAuthorityInKeyVault.ps1') + } +} + +@description('The resource ID of the created Key Vault.') +output keyVaultResourceId string = keyVault.id + +@description('The name of the root certificate secret.') +output rootCertSecretName string = cacertDeploymentScript.properties.outputs.rootCertSecretName + +@description('The name of the certiticate authority key secret.') +output caKeySecretName string = cacertDeploymentScript.properties.outputs.caKeySecretName + +@description('The name of the certificate authority cert secret.') +output caCertSecretName string = cacertDeploymentScript.properties.outputs.caCertSecretName + +@description('The name of the certificate chain secret.') +output certChainSecretName string = cacertDeploymentScript.properties.outputs.certChainSecretName + +@description('The principal ID of the created Managed Identity.') +output managedIdentityPrincipalId string = managedIdentity.properties.principalId diff --git a/avm/res/container-service/managed-cluster/tests/e2e/istio/main.rbac.bicep b/avm/res/container-service/managed-cluster/tests/e2e/istio/main.rbac.bicep new file mode 100644 index 0000000000..56a1220b2f --- /dev/null +++ b/avm/res/container-service/managed-cluster/tests/e2e/istio/main.rbac.bicep @@ -0,0 +1,22 @@ +@description('The resource ID of the Key Vault.') +param keyVaultResourceId string + +@description('The principal ID of the managed identity.') +param principalId string + +resource keyVault 'Microsoft.KeyVault/vaults@2022-07-01' existing = { + name: last(split(keyVaultResourceId, '/')) +} + +resource secretPermissions 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + scope: keyVault + name: guid('msi-${principalId}-KeyVault-Secret-User-RoleAssignment') + properties: { + principalId: principalId + roleDefinitionId: subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + '4633458b-17de-408a-b874-0445c86b69e6' + ) // Key Vault Secrets User + principalType: 'ServicePrincipal' + } +} diff --git a/avm/res/container-service/managed-cluster/tests/e2e/istio/main.test.bicep b/avm/res/container-service/managed-cluster/tests/e2e/istio/main.test.bicep new file mode 100644 index 0000000000..9753edb426 --- /dev/null +++ b/avm/res/container-service/managed-cluster/tests/e2e/istio/main.test.bicep @@ -0,0 +1,91 @@ +targetScope = 'subscription' + +metadata name = 'Using Istio Service Mesh add-on' +metadata description = 'This instance deploys the module with Istio Service Mesh add-on and plug a Certificate Authority from Key Vault.' + +// ========== // +// Parameters // +// ========== // + +@description('Optional. The name of the resource group to deploy for testing purposes.') +@maxLength(90) +param resourceGroupName string = 'dep-${namePrefix}-containerservice.managedclusters-${serviceShort}-rg' + +@description('Optional. The location to deploy resources to.') +param resourceLocation string = deployment().location + +@description('Optional. A short identifier for the kind of deployment. Should be kept short to not run into resource-name length-constraints.') +param serviceShort string = 'csist' + +@description('Optional. A token to inject into the name of each resource.') +param namePrefix string = '#_namePrefix_#' + +// ============ // +// Dependencies // +// ============ // + +// General resources +// ================= +resource resourceGroup 'Microsoft.Resources/resourceGroups@2022-09-01' = { + name: resourceGroupName + location: resourceLocation +} + +module nestedDependencies 'dependencies.bicep' = { + scope: resourceGroup + name: '${uniqueString(deployment().name, resourceLocation)}-nestedDependencies' + params: { + rootOrganization: 'Istio' + caOrganization: 'Istio' + caSubjectName: 'istiod.aks-istio.system.svc' + cacertDeploymentScriptName: 'dep-${namePrefix}-ds-${serviceShort}' + keyVaultName: 'dep-${namePrefix}-kv-${serviceShort}' + managedIdentityName: 'dep-${namePrefix}-msi-${serviceShort}' + } +} + +@batchSize(1) +module testDeployment '../../../main.bicep' = [ + for iteration in ['init', 'idem']: { + scope: resourceGroup + name: '${uniqueString(deployment().name, resourceLocation)}-test-${serviceShort}-${iteration}' + params: { + name: '${namePrefix}${serviceShort}001' + location: resourceLocation + managedIdentities: { + systemAssigned: true + } + primaryAgentPoolProfiles: [ + { + name: 'systempool' + count: 3 + vmSize: 'Standard_DS2_v2' + mode: 'System' + } + ] + istioServiceMeshEnabled: true + istioServiceMeshInternalIngressGatewayEnabled: true + istioServiceMeshRevisions: [ + 'asm-1-22' + ] + istioServiceMeshCertificateAuthority: { + certChainObjectName: nestedDependencies.outputs.certChainSecretName + certObjectName: nestedDependencies.outputs.caCertSecretName + keyObjectName: nestedDependencies.outputs.caKeySecretName + keyVaultResourceId: nestedDependencies.outputs.keyVaultResourceId + rootCertObjectName: nestedDependencies.outputs.rootCertSecretName + } + enableKeyvaultSecretsProvider: true + enableSecretRotation: true + } + } +] + +module secretPermissions 'main.rbac.bicep' = { + scope: resourceGroup + name: '${uniqueString(deployment().name, resourceLocation)}-rbac' + params: { + keyVaultResourceId: nestedDependencies.outputs.keyVaultResourceId + principalId: testDeployment[0].outputs.keyvaultIdentityObjectId + } +} diff --git a/avm/res/container-service/managed-cluster/tests/e2e/istio/scripts/Set-CertificateAuthorityInKeyVault.ps1 b/avm/res/container-service/managed-cluster/tests/e2e/istio/scripts/Set-CertificateAuthorityInKeyVault.ps1 new file mode 100644 index 0000000000..d61d7c8933 --- /dev/null +++ b/avm/res/container-service/managed-cluster/tests/e2e/istio/scripts/Set-CertificateAuthorityInKeyVault.ps1 @@ -0,0 +1,168 @@ +<# +.SYNOPSIS +Generate a new Certificate Authority and store it as secret in the given Key Vault. + +.DESCRIPTION +Generate a new Certificate Authority and store it as secret in the given Key Vault. + +.PARAMETER KeyVaultName +Mandatory. The name of the Key Vault to add a new certificate to, or fetch the secret reference it from + +.PARAMETER RootOrganization +Mandatory. The name of the organization to which the Root Certificate is issued. It helps identify the legal entity that owns the root certificate + +.PARAMETER CAOrganization +Mandatory. The name of the organization to which the Certificate Authority is issued. It helps identify the legal entity that owns the certificate authority + +.PARAMETER CertSubjectName +Mandatory. The subject distinguished name is the name of the user of the certificate authority. The distinguished name for the certificate is a textual representation of the subject or issuer of the certificate + +.EXAMPLE +./Set-CertificateAuthorityInKeyVault.ps1 -KeyVaultName 'myVault' -RootOrganization 'Istio' -CAOrganization 'Istio' -CertSubjectName 'istiod.aks-istio-system.com' + +Generate a Certificate Authority and store it in the Key Vault 'myVault' with the provided organizations and subject name +#> +param( + [Parameter(Mandatory = $true)] + [string] $KeyVaultName, + + [Parameter(Mandatory = $true)] + [string] $RootOrganization, + + [Parameter(Mandatory = $true)] + [string] $CAOrganization, + + [Parameter(Mandatory = $true)] + [string] $CertSubjectName +) + +$rootKeyFile = 'root-key.pem' +$rootKeySize = 4096 + +Write-Verbose ('Generating root key [{0}]' -f $rootKeyFile) -Verbose + +openssl genrsa -out $rootKeyFile $rootKeySize + +$rootKeyContent = Get-Content -Path $rootKeyFile -Raw +$rootKeyContentSecureString = ConvertTo-SecureString -String $rootKeyContent -AsPlainText -Force +$rootKeySecretName = 'root-key' +Set-AzKeyVaultSecret -VaultName $KeyVaultName -Name $rootKeySecretName -SecretValue $rootKeyContentSecureString + +$rootConfFile = 'root-ca.conf' +$rootCommonName = 'Root CA' +$rootConfContent = @" +[ req ] +encrypt_key = no +prompt = no +utf8 = yes +default_md = sha256 +default_bits = $($rootKeySize) +req_extensions = req_ext +x509_extensions = req_ext +distinguished_name = req_dn +[ req_ext ] +subjectKeyIdentifier = hash +basicConstraints = critical, CA:true +keyUsage = critical, digitalSignature, nonRepudiation, keyEncipherment, keyCertSign +[ req_dn ] +O = $($RootOrganization) +CN = $($rootCommonName) +"@ + +Write-Verbose ('Generating openssl config file [{0}]' -f $rootConfFile) -Verbose + +$rootConfContent | Set-Content -Path $rootConfFile + +$rootCertCSRFile = 'root-cert.csr' + +Write-Verbose ('Generating certificate signing request [{0}]' -f $rootCertCSRFile) -Verbose + +openssl req -sha256 -new -key $rootKeyFile -config $rootConfFile -out $rootCertCSRFile + +$rootCertFile = 'root-cert.pem' +$rootCertDays = 3650 + +Write-Verbose ('Generating root cert [{0}]' -f $rootCertFile) -Verbose + +openssl x509 -req -sha256 -days $rootCertDays -signkey $rootKeyFile -extensions req_ext -extfile $rootConfFile -in $rootCertCSRFile -out $rootCertFile + +$rootCertContent = Get-Content -Path $rootCertFile -Raw +$rootCertContentSecureString = ConvertTo-SecureString -String $rootCertContent -AsPlainText -Force +$rootCertSecretName = 'root-cert' +Set-AzKeyVaultSecret -VaultName $KeyVaultName -Name $rootCertSecretName -SecretValue $rootCertContentSecureString + +$caKeyFile = 'ca-key.pem' +$caKeySize = '4096' + +Write-Verbose ('Generating ca key [{0}]' -f $caKeyFile) -Verbose + +openssl genrsa -out $caKeyFile $caKeySize + +$caKeyContent = Get-Content -Path $caKeyFile -Raw +$caKeyContentSecureString = ConvertTo-SecureString -String $caKeyContent -AsPlainText -Force +$caKeySecretName = 'ca-key' +Set-AzKeyVaultSecret -VaultName $KeyVaultName -Name $caKeySecretName -SecretValue $caKeyContentSecureString + +$caConfFile = 'ca.conf' +$caCommonName = 'Intermediate CA' +$caConfLocation = Split-Path -Leaf (Get-Location) +$caConfContent = @" +[ req ] +encrypt_key = no +prompt = no +utf8 = yes +default_md = sha256 +default_bits = $($caKeySize) +req_extensions = req_ext +x509_extensions = req_ext +distinguished_name = req_dn +[ req_ext ] +subjectKeyIdentifier = hash +basicConstraints = critical, CA:true, pathlen:0 +keyUsage = critical, digitalSignature, nonRepudiation, keyEncipherment, keyCertSign +subjectAltName=@san +[ san ] +DNS.1 = $($CertSubjectName) +[ req_dn ] +O = $($CAOrganization) +CN = $($caCommonName) +L = $($caConfLocation) +"@ + +Write-Verbose ('Generating openssl config file [{0}]' -f $caConfFile) -Verbose + +$caConfContent | Set-Content -Path $caConfFile + +$caCertCSRFile = 'ca-cert.csr' + +Write-Verbose ('Generating certificate signing request [{0}]' -f $caCertCSRFile) -Verbose + +openssl req -sha256 -new -key $caKeyFile -config $caConfFile -out $caCertCSRFile + +$caCertFile = 'ca-cert.pem' +$caCertDays = 3650 + +Write-Verbose ('Generating ca cert [{0}]' -f $caCertFile) -Verbose + +openssl x509 -req -sha256 -days $caCertDays -CA $rootCertFile -CAkey $rootKeyFile -CAcreateserial -extensions req_ext -extfile $caConfFile -in $caCertCSRFile -out $caCertFile + +$caCertContent = Get-Content -Path $caCertFile -Raw +$caCertContentSecureString = ConvertTo-SecureString -String $caCertContent -AsPlainText -Force +$caCertSecretName = 'ca-cert' +Set-AzKeyVaultSecret -VaultName $KeyVaultName -Name $caCertSecretName -SecretValue $caCertContentSecureString + +Write-Verbose 'Generating cert chain' -Verbose + +$certChainContent = $caCertContent + $rootCertContent +$certChainContentSecureString = ConvertTo-SecureString -String $certChainContent -AsPlainText -Force +$certChainSecretName = 'cert-chain' +Set-AzKeyVaultSecret -VaultName $KeyVaultName -Name $certChainSecretName -SecretValue $certChainContentSecureString + +# Write into Deployment Script output stream +$DeploymentScriptOutputs = @{ + rootKeySecretName = $rootKeySecretName + rootCertSecretName = $rootCertSecretName + caKeySecretName = $caKeySecretName + caCertSecretName = $caCertSecretName + certChainSecretName = $certChainSecretName +} diff --git a/avm/res/container-service/managed-cluster/version.json b/avm/res/container-service/managed-cluster/version.json index 13669e6601..41fc8c654f 100644 --- a/avm/res/container-service/managed-cluster/version.json +++ b/avm/res/container-service/managed-cluster/version.json @@ -1,7 +1,7 @@ { "$schema": "https://aka.ms/bicep-registry-module-version-file-schema#", - "version": "0.4", + "version": "0.5", "pathFilters": [ "./main.json" ] -} \ No newline at end of file +}