Migrate Logging configuration to JSON parameters file (Azure#236)

cds-snc · Apr 8, 2022 · bfe1f58 · bfe1f58
1 parent cc5f017
commit bfe1f58
Show file tree

Hide file tree

Showing 51 changed files with 1,515 additions and 292 deletions.
diff --git a/.github/workflows/consistency-check.yml b/.github/workflows/consistency-check.yml
@@ -7,6 +7,7 @@ on:
 
 env:
   SCHEMA_FOLDER: schemas/latest/landingzones
+  LOGGING_PATH_FROM_ROOT: config/logging
   SUBSCRIPTIONS_PATH_FROM_ROOT: config/subscriptions
 
 jobs:
@@ -48,6 +49,14 @@ jobs:
           # Validate existing subscription archetype parameter files
           Write-Host "Validate existing subscription archetype parameter files..."
 
+          $LoggingFileFilter="*.json"
+          $LoggingSchemaFile="${{env.SCHEMA_FOLDER}}/lz-platform-logging.json"
+
+          Get-ChildItem -Recurse -Filter $LoggingFileFilter -Path "${{env.LOGGING_PATH_FROM_ROOT}}" | ForEach-Object {
+              Write-Host "Validating: $_ with $LoggingSchemaFile"
+              Get-Content -Raw $_ | Test-Json -SchemaFile $LoggingSchemaFile
+          }
+
           $GenericSubscriptionFileFilter="*generic-subscription*.json"
           $GenericSubscriptionSchemaFile="${{env.SCHEMA_FOLDER}}/lz-generic-subscription.json"
 

diff --git a/.github/workflows/pull-request-check.yml b/.github/workflows/pull-request-check.yml
@@ -10,6 +10,7 @@ on:
       - "**.json"
 env:
   SCHEMA_FOLDER: schemas/latest/landingzones
+  LOGGING_PATH_FROM_ROOT: config/logging
   SUBSCRIPTIONS_PATH_FROM_ROOT: config/subscriptions
 
 jobs:
@@ -51,6 +52,14 @@ jobs:
           # Validate existing subscription archetype parameter files
           Write-Host "Validate existing subscription archetype parameter files..."
 
+          $LoggingFileFilter="*.json"
+          $LoggingSchemaFile="${{env.SCHEMA_FOLDER}}/lz-platform-logging.json"
+
+          Get-ChildItem -Recurse -Filter $LoggingFileFilter -Path "${{env.LOGGING_PATH_FROM_ROOT}}" | ForEach-Object {
+              Write-Host "Validating: $_ with $LoggingSchemaFile"
+              Get-Content -Raw $_ | Test-Json -SchemaFile $LoggingSchemaFile
+          }
+
           $GenericSubscriptionFileFilter="*generic-subscription*.json"
           $GenericSubscriptionSchemaFile="${{env.SCHEMA_FOLDER}}/lz-generic-subscription.json"
 

diff --git a/.pipelines/platform-connectivity-hub-azfw.yml b/.pipelines/platform-connectivity-hub-azfw.yml
@@ -23,6 +23,8 @@ pr: none
 variables:
 - name: devops-org-name
   value: ${{ replace(replace(variables['System.CollectionUri'], 'https://dev.azure.com/' , ''), '/', '') }}
+- name: logging-config-directory
+  value: $(System.DefaultWorkingDirectory)/$(loggingPathFromRoot)/${{ variables['devops-org-name'] }}-${{ variables['Build.SourceBranchName'] }}
 - name: variable-template-file
   value: ${{ variables['devops-org-name'] }}-${{ variables['Build.SourceBranchName'] }}.yml
 - template: ../config/variables/common.yml
@@ -52,8 +54,7 @@ stages:
           - template: templates/steps/load-log-analytics-vars.yml
             parameters:
               logAnalyticsSubscriptionId: $(var-logging-subscriptionId)
-              logAnalyticsResourceGroupName: $(var-logging-logAnalyticsResourceGroupName)
-              logAnalyticsWorkspaceName: $(var-logging-logAnalyticsWorkspaceName)
+              logAnalyticsConfigurationFile: ${{ variables['logging-config-directory'] }}/$(var-logging-configurationFileName)
 
           - template: templates/steps/show-variables.yml
             parameters:

diff --git a/.pipelines/platform-connectivity-hub-nva.yml b/.pipelines/platform-connectivity-hub-nva.yml
@@ -23,6 +23,8 @@ pr: none
 variables:
 - name: devops-org-name
   value: ${{ replace(replace(variables['System.CollectionUri'], 'https://dev.azure.com/' , ''), '/', '') }}
+- name: logging-config-directory
+  value: $(System.DefaultWorkingDirectory)/$(loggingPathFromRoot)/${{ variables['devops-org-name'] }}-${{ variables['Build.SourceBranchName'] }}
 - name: variable-template-file
   value: ${{ variables['devops-org-name'] }}-${{ variables['Build.SourceBranchName'] }}.yml
 - template: ../config/variables/common.yml
@@ -53,8 +55,7 @@ stages:
           - template: templates/steps/load-log-analytics-vars.yml
             parameters:
               logAnalyticsSubscriptionId: $(var-logging-subscriptionId)
-              logAnalyticsResourceGroupName: $(var-logging-logAnalyticsResourceGroupName)
-              logAnalyticsWorkspaceName: $(var-logging-logAnalyticsWorkspaceName)
+              logAnalyticsConfigurationFile: ${{ variables['logging-config-directory'] }}/$(var-logging-configurationFileName)
 
           - template: templates/steps/show-variables.yml
             parameters:

diff --git a/.pipelines/platform-logging.yml b/.pipelines/platform-logging.yml
@@ -33,6 +33,8 @@ resources:
 variables:
 - name: devops-org-name
   value: ${{ replace(replace(variables['System.CollectionUri'], 'https://dev.azure.com/' , ''), '/', '') }}
+- name: logging-config-directory
+  value: $(System.DefaultWorkingDirectory)/$(loggingPathFromRoot)/${{ variables['devops-org-name'] }}-${{ variables['Build.SourceBranchName'] }}
 - name: variable-template-file
   value: ${{ variables['devops-org-name'] }}-${{ variables['Build.SourceBranchName'] }}.yml
 - template: ../config/variables/common.yml
@@ -66,7 +68,8 @@ stages:
           - template: templates/steps/deploy-platform-logging.yml
             parameters:
               description: 'Deploy Logging'
-              moveTemplate: move-subscription.bicep
-              configTemplate: main.bicep
               deployOperation: ${{ variables['deployOperation'] }}
-              workingDir: $(System.DefaultWorkingDirectory)/landingzones
+              loggingManagementGroupId: $(var-logging-managementGroupId)
+              loggingSubscriptionId: $(var-logging-subscriptionId)
+              loggingConfigurationPath: ${{ variables['logging-config-directory'] }}/$(var-logging-configurationFileName)
+              loggingRegion: $(var-logging-region)
diff --git a/.pipelines/policy.yml b/.pipelines/policy.yml
@@ -33,6 +33,8 @@ resources:
 variables:
 - name: devops-org-name
   value: ${{ replace(replace(variables['System.CollectionUri'], 'https://dev.azure.com/' , ''), '/', '') }}
+- name: logging-config-directory
+  value: $(System.DefaultWorkingDirectory)/$(loggingPathFromRoot)/${{ variables['devops-org-name'] }}-${{ variables['Build.SourceBranchName'] }}
 - name: variable-template-file
   value: ${{ variables['devops-org-name'] }}-${{ variables['Build.SourceBranchName'] }}.yml
 - template: ../config/variables/common.yml
@@ -57,7 +59,7 @@ stages:
 
   jobs:
 
-  - deployment: EnvionmentApprovalsAndChecks
+  - deployment: EnvironmentApprovalsAndChecks
     displayName: Environment Approvals and Checks
     environment: ${{ variables['Build.SourceBranchName'] }}
     strategy:
@@ -70,8 +72,8 @@ stages:
   - job: CustomPolicyJob
     displayName: Custom Policy Job
     dependsOn:
-      - EnvionmentApprovalsAndChecks
-    condition: succeeded('EnvionmentApprovalsAndChecks')
+      - EnvironmentApprovalsAndChecks
+    condition: succeeded('EnvironmentApprovalsAndChecks')
 
     steps:
 
@@ -80,8 +82,7 @@ stages:
     - template: templates/steps/load-log-analytics-vars.yml
       parameters:
         logAnalyticsSubscriptionId: $(var-logging-subscriptionId)
-        logAnalyticsResourceGroupName: $(var-logging-logAnalyticsResourceGroupName)
-        logAnalyticsWorkspaceName: $(var-logging-logAnalyticsWorkspaceName)
+        logAnalyticsConfigurationFile: ${{ variables['logging-config-directory'] }}/$(var-logging-configurationFileName)
 
     - template: templates/steps/show-variables.yml
       parameters:
@@ -110,8 +111,8 @@ stages:
   - job: BuiltInPolicyJob
     displayName: Built In Policy Job
     dependsOn:
-      - EnvionmentApprovalsAndChecks
-    condition: succeeded('EnvionmentApprovalsAndChecks')
+      - EnvironmentApprovalsAndChecks
+    condition: succeeded('EnvironmentApprovalsAndChecks')
 
     steps:
 
@@ -120,8 +121,7 @@ stages:
     - template: templates/steps/load-log-analytics-vars.yml
       parameters:
         logAnalyticsSubscriptionId: $(var-logging-subscriptionId)
-        logAnalyticsResourceGroupName: $(var-logging-logAnalyticsResourceGroupName)
-        logAnalyticsWorkspaceName: $(var-logging-logAnalyticsWorkspaceName)
+        logAnalyticsConfigurationFile: ${{ variables['logging-config-directory'] }}/$(var-logging-configurationFileName)
 
     - template: templates/steps/show-variables.yml
       parameters:

diff --git a/.pipelines/subscriptions.yml b/.pipelines/subscriptions.yml
@@ -32,6 +32,8 @@ variables:
   value: ${{ replace(replace(variables['System.CollectionUri'], 'https://dev.azure.com/' , ''), '/', '') }}
 - name: environment
   value: ${{ variables['devops-org-name'] }}-${{ variables['Build.SourceBranchName'] }}
+- name: logging-config-directory
+  value: $(System.DefaultWorkingDirectory)/$(loggingPathFromRoot)/${{ variables['devops-org-name'] }}-${{ variables['Build.SourceBranchName'] }}
 - template: ../config/variables/common.yml
 - template: ../config/variables/${{ variables['environment'] }}.yml
 
@@ -44,7 +46,6 @@ stages:
   displayName: Deploy Subscription Stage
 
   jobs:
-
   # This job is run when the CI trigger is fired based on a change to one or more
   # subscription configuration files *OR* when the pipeline is run manually and the
   # default empty array value is passed for the 'subscriptions' suparameter. 
@@ -83,4 +84,5 @@ stages:
         config: ${{ variables['subscriptionsPathFromRoot'] }}
         enviro: ${{ variables['environment'] }}
         subscription: ${{ subscription }}
-
+        logAnalyticsSubscriptionId: $(var-logging-subscriptionId)
+        logAnalyticsConfigurationFile: ${{ variables['logging-config-directory'] }}/$(var-logging-configurationFileName)
diff --git a/.pipelines/templates/jobs/deploy-subscription.yml b/.pipelines/templates/jobs/deploy-subscription.yml
@@ -14,6 +14,10 @@ parameters:
     type: string
   - name: subscription
     type: string
+  - name: logAnalyticsSubscriptionId
+    type: string
+  - name: logAnalyticsConfigurationFile
+    type: string
 
 jobs:
 
@@ -158,4 +162,5 @@ jobs:
             subguid: $(ParseSubscription.SubGuid)
             subtype: $(ParseSubscription.SubType)
             sublocation: $(ParseSubscription.SubLocation)
-
+            logAnalyticsSubscriptionId: ${{ parameters.logAnalyticsSubscriptionId }}
+            logAnalyticsConfigurationFile: ${{ parameters.logAnalyticsConfigurationFile }}
diff --git a/.pipelines/templates/steps/config-subscription.yml b/.pipelines/templates/steps/config-subscription.yml
@@ -33,7 +33,6 @@ steps:
       Write-Host "Schema File: ${schemaFile}"
 
       Get-Content -Raw "${parameterFile}" | Test-Json -SchemaFile "${schemaFile}"
-
   
 - task: AzureCLI@2
   displayName: Deploy Archetype
@@ -50,12 +49,26 @@ steps:
       deployName=${deployName:0:63}
 
       echo "Configuring subscription ${{ parameters.subscriptionGuid }} using template ${template} ..."
+
+      # Check if the log analytics workspace id is provided in the parameters json.
+      # If present, then do no change it.  Otherwise add it to the json parameter file.
+      LOG_ANALYTICS_WORKSPACE_RESOURCE_ID_IN_PARAMETERS=`jq -r .parameters.logAnalyticsWorkspaceResourceId.value ${{ parameters.filename }}`   
+
+      if [[ $LOG_ANALYTICS_WORKSPACE_RESOURCE_ID_IN_PARAMETERS != null && "$LOG_ANALYTICS_WORKSPACE_RESOURCE_ID_IN_PARAMETERS" != "" ]];
+      then
+        echo "Log Analytics Workspace Resource ID is set in ${{ parameters.filename }} to $LOG_ANALYTICS_WORKSPACE_RESOURCE_ID_IN_PARAMETERS"
+      else
+        echo "Log Analytics Workspace Resource ID is not set in ${{ parameters.filename }}.  Updating ${{ parameters.filename }} with $(var-logging-logAnalyticsWorkspaceResourceId)"
+
+        # use jq to update the json parameter file
+        echo "$( jq '.parameters.logAnalyticsWorkspaceResourceId.value = "$(var-logging-logAnalyticsWorkspaceResourceId)"' ${{ parameters.filename }} )" > ${{ parameters.filename }}
+      fi
+
       az deployment sub $(deployOperation) \
         --subscription ${{ parameters.subscriptionGuid }} \
         --location ${{ parameters.subscriptionLocation }} \
         --template-file $(Build.SourcesDirectory)/${template} \
         --name ${deployName} \
-        --parameters @${{ parameters.filename }} \
-          logAnalyticsWorkspaceResourceId='$(var-logging-logAnalyticsWorkspaceResourceId)'
-          
+        --parameters @${{ parameters.filename }}
+
       $(var-bashPostInjectScript)
diff --git a/.pipelines/templates/steps/deploy-platform-logging.yml b/.pipelines/templates/steps/deploy-platform-logging.yml
@@ -10,30 +10,43 @@
 parameters:
   - name: description
     type: string
-  - name: moveTemplate
-    type: string
-  - name: configTemplate
-    type: string
-  - name: workingDir
-    type: string
   - name: deployOperation
     type: string
     default: create
     values:
       - create
       - what-if
+  - name: loggingManagementGroupId
+    type: string
+  - name: loggingSubscriptionId
+    type: string
+  - name: loggingConfigurationPath
+    type: string
+  - name: loggingRegion
+    type: string
 
 steps:
 
+- task: PowerShell@2
+  displayName: Validate Logging Parameters
+  inputs:
+    targetType: 'inline'
+    script: |
+      $schemaFile="$(Build.SourcesDirectory)/schemas/latest/landingzones/lz-platform-logging.json"
+
+      Write-Host "Parameters File: ${{ parameters.loggingConfigurationPath }}"
+      Write-Host "Schema File: ${schemaFile}"
+
+      Get-Content -Raw "${{ parameters.loggingConfigurationPath }}" | Test-Json -SchemaFile "${schemaFile}"
+
 - template: ./move-subscription.yml
   parameters:
-    managementGroup: $(var-logging-managementGroupId)
-    subscriptionGuid: $(var-logging-subscriptionId)
-    subscriptionLocation: $(deploymentRegion)
+    managementGroup: ${{ parameters.loggingManagementGroupId }}
+    subscriptionGuid: ${{ parameters.loggingSubscriptionId }}
+    subscriptionLocation: ${{ parameters.loggingRegion }}
     templateDirectory: $(Build.SourcesDirectory)/landingzones/utils/mg-move
     templateFile: move-subscription.bicep
-    workingDir: ${{ parameters.workingDir }}/utils/mg-move
-
+    workingDir: $(System.DefaultWorkingDirectory)/landingzones/utils/mg-move
 
 - task: AzureCLI@2
   displayName: Register Resource Providers
@@ -44,9 +57,9 @@ steps:
     inlineScript: |
       $(var-bashPreInjectScript)
 
-      az account set -s $(var-logging-subscriptionId)
+      az account set -s ${{ parameters.loggingSubscriptionId }}
       
-      az provider register -n Microsoft.ContainerService --subscription '$(var-logging-subscriptionId)'
+      az provider register -n Microsoft.ContainerService --subscription '${{ parameters.loggingSubscriptionId }}'
 
       $(var-bashPostInjectScript)
 
@@ -60,23 +73,13 @@ steps:
     inlineScript: |
       $(var-bashPreInjectScript)
 
-      echo "Deploying ${{ parameters.configTemplate }} using ${{ parameters.deployOperation}} operation..."
+      echo "Deploying logging using ${{ parameters.deployOperation}} operation with ${{ parameters.loggingConfigurationPath }} to ${{ parameters.loggingSubscriptionId }} ..."
 
       az deployment sub ${{ parameters.deployOperation }} \
-        --location $(deploymentRegion) \
-        --subscription $(var-logging-subscriptionId) \
-        --template-file ${{ parameters.configTemplate }} \
-        --parameters \
-            serviceHealthAlerts='$(var-logging-serviceHealthAlerts)' \
-            securityCenter='$(var-logging-securityCenter)' \
-            subscriptionRoleAssignments='$(var-logging-subscriptionRoleAssignments)' \
-            subscriptionBudget='$(var-logging-subscriptionBudget)'\
-            subscriptionTags='$(var-logging-subscriptionTags)' \
-            resourceTags='$(var-logging-resourceTags)' \
-            logAnalyticsResourceGroupName='$(var-logging-logAnalyticsResourceGroupName)' \
-            logAnalyticsWorkspaceName='$(var-logging-logAnalyticsWorkspaceName)' \
-            logAnalyticsAutomationAccountName='$(var-logging-logAnalyticsAutomationAccountName)' \
-            logAnalyticsRetentionInDays='$(var-logging-logAnalyticsRetentionInDays)'
+        --location ${{ parameters.loggingRegion }} \
+        --subscription ${{ parameters.loggingSubscriptionId }} \
+        --template-file main.bicep \
+        --parameters @${{ parameters.loggingConfigurationPath }}
 
       $(var-bashPostInjectScript)
-    workingDirectory: '${{ parameters.workingDir }}/lz-platform-logging'
+    workingDirectory: $(System.DefaultWorkingDirectory)/landingzones/lz-platform-logging
diff --git a/.pipelines/templates/steps/deploy-subscription.yml b/.pipelines/templates/steps/deploy-subscription.yml
@@ -28,9 +28,18 @@ parameters:
     type: string
   - name: sublocation
     type: string
+  - name: logAnalyticsSubscriptionId
+    type: string
+  - name: logAnalyticsConfigurationFile
+    type: string
 
 steps:
 
+- template: ./load-log-analytics-vars.yml
+  parameters:
+    logAnalyticsSubscriptionId: ${{ parameters.logAnalyticsSubscriptionId }}
+    logAnalyticsConfigurationFile: ${{ parameters.logAnalyticsConfigurationFile }}
+
 - template: ./move-subscription.yml
   parameters:
     managementGroup: ${{ parameters.mgmtgroup }}
@@ -40,12 +49,6 @@ steps:
     templateFile: move-subscription.bicep
     workingDir: $(Build.SourcesDirectory)/$(subscriptionsPathFromRoot)/${{ parameters.relativePath }}
 
-- template: ./load-log-analytics-vars.yml
-  parameters:
-    logAnalyticsSubscriptionId: $(var-logging-subscriptionId)
-    logAnalyticsResourceGroupName: $(var-logging-logAnalyticsResourceGroupName)
-    logAnalyticsWorkspaceName: $(var-logging-logAnalyticsWorkspaceName)
-
 - task: AzureCLI@2
   displayName: Register Azure Provider Features
   inputs: