Stream Analytics CI/CD Yaml file for the Tech Sample (#549)

* adding the yaml and yaml generator

* update override params

* updating yaml with a sep access key slot

* *adding

* add job config and project config

* updating readme

* lint errors

* Update single_tech_samples/streamanalytics/sampleyaml.yml

Co-authored-by: JM <jason.mostella@microsoft.com>

* cleaning up

* fixing tests

* updating branch and file name

* updating readme

* lint

* lint

* remove bloboutput

Co-authored-by: Sara Norris <sanorris@microsoft.com>
Co-authored-by: JM <jason.mostella@microsoft.com>

single_tech_samples/streamanalytics/JobConfig.json

@@ -0,0 +1,25 @@
+{
+    "ScriptType": "JobConfig",
+    "DataLocale": "en-US",
+    "OutputErrorPolicy": "Retry",
+    "EventsLateArrivalMaxDelayInSeconds": 5,
+    "EventsOutOfOrderMaxDelayInSeconds": 0,
+    "EventsOutOfOrderPolicy": "Adjust",
+    "StreamingUnits": 3,
+    "CompatibilityLevel": "1.2",
+    "UseSystemAssignedIdentity": false,
+    "GlobalStorage": {
+        "AccountName": null,
+        "AccountKey": null,
+        "AuthenticationMode": "ConnectionString"
+    },
+    "ContentStoragePolicy": "SystemAccount",
+    "CustomCodeStorage": {
+        "AccountName": null,
+        "AccountKey": null,
+        "ContainerName": null,
+        "Path": "UserCustomCode.zip"
+    },
+    "DataSourceCredentialDomain": null,
+    "Tags": {}
+}

single_tech_samples/streamanalytics/README.md

@@ -166,3 +166,11 @@ This sample combines [Azure IoT device SDK](https://www.npmjs.com/package/azure-
 ![test result output screen capture](docs/images/e2e-test.PNG)
 
 Within the test file [e2e/e2e.ts](e2e/e2e.ts) there is the `EXPECTED_E2E_LATENCY_MS` defined to be 1s. So this would also need to be adjusted for a real implementation.
+
+#### CI/CD
+
+A sample CI/CD Pipeline YAML file is present in this repo under "samplecicdpipeline.yml". This pipeline runs the tests present under the tests folder,
+sets up the IoTHub, and deploys the ASA job using the contents of the streamingjobs.bicep file. In order to add a new ASA job, please do the following:
+
+1. Include a new bicep file for the additional ASA job, and add it to the main.bicep file.
+2. Add the query into the inlineScript under the parameters of the yaml file where the deployment of main.bicep happens.

single_tech_samples/streamanalytics/asaproj.json

@@ -1,5 +1,10 @@
 {
     "name": "streamanalytics",
     "startFile": "streamanalytics-tech-sample.asaql",
-    "configurations": []
+    "configurations": [
+        {
+            "filePath": "JobConfig.json",
+            "subType": "JobConfig"
+        }
+    ]
 }

single_tech_samples/streamanalytics/samplecicdpipeline.yml

@@ -0,0 +1,77 @@
+trigger:
+- main
+pool:
+  vmImage: ubuntu-latest
+steps:
+
+- task: Npm@1
+  displayName: 'Install Azure stream analytics ci cd'
+  inputs:
+    command: custom
+    verbose: false
+    customCommand: 'install -g azure-streamanalytics-cicd'
+- task: Bash@3
+  inputs:
+    targetType: 'inline'
+    script: azure-streamanalytics-cicd test -project ./asaproj.json -outputPath ./output/
+  displayName: 'Run Unit Tests'
+
+- task: AzureCLI@2  
+  inputs:
+    azureSubscription: '$(subscription)'
+    scriptType: 'bash'
+    scriptLocation: 'inlineScript'
+    inlineScript: 'az group create -n rg-$(app)-$(env) -l $(location)'
+  displayName: 'Create Resource Group'
+
+- task: AzureCLI@2  
+  inputs:
+    azureSubscription: '$(subscription)'
+    scriptType: 'bash'
+    scriptLocation: 'inlineScript'
+    inlineScript: 'az deployment group create -f ./main.bicep -g rg-$(app)-$(env) --parameters query=''@./streamanalytics-tech-sample.asaql'' name=$(app) env=$(env)'
+  displayName: 'Deploy Infrastructure'
+
+- task: AzureCLI@2  
+  inputs:
+    azureSubscription: '$(subscription)'
+    scriptType: 'bash'
+    scriptLocation: 'inlineScript'
+    inlineScript: |
+      az config set extension.use_dynamic_install=yes_without_prompt
+      az iot hub device-identity create --hub-name iot-$(app)-$(env) --device-id iot-$(app)-$(env) --edge-enabled
+  displayName: 'Create device'
+
+- task: AzureCLI@2  
+  inputs:
+    azureSubscription: '$(subscription)'
+    scriptType: 'bash'
+    scriptLocation: 'inlineScript'
+    inlineScript: |
+      az config set extension.use_dynamic_install=yes_without_prompt
+      export DEVICE_CONNECTION_STRING=$(az iot hub device-identity connection-string show --hub-name iot-$(app)-$(env) --device-id iot-$(app)-$(env) --output tsv)
+      export AZURE_STORAGE_CONNECTION_STRING=$(az storage account show-connection-string -n st$(app)$(env) --query connectionString -o tsv)      
+      cd e2e
+      npm install 
+      npm test
+  displayName: 'Run E2E tests'
+
+- task: AzureCLI@2 
+  condition: always() 
+  inputs:
+    azureSubscription: '$(subscription)'
+    scriptType: 'bash'
+    scriptLocation: 'inlineScript'
+    inlineScript: |
+      az config set extension.use_dynamic_install=yes_without_prompt
+      az iot hub device-identity delete --hub-name iot-$(app)-$(env) --device-id iot-$(app)-$(env)
+  displayName: 'Delete device'
+
+- task: AzureCLI@2  
+  condition: always()
+  inputs:
+    azureSubscription: '$(subscription)'
+    scriptType: 'bash'
+    scriptLocation: 'inlineScript'
+    inlineScript: 'az group delete -n rg-$(app)-$(env) -y'    
+  displayName: 'Delete resource group'

single_tech_samples/streamanalytics/test/temperature_less_than_27_degrees.json

@@ -1 +1 @@
-{"messageId":3,"deviceId":"modern-data-warehouse-dataops/single_tech_samples/streamanalytics/test","temperature":26.999999999999999,"humidity":64.86134165355034}
+{"messageId":3,"deviceId":"modern-data-warehouse-dataops/single_tech_samples/streamanalytics/test","temperature":26.999999999999999,"humidity":64.86134165355034}

single_tech_samples/streamanalytics/test/testConfig.json

@@ -15,7 +15,7 @@
       "ExpectedOutputs": [
         {
           "OutputAlias": "bloboutput",
-          "FilePath": "temperature_greater_than_27_degrees_expected.json",
+          "FilePath": "temperature_greater_than_27_degrees.json",
           "Required": true
         }
       ]
@@ -31,13 +31,7 @@
           "ScriptType": "InputMock"
         }
       ],
-      "ExpectedOutputs": [
-        {
-          "OutputAlias": "bloboutput",
-          "FilePath": "/dev/null",
-          "Required": true
-        }
-      ]
+      "ExpectedOutputs": []
     },
     {
       "Name": "temperature_equal_to_27_degrees",
@@ -50,13 +44,7 @@
           "ScriptType": "InputMock"
         }
       ],
-      "ExpectedOutputs": [
-        {
-          "OutputAlias": "bloboutput",
-          "FilePath": "/dev/null",
-          "Required": true
-        }
-      ]
+      "ExpectedOutputs": []
     }
   ]
 }