DevOps pipeline: splitted into stages (#152)

* Fuzzing-pipeline with stages

* stages layout

* stages

.azure/pipelines/ci.yml

@@ -8,6 +8,7 @@ variables:
 
 trigger:
   - master
+  - ci-*
   - refs/tags/v*
 
 pr:
@@ -22,137 +23,98 @@ schedules:
       include:
         - master
 
-jobs:
-  - template: jobs/build_and_test.yml
-    parameters:
-      name: linux
-      vmImage: 'ubuntu-18.04'
-
-  - template: jobs/build_and_test.yml
-    parameters:
-      name: windows
-      vmImage: 'windows-2019'
-
-  - template: jobs/build_and_test.yml
-    parameters:
-      name: mac
-      vmImage: 'macOS-10.15'
-
-  - template: jobs/build_and_test_intrinsics.yml
-    parameters:
-      name: intrinsics
-      vmImage: 'ubuntu-18.04'
+stages:
+  - stage: Build_Test_Windows
+    jobs:
+      - template: jobs/build_and_test.yml
+        parameters:
+          name: windows
+          vmImage: 'windows-2019'
+
+  - stage: Build_Test_Linux
+    dependsOn: [] # break sequential dependency
+    jobs:
+      - template: jobs/build_and_test.yml
+        parameters:
+          name: linux
+          vmImage: 'ubuntu-18.04'
+
+      - template: jobs/build_and_test_intrinsics.yml
+        parameters:
+          name: intrinsics
+          vmImage: 'ubuntu-18.04'
+
+  - stage: Build_Test_MacOS
+    dependsOn: [] # break sequential dependency
+    jobs:
+      - template: jobs/build_and_test.yml
+        parameters:
+          name: mac
+          vmImage: 'macOS-10.15'
+
+  - stage: Demo
+    dependsOn:
+      - Build_Test_Windows
+      - Build_Test_Linux
+      - Build_Test_MacOS
+    jobs:
+      - template: jobs/demo.yml
+        parameters:
+          name: demo_windows
+          vmImage: 'windows-2019'
+
+      - template: jobs/demo.yml
+        parameters:
+          name: demo_linux
+          vmImage: 'ubuntu-18.04'
+
+      - template: jobs/demo.yml
+        parameters:
+          name: demo_mac
+          vmImage: 'macOS-10.15'
+
+  - stage: Build_Docs
+    dependsOn:
+      - Build_Test_Windows
+      - Build_Test_Linux
+      - Build_Test_MacOS
+    jobs:
+      - template: jobs/build_docs.yml
 
-  - job: deploy
+  - stage: Deploy
     dependsOn:
-      - linux
-      - windows
-      - mac
-    pool:
-      vmImage: 'windows-2019'
+      - Build_Test_Windows
+      - Build_Test_Linux
+      - Build_Test_MacOS
+      - Build_Docs
     condition: and( succeeded(), startsWith( variables['Build.SourceBranch'], 'refs/tags' ) )
-    steps:
-      - task: DownloadBuildArtifacts@0
-        inputs:
-          buildType: current
-          artifactName: 'NuGet-Packed'
-          downloadPath: .
-
-      # Hack for file not found when using ./build.sh deploy
-      # With ls -la the files are listed, but can't be found anyway.
-      # I tried quite a lot, relative and absolute path, nothing worked.
-      # This hack does the job.
-      - bash: |
-          chmod u+x ./build.sh
-
-          echo "-------------------------------------------------"
-          echo "Nuget-Packed:"
-          ls -la "NuGet-Packed"
-
-          if [[ "$TAG_NAME" =~ ^v([0-9]+)\.([0-9]+)\.([0-9]+)(-(preview-[0-9]+))$ ]]; then
-            mkdir deploy_myget
-
-            for package in NuGet-Packed/*.nupkg; do
-                mv $package deploy_myget
-            done
-          elif [[ "$TAG_NAME" =~ ^v([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then
-            mkdir deploy_nuget
-
-            for package in NuGet-Packed/*.nupkg; do
-                mv $package deploy_nuget
-            done
-          else
-            echo "no deploy, as $TAG_NAME does not match"
-            echo ##vso[task.complete result=Skipped;]tag does not match for deploy
-          fi
-
-          echo "-------------------------------------------------"
-          echo "myget:"
-          ls -la deploy_myget
-          echo "-------------------------------------------------"
-          echo "nuget:"
-          ls -la deploy_nuget
-          echo "-------------------------------------------------"
-        displayName: 'deploy to myget / nuget'
-
-      - task: NuGetCommand@2
-        inputs:
-          command: push
-          packagesToPush: deploy_myget/*.nupkg
-          nuGetFeedType: external
-          publishFeedCredentials: 'myget - gfoidl'
-        displayName: deploy_myget
-
-      - task: NuGetCommand@2
-        inputs:
-          command: push
-          packagesToPush: deploy_nuget/*.nupkg
-          nuGetFeedType: external
-          publishFeedCredentials: 'nuget - gfoidl'
-        displayName: deploy_nuget
-
-  - template: jobs/demo.yml
-    parameters:
-      name: demo_linux
-      vmImage: 'ubuntu-18.04'
-      dependsOn: linux
-
-  - template: jobs/demo.yml
-    parameters:
-      name: demo_windows
-      vmImage: 'windows-2019'
-      dependsOn: windows
-
-  - template: jobs/demo.yml
-    parameters:
-      name: demo_mac
-      vmImage: 'macOS-10.15'
-      dependsOn: mac
-
-  - template: jobs/benchmarks.yml
-    parameters:
-      name: benchmarks_linux
-      vmImage: 'ubuntu-18.04'
-      dependsOn: linux
-
-  - template: jobs/benchmarks.yml
-    parameters:
-      name: benchmarks_windows
-      vmImage: 'windows-2019'
-      dependsOn: windows
-
-  - template: jobs/benchmarks.yml
-    parameters:
-      name: benchmarks_mac
-      vmImage: 'macOS-10.15'
-      dependsOn: mac
-
-  - template: jobs/benchmarks_net_full.yml
-    parameters:
-      name: benchmarks_net_full
-      vmImage: 'windows-2019'
-      dependsOn: windows
-
-  - template: jobs/api-doc.yml
-    parameters:
-      dependsOn: windows
+    jobs:
+      - template: jobs/deploy_nuget.yml
+      - template: jobs/deploy_docs.yml
+
+  - stage: Benchmarks
+    dependsOn:
+      - Build_Test_Windows
+      - Build_Test_Linux
+      - Build_Test_MacOS
+    condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'), eq(variables['Build.Reason'], 'Schedule'))
+    jobs:
+      - template: jobs/benchmarks.yml
+        parameters:
+          name: benchmarks_windows
+          vmImage: 'windows-2019'
+
+      - template: jobs/benchmarks.yml
+        parameters:
+          name: benchmarks_linux
+          vmImage: 'ubuntu-18.04'
+
+      - template: jobs/benchmarks.yml
+        parameters:
+          name: benchmarks_mac
+          vmImage: 'macOS-10.15'
+
+      - template: jobs/benchmarks_net_full.yml
+        parameters:
+          name: benchmarks_net_full
+          vmImage: 'windows-2019'

.azure/pipelines/fuzz.yml

@@ -21,11 +21,16 @@ schedules:
         - master
     always: true
 
-jobs:
-  - template: jobs/fuzzing.yml
-    parameters:
-      method: Base64_Default_Decode
+stages:
+  - stage: Base64
+    jobs:
+      - template: jobs/fuzzing.yml
+        parameters:
+          method: Base64_Default_Decode
 
-  - template: jobs/fuzzing.yml
-    parameters:
-      method: Base64_Url_Decode
+  - stage: Base64Url
+    dependsOn: [] # break sequential dependency
+    jobs:
+      - template: jobs/fuzzing.yml
+        parameters:
+          method: Base64_Url_Decode

.azure/pipelines/jobs/benchmarks.yml

@@ -1,13 +1,9 @@
 parameters:
   name: ''
   vmImage: ''
-  dependsOn: ''
 
 jobs:
   - job: ${{ parameters.name }}
-    dependsOn:
-      - ${{ parameters.dependsOn }}
-    condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'), eq(variables['Build.Reason'], 'Schedule'))
     pool:
       vmImage: ${{ parameters.vmImage }}
     timeoutInMinutes: 0     # 360 minutes
@@ -17,7 +13,7 @@ jobs:
         inputs:
           version: $(SDK_VERSION)
           includePreviewVersions: true
-      
+
       - bash: |
           chmod u+x *.sh
           ./run-benchmarks.sh

.azure/pipelines/jobs/benchmarks_net_full.yml

@@ -1,13 +1,9 @@
 parameters:
   name: ''
   vmImage: ''
-  dependsOn: ''
 
 jobs:
   - job: ${{ parameters.name }}
-    dependsOn:
-      - ${{ parameters.dependsOn }}
-    condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'), eq(variables['Build.Reason'], 'Schedule'))
     pool:
       vmImage: ${{ parameters.vmImage }}
     timeoutInMinutes: 0     # 360 minutes

.azure/pipelines/jobs/build_and_test.yml

@@ -14,11 +14,12 @@ jobs:
         release-build:
           BUILD_CONFIG: Release
     steps:
-      - task: UseDotNet@2
-        displayName: 'Use dotnet sdk 3.x'
-        inputs:
-          version: $(SDK_VERSION)
-          includePreviewVersions: true
+      # .NET Core 3 SDK is pre-installed
+      #- task: UseDotNet@2
+      #  displayName: 'Use dotnet sdk 3.x'
+      #  inputs:
+      #    version: $(SDK_VERSION)
+      #    includePreviewVersions: true
 
       - bash: |
           export PATH="$(pwd)/dotnet:$PATH"