Split release process (PoC) (#11991)

* Split release process (PoC) This PR adds the necessary infrastructure for the new split release process. Releases are still triggered via the LATEST file but you can choose between the old and the new release process by adding SPLIT-RELEASE at the end of the line. As a first step this publishes all artifacts we currently publish to Github releases to our GCP bucket. changelog_begin changelog_end * drop report-start changelog_begin changelog_end * fix gcp bucket changelog_begin changelog_end * review feedback changelog_begin changelog_end * Update azure-pipelines.yml Co-authored-by: Gary Verhaegen <gary.verhaegen@digitalasset.com> * SPLIT-RELEASE ->SPLIT_RELEASE changelog_begin changelog_end Co-authored-by: Gary Verhaegen <gary.verhaegen@digitalasset.com>
digital-asset · Dec 6, 2021 · 5a019ea · 5a019ea
1 parent f228f95
commit 5a019ea
Show file tree

Hide file tree

Showing 4 changed files with 83 additions and 0 deletions.
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -23,6 +23,7 @@ jobs:
   dependsOn: [ "check_for_release", "Linux", "macOS", "Windows" ]
   condition: and(succeeded(),
                  eq(dependencies.check_for_release.outputs['out.is_release'], 'true'),
+                 eq(dependencies.check_for_release.outputs['out.split_release_process'], 'false')
                  eq(variables['Build.SourceBranchName'], 'main'))
   pool:
     name: 'ubuntu_20_04'
@@ -149,6 +150,64 @@ jobs:
         trigger_sha: '$(trigger_sha)'
     - template: ci/report-end.yml
 
+- job: split_release
+  dependsOn: [ "check_for_release", "Linux", "macOS", "Windows" ]
+  condition: and(succeeded(),
+                 eq(dependencies.check_for_release.outputs['out.is_release'], 'true'),
+                 eq(dependencies.check_for_release.outputs['out.split_release_process'], 'true')
+                 eq(variables['Build.SourceBranchName'], 'main'))
+  pool:
+    name: 'ubuntu_20_04'
+    demands: assignment -equals default
+  variables:
+    release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
+    release_tag: $[ dependencies.check_for_release.outputs['out.release_tag'] ]
+    trigger_sha: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ]
+  steps:
+    - checkout: self
+      persistCredentials: true
+    - bash: |
+        set -euo pipefail
+        git checkout $(release_sha)
+      name: checkout_release
+    - template: ci/bash-lib.yml
+      parameters:
+        var_name: bash-lib
+    - bash: |
+        set -euo pipefail
+        if git tag v$(release_tag) $(release_sha); then
+          git push origin v$(release_tag)
+        fi
+    - task: DownloadPipelineArtifact@0
+      inputs:
+        artifactName: linux-release
+        targetPath: $(Build.StagingDirectory)/release-artifacts
+      condition: and(succeeded(), not(eq(variables['skip-github'], 'TRUE')))
+    - task: DownloadPipelineArtifact@0
+      inputs:
+        artifactName: macos-release
+        targetPath: $(Build.StagingDirectory)/release-artifacts
+      condition: and(succeeded(), not(eq(variables['skip-github'], 'TRUE')))
+    - task: DownloadPipelineArtifact@0
+      inputs:
+        artifactName: windows-release
+        targetPath: $(Build.StagingDirectory)/release-artifacts
+      condition: and(succeeded(), not(eq(variables['skip-github'], 'TRUE')))
+    - bash: |
+        set -eou pipefail
+        eval "$(./dev-env/bin/dade-assist)"
+        mkdir -p $(Build.StagingDirectory)/split-release
+        ./ci/assembly-split-release-artifacts.sh $(release_tag) $(Build.StagingDirectory)/release-artifacts $(Build.StagingDirectory)/split-release
+    - bash: |
+        set -eou piefail
+
+        source $(bash-lib)
+        cd $(Build.StagingDirectory)/split-release
+        gcs "$GCRED" -m cp -r github gs://daml-binaries/split-releases/$(release_tag)/
+    - template: ci/tell-slack-failed.yml
+      parameters:
+        trigger_sha: '$(trigger_sha)'
+
 - job: compat_versions_pr
   dependsOn:
   - git_sha

diff --git a/ci/assembly-split-release-artifacts.sh b/ci/assembly-split-release-artifacts.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+# Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+set -euo pipefail
+
+RELEASE_TAG=$1
+SOURCE_DIR=$2
+OUTPUT_DIR=$3
+
+mkdir -p $OUTPUT_DIR/github
+
+for file in $SOURCE_DIR/github/*; do
+    # Copy as a placeholder for potential tweaks we might want to do here.
+    cp $file $OUTPUT_DIR/github
+done
diff --git a/ci/build.yml b/ci/build.yml
@@ -438,6 +438,11 @@ jobs:
                 setvar release_sha "$(added_line | awk '{print $1}')"
                 RELEASE_TAG="$(added_line | awk '{print $2}')"
                 setvar release_tag "$RELEASE_TAG"
+                if [ $(added_line | awk '{print $3}') == "SPLIT_RELEASE" ]; then
+                  setvar split_release_process true
+                else
+                  setvar split_release_process false
+                fi
             else
                 echo "Release commit should only add one version."
                 exit 1

diff --git a/release.sh b/release.sh
@@ -32,6 +32,7 @@ check() {
     while read line; do
         sha=$(echo "$line" | gawk '{print $1}')
         ver=$(echo "$line" | gawk '{print $2}')
+        split=$(echo "$line" | gawk '{print $3}')
         if ! echo "$ver" | grep -q -P $VERSION_REGEX; then
             echo "Invalid version number in LATEST file, needs manual correction."
             echo "Offending version: '$ver'."
@@ -44,6 +45,9 @@ check() {
                 exit 1
             fi
         fi
+        if [ ! -z "$split" ] && [ "$split" != "SPLIT_RELEASE" ]; then
+            echo "Invalid entry in third column, must be SPLIT_RELEASE or non-existent."
+        fi
     done < LATEST
 }