azure-pipelines.yml

# Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

# Azure Pipelines file, see https://aka.ms/yaml

# Enable builds on all branches
trigger:
  # Build every commit as our release process relies on
  # the release process being built alone.
  batch: false
  branches:
    include:
      - main
      - release/*

# Enable PR triggers that target the main branch
pr: none

jobs:
- template: ci/build.yml

- job: check_perf_test
  pool:
    name: ubuntu_20_04
    demands: assignment -equals default
  steps:
  - bash: |
      TEST_SHA=$(cat ci/cron/perf/test_sha)
      LAST_CHANGES=$(git log -n1 --format=%H daml-lf/scenario-interpreter/src/perf)
      CURRENT_SHA=$(git rev-parse HEAD)
      if [ "$TEST_SHA" != "$LAST_CHANGES" ]; then
          if [ "$LAST_CHANGES" = "$CURRENT_SHA" ]; then
              curl -XPOST \
                   -i \
                   -H 'Content-Type: application/json' \
                   --data "{\"text\":\"<!here> Perf tests seem to have changed. Please manually check:\n\`\`\`\ngit diff $TEST_SHA $LAST_CHANGES -- daml-lf/scenario-interpreter/src/perf\n\`\`\`\nand update accordingly. If the change is benign, update \`ci/cron/perf/test_sha\` to \`$LAST_CHANGES\`. With no intervention, you will no longer get performance reports.\"}" \
                   $(Slack.team-daml)
          else
              echo "Changes detected, but not from this commit."
          fi
      else
          echo "No change detected."
      fi
    displayName: check perf changes

- job: release
  dependsOn: [ "check_for_release", "Linux", "Linux_scala_2_13", "macOS", "Windows" ]
  condition: and(succeeded(),
                 eq(dependencies.check_for_release.outputs['out.is_release'], 'true'),
                 eq(variables['Build.SourceBranchName'], 'main'))
  pool:
    vmImage: "Ubuntu-16.04"
  variables:
    linux-tarball: $[ dependencies.Linux.outputs['publish.tarball'] ]
    macos-tarball: $[ dependencies.macOS.outputs['publish.tarball'] ]
    windows-tarball: $[ dependencies.Windows.outputs['publish.tarball'] ]
    windows-installer: $[ dependencies.Windows.outputs['publish.installer'] ]
    protos-zip: $[ dependencies.Linux.outputs['publish.protos-zip'] ]
    daml-on-sql: $[ dependencies.Linux.outputs['publish.daml-on-sql'] ]
    json-api: $[ dependencies.Linux.outputs['publish.json-api'] ]
    script-runner: $[ dependencies.Linux.outputs['publish.script-runner'] ]
    trigger-runner: $[ dependencies.Linux.outputs['publish.trigger-runner'] ]
    trigger-service: $[ dependencies.Linux.outputs['publish.trigger-service'] ]
    oauth2-middleware: $[ dependencies.Linux.outputs['publish.oauth2-middleware'] ]
    release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
    release_tag: $[ dependencies.check_for_release.outputs['out.release_tag'] ]
    trigger_sha: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ]
  steps:
    - template: ci/report-start.yml
    - checkout: self
      persistCredentials: true
    - template: ci/clean-up.yml
    - template: ci/bash-lib.yml
      parameters:
        var_name: bash-lib
    - bash: |
        set -euo pipefail
        source $(bash-lib)
        if git tag v$(release_tag) $(release_sha); then
          git push origin v$(release_tag)
          mkdir $(Build.StagingDirectory)/release
          mkdir $(Build.StagingDirectory)/artifactory
        else
          setvar skip-github TRUE
        fi
    - task: DownloadPipelineArtifact@0
      inputs:
        artifactName: $(linux-tarball)
        targetPath: $(Build.StagingDirectory)/release
      condition: not(eq(variables['skip-github'], 'TRUE'))
    - task: DownloadPipelineArtifact@0
      inputs:
        artifactName: $(macos-tarball)
        targetPath: $(Build.StagingDirectory)/release
      condition: not(eq(variables['skip-github'], 'TRUE'))
    - task: DownloadPipelineArtifact@0
      inputs:
        artifactName: $(windows-tarball)
        targetPath: $(Build.StagingDirectory)/release
      condition: not(eq(variables['skip-github'], 'TRUE'))
    - task: DownloadPipelineArtifact@0
      inputs:
        artifactName: $(windows-installer)
        targetPath: $(Build.StagingDirectory)/release
      condition: not(eq(variables['skip-github'], 'TRUE'))
    - task: DownloadPipelineArtifact@0
      inputs:
        artifactName: $(protos-zip)
        targetPath: $(Build.StagingDirectory)/release
      condition: not(eq(variables['skip-github'], 'TRUE'))
    - task: DownloadPipelineArtifact@0
      inputs:
        artifactName: $(daml-on-sql)
        targetPath: $(Build.StagingDirectory)/release
      condition: not(eq(variables['skip-github'], 'TRUE'))
    - task: DownloadPipelineArtifact@0
      inputs:
        artifactName: $(json-api)
        targetPath: $(Build.StagingDirectory)/release
      condition: not(eq(variables['skip-github'], 'TRUE'))
    - task: DownloadPipelineArtifact@0
      inputs:
        artifactName: $(trigger-service)
        targetPath: $(Build.StagingDirectory)/release
      condition: not(eq(variables['skip-github'], 'TRUE'))
    - task: DownloadPipelineArtifact@0
      inputs:
        artifactName: $(oauth2-middleware)
        targetPath: $(Build.StagingDirectory)/release
      condition: not(eq(variables['skip-github'], 'TRUE'))
    - task: DownloadPipelineArtifact@0
      inputs:
        artifactName: $(trigger-runner)
        targetPath: $(Build.StagingDirectory)/artifactory
      condition: not(eq(variables['skip-github'], 'TRUE'))
    - task: DownloadPipelineArtifact@0
      inputs:
        artifactName: $(script-runner)
        targetPath: $(Build.StagingDirectory)/artifactory
      condition: not(eq(variables['skip-github'], 'TRUE'))
    - bash: |
        set -euo pipefail
        KEY_FILE=$(mktemp)
        GPG_DIR=$(mktemp -d)
        cleanup() {
            rm -rf $KEY_FILE $GPG_DIR
        }
        trap cleanup EXIT
        echo "$GPG_KEY" | base64 -d > $KEY_FILE
        gpg --homedir $GPG_DIR --no-tty --quiet --import $KEY_FILE
        cd $(Build.StagingDirectory)/release
        sha256sum $(find . -type f | sort) > sha256sums
        # Note: relies on our release artifacts not having spaces in their
        # names. Creates a ${f}.asc with the signature for each $f.
        for f in *; do
            gpg --homedir $GPG_DIR -ab $f
        done
        cd ../artifactory
        for f in *; do
            gpg --homedir $GPG_DIR -ab $f
        done
      env:
        GPG_KEY: $(gpg-code-signing)
    - task: GitHubRelease@0
      inputs:
        gitHubConnection: 'garyverhaegen-da'
        repositoryName: '$(Build.Repository.Name)'
        action: 'create'
        target: '$(release_sha)'
        tagSource: 'manual'
        tag: 'v$(release_tag)'
        assets: $(Build.StagingDirectory)/release/*
        assetUploadMode: 'replace'
        title: '$(release_tag)'
        addChangeLog: false
        isPrerelease: true
        releaseNotesSource: 'input'
        releaseNotes: "This is a pre-release. Use at your own risk."
      condition: not(eq(variables['skip-github'], 'TRUE'))
    - bash: |
        set -euo pipefail

        push() {
          local file repository local_path md5 sha1
          repository=$1
          file=$2
          local_path=$(Build.StagingDirectory)/artifactory/${file}
          md5=$(md5sum ${local_path} | awk '{print $1}')
          sha1=$(sha1sum ${local_path} | awk '{print $1}')
          curl -f \
               -u "$AUTH" \
               -H "X-Checksum-MD5:${md5}" \
               -H "X-Checksum-SHA1:${sha1}" \
               -X PUT \
               -T ${local_path} \
               https://digitalasset.jfrog.io/artifactory/${repository}/$(release_tag)/${file}
        }

        push daml-trigger-runner $(trigger-runner)
        push daml-trigger-runner $(trigger-runner).asc
        push daml-script-runner $(script-runner)
        push daml-script-runner $(script-runner).asc
      env:
        AUTH: $(ARTIFACTORY_USERNAME):$(ARTIFACTORY_PASSWORD)
      condition: not(eq(variables['skip-github'], 'TRUE'))
    - template: ci/bash-lib.yml
      parameters:
        var_name: bash-lib
    - bash: |
        set -euo pipefail

        source $(bash-lib)

        cd $(Build.StagingDirectory)/release
        for f in *; do
            gcs "$GCRED" cp "$f" "gs://daml-data/releases/$(release_tag)/github/$f"
        done

        cd $(Build.StagingDirectory)/artifactory
        for f in *; do
            gcs "$GCRED" cp "$f" "gs://daml-data/releases/$(release_tag)/artifactory/$f"
        done
      name: backup_to_gcs
      env:
        GCRED: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
      condition: not(eq(variables['skip-github'], 'TRUE'))
    - bash: |
        set -euo pipefail
        msg=$(git log -n1 --format=%b HEAD | head -1)
        if [ "$msg" = "This PR has been created by a script, which is not very smart and does not have all the context. Please do double-check that the version prefix is correct before merging." ]; then
            pr_handler=@$(head -1 release/rotation | awk '{print $1}')
        else
            pr_handler="!here"
        fi
        curl -XPOST \
             -i \
             -H 'Content-Type: application/json' \
             --data "{\"text\":\"<${pr_handler}> Release \`$(release_tag)\` is ready for testing. See <https://github.com/digital-asset/daml/blob/main/release/RELEASE.md|release instructions>. (<https://dev.azure.com/digitalasset/daml/_build/results?buildId=$(Build.BuildId)|build>, <https://github.com/digital-asset/daml/commit/$(trigger_sha)|trigger commit>, <https://github.com/digital-asset/daml/commit/$(release_sha)|target commit>)\"}" \
             $(Slack.team-daml)
    - template: ci/tell-slack-failed.yml
      parameters:
        trigger_sha: '$(trigger_sha)'
    - template: ci/report-end.yml

- job: compat_versions_pr
  dependsOn:
  - git_sha
  - release
  - check_for_release
  pool:
    name: ubuntu_20_04
    demands: assignment -equals default
  variables:
    release_tag: $[ dependencies.check_for_release.outputs['out.release_tag'] ]
    branch_sha: $[ dependencies.git_sha.outputs['out.branch'] ]
  steps:
  - checkout: self
    persistCredentials: true
  - bash: ci/dev-env-install.sh
  - template: ci/bash-lib.yml
    parameters:
      var_name: bash_lib
  - bash: |
      set -euo pipefail
      eval "$(./dev-env/bin/dade-assist)"

      source $(bash_lib)

      DELAY=1
      while ! curl --fail -I https://repo1.maven.org/maven2/com/daml/ledger-api-test-tool/$(release_tag)/ledger-api-test-tool-$(release_tag).jar; do
          sleep $DELAY
          DELAY=$(( DELAY * 2 ))
          if (( $DELAY > 2000 )); then
              echo "Too many attempts waiting for Maven."
              exit 1
          fi
      done

      trap "git checkout $(branch_sha)" EXIT
      git checkout origin/main
      cp .bazelrc compatibility/
      compatibility/update-versions.sh
      git add compatibility/versions.bzl compatibility/maven_install.json

      BRANCH=update-compat-versions-for-$(release_tag)-$(Build.BuildId)
      TITLE="update compat versions for $(release_tag)"
      open_pr "$BRANCH" "$TITLE"

      setvar "branch" "$BRANCH"
    name: out

  - template: ci/tell-slack-failed.yml
    parameters:
      trigger_sha: '$(trigger_sha)'

- job: compat_versions_pr_trigger_daily
  dependsOn: compat_versions_pr
  pool:
    vmImage: "ubuntu-18.04"
  variables:
    branch: $[ dependencies.compat_versions_pr.outputs['out.branch'] ]
  steps:
  - checkout: none
  - bash: |
      set -euo pipefail
      az extension add --name azure-devops
      echo "$(System.AccessToken)" | az devops login --org "https://dev.azure.com/digitalasset"
      az pipelines build queue --branch $(branch) --definition-name "digital-asset.daml-daily-compat" --org "https://dev.azure.com/digitalasset" --project daml
      az pipelines build queue --branch $(branch) --definition-name "PRs" --org "https://dev.azure.com/digitalasset" --project daml