# Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # Azure Pipelines file, see https://aka.ms/yaml # Enable builds on all branches trigger: # Build every commit as our release process relies on # the release process being built alone. batch: false branches: include: - master - release/* # Enable PR triggers that target the master branch pr: autoCancel: true # cancel previous builds on push branches: include: - master - release/* jobs: - job: git_sha pool: name: 'linux-pool' demands: assignment -equals default steps: - bash: | set -euo pipefail if [ "$(Build.Reason)" == "PullRequest" ]; then echo "##vso[task.setvariable variable=branch;isOutput=true]$(git rev-parse HEAD^2)" echo "##vso[task.setvariable variable=master;isOutput=true]$(git rev-parse HEAD^1)" echo "##vso[task.setvariable variable=fork_point;isOutput=true]$(git merge-base $(git rev-parse HEAD^1) $(git rev-parse HEAD^2))" else echo "##vso[task.setvariable variable=branch;isOutput=true]$(git rev-parse HEAD)" echo "##vso[task.setvariable variable=master;isOutput=true]$(git rev-parse HEAD^1)" echo "##vso[task.setvariable variable=fork_point;isOutput=true]$(git rev-parse HEAD^1)" fi name: out - job: check_standard_change_label dependsOn: - git_sha variables: fork_sha: $[ dependencies.git_sha.outputs['out.fork_point'] ] branch_sha: $[ dependencies.git_sha.outputs['out.branch'] ] condition: eq(variables['Build.Reason'], 'PullRequest') pool: name: 'linux-pool' demands: assignment -equals default steps: - checkout: self - bash: | set -euo pipefail has_changed () { git diff $(fork_sha) $(branch_sha) --name-only | grep -q "^$1/" } fail_if_missing_std_change_label () { curl https://api.github.com/repos/digital-asset/daml/pulls/$PR -s | jq -r '.labels[].name' | grep -q '^Standard-Change$' } if has_changed "infra" || has_changed "LATEST"; then fail_if_missing_std_change_label fi env: PR: $(System.PullRequest.PullRequestNumber) - job: check_changelog_entry dependsOn: - git_sha variables: fork_sha: $[ dependencies.git_sha.outputs['out.fork_point'] ] condition: eq(variables['Build.Reason'], 'PullRequest') pool: name: 'linux-pool' demands: assignment -equals default steps: - checkout: self - bash: ci/check-changelog.sh $(fork_sha) - job: Linux dependsOn: - check_for_release variables: release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ] release_tag: $[ coalesce(dependencies.check_for_release.outputs['out.release_tag'], '0.0.0') ] trigger_sha: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ] is_release: $[ dependencies.check_for_release.outputs['out.is_release'] ] timeoutInMinutes: 360 pool: name: 'linux-pool' demands: assignment -equals default steps: - template: ci/report-start.yml - checkout: self - bash: | set -euo pipefail git checkout $(release_sha) git checkout $(trigger_sha) -- docs/source/support/release-notes.rst name: checkout_release condition: eq(variables.is_release, 'true') - template: ci/build-unix.yml parameters: release_tag: $(release_tag) name: 'linux' is_release: variables.is_release - bash: | set -euo pipefail eval "$(./dev-env/bin/dade-assist)" bazel build //release:release ./bazel-bin/release/release --release-dir "$(mktemp -d)" condition: and(succeeded(), ne(variables['is_release'], 'true')) - template: ci/tell-slack-failed.yml parameters: trigger_sha: '$(trigger_sha)' - template: ci/report-end.yml - job: macOS dependsOn: - check_for_release timeoutInMinutes: 360 pool: name: macOS-pool variables: release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ] release_tag: $[ coalesce(dependencies.check_for_release.outputs['out.release_tag'], '0.0.0') ] trigger_sha: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ] is_release: $[ dependencies.check_for_release.outputs['out.is_release'] ] steps: - template: ci/report-start.yml - checkout: self - bash: | set -euo pipefail git checkout $(release_sha) git checkout $(trigger_sha) -- docs/source/support/release-notes.rst name: checkout_release condition: eq(variables.is_release, 'true') - template: ci/build-unix.yml parameters: release_tag: $(release_tag) name: macos is_release: variables.is_release - template: ci/tell-slack-failed.yml parameters: trigger_sha: '$(trigger_sha)' - template: ci/report-end.yml - template: ci/patch_bazel_windows/compile.yml parameters: final_job_name: patch_bazel_windows - job: Windows dependsOn: - check_for_release - patch_bazel_windows variables: release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ] release_tag: $[ coalesce(dependencies.check_for_release.outputs['out.release_tag'], '0.0.0') ] trigger_sha: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ] is_release: $[ dependencies.check_for_release.outputs['out.is_release'] ] timeoutInMinutes: 360 pool: name: 'windows-pool' demands: assignment -equals default steps: - template: ci/report-start.yml - checkout: self - bash: | set -euo pipefail git checkout $(release_sha) git checkout $(trigger_sha) -- docs/source/support/release-notes.rst name: checkout_release condition: eq(variables.is_release, 'true') - template: ci/build-windows.yml parameters: release_tag: $(release_tag) is_release: variables.is_release - task: PublishBuildArtifacts@1 condition: succeededOrFailed() inputs: pathtoPublish: '$(Build.StagingDirectory)' artifactName: 'Bazel Logs' - template: ci/tell-slack-failed.yml parameters: trigger_sha: '$(trigger_sha)' - template: ci/report-end.yml - job: compatibility_linux dependsOn: - check_for_release timeoutInMinutes: 60 pool: name: linux-pool demands: assignment -equals default steps: - template: ci/report-start.yml - checkout: self - template: ci/compatibility.yml parameters: test_flags: '--quick' - template: ci/tell-slack-failed.yml - template: ci/report-end.yml - job: compatibility_macos dependsOn: - check_for_release timeoutInMinutes: 60 pool: name: macOS-pool steps: - template: ci/report-start.yml - checkout: self - template: ci/compatibility.yml parameters: test_flags: '--quick' - template: ci/tell-slack-failed.yml - template: ci/report-end.yml - job: compatibility_windows dependsOn: - check_for_release - patch_bazel_windows timeoutInMinutes: 60 pool: name: 'windows-pool' demands: assignment -equals default steps: - template: ci/report-start.yml - checkout: self - template: ci/compatibility-windows.yml parameters: test_flags: '--quick' - template: ci/tell-slack-failed.yml - template: ci/report-end.yml - task: PublishBuildArtifacts@1 condition: succeededOrFailed() inputs: pathtoPublish: '$(Build.StagingDirectory)' artifactName: 'Bazel Compatibility Logs' - job: check_for_release dependsOn: - git_sha variables: branch_sha: $[ dependencies.git_sha.outputs['out.branch'] ] fork_sha: $[ dependencies.git_sha.outputs['out.fork_point'] ] pool: name: "linux-pool" demands: assignment -equals default steps: - bash: | set -euo pipefail ./release.sh check is_release_commit() { changed="$(git diff-tree --no-commit-id --name-only -r $(branch_sha) $(fork_sha) | sort)" stable=$(printf "LATEST\ndocs/source/support/release-notes.rst" | sort) snapshot="LATEST" [ "$snapshot" = "$changed" ] || [ "$stable" = "$changed" ] } if is_release_commit; then echo "##vso[task.setvariable variable=is_release;isOutput=true]true" echo "##vso[task.setvariable variable=trigger_sha;isOutput=true]$(branch_sha)" echo "##vso[task.setvariable variable=release_sha;isOutput=true]$(cat LATEST | awk '{print $1}')" echo "##vso[task.setvariable variable=release_tag;isOutput=true]$(cat LATEST | awk '{print $2}')" else echo "##vso[task.setvariable variable=is_release;isOutput=true]false" fi name: out - job: check_perf_test pool: name: linux-pool demands: assignment -equals default condition: eq(variables['Build.Reason'], 'IndividualCI') steps: - bash: | TEST_SHA=$(cat ci/cron/perf/test_sha) LAST_CHANGES=$(git log -n1 --format=%H daml-lf/scenario-interpreter/src/perf) CURRENT_SHA=$(git rev-parse HEAD) if [ "$TEST_SHA" != "$LAST_CHANGES" ]; then if [ "$LAST_CHANGES" = "$CURRENT_SHA" ]; then curl -XPOST \ -i \ -H 'Content-Type: application/json' \ --data "{\"text\":\" Perf tests seem to have changed. Please manually check:\n\`\`\`\ngit diff $TEST_SHA $LAST_CHANGES -- daml-lf/scenario-interpreter/src/perf\n\`\`\`\nand update accordingly. If the change is benign, update \`ci/cron/perf/test_sha\` to \`$LAST_CHANGES\`. With no intervention, you will no longer get performance reports.\"}" \ $(Slack.team-daml) else echo "Changes detected, but not from this commit." fi else echo "No change detected." fi displayName: check perf changes - job: release dependsOn: [ "check_for_release", "Linux", "macOS", "Windows" ] condition: and(succeeded(), eq(dependencies.check_for_release.outputs['out.is_release'], 'true'), or(eq(variables['Build.SourceBranchName'], 'master'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'))) pool: vmImage: "Ubuntu-16.04" variables: linux-tarball: $[ dependencies.Linux.outputs['publish.tarball'] ] macos-tarball: $[ dependencies.macOS.outputs['publish.tarball'] ] windows-tarball: $[ dependencies.Windows.outputs['publish.tarball'] ] windows-installer: $[ dependencies.Windows.outputs['publish.installer'] ] protos-zip: $[ dependencies.Linux.outputs['publish.protos-zip'] ] release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ] release_tag: $[ dependencies.check_for_release.outputs['out.release_tag'] ] trigger_sha: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ] steps: - template: ci/report-start.yml - checkout: self persistCredentials: true - bash: | set -euxo pipefail if git tag v$(release_tag) $(release_sha); then git push origin v$(release_tag) mkdir $(Build.StagingDirectory)/release else echo "##vso[task.setvariable variable=skip-github]TRUE" fi - task: DownloadPipelineArtifact@0 inputs: artifactName: $(linux-tarball) targetPath: $(Build.StagingDirectory)/release condition: not(eq(variables['skip-github'], 'TRUE')) - task: DownloadPipelineArtifact@0 inputs: artifactName: $(macos-tarball) targetPath: $(Build.StagingDirectory)/release condition: not(eq(variables['skip-github'], 'TRUE')) - task: DownloadPipelineArtifact@0 inputs: artifactName: $(windows-tarball) targetPath: $(Build.StagingDirectory)/release condition: not(eq(variables['skip-github'], 'TRUE')) - task: DownloadPipelineArtifact@0 inputs: artifactName: $(windows-installer) targetPath: $(Build.StagingDirectory)/release condition: not(eq(variables['skip-github'], 'TRUE')) - task: DownloadPipelineArtifact@0 inputs: artifactName: $(protos-zip) targetPath: $(Build.StagingDirectory)/release condition: not(eq(variables['skip-github'], 'TRUE')) - bash: | set -euo pipefail KEY_FILE=$(mktemp) GPG_DIR=$(mktemp -d) cleanup() { rm -rf $KEY_FILE $GPG_DIR } trap cleanup EXIT echo "$GPG_KEY" | base64 -d > $KEY_FILE gpg --homedir $GPG_DIR --no-tty --quiet --import $KEY_FILE cd $(Build.StagingDirectory)/release # Note: relies on our release artifacts not having spaces in their # names. Creates a ${f}.asc with the signature for each $f. for f in *; do gpg --homedir $GPG_DIR -ab $f done env: GPG_KEY: $(gpg-code-signing) - task: GitHubRelease@0 inputs: gitHubConnection: 'garyverhaegen-da' repositoryName: '$(Build.Repository.Name)' action: 'create' target: '$(release_sha)' tagSource: 'manual' tag: 'v$(release_tag)' assets: $(Build.StagingDirectory)/release/* assetUploadMode: 'replace' title: '$(release_tag)' addChangeLog: false isPrerelease: true condition: not(eq(variables['skip-github'], 'TRUE')) - template: ci/tell-slack-failed.yml parameters: trigger_sha: '$(trigger_sha)' - template: ci/report-end.yml - job: write_ledger_dump dependsOn: [ "check_for_release" ] pool: vmImage: "Ubuntu-16.04" condition: and(eq(dependencies.check_for_release.outputs['out.is_release'], 'true'), or(eq(variables['Build.SourceBranchName'], 'master'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'))) variables: release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ] release_tag: $[ dependencies.check_for_release.outputs['out.release_tag'] ] trigger_sha: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ] steps: - checkout: self - bash: | set -euo pipefail git checkout $(release_sha) export DAML_SDK_RELEASE_VERSION=$(release_tag) sudo mkdir -p /nix sudo chown $USER /nix curl -sfL https://nixos.org/releases/nix/nix-2.3.3/install | bash eval "$(dev-env/bin/dade-assist)" GCS_KEY=$(mktemp) cleanup () { rm -f $GCS_KEY } trap cleanup EXIT echo "$GOOGLE_APPLICATION_CREDENTIALS_CONTENT" > $GCS_KEY gcloud auth activate-service-account --key-file=$GCS_KEY export BOTO_CONFIG=/dev/null bazel build //ledger/participant-state/kvutils:reference-ledger-dump gsutil cp bazel-bin/ledger/participant-state/kvutils/reference-ledger-dump.out \ gs://daml-dumps/release/ledger/api-server-damlonx/reference-v2/reference-ledger-dump-$(release_tag) env: GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT) - template: ci/tell-slack-failed.yml parameters: trigger_sha: '$(trigger_sha)' - job: collect_build_data condition: always() dependsOn: - Linux - macOS - Windows - release - write_ledger_dump - git_sha - compatibility_macos - compatibility_linux - compatibility_windows pool: name: "linux-pool" demands: assignment -equals default variables: Linux.start: $[ dependencies.Linux.outputs['start.time'] ] Linux.machine: $[ dependencies.Linux.outputs['start.machine'] ] Linux.end: $[ dependencies.Linux.outputs['end.time'] ] Linux.status: $[ dependencies.Linux.result ] macOS.start: $[ dependencies.macOS.outputs['start.time'] ] macOS.machine: $[ dependencies.macOS.outputs['start.machine'] ] macOS.end: $[ dependencies.macOS.outputs['end.time'] ] macOS.status: $[ dependencies.macOS.result ] Windows.start: $[ dependencies.Windows.outputs['start.time'] ] Windows.machine: $[ dependencies.Windows.outputs['start.machine'] ] Windows.end: $[ dependencies.Windows.outputs['end.time'] ] Windows.status: $[ dependencies.Windows.result ] release.start: $[ dependencies.release.outputs['start.time'] ] release.machine: $[ dependencies.release.outputs['start.machine'] ] release.end: $[ dependencies.release.outputs['end.time'] ] release.status: $[ dependencies.release.result ] dump.start: $[ dependencies.write_ledger_dump.outputs['start.time'] ] dump.machine: $[ dependencies.write_ledger_dump.outputs['start.machine'] ] dump.end: $[ dependencies.write_ledger_dump.outputs['end.time'] ] dump.status: $[ dependencies.write_ledger_dump.result ] compatibility_linux.start: $[ dependencies.compatibility_linux.outputs['start.time'] ] compatibility_linux.machine: $[ dependencies.compatibility_linux.outputs['start.machine'] ] compatibility_linux.end: $[ dependencies.compatibility_linux.outputs['end.time'] ] compatibility_linux.status: $[ dependencies.compatibility_linux.result ] compatibility_macos.start: $[ dependencies.compatibility_macos.outputs['start.time'] ] compatibility_macos.machine: $[ dependencies.compatibility_macos.outputs['start.machine'] ] compatibility_macos.end: $[ dependencies.compatibility_macos.outputs['end.time'] ] compatibility_macos.status: $[ dependencies.compatibility_macos.result ] compatibility_windows.start: $[ dependencies.compatibility_windows.outputs['start.time'] ] compatibility_windows.machine: $[ dependencies.compatibility_windows.outputs['start.machine'] ] compatibility_windows.end: $[ dependencies.compatibility_windows.outputs['end.time'] ] compatibility_windows.status: $[ dependencies.compatibility_windows.result ] branch_sha: $[ dependencies.git_sha.outputs['out.branch'] ] master_sha: $[ dependencies.git_sha.outputs['out.master'] ] fork_sha: $[ dependencies.git_sha.outputs['out.fork_point'] ] # Using expression syntax so we get an empty string if not set, rather # than the raw $(VarName) string. Expression syntax works on the # variables key, but not on the env one, so we need an extra indirection. # Note: These Azure variables are only set for PR builds. pr.num: $[ variables['System.PullRequest.PullRequestNumber'] ] pr.branch: $[ variables['System.PullRequest.SourceBranch'] ] steps: - bash: | set -euo pipefail eval "$(./dev-env/bin/dade-assist)" REPORT=$(mktemp) cat >$REPORT < $REPORT_GZ GCS_KEY=$(mktemp) cleanup() { rm -rf $GCS_KEY } trap cleanup EXIT # Application credentials will not be set for forks. We give up on # tracking those for now. "Not set" in Azure world means set to the # expression Azure would otherwise substitute, i.e. the literal value # of the string in the `env:` block below. if [[ "${GOOGLE_APPLICATION_CREDENTIALS_CONTENT:1:${#GOOGLE_APPLICATION_CREDENTIALS_CONTENT}-1}" != '(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)' ]]; then echo "$GOOGLE_APPLICATION_CREDENTIALS_CONTENT" > $GCS_KEY gcloud auth activate-service-account --key-file=$GCS_KEY BOTO_CONFIG=/dev/null gsutil cp $REPORT_GZ gs://daml-data/builds/$(Build.BuildId)_$(date -u +%Y%m%d_%H%M%SZ).json.gz else echo "Could not save build data: no credentials. Data was:" cat $REPORT fi # Linux, macOS and Windows are always required and should always # succeed. # # release and write_ledger_dump only run on releases and are skipped # otherwise. if [[ "$(Linux.status)" != "Succeeded" || "$(macOS.status)" != "Succeeded" || "$(Windows.status)" != "Succeeded" || "$(compatibility_linux.status)" != "Succeeded" || "$(compatibility_macos.status)" != "Succeeded" || "$(compatibility_windows.status)" != "Succeeded" || "$(dump.status)" == "Canceled" || "$(release.status)" == "Canceled" ]]; then exit 1 fi env: GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT) # Commit message is always set COMMIT_MSG: $(Build.SourceVersionMessage) # Because these variables are always set (in the variables block), # hopefully these should be set as expected (i.e. either correct # value or empty string, but not $(Azure.Variable.Name)). PR_NUM: $(pr.num) PR_BRANCH: $(pr.branch) - job: notify_user condition: and(eq(variables['Build.Reason'], 'PullRequest'), not(canceled())) dependsOn: - git_sha - collect_build_data pool: name: 'linux-pool' demands: assignment -equals default variables: pr.num: $[ variables['System.PullRequest.PullRequestNumber'] ] branch_sha: $[ dependencies.git_sha.outputs['out.branch'] ] status: $[ dependencies.collect_build_data.result ] steps: - bash: | set -euo pipefail tell_slack() { local MESSAGE=$1 local USER_ID=$2 curl -XPOST \ -i \ -H 'Content-Type: application/json' \ --data "{\"text\":\"<@${USER_ID}> for has completed with status ${MESSAGE}.\"}" \ $(Slack.team-daml-ci) } EMAIL=$(git log -n 1 --format=%ae $(branch_sha)) user_registered() { cat ci/slack_user_ids | grep $EMAIL } user_id() { echo $(cat ci/slack_user_ids | grep $EMAIL | awk '{print $2}') } if user_registered; then tell_slack "$(status)" "$(user_id)" else echo "User $(user_id) did not opt in for notifications." fi