daml/azure-pipelines.yml
Moritz Kiefer 1243afc4a1
Remove reference to release-notes.rst (#7524)
* Remove reference to release-notes.rst

https://github.com/digital-asset/daml/pull/7458 shuffled this
around. While we could update it, it doesn’t really make any sense. We
post our release notes to the blog now and not in the docs so this
whole checkout procedure is redundant. This is also true if we wanted
to make a bugfix release for a release < 1.5 where this file still
existed. The trigger_sha is always on master (following our current
release process) so the file would still not exist.

I did also remove it from the docs cronjob. We never reupload old docs
so this doesn’t make a difference.

changelog_begin
changelog_end

* Stupid whitespace change because windows is pissing me off

changelog_begin
changelog_end
2020-09-30 11:34:43 +00:00

891 lines
35 KiB
YAML

# Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# Azure Pipelines file, see https://aka.ms/yaml
# Enable builds on all branches
trigger:
# Build every commit as our release process relies on
# the release process being built alone.
batch: false
branches:
include:
- master
- release/*
# Enable PR triggers that target the master branch
pr:
autoCancel: true # cancel previous builds on push
branches:
include:
- master
- release/*
jobs:
- job: git_sha
pool:
name: 'linux-pool'
demands: assignment -equals default
steps:
- bash: |
set -euo pipefail
if [ "$(Build.Reason)" == "PullRequest" ]; then
echo "##vso[task.setvariable variable=branch;isOutput=true]$(git rev-parse HEAD^2)"
echo "##vso[task.setvariable variable=master;isOutput=true]$(git rev-parse HEAD^1)"
echo "##vso[task.setvariable variable=fork_point;isOutput=true]$(git merge-base $(git rev-parse HEAD^1) $(git rev-parse HEAD^2))"
else
echo "##vso[task.setvariable variable=branch;isOutput=true]$(git rev-parse HEAD)"
echo "##vso[task.setvariable variable=master;isOutput=true]$(git rev-parse HEAD^1)"
echo "##vso[task.setvariable variable=fork_point;isOutput=true]$(git rev-parse HEAD^1)"
fi
name: out
- job: check_standard_change_label
dependsOn:
- git_sha
variables:
fork_sha: $[ dependencies.git_sha.outputs['out.fork_point'] ]
branch_sha: $[ dependencies.git_sha.outputs['out.branch'] ]
condition: eq(variables['Build.Reason'], 'PullRequest')
pool:
name: 'linux-pool'
demands: assignment -equals default
steps:
- checkout: self
- bash: |
set -euo pipefail
has_changed () {
git diff $(fork_sha) $(branch_sha) --name-only | grep -q "^$1"
}
fail_if_missing_std_change_label () {
curl https://api.github.com/repos/digital-asset/daml/pulls/$PR -s | jq -r '.labels[].name' | grep -q '^Standard-Change$'
}
if has_changed "infra/" || has_changed "LATEST"; then
fail_if_missing_std_change_label
fi
env:
PR: $(System.PullRequest.PullRequestNumber)
- job: check_changelog_entry
dependsOn:
- git_sha
variables:
fork_sha: $[ dependencies.git_sha.outputs['out.fork_point'] ]
condition: eq(variables['Build.Reason'], 'PullRequest')
pool:
name: 'linux-pool'
demands: assignment -equals default
steps:
- checkout: self
- bash: ci/check-changelog.sh $(fork_sha)
- template: ci/da-ghc-lib/compile.yml
parameters:
final_job_name: da_ghc_lib
- job: Linux
dependsOn:
- da_ghc_lib
- check_for_release
variables:
release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
release_tag: $[ coalesce(dependencies.check_for_release.outputs['out.release_tag'], '0.0.0') ]
trigger_sha: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ]
is_release: $[ dependencies.check_for_release.outputs['out.is_release'] ]
timeoutInMinutes: 360
pool:
name: 'linux-pool'
demands: assignment -equals default
steps:
- template: ci/report-start.yml
- checkout: self
- bash: |
set -euo pipefail
git checkout $(release_sha)
name: checkout_release
condition: eq(variables.is_release, 'true')
- template: ci/build-unix.yml
parameters:
release_tag: $(release_tag)
name: 'linux'
is_release: variables.is_release
- bash: |
set -euo pipefail
eval "$(./dev-env/bin/dade-assist)"
bazel build //release:release
./bazel-bin/release/release --release-dir "$(mktemp -d)"
condition: and(succeeded(), ne(variables['is_release'], 'true'))
- template: ci/tell-slack-failed.yml
parameters:
trigger_sha: '$(trigger_sha)'
- template: ci/report-end.yml
- job: macOS
dependsOn:
- da_ghc_lib
- check_for_release
timeoutInMinutes: 360
pool:
name: macOS-pool
variables:
release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
release_tag: $[ coalesce(dependencies.check_for_release.outputs['out.release_tag'], '0.0.0') ]
trigger_sha: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ]
is_release: $[ dependencies.check_for_release.outputs['out.is_release'] ]
steps:
- template: ci/report-start.yml
- template: ci/clear-shared-segments-macos.yml
- checkout: self
- bash: |
set -euo pipefail
git checkout $(release_sha)
name: checkout_release
condition: eq(variables.is_release, 'true')
- template: ci/build-unix.yml
parameters:
release_tag: $(release_tag)
name: macos
is_release: variables.is_release
- template: ci/tell-slack-failed.yml
parameters:
trigger_sha: '$(trigger_sha)'
- template: ci/report-end.yml
- template: ci/patch_bazel_windows/compile.yml
parameters:
final_job_name: patch_bazel_windows
- job: Windows
dependsOn:
- da_ghc_lib
- check_for_release
- patch_bazel_windows
variables:
release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
release_tag: $[ coalesce(dependencies.check_for_release.outputs['out.release_tag'], '0.0.0') ]
trigger_sha: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ]
is_release: $[ dependencies.check_for_release.outputs['out.is_release'] ]
timeoutInMinutes: 360
pool:
name: 'windows-pool'
demands: assignment -equals default
steps:
- template: ci/report-start.yml
- checkout: self
- bash: |
set -euo pipefail
git checkout $(release_sha)
name: checkout_release
condition: eq(variables.is_release, 'true')
- template: ci/build-windows.yml
parameters:
release_tag: $(release_tag)
is_release: variables.is_release
- task: PublishBuildArtifacts@1
condition: succeededOrFailed()
inputs:
pathtoPublish: '$(Build.StagingDirectory)'
artifactName: 'Bazel Logs'
- template: ci/tell-slack-failed.yml
parameters:
trigger_sha: '$(trigger_sha)'
- template: ci/report-end.yml
- job: compatibility_ts_libs
dependsOn:
- da_ghc_lib
- check_for_release
timeoutInMinutes: 360
pool:
name: linux-pool
demands: assignment -equals default
steps:
- template: ci/report-start.yml
- checkout: self
- template: ci/compatibility_ts_libs.yml
- template: ci/tell-slack-failed.yml
- template: ci/report-end.yml
- job: compatibility_linux
dependsOn:
- da_ghc_lib
- check_for_release
- compatibility_ts_libs
timeoutInMinutes: 360
pool:
name: linux-pool
demands: assignment -equals default
steps:
- template: ci/report-start.yml
- checkout: self
- template: ci/compatibility.yml
parameters:
test_flags: '--quick'
- template: ci/tell-slack-failed.yml
- template: ci/report-end.yml
- job: compatibility_macos
dependsOn:
- da_ghc_lib
- check_for_release
- compatibility_ts_libs
timeoutInMinutes: 360
pool:
name: macOS-pool
steps:
- template: ci/report-start.yml
- template: ci/clear-shared-segments-macos.yml
- checkout: self
- template: ci/compatibility.yml
parameters:
test_flags: '--quick'
- template: ci/tell-slack-failed.yml
- template: ci/report-end.yml
- job: compatibility_windows
dependsOn:
- da_ghc_lib
- check_for_release
- compatibility_ts_libs
- patch_bazel_windows
timeoutInMinutes: 360
pool:
name: 'windows-pool'
demands: assignment -equals default
steps:
- template: ci/report-start.yml
- checkout: self
- template: ci/compatibility-windows.yml
parameters:
test_flags: '--quick'
- template: ci/tell-slack-failed.yml
- template: ci/report-end.yml
- task: PublishBuildArtifacts@1
condition: succeededOrFailed()
inputs:
pathtoPublish: '$(Build.StagingDirectory)'
artifactName: 'Bazel Compatibility Logs'
- job: check_for_release
dependsOn:
- git_sha
variables:
branch_sha: $[ dependencies.git_sha.outputs['out.branch'] ]
fork_sha: $[ dependencies.git_sha.outputs['out.fork_point'] ]
pool:
name: "linux-pool"
demands: assignment -equals default
steps:
- bash: |
set -euo pipefail
./release.sh check
changes_release_files() {
changed="$(git diff-tree --no-commit-id --name-only -r $(fork_sha) $(branch_sha) | sort)"
[ "LATEST" = "$changed" ]
}
changes_one_line_in_latest() {
changed="$(git diff-tree --no-commit-id --numstat -r $(fork_sha) $(branch_sha) -- LATEST | awk '{print $1 "_" $2}')"
add_one="1_0"
change_one="1_1"
[[ "$add_one" == "$changed" || "$change_one" == "$changed" ]]
}
setvar() {
echo "Setting '$1' to '$2'"
echo "##vso[task.setvariable variable=$1;isOutput=true]$2"
}
added_line() {
echo "$(git diff $(fork_sha) $(branch_sha) -- LATEST | tail -n+6 | grep '^\+' | cut -c2-)"
}
if changes_release_files; then
if changes_one_line_in_latest; then
setvar is_release true
setvar trigger_sha $(branch_sha)
setvar release_sha "$(added_line | awk '{print $1}')"
setvar release_tag "$(added_line | awk '{print $2}')"
else
echo "Release commit should only add one version."
exit 1
fi
else
setvar is_release false
fi
name: out
- job: check_perf_test
pool:
name: linux-pool
demands: assignment -equals default
condition: eq(variables['Build.Reason'], 'IndividualCI')
steps:
- bash: |
TEST_SHA=$(cat ci/cron/perf/test_sha)
LAST_CHANGES=$(git log -n1 --format=%H daml-lf/scenario-interpreter/src/perf)
CURRENT_SHA=$(git rev-parse HEAD)
if [ "$TEST_SHA" != "$LAST_CHANGES" ]; then
if [ "$LAST_CHANGES" = "$CURRENT_SHA" ]; then
curl -XPOST \
-i \
-H 'Content-Type: application/json' \
--data "{\"text\":\"<!here> Perf tests seem to have changed. Please manually check:\n\`\`\`\ngit diff $TEST_SHA $LAST_CHANGES -- daml-lf/scenario-interpreter/src/perf\n\`\`\`\nand update accordingly. If the change is benign, update \`ci/cron/perf/test_sha\` to \`$LAST_CHANGES\`. With no intervention, you will no longer get performance reports.\"}" \
$(Slack.team-daml)
else
echo "Changes detected, but not from this commit."
fi
else
echo "No change detected."
fi
displayName: check perf changes
- job: release
dependsOn: [ "check_for_release", "Linux", "macOS", "Windows" ]
condition: and(succeeded(),
eq(dependencies.check_for_release.outputs['out.is_release'], 'true'),
eq(variables['Build.SourceBranchName'], 'master'))
pool:
vmImage: "Ubuntu-16.04"
variables:
linux-tarball: $[ dependencies.Linux.outputs['publish.tarball'] ]
macos-tarball: $[ dependencies.macOS.outputs['publish.tarball'] ]
windows-tarball: $[ dependencies.Windows.outputs['publish.tarball'] ]
windows-installer: $[ dependencies.Windows.outputs['publish.installer'] ]
protos-zip: $[ dependencies.Linux.outputs['publish.protos-zip'] ]
daml-on-sql: $[ dependencies.Linux.outputs['publish.daml-on-sql'] ]
json-api: $[ dependencies.Linux.outputs['publish.json-api'] ]
script-runner: $[ dependencies.Linux.outputs['publish.script-runner'] ]
trigger-runner: $[ dependencies.Linux.outputs['publish.trigger-runner'] ]
release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
release_tag: $[ dependencies.check_for_release.outputs['out.release_tag'] ]
trigger_sha: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ]
steps:
- template: ci/report-start.yml
- checkout: self
persistCredentials: true
- bash: |
set -euo pipefail
if git tag v$(release_tag) $(release_sha); then
git push origin v$(release_tag)
mkdir $(Build.StagingDirectory)/release
mkdir $(Build.StagingDirectory)/artifactory
else
echo "##vso[task.setvariable variable=skip-github]TRUE"
fi
- task: DownloadPipelineArtifact@0
inputs:
artifactName: $(linux-tarball)
targetPath: $(Build.StagingDirectory)/release
condition: not(eq(variables['skip-github'], 'TRUE'))
- task: DownloadPipelineArtifact@0
inputs:
artifactName: $(macos-tarball)
targetPath: $(Build.StagingDirectory)/release
condition: not(eq(variables['skip-github'], 'TRUE'))
- task: DownloadPipelineArtifact@0
inputs:
artifactName: $(windows-tarball)
targetPath: $(Build.StagingDirectory)/release
condition: not(eq(variables['skip-github'], 'TRUE'))
- task: DownloadPipelineArtifact@0
inputs:
artifactName: $(windows-installer)
targetPath: $(Build.StagingDirectory)/release
condition: not(eq(variables['skip-github'], 'TRUE'))
- task: DownloadPipelineArtifact@0
inputs:
artifactName: $(protos-zip)
targetPath: $(Build.StagingDirectory)/release
condition: not(eq(variables['skip-github'], 'TRUE'))
- task: DownloadPipelineArtifact@0
inputs:
artifactName: $(daml-on-sql)
targetPath: $(Build.StagingDirectory)/release
condition: not(eq(variables['skip-github'], 'TRUE'))
- task: DownloadPipelineArtifact@0
inputs:
artifactName: $(json-api)
targetPath: $(Build.StagingDirectory)/release
condition: not(eq(variables['skip-github'], 'TRUE'))
- task: DownloadPipelineArtifact@0
inputs:
artifactName: $(trigger-runner)
targetPath: $(Build.StagingDirectory)/artifactory
condition: not(eq(variables['skip-github'], 'TRUE'))
- task: DownloadPipelineArtifact@0
inputs:
artifactName: $(script-runner)
targetPath: $(Build.StagingDirectory)/artifactory
condition: not(eq(variables['skip-github'], 'TRUE'))
- bash: |
set -euo pipefail
KEY_FILE=$(mktemp)
GPG_DIR=$(mktemp -d)
cleanup() {
rm -rf $KEY_FILE $GPG_DIR
}
trap cleanup EXIT
echo "$GPG_KEY" | base64 -d > $KEY_FILE
gpg --homedir $GPG_DIR --no-tty --quiet --import $KEY_FILE
cd $(Build.StagingDirectory)/release
sha256sum $(find . -type f | sort) > sha256sums
# Note: relies on our release artifacts not having spaces in their
# names. Creates a ${f}.asc with the signature for each $f.
for f in *; do
gpg --homedir $GPG_DIR -ab $f
done
cd ../artifactory
for f in *; do
gpg --homedir $GPG_DIR -ab $f
done
env:
GPG_KEY: $(gpg-code-signing)
- task: GitHubRelease@0
inputs:
gitHubConnection: 'garyverhaegen-da'
repositoryName: '$(Build.Repository.Name)'
action: 'create'
target: '$(release_sha)'
tagSource: 'manual'
tag: 'v$(release_tag)'
assets: $(Build.StagingDirectory)/release/*
assetUploadMode: 'replace'
title: '$(release_tag)'
addChangeLog: false
isPrerelease: true
releaseNotesSource: 'input'
releaseNotes: "This is a pre-release. Use at your own risk."
condition: not(eq(variables['skip-github'], 'TRUE'))
- bash: |
set -euo pipefail
GCS_KEY=$(mktemp)
cleanup () {
rm -f $GCS_KEY
}
trap cleanup EXIT
echo "$GOOGLE_APPLICATION_CREDENTIALS_CONTENT" > $GCS_KEY
gcloud auth activate-service-account --key-file=$GCS_KEY
export BOTO_CONFIG=/dev/null
gsutil cp $(Build.StagingDirectory)/release/$(daml-on-sql) \
gs://daml-binaries/daml-on-sql/$(daml-on-sql)
gsutil cp $(Build.StagingDirectory)/release/$(daml-on-sql).asc \
gs://daml-binaries/daml-on-sql/$(daml-on-sql).asc
gsutil cp $(Build.StagingDirectory)/release/$(json-api) \
gs://daml-binaries/json-api/$(json-api)
gsutil cp $(Build.StagingDirectory)/release/$(json-api).asc \
gs://daml-binaries/json-api/$(json-api).asc
env:
GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
condition: not(eq(variables['skip-github'], 'TRUE'))
- bash: |
set -euo pipefail
push() {
local file repository
repository=$1
file=$2
curl -f \
-u "$AUTH" \
-H "X-Checksum-MD5:$(md5sum ${local} | awk '{print $1}')" \
-H "X-Checksum-SHA1:$(sha1sum ${local} | awk '{print $1}')" \
-X PUT \
-T $(Build.StagingDirectory)/artifactory/${file} \
https://digitalasset.jfrog.io/artifactory/${repository}/$(release_tag)/${file}
}
push daml-trigger-runner $(trigger-runner)
push daml-trigger-runner $(trigger-runner).asc
push daml-script-runner $(script-runner)
push daml-script-runner $(script-runner).asc
env:
AUTH: $(ARTIFACTORY_USERPASS)
condition: not(eq(variables['skip-github'], 'TRUE'))
- bash: |
set -euo pipefail
pr_handler=$(head -1 release/rotation | awk '{print $1}')
curl -XPOST \
-i \
-H 'Content-Type: application/json' \
--data "{\"text\":\"<@${pr_handler}> Release \`$(release_tag)\` is ready for testing. (<https://dev.azure.com/digitalasset/daml/_build/results?buildId=$(Build.BuildId)|build>, <https://github.com/digital-asset/daml/commit/$(trigger_sha)|trigger commit>, <https://github.com/digital-asset/daml/commit/$(release_sha)|target commit>)\"}" \
$(Slack.team-daml)
- template: ci/tell-slack-failed.yml
parameters:
trigger_sha: '$(trigger_sha)'
- template: ci/report-end.yml
- job: compat_versions_pr
dependsOn:
- release
- check_for_release
pool:
name: linux-pool
demands: assignment -equals default
variables:
release_tag: $[ dependencies.check_for_release.outputs['out.release_tag'] ]
steps:
- checkout: self
persistCredentials: true
- bash: ci/dev-env-install.sh
- template: ci/bash-lib.yml
parameters:
var_name: bash_lib
- bash: |
set -euo pipefail
eval "$(./dev-env/bin/dade-assist)"
source $(bash_lib)
setvar() {
echo "Setting '$1' to '$2'"
echo "##vso[task.setvariable variable=$1;isOutput=true]$2"
}
DELAY=1
while ! curl --fail -I https://repo1.maven.org/maven2/com/daml/ledger-api-test-tool/$(release_tag)/ledger-api-test-tool-$(release_tag).jar; do
sleep $DELAY
DELAY=$(( DELAY * 2 ))
if (( $DELAY > 2000 )); then
echo "Too many attempts waiting for Maven."
exit 1
fi
done
AUTH="$(get_gh_auth_header)"
git checkout origin/master
BRANCH=update-compat-versions-for-$(release_tag)
# if this is a rerun, branch might already exist
git branch -D $BRANCH || true
git checkout -b $BRANCH
cp .bazelrc compatibility/
compatibility/update-versions.sh
# Note that maven_install.json might be unchanged but
# git add is happy about that.
git add compatibility/versions.bzl compatibility/maven_install.json
git -c user.name="Azure Pipelines DAML Build" \
-c user.email="support@digitalasset.com" \
commit \
-m "$(printf "update compat versions for $(release_tag)\n\nCHANGELOG_BEGIN\nCHANGELOG_END\n")"
git push origin $BRANCH:$BRANCH
curl -H "Content-Type: application/json" \
-H "$AUTH" \
--silent \
--fail \
--include \
--location \
-d "{\"title\": \"update compat versions for $(release_tag)\", \"head\": \"$BRANCH\", \"base\": \"master\"}" \
https://api.github.com/repos/digital-asset/daml/pulls
setvar "branch" "$BRANCH"
name: out
- template: ci/tell-slack-failed.yml
parameters:
trigger_sha: '$(trigger_sha)'
- job: compat_versions_pr_trigger_daily
dependsOn: compat_versions_pr
pool:
vmImage: "ubuntu-latest"
variables:
branch: $[ dependencies.compat_versions_pr.outputs['out.branch'] ]
steps:
- checkout: none
- bash: |
set -euo pipefail
az extension add --name azure-devops
echo "$(System.AccessToken)" | az devops login --org "https://dev.azure.com/digitalasset"
az pipelines build queue --branch $(branch) --definition-name "digital-asset.daml-daily-compat" --org "https://dev.azure.com/digitalasset" --project daml
az pipelines build queue --branch $(branch) --definition-name "digital-asset.daml" --org "https://dev.azure.com/digitalasset" --project daml
- job: write_ledger_dump
dependsOn:
- check_for_release
- da_ghc_lib
pool:
vmImage: "Ubuntu-16.04"
condition: and(eq(dependencies.check_for_release.outputs['out.is_release'], 'true'),
eq(variables['Build.SourceBranchName'], 'master'))
variables:
release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
release_tag: $[ dependencies.check_for_release.outputs['out.release_tag'] ]
trigger_sha: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ]
steps:
- checkout: self
- bash: |
set -euo pipefail
git checkout $(release_sha)
export DAML_SDK_RELEASE_VERSION=$(release_tag)
sudo mkdir -p /nix
sudo chown $USER /nix
curl -sSfL https://nixos.org/releases/nix/nix-2.3.3/install | bash
eval "$(dev-env/bin/dade-assist)"
GCS_KEY=$(mktemp)
cleanup () {
rm -f $GCS_KEY
}
trap cleanup EXIT
echo "$GOOGLE_APPLICATION_CREDENTIALS_CONTENT" > $GCS_KEY
gcloud auth activate-service-account --key-file=$GCS_KEY
export BOTO_CONFIG=/dev/null
bazel build //ledger/participant-state/kvutils:reference-ledger-dump
gsutil cp bazel-bin/ledger/participant-state/kvutils/reference-ledger-dump.out \
gs://daml-dumps/release/ledger/api-server-damlonx/reference-v2/reference-ledger-dump-$(release_tag)
env:
GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
- template: ci/tell-slack-failed.yml
parameters:
trigger_sha: '$(trigger_sha)'
- job: collect_build_data
condition: always()
dependsOn:
- Linux
- macOS
- Windows
- release
- write_ledger_dump
- git_sha
- compatibility_macos
- compatibility_linux
- compatibility_windows
pool:
name: "linux-pool"
demands: assignment -equals default
variables:
Linux.start: $[ dependencies.Linux.outputs['start.time'] ]
Linux.machine: $[ dependencies.Linux.outputs['start.machine'] ]
Linux.end: $[ dependencies.Linux.outputs['end.time'] ]
Linux.status: $[ dependencies.Linux.result ]
macOS.start: $[ dependencies.macOS.outputs['start.time'] ]
macOS.machine: $[ dependencies.macOS.outputs['start.machine'] ]
macOS.end: $[ dependencies.macOS.outputs['end.time'] ]
macOS.status: $[ dependencies.macOS.result ]
Windows.start: $[ dependencies.Windows.outputs['start.time'] ]
Windows.machine: $[ dependencies.Windows.outputs['start.machine'] ]
Windows.end: $[ dependencies.Windows.outputs['end.time'] ]
Windows.status: $[ dependencies.Windows.result ]
release.start: $[ dependencies.release.outputs['start.time'] ]
release.machine: $[ dependencies.release.outputs['start.machine'] ]
release.end: $[ dependencies.release.outputs['end.time'] ]
release.status: $[ dependencies.release.result ]
dump.start: $[ dependencies.write_ledger_dump.outputs['start.time'] ]
dump.machine: $[ dependencies.write_ledger_dump.outputs['start.machine'] ]
dump.end: $[ dependencies.write_ledger_dump.outputs['end.time'] ]
dump.status: $[ dependencies.write_ledger_dump.result ]
compatibility_linux.start: $[ dependencies.compatibility_linux.outputs['start.time'] ]
compatibility_linux.machine: $[ dependencies.compatibility_linux.outputs['start.machine'] ]
compatibility_linux.end: $[ dependencies.compatibility_linux.outputs['end.time'] ]
compatibility_linux.status: $[ dependencies.compatibility_linux.result ]
compatibility_macos.start: $[ dependencies.compatibility_macos.outputs['start.time'] ]
compatibility_macos.machine: $[ dependencies.compatibility_macos.outputs['start.machine'] ]
compatibility_macos.end: $[ dependencies.compatibility_macos.outputs['end.time'] ]
compatibility_macos.status: $[ dependencies.compatibility_macos.result ]
compatibility_windows.start: $[ dependencies.compatibility_windows.outputs['start.time'] ]
compatibility_windows.machine: $[ dependencies.compatibility_windows.outputs['start.machine'] ]
compatibility_windows.end: $[ dependencies.compatibility_windows.outputs['end.time'] ]
compatibility_windows.status: $[ dependencies.compatibility_windows.result ]
branch_sha: $[ dependencies.git_sha.outputs['out.branch'] ]
master_sha: $[ dependencies.git_sha.outputs['out.master'] ]
fork_sha: $[ dependencies.git_sha.outputs['out.fork_point'] ]
# Using expression syntax so we get an empty string if not set, rather
# than the raw $(VarName) string. Expression syntax works on the
# variables key, but not on the env one, so we need an extra indirection.
# Note: These Azure variables are only set for PR builds.
pr.num: $[ variables['System.PullRequest.PullRequestNumber'] ]
pr.branch: $[ variables['System.PullRequest.SourceBranch'] ]
steps:
- template: ci/bash-lib.yml
parameters:
var_name: bash_lib
- bash: |
set -euo pipefail
eval "$(./dev-env/bin/dade-assist)"
source $(bash_lib)
REPORT=$(mktemp)
cat >$REPORT <<END
{"jobs": {"Linux": {"start": "$(Linux.start)",
"machine": "$(Linux.machine)",
"end": "$(Linux.end)",
"status": "$(Linux.status)"},
"macOS": {"start": "$(macOS.start)",
"machine": "$(macOS.machine)",
"end": "$(macOS.end)",
"status": "$(macOS.status)"},
"Windows": {"start": "$(Windows.start)",
"machine": "$(Windows.machine)",
"end": "$(Windows.end)",
"status": "$(Windows.status)"},
"write_ledger_dump": {"start": "$(dump.start)",
"machine": "$(dump.machine)",
"end": "$(dump.end)",
"status": "$(dump.status)"},
"release": {"start": "$(release.start)",
"machine": "$(release.machine)",
"end": "$(release.end)",
"status": "$(release.status)"},
"compatibility_linux": {"start": "$(compatibility_linux.start)",
"machine": "$(compatibility_linux.machine)",
"end": "$(compatibility_linux.end)",
"status": "$(compatibility_linux.status)"},
"compatibility_macos": {"start": "$(compatibility_macos.start)",
"machine": "$(compatibility_macos.machine)",
"end": "$(compatibility_macos.end)",
"status": "$(compatibility_macos.status)"},
"compatibility_windows": {"start": "$(compatibility_windows.start)",
"machine": "$(compatibility_windows.machine)",
"end": "$(compatibility_windows.end)",
"status": "$(compatibility_windows.status)"}},
"id": "$(Build.BuildId)",
"url": "https://dev.azure.com/digitalasset/daml/_build/results?buildId=$(Build.BuildId)",
"name": "$(Build.DefinitionName)",
"version": "$(Build.DefinitionVersion)",
"queued_by": "$(Build.QueuedBy)",
"reason": "$(Build.Reason)",
"branch": "$(Build.SourceBranch)",
"merge_commit": "$(Build.SourceVersion)",
"branch_commit": "$(branch_sha)",
"master_commit": "$(master_sha)",
"fork_point_commit": "$(fork_sha)",
"commit_message": $(echo -n "$COMMIT_MSG" | jq -sR),
"is_fork": "$(System.PullRequest.IsFork)",
"pr": "$PR_NUM",
"pr_url": "https://github.com/digital-asset/daml/pull/$PR_NUM",
"pr_source_branch": "$PR_BRANCH"}
END
# Test above JSON is well formed
cat $REPORT | jq '.'
REPORT_GZ=$(mktemp)
cat $REPORT | gzip -9 > $REPORT_GZ
# Application credentials will not be set for forks. We give up on
# tracking those for now. "Not set" in Azure world means set to the
# expression Azure would otherwise substitute, i.e. the literal value
# of the string in the `env:` block below.
if [[ "${GCRED:1:${#GCRED}-1}" != '(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)' ]]; then
save_gcp_data "$GCRED" "$REPORT_GZ" "gs://daml-data/builds/$(Build.BuildId)_$(date -u +%Y%m%d_%H%M%SZ).json.gz"
else
echo "Could not save build data: no credentials."
fi
# Linux, macOS and Windows are always required and should always
# succeed.
#
# release and write_ledger_dump only run on releases and are skipped
# otherwise.
if [[ "$(Linux.status)" != "Succeeded"
|| "$(macOS.status)" != "Succeeded"
|| "$(Windows.status)" != "Succeeded"
|| "$(compatibility_linux.status)" != "Succeeded"
|| "$(compatibility_macos.status)" != "Succeeded"
|| "$(compatibility_windows.status)" != "Succeeded"
|| "$(dump.status)" == "Canceled"
|| "$(release.status)" == "Canceled" ]]; then
exit 1
fi
env:
GCRED: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
# Commit message is always set
COMMIT_MSG: $(Build.SourceVersionMessage)
# Because these variables are always set (in the variables block),
# hopefully these should be set as expected (i.e. either correct
# value or empty string, but not $(Azure.Variable.Name)).
PR_NUM: $(pr.num)
PR_BRANCH: $(pr.branch)
- job: notify_release_pr
condition: and(not(canceled()),
startsWith(variables['Build.SourceBranchName'], 'auto-release-pr-'),
eq(dependencies.check_for_release.outputs['out.is_release'], 'true'))
dependsOn:
- git_sha
- collect_build_data
- check_for_release
pool:
name: 'linux-pool'
demands: assignment -equals default
variables:
branch_sha: $[ dependencies.git_sha.outputs['out.branch'] ]
status: $[ dependencies.collect_build_data.result ]
is_release: $[ dependencies.check_for_release.outputs['out.is_release'] ]
steps:
- checkout: self
persistCredentials: true
- template: ci/bash-lib.yml
parameters:
var_name: bash-lib
- bash: |
set -euo pipefail
source $(bash-lib)
AUTH="$(get_gh_auth_header)"
PR=$(curl -H "$AUTH" \
-H "Accept: application/vnd.github.groot-preview+json" \
-s -f \
"https://api.github.com/repos/digital-asset/daml/commits/$(git rev-parse HEAD)/pulls" \
| jq '.[0].number' \
|| echo "")
# Note: if we somehow fail to resolve the PR number from the GitHub
# API, there is still value in getting the notification on Slack, as
# we do have the build number and from there we can click through to
# the PR. Hence the `|| echo ""`.
PR_HANDLER=$(head -1 release/rotation | awk '{print $1}')
tell_slack "<@${PR_HANDLER}> <https://dev.azure.com/digitalasset/daml/_build/results?buildId=$(Build.BuildId)|Build $(Build.BuildId)> for release PR <https://github.com/digital-asset/daml/pull/${PR}|#${PR}> has completed with status $(status). See <https://github.com/digital-asset/daml/blob/master/release/RELEASE.md|RELEASE.md> for what to do next."
- job: notify_user
condition: and(eq(variables['Build.Reason'], 'PullRequest'), not(canceled()))
dependsOn:
- git_sha
- collect_build_data
- check_for_release
pool:
name: 'linux-pool'
demands: assignment -equals default
variables:
pr.num: $[ variables['System.PullRequest.PullRequestNumber'] ]
branch_sha: $[ dependencies.git_sha.outputs['out.branch'] ]
status: $[ dependencies.collect_build_data.result ]
steps:
- bash: |
set -euo pipefail
tell_slack() {
local MESSAGE=$1
local USER_ID=$2
curl -XPOST \
-i \
-H 'Content-Type: application/json' \
--data "{\"text\":\"<@${USER_ID}> <https://dev.azure.com/digitalasset/daml/_build/results?buildId=$(Build.BuildId)|Build $(Build.BuildId)> for <https://github.com/digital-asset/daml/pull/$(pr.num)|PR $(pr.num)> has completed with status ${MESSAGE}.\"}" \
$(Slack.team-daml-ci)
}
EMAIL=$(git log -n 1 --format=%ae $(branch_sha))
user_registered() {
cat ci/slack_user_ids | grep $EMAIL
}
user_id() {
echo $(cat ci/slack_user_ids | grep $EMAIL | awk '{print $2}')
}
if user_registered; then
tell_slack "$(status)" "$(user_id)"
else
echo "User $(user_id) did not opt in for notifications."
fi