mirror of
https://github.com/digital-asset/daml.git
synced 2024-09-20 01:07:18 +03:00
7d6e8f8d5c
* Do not checkout release commit in scala 2.13 At least for now, this fails because older commits don’t have the necessary infrastructure so far. At some point we probably do want to run this on release commits but that can wait a bit and this unblocks release PRs for bugfix releases. changelog_begin changelog_end * Update azure-pipelines.yml Co-authored-by: Gary Verhaegen <gary.verhaegen@digitalasset.com> Co-authored-by: Gary Verhaegen <gary.verhaegen@digitalasset.com>
887 lines
35 KiB
YAML
887 lines
35 KiB
YAML
# Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
# Azure Pipelines file, see https://aka.ms/yaml
|
|
|
|
# Enable builds on all branches
|
|
trigger:
|
|
# Build every commit as our release process relies on
|
|
# the release process being built alone.
|
|
batch: false
|
|
branches:
|
|
include:
|
|
- master
|
|
- release/*
|
|
|
|
# Enable PR triggers that target the master branch
|
|
pr:
|
|
autoCancel: true # cancel previous builds on push
|
|
branches:
|
|
include:
|
|
- master
|
|
- release/*
|
|
|
|
jobs:
|
|
- job: git_sha
|
|
pool:
|
|
name: 'linux-pool'
|
|
demands: assignment -equals default
|
|
steps:
|
|
- bash: |
|
|
set -euo pipefail
|
|
if [ "$(Build.Reason)" == "PullRequest" ]; then
|
|
echo "##vso[task.setvariable variable=branch;isOutput=true]$(git rev-parse HEAD^2)"
|
|
echo "##vso[task.setvariable variable=master;isOutput=true]$(git rev-parse HEAD^1)"
|
|
echo "##vso[task.setvariable variable=fork_point;isOutput=true]$(git merge-base $(git rev-parse HEAD^1) $(git rev-parse HEAD^2))"
|
|
else
|
|
echo "##vso[task.setvariable variable=branch;isOutput=true]$(git rev-parse HEAD)"
|
|
echo "##vso[task.setvariable variable=master;isOutput=true]$(git rev-parse HEAD^1)"
|
|
echo "##vso[task.setvariable variable=fork_point;isOutput=true]$(git rev-parse HEAD^1)"
|
|
fi
|
|
name: out
|
|
|
|
- job: check_standard_change_label
|
|
dependsOn:
|
|
- git_sha
|
|
variables:
|
|
fork_sha: $[ dependencies.git_sha.outputs['out.fork_point'] ]
|
|
branch_sha: $[ dependencies.git_sha.outputs['out.branch'] ]
|
|
condition: eq(variables['Build.Reason'], 'PullRequest')
|
|
pool:
|
|
name: 'linux-pool'
|
|
demands: assignment -equals default
|
|
steps:
|
|
- checkout: self
|
|
- bash: |
|
|
set -euo pipefail
|
|
|
|
has_changed () {
|
|
git diff $(fork_sha) $(branch_sha) --name-only | grep -q "^$1"
|
|
}
|
|
|
|
fail_if_missing_std_change_label () {
|
|
curl https://api.github.com/repos/digital-asset/daml/pulls/$PR -s | jq -r '.labels[].name' | grep -q '^Standard-Change$'
|
|
}
|
|
|
|
if has_changed "infra/" || has_changed "LATEST"; then
|
|
fail_if_missing_std_change_label
|
|
fi
|
|
env:
|
|
PR: $(System.PullRequest.PullRequestNumber)
|
|
|
|
- job: check_changelog_entry
|
|
dependsOn:
|
|
- git_sha
|
|
variables:
|
|
fork_sha: $[ dependencies.git_sha.outputs['out.fork_point'] ]
|
|
branch_sha: $[ dependencies.git_sha.outputs['out.branch'] ]
|
|
pr.num: $[ variables['System.PullRequest.PullRequestNumber'] ]
|
|
condition: eq(variables['Build.Reason'], 'PullRequest')
|
|
pool:
|
|
name: 'linux-pool'
|
|
demands: assignment -equals default
|
|
steps:
|
|
- checkout: self
|
|
- template: ci/bash-lib.yml
|
|
parameters:
|
|
var_name: bash-lib
|
|
- bash: |
|
|
set -euo pipefail
|
|
source $(bash-lib)
|
|
git checkout $(branch_sha)
|
|
if ci/check-changelog.sh $(fork_sha); then
|
|
exit 0
|
|
else
|
|
user=$(user_slack_handle $(branch_sha))
|
|
if [ "$user" != "" ]; then
|
|
tell_slack "<@${user}> <https://dev.azure.com/digitalasset/daml/_build/results?buildId=$(Build.BuildId)|Build $(Build.BuildId)> for <https://github.com/digital-asset/daml/pull/$(pr.num)|PR $(pr.num)> is missing a changelog entry." \
|
|
"$(Slack.team-daml-ci)"
|
|
fi
|
|
exit 1
|
|
fi
|
|
|
|
- template: ci/da-ghc-lib/compile.yml
|
|
parameters:
|
|
final_job_name: da_ghc_lib
|
|
|
|
- job: Linux
|
|
dependsOn:
|
|
- da_ghc_lib
|
|
- check_for_release
|
|
variables:
|
|
release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
|
|
release_tag: $[ coalesce(dependencies.check_for_release.outputs['out.release_tag'], '0.0.0') ]
|
|
trigger_sha: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ]
|
|
is_release: $[ dependencies.check_for_release.outputs['out.is_release'] ]
|
|
timeoutInMinutes: 360
|
|
pool:
|
|
name: 'linux-pool'
|
|
demands: assignment -equals default
|
|
steps:
|
|
- template: ci/report-start.yml
|
|
- checkout: self
|
|
- bash: |
|
|
set -euo pipefail
|
|
git checkout $(release_sha)
|
|
name: checkout_release
|
|
condition: eq(variables.is_release, 'true')
|
|
- template: ci/build-unix.yml
|
|
parameters:
|
|
release_tag: $(release_tag)
|
|
name: 'linux'
|
|
is_release: variables.is_release
|
|
- bash: |
|
|
set -euo pipefail
|
|
eval "$(./dev-env/bin/dade-assist)"
|
|
bazel build //release:release
|
|
./bazel-bin/release/release --release-dir "$(mktemp -d)"
|
|
condition: and(succeeded(), ne(variables['is_release'], 'true'))
|
|
- template: ci/tell-slack-failed.yml
|
|
parameters:
|
|
trigger_sha: '$(trigger_sha)'
|
|
- template: ci/report-end.yml
|
|
|
|
- job: macOS
|
|
dependsOn:
|
|
- da_ghc_lib
|
|
- check_for_release
|
|
timeoutInMinutes: 360
|
|
pool:
|
|
name: macOS-pool
|
|
demands: assignment -equals default
|
|
variables:
|
|
release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
|
|
release_tag: $[ coalesce(dependencies.check_for_release.outputs['out.release_tag'], '0.0.0') ]
|
|
trigger_sha: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ]
|
|
is_release: $[ dependencies.check_for_release.outputs['out.is_release'] ]
|
|
steps:
|
|
- template: ci/report-start.yml
|
|
- template: ci/clear-shared-segments-macos.yml
|
|
- checkout: self
|
|
- bash: |
|
|
set -euo pipefail
|
|
git checkout $(release_sha)
|
|
name: checkout_release
|
|
condition: eq(variables.is_release, 'true')
|
|
- template: ci/build-unix.yml
|
|
parameters:
|
|
release_tag: $(release_tag)
|
|
name: macos
|
|
is_release: variables.is_release
|
|
- template: ci/tell-slack-failed.yml
|
|
parameters:
|
|
trigger_sha: '$(trigger_sha)'
|
|
- template: ci/report-end.yml
|
|
|
|
- template: ci/patch_bazel_windows/compile.yml
|
|
parameters:
|
|
final_job_name: patch_bazel_windows
|
|
|
|
- job: Windows
|
|
dependsOn:
|
|
- da_ghc_lib
|
|
- check_for_release
|
|
- patch_bazel_windows
|
|
variables:
|
|
release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
|
|
release_tag: $[ coalesce(dependencies.check_for_release.outputs['out.release_tag'], '0.0.0') ]
|
|
trigger_sha: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ]
|
|
is_release: $[ dependencies.check_for_release.outputs['out.is_release'] ]
|
|
timeoutInMinutes: 360
|
|
pool:
|
|
name: 'windows-pool'
|
|
demands: assignment -equals default
|
|
steps:
|
|
- template: ci/report-start.yml
|
|
- checkout: self
|
|
- bash: |
|
|
set -euo pipefail
|
|
git checkout $(release_sha)
|
|
name: checkout_release
|
|
condition: eq(variables.is_release, 'true')
|
|
- template: ci/build-windows.yml
|
|
parameters:
|
|
release_tag: $(release_tag)
|
|
is_release: variables.is_release
|
|
- task: PublishBuildArtifacts@1
|
|
condition: succeededOrFailed()
|
|
inputs:
|
|
pathtoPublish: '$(Build.StagingDirectory)'
|
|
artifactName: 'Bazel Logs'
|
|
- template: ci/tell-slack-failed.yml
|
|
parameters:
|
|
trigger_sha: '$(trigger_sha)'
|
|
- template: ci/report-end.yml
|
|
|
|
- job: Linux_scala_2_13
|
|
dependsOn:
|
|
- da_ghc_lib
|
|
timeoutInMinutes: 360
|
|
pool:
|
|
name: 'linux-pool'
|
|
demands: assignment -equals default
|
|
steps:
|
|
- template: ci/report-start.yml
|
|
- checkout: self
|
|
- bash: ci/dev-env-install.sh
|
|
displayName: 'Build/Install the Developer Environment'
|
|
- bash: |
|
|
source dev-env/lib/ensure-nix
|
|
ci/dev-env-push.py
|
|
displayName: 'Push Developer Environment build results'
|
|
condition: eq(variables['System.PullRequest.IsFork'], 'False')
|
|
env:
|
|
# to upload to the Nix cache
|
|
GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
|
|
NIX_SECRET_KEY_CONTENT: $(NIX_SECRET_KEY_CONTENT)
|
|
|
|
- bash: ci/configure-bazel.sh
|
|
displayName: 'Configure Bazel'
|
|
env:
|
|
IS_FORK: $(System.PullRequest.IsFork)
|
|
# to upload to the bazel cache
|
|
GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
|
|
- bash: |
|
|
set -euo pipefail
|
|
eval "$(./dev-env/bin/dade-assist)"
|
|
bazel build --config scala_2_13 //libs-scala/...
|
|
# gatling-utils tests fail with a ClassNotFoundException for scala.collection.SeqLike
|
|
bazel test --config scala_2_13 -- //libs-scala/... -//libs-scala/gatling-utils/...
|
|
displayName: 'Build'
|
|
- template: ci/tell-slack-failed.yml
|
|
parameters:
|
|
trigger_sha: '$(trigger_sha)'
|
|
- template: ci/report-end.yml
|
|
|
|
- job: compatibility_ts_libs
|
|
dependsOn:
|
|
- da_ghc_lib
|
|
- check_for_release
|
|
timeoutInMinutes: 360
|
|
pool:
|
|
name: linux-pool
|
|
demands: assignment -equals default
|
|
steps:
|
|
- template: ci/report-start.yml
|
|
- checkout: self
|
|
- template: ci/compatibility_ts_libs.yml
|
|
- template: ci/tell-slack-failed.yml
|
|
- template: ci/report-end.yml
|
|
|
|
- job: compatibility_linux
|
|
dependsOn:
|
|
- da_ghc_lib
|
|
- check_for_release
|
|
- compatibility_ts_libs
|
|
timeoutInMinutes: 360
|
|
pool:
|
|
name: linux-pool
|
|
demands: assignment -equals default
|
|
steps:
|
|
- template: ci/report-start.yml
|
|
- checkout: self
|
|
- template: ci/compatibility.yml
|
|
parameters:
|
|
test_flags: '--quick'
|
|
- template: ci/tell-slack-failed.yml
|
|
- template: ci/report-end.yml
|
|
|
|
- job: compatibility_macos
|
|
dependsOn:
|
|
- da_ghc_lib
|
|
- check_for_release
|
|
- compatibility_ts_libs
|
|
timeoutInMinutes: 360
|
|
pool:
|
|
name: macOS-pool
|
|
demands: assignment -equals default
|
|
steps:
|
|
- template: ci/report-start.yml
|
|
- template: ci/clear-shared-segments-macos.yml
|
|
- checkout: self
|
|
- template: ci/compatibility.yml
|
|
parameters:
|
|
test_flags: '--quick'
|
|
- template: ci/tell-slack-failed.yml
|
|
- template: ci/report-end.yml
|
|
|
|
- job: compatibility_windows
|
|
dependsOn:
|
|
- da_ghc_lib
|
|
- check_for_release
|
|
- compatibility_ts_libs
|
|
- patch_bazel_windows
|
|
timeoutInMinutes: 360
|
|
pool:
|
|
name: 'windows-pool'
|
|
demands: assignment -equals default
|
|
steps:
|
|
- template: ci/report-start.yml
|
|
- checkout: self
|
|
- template: ci/compatibility-windows.yml
|
|
parameters:
|
|
test_flags: '--quick'
|
|
- template: ci/tell-slack-failed.yml
|
|
- template: ci/report-end.yml
|
|
- task: PublishBuildArtifacts@1
|
|
condition: succeededOrFailed()
|
|
inputs:
|
|
pathtoPublish: '$(Build.StagingDirectory)'
|
|
artifactName: 'Bazel Compatibility Logs'
|
|
|
|
- job: check_for_release
|
|
dependsOn:
|
|
- git_sha
|
|
variables:
|
|
branch_sha: $[ dependencies.git_sha.outputs['out.branch'] ]
|
|
fork_sha: $[ dependencies.git_sha.outputs['out.fork_point'] ]
|
|
pool:
|
|
name: "linux-pool"
|
|
demands: assignment -equals default
|
|
steps:
|
|
- bash: |
|
|
set -euo pipefail
|
|
|
|
./release.sh check
|
|
|
|
changes_release_files() {
|
|
changed="$(git diff-tree --no-commit-id --name-only -r $(fork_sha) $(branch_sha) | sort)"
|
|
[ "LATEST" = "$changed" ]
|
|
}
|
|
|
|
changes_one_line_in_latest() {
|
|
changed="$(git diff-tree --no-commit-id --numstat -r $(fork_sha) $(branch_sha) -- LATEST | awk '{print $1 "_" $2}')"
|
|
add_one="1_0"
|
|
change_one="1_1"
|
|
[[ "$add_one" == "$changed" || "$change_one" == "$changed" ]]
|
|
}
|
|
|
|
setvar() {
|
|
echo "Setting '$1' to '$2'"
|
|
echo "##vso[task.setvariable variable=$1;isOutput=true]$2"
|
|
}
|
|
|
|
added_line() {
|
|
echo "$(git diff $(fork_sha) $(branch_sha) -- LATEST | tail -n+6 | grep '^\+' | cut -c2-)"
|
|
}
|
|
|
|
if changes_release_files; then
|
|
if changes_one_line_in_latest; then
|
|
setvar is_release true
|
|
setvar trigger_sha $(branch_sha)
|
|
setvar release_sha "$(added_line | awk '{print $1}')"
|
|
setvar release_tag "$(added_line | awk '{print $2}')"
|
|
else
|
|
echo "Release commit should only add one version."
|
|
exit 1
|
|
fi
|
|
else
|
|
setvar is_release false
|
|
fi
|
|
name: out
|
|
|
|
- job: check_perf_test
|
|
pool:
|
|
name: linux-pool
|
|
demands: assignment -equals default
|
|
condition: eq(variables['Build.Reason'], 'IndividualCI')
|
|
steps:
|
|
- bash: |
|
|
TEST_SHA=$(cat ci/cron/perf/test_sha)
|
|
LAST_CHANGES=$(git log -n1 --format=%H daml-lf/scenario-interpreter/src/perf)
|
|
CURRENT_SHA=$(git rev-parse HEAD)
|
|
if [ "$TEST_SHA" != "$LAST_CHANGES" ]; then
|
|
if [ "$LAST_CHANGES" = "$CURRENT_SHA" ]; then
|
|
curl -XPOST \
|
|
-i \
|
|
-H 'Content-Type: application/json' \
|
|
--data "{\"text\":\"<!here> Perf tests seem to have changed. Please manually check:\n\`\`\`\ngit diff $TEST_SHA $LAST_CHANGES -- daml-lf/scenario-interpreter/src/perf\n\`\`\`\nand update accordingly. If the change is benign, update \`ci/cron/perf/test_sha\` to \`$LAST_CHANGES\`. With no intervention, you will no longer get performance reports.\"}" \
|
|
$(Slack.team-daml)
|
|
else
|
|
echo "Changes detected, but not from this commit."
|
|
fi
|
|
else
|
|
echo "No change detected."
|
|
fi
|
|
displayName: check perf changes
|
|
|
|
- job: release
|
|
dependsOn: [ "check_for_release", "Linux", "macOS", "Windows" ]
|
|
condition: and(succeeded(),
|
|
eq(dependencies.check_for_release.outputs['out.is_release'], 'true'),
|
|
eq(variables['Build.SourceBranchName'], 'master'))
|
|
pool:
|
|
vmImage: "Ubuntu-16.04"
|
|
variables:
|
|
linux-tarball: $[ dependencies.Linux.outputs['publish.tarball'] ]
|
|
macos-tarball: $[ dependencies.macOS.outputs['publish.tarball'] ]
|
|
windows-tarball: $[ dependencies.Windows.outputs['publish.tarball'] ]
|
|
windows-installer: $[ dependencies.Windows.outputs['publish.installer'] ]
|
|
protos-zip: $[ dependencies.Linux.outputs['publish.protos-zip'] ]
|
|
daml-on-sql: $[ dependencies.Linux.outputs['publish.daml-on-sql'] ]
|
|
json-api: $[ dependencies.Linux.outputs['publish.json-api'] ]
|
|
script-runner: $[ dependencies.Linux.outputs['publish.script-runner'] ]
|
|
trigger-runner: $[ dependencies.Linux.outputs['publish.trigger-runner'] ]
|
|
release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
|
|
release_tag: $[ dependencies.check_for_release.outputs['out.release_tag'] ]
|
|
trigger_sha: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ]
|
|
steps:
|
|
- template: ci/report-start.yml
|
|
- checkout: self
|
|
persistCredentials: true
|
|
- bash: |
|
|
set -euo pipefail
|
|
if git tag v$(release_tag) $(release_sha); then
|
|
git push origin v$(release_tag)
|
|
mkdir $(Build.StagingDirectory)/release
|
|
mkdir $(Build.StagingDirectory)/artifactory
|
|
else
|
|
echo "##vso[task.setvariable variable=skip-github]TRUE"
|
|
fi
|
|
- task: DownloadPipelineArtifact@0
|
|
inputs:
|
|
artifactName: $(linux-tarball)
|
|
targetPath: $(Build.StagingDirectory)/release
|
|
condition: not(eq(variables['skip-github'], 'TRUE'))
|
|
- task: DownloadPipelineArtifact@0
|
|
inputs:
|
|
artifactName: $(macos-tarball)
|
|
targetPath: $(Build.StagingDirectory)/release
|
|
condition: not(eq(variables['skip-github'], 'TRUE'))
|
|
- task: DownloadPipelineArtifact@0
|
|
inputs:
|
|
artifactName: $(windows-tarball)
|
|
targetPath: $(Build.StagingDirectory)/release
|
|
condition: not(eq(variables['skip-github'], 'TRUE'))
|
|
- task: DownloadPipelineArtifact@0
|
|
inputs:
|
|
artifactName: $(windows-installer)
|
|
targetPath: $(Build.StagingDirectory)/release
|
|
condition: not(eq(variables['skip-github'], 'TRUE'))
|
|
- task: DownloadPipelineArtifact@0
|
|
inputs:
|
|
artifactName: $(protos-zip)
|
|
targetPath: $(Build.StagingDirectory)/release
|
|
condition: not(eq(variables['skip-github'], 'TRUE'))
|
|
- task: DownloadPipelineArtifact@0
|
|
inputs:
|
|
artifactName: $(daml-on-sql)
|
|
targetPath: $(Build.StagingDirectory)/release
|
|
condition: not(eq(variables['skip-github'], 'TRUE'))
|
|
- task: DownloadPipelineArtifact@0
|
|
inputs:
|
|
artifactName: $(json-api)
|
|
targetPath: $(Build.StagingDirectory)/release
|
|
condition: not(eq(variables['skip-github'], 'TRUE'))
|
|
- task: DownloadPipelineArtifact@0
|
|
inputs:
|
|
artifactName: $(trigger-runner)
|
|
targetPath: $(Build.StagingDirectory)/artifactory
|
|
condition: not(eq(variables['skip-github'], 'TRUE'))
|
|
- task: DownloadPipelineArtifact@0
|
|
inputs:
|
|
artifactName: $(script-runner)
|
|
targetPath: $(Build.StagingDirectory)/artifactory
|
|
condition: not(eq(variables['skip-github'], 'TRUE'))
|
|
- bash: |
|
|
set -euo pipefail
|
|
KEY_FILE=$(mktemp)
|
|
GPG_DIR=$(mktemp -d)
|
|
cleanup() {
|
|
rm -rf $KEY_FILE $GPG_DIR
|
|
}
|
|
trap cleanup EXIT
|
|
echo "$GPG_KEY" | base64 -d > $KEY_FILE
|
|
gpg --homedir $GPG_DIR --no-tty --quiet --import $KEY_FILE
|
|
cd $(Build.StagingDirectory)/release
|
|
sha256sum $(find . -type f | sort) > sha256sums
|
|
# Note: relies on our release artifacts not having spaces in their
|
|
# names. Creates a ${f}.asc with the signature for each $f.
|
|
for f in *; do
|
|
gpg --homedir $GPG_DIR -ab $f
|
|
done
|
|
cd ../artifactory
|
|
for f in *; do
|
|
gpg --homedir $GPG_DIR -ab $f
|
|
done
|
|
env:
|
|
GPG_KEY: $(gpg-code-signing)
|
|
- task: GitHubRelease@0
|
|
inputs:
|
|
gitHubConnection: 'garyverhaegen-da'
|
|
repositoryName: '$(Build.Repository.Name)'
|
|
action: 'create'
|
|
target: '$(release_sha)'
|
|
tagSource: 'manual'
|
|
tag: 'v$(release_tag)'
|
|
assets: $(Build.StagingDirectory)/release/*
|
|
assetUploadMode: 'replace'
|
|
title: '$(release_tag)'
|
|
addChangeLog: false
|
|
isPrerelease: true
|
|
releaseNotesSource: 'input'
|
|
releaseNotes: "This is a pre-release. Use at your own risk."
|
|
condition: not(eq(variables['skip-github'], 'TRUE'))
|
|
- bash: |
|
|
set -euo pipefail
|
|
|
|
push() {
|
|
local file repository
|
|
repository=$1
|
|
file=$2
|
|
curl -f \
|
|
-u "$AUTH" \
|
|
-H "X-Checksum-MD5:$(md5sum ${local} | awk '{print $1}')" \
|
|
-H "X-Checksum-SHA1:$(sha1sum ${local} | awk '{print $1}')" \
|
|
-X PUT \
|
|
-T $(Build.StagingDirectory)/artifactory/${file} \
|
|
https://digitalasset.jfrog.io/artifactory/${repository}/$(release_tag)/${file}
|
|
}
|
|
|
|
push daml-trigger-runner $(trigger-runner)
|
|
push daml-trigger-runner $(trigger-runner).asc
|
|
push daml-script-runner $(script-runner)
|
|
push daml-script-runner $(script-runner).asc
|
|
env:
|
|
AUTH: $(ARTIFACTORY_USERPASS)
|
|
condition: not(eq(variables['skip-github'], 'TRUE'))
|
|
- template: ci/bash-lib.yml
|
|
parameters:
|
|
var_name: bash-lib
|
|
- bash: |
|
|
set -euo pipefail
|
|
|
|
source $(bash-lib)
|
|
|
|
cd $(Build.StagingDirectory)/release
|
|
for f in *; do
|
|
gcs "$GCRED" cp "$f" "gs://daml-data/releases/$(release_tag)/github/$f"
|
|
done
|
|
|
|
cd $(Build.StagingDirectory)/artifactory
|
|
for f in *; do
|
|
gcs "$GCRED" cp "$f" "gs://daml-data/releases/$(release_tag)/artifactory/$f"
|
|
done
|
|
name: backup_to_gcs
|
|
env:
|
|
GCRED: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
|
|
condition: not(eq(variables['skip-github'], 'TRUE'))
|
|
- bash: |
|
|
set -euo pipefail
|
|
pr_handler=$(head -1 release/rotation | awk '{print $1}')
|
|
curl -XPOST \
|
|
-i \
|
|
-H 'Content-Type: application/json' \
|
|
--data "{\"text\":\"<@${pr_handler}> Release \`$(release_tag)\` is ready for testing. See <https://github.com/digital-asset/daml/blob/master/release/RELEASE.md|release instructions>. (<https://dev.azure.com/digitalasset/daml/_build/results?buildId=$(Build.BuildId)|build>, <https://github.com/digital-asset/daml/commit/$(trigger_sha)|trigger commit>, <https://github.com/digital-asset/daml/commit/$(release_sha)|target commit>)\"}" \
|
|
$(Slack.team-daml)
|
|
- template: ci/tell-slack-failed.yml
|
|
parameters:
|
|
trigger_sha: '$(trigger_sha)'
|
|
- template: ci/report-end.yml
|
|
|
|
- job: compat_versions_pr
|
|
dependsOn:
|
|
- git_sha
|
|
- release
|
|
- check_for_release
|
|
pool:
|
|
name: linux-pool
|
|
demands: assignment -equals default
|
|
variables:
|
|
release_tag: $[ dependencies.check_for_release.outputs['out.release_tag'] ]
|
|
branch_sha: $[ dependencies.git_sha.outputs['out.branch'] ]
|
|
steps:
|
|
- checkout: self
|
|
persistCredentials: true
|
|
- bash: ci/dev-env-install.sh
|
|
- template: ci/bash-lib.yml
|
|
parameters:
|
|
var_name: bash_lib
|
|
- bash: |
|
|
set -euo pipefail
|
|
eval "$(./dev-env/bin/dade-assist)"
|
|
|
|
source $(bash_lib)
|
|
|
|
setvar() {
|
|
echo "Setting '$1' to '$2'"
|
|
echo "##vso[task.setvariable variable=$1;isOutput=true]$2"
|
|
}
|
|
|
|
DELAY=1
|
|
while ! curl --fail -I https://repo1.maven.org/maven2/com/daml/ledger-api-test-tool/$(release_tag)/ledger-api-test-tool-$(release_tag).jar; do
|
|
sleep $DELAY
|
|
DELAY=$(( DELAY * 2 ))
|
|
if (( $DELAY > 2000 )); then
|
|
echo "Too many attempts waiting for Maven."
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
trap "git checkout $(branch_sha)" EXIT
|
|
git checkout origin/master
|
|
cp .bazelrc compatibility/
|
|
compatibility/update-versions.sh
|
|
git add compatibility/versions.bzl compatibility/maven_install.json
|
|
|
|
BRANCH=update-compat-versions-for-$(release_tag)-$(Build.BuildId)
|
|
TITLE="update compat versions for $(release_tag)"
|
|
open_pr "$BRANCH" "$TITLE"
|
|
|
|
setvar "branch" "$BRANCH"
|
|
name: out
|
|
|
|
- template: ci/tell-slack-failed.yml
|
|
parameters:
|
|
trigger_sha: '$(trigger_sha)'
|
|
|
|
- job: compat_versions_pr_trigger_daily
|
|
dependsOn: compat_versions_pr
|
|
pool:
|
|
vmImage: "ubuntu-latest"
|
|
variables:
|
|
branch: $[ dependencies.compat_versions_pr.outputs['out.branch'] ]
|
|
steps:
|
|
- checkout: none
|
|
- bash: |
|
|
set -euo pipefail
|
|
az extension add --name azure-devops
|
|
echo "$(System.AccessToken)" | az devops login --org "https://dev.azure.com/digitalasset"
|
|
az pipelines build queue --branch $(branch) --definition-name "digital-asset.daml-daily-compat" --org "https://dev.azure.com/digitalasset" --project daml
|
|
az pipelines build queue --branch $(branch) --definition-name "digital-asset.daml" --org "https://dev.azure.com/digitalasset" --project daml
|
|
|
|
- job: collect_build_data
|
|
condition: always()
|
|
dependsOn:
|
|
- Linux
|
|
- macOS
|
|
- Windows
|
|
- release
|
|
- git_sha
|
|
- compatibility_macos
|
|
- compatibility_linux
|
|
- compatibility_windows
|
|
- Linux_scala_2_13
|
|
pool:
|
|
name: "linux-pool"
|
|
demands: assignment -equals default
|
|
variables:
|
|
Linux.start: $[ dependencies.Linux.outputs['start.time'] ]
|
|
Linux.machine: $[ dependencies.Linux.outputs['start.machine'] ]
|
|
Linux.end: $[ dependencies.Linux.outputs['end.time'] ]
|
|
Linux.status: $[ dependencies.Linux.result ]
|
|
macOS.start: $[ dependencies.macOS.outputs['start.time'] ]
|
|
macOS.machine: $[ dependencies.macOS.outputs['start.machine'] ]
|
|
macOS.end: $[ dependencies.macOS.outputs['end.time'] ]
|
|
macOS.status: $[ dependencies.macOS.result ]
|
|
Windows.start: $[ dependencies.Windows.outputs['start.time'] ]
|
|
Windows.machine: $[ dependencies.Windows.outputs['start.machine'] ]
|
|
Windows.end: $[ dependencies.Windows.outputs['end.time'] ]
|
|
Windows.status: $[ dependencies.Windows.result ]
|
|
Linux_scala_2_13.start: $[ dependencies.Linux_scala_2_13.outputs['start.time'] ]
|
|
Linux_scala_2_13.machine: $[ dependencies.Linux_scala_2_13.outputs['start.machine'] ]
|
|
Linux_scala_2_13.end: $[ dependencies.Linux_scala_2_13.outputs['end.time'] ]
|
|
Linux_scala_2_13.status: $[ dependencies.Linux_scala_2_13.result ]
|
|
release.start: $[ dependencies.release.outputs['start.time'] ]
|
|
release.machine: $[ dependencies.release.outputs['start.machine'] ]
|
|
release.end: $[ dependencies.release.outputs['end.time'] ]
|
|
release.status: $[ dependencies.release.result ]
|
|
compatibility_linux.start: $[ dependencies.compatibility_linux.outputs['start.time'] ]
|
|
compatibility_linux.machine: $[ dependencies.compatibility_linux.outputs['start.machine'] ]
|
|
compatibility_linux.end: $[ dependencies.compatibility_linux.outputs['end.time'] ]
|
|
compatibility_linux.status: $[ dependencies.compatibility_linux.result ]
|
|
compatibility_macos.start: $[ dependencies.compatibility_macos.outputs['start.time'] ]
|
|
compatibility_macos.machine: $[ dependencies.compatibility_macos.outputs['start.machine'] ]
|
|
compatibility_macos.end: $[ dependencies.compatibility_macos.outputs['end.time'] ]
|
|
compatibility_macos.status: $[ dependencies.compatibility_macos.result ]
|
|
compatibility_windows.start: $[ dependencies.compatibility_windows.outputs['start.time'] ]
|
|
compatibility_windows.machine: $[ dependencies.compatibility_windows.outputs['start.machine'] ]
|
|
compatibility_windows.end: $[ dependencies.compatibility_windows.outputs['end.time'] ]
|
|
compatibility_windows.status: $[ dependencies.compatibility_windows.result ]
|
|
|
|
branch_sha: $[ dependencies.git_sha.outputs['out.branch'] ]
|
|
master_sha: $[ dependencies.git_sha.outputs['out.master'] ]
|
|
fork_sha: $[ dependencies.git_sha.outputs['out.fork_point'] ]
|
|
|
|
# Using expression syntax so we get an empty string if not set, rather
|
|
# than the raw $(VarName) string. Expression syntax works on the
|
|
# variables key, but not on the env one, so we need an extra indirection.
|
|
# Note: These Azure variables are only set for PR builds.
|
|
pr.num: $[ variables['System.PullRequest.PullRequestNumber'] ]
|
|
pr.branch: $[ variables['System.PullRequest.SourceBranch'] ]
|
|
steps:
|
|
- template: ci/bash-lib.yml
|
|
parameters:
|
|
var_name: bash_lib
|
|
- bash: |
|
|
set -euo pipefail
|
|
|
|
eval "$(./dev-env/bin/dade-assist)"
|
|
source $(bash_lib)
|
|
|
|
REPORT=$(mktemp)
|
|
cat >$REPORT <<END
|
|
{"jobs": {"Linux": {"start": "$(Linux.start)",
|
|
"machine": "$(Linux.machine)",
|
|
"end": "$(Linux.end)",
|
|
"status": "$(Linux.status)"},
|
|
"macOS": {"start": "$(macOS.start)",
|
|
"machine": "$(macOS.machine)",
|
|
"end": "$(macOS.end)",
|
|
"status": "$(macOS.status)"},
|
|
"Windows": {"start": "$(Windows.start)",
|
|
"machine": "$(Windows.machine)",
|
|
"end": "$(Windows.end)",
|
|
"status": "$(Windows.status)"},
|
|
"release": {"start": "$(release.start)",
|
|
"machine": "$(release.machine)",
|
|
"end": "$(release.end)",
|
|
"status": "$(release.status)"},
|
|
"compatibility_linux": {"start": "$(compatibility_linux.start)",
|
|
"machine": "$(compatibility_linux.machine)",
|
|
"end": "$(compatibility_linux.end)",
|
|
"status": "$(compatibility_linux.status)"},
|
|
"compatibility_macos": {"start": "$(compatibility_macos.start)",
|
|
"machine": "$(compatibility_macos.machine)",
|
|
"end": "$(compatibility_macos.end)",
|
|
"status": "$(compatibility_macos.status)"},
|
|
"compatibility_windows": {"start": "$(compatibility_windows.start)",
|
|
"machine": "$(compatibility_windows.machine)",
|
|
"end": "$(compatibility_windows.end)",
|
|
"status": "$(compatibility_windows.status)"}},
|
|
"id": "$(Build.BuildId)",
|
|
"url": "https://dev.azure.com/digitalasset/daml/_build/results?buildId=$(Build.BuildId)",
|
|
"name": "$(Build.DefinitionName)",
|
|
"version": "$(Build.DefinitionVersion)",
|
|
"queued_by": "$(Build.QueuedBy)",
|
|
"reason": "$(Build.Reason)",
|
|
"branch": "$(Build.SourceBranch)",
|
|
"merge_commit": "$(Build.SourceVersion)",
|
|
"branch_commit": "$(branch_sha)",
|
|
"master_commit": "$(master_sha)",
|
|
"fork_point_commit": "$(fork_sha)",
|
|
"commit_message": $(echo -n "$COMMIT_MSG" | jq -sR),
|
|
"is_fork": "$(System.PullRequest.IsFork)",
|
|
"pr": "$PR_NUM",
|
|
"pr_url": "https://github.com/digital-asset/daml/pull/$PR_NUM",
|
|
"pr_source_branch": "$PR_BRANCH"}
|
|
END
|
|
# Test above JSON is well formed
|
|
cat $REPORT | jq '.'
|
|
REPORT_GZ=$(mktemp)
|
|
cat $REPORT | gzip -9 > $REPORT_GZ
|
|
# Application credentials will not be set for forks. We give up on
|
|
# tracking those for now. "Not set" in Azure world means set to the
|
|
# expression Azure would otherwise substitute, i.e. the literal value
|
|
# of the string in the `env:` block below.
|
|
if [[ "${GCRED:1:${#GCRED}-1}" != '(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)' ]]; then
|
|
gcs "$GCRED" cp "$REPORT_GZ" "gs://daml-data/builds/$(Build.BuildId)_$(date -u +%Y%m%d_%H%M%SZ).json.gz"
|
|
else
|
|
echo "Could not save build data: no credentials."
|
|
fi
|
|
|
|
# Linux, macOS and Windows are always required and should always
|
|
# succeed.
|
|
#
|
|
# release only run on releases and is skipped otherwise.
|
|
if [[ "$(Linux.status)" != "Succeeded"
|
|
|| "$(macOS.status)" != "Succeeded"
|
|
|| "$(Windows.status)" != "Succeeded"
|
|
|| "$(compatibility_linux.status)" != "Succeeded"
|
|
|| "$(compatibility_macos.status)" != "Succeeded"
|
|
|| "$(compatibility_windows.status)" != "Succeeded"
|
|
|| "$(release.status)" == "Canceled" ]]; then
|
|
exit 1
|
|
fi
|
|
env:
|
|
GCRED: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
|
|
# Commit message is always set
|
|
COMMIT_MSG: $(Build.SourceVersionMessage)
|
|
# Because these variables are always set (in the variables block),
|
|
# hopefully these should be set as expected (i.e. either correct
|
|
# value or empty string, but not $(Azure.Variable.Name)).
|
|
PR_NUM: $(pr.num)
|
|
PR_BRANCH: $(pr.branch)
|
|
|
|
- job: notify_release_pr
|
|
condition: and(not(canceled()),
|
|
startsWith(variables['Build.SourceBranchName'], 'auto-release-pr-'),
|
|
eq(dependencies.check_for_release.outputs['out.is_release'], 'true'))
|
|
dependsOn:
|
|
- git_sha
|
|
- collect_build_data
|
|
- check_for_release
|
|
pool:
|
|
name: 'linux-pool'
|
|
demands: assignment -equals default
|
|
variables:
|
|
branch_sha: $[ dependencies.git_sha.outputs['out.branch'] ]
|
|
status: $[ dependencies.collect_build_data.result ]
|
|
is_release: $[ dependencies.check_for_release.outputs['out.is_release'] ]
|
|
steps:
|
|
- checkout: self
|
|
persistCredentials: true
|
|
- template: ci/bash-lib.yml
|
|
parameters:
|
|
var_name: bash-lib
|
|
- bash: |
|
|
set -euo pipefail
|
|
|
|
source $(bash-lib)
|
|
|
|
AUTH="$(get_gh_auth_header)"
|
|
PR=$(curl -H "$AUTH" \
|
|
-H "Accept: application/vnd.github.groot-preview+json" \
|
|
-s -f \
|
|
"https://api.github.com/repos/digital-asset/daml/commits/$(git rev-parse HEAD)/pulls" \
|
|
| jq '.[0].number' \
|
|
|| echo "")
|
|
# Note: if we somehow fail to resolve the PR number from the GitHub
|
|
# API, there is still value in getting the notification on Slack, as
|
|
# we do have the build number and from there we can click through to
|
|
# the PR. Hence the `|| echo ""`.
|
|
PR_HANDLER=$(head -1 release/rotation | awk '{print $1}')
|
|
|
|
case "$(status)" in
|
|
Succeeded*)
|
|
msg="has succeeded! Next step is to approve & merge."
|
|
;;
|
|
Failed*)
|
|
msg="has failed. Please investigate."
|
|
;;
|
|
*)
|
|
# Should not be reached, but who knows?
|
|
msg="has completed with status $(status). See <https://github.com/digital-asset/daml/blob/master/release/RELEASE.md|RELEASE.md> for what to do next."
|
|
;;
|
|
esac
|
|
tell_slack "<@${PR_HANDLER}> <https://dev.azure.com/digitalasset/daml/_build/results?buildId=$(Build.BuildId)|Build $(Build.BuildId)> for release PR <https://github.com/digital-asset/daml/pull/${PR}|#${PR}> $msg"
|
|
|
|
- job: notify_user
|
|
condition: and(eq(variables['Build.Reason'], 'PullRequest'), not(canceled()))
|
|
dependsOn:
|
|
- git_sha
|
|
- collect_build_data
|
|
- check_for_release
|
|
- check_changelog_entry
|
|
pool:
|
|
name: 'linux-pool'
|
|
demands: assignment -equals default
|
|
variables:
|
|
pr.num: $[ variables['System.PullRequest.PullRequestNumber'] ]
|
|
branch_sha: $[ dependencies.git_sha.outputs['out.branch'] ]
|
|
build_status: $[ dependencies.collect_build_data.result ]
|
|
changelog_status: $[ dependencies.check_changelog_entry.result ]
|
|
steps:
|
|
- template: ci/bash-lib.yml
|
|
parameters:
|
|
var_name: bash-lib
|
|
- bash: |
|
|
set -euo pipefail
|
|
source $(bash-lib)
|
|
|
|
user=$(user_slack_handle $(branch_sha))
|
|
if [ "$user" != "" ]; then
|
|
tell_slack "<@${user}> <https://dev.azure.com/digitalasset/daml/_build/results?buildId=$(Build.BuildId)|Build $(Build.BuildId)> for <https://github.com/digital-asset/daml/pull/$(pr.num)|PR $(pr.num)> has completed with status $(build_status) (changelog: $(changelog_status))." \
|
|
"$(Slack.team-daml-ci)"
|
|
fi
|