mirror of
https://github.com/digital-asset/daml.git
synced 2024-11-10 10:46:11 +03:00
1243afc4a1
* Remove reference to release-notes.rst https://github.com/digital-asset/daml/pull/7458 shuffled this around. While we could update it, it doesn’t really make any sense. We post our release notes to the blog now and not in the docs so this whole checkout procedure is redundant. This is also true if we wanted to make a bugfix release for a release < 1.5 where this file still existed. The trigger_sha is always on master (following our current release process) so the file would still not exist. I did also remove it from the docs cronjob. We never reupload old docs so this doesn’t make a difference. changelog_begin changelog_end * Stupid whitespace change because windows is pissing me off changelog_begin changelog_end
891 lines
35 KiB
YAML
891 lines
35 KiB
YAML
# Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
# Azure Pipelines file, see https://aka.ms/yaml
|
|
|
|
# Enable builds on all branches
|
|
trigger:
|
|
# Build every commit as our release process relies on
|
|
# the release process being built alone.
|
|
batch: false
|
|
branches:
|
|
include:
|
|
- master
|
|
- release/*
|
|
|
|
# Enable PR triggers that target the master branch
|
|
pr:
|
|
autoCancel: true # cancel previous builds on push
|
|
branches:
|
|
include:
|
|
- master
|
|
- release/*
|
|
|
|
jobs:
|
|
- job: git_sha
|
|
pool:
|
|
name: 'linux-pool'
|
|
demands: assignment -equals default
|
|
steps:
|
|
- bash: |
|
|
set -euo pipefail
|
|
if [ "$(Build.Reason)" == "PullRequest" ]; then
|
|
echo "##vso[task.setvariable variable=branch;isOutput=true]$(git rev-parse HEAD^2)"
|
|
echo "##vso[task.setvariable variable=master;isOutput=true]$(git rev-parse HEAD^1)"
|
|
echo "##vso[task.setvariable variable=fork_point;isOutput=true]$(git merge-base $(git rev-parse HEAD^1) $(git rev-parse HEAD^2))"
|
|
else
|
|
echo "##vso[task.setvariable variable=branch;isOutput=true]$(git rev-parse HEAD)"
|
|
echo "##vso[task.setvariable variable=master;isOutput=true]$(git rev-parse HEAD^1)"
|
|
echo "##vso[task.setvariable variable=fork_point;isOutput=true]$(git rev-parse HEAD^1)"
|
|
fi
|
|
name: out
|
|
|
|
- job: check_standard_change_label
|
|
dependsOn:
|
|
- git_sha
|
|
variables:
|
|
fork_sha: $[ dependencies.git_sha.outputs['out.fork_point'] ]
|
|
branch_sha: $[ dependencies.git_sha.outputs['out.branch'] ]
|
|
condition: eq(variables['Build.Reason'], 'PullRequest')
|
|
pool:
|
|
name: 'linux-pool'
|
|
demands: assignment -equals default
|
|
steps:
|
|
- checkout: self
|
|
- bash: |
|
|
set -euo pipefail
|
|
|
|
has_changed () {
|
|
git diff $(fork_sha) $(branch_sha) --name-only | grep -q "^$1"
|
|
}
|
|
|
|
fail_if_missing_std_change_label () {
|
|
curl https://api.github.com/repos/digital-asset/daml/pulls/$PR -s | jq -r '.labels[].name' | grep -q '^Standard-Change$'
|
|
}
|
|
|
|
if has_changed "infra/" || has_changed "LATEST"; then
|
|
fail_if_missing_std_change_label
|
|
fi
|
|
env:
|
|
PR: $(System.PullRequest.PullRequestNumber)
|
|
|
|
- job: check_changelog_entry
|
|
dependsOn:
|
|
- git_sha
|
|
variables:
|
|
fork_sha: $[ dependencies.git_sha.outputs['out.fork_point'] ]
|
|
condition: eq(variables['Build.Reason'], 'PullRequest')
|
|
pool:
|
|
name: 'linux-pool'
|
|
demands: assignment -equals default
|
|
steps:
|
|
- checkout: self
|
|
- bash: ci/check-changelog.sh $(fork_sha)
|
|
|
|
- template: ci/da-ghc-lib/compile.yml
|
|
parameters:
|
|
final_job_name: da_ghc_lib
|
|
|
|
- job: Linux
|
|
dependsOn:
|
|
- da_ghc_lib
|
|
- check_for_release
|
|
variables:
|
|
release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
|
|
release_tag: $[ coalesce(dependencies.check_for_release.outputs['out.release_tag'], '0.0.0') ]
|
|
trigger_sha: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ]
|
|
is_release: $[ dependencies.check_for_release.outputs['out.is_release'] ]
|
|
timeoutInMinutes: 360
|
|
pool:
|
|
name: 'linux-pool'
|
|
demands: assignment -equals default
|
|
steps:
|
|
- template: ci/report-start.yml
|
|
- checkout: self
|
|
- bash: |
|
|
set -euo pipefail
|
|
git checkout $(release_sha)
|
|
name: checkout_release
|
|
condition: eq(variables.is_release, 'true')
|
|
- template: ci/build-unix.yml
|
|
parameters:
|
|
release_tag: $(release_tag)
|
|
name: 'linux'
|
|
is_release: variables.is_release
|
|
- bash: |
|
|
set -euo pipefail
|
|
eval "$(./dev-env/bin/dade-assist)"
|
|
bazel build //release:release
|
|
./bazel-bin/release/release --release-dir "$(mktemp -d)"
|
|
condition: and(succeeded(), ne(variables['is_release'], 'true'))
|
|
- template: ci/tell-slack-failed.yml
|
|
parameters:
|
|
trigger_sha: '$(trigger_sha)'
|
|
- template: ci/report-end.yml
|
|
|
|
- job: macOS
|
|
dependsOn:
|
|
- da_ghc_lib
|
|
- check_for_release
|
|
timeoutInMinutes: 360
|
|
pool:
|
|
name: macOS-pool
|
|
variables:
|
|
release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
|
|
release_tag: $[ coalesce(dependencies.check_for_release.outputs['out.release_tag'], '0.0.0') ]
|
|
trigger_sha: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ]
|
|
is_release: $[ dependencies.check_for_release.outputs['out.is_release'] ]
|
|
steps:
|
|
- template: ci/report-start.yml
|
|
- template: ci/clear-shared-segments-macos.yml
|
|
- checkout: self
|
|
- bash: |
|
|
set -euo pipefail
|
|
git checkout $(release_sha)
|
|
name: checkout_release
|
|
condition: eq(variables.is_release, 'true')
|
|
- template: ci/build-unix.yml
|
|
parameters:
|
|
release_tag: $(release_tag)
|
|
name: macos
|
|
is_release: variables.is_release
|
|
- template: ci/tell-slack-failed.yml
|
|
parameters:
|
|
trigger_sha: '$(trigger_sha)'
|
|
- template: ci/report-end.yml
|
|
|
|
- template: ci/patch_bazel_windows/compile.yml
|
|
parameters:
|
|
final_job_name: patch_bazel_windows
|
|
|
|
- job: Windows
|
|
dependsOn:
|
|
- da_ghc_lib
|
|
- check_for_release
|
|
- patch_bazel_windows
|
|
variables:
|
|
release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
|
|
release_tag: $[ coalesce(dependencies.check_for_release.outputs['out.release_tag'], '0.0.0') ]
|
|
trigger_sha: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ]
|
|
is_release: $[ dependencies.check_for_release.outputs['out.is_release'] ]
|
|
timeoutInMinutes: 360
|
|
pool:
|
|
name: 'windows-pool'
|
|
demands: assignment -equals default
|
|
steps:
|
|
- template: ci/report-start.yml
|
|
- checkout: self
|
|
- bash: |
|
|
set -euo pipefail
|
|
git checkout $(release_sha)
|
|
name: checkout_release
|
|
condition: eq(variables.is_release, 'true')
|
|
- template: ci/build-windows.yml
|
|
parameters:
|
|
release_tag: $(release_tag)
|
|
is_release: variables.is_release
|
|
- task: PublishBuildArtifacts@1
|
|
condition: succeededOrFailed()
|
|
inputs:
|
|
pathtoPublish: '$(Build.StagingDirectory)'
|
|
artifactName: 'Bazel Logs'
|
|
- template: ci/tell-slack-failed.yml
|
|
parameters:
|
|
trigger_sha: '$(trigger_sha)'
|
|
- template: ci/report-end.yml
|
|
|
|
- job: compatibility_ts_libs
|
|
dependsOn:
|
|
- da_ghc_lib
|
|
- check_for_release
|
|
timeoutInMinutes: 360
|
|
pool:
|
|
name: linux-pool
|
|
demands: assignment -equals default
|
|
steps:
|
|
- template: ci/report-start.yml
|
|
- checkout: self
|
|
- template: ci/compatibility_ts_libs.yml
|
|
- template: ci/tell-slack-failed.yml
|
|
- template: ci/report-end.yml
|
|
|
|
- job: compatibility_linux
|
|
dependsOn:
|
|
- da_ghc_lib
|
|
- check_for_release
|
|
- compatibility_ts_libs
|
|
timeoutInMinutes: 360
|
|
pool:
|
|
name: linux-pool
|
|
demands: assignment -equals default
|
|
steps:
|
|
- template: ci/report-start.yml
|
|
- checkout: self
|
|
- template: ci/compatibility.yml
|
|
parameters:
|
|
test_flags: '--quick'
|
|
- template: ci/tell-slack-failed.yml
|
|
- template: ci/report-end.yml
|
|
|
|
- job: compatibility_macos
|
|
dependsOn:
|
|
- da_ghc_lib
|
|
- check_for_release
|
|
- compatibility_ts_libs
|
|
timeoutInMinutes: 360
|
|
pool:
|
|
name: macOS-pool
|
|
steps:
|
|
- template: ci/report-start.yml
|
|
- template: ci/clear-shared-segments-macos.yml
|
|
- checkout: self
|
|
- template: ci/compatibility.yml
|
|
parameters:
|
|
test_flags: '--quick'
|
|
- template: ci/tell-slack-failed.yml
|
|
- template: ci/report-end.yml
|
|
|
|
- job: compatibility_windows
|
|
dependsOn:
|
|
- da_ghc_lib
|
|
- check_for_release
|
|
- compatibility_ts_libs
|
|
- patch_bazel_windows
|
|
timeoutInMinutes: 360
|
|
pool:
|
|
name: 'windows-pool'
|
|
demands: assignment -equals default
|
|
steps:
|
|
- template: ci/report-start.yml
|
|
- checkout: self
|
|
- template: ci/compatibility-windows.yml
|
|
parameters:
|
|
test_flags: '--quick'
|
|
- template: ci/tell-slack-failed.yml
|
|
- template: ci/report-end.yml
|
|
- task: PublishBuildArtifacts@1
|
|
condition: succeededOrFailed()
|
|
inputs:
|
|
pathtoPublish: '$(Build.StagingDirectory)'
|
|
artifactName: 'Bazel Compatibility Logs'
|
|
|
|
- job: check_for_release
|
|
dependsOn:
|
|
- git_sha
|
|
variables:
|
|
branch_sha: $[ dependencies.git_sha.outputs['out.branch'] ]
|
|
fork_sha: $[ dependencies.git_sha.outputs['out.fork_point'] ]
|
|
pool:
|
|
name: "linux-pool"
|
|
demands: assignment -equals default
|
|
steps:
|
|
- bash: |
|
|
set -euo pipefail
|
|
|
|
./release.sh check
|
|
|
|
changes_release_files() {
|
|
changed="$(git diff-tree --no-commit-id --name-only -r $(fork_sha) $(branch_sha) | sort)"
|
|
[ "LATEST" = "$changed" ]
|
|
}
|
|
|
|
changes_one_line_in_latest() {
|
|
changed="$(git diff-tree --no-commit-id --numstat -r $(fork_sha) $(branch_sha) -- LATEST | awk '{print $1 "_" $2}')"
|
|
add_one="1_0"
|
|
change_one="1_1"
|
|
[[ "$add_one" == "$changed" || "$change_one" == "$changed" ]]
|
|
}
|
|
|
|
setvar() {
|
|
echo "Setting '$1' to '$2'"
|
|
echo "##vso[task.setvariable variable=$1;isOutput=true]$2"
|
|
}
|
|
|
|
added_line() {
|
|
echo "$(git diff $(fork_sha) $(branch_sha) -- LATEST | tail -n+6 | grep '^\+' | cut -c2-)"
|
|
}
|
|
|
|
if changes_release_files; then
|
|
if changes_one_line_in_latest; then
|
|
setvar is_release true
|
|
setvar trigger_sha $(branch_sha)
|
|
setvar release_sha "$(added_line | awk '{print $1}')"
|
|
setvar release_tag "$(added_line | awk '{print $2}')"
|
|
else
|
|
echo "Release commit should only add one version."
|
|
exit 1
|
|
fi
|
|
else
|
|
setvar is_release false
|
|
fi
|
|
name: out
|
|
|
|
- job: check_perf_test
|
|
pool:
|
|
name: linux-pool
|
|
demands: assignment -equals default
|
|
condition: eq(variables['Build.Reason'], 'IndividualCI')
|
|
steps:
|
|
- bash: |
|
|
TEST_SHA=$(cat ci/cron/perf/test_sha)
|
|
LAST_CHANGES=$(git log -n1 --format=%H daml-lf/scenario-interpreter/src/perf)
|
|
CURRENT_SHA=$(git rev-parse HEAD)
|
|
if [ "$TEST_SHA" != "$LAST_CHANGES" ]; then
|
|
if [ "$LAST_CHANGES" = "$CURRENT_SHA" ]; then
|
|
curl -XPOST \
|
|
-i \
|
|
-H 'Content-Type: application/json' \
|
|
--data "{\"text\":\"<!here> Perf tests seem to have changed. Please manually check:\n\`\`\`\ngit diff $TEST_SHA $LAST_CHANGES -- daml-lf/scenario-interpreter/src/perf\n\`\`\`\nand update accordingly. If the change is benign, update \`ci/cron/perf/test_sha\` to \`$LAST_CHANGES\`. With no intervention, you will no longer get performance reports.\"}" \
|
|
$(Slack.team-daml)
|
|
else
|
|
echo "Changes detected, but not from this commit."
|
|
fi
|
|
else
|
|
echo "No change detected."
|
|
fi
|
|
displayName: check perf changes
|
|
|
|
- job: release
|
|
dependsOn: [ "check_for_release", "Linux", "macOS", "Windows" ]
|
|
condition: and(succeeded(),
|
|
eq(dependencies.check_for_release.outputs['out.is_release'], 'true'),
|
|
eq(variables['Build.SourceBranchName'], 'master'))
|
|
pool:
|
|
vmImage: "Ubuntu-16.04"
|
|
variables:
|
|
linux-tarball: $[ dependencies.Linux.outputs['publish.tarball'] ]
|
|
macos-tarball: $[ dependencies.macOS.outputs['publish.tarball'] ]
|
|
windows-tarball: $[ dependencies.Windows.outputs['publish.tarball'] ]
|
|
windows-installer: $[ dependencies.Windows.outputs['publish.installer'] ]
|
|
protos-zip: $[ dependencies.Linux.outputs['publish.protos-zip'] ]
|
|
daml-on-sql: $[ dependencies.Linux.outputs['publish.daml-on-sql'] ]
|
|
json-api: $[ dependencies.Linux.outputs['publish.json-api'] ]
|
|
script-runner: $[ dependencies.Linux.outputs['publish.script-runner'] ]
|
|
trigger-runner: $[ dependencies.Linux.outputs['publish.trigger-runner'] ]
|
|
release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
|
|
release_tag: $[ dependencies.check_for_release.outputs['out.release_tag'] ]
|
|
trigger_sha: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ]
|
|
steps:
|
|
- template: ci/report-start.yml
|
|
- checkout: self
|
|
persistCredentials: true
|
|
- bash: |
|
|
set -euo pipefail
|
|
if git tag v$(release_tag) $(release_sha); then
|
|
git push origin v$(release_tag)
|
|
mkdir $(Build.StagingDirectory)/release
|
|
mkdir $(Build.StagingDirectory)/artifactory
|
|
else
|
|
echo "##vso[task.setvariable variable=skip-github]TRUE"
|
|
fi
|
|
- task: DownloadPipelineArtifact@0
|
|
inputs:
|
|
artifactName: $(linux-tarball)
|
|
targetPath: $(Build.StagingDirectory)/release
|
|
condition: not(eq(variables['skip-github'], 'TRUE'))
|
|
- task: DownloadPipelineArtifact@0
|
|
inputs:
|
|
artifactName: $(macos-tarball)
|
|
targetPath: $(Build.StagingDirectory)/release
|
|
condition: not(eq(variables['skip-github'], 'TRUE'))
|
|
- task: DownloadPipelineArtifact@0
|
|
inputs:
|
|
artifactName: $(windows-tarball)
|
|
targetPath: $(Build.StagingDirectory)/release
|
|
condition: not(eq(variables['skip-github'], 'TRUE'))
|
|
- task: DownloadPipelineArtifact@0
|
|
inputs:
|
|
artifactName: $(windows-installer)
|
|
targetPath: $(Build.StagingDirectory)/release
|
|
condition: not(eq(variables['skip-github'], 'TRUE'))
|
|
- task: DownloadPipelineArtifact@0
|
|
inputs:
|
|
artifactName: $(protos-zip)
|
|
targetPath: $(Build.StagingDirectory)/release
|
|
condition: not(eq(variables['skip-github'], 'TRUE'))
|
|
- task: DownloadPipelineArtifact@0
|
|
inputs:
|
|
artifactName: $(daml-on-sql)
|
|
targetPath: $(Build.StagingDirectory)/release
|
|
condition: not(eq(variables['skip-github'], 'TRUE'))
|
|
- task: DownloadPipelineArtifact@0
|
|
inputs:
|
|
artifactName: $(json-api)
|
|
targetPath: $(Build.StagingDirectory)/release
|
|
condition: not(eq(variables['skip-github'], 'TRUE'))
|
|
- task: DownloadPipelineArtifact@0
|
|
inputs:
|
|
artifactName: $(trigger-runner)
|
|
targetPath: $(Build.StagingDirectory)/artifactory
|
|
condition: not(eq(variables['skip-github'], 'TRUE'))
|
|
- task: DownloadPipelineArtifact@0
|
|
inputs:
|
|
artifactName: $(script-runner)
|
|
targetPath: $(Build.StagingDirectory)/artifactory
|
|
condition: not(eq(variables['skip-github'], 'TRUE'))
|
|
- bash: |
|
|
set -euo pipefail
|
|
KEY_FILE=$(mktemp)
|
|
GPG_DIR=$(mktemp -d)
|
|
cleanup() {
|
|
rm -rf $KEY_FILE $GPG_DIR
|
|
}
|
|
trap cleanup EXIT
|
|
echo "$GPG_KEY" | base64 -d > $KEY_FILE
|
|
gpg --homedir $GPG_DIR --no-tty --quiet --import $KEY_FILE
|
|
cd $(Build.StagingDirectory)/release
|
|
sha256sum $(find . -type f | sort) > sha256sums
|
|
# Note: relies on our release artifacts not having spaces in their
|
|
# names. Creates a ${f}.asc with the signature for each $f.
|
|
for f in *; do
|
|
gpg --homedir $GPG_DIR -ab $f
|
|
done
|
|
cd ../artifactory
|
|
for f in *; do
|
|
gpg --homedir $GPG_DIR -ab $f
|
|
done
|
|
env:
|
|
GPG_KEY: $(gpg-code-signing)
|
|
- task: GitHubRelease@0
|
|
inputs:
|
|
gitHubConnection: 'garyverhaegen-da'
|
|
repositoryName: '$(Build.Repository.Name)'
|
|
action: 'create'
|
|
target: '$(release_sha)'
|
|
tagSource: 'manual'
|
|
tag: 'v$(release_tag)'
|
|
assets: $(Build.StagingDirectory)/release/*
|
|
assetUploadMode: 'replace'
|
|
title: '$(release_tag)'
|
|
addChangeLog: false
|
|
isPrerelease: true
|
|
releaseNotesSource: 'input'
|
|
releaseNotes: "This is a pre-release. Use at your own risk."
|
|
condition: not(eq(variables['skip-github'], 'TRUE'))
|
|
- bash: |
|
|
set -euo pipefail
|
|
|
|
GCS_KEY=$(mktemp)
|
|
cleanup () {
|
|
rm -f $GCS_KEY
|
|
}
|
|
trap cleanup EXIT
|
|
echo "$GOOGLE_APPLICATION_CREDENTIALS_CONTENT" > $GCS_KEY
|
|
gcloud auth activate-service-account --key-file=$GCS_KEY
|
|
export BOTO_CONFIG=/dev/null
|
|
|
|
gsutil cp $(Build.StagingDirectory)/release/$(daml-on-sql) \
|
|
gs://daml-binaries/daml-on-sql/$(daml-on-sql)
|
|
gsutil cp $(Build.StagingDirectory)/release/$(daml-on-sql).asc \
|
|
gs://daml-binaries/daml-on-sql/$(daml-on-sql).asc
|
|
gsutil cp $(Build.StagingDirectory)/release/$(json-api) \
|
|
gs://daml-binaries/json-api/$(json-api)
|
|
gsutil cp $(Build.StagingDirectory)/release/$(json-api).asc \
|
|
gs://daml-binaries/json-api/$(json-api).asc
|
|
env:
|
|
GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
|
|
condition: not(eq(variables['skip-github'], 'TRUE'))
|
|
- bash: |
|
|
set -euo pipefail
|
|
|
|
push() {
|
|
local file repository
|
|
repository=$1
|
|
file=$2
|
|
curl -f \
|
|
-u "$AUTH" \
|
|
-H "X-Checksum-MD5:$(md5sum ${local} | awk '{print $1}')" \
|
|
-H "X-Checksum-SHA1:$(sha1sum ${local} | awk '{print $1}')" \
|
|
-X PUT \
|
|
-T $(Build.StagingDirectory)/artifactory/${file} \
|
|
https://digitalasset.jfrog.io/artifactory/${repository}/$(release_tag)/${file}
|
|
}
|
|
|
|
push daml-trigger-runner $(trigger-runner)
|
|
push daml-trigger-runner $(trigger-runner).asc
|
|
push daml-script-runner $(script-runner)
|
|
push daml-script-runner $(script-runner).asc
|
|
env:
|
|
AUTH: $(ARTIFACTORY_USERPASS)
|
|
condition: not(eq(variables['skip-github'], 'TRUE'))
|
|
- bash: |
|
|
set -euo pipefail
|
|
pr_handler=$(head -1 release/rotation | awk '{print $1}')
|
|
curl -XPOST \
|
|
-i \
|
|
-H 'Content-Type: application/json' \
|
|
--data "{\"text\":\"<@${pr_handler}> Release \`$(release_tag)\` is ready for testing. (<https://dev.azure.com/digitalasset/daml/_build/results?buildId=$(Build.BuildId)|build>, <https://github.com/digital-asset/daml/commit/$(trigger_sha)|trigger commit>, <https://github.com/digital-asset/daml/commit/$(release_sha)|target commit>)\"}" \
|
|
$(Slack.team-daml)
|
|
- template: ci/tell-slack-failed.yml
|
|
parameters:
|
|
trigger_sha: '$(trigger_sha)'
|
|
- template: ci/report-end.yml
|
|
|
|
- job: compat_versions_pr
|
|
dependsOn:
|
|
- release
|
|
- check_for_release
|
|
pool:
|
|
name: linux-pool
|
|
demands: assignment -equals default
|
|
variables:
|
|
release_tag: $[ dependencies.check_for_release.outputs['out.release_tag'] ]
|
|
steps:
|
|
- checkout: self
|
|
persistCredentials: true
|
|
- bash: ci/dev-env-install.sh
|
|
- template: ci/bash-lib.yml
|
|
parameters:
|
|
var_name: bash_lib
|
|
- bash: |
|
|
set -euo pipefail
|
|
eval "$(./dev-env/bin/dade-assist)"
|
|
|
|
source $(bash_lib)
|
|
|
|
setvar() {
|
|
echo "Setting '$1' to '$2'"
|
|
echo "##vso[task.setvariable variable=$1;isOutput=true]$2"
|
|
}
|
|
|
|
DELAY=1
|
|
while ! curl --fail -I https://repo1.maven.org/maven2/com/daml/ledger-api-test-tool/$(release_tag)/ledger-api-test-tool-$(release_tag).jar; do
|
|
sleep $DELAY
|
|
DELAY=$(( DELAY * 2 ))
|
|
if (( $DELAY > 2000 )); then
|
|
echo "Too many attempts waiting for Maven."
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
AUTH="$(get_gh_auth_header)"
|
|
|
|
git checkout origin/master
|
|
BRANCH=update-compat-versions-for-$(release_tag)
|
|
# if this is a rerun, branch might already exist
|
|
git branch -D $BRANCH || true
|
|
git checkout -b $BRANCH
|
|
cp .bazelrc compatibility/
|
|
compatibility/update-versions.sh
|
|
# Note that maven_install.json might be unchanged but
|
|
# git add is happy about that.
|
|
git add compatibility/versions.bzl compatibility/maven_install.json
|
|
git -c user.name="Azure Pipelines DAML Build" \
|
|
-c user.email="support@digitalasset.com" \
|
|
commit \
|
|
-m "$(printf "update compat versions for $(release_tag)\n\nCHANGELOG_BEGIN\nCHANGELOG_END\n")"
|
|
git push origin $BRANCH:$BRANCH
|
|
curl -H "Content-Type: application/json" \
|
|
-H "$AUTH" \
|
|
--silent \
|
|
--fail \
|
|
--include \
|
|
--location \
|
|
-d "{\"title\": \"update compat versions for $(release_tag)\", \"head\": \"$BRANCH\", \"base\": \"master\"}" \
|
|
https://api.github.com/repos/digital-asset/daml/pulls
|
|
setvar "branch" "$BRANCH"
|
|
name: out
|
|
|
|
- template: ci/tell-slack-failed.yml
|
|
parameters:
|
|
trigger_sha: '$(trigger_sha)'
|
|
|
|
- job: compat_versions_pr_trigger_daily
|
|
dependsOn: compat_versions_pr
|
|
pool:
|
|
vmImage: "ubuntu-latest"
|
|
variables:
|
|
branch: $[ dependencies.compat_versions_pr.outputs['out.branch'] ]
|
|
steps:
|
|
- checkout: none
|
|
- bash: |
|
|
set -euo pipefail
|
|
az extension add --name azure-devops
|
|
echo "$(System.AccessToken)" | az devops login --org "https://dev.azure.com/digitalasset"
|
|
az pipelines build queue --branch $(branch) --definition-name "digital-asset.daml-daily-compat" --org "https://dev.azure.com/digitalasset" --project daml
|
|
az pipelines build queue --branch $(branch) --definition-name "digital-asset.daml" --org "https://dev.azure.com/digitalasset" --project daml
|
|
|
|
- job: write_ledger_dump
|
|
dependsOn:
|
|
- check_for_release
|
|
- da_ghc_lib
|
|
pool:
|
|
vmImage: "Ubuntu-16.04"
|
|
condition: and(eq(dependencies.check_for_release.outputs['out.is_release'], 'true'),
|
|
eq(variables['Build.SourceBranchName'], 'master'))
|
|
variables:
|
|
release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
|
|
release_tag: $[ dependencies.check_for_release.outputs['out.release_tag'] ]
|
|
trigger_sha: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ]
|
|
steps:
|
|
- checkout: self
|
|
- bash: |
|
|
set -euo pipefail
|
|
|
|
git checkout $(release_sha)
|
|
export DAML_SDK_RELEASE_VERSION=$(release_tag)
|
|
|
|
sudo mkdir -p /nix
|
|
sudo chown $USER /nix
|
|
curl -sSfL https://nixos.org/releases/nix/nix-2.3.3/install | bash
|
|
eval "$(dev-env/bin/dade-assist)"
|
|
GCS_KEY=$(mktemp)
|
|
cleanup () {
|
|
rm -f $GCS_KEY
|
|
}
|
|
trap cleanup EXIT
|
|
echo "$GOOGLE_APPLICATION_CREDENTIALS_CONTENT" > $GCS_KEY
|
|
gcloud auth activate-service-account --key-file=$GCS_KEY
|
|
export BOTO_CONFIG=/dev/null
|
|
|
|
bazel build //ledger/participant-state/kvutils:reference-ledger-dump
|
|
gsutil cp bazel-bin/ledger/participant-state/kvutils/reference-ledger-dump.out \
|
|
gs://daml-dumps/release/ledger/api-server-damlonx/reference-v2/reference-ledger-dump-$(release_tag)
|
|
env:
|
|
GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
|
|
- template: ci/tell-slack-failed.yml
|
|
parameters:
|
|
trigger_sha: '$(trigger_sha)'
|
|
|
|
- job: collect_build_data
|
|
condition: always()
|
|
dependsOn:
|
|
- Linux
|
|
- macOS
|
|
- Windows
|
|
- release
|
|
- write_ledger_dump
|
|
- git_sha
|
|
- compatibility_macos
|
|
- compatibility_linux
|
|
- compatibility_windows
|
|
pool:
|
|
name: "linux-pool"
|
|
demands: assignment -equals default
|
|
variables:
|
|
Linux.start: $[ dependencies.Linux.outputs['start.time'] ]
|
|
Linux.machine: $[ dependencies.Linux.outputs['start.machine'] ]
|
|
Linux.end: $[ dependencies.Linux.outputs['end.time'] ]
|
|
Linux.status: $[ dependencies.Linux.result ]
|
|
macOS.start: $[ dependencies.macOS.outputs['start.time'] ]
|
|
macOS.machine: $[ dependencies.macOS.outputs['start.machine'] ]
|
|
macOS.end: $[ dependencies.macOS.outputs['end.time'] ]
|
|
macOS.status: $[ dependencies.macOS.result ]
|
|
Windows.start: $[ dependencies.Windows.outputs['start.time'] ]
|
|
Windows.machine: $[ dependencies.Windows.outputs['start.machine'] ]
|
|
Windows.end: $[ dependencies.Windows.outputs['end.time'] ]
|
|
Windows.status: $[ dependencies.Windows.result ]
|
|
release.start: $[ dependencies.release.outputs['start.time'] ]
|
|
release.machine: $[ dependencies.release.outputs['start.machine'] ]
|
|
release.end: $[ dependencies.release.outputs['end.time'] ]
|
|
release.status: $[ dependencies.release.result ]
|
|
dump.start: $[ dependencies.write_ledger_dump.outputs['start.time'] ]
|
|
dump.machine: $[ dependencies.write_ledger_dump.outputs['start.machine'] ]
|
|
dump.end: $[ dependencies.write_ledger_dump.outputs['end.time'] ]
|
|
dump.status: $[ dependencies.write_ledger_dump.result ]
|
|
compatibility_linux.start: $[ dependencies.compatibility_linux.outputs['start.time'] ]
|
|
compatibility_linux.machine: $[ dependencies.compatibility_linux.outputs['start.machine'] ]
|
|
compatibility_linux.end: $[ dependencies.compatibility_linux.outputs['end.time'] ]
|
|
compatibility_linux.status: $[ dependencies.compatibility_linux.result ]
|
|
compatibility_macos.start: $[ dependencies.compatibility_macos.outputs['start.time'] ]
|
|
compatibility_macos.machine: $[ dependencies.compatibility_macos.outputs['start.machine'] ]
|
|
compatibility_macos.end: $[ dependencies.compatibility_macos.outputs['end.time'] ]
|
|
compatibility_macos.status: $[ dependencies.compatibility_macos.result ]
|
|
compatibility_windows.start: $[ dependencies.compatibility_windows.outputs['start.time'] ]
|
|
compatibility_windows.machine: $[ dependencies.compatibility_windows.outputs['start.machine'] ]
|
|
compatibility_windows.end: $[ dependencies.compatibility_windows.outputs['end.time'] ]
|
|
compatibility_windows.status: $[ dependencies.compatibility_windows.result ]
|
|
|
|
branch_sha: $[ dependencies.git_sha.outputs['out.branch'] ]
|
|
master_sha: $[ dependencies.git_sha.outputs['out.master'] ]
|
|
fork_sha: $[ dependencies.git_sha.outputs['out.fork_point'] ]
|
|
|
|
# Using expression syntax so we get an empty string if not set, rather
|
|
# than the raw $(VarName) string. Expression syntax works on the
|
|
# variables key, but not on the env one, so we need an extra indirection.
|
|
# Note: These Azure variables are only set for PR builds.
|
|
pr.num: $[ variables['System.PullRequest.PullRequestNumber'] ]
|
|
pr.branch: $[ variables['System.PullRequest.SourceBranch'] ]
|
|
steps:
|
|
- template: ci/bash-lib.yml
|
|
parameters:
|
|
var_name: bash_lib
|
|
- bash: |
|
|
set -euo pipefail
|
|
|
|
eval "$(./dev-env/bin/dade-assist)"
|
|
source $(bash_lib)
|
|
|
|
REPORT=$(mktemp)
|
|
cat >$REPORT <<END
|
|
{"jobs": {"Linux": {"start": "$(Linux.start)",
|
|
"machine": "$(Linux.machine)",
|
|
"end": "$(Linux.end)",
|
|
"status": "$(Linux.status)"},
|
|
"macOS": {"start": "$(macOS.start)",
|
|
"machine": "$(macOS.machine)",
|
|
"end": "$(macOS.end)",
|
|
"status": "$(macOS.status)"},
|
|
"Windows": {"start": "$(Windows.start)",
|
|
"machine": "$(Windows.machine)",
|
|
"end": "$(Windows.end)",
|
|
"status": "$(Windows.status)"},
|
|
"write_ledger_dump": {"start": "$(dump.start)",
|
|
"machine": "$(dump.machine)",
|
|
"end": "$(dump.end)",
|
|
"status": "$(dump.status)"},
|
|
"release": {"start": "$(release.start)",
|
|
"machine": "$(release.machine)",
|
|
"end": "$(release.end)",
|
|
"status": "$(release.status)"},
|
|
"compatibility_linux": {"start": "$(compatibility_linux.start)",
|
|
"machine": "$(compatibility_linux.machine)",
|
|
"end": "$(compatibility_linux.end)",
|
|
"status": "$(compatibility_linux.status)"},
|
|
"compatibility_macos": {"start": "$(compatibility_macos.start)",
|
|
"machine": "$(compatibility_macos.machine)",
|
|
"end": "$(compatibility_macos.end)",
|
|
"status": "$(compatibility_macos.status)"},
|
|
"compatibility_windows": {"start": "$(compatibility_windows.start)",
|
|
"machine": "$(compatibility_windows.machine)",
|
|
"end": "$(compatibility_windows.end)",
|
|
"status": "$(compatibility_windows.status)"}},
|
|
"id": "$(Build.BuildId)",
|
|
"url": "https://dev.azure.com/digitalasset/daml/_build/results?buildId=$(Build.BuildId)",
|
|
"name": "$(Build.DefinitionName)",
|
|
"version": "$(Build.DefinitionVersion)",
|
|
"queued_by": "$(Build.QueuedBy)",
|
|
"reason": "$(Build.Reason)",
|
|
"branch": "$(Build.SourceBranch)",
|
|
"merge_commit": "$(Build.SourceVersion)",
|
|
"branch_commit": "$(branch_sha)",
|
|
"master_commit": "$(master_sha)",
|
|
"fork_point_commit": "$(fork_sha)",
|
|
"commit_message": $(echo -n "$COMMIT_MSG" | jq -sR),
|
|
"is_fork": "$(System.PullRequest.IsFork)",
|
|
"pr": "$PR_NUM",
|
|
"pr_url": "https://github.com/digital-asset/daml/pull/$PR_NUM",
|
|
"pr_source_branch": "$PR_BRANCH"}
|
|
END
|
|
# Test above JSON is well formed
|
|
cat $REPORT | jq '.'
|
|
REPORT_GZ=$(mktemp)
|
|
cat $REPORT | gzip -9 > $REPORT_GZ
|
|
# Application credentials will not be set for forks. We give up on
|
|
# tracking those for now. "Not set" in Azure world means set to the
|
|
# expression Azure would otherwise substitute, i.e. the literal value
|
|
# of the string in the `env:` block below.
|
|
if [[ "${GCRED:1:${#GCRED}-1}" != '(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)' ]]; then
|
|
save_gcp_data "$GCRED" "$REPORT_GZ" "gs://daml-data/builds/$(Build.BuildId)_$(date -u +%Y%m%d_%H%M%SZ).json.gz"
|
|
else
|
|
echo "Could not save build data: no credentials."
|
|
fi
|
|
|
|
# Linux, macOS and Windows are always required and should always
|
|
# succeed.
|
|
#
|
|
# release and write_ledger_dump only run on releases and are skipped
|
|
# otherwise.
|
|
if [[ "$(Linux.status)" != "Succeeded"
|
|
|| "$(macOS.status)" != "Succeeded"
|
|
|| "$(Windows.status)" != "Succeeded"
|
|
|| "$(compatibility_linux.status)" != "Succeeded"
|
|
|| "$(compatibility_macos.status)" != "Succeeded"
|
|
|| "$(compatibility_windows.status)" != "Succeeded"
|
|
|| "$(dump.status)" == "Canceled"
|
|
|| "$(release.status)" == "Canceled" ]]; then
|
|
exit 1
|
|
fi
|
|
env:
|
|
GCRED: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
|
|
# Commit message is always set
|
|
COMMIT_MSG: $(Build.SourceVersionMessage)
|
|
# Because these variables are always set (in the variables block),
|
|
# hopefully these should be set as expected (i.e. either correct
|
|
# value or empty string, but not $(Azure.Variable.Name)).
|
|
PR_NUM: $(pr.num)
|
|
PR_BRANCH: $(pr.branch)
|
|
|
|
- job: notify_release_pr
|
|
condition: and(not(canceled()),
|
|
startsWith(variables['Build.SourceBranchName'], 'auto-release-pr-'),
|
|
eq(dependencies.check_for_release.outputs['out.is_release'], 'true'))
|
|
dependsOn:
|
|
- git_sha
|
|
- collect_build_data
|
|
- check_for_release
|
|
pool:
|
|
name: 'linux-pool'
|
|
demands: assignment -equals default
|
|
variables:
|
|
branch_sha: $[ dependencies.git_sha.outputs['out.branch'] ]
|
|
status: $[ dependencies.collect_build_data.result ]
|
|
is_release: $[ dependencies.check_for_release.outputs['out.is_release'] ]
|
|
steps:
|
|
- checkout: self
|
|
persistCredentials: true
|
|
- template: ci/bash-lib.yml
|
|
parameters:
|
|
var_name: bash-lib
|
|
- bash: |
|
|
set -euo pipefail
|
|
|
|
source $(bash-lib)
|
|
|
|
AUTH="$(get_gh_auth_header)"
|
|
PR=$(curl -H "$AUTH" \
|
|
-H "Accept: application/vnd.github.groot-preview+json" \
|
|
-s -f \
|
|
"https://api.github.com/repos/digital-asset/daml/commits/$(git rev-parse HEAD)/pulls" \
|
|
| jq '.[0].number' \
|
|
|| echo "")
|
|
# Note: if we somehow fail to resolve the PR number from the GitHub
|
|
# API, there is still value in getting the notification on Slack, as
|
|
# we do have the build number and from there we can click through to
|
|
# the PR. Hence the `|| echo ""`.
|
|
PR_HANDLER=$(head -1 release/rotation | awk '{print $1}')
|
|
|
|
tell_slack "<@${PR_HANDLER}> <https://dev.azure.com/digitalasset/daml/_build/results?buildId=$(Build.BuildId)|Build $(Build.BuildId)> for release PR <https://github.com/digital-asset/daml/pull/${PR}|#${PR}> has completed with status $(status). See <https://github.com/digital-asset/daml/blob/master/release/RELEASE.md|RELEASE.md> for what to do next."
|
|
|
|
- job: notify_user
|
|
condition: and(eq(variables['Build.Reason'], 'PullRequest'), not(canceled()))
|
|
dependsOn:
|
|
- git_sha
|
|
- collect_build_data
|
|
- check_for_release
|
|
pool:
|
|
name: 'linux-pool'
|
|
demands: assignment -equals default
|
|
variables:
|
|
pr.num: $[ variables['System.PullRequest.PullRequestNumber'] ]
|
|
branch_sha: $[ dependencies.git_sha.outputs['out.branch'] ]
|
|
status: $[ dependencies.collect_build_data.result ]
|
|
steps:
|
|
- bash: |
|
|
set -euo pipefail
|
|
|
|
tell_slack() {
|
|
local MESSAGE=$1
|
|
local USER_ID=$2
|
|
curl -XPOST \
|
|
-i \
|
|
-H 'Content-Type: application/json' \
|
|
--data "{\"text\":\"<@${USER_ID}> <https://dev.azure.com/digitalasset/daml/_build/results?buildId=$(Build.BuildId)|Build $(Build.BuildId)> for <https://github.com/digital-asset/daml/pull/$(pr.num)|PR $(pr.num)> has completed with status ${MESSAGE}.\"}" \
|
|
$(Slack.team-daml-ci)
|
|
}
|
|
|
|
EMAIL=$(git log -n 1 --format=%ae $(branch_sha))
|
|
user_registered() {
|
|
cat ci/slack_user_ids | grep $EMAIL
|
|
}
|
|
|
|
user_id() {
|
|
echo $(cat ci/slack_user_ids | grep $EMAIL | awk '{print $2}')
|
|
}
|
|
|
|
if user_registered; then
|
|
tell_slack "$(status)" "$(user_id)"
|
|
else
|
|
echo "User $(user_id) did not opt in for notifications."
|
|
fi
|