daml/ci/build.yml
Gary Verhaegen b0c0306696
fix release: skipping compat should not fail (#9227)
In #9169, I changed the compat jobs to not run on releases.
Unfortunately I forgot to update the `collect_build_data` job to know
about that. Hopefully after this has been merged we'll be able to rerun
\#9221.

CHANGELOG_BEGIN
CHANGELOG_END
2021-03-24 11:18:58 +00:00

548 lines
21 KiB
YAML
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

# Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
jobs:
- job: git_sha
pool:
name: 'ubuntu_20_04'
demands: assignment -equals default
steps:
- template: bash-lib.yml
parameters:
var_name: bash-lib
- bash: |
set -euo pipefail
source $(bash-lib)
if [ "$(Build.Reason)" == "PullRequest" ]; then
setvar branch "$(git rev-parse HEAD^2)"
setvar main "$(git rev-parse HEAD^1)"
setvar fork_point "$(git merge-base $(git rev-parse HEAD^1) $(git rev-parse HEAD^2))"
else
setvar branch "$(git rev-parse HEAD)"
setvar main "$(git rev-parse HEAD^1)"
setvar fork_point "$(git rev-parse HEAD^1)"
fi
name: out
- template: da-ghc-lib/compile.yml
parameters:
final_job_name: da_ghc_lib
- job: Linux
dependsOn:
- da_ghc_lib
- check_for_release
variables:
release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
release_tag: $[ coalesce(dependencies.check_for_release.outputs['out.release_tag'], '0.0.0') ]
trigger_sha: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ]
is_release: $[ dependencies.check_for_release.outputs['out.is_release'] ]
timeoutInMinutes: 360
pool:
name: 'ubuntu_20_04'
demands: assignment -equals default
steps:
- template: report-start.yml
- checkout: self
- bash: |
set -euo pipefail
git checkout $(release_sha)
name: checkout_release
condition: and(succeeded(), eq(variables.is_release, 'true'))
- template: clean-up.yml
- template: build-unix.yml
parameters:
release_tag: $(release_tag)
name: 'linux'
is_release: variables.is_release
- bash: |
set -euo pipefail
eval "$(./dev-env/bin/dade-assist)"
bazel build //release:release
./bazel-bin/release/release --release-dir "$(mktemp -d)"
condition: and(succeeded(), ne(variables['is_release'], 'true'))
- task: PublishBuildArtifacts@1
inputs:
pathtoPublish: 'bazel-bin/docs/html.tar.gz'
artifactName: 'Docs bundle'
- template: tell-slack-failed.yml
parameters:
trigger_sha: '$(trigger_sha)'
- template: report-end.yml
- job: Linux_scala_2_13
dependsOn:
- da_ghc_lib
- check_for_release
variables:
release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
release_tag: $[ coalesce(dependencies.check_for_release.outputs['out.release_tag'], '0.0.0') ]
trigger_sha: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ]
# We still need to be able to run builds for releases that dont yet support Scala 2.13
# The easiest option seems to be to still run the Scala 2.13 job but
# pretend its not a release build and thereby just run it against main
# so thats what we do here.
is_release: $[ and(eq(dependencies.check_for_release.outputs['out.is_release'], 'true'), eq(dependencies.check_for_release.outputs['out.scala_2_13'], 'true')) ]
scala_2_13: $[ dependencies.check_for_release.outputs['out.scala_2_13'] ]
timeoutInMinutes: 360
pool:
name: 'ubuntu_20_04'
demands: assignment -equals default
steps:
- template: report-start.yml
- checkout: self
- bash: |
set -euo pipefail
git checkout $(release_sha)
name: checkout_release
condition: and(succeeded(), eq(variables.is_release, 'true'))
- template: clean-up.yml
- template: build-unix.yml
parameters:
release_tag: $(release_tag)
name: 'linux-scala-2.13'
is_release: variables.is_release
scala_version: "2.13.3"
- template: tell-slack-failed.yml
parameters:
trigger_sha: '$(trigger_sha)'
- template: report-end.yml
- job: macOS
dependsOn:
- da_ghc_lib
- check_for_release
timeoutInMinutes: 360
pool:
name: macOS-pool
demands: assignment -equals default
variables:
release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
release_tag: $[ coalesce(dependencies.check_for_release.outputs['out.release_tag'], '0.0.0') ]
trigger_sha: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ]
is_release: $[ dependencies.check_for_release.outputs['out.is_release'] ]
steps:
- template: report-start.yml
- template: clear-shared-segments-macos.yml
- checkout: self
- bash: |
set -euo pipefail
git checkout $(release_sha)
name: checkout_release
condition: and(succeeded(), eq(variables.is_release, 'true'))
- template: clean-up.yml
- template: build-unix.yml
parameters:
release_tag: $(release_tag)
name: macos
is_release: variables.is_release
- template: tell-slack-failed.yml
parameters:
trigger_sha: '$(trigger_sha)'
- template: report-end.yml
- template: patch_bazel_windows/compile.yml
parameters:
final_job_name: patch_bazel_windows
- job: Windows
dependsOn:
- da_ghc_lib
- check_for_release
- patch_bazel_windows
variables:
release_sha: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
release_tag: $[ coalesce(dependencies.check_for_release.outputs['out.release_tag'], '0.0.0') ]
trigger_sha: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ]
is_release: $[ dependencies.check_for_release.outputs['out.is_release'] ]
timeoutInMinutes: 360
pool:
name: 'windows-pool'
demands: assignment -equals default
steps:
- template: report-start.yml
- checkout: self
- bash: |
set -euo pipefail
git checkout $(release_sha)
name: checkout_release
condition: and(succeeded(), eq(variables.is_release, 'true'))
- template: build-windows.yml
parameters:
release_tag: $(release_tag)
is_release: variables.is_release
- task: PublishBuildArtifacts@1
condition: succeededOrFailed()
inputs:
pathtoPublish: '$(Build.StagingDirectory)/logs'
artifactName: 'Bazel Logs'
- template: tell-slack-failed.yml
parameters:
trigger_sha: '$(trigger_sha)'
- template: report-end.yml
- job: Linux_oracle
dependsOn:
- da_ghc_lib
timeoutInMinutes: 360
pool:
name: 'ubuntu_20_04'
demands: assignment -equals default
steps:
- template: report-start.yml
- checkout: self
- bash: ci/dev-env-install.sh
displayName: 'Build/Install the Developer Environment'
- template: clean-up.yml
- bash: |
source dev-env/lib/ensure-nix
ci/dev-env-push.py
displayName: 'Push Developer Environment build results'
condition: and(succeeded(), eq(variables['System.PullRequest.IsFork'], 'False'))
env:
# to upload to the Nix cache
GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
NIX_SECRET_KEY_CONTENT: $(NIX_SECRET_KEY_CONTENT)
- bash: ci/configure-bazel.sh
displayName: 'Configure Bazel'
env:
IS_FORK: $(System.PullRequest.IsFork)
# to upload to the bazel cache
GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
- bash: |
set -euo pipefail
eval "$(./dev-env/bin/dade-assist)"
export ORACLE_PORT=1521
export ORACLE_PWD=hunter2
docker login --username $ARTIFACTORY_USERNAME --password $ARTIFACTORY_PASSWORD digitalasset-daml-on-oracle-docker.jfrog.io
docker pull digitalasset-daml-on-oracle-docker.jfrog.io/oracle-xe-preloaded:2021-02-03-5a88b29
# Cleanup stray containers that might still be running from
# another build that didnt get shut down cleanly.
docker rm -f oracle || true
docker run -d --rm --name oracle -p $ORACLE_PORT:1521 -e ORACLE_PWD=$ORACLE_PWD digitalasset-daml-on-oracle-docker.jfrog.io/oracle-xe-preloaded:2021-02-03-5a88b29
function cleanup() {
docker rm -f oracle
}
trap cleanup EXIT
testConnection() {
docker exec oracle bash -c 'sqlplus -L system/hunter2@//localhost:1521/XEPDB1 <<< "select * from dba_users;"; exit $?' >/dev/null
}
until testConnection
do
echo "Could not connect to Oracle, trying again..."
sleep 1
done
# Actually run some tests
bazel test --//ledger-service/http-json-oracle:oracle_testing=yes \
--test_env ORACLE_USERNAME=system --test_env ORACLE_PORT --test_env ORACLE_PWD \
//ledger-service/http-json-oracle/...
env:
ARTIFACTORY_USERNAME: $(ARTIFACTORY_USERNAME)
ARTIFACTORY_PASSWORD: $(ARTIFACTORY_PASSWORD)
displayName: 'Build'
condition: and(succeeded(), eq(variables['System.PullRequest.IsFork'], 'False'))
- template: tell-slack-failed.yml
parameters:
trigger_sha: '$(trigger_sha)'
- template: report-end.yml
- job: compatibility_ts_libs
dependsOn:
- da_ghc_lib
- check_for_release
condition: and(succeeded(),
not(eq(dependencies.check_for_release.outputs['out.is_release'], 'true')))
timeoutInMinutes: 360
pool:
name: ubuntu_20_04
demands: assignment -equals default
steps:
- template: report-start.yml
- checkout: self
- template: clean-up.yml
- template: compatibility_ts_libs.yml
- template: tell-slack-failed.yml
- template: report-end.yml
- job: compatibility_linux
dependsOn:
- da_ghc_lib
- check_for_release
- compatibility_ts_libs
timeoutInMinutes: 360
pool:
name: ubuntu_20_04
demands: assignment -equals default
steps:
- template: report-start.yml
- checkout: self
- template: clean-up.yml
- template: compatibility.yml
parameters:
test_flags: '--quick'
- template: tell-slack-failed.yml
- template: report-end.yml
- job: compatibility_macos
dependsOn:
- da_ghc_lib
- check_for_release
- compatibility_ts_libs
timeoutInMinutes: 360
pool:
name: macOS-pool
demands: assignment -equals default
steps:
- template: report-start.yml
- template: clear-shared-segments-macos.yml
- checkout: self
- template: clean-up.yml
- template: compatibility.yml
parameters:
test_flags: '--quick'
- template: tell-slack-failed.yml
- template: report-end.yml
- job: compatibility_windows
dependsOn:
- da_ghc_lib
- check_for_release
- compatibility_ts_libs
- patch_bazel_windows
timeoutInMinutes: 360
pool:
name: 'windows-pool'
demands: assignment -equals default
steps:
- template: report-start.yml
- checkout: self
- template: compatibility-windows.yml
parameters:
test_flags: '--quick'
- template: tell-slack-failed.yml
- template: report-end.yml
- task: PublishBuildArtifacts@1
condition: succeededOrFailed()
inputs:
pathtoPublish: '$(Build.StagingDirectory)'
artifactName: 'Bazel Compatibility Logs'
- job: check_for_release
dependsOn:
- git_sha
variables:
branch_sha: $[ dependencies.git_sha.outputs['out.branch'] ]
fork_sha: $[ dependencies.git_sha.outputs['out.fork_point'] ]
pool:
name: "ubuntu_20_04"
demands: assignment -equals default
steps:
- template: bash-lib.yml
parameters:
var_name: bash-lib
- bash: |
set -euo pipefail
eval "$(./dev-env/bin/dade-assist)"
source $(bash-lib)
./release.sh check
changes_release_files() {
changed="$(git diff-tree --no-commit-id --name-only -r $(fork_sha) $(branch_sha) | sort)"
[ "LATEST" = "$changed" ]
}
changes_one_line_in_latest() {
changed="$(git diff-tree --no-commit-id --numstat -r $(fork_sha) $(branch_sha) -- LATEST | awk '{print $1 "_" $2}')"
add_one="1_0"
change_one="1_1"
[[ "$add_one" == "$changed" || "$change_one" == "$changed" ]]
}
added_line() {
echo "$(git diff $(fork_sha) $(branch_sha) -- LATEST | tail -n+6 | grep '^\+' | cut -c2-)"
}
if changes_release_files; then
if changes_one_line_in_latest; then
setvar is_release true
setvar trigger_sha $(branch_sha)
setvar release_sha "$(added_line | awk '{print $1}')"
RELEASE_TAG="$(added_line | awk '{print $2}')"
setvar release_tag "$RELEASE_TAG"
else
echo "Release commit should only add one version."
exit 1
fi
else
RELEASE_TAG="0.0.0"
setvar is_release false
fi
# This is the last snapshot that does not support Scala 2.13.
CMP="$(semver compare "$RELEASE_TAG" '1.11.0-snapshot.20210212.6300.0.ad161d7f')"
if [[ $CMP == '1' || "$RELEASE_TAG" == '0.0.0' ]]; then
setvar scala_2_13 true
else
setvar scala_2_13 false
fi
name: out
- job: collect_build_data
condition: always()
dependsOn:
- Linux
- Linux_scala_2_13
- macOS
- Windows
- release
- git_sha
- compatibility_macos
- compatibility_linux
- compatibility_windows
- check_for_release
pool:
name: "ubuntu_20_04"
demands: assignment -equals default
variables:
Linux.start: $[ dependencies.Linux.outputs['start.time'] ]
Linux.machine: $[ dependencies.Linux.outputs['start.machine'] ]
Linux.end: $[ dependencies.Linux.outputs['end.time'] ]
Linux.status: $[ dependencies.Linux.result ]
Linux_scala_2_13.start: $[ dependencies.Linux_scala_2_13.outputs['start.time'] ]
Linux_scala_2_13.machine: $[ dependencies.Linux_scala_2_13.outputs['start.machine'] ]
Linux_scala_2_13.end: $[ dependencies.Linux_scala_2_13.outputs['end.time'] ]
Linux_scala_2_13.status: $[ dependencies.Linux_scala_2_13.result ]
macOS.start: $[ dependencies.macOS.outputs['start.time'] ]
macOS.machine: $[ dependencies.macOS.outputs['start.machine'] ]
macOS.end: $[ dependencies.macOS.outputs['end.time'] ]
macOS.status: $[ dependencies.macOS.result ]
Windows.start: $[ dependencies.Windows.outputs['start.time'] ]
Windows.machine: $[ dependencies.Windows.outputs['start.machine'] ]
Windows.end: $[ dependencies.Windows.outputs['end.time'] ]
Windows.status: $[ dependencies.Windows.result ]
release.start: $[ dependencies.release.outputs['start.time'] ]
release.machine: $[ dependencies.release.outputs['start.machine'] ]
release.end: $[ dependencies.release.outputs['end.time'] ]
release.status: $[ dependencies.release.result ]
compatibility_linux.start: $[ dependencies.compatibility_linux.outputs['start.time'] ]
compatibility_linux.machine: $[ dependencies.compatibility_linux.outputs['start.machine'] ]
compatibility_linux.end: $[ dependencies.compatibility_linux.outputs['end.time'] ]
compatibility_linux.status: $[ dependencies.compatibility_linux.result ]
compatibility_macos.start: $[ dependencies.compatibility_macos.outputs['start.time'] ]
compatibility_macos.machine: $[ dependencies.compatibility_macos.outputs['start.machine'] ]
compatibility_macos.end: $[ dependencies.compatibility_macos.outputs['end.time'] ]
compatibility_macos.status: $[ dependencies.compatibility_macos.result ]
compatibility_windows.start: $[ dependencies.compatibility_windows.outputs['start.time'] ]
compatibility_windows.machine: $[ dependencies.compatibility_windows.outputs['start.machine'] ]
compatibility_windows.end: $[ dependencies.compatibility_windows.outputs['end.time'] ]
compatibility_windows.status: $[ dependencies.compatibility_windows.result ]
branch_sha: $[ dependencies.git_sha.outputs['out.branch'] ]
main_sha: $[ dependencies.git_sha.outputs['out.main'] ]
fork_sha: $[ dependencies.git_sha.outputs['out.fork_point'] ]
is_release: $[ dependencies.check_for_release.outputs['out.is_release'] ]
# Using expression syntax so we get an empty string if not set, rather
# than the raw $(VarName) string. Expression syntax works on the
# variables key, but not on the env one, so we need an extra indirection.
# Note: These Azure variables are only set for PR builds.
pr.num: $[ variables['System.PullRequest.PullRequestNumber'] ]
pr.branch: $[ variables['System.PullRequest.SourceBranch'] ]
steps:
- template: bash-lib.yml
parameters:
var_name: bash_lib
- bash: |
set -euo pipefail
eval "$(./dev-env/bin/dade-assist)"
source $(bash_lib)
REPORT=$(mktemp)
cat >$REPORT <<END
{"jobs": {"Linux": {"start": "$(Linux.start)",
"machine": "$(Linux.machine)",
"end": "$(Linux.end)",
"status": "$(Linux.status)"},
"macOS": {"start": "$(macOS.start)",
"machine": "$(macOS.machine)",
"end": "$(macOS.end)",
"status": "$(macOS.status)"},
"Windows": {"start": "$(Windows.start)",
"machine": "$(Windows.machine)",
"end": "$(Windows.end)",
"status": "$(Windows.status)"},
"release": {"start": "$(release.start)",
"machine": "$(release.machine)",
"end": "$(release.end)",
"status": "$(release.status)"},
"compatibility_linux": {"start": "$(compatibility_linux.start)",
"machine": "$(compatibility_linux.machine)",
"end": "$(compatibility_linux.end)",
"status": "$(compatibility_linux.status)"},
"compatibility_macos": {"start": "$(compatibility_macos.start)",
"machine": "$(compatibility_macos.machine)",
"end": "$(compatibility_macos.end)",
"status": "$(compatibility_macos.status)"},
"compatibility_windows": {"start": "$(compatibility_windows.start)",
"machine": "$(compatibility_windows.machine)",
"end": "$(compatibility_windows.end)",
"status": "$(compatibility_windows.status)"}},
"id": "$(Build.BuildId)",
"url": "https://dev.azure.com/digitalasset/daml/_build/results?buildId=$(Build.BuildId)",
"name": "$(Build.DefinitionName)",
"version": "$(Build.DefinitionVersion)",
"queued_by": "$(Build.QueuedBy)",
"reason": "$(Build.Reason)",
"branch": "$(Build.SourceBranch)",
"merge_commit": "$(Build.SourceVersion)",
"branch_commit": "$(branch_sha)",
"main_commit": "$(main_sha)",
"fork_point_commit": "$(fork_sha)",
"commit_message": $(echo -n "$COMMIT_MSG" | jq -sR),
"is_fork": "$(System.PullRequest.IsFork)",
"pr": "$PR_NUM",
"pr_url": "https://github.com/digital-asset/daml/pull/$PR_NUM",
"pr_source_branch": "$PR_BRANCH",
"is_release": $(is_release)}
END
# Test above JSON is well formed
cat $REPORT | jq '.'
REPORT_GZ=$(mktemp)
cat $REPORT | gzip -9 > $REPORT_GZ
# Application credentials will not be set for forks. We give up on
# tracking those for now. "Not set" in Azure world means set to the
# expression Azure would otherwise substitute, i.e. the literal value
# of the string in the `env:` block below.
if [[ "${GCRED:1:${#GCRED}-1}" != '(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)' ]]; then
gcs "$GCRED" cp "$REPORT_GZ" "gs://daml-data/builds/$(Build.BuildId)_$(date -u +%Y%m%d_%H%M%SZ).json.gz"
else
echo "Could not save build data: no credentials."
fi
# Linux, macOS and Windows are always required and should always
# succeed.
#
# release only run on releases and is skipped otherwise.
if [[ "$(Linux.status)" != "Succeeded"
|| "$(macOS.status)" != "Succeeded"
|| "$(Windows.status)" != "Succeeded"
|| ("$(is_release)" == "false" && "$(compatibility_linux.status)" != "Succeeded")
|| ("$(is_release)" == "false" && "$(compatibility_macos.status)" != "Succeeded")
|| ("$(is_release)" == "false" && "$(compatibility_windows.status)" != "Succeeded")
|| "$(release.status)" == "Canceled" ]]; then
exit 1
fi
env:
GCRED: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
# Commit message is always set
COMMIT_MSG: $(Build.SourceVersionMessage)
# Because these variables are always set (in the variables block),
# hopefully these should be set as expected (i.e. either correct
# value or empty string, but not $(Azure.Variable.Name)).
PR_NUM: $(pr.num)
PR_BRANCH: $(pr.branch)