daml/ci/build.yml
Gary Verhaegen c04fa81d6a
ci: bump Windows workdirs (#12918)
Since #12645, we added a new pipeline, so we need to add a corresponding
entry.

As for #12645, the content of the files and the directory structure is
taken directly from a live CI node, as printed by the (now-named)
`workdirs` step.

CHANGELOG_BEGIN
CHANGELOG_END
2022-02-14 18:49:32 +00:00

552 lines
21 KiB
YAML
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

# Copyright (c) 2022 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
jobs:
- job: fix_bazel_cache
timeoutInMinutes: 120
condition: eq(variables['System.PullRequest.IsFork'], 'False')
pool:
name: 'ubuntu_20_04'
demands: assignment -equals default
steps:
- checkout: self
- bash: ci/dev-env-install.sh
displayName: 'Build/Install the Developer Environment'
- template: bash-lib.yml
parameters:
var_name: bash-lib
- bash: |
set -euo pipefail
eval "$(dev-env/bin/dade assist)"
bazel build //ci/cron:cron
key=$(mktemp)
cleanup="rm -rf $key ~/.config/gcloud"
trap "$cleanup" EXIT
echo "$GCRED" > $key
gcloud auth activate-service-account --key-file=$key
export BOTO_CONFIG=/dev/null
# 90 minutes should provide enough overlap for an hourly
# cronjob.
./bazel-bin/ci/cron/cron bazel-cache --age 90 --delete
env:
GCRED: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
- template: tell-slack-failed.yml
- job: git_sha
pool:
name: 'ubuntu_20_04'
demands: assignment -equals default
steps:
- template: bash-lib.yml
parameters:
var_name: bash-lib
- bash: |
set -euo pipefail
source $(bash-lib)
if [ "$(Build.Reason)" == "PullRequest" ]; then
setvar branch "$(git rev-parse HEAD^2)"
setvar main "$(git rev-parse HEAD^1)"
setvar fork_point "$(git merge-base $(git rev-parse HEAD^1) $(git rev-parse HEAD^2))"
else
setvar branch "$(git rev-parse HEAD)"
setvar main "$(git rev-parse HEAD^1)"
setvar fork_point "$(git rev-parse HEAD^1)"
fi
name: out
- template: da-ghc-lib/compile.yml
parameters:
final_job_name: da_ghc_lib
- job: Linux
dependsOn:
- da_ghc_lib
- check_for_release
- git_sha
variables:
- name: release_sha
value: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
- name: release_tag
value: $[ coalesce(dependencies.check_for_release.outputs['out.release_tag'], '0.0.0') ]
- name: trigger_sha
value: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ]
- name: is_release
value: $[ dependencies.check_for_release.outputs['out.is_release'] ]
- template: job-variables.yml
timeoutInMinutes: 360
pool:
name: 'ubuntu_20_04'
demands: assignment -equals default
steps:
- template: report-start.yml
- checkout: self
- bash: |
set -euo pipefail
git checkout $(release_sha)
name: checkout_release
condition: and(succeeded(), eq(variables.is_release, 'true'))
- template: clean-up.yml
- template: build-unix.yml
parameters:
release_tag: $(release_tag)
name: 'linux'
is_release: variables.is_release
- template: upload-bazel-metrics.yml
- bash: |
set -euo pipefail
eval "$(./dev-env/bin/dade-assist)"
bazel build //release:release
./bazel-bin/release/release --release-dir "$(mktemp -d)"
condition: and(succeeded(), ne(variables['is_release'], 'true'))
- task: PublishBuildArtifacts@1
inputs:
pathtoPublish: 'bazel-bin/docs/html.tar.gz'
artifactName: 'Docs bundle'
- template: tell-slack-failed.yml
parameters:
trigger_sha: '$(trigger_sha)'
- template: report-end.yml
- job: macOS
dependsOn:
- da_ghc_lib
- check_for_release
- git_sha
timeoutInMinutes: 360
pool:
name: macOS-pool
demands: assignment -equals default
variables:
- name: release_sha
value: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
- name: release_tag
value: $[ coalesce(dependencies.check_for_release.outputs['out.release_tag'], '0.0.0') ]
- name: trigger_sha
value: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ]
- name: is_release
value: $[ dependencies.check_for_release.outputs['out.is_release'] ]
- template: job-variables.yml
steps:
- template: report-start.yml
- template: clear-shared-segments-macos.yml
- checkout: self
- bash: |
set -euo pipefail
git checkout $(release_sha)
name: checkout_release
condition: and(succeeded(), eq(variables.is_release, 'true'))
- template: clean-up.yml
- template: build-unix.yml
parameters:
release_tag: $(release_tag)
name: macos
is_release: variables.is_release
- template: upload-bazel-metrics.yml
- template: tell-slack-failed.yml
parameters:
trigger_sha: '$(trigger_sha)'
- template: report-end.yml
- template: patch_bazel_windows/compile.yml
parameters:
final_job_name: patch_bazel_windows
- job: Windows
dependsOn:
- da_ghc_lib
- check_for_release
- patch_bazel_windows
- git_sha
variables:
- name: release_sha
value: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
- name: release_tag
value: $[ coalesce(dependencies.check_for_release.outputs['out.release_tag'], '0.0.0') ]
- name: trigger_sha
value: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ]
- name: is_release
value: $[ dependencies.check_for_release.outputs['out.is_release'] ]
- name: is_split_release
value: $[ dependencies.check_for_release.outputs['out.split_release_process'] ]
- name: skip_tests
value: $[ and(eq(variables.is_release, 'true'),
eq(variables['Build.SourceBranchName'], 'main')) ]
- template: job-variables.yml
timeoutInMinutes: 360
pool:
name: 'windows-pool'
demands: assignment -equals default
steps:
- template: report-start.yml
- checkout: self
- bash: |
set -euo pipefail
for f in $(find /d/a/SourceRootMapping -type f); do
echo "-----"
echo $f
echo "-----"
cat $f
echo "-----"
done
name: workdirs
- bash: |
set -euo pipefail
git checkout $(release_sha)
name: checkout_release
condition: and(succeeded(), eq(variables.is_release, 'true'))
- template: build-windows.yml
parameters:
release_tag: $(release_tag)
# Azure pipelines variable and parameter expansion is utter garbage.
# For whatever reason `env` values only seem to be able to use macro syntax
# and not runtime expression. is_release however is a runtime variable
# so template conditions wont work. Therefore we define the variable here
# with a runtime expression, set the parameter to the (unexpanded) string "$(skip_tests)"
# and then splice that in via a template parameter.
skip_tests: $(skip_tests)
is_release: variables.is_release
is_split_release: $(is_split_release)
- template: upload-bazel-metrics.yml
- task: PublishBuildArtifacts@1
condition: succeededOrFailed()
inputs:
pathtoPublish: '$(Build.StagingDirectory)/logs'
artifactName: 'Bazel Logs'
- template: tell-slack-failed.yml
parameters:
trigger_sha: '$(trigger_sha)'
- template: report-end.yml
- job: Linux_oracle
dependsOn:
- da_ghc_lib
timeoutInMinutes: 360
pool:
name: 'ubuntu_20_04'
demands: assignment -equals default
steps:
- template: report-start.yml
- checkout: self
- bash: ci/dev-env-install.sh
displayName: 'Build/Install the Developer Environment'
- template: clean-up.yml
- bash: |
source dev-env/lib/ensure-nix
ci/dev-env-push.py
displayName: 'Push Developer Environment build results'
condition: and(succeeded(), eq(variables['System.PullRequest.IsFork'], 'False'))
env:
# to upload to the Nix cache
GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
NIX_SECRET_KEY_CONTENT: $(NIX_SECRET_KEY_CONTENT)
- bash: ci/configure-bazel.sh
displayName: 'Configure Bazel'
env:
IS_FORK: $(System.PullRequest.IsFork)
# to upload to the bazel cache
GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
- bash: |
set -euo pipefail
eval "$(./dev-env/bin/dade-assist)"
docker login --username "$DOCKER_LOGIN" --password "$DOCKER_PASSWORD"
IMAGE=$(cat ci/oracle_image)
docker pull $IMAGE
# Cleanup stray containers that might still be running from
# another build that didnt get shut down cleanly.
docker rm -f oracle || true
# Oracle does not like if you connect to it via localhost if its running in the container.
# Interestingly it works if you use the external IP of the host so the issue is
# not the host it is listening on (it claims for that to be 0.0.0.0).
# --network host is a cheap escape hatch for this.
docker run -d --rm --name oracle --network host -e ORACLE_PWD=$ORACLE_PWD $IMAGE
function cleanup() {
docker rm -f oracle
}
trap cleanup EXIT
testConnection() {
docker exec oracle bash -c 'sqlplus -L '"$ORACLE_USERNAME"'/'"$ORACLE_PWD"'@//localhost:'"$ORACLE_PORT"'/ORCLPDB1 <<< "select * from dba_users;"; exit $?' >/dev/null
}
until testConnection
do
echo "Could not connect to Oracle, trying again..."
sleep 1
done
# Actually run some tests
# Note: Oracle tests all run sequentially because they all access the same Oracle instance,
# and we sometimes observe transient connection issues when running tests in parallel.
bazel test \
--config=oracle \
--test_strategy=exclusive \
--test_tag_filters=+oracle \
//...
oracle_logs=$(Build.StagingDirectory)/oracle-logs
mkdir $oracle_logs
for path in $(docker exec oracle bash -c 'find /opt/oracle/diag/rdbms/ -type f'); do
# $path starts with a slash
mkdir -p $(dirname ${oracle_logs}${path})
docker exec oracle bash -c "cat $path" > ${oracle_logs}${path}
done
env:
DOCKER_LOGIN: $(DOCKER_LOGIN)
DOCKER_PASSWORD: $(DOCKER_PASSWORD)
displayName: 'Build'
condition: and(succeeded(), eq(variables['System.PullRequest.IsFork'], 'False'))
- task: PublishBuildArtifacts@1
condition: failed()
displayName: 'Publish the bazel test logs'
inputs:
pathtoPublish: 'bazel-testlogs/'
artifactName: 'Test logs Oracle'
- task: PublishBuildArtifacts@1
condition: failed()
displayName: 'Publish Oracle image logs'
inputs:
pathtoPublish: '$(Build.StagingDirectory)/oracle-logs'
artifactName: 'Oracle image logs'
- template: tell-slack-failed.yml
parameters:
trigger_sha: '$(trigger_sha)'
- template: report-end.yml
- job: platform_independence_test
condition: and(succeeded(),
eq(dependencies.check_for_release.outputs['out.is_release'], 'false'))
dependsOn:
- Windows
- Linux
- macOS
pool:
name: 'ubuntu_20_04'
demands: assignment -equals default
steps:
- checkout: self
- bash: ci/dev-env-install.sh
displayName: 'Build/Install the Developer Environment'
- task: DownloadPipelineArtifact@2
inputs:
artifactName: platform-independence-dar-linux
targetPath: $(Build.StagingDirectory)/platform-independence/linux/
- task: DownloadPipelineArtifact@2
inputs:
artifactName: platform-independence-dar-windows
targetPath: $(Build.StagingDirectory)/platform-independence/windows/
- task: DownloadPipelineArtifact@2
inputs:
artifactName: platform-independence-dar-macos
targetPath: $(Build.StagingDirectory)/platform-independence/macos/
- bash: |
set -euo pipefail
eval "$(./dev-env/bin/dade-assist)"
DIR1=$(mktemp -d)
DIR2=$(mktemp -d)
DIR3=$(mktemp -d)
trap "rm -rf $DIR1; rm -rf $DIR2; rm -rf $DIR3" EXIT
unzip -d $DIR1 $(Build.StagingDirectory)/platform-independence/linux/platform-independence.dar
unzip -d $DIR2 $(Build.StagingDirectory)/platform-independence/windows/platform-independence.dar
unzip -d $DIR3 $(Build.StagingDirectory)/platform-independence/macos/platform-independence.dar
# hie/hi files may differ.
diff -r --strip-trailing-cr -x '*.hie' -x '*.hi' $DIR1 $DIR2
diff -r --strip-trailing-cr -x '*.hie' -x '*.hi' $DIR1 $DIR3
displayName: 'Compare platform-independence dars of different platforms.'
- job: compatibility_ts_libs
dependsOn:
- da_ghc_lib
- check_for_release
condition: and(succeeded(),
not(eq(dependencies.check_for_release.outputs['out.is_release'], 'true')))
timeoutInMinutes: 360
pool:
name: ubuntu_20_04
demands: assignment -equals default
steps:
- template: report-start.yml
- checkout: self
- template: clean-up.yml
- template: compatibility_ts_libs.yml
- template: tell-slack-failed.yml
- template: report-end.yml
- job: compatibility_linux
dependsOn:
- da_ghc_lib
- check_for_release
- compatibility_ts_libs
timeoutInMinutes: 360
pool:
name: ubuntu_20_04
demands: assignment -equals default
steps:
- template: report-start.yml
- checkout: self
- template: clean-up.yml
- template: compatibility.yml
parameters:
test_flags: '--quick'
- template: tell-slack-failed.yml
- template: report-end.yml
# For main and PRs targeting main, we simply check against the most recent stable tag.
- job: compatibility_stable_protobuf
pool:
name: ubuntu_20_04
demands: assignment -equals default
steps:
- checkout: self
- bash: ci/check-protobuf-stability.sh
- template: tell-slack-failed.yml
- job: collect_build_data
condition: always()
dependsOn:
- Linux
- Linux_oracle
- macOS
- Windows
- release
- git_sha
- compatibility_linux
- compatibility_stable_protobuf
- check_for_release
pool:
name: "ubuntu_20_04"
demands: assignment -equals default
variables:
Linux.start: $[ dependencies.Linux.outputs['start.time'] ]
Linux.machine: $[ dependencies.Linux.outputs['start.machine'] ]
Linux.end: $[ dependencies.Linux.outputs['end.time'] ]
Linux.status: $[ dependencies.Linux.result ]
Linux_oracle.start: $[ dependencies.Linux_oracle.outputs['start.time'] ]
Linux_oracle.machine: $[ dependencies.Linux_oracle.outputs['start.machine'] ]
Linux_oracle.end: $[ dependencies.Linux_oracle.outputs['end.time'] ]
Linux_oracle.status: $[ dependencies.Linux_oracle.result ]
macOS.start: $[ dependencies.macOS.outputs['start.time'] ]
macOS.machine: $[ dependencies.macOS.outputs['start.machine'] ]
macOS.end: $[ dependencies.macOS.outputs['end.time'] ]
macOS.status: $[ dependencies.macOS.result ]
Windows.start: $[ dependencies.Windows.outputs['start.time'] ]
Windows.machine: $[ dependencies.Windows.outputs['start.machine'] ]
Windows.end: $[ dependencies.Windows.outputs['end.time'] ]
Windows.status: $[ dependencies.Windows.result ]
release.start: $[ dependencies.release.outputs['start.time'] ]
release.machine: $[ dependencies.release.outputs['start.machine'] ]
release.end: $[ dependencies.release.outputs['end.time'] ]
release.status: $[ dependencies.release.result ]
compatibility_linux.start: $[ dependencies.compatibility_linux.outputs['start.time'] ]
compatibility_linux.machine: $[ dependencies.compatibility_linux.outputs['start.machine'] ]
compatibility_linux.end: $[ dependencies.compatibility_linux.outputs['end.time'] ]
compatibility_linux.status: $[ dependencies.compatibility_linux.result ]
branch_sha: $[ dependencies.git_sha.outputs['out.branch'] ]
main_sha: $[ dependencies.git_sha.outputs['out.main'] ]
fork_sha: $[ dependencies.git_sha.outputs['out.fork_point'] ]
is_release: $[ dependencies.check_for_release.outputs['out.is_release'] ]
# Using expression syntax so we get an empty string if not set, rather
# than the raw $(VarName) string. Expression syntax works on the
# variables key, but not on the env one, so we need an extra indirection.
# Note: These Azure variables are only set for PR builds.
pr.num: $[ variables['System.PullRequest.PullRequestNumber'] ]
pr.branch: $[ variables['System.PullRequest.SourceBranch'] ]
steps:
- template: bash-lib.yml
parameters:
var_name: bash_lib
- bash: |
set -euo pipefail
eval "$(./dev-env/bin/dade-assist)"
source $(bash_lib)
REPORT=$(mktemp)
cat >$REPORT <<END
{"jobs": {"Linux": {"start": "$(Linux.start)",
"machine": "$(Linux.machine)",
"end": "$(Linux.end)",
"status": "$(Linux.status)"},
"Linux_oracle": {"start": "$(Linux_oracle.start)",
"machine": "$(Linux_oracle.machine)",
"end": "$(Linux_oracle.end)",
"status": "$(Linux_oracle.status)"},
"macOS": {"start": "$(macOS.start)",
"machine": "$(macOS.machine)",
"end": "$(macOS.end)",
"status": "$(macOS.status)"},
"Windows": {"start": "$(Windows.start)",
"machine": "$(Windows.machine)",
"end": "$(Windows.end)",
"status": "$(Windows.status)"},
"release": {"start": "$(release.start)",
"machine": "$(release.machine)",
"end": "$(release.end)",
"status": "$(release.status)"},
"compatibility_linux": {"start": "$(compatibility_linux.start)",
"machine": "$(compatibility_linux.machine)",
"end": "$(compatibility_linux.end)",
"status": "$(compatibility_linux.status)"}},
"id": "$(Build.BuildId)",
"url": "https://dev.azure.com/digitalasset/daml/_build/results?buildId=$(Build.BuildId)",
"name": "$(Build.DefinitionName)",
"version": "$(Build.DefinitionVersion)",
"queued_by": "$(Build.QueuedBy)",
"reason": "$(Build.Reason)",
"branch": "$(Build.SourceBranch)",
"merge_commit": "$(Build.SourceVersion)",
"branch_commit": "$(branch_sha)",
"main_commit": "$(main_sha)",
"fork_point_commit": "$(fork_sha)",
"commit_message": $(echo -n "$COMMIT_MSG" | jq -sR),
"is_fork": "$(System.PullRequest.IsFork)",
"pr": "$PR_NUM",
"pr_url": "https://github.com/digital-asset/daml/pull/$PR_NUM",
"pr_source_branch": "$PR_BRANCH",
"is_release": $(is_release)}
END
# Test above JSON is well formed
cat $REPORT | jq '.'
REPORT_GZ=$(mktemp)
cat $REPORT | gzip -9 > $REPORT_GZ
# Application credentials will not be set for forks. We give up on
# tracking those for now. "Not set" in Azure world means set to the
# expression Azure would otherwise substitute, i.e. the literal value
# of the string in the `env:` block below.
if [[ "${GCRED:1:${#GCRED}-1}" != '(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)' ]]; then
gcs "$GCRED" cp "$REPORT_GZ" "gs://daml-data/builds/$(Build.BuildId)_$(date -u +%Y%m%d_%H%M%SZ).json.gz"
else
echo "Could not save build data: no credentials."
fi
# Linux, macOS and Windows are always required and should always
# succeed.
#
# release only run on releases and is skipped otherwise.
if [[ !("$(Linux.status)" == "Succeeded" || "$(Linux.status)" == "SucceededWithIssues")
|| "$(Linux_oracle.status)" != "Succeeded"
|| !("$(macOS.status)" == "Succeeded" || "$(macOS.status)" == "SucceededWithIssues")
|| "$(Windows.status)" != "Succeeded"
|| ("$(is_release)" == "false" && "$(compatibility_linux.status)" != "Succeeded")
|| "$(release.status)" == "Canceled" ]]; then
exit 1
fi
env:
GCRED: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
# Commit message is always set
COMMIT_MSG: $(Build.SourceVersionMessage)
# Because these variables are always set (in the variables block),
# hopefully these should be set as expected (i.e. either correct
# value or empty string, but not $(Azure.Variable.Name)).
PR_NUM: $(pr.num)
PR_BRANCH: $(pr.branch)