daml/ci/build.yml
Gary Verhaegen 4fa880a5fc
don't run fix-bazel-cache on PRs (#16377)
I'm annoyed at how slow it runs. The hourly cron should be good enough.
2023-02-23 15:59:57 +01:00

361 lines
12 KiB
YAML
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

# Copyright (c) 2023 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
jobs:
- job: git_sha
pool:
name: 'ubuntu_20_04'
demands: assignment -equals default
steps:
- template: bash-lib.yml
parameters:
var_name: bash-lib
- bash: |
set -euo pipefail
source $(bash-lib)
if [ "$(Build.Reason)" == "PullRequest" ]; then
setvar branch "$(git rev-parse HEAD^2)"
setvar main "$(git rev-parse HEAD^1)"
setvar fork_point "$(git merge-base $(git rev-parse HEAD^1) $(git rev-parse HEAD^2))"
else
setvar branch "$(git rev-parse HEAD)"
setvar main "$(git rev-parse HEAD^1)"
setvar fork_point "$(git rev-parse HEAD^1)"
fi
name: out
- job: Linux
dependsOn:
- check_for_release
variables:
- name: release_sha
value: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
- name: release_tag
value: $[ coalesce(dependencies.check_for_release.outputs['out.release_tag'], '0.0.0') ]
- name: trigger_sha
value: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ]
- name: is_release
value: $[ dependencies.check_for_release.outputs['out.is_release'] ]
- template: job-variables.yml
timeoutInMinutes: 360
pool:
name: 'ubuntu_20_04'
demands: assignment -equals default
steps:
- template: report-start.yml
- checkout: self
- bash: |
set -euo pipefail
git checkout $(release_sha)
name: checkout_release
condition: and(succeeded(), eq(variables.is_release, 'true'))
- template: clean-up.yml
- template: build-unix.yml
parameters:
release_tag: $(release_tag)
name: 'linux'
is_release: variables.is_release
- template: upload-bazel-metrics.yml
- bash: |
set -euo pipefail
eval "$(./dev-env/bin/dade-assist)"
bazel build //release:release
./bazel-bin/release/release --release-dir "$(mktemp -d)"
condition: and(succeeded(), ne(variables['is_release'], 'true'))
- template: tell-slack-failed.yml
parameters:
trigger_sha: '$(trigger_sha)'
- template: report-end.yml
- template: macOS.yml
parameters:
job_name: macOS
name: macos
assignment: default
- template: macOS.yml
parameters:
job_name: m1
name: m1
assignment: m1-builds
- job: Windows
dependsOn:
- check_for_release
variables:
- name: release_sha
value: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
- name: release_tag
value: $[ coalesce(dependencies.check_for_release.outputs['out.release_tag'], '0.0.0') ]
- name: trigger_sha
value: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ]
- name: is_release
value: $[ dependencies.check_for_release.outputs['out.is_release'] ]
- name: is_split_release
value: $[ dependencies.check_for_release.outputs['out.split_release_process'] ]
- name: skip_tests
value: $[ and(eq(variables.is_release, 'true'),
eq(variables['Build.SourceBranchName'], 'main')) ]
- template: job-variables.yml
timeoutInMinutes: 360
pool:
name: 'windows-pool'
demands: assignment -equals default
steps:
- template: report-start.yml
- checkout: self
- bash: |
set -euo pipefail
for f in $(find /d/a/SourceRootMapping -type f); do
echo "-----"
echo $f
echo "-----"
cat $f
echo "-----"
done
name: workdirs
- bash: |
set -euo pipefail
git checkout $(release_sha)
name: checkout_release
condition: and(succeeded(), eq(variables.is_release, 'true'))
- template: build-windows.yml
parameters:
release_tag: $(release_tag)
# Azure pipelines variable and parameter expansion is utter garbage.
# For whatever reason `env` values only seem to be able to use macro syntax
# and not runtime expression. is_release however is a runtime variable
# so template conditions wont work. Therefore we define the variable here
# with a runtime expression, set the parameter to the (unexpanded) string "$(skip_tests)"
# and then splice that in via a template parameter.
skip_tests: $(skip_tests)
is_release: variables.is_release
is_split_release: $(is_split_release)
- template: upload-bazel-metrics.yml
- task: PublishBuildArtifacts@1
condition: succeededOrFailed()
inputs:
pathtoPublish: '$(Build.StagingDirectory)/logs'
artifactName: 'Bazel Logs'
- template: tell-slack-failed.yml
parameters:
trigger_sha: '$(trigger_sha)'
- template: report-end.yml
- job: Linux_oracle
timeoutInMinutes: 360
pool:
name: 'ubuntu_20_04'
demands: assignment -equals default
steps:
- template: report-start.yml
- checkout: self
- bash: ci/dev-env-install.sh
displayName: 'Build/Install the Developer Environment'
- template: clean-up.yml
- bash: |
source dev-env/lib/ensure-nix
ci/dev-env-push.py
displayName: 'Push Developer Environment build results'
condition: and(succeeded(), eq(variables['System.PullRequest.IsFork'], 'False'))
env:
# to upload to the Nix cache
GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
NIX_SECRET_KEY_CONTENT: $(NIX_SECRET_KEY_CONTENT)
- bash: ci/configure-bazel.sh
displayName: 'Configure Bazel'
env:
IS_FORK: $(System.PullRequest.IsFork)
# to upload to the bazel cache
GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
- bash: |
set -euo pipefail
eval "$(./dev-env/bin/dade-assist)"
docker login --username "$DOCKER_LOGIN" --password "$DOCKER_PASSWORD"
IMAGE=$(cat ci/oracle_image)
docker pull $IMAGE
# Cleanup stray containers that might still be running from
# another build that didnt get shut down cleanly.
docker rm -f oracle || true
# Oracle does not like if you connect to it via localhost if its running in the container.
# Interestingly it works if you use the external IP of the host so the issue is
# not the host it is listening on (it claims for that to be 0.0.0.0).
# --network host is a cheap escape hatch for this.
docker run -d --rm --name oracle --network host -e ORACLE_PWD=$ORACLE_PWD $IMAGE
function cleanup() {
docker rm -f oracle
}
trap cleanup EXIT
testConnection() {
docker exec oracle bash -c 'sqlplus -L '"$ORACLE_USERNAME"'/'"$ORACLE_PWD"'@//localhost:'"$ORACLE_PORT"'/ORCLPDB1 <<< "select * from dba_users;"; exit $?' >/dev/null
}
until testConnection
do
echo "Could not connect to Oracle, trying again..."
sleep 1
done
# Actually run some tests
# Note: Oracle tests all run sequentially because they all access the same Oracle instance,
# and we sometimes observe transient connection issues when running tests in parallel.
bazel test \
--config=oracle \
--test_strategy=exclusive \
--test_tag_filters=+oracle \
//...
oracle_logs=$(Build.StagingDirectory)/oracle-logs
mkdir $oracle_logs
for path in $(docker exec oracle bash -c 'find /opt/oracle/diag/rdbms/ -type f'); do
# $path starts with a slash
mkdir -p $(dirname ${oracle_logs}${path})
docker exec oracle bash -c "cat $path" > ${oracle_logs}${path}
done
env:
DOCKER_LOGIN: $(DOCKER_LOGIN)
DOCKER_PASSWORD: $(DOCKER_PASSWORD)
displayName: 'Build'
condition: and(succeeded(), eq(variables['System.PullRequest.IsFork'], 'False'))
- task: PublishBuildArtifacts@1
condition: failed()
displayName: 'Publish the bazel test logs'
inputs:
pathtoPublish: 'bazel-testlogs/'
artifactName: 'Test logs Oracle'
- task: PublishBuildArtifacts@1
condition: failed()
displayName: 'Publish Oracle image logs'
inputs:
pathtoPublish: '$(Build.StagingDirectory)/oracle-logs'
artifactName: 'Oracle image logs'
- template: tell-slack-failed.yml
parameters:
trigger_sha: '$(trigger_sha)'
- template: report-end.yml
- job: platform_independence_test
condition: and(succeeded(),
eq(dependencies.check_for_release.outputs['out.is_release'], 'false'))
dependsOn:
- Windows
- Linux
- macOS
pool:
name: 'ubuntu_20_04'
demands: assignment -equals default
steps:
- checkout: self
- bash: ci/dev-env-install.sh
displayName: 'Build/Install the Developer Environment'
- task: DownloadPipelineArtifact@2
inputs:
artifactName: platform-independence-dar-linux
targetPath: $(Build.StagingDirectory)/platform-independence/linux/
- task: DownloadPipelineArtifact@2
inputs:
artifactName: platform-independence-dar-windows
targetPath: $(Build.StagingDirectory)/platform-independence/windows/
- task: DownloadPipelineArtifact@2
inputs:
artifactName: platform-independence-dar-macos
targetPath: $(Build.StagingDirectory)/platform-independence/macos/
- bash: |
set -euo pipefail
eval "$(./dev-env/bin/dade-assist)"
DIR1=$(mktemp -d)
DIR2=$(mktemp -d)
DIR3=$(mktemp -d)
trap "rm -rf $DIR1; rm -rf $DIR2; rm -rf $DIR3" EXIT
unzip -d $DIR1 $(Build.StagingDirectory)/platform-independence/linux/platform-independence.dar
unzip -d $DIR2 $(Build.StagingDirectory)/platform-independence/windows/platform-independence.dar
unzip -d $DIR3 $(Build.StagingDirectory)/platform-independence/macos/platform-independence.dar
# hie/hi files may differ.
diff -r --strip-trailing-cr -x '*.hie' -x '*.hi' $DIR1 $DIR2
diff -r --strip-trailing-cr -x '*.hie' -x '*.hi' $DIR1 $DIR3
displayName: 'Compare platform-independence dars of different platforms.'
- job: compatibility_ts_libs
dependsOn:
- check_for_release
condition: and(succeeded(),
not(eq(dependencies.check_for_release.outputs['out.is_release'], 'true')))
timeoutInMinutes: 360
pool:
name: ubuntu_20_04
demands: assignment -equals default
steps:
- template: report-start.yml
- checkout: self
- template: clean-up.yml
- template: compatibility_ts_libs.yml
- template: tell-slack-failed.yml
- template: report-end.yml
- job: compatibility_linux
dependsOn:
- check_for_release
- compatibility_ts_libs
timeoutInMinutes: 360
pool:
name: ubuntu_20_04
demands: assignment -equals default
steps:
- template: report-start.yml
- checkout: self
- template: clean-up.yml
- template: compatibility.yml
parameters:
test_flags: '--quick'
- template: tell-slack-failed.yml
- template: report-end.yml
# For main and PRs targeting main, we simply check against the most recent stable tag.
- job: compatibility_stable_protobuf
pool:
name: ubuntu_20_04
demands: assignment -equals default
steps:
- checkout: self
fetchTags: true
- bash: ci/check-protobuf-stability.sh
- template: tell-slack-failed.yml
- job: collect_build_data_failed
condition: failed()
dependsOn:
- Linux
- Linux_oracle
- macOS
- Windows
- release
- git_sha
- compatibility_linux
- compatibility_stable_protobuf
- check_for_release
pool:
name: "ubuntu_20_04"
demands: assignment -equals default
- job: collect_build_data
condition: always()
dependsOn:
- collect_build_data_failed
variables:
failed_result: $[ dependencies.collect_build_data_failed.result ]
pool:
name: "ubuntu_20_04"
demands: assignment -equals default
steps:
- bash: "exit 1"
# Since 'collect_build_data_failed' only runs when 'failed()', if it was
# successful that means at least one of its dependencies failed, so we set
# the result of 'collect_build_data' to 'Failed' with 'exit 1'
condition: eq(variables.failed_result, 'Succeeded')