daml/ci/build.yml

629 lines
24 KiB
YAML
Raw Normal View History

# Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
jobs:
- job: fix_bazel_cache
timeoutInMinutes: 120
condition: eq(variables['System.PullRequest.IsFork'], 'False')
pool:
name: 'ubuntu_20_04'
demands: assignment -equals default
steps:
- checkout: self
- bash: ci/dev-env-install.sh
displayName: 'Build/Install the Developer Environment'
- template: bash-lib.yml
parameters:
var_name: bash-lib
- bash: |
set -euo pipefail
eval "$(dev-env/bin/dade assist)"
bazel build //ci/cron:cron
key=$(mktemp)
cleanup="rm -rf $key ~/.config/gcloud"
trap "$cleanup" EXIT
echo "$GCRED" > $key
gcloud auth activate-service-account --key-file=$key
export BOTO_CONFIG=/dev/null
# 90 minutes should provide enough overlap for an hourly
# cronjob.
./bazel-bin/ci/cron/cron bazel-cache --age 90 --delete
env:
GCRED: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
- template: tell-slack-failed.yml
- job: git_sha
pool:
name: 'ubuntu_20_04'
demands: assignment -equals default
steps:
- template: bash-lib.yml
parameters:
var_name: bash-lib
- bash: |
set -euo pipefail
source $(bash-lib)
if [ "$(Build.Reason)" == "PullRequest" ]; then
setvar branch "$(git rev-parse HEAD^2)"
setvar main "$(git rev-parse HEAD^1)"
setvar fork_point "$(git merge-base $(git rev-parse HEAD^1) $(git rev-parse HEAD^2))"
else
setvar branch "$(git rev-parse HEAD)"
setvar main "$(git rev-parse HEAD^1)"
setvar fork_point "$(git rev-parse HEAD^1)"
fi
name: out
- template: da-ghc-lib/compile.yml
parameters:
final_job_name: da_ghc_lib
- job: Linux
dependsOn:
- da_ghc_lib
- check_for_release
- git_sha
variables:
- name: release_sha
value: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
- name: release_tag
value: $[ coalesce(dependencies.check_for_release.outputs['out.release_tag'], '0.0.0') ]
- name: trigger_sha
value: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ]
- name: is_release
value: $[ dependencies.check_for_release.outputs['out.is_release'] ]
- template: job-variables.yml
timeoutInMinutes: 360
pool:
name: 'ubuntu_20_04'
demands: assignment -equals default
steps:
- template: report-start.yml
- checkout: self
- bash: |
set -euo pipefail
git checkout $(release_sha)
name: checkout_release
condition: and(succeeded(), eq(variables.is_release, 'true'))
- template: clean-up.yml
- template: build-unix.yml
parameters:
release_tag: $(release_tag)
name: 'linux'
is_release: variables.is_release
- template: upload-bazel-metrics.yml
- bash: |
set -euo pipefail
eval "$(./dev-env/bin/dade-assist)"
bazel build //release:release
./bazel-bin/release/release --release-dir "$(mktemp -d)"
condition: and(succeeded(), ne(variables['is_release'], 'true'))
- task: PublishBuildArtifacts@1
inputs:
pathtoPublish: 'bazel-bin/docs/html.tar.gz'
artifactName: 'Docs bundle'
- template: tell-slack-failed.yml
parameters:
trigger_sha: '$(trigger_sha)'
- template: report-end.yml
- job: Linux_scala_2_12
# We disable this for release builds. The actual release happens via the regular Linux builds
# and running tests on two scala versions doesnt add any value if the tests already
# passed on the respective PRs that added the changes. The only difference here
# is the release version which should not affect things differently depending on the
# Scala version.
condition: and(succeeded(),
eq(dependencies.check_for_release.outputs['out.is_release'], 'false'))
dependsOn:
- da_ghc_lib
- check_for_release
- git_sha
variables:
- name: release_sha
value: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
- name: release_tag
value: $[ coalesce(dependencies.check_for_release.outputs['out.release_tag'], '0.0.0') ]
- name: trigger_sha
value: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ]
- name: is_release
value: $[ eq(dependencies.check_for_release.outputs['out.is_release'], 'true') ]
- template: job-variables.yml
timeoutInMinutes: 360
pool:
name: 'ubuntu_20_04'
demands: assignment -equals default
steps:
- template: report-start.yml
- checkout: self
- bash: |
set -euo pipefail
git checkout $(release_sha)
name: checkout_release
condition: and(succeeded(), eq(variables.is_release, 'true'))
- template: clean-up.yml
- template: build-unix.yml
parameters:
release_tag: $(release_tag)
name: 'linux-scala-2.12'
is_release: variables.is_release
scala_version: "2.12.14"
- template: upload-bazel-metrics.yml
- template: tell-slack-failed.yml
parameters:
trigger_sha: '$(trigger_sha)'
- template: report-end.yml
- job: macOS
dependsOn:
- da_ghc_lib
- check_for_release
- git_sha
timeoutInMinutes: 360
pool:
name: macOS-pool
demands: assignment -equals default
variables:
- name: release_sha
value: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
- name: release_tag
value: $[ coalesce(dependencies.check_for_release.outputs['out.release_tag'], '0.0.0') ]
- name: trigger_sha
value: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ]
- name: is_release
value: $[ dependencies.check_for_release.outputs['out.is_release'] ]
- template: job-variables.yml
steps:
- template: report-start.yml
- template: clear-shared-segments-macos.yml
- checkout: self
- bash: |
set -euo pipefail
git checkout $(release_sha)
name: checkout_release
condition: and(succeeded(), eq(variables.is_release, 'true'))
- template: clean-up.yml
- template: build-unix.yml
parameters:
release_tag: $(release_tag)
name: macos
is_release: variables.is_release
- template: upload-bazel-metrics.yml
- template: tell-slack-failed.yml
parameters:
trigger_sha: '$(trigger_sha)'
- template: report-end.yml
- template: patch_bazel_windows/compile.yml
parameters:
final_job_name: patch_bazel_windows
- job: Windows
dependsOn:
- da_ghc_lib
- check_for_release
- patch_bazel_windows
- git_sha
variables:
- name: release_sha
value: $[ dependencies.check_for_release.outputs['out.release_sha'] ]
- name: release_tag
value: $[ coalesce(dependencies.check_for_release.outputs['out.release_tag'], '0.0.0') ]
- name: trigger_sha
value: $[ dependencies.check_for_release.outputs['out.trigger_sha'] ]
- name: is_release
value: $[ dependencies.check_for_release.outputs['out.is_release'] ]
- name: skip_tests
value: $[ and(eq(variables.is_release, 'true'),
Try to fix windows skipping again (#9893) This took me embarassingly long to understand and debug (partially because afaict Azure is broken): The issue is that in the current state, parameters.is_release is not expanded when setting the env var. That makes sense. The variable is only set at runtime but the ${{}} template expressions are expanded before that (it works below in the condition since that’s not in a ${{}} and is evaluated at runtime). Now if we look at the other env var that does work (the release tag) we can see something interesting. We set it to the macro $(release_tag) in build.yml. However, that is not expanded since template expansion happens way earlier. So the template parameter is set to the literal string "$(release_tag)". We then splice that in via template expansion ${{parameters.release_tag}} and then at runtime azure will expand the macro. Just changing is_release to a macro however would break the use in the condition (I think you might be able to fix that if you put it in a string but that just seems even more hacky). So this PR instead defines a new variable skip_tests which we define in the job and the splice it in via a macro. Confusingly, `$[variables.is_release]` is not expanded in an env definition. Afaict this is simply a bug. The only difference between macros and runtime expressions according to the docs is that runtime expressions need to replace the full RHs and that macros can only reference a single variable. Wouldn’t help much here either anyway if we want to stick to the parameter instead of referencing a variable directly (which maybe we don’t, it doesn’t seem to help much but that’s a separate question). changelog_begin changelog_end
2021-06-03 12:24:52 +03:00
eq(variables['Build.SourceBranchName'], 'main')) ]
- template: job-variables.yml
timeoutInMinutes: 360
pool:
name: 'windows-pool'
demands: assignment -equals default
steps:
- template: report-start.yml
- checkout: self
- bash: |
set -euo pipefail
git checkout $(release_sha)
name: checkout_release
condition: and(succeeded(), eq(variables.is_release, 'true'))
- template: build-windows.yml
parameters:
release_tag: $(release_tag)
Try to fix windows skipping again (#9893) This took me embarassingly long to understand and debug (partially because afaict Azure is broken): The issue is that in the current state, parameters.is_release is not expanded when setting the env var. That makes sense. The variable is only set at runtime but the ${{}} template expressions are expanded before that (it works below in the condition since that’s not in a ${{}} and is evaluated at runtime). Now if we look at the other env var that does work (the release tag) we can see something interesting. We set it to the macro $(release_tag) in build.yml. However, that is not expanded since template expansion happens way earlier. So the template parameter is set to the literal string "$(release_tag)". We then splice that in via template expansion ${{parameters.release_tag}} and then at runtime azure will expand the macro. Just changing is_release to a macro however would break the use in the condition (I think you might be able to fix that if you put it in a string but that just seems even more hacky). So this PR instead defines a new variable skip_tests which we define in the job and the splice it in via a macro. Confusingly, `$[variables.is_release]` is not expanded in an env definition. Afaict this is simply a bug. The only difference between macros and runtime expressions according to the docs is that runtime expressions need to replace the full RHs and that macros can only reference a single variable. Wouldn’t help much here either anyway if we want to stick to the parameter instead of referencing a variable directly (which maybe we don’t, it doesn’t seem to help much but that’s a separate question). changelog_begin changelog_end
2021-06-03 12:24:52 +03:00
# Azure pipelines variable and parameter expansion is utter garbage.
# For whatever reason `env` values only seem to be able to use macro syntax
# and not runtime expression. is_release however is a runtime variable
# so template conditions wont work. Therefore we define the variable here
# with a runtime expression, set the parameter to the (unexpanded) string "$(skip_tests)"
# and then splice that in via a template parameter.
skip_tests: $(skip_tests)
is_release: variables.is_release
- template: upload-bazel-metrics.yml
- task: PublishBuildArtifacts@1
condition: succeededOrFailed()
inputs:
pathtoPublish: '$(Build.StagingDirectory)/logs'
artifactName: 'Bazel Logs'
- template: tell-slack-failed.yml
parameters:
trigger_sha: '$(trigger_sha)'
- template: report-end.yml
- job: Linux_oracle
dependsOn:
- da_ghc_lib
timeoutInMinutes: 360
pool:
name: 'ubuntu_20_04'
demands: assignment -equals default
steps:
- template: report-start.yml
- checkout: self
- bash: ci/dev-env-install.sh
displayName: 'Build/Install the Developer Environment'
- template: clean-up.yml
- bash: |
source dev-env/lib/ensure-nix
ci/dev-env-push.py
displayName: 'Push Developer Environment build results'
condition: and(succeeded(), eq(variables['System.PullRequest.IsFork'], 'False'))
env:
# to upload to the Nix cache
GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
NIX_SECRET_KEY_CONTENT: $(NIX_SECRET_KEY_CONTENT)
- bash: ci/configure-bazel.sh
displayName: 'Configure Bazel'
env:
IS_FORK: $(System.PullRequest.IsFork)
# to upload to the bazel cache
GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
- bash: |
set -euo pipefail
eval "$(./dev-env/bin/dade-assist)"
docker login --username "$DOCKER_LOGIN" --password "$DOCKER_PASSWORD"
IMAGE=$(cat ci/oracle_image)
docker pull $IMAGE
# Cleanup stray containers that might still be running from
# another build that didnt get shut down cleanly.
docker rm -f oracle || true
# Oracle does not like if you connect to it via localhost if its running in the container.
# Interestingly it works if you use the external IP of the host so the issue is
# not the host it is listening on (it claims for that to be 0.0.0.0).
# --network host is a cheap escape hatch for this.
docker run -d --rm --name oracle --network host -e ORACLE_PWD=$ORACLE_PWD $IMAGE
function cleanup() {
docker rm -f oracle
}
trap cleanup EXIT
testConnection() {
docker exec oracle bash -c 'sqlplus -L '"$ORACLE_USERNAME"'/'"$ORACLE_PWD"'@//localhost:'"$ORACLE_PORT"'/ORCLPDB1 <<< "select * from dba_users;"; exit $?' >/dev/null
}
until testConnection
do
echo "Could not connect to Oracle, trying again..."
sleep 1
done
experimental Oracle support in json-api (#8596) * separate OracleQueries from PostgresQueries - with some changes from 8161e63189 courtesy @cocreature Co-authored-by: Moritz Kiefer <moritz.kiefer@purelyfunctional.org> * abstract BIGINT * json, signatories, observers columns * compatible lastOffset Co-authored-by: Moritz Kiefer <moritz.kiefer@purelyfunctional.org> * oracle functions for select (single template ID), insert Co-authored-by: Moritz Kiefer <moritz.kiefer@purelyfunctional.org> * add oracle branch to integration tests * oracle CLI configuration for json-api * run integration tests with ojdbc in classpath * update maven_install for ojdbc * drop table if exists for Oracle * make create DDLs and drops more planned out; drop in reverse order for Oracle integrity * repin maven * port agreement_text * port (by removal) array part of ledger offset update * use CASE instead of JSON map lookup for multiparty offset update * simplify self types * fix contract archival * repin * remove selectContracts in favor of selectContractsMultiTemplate * move Oracle test execution to separate build target * move websocket test to itlib * make a bad array instance for Oracle * report actually-available JDBC drivers only * configure Oracle test from CI * attempt with platforms and constraints * a mismash of bazel to get it to conditionally enable oracle testing * fix dep resolution in Scala 2.13 * make the Oracle test a stub (inits and does empty DB query) * remove commented unused deps * no changelog CHANGELOG_BEGIN CHANGELOG_END * repin * we never supply a value for the surrogate ID columns - suggested by @cocreature; thanks * add not null to json in DB-specific place - suggested by @cocreature; thanks * why DBContractKey - suggested by @cocreature; thanks * textType isn't finalized - suggested by @cocreature; thanks Co-authored-by: Moritz Kiefer <moritz.kiefer@purelyfunctional.org>
2021-02-17 11:50:35 +03:00
# Actually run some tests
bazel test \
--config=oracle \
//ledger-service/http-json-oracle/... \
Ledger Api Oracle support (#9277) * WIP : first cut at changed schema files for oracle Define Oracle as DbType and handle necessary case match switches for it recomputed shas for oracle migration scripts Oracle fixtures get things compiling Able to connect to Oracle Working through getting schema definitions functional with Oracle runnable schema definitions only for active tables on oracle delete commented lines in schema scripts use oracle enterprise correct inadvertently changed postgres schemas WIP - latest oracle-ificiation passing upload packages spec add additional test for package upload entry read correct typo in oracle database spec name use BLOB for parties ledger_offset package_entries use hex version of offset for range queries reformat and update shas for sql scripts binary numeric implicit conversion for oracle correct duplicate exception text for oracle parties test passing on oracle add additional column to hold hex offset for party_entries party_entries working for all dbs scalafmt Configuration ledger_offset should be BLOB update sha of oracle sql files enable passing tests in order remove misleading null comments define additional custom VARRAY types add participant-integration-api-oracle tests to linux-oracle job Add TODO for places where we need to deal with separate implicit imports for Oracle vs Postgres/H2 oracle implicit conversions for custom arrays and other problematic types Do not override default debug level for all tests in participant-integration-api CHANGELOG_BEGIN Ledger API and Indexer Oracle Support CHANGELOG_END passing TransactionWriterSpec passing JdbcLedgerDaoCompletionsSpec JdbcLedgerDaoDivulgenceSpec passing JdbcLedgerDaoContractsSpec All Oracle tests passing apart from one post-commit validation test * Remove JdbcLedgerDaoValidatedOracleSpec as this is only relevant for classic postgres-backed sandbox * rebase to master -- offsets are now varchar2 rather than blob * remove use of DBMS_LOB operations * remove all greater than/less than variants for DBMS_LOB * revert postgres files that need not be touched * code review feedback : avoid code duplication * avoid indirection in type names for oracle arrays * code review: HexString implicit conversions are not needed * code review: Oracle case is not yet implemented for appendonlydao * code review: Oracle case is not yet implemented for appendonlydao (cleanup import) * code review: revert files that should not be touched * address code review feedback: db specific imports for command completion become part of queries * code review: perform db-specific reserved word escape to avoid case match * code review: remove all dbms_lob comparison operations * use simpler insert into with ignore dupes hint for oracle * code review: avoid db specific match case in events range, use db specific limitClause * code review: restore group by on Binary and Array fields for H2 and Postgres, disable for Oracle * code review: restore group by on Binary and Array fields for H2 and Postgres, disable for Oracle * code review: restore group by on binary and array fields for non-oracle dbs, honour the calculation of limit size from QueryParty.ByArith * code review: honour the calculation of limit size from QueryParty.ByArith * code review: drop user after oracle test * code review: remove drop user as it throws errors due to dangling sessions * code review: revert incorrectly changed postgres schema files * code review: clean up TODOs * Remove // before hostname for consistency with other oracle connection strings * code review: unambiguously scope table column referenced in select and where queries * code review: correct duplicate table alias
2021-04-27 01:34:17 +03:00
//triggers/service:test-oracle \
10050 append only schema on oracle (#10051) * Oracle compliant append only schema CHANGELOG_BEGIN CHANGELOG_END WIP : oracle on new appendonly schema * diff to postgres dump, create consolidated view * diff to postgres dump to ensure all oracle setup is equiv * recompute sha for changed oracle flyway scripts * drop old tables to prevent clash on name of new participant_events view * recompute sha for flyway script * prelim oracle StorageBackend * Adds support for special preparedStatement for oracle * Add support and wires to setObject by default everywhere * Add the full OracleField suite with TODOs for convenience * Wires OracleField suite to OracleFieldStrategy changelog_begin changelog_end * enable debug version of oracle driver * conversion Instant -> Timestamp for oracle * WIP: primitive println debugging * Passing PackagesSpec with appendonlyschema on Oracle Rename size column to siz to avoid reserved word clash, including migration script for postgres * include sha for new postgres migration script * add missing copyright header * cleanup * passing party spec for appendonly on oracle * passing configuration spec for appendonly on oracle * scalafmt * bazel buildifier reformat * use db generic FETCH NEXT n ROW ONLY rather than limit for cross db compat * siz instead of size for packages table on all dbs and schema * revert enabling oracle jdbc debug * Support Array[String] -> String conversion (and vice versa) for JSON array Remove as aliases for tables as this does not work with oracle Extract submitters clause for all db types Use append transaction injector for oracle append only spec * scalafmt * correct oracle failing active contract spec tests * wire in JdbcLedgerDaoCompletionsSpec * remove semi-colons for ending statements that are problematic for oracle driver * all tests up to divulgence passing for append only on oracle * all appendonly tests passing on oracle * remove ignore on fall back to limit-based query with consistent results * do not change name of size column in packages table for mutable schema for all DBs * do not change name of size column in packages table for mutable schema for all DBs * standalone oracle appendonly schema script regen shas on flyway scripts revert some cosmetic refactoring in CommonStorageBackend * Fixes conversion to parties from Oracle-JSON at flatEventWitnessesColumn * Switches from composit SQLs to single SQLs at prepared statements to accommodate Oracle limitation * Fixes arrayIntersectionWhereClause by applying patch from mutable Oracle schema integration * Fixes queries with empty startExclusive Offsets * First draw adding Oracle conformance test suites to CI * wire in the oracle conformance tests for CI * Use cross-db fetch next $n rows only syntax instead of limit syntax that works only for postgres/h2 * rename siz to package_size * recompute shas * scalafmt and include sha check for oracle append only flyway script * correct missing package_size rename * remove some todos -- correct corrupted V1__Init.sql * Update ledger/ledger-on-sql/src/test/lib/scala/com/daml/ledger/on/sql/MainWithEphemeralOracleUser.scala Co-authored-by: Robert Autenrieth <31539813+rautenrieth-da@users.noreply.github.com> * correct version number for postgres rename column scripts * remove unnecessary migration tables for oracle append only * review feedback: rename createEventFilter as requested, remove todos * review feedback: case consistency * review feedback: update todos with issue markers * review feedback: cleanup * review feedback: OracleField and OracleSchema cleanup * Fixing Table generators to use preparedData for convenience * Placing TODOs for refactorings later * Renames initial append-only oracle script, for convenience * Falls back to original behavior as far prepared statements go at couple of queries Co-authored-by: Marton Nagy <marton.nagy@digitalasset.com> Co-authored-by: Robert Autenrieth <31539813+rautenrieth-da@users.noreply.github.com>
2021-06-29 01:51:48 +03:00
//ledger/participant-integration-api:participant-integration-api-tests-oracle \
//ledger/ledger-on-sql:conformance-test-oracle \
//ledger/ledger-on-sql:conformance-test-append-only-oracle
env:
DOCKER_LOGIN: $(DOCKER_LOGIN)
DOCKER_PASSWORD: $(DOCKER_PASSWORD)
displayName: 'Build'
condition: and(succeeded(), eq(variables['System.PullRequest.IsFork'], 'False'))
- task: PublishBuildArtifacts@1
condition: failed()
displayName: 'Publish the bazel test logs'
inputs:
pathtoPublish: 'bazel-testlogs/'
artifactName: 'Test logs Oracle'
- template: tell-slack-failed.yml
parameters:
trigger_sha: '$(trigger_sha)'
- template: report-end.yml
- job: platform_independence_test
dependsOn:
- Windows
- Linux
- macOS
pool:
name: 'ubuntu_20_04'
demands: assignment -equals default
steps:
- checkout: self
- bash: ci/dev-env-install.sh
displayName: 'Build/Install the Developer Environment'
- task: DownloadPipelineArtifact@2
inputs:
artifactName: platform-independence-dar-linux
targetPath: $(Build.StagingDirectory)/platform-independence/linux/
- task: DownloadPipelineArtifact@2
inputs:
artifactName: platform-independence-dar-windows
targetPath: $(Build.StagingDirectory)/platform-independence/windows/
- task: DownloadPipelineArtifact@2
inputs:
artifactName: platform-independence-dar-macos
targetPath: $(Build.StagingDirectory)/platform-independence/macos/
- bash: |
set -euo pipefail
eval "$(./dev-env/bin/dade-assist)"
DIR1=$(mktemp -d)
DIR2=$(mktemp -d)
DIR3=$(mktemp -d)
trap "rm -rf $DIR1; rm -rf $DIR2; rm -rf $DIR3" EXIT
unzip -d $DIR1 $(Build.StagingDirectory)/platform-independence/linux/platform-independence.dar
unzip -d $DIR2 $(Build.StagingDirectory)/platform-independence/windows/platform-independence.dar
unzip -d $DIR3 $(Build.StagingDirectory)/platform-independence/macos/platform-independence.dar
# hie/hi files may differ.
diff -r --strip-trailing-cr -x '*.hie' -x '*.hi' $DIR1 $DIR2
diff -r --strip-trailing-cr -x '*.hie' -x '*.hi' $DIR1 $DIR3
displayName: 'Compare platform-independence dars of different platforms.'
- job: compatibility_ts_libs
dependsOn:
- da_ghc_lib
- check_for_release
condition: and(succeeded(),
not(eq(dependencies.check_for_release.outputs['out.is_release'], 'true')))
timeoutInMinutes: 360
pool:
name: ubuntu_20_04
demands: assignment -equals default
steps:
- template: report-start.yml
- checkout: self
- template: clean-up.yml
- template: compatibility_ts_libs.yml
- template: tell-slack-failed.yml
- template: report-end.yml
- job: compatibility_linux
dependsOn:
- da_ghc_lib
- check_for_release
- compatibility_ts_libs
timeoutInMinutes: 360
pool:
name: ubuntu_20_04
demands: assignment -equals default
steps:
- template: report-start.yml
- checkout: self
- template: clean-up.yml
- template: compatibility.yml
parameters:
test_flags: '--quick'
- template: tell-slack-failed.yml
- template: report-end.yml
- job: check_for_release
dependsOn:
- git_sha
variables:
branch_sha: $[ dependencies.git_sha.outputs['out.branch'] ]
fork_sha: $[ dependencies.git_sha.outputs['out.fork_point'] ]
pool:
name: "ubuntu_20_04"
demands: assignment -equals default
steps:
- template: bash-lib.yml
parameters:
var_name: bash-lib
- bash: |
set -euo pipefail
eval "$(./dev-env/bin/dade-assist)"
source $(bash-lib)
./release.sh check
changes_release_files() {
changed="$(git diff-tree --no-commit-id --name-only -r $(fork_sha) $(branch_sha) | sort)"
[ "LATEST" = "$changed" ]
}
changes_one_line_in_latest() {
changed="$(git diff-tree --no-commit-id --numstat -r $(fork_sha) $(branch_sha) -- LATEST | awk '{print $1 "_" $2}')"
add_one="1_0"
change_one="1_1"
[[ "$add_one" == "$changed" || "$change_one" == "$changed" ]]
}
added_line() {
echo "$(git diff $(fork_sha) $(branch_sha) -- LATEST | tail -n+6 | grep '^\+' | cut -c2-)"
}
if changes_release_files; then
if changes_one_line_in_latest; then
setvar is_release true
setvar trigger_sha $(branch_sha)
setvar release_sha "$(added_line | awk '{print $1}')"
RELEASE_TAG="$(added_line | awk '{print $2}')"
setvar release_tag "$RELEASE_TAG"
else
echo "Release commit should only add one version."
exit 1
fi
else
RELEASE_TAG="0.0.0"
setvar is_release false
fi
# This is the last snapshot that does not support Scala 2.13.
CMP="$(semver compare "$RELEASE_TAG" '1.11.0-snapshot.20210212.6300.0.ad161d7f')"
if [[ $CMP == '1' || "$RELEASE_TAG" == '0.0.0' ]]; then
setvar scala_2_13 true
else
setvar scala_2_13 false
fi
name: out
- job: collect_build_data
condition: always()
dependsOn:
- Linux
- Linux_oracle
- Linux_scala_2_12
- macOS
- Windows
- release
- git_sha
- compatibility_linux
- check_for_release
pool:
name: "ubuntu_20_04"
demands: assignment -equals default
variables:
Linux.start: $[ dependencies.Linux.outputs['start.time'] ]
Linux.machine: $[ dependencies.Linux.outputs['start.machine'] ]
Linux.end: $[ dependencies.Linux.outputs['end.time'] ]
Linux.status: $[ dependencies.Linux.result ]
Linux_oracle.start: $[ dependencies.Linux_oracle.outputs['start.time'] ]
Linux_oracle.machine: $[ dependencies.Linux_oracle.outputs['start.machine'] ]
Linux_oracle.end: $[ dependencies.Linux_oracle.outputs['end.time'] ]
Linux_oracle.status: $[ dependencies.Linux_oracle.result ]
Linux_scala_2_12.start: $[ dependencies.Linux_scala_2_12.outputs['start.time'] ]
Linux_scala_2_12.machine: $[ dependencies.Linux_scala_2_12.outputs['start.machine'] ]
Linux_scala_2_12.end: $[ dependencies.Linux_scala_2_12.outputs['end.time'] ]
Linux_scala_2_12.status: $[ dependencies.Linux_scala_2_12.result ]
macOS.start: $[ dependencies.macOS.outputs['start.time'] ]
macOS.machine: $[ dependencies.macOS.outputs['start.machine'] ]
macOS.end: $[ dependencies.macOS.outputs['end.time'] ]
macOS.status: $[ dependencies.macOS.result ]
Windows.start: $[ dependencies.Windows.outputs['start.time'] ]
Windows.machine: $[ dependencies.Windows.outputs['start.machine'] ]
Windows.end: $[ dependencies.Windows.outputs['end.time'] ]
Windows.status: $[ dependencies.Windows.result ]
release.start: $[ dependencies.release.outputs['start.time'] ]
release.machine: $[ dependencies.release.outputs['start.machine'] ]
release.end: $[ dependencies.release.outputs['end.time'] ]
release.status: $[ dependencies.release.result ]
compatibility_linux.start: $[ dependencies.compatibility_linux.outputs['start.time'] ]
compatibility_linux.machine: $[ dependencies.compatibility_linux.outputs['start.machine'] ]
compatibility_linux.end: $[ dependencies.compatibility_linux.outputs['end.time'] ]
compatibility_linux.status: $[ dependencies.compatibility_linux.result ]
branch_sha: $[ dependencies.git_sha.outputs['out.branch'] ]
main_sha: $[ dependencies.git_sha.outputs['out.main'] ]
fork_sha: $[ dependencies.git_sha.outputs['out.fork_point'] ]
is_release: $[ dependencies.check_for_release.outputs['out.is_release'] ]
# Using expression syntax so we get an empty string if not set, rather
# than the raw $(VarName) string. Expression syntax works on the
# variables key, but not on the env one, so we need an extra indirection.
# Note: These Azure variables are only set for PR builds.
pr.num: $[ variables['System.PullRequest.PullRequestNumber'] ]
pr.branch: $[ variables['System.PullRequest.SourceBranch'] ]
steps:
- template: bash-lib.yml
parameters:
var_name: bash_lib
- bash: |
set -euo pipefail
eval "$(./dev-env/bin/dade-assist)"
source $(bash_lib)
REPORT=$(mktemp)
cat >$REPORT <<END
{"jobs": {"Linux": {"start": "$(Linux.start)",
"machine": "$(Linux.machine)",
"end": "$(Linux.end)",
"status": "$(Linux.status)"},
"Linux_oracle": {"start": "$(Linux_oracle.start)",
"machine": "$(Linux_oracle.machine)",
"end": "$(Linux_oracle.end)",
"status": "$(Linux_oracle.status)"},
"Linux_scala_2_12": {"start": "$(Linux_scala_2_12.start)",
"machine": "$(Linux_scala_2_12.machine)",
"end": "$(Linux_scala_2_12.end)",
"status": "$(Linux_scala_2_12.status)"},
"macOS": {"start": "$(macOS.start)",
"machine": "$(macOS.machine)",
"end": "$(macOS.end)",
"status": "$(macOS.status)"},
"Windows": {"start": "$(Windows.start)",
"machine": "$(Windows.machine)",
"end": "$(Windows.end)",
"status": "$(Windows.status)"},
"release": {"start": "$(release.start)",
"machine": "$(release.machine)",
"end": "$(release.end)",
"status": "$(release.status)"},
"compatibility_linux": {"start": "$(compatibility_linux.start)",
"machine": "$(compatibility_linux.machine)",
"end": "$(compatibility_linux.end)",
"status": "$(compatibility_linux.status)"}},
"id": "$(Build.BuildId)",
"url": "https://dev.azure.com/digitalasset/daml/_build/results?buildId=$(Build.BuildId)",
"name": "$(Build.DefinitionName)",
"version": "$(Build.DefinitionVersion)",
"queued_by": "$(Build.QueuedBy)",
"reason": "$(Build.Reason)",
"branch": "$(Build.SourceBranch)",
"merge_commit": "$(Build.SourceVersion)",
"branch_commit": "$(branch_sha)",
"main_commit": "$(main_sha)",
"fork_point_commit": "$(fork_sha)",
"commit_message": $(echo -n "$COMMIT_MSG" | jq -sR),
"is_fork": "$(System.PullRequest.IsFork)",
"pr": "$PR_NUM",
"pr_url": "https://github.com/digital-asset/daml/pull/$PR_NUM",
"pr_source_branch": "$PR_BRANCH",
"is_release": $(is_release)}
END
# Test above JSON is well formed
cat $REPORT | jq '.'
REPORT_GZ=$(mktemp)
cat $REPORT | gzip -9 > $REPORT_GZ
# Application credentials will not be set for forks. We give up on
# tracking those for now. "Not set" in Azure world means set to the
# expression Azure would otherwise substitute, i.e. the literal value
# of the string in the `env:` block below.
if [[ "${GCRED:1:${#GCRED}-1}" != '(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)' ]]; then
gcs "$GCRED" cp "$REPORT_GZ" "gs://daml-data/builds/$(Build.BuildId)_$(date -u +%Y%m%d_%H%M%SZ).json.gz"
else
echo "Could not save build data: no credentials."
fi
# Linux, macOS and Windows are always required and should always
# succeed.
#
# release only run on releases and is skipped otherwise.
if [[ !("$(Linux.status)" == "Succeeded" || "$(Linux.status)" == "SucceededWithIssues")
|| "$(Linux_oracle.status)" != "Succeeded"
|| !("$(macOS.status)" == "Succeeded" || "$(macOS.status)" == "SucceededWithIssues")
|| "$(Windows.status)" != "Succeeded"
|| ("$(is_release)" == "false" && "$(compatibility_linux.status)" != "Succeeded")
|| ("$(is_release)" == "false" && "$(Linux_scala_2_12.status)" != "Succeeded")
|| "$(release.status)" == "Canceled" ]]; then
exit 1
fi
env:
GCRED: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
# Commit message is always set
COMMIT_MSG: $(Build.SourceVersionMessage)
# Because these variables are always set (in the variables block),
# hopefully these should be set as expected (i.e. either correct
# value or empty string, but not $(Azure.Variable.Name)).
PR_NUM: $(pr.num)
PR_BRANCH: $(pr.branch)