mirror of
https://github.com/digital-asset/daml.git
synced 2024-09-20 01:07:18 +03:00
9a98fa04f5
CHANGELOG_BEGIN CHANGELOG_END
698 lines
28 KiB
YAML
698 lines
28 KiB
YAML
# Copyright (c) 2022 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
|
||
# SPDX-License-Identifier: Apache-2.0
|
||
|
||
# Do not run on PRs
|
||
pr: none
|
||
|
||
# Do not run on merge to main
|
||
trigger: none
|
||
|
||
# Do run on a schedule (daily)
|
||
#
|
||
# Note: machines are killed every day at 4AM UTC, so we need to either:
|
||
# - run sufficiently before that that this doesn't get killed, or
|
||
# - run sufficiently after that that machines are initialized.
|
||
#
|
||
# Targeting 6AM UTC seems to fit that.
|
||
schedules:
|
||
- cron: "0 6 * * *"
|
||
displayName: daily checks and reporting
|
||
branches:
|
||
include:
|
||
- main
|
||
always: true
|
||
|
||
jobs:
|
||
- job: compatibility_ts_libs
|
||
timeoutInMinutes: 60
|
||
pool:
|
||
name: ubuntu_20_04
|
||
demands: assignment -equals default
|
||
steps:
|
||
- checkout: self
|
||
- template: ../clean-up.yml
|
||
- template: ../compatibility_ts_libs.yml
|
||
|
||
- job: compatibility
|
||
dependsOn: compatibility_ts_libs
|
||
timeoutInMinutes: 720
|
||
strategy:
|
||
matrix:
|
||
linux:
|
||
pool: ubuntu_20_04
|
||
macos:
|
||
pool: macOS-pool
|
||
pool:
|
||
name: $(pool)
|
||
demands: assignment -equals default
|
||
steps:
|
||
- checkout: self
|
||
- ${{ if eq(variables['pool'], 'macos-pool') }}:
|
||
- template: ../clear-shared-segments-macos.yml
|
||
- template: ../clean-up.yml
|
||
- template: ../compatibility.yml
|
||
|
||
- job: compatibility_windows
|
||
dependsOn: compatibility_ts_libs
|
||
timeoutInMinutes: 720
|
||
pool:
|
||
name: windows-pool
|
||
demands: assignment -equals default
|
||
steps:
|
||
- checkout: self
|
||
- template: ../compatibility-windows.yml
|
||
- task: PublishBuildArtifacts@1
|
||
condition: succeededOrFailed()
|
||
inputs:
|
||
pathtoPublish: '$(Build.StagingDirectory)'
|
||
artifactName: 'Bazel Compatibility Logs'
|
||
|
||
- job: perf_speedy
|
||
timeoutInMinutes: 120
|
||
pool:
|
||
name: "ubuntu_20_04"
|
||
demands: assignment -equals default
|
||
steps:
|
||
- checkout: self
|
||
- bash: ci/dev-env-install.sh
|
||
displayName: 'Build/Install the Developer Environment'
|
||
- bash: ci/configure-bazel.sh
|
||
displayName: 'Configure Bazel for root workspace'
|
||
env:
|
||
IS_FORK: $(System.PullRequest.IsFork)
|
||
# to upload to the bazel cache
|
||
GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
|
||
- template: ../bash-lib.yml
|
||
parameters:
|
||
var_name: bash_lib
|
||
- bash: |
|
||
set -euo pipefail
|
||
eval "$(dev-env/bin/dade assist)"
|
||
source $(bash_lib)
|
||
|
||
OUT="$(Build.StagingDirectory)/perf-results-speedy.json"
|
||
|
||
START=$(git log -n1 --format=%cd --date=format:%Y%m%d).$(git rev-list --count HEAD).$(Build.BuildId).$(git log -n1 --format=%h --abbrev=8)
|
||
|
||
tmp=$(mktemp -d)/out.json
|
||
bazel run daml-lf/scenario-interpreter:scenario-perf -- -rf json -rff $tmp >&2
|
||
perf=$(printf '%.2f' $(cat $tmp | jq '.[0].primaryMetric.score'))
|
||
if [ "" = "$perf" ]; then exit 1; fi
|
||
sha=$(git rev-parse HEAD)
|
||
echo '{"current-perf": '$perf', "current-sha": "'$sha'"}' > "$OUT"
|
||
cat "$OUT"
|
||
|
||
gcs "$GCRED" cp "$OUT" gs://daml-data/perf/speedy/$START.json
|
||
|
||
setvar speedy_perf "$perf ms/op"
|
||
|
||
displayName: measure perf
|
||
name: out
|
||
env:
|
||
GCRED: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
|
||
- task: PublishPipelineArtifact@0
|
||
inputs:
|
||
targetPath: $(Build.StagingDirectory)/perf-results-speedy.json
|
||
artifactName: perf-speedy
|
||
|
||
- job: perf_http_json
|
||
timeoutInMinutes: 120
|
||
pool:
|
||
name: "ubuntu_20_04"
|
||
demands: assignment -equals default
|
||
steps:
|
||
- checkout: self
|
||
- bash: ci/dev-env-install.sh
|
||
displayName: 'Build/Install the Developer Environment'
|
||
- bash: ci/configure-bazel.sh
|
||
displayName: 'Configure Bazel for root workspace'
|
||
env:
|
||
IS_FORK: $(System.PullRequest.IsFork)
|
||
# to upload to the bazel cache
|
||
GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
|
||
- template: ../bash-lib.yml
|
||
parameters:
|
||
var_name: bash_lib
|
||
- bash: |
|
||
set -euo pipefail
|
||
eval "$(dev-env/bin/dade assist)"
|
||
source $(bash_lib)
|
||
|
||
SCENARIOS="\
|
||
com.daml.http.perf.scenario.CreateCommand \
|
||
com.daml.http.perf.scenario.ExerciseCommand \
|
||
com.daml.http.perf.scenario.CreateAndExerciseCommand \
|
||
com.daml.http.perf.scenario.AsyncQueryConstantAcs \
|
||
com.daml.http.perf.scenario.SyncQueryConstantAcs \
|
||
com.daml.http.perf.scenario.SyncQueryNewAcs \
|
||
com.daml.http.perf.scenario.SyncQueryVariableAcs \
|
||
"
|
||
|
||
bazel build //docs:quickstart-model
|
||
DAR="${PWD}/bazel-bin/docs/quickstart-model.dar"
|
||
|
||
START=$(git log -n1 --format=%cd --date=format:%Y%m%d).$(git rev-list --count HEAD).$(Build.BuildId).$(git log -n1 --format=%h --abbrev=8)
|
||
REPORT_ID="http_json_perf_results_${START}"
|
||
OUT="$(Build.StagingDirectory)/${REPORT_ID}"
|
||
|
||
for scenario in $SCENARIOS; do
|
||
bazel run //ledger-service/http-json-perf:http-json-perf-binary-ce -- \
|
||
--scenario=${scenario} \
|
||
--dars=${DAR} \
|
||
--reports-dir=${OUT}
|
||
done
|
||
|
||
GZIP=-9 tar -zcvf ${OUT}.tgz ${OUT}
|
||
|
||
gcs "$GCRED" cp "$OUT.tgz" "gs://daml-data/perf/http-json/${REPORT_ID}.tgz"
|
||
|
||
displayName: measure http-json performance
|
||
env:
|
||
GCRED: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
|
||
|
||
- job: perf_http_json_querystore
|
||
timeoutInMinutes: 120
|
||
strategy:
|
||
matrix:
|
||
postgres:
|
||
querystore: postgres
|
||
oracle:
|
||
querystore: oracle
|
||
pool:
|
||
name: "ubuntu_20_04"
|
||
demands: assignment -equals default
|
||
steps:
|
||
- checkout: self
|
||
- bash: ci/dev-env-install.sh
|
||
displayName: 'Build/Install the Developer Environment'
|
||
- bash: ci/configure-bazel.sh
|
||
displayName: 'Configure Bazel for root workspace'
|
||
env:
|
||
IS_FORK: $(System.PullRequest.IsFork)
|
||
# to upload to the bazel cache
|
||
GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
|
||
- template: ../bash-lib.yml
|
||
parameters:
|
||
var_name: bash_lib
|
||
- bash: |
|
||
set -euo pipefail
|
||
eval "$(dev-env/bin/dade assist)"
|
||
source $(bash_lib)
|
||
|
||
QUERY_STORE=$(querystore)
|
||
|
||
#setup oracle docker vm for perf tests against oracle.
|
||
if [[ "$QUERY_STORE" == "oracle" ]]; then
|
||
docker login --username "$DOCKER_LOGIN" --password "$DOCKER_PASSWORD"
|
||
IMAGE=$(cat ci/oracle_image)
|
||
docker pull $IMAGE
|
||
# Cleanup stray containers that might still be running from
|
||
# another build that didn’t get shut down cleanly.
|
||
docker rm -f oracle || true
|
||
# Oracle does not like if you connect to it via localhost if it’s running in the container.
|
||
# Interestingly it works if you use the external IP of the host so the issue is
|
||
# not the host it is listening on (it claims for that to be 0.0.0.0).
|
||
# --network host is a cheap escape hatch for this.
|
||
docker run -d --rm --name oracle --network host -e ORACLE_PWD=$ORACLE_PWD $IMAGE
|
||
function cleanup() {
|
||
docker rm -f oracle
|
||
}
|
||
trap cleanup EXIT
|
||
testConnection() {
|
||
docker exec oracle bash -c 'sqlplus -L '"$ORACLE_USERNAME"'/'"$ORACLE_PWD"'@//localhost:'"$ORACLE_PORT"'/ORCLPDB1 <<< "select * from dba_users;"; exit $?' >/dev/null
|
||
}
|
||
|
||
# dont want to wait forever to test connection , 15m is more than sufficient here.
|
||
declare -xf testConnection
|
||
timeout 15m bash -c 'until testConnection; do echo "Could not connect to Oracle, trying again..." ; sleep 1 ; done'
|
||
fi
|
||
|
||
bazel build //ledger-service/http-json-perf/...
|
||
DAR="${PWD}/bazel-bin/ledger-service/http-json-perf/LargeAcs.dar"
|
||
|
||
# {
|
||
# "https://daml.com/ledger-api": {
|
||
# "ledgerId": "MyLedger",
|
||
# "applicationId": "foobar",
|
||
# "actAs": [
|
||
# "Alice"
|
||
# ]
|
||
# }
|
||
# }
|
||
|
||
METADATA=$(git log -n1 --format=%cd --date=format:%Y%m%d).$(git rev-list --count HEAD).$(Build.BuildId).$(git log -n1 --format=%h --abbrev=8)
|
||
REPORT_ID="http_json_perf_${QUERY_STORE}_results_${METADATA}"
|
||
OUT="$(Build.StagingDirectory)/${REPORT_ID}"
|
||
LOG_DIR="$(Build.StagingDirectory)/log"
|
||
|
||
mkdir -p $LOG_DIR
|
||
|
||
# Overall we want to populate the test cache , run some fetch by key queries and then some index queries
|
||
|
||
export NUM_RECORDS=100000 # 100k ACS for a single template
|
||
export NUM_QUERIES=10000 # 10k queries in total
|
||
export NUM_READERS=100 # 100 users in parallel.
|
||
|
||
READ_PERF_KEYS="\
|
||
fetchByKey \
|
||
fetchByQuery \
|
||
"
|
||
RESULT=""
|
||
|
||
if [[ "$QUERY_STORE" == "oracle" ]]
|
||
then
|
||
# We run test cases in isolation and retain data between them.
|
||
TEST_CASES="\
|
||
populateCache \
|
||
fetchByKey \
|
||
fetchByQuery \
|
||
"
|
||
for CASE in $TEST_CASES; do
|
||
RUN_MODE=${CASE} \
|
||
USE_DEFAULT_USER=true \
|
||
RETAIN_DATA=true \
|
||
bazel run //ledger-service/http-json-perf:http-json-perf-binary-ee -- \
|
||
--scenario=com.daml.http.perf.scenario.MultiUserQueryScenario \
|
||
--dars=${DAR} \
|
||
--reports-dir=${OUT}/${CASE} \
|
||
--query-store-index=${QUERY_STORE} > "${LOG_DIR}/${CASE}_log.out"
|
||
done
|
||
for KEY in $READ_PERF_KEYS; do
|
||
# capture the avg, stddev, p90, p99, requests_per_second numbers from gatling summary csv
|
||
perf=$(cat ${OUT}/${KEY}/*/summary.csv | tail -n 1 | awk -F, '{printf "%.1f, %.1f, %.1f, %.1f, %.1f", $11, $12, $6, $8, $13 }')
|
||
RESULT="${RESULT}json_${QUERY_STORE}/${KEY}: $perf\n"
|
||
done
|
||
elif [[ "$QUERY_STORE" == "postgres" ]]
|
||
then
|
||
# the test case to run in MultiUserQueryScenario for postgres
|
||
TEST_CASE="populateAndFetch"
|
||
|
||
RUN_MODE=${TEST_CASE} \
|
||
bazel run //ledger-service/http-json-perf:http-json-perf-binary-ee -- \
|
||
--scenario=com.daml.http.perf.scenario.MultiUserQueryScenario \
|
||
--dars=${DAR} \
|
||
--reports-dir=${OUT}/${TEST_CASE} \
|
||
--query-store-index=${QUERY_STORE} > "${LOG_DIR}/${TEST_CASE}_log.out"
|
||
|
||
for KEY in $READ_PERF_KEYS; do
|
||
# capture the avg, stddev, p90, p99, requests_per_second numbers from gatling summary csv
|
||
perf=$(cat ${OUT}/${TEST_CASE}/*/summary.csv | grep -i "$KEY" | awk -F, '{printf "%.1f, %.1f, %.1f, %.1f, %.1f", $11, $12, $6, $8, $13 }')
|
||
RESULT="${RESULT}json_${QUERY_STORE}/${KEY}: $perf\n"
|
||
done
|
||
fi
|
||
|
||
RESULT=${RESULT%"\n"}
|
||
setvar ${QUERY_STORE}_perf_results "$RESULT"
|
||
|
||
GZIP=-9 tar -zcf ${OUT}.tgz ${OUT}
|
||
|
||
gcs "$GCRED" cp "$OUT.tgz" "gs://daml-data/perf/http-json-${QUERY_STORE}/${REPORT_ID}.tgz"
|
||
|
||
displayName: http-json-$(querystore) perf
|
||
name: out
|
||
env:
|
||
GCRED: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
|
||
DOCKER_LOGIN: $(DOCKER_LOGIN)
|
||
DOCKER_PASSWORD: $(DOCKER_PASSWORD)
|
||
- task: PublishPipelineArtifact@0
|
||
inputs:
|
||
targetPath: $(Build.StagingDirectory)/log
|
||
artifactName: perf_http_json_$(querystore)
|
||
|
||
- job: check_releases
|
||
timeoutInMinutes: 360
|
||
pool:
|
||
name: ubuntu_20_04
|
||
demands: assignment -equals default
|
||
steps:
|
||
- checkout: self
|
||
- bash: ci/dev-env-install.sh
|
||
displayName: 'Build/Install the Developer Environment'
|
||
- template: ../bash-lib.yml
|
||
parameters:
|
||
var_name: bash_lib
|
||
- bash: |
|
||
set -euo pipefail
|
||
eval "$(dev-env/bin/dade assist)"
|
||
|
||
bazel build //ci/cron:cron
|
||
bazel-bin/ci/cron/cron check --bash-lib $(bash_lib) --gcp-creds "$GCRED"
|
||
displayName: check releases
|
||
env:
|
||
GCRED: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
|
||
|
||
- job: blackduck_scan
|
||
timeoutInMinutes: 1200
|
||
condition: eq(variables['Build.SourceBranchName'], 'main')
|
||
pool:
|
||
name: ubuntu_20_04
|
||
demands: assignment -equals default
|
||
steps:
|
||
- checkout: self
|
||
persistCredentials: true
|
||
- bash: ci/dev-env-install.sh
|
||
displayName: 'Build/Install the Developer Environment'
|
||
- bash: |
|
||
set -euo pipefail
|
||
eval "$(dev-env/bin/dade assist)"
|
||
export LC_ALL=en_US.UTF-8
|
||
|
||
bazel build //...
|
||
# Make sure that Bazel query works
|
||
bazel query 'deps(//...)' >/dev/null
|
||
displayName: 'Build'
|
||
- bash: |
|
||
set -euo pipefail
|
||
eval "$(dev-env/bin/dade-assist)"
|
||
|
||
#needs to be specified since blackduck can not scan all bazel
|
||
#dependency types in one go, haskell has to be scanned separatey and
|
||
#code location name uniquely identified to avoid stomping
|
||
BAZEL_DEPENDENCY_TYPE="haskell_cabal_library"
|
||
|
||
bash <(curl -s https://raw.githubusercontent.com/DACH-NY/security-blackduck/master/synopsys-detect) \
|
||
ci-build digital-asset_daml $(Build.SourceBranchName) \
|
||
--logging.level.com.synopsys.integration=DEBUG \
|
||
--detect.tools=BAZEL \
|
||
--detect.bazel.target=//... \
|
||
--detect.bazel.dependency.type=${BAZEL_DEPENDENCY_TYPE} \
|
||
--detect.notices.report=true \
|
||
--detect.code.location.name=digital-asset_daml_${BAZEL_DEPENDENCY_TYPE} \
|
||
--detect.timeout=1500
|
||
displayName: 'Blackduck Bazel Haskell Scan'
|
||
env:
|
||
BLACKDUCK_HUBDETECT_TOKEN: $(BLACKDUCK_HUBDETECT_TOKEN)
|
||
- bash: |
|
||
set -euo pipefail
|
||
eval "$(dev-env/bin/dade-assist)"
|
||
|
||
#needs to be specified since blackduck can not scan all bazel
|
||
#dependency types in one go, java has to be scanned separatey and
|
||
#code location name uniquely identified to avoid stomping
|
||
BAZEL_DEPENDENCY_TYPE="maven_install"
|
||
|
||
bash <(curl -s https://raw.githubusercontent.com/DACH-NY/security-blackduck/master/synopsys-detect) \
|
||
ci-build digital-asset_daml $(Build.SourceBranchName) \
|
||
--logging.level.com.synopsys.integration=DEBUG \
|
||
--detect.tools=BAZEL \
|
||
--detect.bazel.target=//... \
|
||
--detect.bazel.dependency.type=${BAZEL_DEPENDENCY_TYPE} \
|
||
--detect.notices.report=true \
|
||
--detect.code.location.name=digital-asset_daml_${BAZEL_DEPENDENCY_TYPE} \
|
||
--detect.timeout=1500
|
||
displayName: 'Blackduck Bazel JVM Scan'
|
||
env:
|
||
BLACKDUCK_HUBDETECT_TOKEN: $(BLACKDUCK_HUBDETECT_TOKEN)
|
||
- bash: |
|
||
set -euo pipefail
|
||
eval "$(dev-env/bin/dade-assist)"
|
||
|
||
(cd language-support/ts && yarn install)
|
||
|
||
bash <(curl -s https://raw.githubusercontent.com/DACH-NY/security-blackduck/master/synopsys-detect) \
|
||
ci-build digital-asset_daml $(Build.SourceBranchName) \
|
||
--logging.level.com.synopsys.integration=DEBUG \
|
||
--detect.tools=DETECTOR \
|
||
--detect.included.detector.types=YARN,NPM,CLANG \
|
||
--detect.npm.dependency.types.excluded=DEV \
|
||
--detect.yarn.dependency.types.excluded=DEV \
|
||
--detect.follow.symbolic.links=false \
|
||
--detect.excluded.directories=language-support/ts/daml-ledger,language-support/ts/daml-types,language-support/ts/daml-react,language-support/ts/codegen/tests/ts,bazel-out,bazel-bin,.bazel-cache,bazel-testlogs,bazel-daml,bazel-s,node_modules,dev-env,result-*\
|
||
--detect.blackduck.signature.scanner.exclusion.name.patterns=language-support/ts/daml-ledger,language-support/ts/daml-types,language-support/ts/daml-react.bazel-out,bazel-bin,.bazel-cache,bazel-testlogs,bazel-daml,bazel-s,node_modules,dev-env,result-* \
|
||
--detect.detector.search.exclusion.paths=language-support/ts/daml-ledger,language-support/ts/daml-types,language-support/ts/daml-react,bazel-out,bazel-bin,.bazel-cache,bazel-testlogs,bazel-daml,bazel-s,node_modules,dev-env,result-* \
|
||
--detect.notices.report=true \
|
||
--detect.code.location.name=digital-asset_daml_npm \
|
||
--detect.timeout=1500
|
||
displayName: 'Blackduck Npm Scan'
|
||
env:
|
||
BLACKDUCK_HUBDETECT_TOKEN: $(BLACKDUCK_HUBDETECT_TOKEN)
|
||
- bash: |
|
||
set -euo pipefail
|
||
eval "$(dev-env/bin/dade-assist)"
|
||
|
||
bash <(curl -s https://raw.githubusercontent.com/DACH-NY/security-blackduck/master/synopsys-detect) \
|
||
ci-build digital-asset_daml $(Build.SourceBranchName) \
|
||
--logging.level.com.synopsys.integration=DEBUG \
|
||
--detect.tools=DETECTOR \
|
||
--detect.included.detector.types=PIP,POETRY \
|
||
--detect.follow.symbolic.links=false \
|
||
--detect.excluded.directories=bazel-out,bazel-bin,.bazel-cache,bazel-testlogs,bazel-daml,bazel-s,node_modules,dev-env,result-* \
|
||
--detect.blackduck.signature.scanner.exclusion.name.patterns=.bazel-out,bazel-bin,.bazel-cache,bazel-testlogs,bazel-daml,bazel-s,node_modules,dev-env,result-*,*_bazel_vsts* \
|
||
--detect.detector.search.exclusion.paths=bazel-out,bazel-bin,.bazel-cache,bazel-testlogs,bazel-daml,bazel-s,node_modules,dev-env,result-*,*_bazel_vsts* \
|
||
--detect.notices.report=true \
|
||
--detect.code.location.name=digital-asset_daml_python \
|
||
--detect.timeout=1500
|
||
displayName: 'Blackduck Python Scan'
|
||
env:
|
||
BLACKDUCK_HUBDETECT_TOKEN: $(BLACKDUCK_HUBDETECT_TOKEN)
|
||
- bash: |
|
||
set -euo pipefail
|
||
eval "$(dev-env/bin/dade-assist)"
|
||
|
||
bash <(curl -s https://raw.githubusercontent.com/DACH-NY/security-blackduck/master/synopsys-detect) \
|
||
ci-build digital-asset_daml $(Build.SourceBranchName) \
|
||
--logging.level.com.synopsys.integration=DEBUG \
|
||
--detect.tools=DETECTOR \
|
||
--detect.included.detector.types=GIT,GO_MOD,GO_DEP,GO_VNDR,GO_VENDOR,GO_GRADLE \
|
||
--detect.follow.symbolic.links=false \
|
||
--detect.go.path=$(bazel info execution_root)/external/go_sdk/bin/go \
|
||
--detect.excluded.directories=bazel-out,bazel-bin,.bazel-cache,bazel-testlogs,bazel-daml,bazel-s,node_modules,dev-env,result-* \
|
||
--detect.detector.search.exclusion.paths=bazel-out,bazel-bin,.bazel-cache,bazel-testlogs,bazel-daml,bazel-s,node_modules,dev-env,result-*\
|
||
--detect.policy.check.fail.on.severities=MAJOR,CRITICAL,BLOCKER \
|
||
--detect.code.location.name=digital-asset_daml_go \
|
||
--detect.timeout=1500
|
||
displayName: 'Blackduck Go Scan'
|
||
env:
|
||
BLACKDUCK_HUBDETECT_TOKEN: $(BLACKDUCK_HUBDETECT_TOKEN)
|
||
- template: ../bash-lib.yml
|
||
parameters:
|
||
var_name: bash_lib
|
||
- bash: |
|
||
set -euo pipefail
|
||
eval "$(./dev-env/bin/dade-assist)"
|
||
source $(bash_lib)
|
||
|
||
branch="notices-update-$(Build.BuildId)"
|
||
|
||
tr -d '\015' <*_Black_Duck_Notices_Report.txt | grep -v digital-asset_daml >NOTICES
|
||
if git diff --exit-code -- NOTICES; then
|
||
echo "NOTICES file already up-to-date."
|
||
setvar need_to_build false
|
||
else
|
||
git add NOTICES
|
||
open_pr "$branch" "update NOTICES file"
|
||
setvar need_to_build true
|
||
fi
|
||
displayName: open PR
|
||
name: out
|
||
condition: and(succeeded(),
|
||
eq(variables['Build.SourceBranchName'], 'main'))
|
||
|
||
- job: run_notices_pr_build
|
||
timeoutInMinutes: 60
|
||
dependsOn: ["blackduck_scan"]
|
||
condition: and(succeeded(),
|
||
eq(variables['Build.SourceBranchName'], 'main'))
|
||
pool:
|
||
name: 'ubuntu_20_04'
|
||
demands: assignment -equals default
|
||
variables:
|
||
need_to_build: $[ dependencies.blackduck_scan.outputs['out.need_to_build'] ]
|
||
steps:
|
||
- bash: |
|
||
eval "$(./dev-env/bin/dade-assist)"
|
||
if [ "$(need_to_build)" == "true" ]; then
|
||
branch="notices-update-$(Build.BuildId)"
|
||
az extension add --name azure-devops
|
||
trap "az devops logout" EXIT
|
||
echo "$(System.AccessToken)" | az devops login --org "https://dev.azure.com/digitalasset"
|
||
az pipelines build queue --branch "$branch" \
|
||
--definition-name "PRs" \
|
||
--org "https://dev.azure.com/digitalasset" \
|
||
--project daml
|
||
fi
|
||
|
||
- job: bump_canton
|
||
timeoutInMinutes: 60
|
||
pool:
|
||
name: ubuntu_20_04
|
||
demands: assignment -equals default
|
||
steps:
|
||
- checkout: self
|
||
persistCredentials: true
|
||
- template: ../bash-lib.yml
|
||
parameters:
|
||
var_name: bash_lib
|
||
- bash: |
|
||
set -euo pipefail
|
||
|
||
eval "$(./dev-env/bin/dade-assist)"
|
||
source $(bash_lib)
|
||
|
||
git fetch
|
||
git checkout origin/main
|
||
|
||
canton_version=$(curl -u $AUTH \
|
||
--fail \
|
||
--location \
|
||
--silent \
|
||
https://digitalasset.jfrog.io/artifactory/api/storage/assembly/canton \
|
||
| jq -r '.children[].uri' \
|
||
| sed -e 's/^\///' \
|
||
| grep '^[0-9]\{8\}$' \
|
||
| sort \
|
||
| tail -1)
|
||
dir=$(pwd)
|
||
cd $(mktemp -d)
|
||
curl -u $AUTH \
|
||
--fail \
|
||
--location \
|
||
--silent \
|
||
https://digitalasset.jfrog.io/artifactory/assembly/canton/$canton_version/canton-open-source-$canton_version.tar.gz \
|
||
> tarball.tar.gz
|
||
canton_sha=$(sha256sum tarball.tar.gz | awk '{print $1}')
|
||
tar xzf tarball.tar.gz
|
||
rm tarball.tar.gz
|
||
canton_prefix=$(ls)
|
||
mv $canton_prefix canton
|
||
canton_local_path=external/$(ls canton/lib/*.jar)
|
||
cd $dir
|
||
|
||
bad_file=daml-script/test/src/test-utils/com/daml/lf/engine/script/test/MultiParticipantFixture.scala
|
||
|
||
sed -i 's|"external/canton/lib/.*|"'$canton_local_path'"|' $bad_file
|
||
cat <<EOF > canton_dep.bzl
|
||
# Copyright (c) 2022 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
|
||
# SPDX-License-Identifier: Apache-2.0
|
||
|
||
canton = {
|
||
"sha": "$canton_sha",
|
||
"prefix": "$canton_prefix",
|
||
"url": "https://www.canton.io/releases/canton-open-source-$canton_version.tar.gz",
|
||
}
|
||
EOF
|
||
|
||
branch="canton-update-$canton_version"
|
||
|
||
if git diff --exit-code -- canton_dep.bzl $bad_file; then
|
||
echo "Already up-to-date with latest Canton snapshot."
|
||
setvar need_to_build false
|
||
else
|
||
if [ "main" = "$(Build.SourceBranchName)" ]; then
|
||
git add canton_dep.bzl $bad_file
|
||
open_pr "$branch" "bump canton to $canton_version"
|
||
az extension add --name azure-devops
|
||
trap "az devops logout" EXIT
|
||
echo "$(System.AccessToken)" | az devops login --org "https://dev.azure.com/digitalasset"
|
||
az pipelines build queue --branch "$branch" \
|
||
--definition-name "PRs" \
|
||
--org "https://dev.azure.com/digitalasset" \
|
||
--project daml
|
||
else
|
||
echo "Would open PR if this were running on main."
|
||
fi
|
||
fi
|
||
env:
|
||
AUTH: $(ARTIFACTORY_USERNAME):$(ARTIFACTORY_PASSWORD)
|
||
|
||
- job: compat_versions_pr
|
||
condition: eq(variables['Build.SourceBranchName'], 'main')
|
||
pool:
|
||
name: ubuntu_20_04
|
||
demands: assignment -equals default
|
||
steps:
|
||
- checkout: self
|
||
persistCredentials: true
|
||
- bash: ci/dev-env-install.sh
|
||
- template: ../bash-lib.yml
|
||
parameters:
|
||
var_name: bash_lib
|
||
- bash: |
|
||
set -euo pipefail
|
||
eval "$(./dev-env/bin/dade-assist)"
|
||
|
||
## refresh tags, in case someone deleted one
|
||
git fetch --prune --prune-tags
|
||
|
||
## get latest version, in case it's already up to date
|
||
git fetch origin
|
||
git merge origin/main
|
||
|
||
source $(bash_lib)
|
||
|
||
cp .bazelrc compatibility/
|
||
compatibility/update-versions.sh
|
||
git add compatibility/versions.bzl compatibility/maven_install.json
|
||
|
||
|
||
if ! git diff --cached --quiet; then
|
||
BRANCH=update-compat-versions-$(date -I)
|
||
TITLE="update compat versions ($(date -I))"
|
||
open_pr "$BRANCH" "$TITLE"
|
||
trigger_azure $(System.AccessToken) PRs --branch $BRANCH
|
||
trigger_azure $(System.AccessToken) digital-asset.daml-daily-compat --branch $BRANCH
|
||
else
|
||
echo "No changes"
|
||
fi
|
||
|
||
- job: report
|
||
dependsOn: [compatibility_ts_libs, compatibility, compatibility_windows,
|
||
perf_speedy, perf_http_json, perf_http_json_querystore, check_releases,
|
||
blackduck_scan, run_notices_pr_build, bump_canton, compat_versions_pr]
|
||
condition: and(succeededOrFailed(),
|
||
eq(variables['Build.SourceBranchName'], 'main'))
|
||
pool:
|
||
name: ubuntu_20_04
|
||
demands: assignment -equals default
|
||
variables:
|
||
compatibility_ts_libs: $[ dependencies.compatibility_ts_libs.result ]
|
||
compatibility: $[ dependencies.compatibility.result ]
|
||
compatibility_windows: $[ dependencies.compatibility_windows.result ]
|
||
perf_speedy: $[ dependencies.perf_speedy.result ]
|
||
speedy_perf: $[ dependencies.perf_speedy.outputs['out.speedy_perf'] ]
|
||
perf_http_json: $[ dependencies.perf_http_json.result ]
|
||
perf_http_json_querystore: $[ dependencies.perf_http_json_querystore.result ]
|
||
oracle_perf_results: $[ dependencies.perf_http_json_querystore.outputs['oracle.out.oracle_perf_results'] ]
|
||
postgres_perf_results: $[ dependencies.perf_http_json_querystore.outputs['postgres.out.postgres_perf_results'] ]
|
||
check_releases: $[ dependencies.check_releases.result ]
|
||
blackduck_scan: $[ dependencies.blackduck_scan.result ]
|
||
run_notices_pr_build: $[ dependencies.run_notices_pr_build.result ]
|
||
bump_canton: $[ dependencies.bump_canton.result ]
|
||
compat_versions_pr: $[ dependencies.compat_versions_pr.result ]
|
||
steps:
|
||
- template: ../bash-lib.yml
|
||
parameters:
|
||
var_name: bash_lib
|
||
- bash: |
|
||
set -euo pipefail
|
||
|
||
eval "$(./dev-env/bin/dade-assist)"
|
||
source $(bash_lib)
|
||
|
||
COMMIT_TITLE="$(escape_slack "$(git log --pretty=format:%s -n1)")"
|
||
COMMIT_LINK="<https://dev.azure.com/digitalasset/daml/_build/results?buildId=$(Build.BuildId)|${COMMIT_TITLE}>"
|
||
|
||
if [[ "$(compatibility_ts_libs)" == "Succeeded"
|
||
&& "$(compatibility)" == "Succeeded"
|
||
&& "$(compatibility_windows)" == "Succeeded"
|
||
&& "$(perf_speedy)" == "Succeeded"
|
||
&& "$(perf_http_json)" == "Succeeded"
|
||
&& "$(perf_http_json_querystore)" == "Succeeded"
|
||
&& "$(check_releases)" == "Succeeded"
|
||
&& "$(bump_canton)" == "Succeeded"
|
||
&& ("$(blackduck_scan)" == "Succeeded" || "$(blackduck_scan)" == "Skipped")
|
||
&& ("$(run_notices_pr_build)" == "Succeeded" || "$(run_notices_pr_build)" == "Skipped")
|
||
&& "$(compat_versions_pr)" == "Succeeded"
|
||
]]; then
|
||
MSG="Daily tests passed: $COMMIT_LINK"
|
||
REPORT='```
|
||
speedy_perf: $(speedy_perf)
|
||
$(oracle_perf_results)
|
||
$(postgres_perf_results)
|
||
```
|
||
'
|
||
tell_slack "$(echo -e "$MSG\n$REPORT")" "$(Slack.ci-failures-daml)"
|
||
else
|
||
tell_slack "Daily tests failed: $COMMIT_LINK." "$(Slack.ci-failures-daml)"
|
||
fi
|