mirror of
https://github.com/digital-asset/daml.git
synced 2024-09-20 01:07:18 +03:00
9e1e42d27c
This should hopefully get CI working again. There are two changes in here: 1. We can no longer change our patched Bazel. I didn’t switch away from the current patched version for now (we upload it to gcp bucket so it still works fine even if we cannot build it) but if we upgrade, we need to go to an unpatched version for now. 2. We need to get `az` from dev-env. I tested the self service compat job stuff and it works fine with this but there is a chance other parts don’t. changelog_begin changelog_end
576 lines
24 KiB
YAML
576 lines
24 KiB
YAML
# Copyright (c) 2022 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
|
||
# SPDX-License-Identifier: Apache-2.0
|
||
|
||
# Do not run on PRs
|
||
pr: none
|
||
|
||
# Do not run on merge to main
|
||
trigger: none
|
||
|
||
# Do run on a schedule (daily)
|
||
#
|
||
# Note: machines are killed every day at 4AM UTC, so we need to either:
|
||
# - run sufficiently before that that this doesn't get killed, or
|
||
# - run sufficiently after that that machines are initialized.
|
||
#
|
||
# Targeting 6AM UTC seems to fit that.
|
||
schedules:
|
||
- cron: "0 6 * * *"
|
||
displayName: daily checks and reporting
|
||
branches:
|
||
include:
|
||
- main
|
||
always: true
|
||
|
||
jobs:
|
||
- job: compatibility_ts_libs
|
||
timeoutInMinutes: 60
|
||
pool:
|
||
name: ubuntu_20_04
|
||
demands: assignment -equals default
|
||
steps:
|
||
- checkout: self
|
||
- template: ../clean-up.yml
|
||
- template: ../compatibility_ts_libs.yml
|
||
|
||
- job: compatibility
|
||
dependsOn: compatibility_ts_libs
|
||
timeoutInMinutes: 720
|
||
strategy:
|
||
matrix:
|
||
linux:
|
||
pool: ubuntu_20_04
|
||
macos:
|
||
pool: macOS-pool
|
||
pool:
|
||
name: $(pool)
|
||
demands: assignment -equals default
|
||
steps:
|
||
- checkout: self
|
||
- ${{ if eq(variables['pool'], 'macos-pool') }}:
|
||
- template: ../clear-shared-segments-macos.yml
|
||
- template: ../clean-up.yml
|
||
- template: ../compatibility.yml
|
||
|
||
- job: compatibility_windows
|
||
dependsOn: compatibility_ts_libs
|
||
timeoutInMinutes: 720
|
||
pool:
|
||
name: windows-pool
|
||
demands: assignment -equals default
|
||
steps:
|
||
- checkout: self
|
||
- template: ../compatibility-windows.yml
|
||
- task: PublishBuildArtifacts@1
|
||
condition: succeededOrFailed()
|
||
inputs:
|
||
pathtoPublish: '$(Build.StagingDirectory)'
|
||
artifactName: 'Bazel Compatibility Logs'
|
||
|
||
- job: perf_speedy
|
||
timeoutInMinutes: 120
|
||
pool:
|
||
name: "ubuntu_20_04"
|
||
demands: assignment -equals default
|
||
steps:
|
||
- checkout: self
|
||
- bash: ci/dev-env-install.sh
|
||
displayName: 'Build/Install the Developer Environment'
|
||
- bash: ci/configure-bazel.sh
|
||
displayName: 'Configure Bazel for root workspace'
|
||
env:
|
||
IS_FORK: $(System.PullRequest.IsFork)
|
||
# to upload to the bazel cache
|
||
GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
|
||
- template: ../bash-lib.yml
|
||
parameters:
|
||
var_name: bash_lib
|
||
- bash: |
|
||
set -euo pipefail
|
||
eval "$(dev-env/bin/dade assist)"
|
||
source $(bash_lib)
|
||
|
||
OUT="$(Build.StagingDirectory)/perf-results-speedy.json"
|
||
|
||
START=$(git log -n1 --format=%cd --date=format:%Y%m%d).$(git rev-list --count HEAD).$(Build.BuildId).$(git log -n1 --format=%h --abbrev=8)
|
||
|
||
tmp=$(mktemp -d)/out.json
|
||
bazel run daml-lf/scenario-interpreter:scenario-perf -- -rf json -rff $tmp >&2
|
||
perf=$(printf '%.2f' $(cat $tmp | jq '.[0].primaryMetric.score'))
|
||
if [ "" = "$perf" ]; then exit 1; fi
|
||
sha=$(git rev-parse HEAD)
|
||
echo '{"current-perf": '$perf', "current-sha": "'$sha'"}' > "$OUT"
|
||
cat "$OUT"
|
||
|
||
gcs "$GCRED" cp "$OUT" gs://daml-data/perf/speedy/$START.json
|
||
|
||
setvar speedy_perf "$perf ms/op"
|
||
|
||
displayName: measure perf
|
||
name: out
|
||
env:
|
||
GCRED: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
|
||
- task: PublishPipelineArtifact@0
|
||
inputs:
|
||
targetPath: $(Build.StagingDirectory)/perf-results-speedy.json
|
||
artifactName: perf-speedy
|
||
|
||
- job: perf_http_json
|
||
timeoutInMinutes: 120
|
||
pool:
|
||
name: "ubuntu_20_04"
|
||
demands: assignment -equals default
|
||
steps:
|
||
- checkout: self
|
||
- bash: ci/dev-env-install.sh
|
||
displayName: 'Build/Install the Developer Environment'
|
||
- bash: ci/configure-bazel.sh
|
||
displayName: 'Configure Bazel for root workspace'
|
||
env:
|
||
IS_FORK: $(System.PullRequest.IsFork)
|
||
# to upload to the bazel cache
|
||
GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
|
||
- template: ../bash-lib.yml
|
||
parameters:
|
||
var_name: bash_lib
|
||
- bash: |
|
||
set -euo pipefail
|
||
eval "$(dev-env/bin/dade assist)"
|
||
source $(bash_lib)
|
||
|
||
SCENARIOS="\
|
||
com.daml.http.perf.scenario.CreateCommand \
|
||
com.daml.http.perf.scenario.ExerciseCommand \
|
||
com.daml.http.perf.scenario.CreateAndExerciseCommand \
|
||
com.daml.http.perf.scenario.AsyncQueryConstantAcs \
|
||
com.daml.http.perf.scenario.SyncQueryConstantAcs \
|
||
com.daml.http.perf.scenario.SyncQueryNewAcs \
|
||
com.daml.http.perf.scenario.SyncQueryVariableAcs \
|
||
"
|
||
|
||
bazel build //docs:quickstart-model
|
||
DAR="${PWD}/bazel-bin/docs/quickstart-model.dar"
|
||
|
||
JWT="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJodHRwczovL2RhbWwuY29tL2xlZGdlci1hcGkiOnsibGVkZ2VySWQiOiJNeUxlZGdlciIsImFwcGxpY2F0aW9uSWQiOiJmb29iYXIiLCJhY3RBcyI6WyJBbGljZSJdfX0.VdDI96mw5hrfM5ZNxLyetSVwcD7XtLT4dIdHIOa9lcU"
|
||
|
||
START=$(git log -n1 --format=%cd --date=format:%Y%m%d).$(git rev-list --count HEAD).$(Build.BuildId).$(git log -n1 --format=%h --abbrev=8)
|
||
REPORT_ID="http_json_perf_results_${START}"
|
||
OUT="$(Build.StagingDirectory)/${REPORT_ID}"
|
||
|
||
for scenario in $SCENARIOS; do
|
||
bazel run //ledger-service/http-json-perf:http-json-perf-binary-ce -- \
|
||
--scenario=${scenario} \
|
||
--dars=${DAR} \
|
||
--reports-dir=${OUT} \
|
||
--jwt=${JWT}
|
||
done
|
||
|
||
GZIP=-9 tar -zcvf ${OUT}.tgz ${OUT}
|
||
|
||
gcs "$GCRED" cp "$OUT.tgz" "gs://daml-data/perf/http-json/${REPORT_ID}.tgz"
|
||
|
||
displayName: measure http-json performance
|
||
env:
|
||
GCRED: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
|
||
|
||
- job: perf_http_json_querystore
|
||
timeoutInMinutes: 120
|
||
strategy:
|
||
matrix:
|
||
postgres:
|
||
querystore: postgres
|
||
oracle:
|
||
querystore: oracle
|
||
pool:
|
||
name: "ubuntu_20_04"
|
||
demands: assignment -equals default
|
||
steps:
|
||
- checkout: self
|
||
- bash: ci/dev-env-install.sh
|
||
displayName: 'Build/Install the Developer Environment'
|
||
- bash: ci/configure-bazel.sh
|
||
displayName: 'Configure Bazel for root workspace'
|
||
env:
|
||
IS_FORK: $(System.PullRequest.IsFork)
|
||
# to upload to the bazel cache
|
||
GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
|
||
- template: ../bash-lib.yml
|
||
parameters:
|
||
var_name: bash_lib
|
||
- bash: |
|
||
set -euo pipefail
|
||
eval "$(dev-env/bin/dade assist)"
|
||
source $(bash_lib)
|
||
|
||
QUERY_STORE=$(querystore)
|
||
|
||
#setup oracle docker vm for perf tests against oracle.
|
||
if [[ "$QUERY_STORE" == "oracle" ]]; then
|
||
docker login --username "$DOCKER_LOGIN" --password "$DOCKER_PASSWORD"
|
||
IMAGE=$(cat ci/oracle_image)
|
||
docker pull $IMAGE
|
||
# Cleanup stray containers that might still be running from
|
||
# another build that didn’t get shut down cleanly.
|
||
docker rm -f oracle || true
|
||
# Oracle does not like if you connect to it via localhost if it’s running in the container.
|
||
# Interestingly it works if you use the external IP of the host so the issue is
|
||
# not the host it is listening on (it claims for that to be 0.0.0.0).
|
||
# --network host is a cheap escape hatch for this.
|
||
docker run -d --rm --name oracle --network host -e ORACLE_PWD=$ORACLE_PWD $IMAGE
|
||
function cleanup() {
|
||
docker rm -f oracle
|
||
}
|
||
trap cleanup EXIT
|
||
testConnection() {
|
||
docker exec oracle bash -c 'sqlplus -L '"$ORACLE_USERNAME"'/'"$ORACLE_PWD"'@//localhost:'"$ORACLE_PORT"'/ORCLPDB1 <<< "select * from dba_users;"; exit $?' >/dev/null
|
||
}
|
||
|
||
# dont want to wait forever to test connection , 15m is more than sufficient here.
|
||
declare -xf testConnection
|
||
timeout 15m bash -c 'until testConnection; do echo "Could not connect to Oracle, trying again..." ; sleep 1 ; done'
|
||
fi
|
||
|
||
bazel build //ledger-service/http-json-perf/...
|
||
DAR="${PWD}/bazel-bin/ledger-service/http-json-perf/LargeAcs.dar"
|
||
|
||
# {
|
||
# "https://daml.com/ledger-api": {
|
||
# "ledgerId": "MyLedger",
|
||
# "applicationId": "foobar",
|
||
# "actAs": [
|
||
# "Alice"
|
||
# ]
|
||
# }
|
||
# }
|
||
JWT="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJodHRwczovL2RhbWwuY29tL2xlZGdlci1hcGkiOnsibGVkZ2VySWQiOiJNeUxlZGdlciIsImFwcGxpY2F0aW9uSWQiOiJmb29iYXIiLCJhY3RBcyI6WyJBbGljZSJdfX0.VdDI96mw5hrfM5ZNxLyetSVwcD7XtLT4dIdHIOa9lcU"
|
||
|
||
METADATA=$(git log -n1 --format=%cd --date=format:%Y%m%d).$(git rev-list --count HEAD).$(Build.BuildId).$(git log -n1 --format=%h --abbrev=8)
|
||
REPORT_ID="http_json_perf_${QUERY_STORE}_results_${METADATA}"
|
||
OUT="$(Build.StagingDirectory)/${REPORT_ID}"
|
||
LOG_DIR="$(Build.StagingDirectory)/log"
|
||
|
||
mkdir -p $LOG_DIR
|
||
|
||
# Overall we want to populate the test cache , run some fetch by key queries and then some index queries
|
||
|
||
export NUM_RECORDS=100000 # 100k ACS for a single template
|
||
export NUM_QUERIES=10000 # 10k queries in total
|
||
export NUM_READERS=100 # 100 users in parallel.
|
||
|
||
READ_PERF_KEYS="\
|
||
fetchByKey \
|
||
fetchByQuery \
|
||
"
|
||
RESULT=""
|
||
|
||
if [[ "$QUERY_STORE" == "oracle" ]]
|
||
then
|
||
# We run test cases in isolation and retain data between them.
|
||
TEST_CASES="\
|
||
populateCache \
|
||
fetchByKey \
|
||
fetchByQuery \
|
||
"
|
||
for CASE in $TEST_CASES; do
|
||
RUN_MODE=${CASE} \
|
||
USE_DEFAULT_USER=true \
|
||
RETAIN_DATA=true \
|
||
bazel run //ledger-service/http-json-perf:http-json-perf-binary-ee -- \
|
||
--scenario=com.daml.http.perf.scenario.MultiUserQueryScenario \
|
||
--dars=${DAR} \
|
||
--reports-dir=${OUT}/${CASE} \
|
||
--jwt=${JWT} \
|
||
--query-store-index=${QUERY_STORE} > "${LOG_DIR}/${CASE}_log.out"
|
||
done
|
||
for KEY in $READ_PERF_KEYS; do
|
||
# capture the avg, stddev, p90, p99, requests_per_second numbers from gatling summary csv
|
||
perf=$(cat ${OUT}/${KEY}/*/summary.csv | tail -n 1 | awk -F, '{printf "%.1f, %.1f, %.1f, %.1f, %.1f", $11, $12, $6, $8, $13 }')
|
||
RESULT="${RESULT}json_${QUERY_STORE}/${KEY}: $perf\n"
|
||
done
|
||
elif [[ "$QUERY_STORE" == "postgres" ]]
|
||
then
|
||
# the test case to run in MultiUserQueryScenario for postgres
|
||
TEST_CASE="populateAndFetch"
|
||
|
||
RUN_MODE=${TEST_CASE} \
|
||
bazel run //ledger-service/http-json-perf:http-json-perf-binary-ee -- \
|
||
--scenario=com.daml.http.perf.scenario.MultiUserQueryScenario \
|
||
--dars=${DAR} \
|
||
--reports-dir=${OUT}/${TEST_CASE} \
|
||
--jwt=${JWT} \
|
||
--query-store-index=${QUERY_STORE} > "${LOG_DIR}/${TEST_CASE}_log.out"
|
||
|
||
for KEY in $READ_PERF_KEYS; do
|
||
# capture the avg, stddev, p90, p99, requests_per_second numbers from gatling summary csv
|
||
perf=$(cat ${OUT}/${TEST_CASE}/*/summary.csv | grep -i "$KEY" | awk -F, '{printf "%.1f, %.1f, %.1f, %.1f, %.1f", $11, $12, $6, $8, $13 }')
|
||
RESULT="${RESULT}json_${QUERY_STORE}/${KEY}: $perf\n"
|
||
done
|
||
fi
|
||
|
||
RESULT=${RESULT%"\n"}
|
||
setvar ${QUERY_STORE}_perf_results "$RESULT"
|
||
|
||
GZIP=-9 tar -zcf ${OUT}.tgz ${OUT}
|
||
|
||
gcs "$GCRED" cp "$OUT.tgz" "gs://daml-data/perf/http-json-${QUERY_STORE}/${REPORT_ID}.tgz"
|
||
|
||
displayName: http-json-$(querystore) perf
|
||
name: out
|
||
env:
|
||
GCRED: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
|
||
DOCKER_LOGIN: $(DOCKER_LOGIN)
|
||
DOCKER_PASSWORD: $(DOCKER_PASSWORD)
|
||
- task: PublishPipelineArtifact@0
|
||
inputs:
|
||
targetPath: $(Build.StagingDirectory)/log
|
||
artifactName: perf_http_json_$(querystore)
|
||
|
||
- job: check_releases
|
||
timeoutInMinutes: 360
|
||
pool:
|
||
name: ubuntu_20_04
|
||
demands: assignment -equals default
|
||
steps:
|
||
- checkout: self
|
||
- bash: ci/dev-env-install.sh
|
||
displayName: 'Build/Install the Developer Environment'
|
||
- template: ../bash-lib.yml
|
||
parameters:
|
||
var_name: bash_lib
|
||
- bash: |
|
||
set -euo pipefail
|
||
eval "$(dev-env/bin/dade assist)"
|
||
|
||
bazel build //ci/cron:cron
|
||
bazel-bin/ci/cron/cron check --bash-lib $(bash_lib) --gcp-creds "$GCRED"
|
||
displayName: check releases
|
||
env:
|
||
GCRED: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
|
||
|
||
- job: blackduck_scan
|
||
timeoutInMinutes: 1200
|
||
condition: eq(variables['Build.SourceBranchName'], 'main')
|
||
pool:
|
||
name: ubuntu_20_04
|
||
demands: assignment -equals default
|
||
steps:
|
||
- checkout: self
|
||
persistCredentials: true
|
||
- bash: ci/dev-env-install.sh
|
||
displayName: 'Build/Install the Developer Environment'
|
||
- bash: |
|
||
set -euo pipefail
|
||
eval "$(dev-env/bin/dade assist)"
|
||
export LC_ALL=en_US.UTF-8
|
||
|
||
bazel build //...
|
||
# Make sure that Bazel query works
|
||
bazel query 'deps(//...)' >/dev/null
|
||
displayName: 'Build'
|
||
- bash: |
|
||
set -euo pipefail
|
||
eval "$(dev-env/bin/dade-assist)"
|
||
|
||
#needs to be specified since blackduck can not scan all bazel
|
||
#dependency types in one go, haskell has to be scanned separatey and
|
||
#code location name uniquely identified to avoid stomping
|
||
BAZEL_DEPENDENCY_TYPE="haskell_cabal_library"
|
||
|
||
bash <(curl -s https://raw.githubusercontent.com/DACH-NY/security-blackduck/master/synopsys-detect) \
|
||
ci-build digital-asset_daml $(Build.SourceBranchName) \
|
||
--logging.level.com.synopsys.integration=DEBUG \
|
||
--detect.tools=BAZEL \
|
||
--detect.bazel.target=//... \
|
||
--detect.bazel.dependency.type=${BAZEL_DEPENDENCY_TYPE} \
|
||
--detect.notices.report=true \
|
||
--detect.code.location.name=digital-asset_daml_${BAZEL_DEPENDENCY_TYPE} \
|
||
--detect.timeout=1500
|
||
displayName: 'Blackduck Bazel Haskell Scan'
|
||
env:
|
||
BLACKDUCK_HUBDETECT_TOKEN: $(BLACKDUCK_HUBDETECT_TOKEN)
|
||
- bash: |
|
||
set -euo pipefail
|
||
eval "$(dev-env/bin/dade-assist)"
|
||
|
||
#needs to be specified since blackduck can not scan all bazel
|
||
#dependency types in one go, java has to be scanned separatey and
|
||
#code location name uniquely identified to avoid stomping
|
||
BAZEL_DEPENDENCY_TYPE="maven_install"
|
||
|
||
bash <(curl -s https://raw.githubusercontent.com/DACH-NY/security-blackduck/master/synopsys-detect) \
|
||
ci-build digital-asset_daml $(Build.SourceBranchName) \
|
||
--logging.level.com.synopsys.integration=DEBUG \
|
||
--detect.tools=BAZEL \
|
||
--detect.bazel.target=//... \
|
||
--detect.bazel.dependency.type=${BAZEL_DEPENDENCY_TYPE} \
|
||
--detect.notices.report=true \
|
||
--detect.code.location.name=digital-asset_daml_${BAZEL_DEPENDENCY_TYPE} \
|
||
--detect.timeout=1500
|
||
displayName: 'Blackduck Bazel JVM Scan'
|
||
env:
|
||
BLACKDUCK_HUBDETECT_TOKEN: $(BLACKDUCK_HUBDETECT_TOKEN)
|
||
- bash: |
|
||
set -euo pipefail
|
||
eval "$(dev-env/bin/dade-assist)"
|
||
|
||
(cd language-support/ts && yarn install)
|
||
|
||
bash <(curl -s https://raw.githubusercontent.com/DACH-NY/security-blackduck/master/synopsys-detect) \
|
||
ci-build digital-asset_daml $(Build.SourceBranchName) \
|
||
--logging.level.com.synopsys.integration=DEBUG \
|
||
--detect.tools=DETECTOR \
|
||
--detect.included.detector.types=YARN,NPM,CLANG \
|
||
--detect.npm.dependency.types.excluded=DEV \
|
||
--detect.yarn.dependency.types.excluded=DEV \
|
||
--detect.follow.symbolic.links=false \
|
||
--detect.excluded.directories=language-support/ts/daml-ledger,language-support/ts/daml-types,language-support/ts/daml-react,language-support/ts/codegen/tests/ts,bazel-out,bazel-bin,.bazel-cache,bazel-testlogs,bazel-daml,bazel-s,node_modules,dev-env,result-*\
|
||
--detect.blackduck.signature.scanner.exclusion.name.patterns=language-support/ts/daml-ledger,language-support/ts/daml-types,language-support/ts/daml-react.bazel-out,bazel-bin,.bazel-cache,bazel-testlogs,bazel-daml,bazel-s,node_modules,dev-env,result-* \
|
||
--detect.detector.search.exclusion.paths=language-support/ts/daml-ledger,language-support/ts/daml-types,language-support/ts/daml-react,bazel-out,bazel-bin,.bazel-cache,bazel-testlogs,bazel-daml,bazel-s,node_modules,dev-env,result-* \
|
||
--detect.notices.report=true \
|
||
--detect.code.location.name=digital-asset_daml_npm \
|
||
--detect.timeout=1500
|
||
displayName: 'Blackduck Npm Scan'
|
||
env:
|
||
BLACKDUCK_HUBDETECT_TOKEN: $(BLACKDUCK_HUBDETECT_TOKEN)
|
||
- bash: |
|
||
set -euo pipefail
|
||
eval "$(dev-env/bin/dade-assist)"
|
||
|
||
bash <(curl -s https://raw.githubusercontent.com/DACH-NY/security-blackduck/master/synopsys-detect) \
|
||
ci-build digital-asset_daml $(Build.SourceBranchName) \
|
||
--logging.level.com.synopsys.integration=DEBUG \
|
||
--detect.tools=DETECTOR \
|
||
--detect.included.detector.types=PIP,POETRY \
|
||
--detect.follow.symbolic.links=false \
|
||
--detect.excluded.directories=bazel-out,bazel-bin,.bazel-cache,bazel-testlogs,bazel-daml,bazel-s,node_modules,dev-env,result-* \
|
||
--detect.blackduck.signature.scanner.exclusion.name.patterns=.bazel-out,bazel-bin,.bazel-cache,bazel-testlogs,bazel-daml,bazel-s,node_modules,dev-env,result-*,*_bazel_vsts* \
|
||
--detect.detector.search.exclusion.paths=bazel-out,bazel-bin,.bazel-cache,bazel-testlogs,bazel-daml,bazel-s,node_modules,dev-env,result-*,*_bazel_vsts* \
|
||
--detect.notices.report=true \
|
||
--detect.code.location.name=digital-asset_daml_python \
|
||
--detect.timeout=1500
|
||
displayName: 'Blackduck Python Scan'
|
||
env:
|
||
BLACKDUCK_HUBDETECT_TOKEN: $(BLACKDUCK_HUBDETECT_TOKEN)
|
||
- bash: |
|
||
set -euo pipefail
|
||
eval "$(dev-env/bin/dade-assist)"
|
||
|
||
bash <(curl -s https://raw.githubusercontent.com/DACH-NY/security-blackduck/master/synopsys-detect) \
|
||
ci-build digital-asset_daml $(Build.SourceBranchName) \
|
||
--logging.level.com.synopsys.integration=DEBUG \
|
||
--detect.tools=DETECTOR \
|
||
--detect.included.detector.types=GIT,GO_MOD,GO_DEP,GO_VNDR,GO_VENDOR,GO_GRADLE \
|
||
--detect.follow.symbolic.links=false \
|
||
--detect.go.path=$(bazel info execution_root)/external/go_sdk/bin/go \
|
||
--detect.excluded.directories=bazel-out,bazel-bin,.bazel-cache,bazel-testlogs,bazel-daml,bazel-s,node_modules,dev-env,result-* \
|
||
--detect.detector.search.exclusion.paths=bazel-out,bazel-bin,.bazel-cache,bazel-testlogs,bazel-daml,bazel-s,node_modules,dev-env,result-*\
|
||
--detect.policy.check.fail.on.severities=MAJOR,CRITICAL,BLOCKER \
|
||
--detect.code.location.name=digital-asset_daml_go \
|
||
--detect.timeout=1500
|
||
displayName: 'Blackduck Go Scan'
|
||
env:
|
||
BLACKDUCK_HUBDETECT_TOKEN: $(BLACKDUCK_HUBDETECT_TOKEN)
|
||
- template: ../bash-lib.yml
|
||
parameters:
|
||
var_name: bash_lib
|
||
- bash: |
|
||
set -euo pipefail
|
||
eval "$(./dev-env/bin/dade-assist)"
|
||
source $(bash_lib)
|
||
|
||
branch="notices-update-$(Build.BuildId)"
|
||
|
||
tr -d '\015' <*_Black_Duck_Notices_Report.txt | grep -v digital-asset_daml >NOTICES
|
||
if git diff --exit-code -- NOTICES; then
|
||
echo "NOTICES file already up-to-date."
|
||
setvar need_to_build false
|
||
else
|
||
git add NOTICES
|
||
open_pr "$branch" "update NOTICES file"
|
||
setvar need_to_build true
|
||
fi
|
||
displayName: open PR
|
||
name: out
|
||
condition: and(succeeded(),
|
||
eq(variables['Build.SourceBranchName'], 'main'))
|
||
|
||
- job: run_notices_pr_build
|
||
timeoutInMinutes: 60
|
||
dependsOn: ["blackduck_scan"]
|
||
condition: and(succeeded(),
|
||
eq(variables['Build.SourceBranchName'], 'main'))
|
||
pool:
|
||
name: 'ubuntu_20_04'
|
||
demands: assignment -equals default
|
||
variables:
|
||
need_to_build: $[ dependencies.blackduck_scan.outputs['out.need_to_build'] ]
|
||
steps:
|
||
- bash: |
|
||
eval "$(./dev-env/bin/dade-assist)"
|
||
if [ "$(need_to_build)" == "true" ]; then
|
||
branch="notices-update-$(Build.BuildId)"
|
||
az extension add --name azure-devops
|
||
trap "az devops logout" EXIT
|
||
echo "$(System.AccessToken)" | az devops login --org "https://dev.azure.com/digitalasset"
|
||
az pipelines build queue --branch "$branch" \
|
||
--definition-name "PRs" \
|
||
--org "https://dev.azure.com/digitalasset" \
|
||
--project daml
|
||
fi
|
||
|
||
- job: report
|
||
dependsOn: [compatibility_ts_libs, compatibility, compatibility_windows,
|
||
perf_speedy, perf_http_json, perf_http_json_querystore, check_releases,
|
||
blackduck_scan, run_notices_pr_build]
|
||
condition: and(succeededOrFailed(),
|
||
eq(variables['Build.SourceBranchName'], 'main'))
|
||
pool:
|
||
name: ubuntu_20_04
|
||
demands: assignment -equals default
|
||
variables:
|
||
compatibility_ts_libs: $[ dependencies.compatibility_ts_libs.result ]
|
||
compatibility: $[ dependencies.compatibility.result ]
|
||
compatibility_windows: $[ dependencies.compatibility_windows.result ]
|
||
perf_speedy: $[ dependencies.perf_speedy.result ]
|
||
speedy_perf: $[ dependencies.perf_speedy.outputs['out.speedy_perf'] ]
|
||
perf_http_json: $[ dependencies.perf_http_json.result ]
|
||
perf_http_json_querystore: $[ dependencies.perf_http_json_querystore.result ]
|
||
oracle_perf_results: $[ dependencies.perf_http_json_querystore.outputs['oracle.out.oracle_perf_results'] ]
|
||
postgres_perf_results: $[ dependencies.perf_http_json_querystore.outputs['postgres.out.postgres_perf_results'] ]
|
||
check_releases: $[ dependencies.check_releases.result ]
|
||
blackduck_scan: $[ dependencies.blackduck_scan.result ]
|
||
run_notices_pr_build: $[ dependencies.run_notices_pr_build.result ]
|
||
steps:
|
||
- template: ../bash-lib.yml
|
||
parameters:
|
||
var_name: bash_lib
|
||
- bash: |
|
||
set -euo pipefail
|
||
|
||
eval "$(./dev-env/bin/dade-assist)"
|
||
source $(bash_lib)
|
||
|
||
COMMIT_TITLE="$(escape_slack "$(git log --pretty=format:%s -n1)")"
|
||
COMMIT_LINK="<https://dev.azure.com/digitalasset/daml/_build/results?buildId=$(Build.BuildId)|${COMMIT_TITLE}>"
|
||
|
||
if [[ "$(compatibility_ts_libs)" == "Succeeded"
|
||
&& "$(compatibility)" == "Succeeded"
|
||
&& "$(compatibility_windows)" == "Succeeded"
|
||
&& "$(perf_speedy)" == "Succeeded"
|
||
&& "$(perf_http_json)" == "Succeeded"
|
||
&& "$(perf_http_json_querystore)" == "Succeeded"
|
||
&& "$(check_releases)" == "Succeeded"
|
||
&& ("$(blackduck_scan)" == "Succeeded" || "$(blackduck_scan)" == "Skipped")
|
||
&& ("$(run_notices_pr_build)" == "Succeeded" || "$(run_notices_pr_build)" == "Skipped") ]]; then
|
||
MSG="Daily tests passed: $COMMIT_LINK"
|
||
REPORT='```
|
||
speedy_perf: $(speedy_perf)
|
||
$(oracle_perf_results)
|
||
$(postgres_perf_results)
|
||
```
|
||
'
|
||
tell_slack "$(echo -e "$MSG\n$REPORT")" "$(Slack.ci-failures-daml)"
|
||
else
|
||
tell_slack "Daily tests failed: $COMMIT_LINK." "$(Slack.ci-failures-daml)"
|
||
fi
|