daml/ci/cron/daily-compat.yml

# Copyright (c) 2022 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

# Do not run on PRs
pr: none

# Do not run on merge to main
trigger: none

# Do run on a schedule (daily)
#
# Note: machines are killed every day at 4AM UTC, so we need to either:
# - run sufficiently before that that this doesn't get killed, or
# - run sufficiently after that that machines are initialized.
#
# Targeting 6AM UTC seems to fit that.
schedules:
- cron: "0 6 * * *"
  displayName: daily checks and reporting
  branches:
    include:
    - main
  always: true

jobs:
  - job: compatibility_ts_libs
    timeoutInMinutes: 60
    pool:
      name: ubuntu_20_04
      demands: assignment -equals default
    steps:
      - checkout: self
      - template: ../clean-up.yml
      - template: ../compatibility_ts_libs.yml

  - job: compatibility
    dependsOn: compatibility_ts_libs
    timeoutInMinutes: 720
    strategy:
      matrix:
        linux:
          pool: ubuntu_20_04
        macos:
          pool: macOS-pool
    pool:
      name: $(pool)
      demands: assignment -equals default
    steps:
      - checkout: self
      - ${{ if eq(variables['pool'], 'macos-pool') }}:
        - template: ../clear-shared-segments-macos.yml
      - template: ../clean-up.yml
      - template: ../compatibility.yml

  - job: compatibility_windows
    dependsOn: compatibility_ts_libs
    timeoutInMinutes: 720
    pool:
      name: windows-pool
      demands: assignment -equals default
    steps:
      - checkout: self
      - template: ../compatibility-windows.yml
      - task: PublishBuildArtifacts@1
        condition: succeededOrFailed()
        inputs:
          pathtoPublish: '$(Build.StagingDirectory)'
          artifactName: 'Bazel Compatibility Logs'

  - job: perf_speedy
    timeoutInMinutes: 120
    pool:
      name: "ubuntu_20_04"
      demands: assignment -equals default
    steps:
      - checkout: self
      - bash: ci/dev-env-install.sh
        displayName: 'Build/Install the Developer Environment'
      - bash: ci/configure-bazel.sh
        displayName: 'Configure Bazel for root workspace'
        env:
          IS_FORK: $(System.PullRequest.IsFork)
          # to upload to the bazel cache
          GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
      - template: ../bash-lib.yml
        parameters:
          var_name: bash_lib
      - bash: |
          set -euo pipefail
          eval "$(dev-env/bin/dade assist)"
          source $(bash_lib)

          OUT="$(Build.StagingDirectory)/perf-results-speedy.json"

          START=$(git log -n1 --format=%cd --date=format:%Y%m%d).$(git rev-list --count HEAD).$(Build.BuildId).$(git log -n1 --format=%h --abbrev=8)

          tmp=$(mktemp -d)/out.json
          bazel run daml-lf/scenario-interpreter:scenario-perf -- -rf json -rff $tmp >&2
          perf=$(printf '%.2f' $(cat $tmp | jq '.[0].primaryMetric.score'))
          if [ "" = "$perf" ]; then exit 1; fi
          sha=$(git rev-parse HEAD)
          echo '{"current-perf": '$perf', "current-sha": "'$sha'"}' > "$OUT"
          cat "$OUT"

          gcs "$GCRED" cp "$OUT" gs://daml-data/perf/speedy/$START.json

          setvar speedy_perf "$perf ms/op"          

        displayName: measure perf
        name: out
        env:
          GCRED: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
      - task: PublishPipelineArtifact@0
        inputs:
          targetPath: $(Build.StagingDirectory)/perf-results-speedy.json
          artifactName: perf-speedy

  - job: perf_http_json
    timeoutInMinutes: 120
    pool:
      name: "ubuntu_20_04"
      demands: assignment -equals default
    steps:
      - checkout: self
      - bash: ci/dev-env-install.sh
        displayName: 'Build/Install the Developer Environment'
      - bash: ci/configure-bazel.sh
        displayName: 'Configure Bazel for root workspace'
        env:
          IS_FORK: $(System.PullRequest.IsFork)
          # to upload to the bazel cache
          GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
      - template: ../bash-lib.yml
        parameters:
          var_name: bash_lib
      - bash: |
          set -euo pipefail
          eval "$(dev-env/bin/dade assist)"
          source $(bash_lib)

          SCENARIOS="\
           com.daml.http.perf.scenario.CreateCommand \
           com.daml.http.perf.scenario.ExerciseCommand \
           com.daml.http.perf.scenario.CreateAndExerciseCommand \
           com.daml.http.perf.scenario.AsyncQueryConstantAcs \
           com.daml.http.perf.scenario.SyncQueryConstantAcs \
           com.daml.http.perf.scenario.SyncQueryNewAcs \
           com.daml.http.perf.scenario.SyncQueryVariableAcs \
          "

          bazel build //docs:quickstart-model
          DAR="${PWD}/bazel-bin/docs/quickstart-model.dar"

          START=$(git log -n1 --format=%cd --date=format:%Y%m%d).$(git rev-list --count HEAD).$(Build.BuildId).$(git log -n1 --format=%h --abbrev=8)
          REPORT_ID="http_json_perf_results_${START}"
          OUT="$(Build.StagingDirectory)/${REPORT_ID}"

          for scenario in $SCENARIOS; do
            bazel run //ledger-service/http-json-perf:http-json-perf-binary-ce -- \
            --scenario=${scenario} \
            --dars=${DAR} \
            --reports-dir=${OUT}
          done

          GZIP=-9 tar -zcvf ${OUT}.tgz ${OUT}

          gcs "$GCRED" cp "$OUT.tgz" "gs://daml-data/perf/http-json/${REPORT_ID}.tgz"          

        displayName: measure http-json performance
        env:
          GCRED: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)

  - job: perf_http_json_querystore
    timeoutInMinutes: 120
    strategy:
      matrix:
        postgres:
          querystore: postgres
        oracle:
          querystore: oracle
    pool:
      name: "ubuntu_20_04"
      demands: assignment -equals default
    steps:
      - checkout: self
      - bash: ci/dev-env-install.sh
        displayName: 'Build/Install the Developer Environment'
      - bash: ci/configure-bazel.sh
        displayName: 'Configure Bazel for root workspace'
        env:
          IS_FORK: $(System.PullRequest.IsFork)
          # to upload to the bazel cache
          GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
      - template: ../bash-lib.yml
        parameters:
          var_name: bash_lib
      - bash: |
          set -euo pipefail
          eval "$(dev-env/bin/dade assist)"
          source $(bash_lib)

          QUERY_STORE=$(querystore)

          #setup oracle docker vm for perf tests against oracle.
          if [[ "$QUERY_STORE" == "oracle" ]]; then
              docker login --username "$DOCKER_LOGIN" --password "$DOCKER_PASSWORD"
              IMAGE=$(cat ci/oracle_image)
              docker pull $IMAGE
              # Cleanup stray containers that might still be running from
              # another build that didn’t get shut down cleanly.
              docker rm -f oracle || true
              # Oracle does not like if you connect to it via localhost if it’s running in the container.
              # Interestingly it works if you use the external IP of the host so the issue is
              # not the host it is listening on (it claims for that to be 0.0.0.0).
              # --network host is a cheap escape hatch for this.
              docker run -d --rm --name oracle --network host -e ORACLE_PWD=$ORACLE_PWD $IMAGE
              function cleanup() {
              docker rm -f oracle
            }
              trap cleanup EXIT
              testConnection() {
              docker exec oracle bash -c 'sqlplus -L '"$ORACLE_USERNAME"'/'"$ORACLE_PWD"'@//localhost:'"$ORACLE_PORT"'/ORCLPDB1 <<< "select * from dba_users;"; exit $?' >/dev/null
            }

            # dont want to wait forever to test connection , 15m is more than sufficient here.
              declare -xf testConnection
              timeout 15m bash -c 'until testConnection; do echo "Could not connect to Oracle, trying again..." ; sleep 1 ; done'
          fi

          bazel build //ledger-service/http-json-perf/...
          DAR="${PWD}/bazel-bin/ledger-service/http-json-perf/LargeAcs.dar"

          # {
          #  "https://daml.com/ledger-api": {
          #    "ledgerId": "MyLedger",
          #    "applicationId": "foobar",
          #    "actAs": [
          #      "Alice"
          #    ]
          #  }
          # }

          METADATA=$(git log -n1 --format=%cd --date=format:%Y%m%d).$(git rev-list --count HEAD).$(Build.BuildId).$(git log -n1 --format=%h --abbrev=8)
          REPORT_ID="http_json_perf_${QUERY_STORE}_results_${METADATA}"
          OUT="$(Build.StagingDirectory)/${REPORT_ID}"
          LOG_DIR="$(Build.StagingDirectory)/log"

          mkdir -p $LOG_DIR

          # Overall we want to populate the test cache , run some fetch by key queries and then some index queries

          export NUM_RECORDS=100000 # 100k ACS for a single template
          export NUM_QUERIES=10000 # 10k queries in total
          export NUM_READERS=100 # 100 users in parallel.

          READ_PERF_KEYS="\
           fetchByKey \
           fetchByQuery \
          "
          RESULT=""

          if [[ "$QUERY_STORE" == "oracle" ]]
          then
            # We run test cases in isolation and retain data between them.
            TEST_CASES="\
             populateCache \
             fetchByKey \
             fetchByQuery \
            "
            for CASE in $TEST_CASES; do
              RUN_MODE=${CASE} \
              USE_DEFAULT_USER=true \
              RETAIN_DATA=true \
              bazel run //ledger-service/http-json-perf:http-json-perf-binary-ee -- \
                --scenario=com.daml.http.perf.scenario.MultiUserQueryScenario \
                --dars=${DAR} \
                --reports-dir=${OUT}/${CASE} \
                --query-store-index=${QUERY_STORE} > "${LOG_DIR}/${CASE}_log.out"
            done
            for KEY in $READ_PERF_KEYS; do
              # capture the avg, stddev, p90, p99, requests_per_second numbers from gatling summary csv
              perf=$(cat ${OUT}/${KEY}/*/summary.csv | tail -n 1 | awk -F, '{printf "%.1f, %.1f, %.1f, %.1f, %.1f", $11, $12, $6, $8, $13 }')
              RESULT="${RESULT}json_${QUERY_STORE}/${KEY}: $perf\n"
            done
          elif [[ "$QUERY_STORE" == "postgres" ]]
          then
            # the test case to run in MultiUserQueryScenario for postgres
            TEST_CASE="populateAndFetch"

            RUN_MODE=${TEST_CASE} \
            bazel run //ledger-service/http-json-perf:http-json-perf-binary-ee -- \
            --scenario=com.daml.http.perf.scenario.MultiUserQueryScenario \
            --dars=${DAR} \
            --reports-dir=${OUT}/${TEST_CASE} \
            --query-store-index=${QUERY_STORE} > "${LOG_DIR}/${TEST_CASE}_log.out"

            for KEY in $READ_PERF_KEYS; do
              # capture the avg, stddev, p90, p99, requests_per_second numbers from gatling summary csv
              perf=$(cat ${OUT}/${TEST_CASE}/*/summary.csv | grep -i "$KEY" | awk -F, '{printf "%.1f, %.1f, %.1f, %.1f, %.1f", $11, $12, $6, $8, $13 }')
              RESULT="${RESULT}json_${QUERY_STORE}/${KEY}: $perf\n"
            done
          fi

          RESULT=${RESULT%"\n"}
          setvar ${QUERY_STORE}_perf_results "$RESULT"

          GZIP=-9 tar -zcf ${OUT}.tgz ${OUT}

          gcs "$GCRED" cp "$OUT.tgz" "gs://daml-data/perf/http-json-${QUERY_STORE}/${REPORT_ID}.tgz"          

        displayName: http-json-$(querystore) perf
        name: out
        env:
          GCRED: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
          DOCKER_LOGIN: $(DOCKER_LOGIN)
          DOCKER_PASSWORD: $(DOCKER_PASSWORD)
      - task: PublishPipelineArtifact@0
        inputs:
          targetPath: $(Build.StagingDirectory)/log
          artifactName: perf_http_json_$(querystore)

  - job: check_releases
    timeoutInMinutes: 360
    pool:
      name: ubuntu_20_04
      demands: assignment -equals default
    steps:
      - checkout: self
      - bash: ci/dev-env-install.sh
        displayName: 'Build/Install the Developer Environment'
      - template: ../bash-lib.yml
        parameters:
          var_name: bash_lib
      - bash: |
          set -euo pipefail
          eval "$(dev-env/bin/dade assist)"

          bazel build //ci/cron:cron
          bazel-bin/ci/cron/cron check --bash-lib $(bash_lib) --gcp-creds "$GCRED"          
        displayName: check releases
        env:
          GCRED: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)

  - job: blackduck_scan
    timeoutInMinutes: 1200
    condition: eq(variables['Build.SourceBranchName'], 'main')
    pool:
      name: ubuntu_20_04
      demands: assignment -equals default
    steps:
      - checkout: self
        persistCredentials: true
      - bash: ci/dev-env-install.sh
        displayName: 'Build/Install the Developer Environment'
      - bash: |
          set -euo pipefail
          eval "$(dev-env/bin/dade assist)"
          export LC_ALL=en_US.UTF-8

          bazel build //...
          # Make sure that Bazel query works
          bazel query 'deps(//...)' >/dev/null          
        displayName: 'Build'
      - bash: |
          set -euo pipefail
          eval "$(dev-env/bin/dade-assist)"

          #needs to be specified since blackduck can not scan all bazel
          #dependency types in one go, haskell has to be scanned separatey and
          #code location name uniquely identified to avoid stomping
          BAZEL_DEPENDENCY_TYPE="haskell_cabal_library"

          bash <(curl -s https://raw.githubusercontent.com/DACH-NY/security-blackduck/master/synopsys-detect) \
          ci-build digital-asset_daml $(Build.SourceBranchName) \
          --logging.level.com.synopsys.integration=DEBUG \
          --detect.tools=BAZEL \
          --detect.bazel.target=//... \
          --detect.bazel.dependency.type=${BAZEL_DEPENDENCY_TYPE} \
          --detect.notices.report=true \
          --detect.code.location.name=digital-asset_daml_${BAZEL_DEPENDENCY_TYPE} \
          --detect.timeout=1500          
        displayName: 'Blackduck Bazel Haskell Scan'
        env:
          BLACKDUCK_HUBDETECT_TOKEN: $(BLACKDUCK_HUBDETECT_TOKEN)
      - bash: |
          set -euo pipefail
          eval "$(dev-env/bin/dade-assist)"

          #needs to be specified since blackduck can not scan all bazel
          #dependency types in one go, java has to be scanned separatey and
          #code location name uniquely identified to avoid stomping
          BAZEL_DEPENDENCY_TYPE="maven_install"

          bash <(curl -s https://raw.githubusercontent.com/DACH-NY/security-blackduck/master/synopsys-detect) \
          ci-build digital-asset_daml $(Build.SourceBranchName) \
          --logging.level.com.synopsys.integration=DEBUG \
          --detect.tools=BAZEL \
          --detect.bazel.target=//... \
          --detect.bazel.dependency.type=${BAZEL_DEPENDENCY_TYPE} \
          --detect.notices.report=true \
          --detect.code.location.name=digital-asset_daml_${BAZEL_DEPENDENCY_TYPE} \
          --detect.timeout=1500          
        displayName: 'Blackduck Bazel JVM Scan'
        env:
          BLACKDUCK_HUBDETECT_TOKEN: $(BLACKDUCK_HUBDETECT_TOKEN)
      - bash: |
          set -euo pipefail
          eval "$(dev-env/bin/dade-assist)"

          (cd language-support/ts && yarn install)

          bash <(curl -s https://raw.githubusercontent.com/DACH-NY/security-blackduck/master/synopsys-detect) \
          ci-build digital-asset_daml $(Build.SourceBranchName) \
          --logging.level.com.synopsys.integration=DEBUG \
          --detect.tools=DETECTOR \
          --detect.included.detector.types=YARN,NPM,CLANG \
          --detect.npm.dependency.types.excluded=DEV \
          --detect.yarn.dependency.types.excluded=DEV \
          --detect.follow.symbolic.links=false \
          --detect.excluded.directories=language-support/ts/daml-ledger,language-support/ts/daml-types,language-support/ts/daml-react,language-support/ts/codegen/tests/ts,bazel-out,bazel-bin,.bazel-cache,bazel-testlogs,bazel-daml,bazel-s,node_modules,dev-env,result-*\
          --detect.blackduck.signature.scanner.exclusion.name.patterns=language-support/ts/daml-ledger,language-support/ts/daml-types,language-support/ts/daml-react.bazel-out,bazel-bin,.bazel-cache,bazel-testlogs,bazel-daml,bazel-s,node_modules,dev-env,result-* \
          --detect.detector.search.exclusion.paths=language-support/ts/daml-ledger,language-support/ts/daml-types,language-support/ts/daml-react,bazel-out,bazel-bin,.bazel-cache,bazel-testlogs,bazel-daml,bazel-s,node_modules,dev-env,result-* \
          --detect.notices.report=true \
          --detect.code.location.name=digital-asset_daml_npm \
          --detect.timeout=1500          
        displayName: 'Blackduck Npm Scan'
        env:
          BLACKDUCK_HUBDETECT_TOKEN: $(BLACKDUCK_HUBDETECT_TOKEN)
      - bash: |
          set -euo pipefail
          eval "$(dev-env/bin/dade-assist)"

          bash <(curl -s https://raw.githubusercontent.com/DACH-NY/security-blackduck/master/synopsys-detect) \
          ci-build digital-asset_daml $(Build.SourceBranchName) \
          --logging.level.com.synopsys.integration=DEBUG \
          --detect.tools=DETECTOR \
          --detect.included.detector.types=PIP,POETRY \
          --detect.follow.symbolic.links=false \
          --detect.excluded.directories=bazel-out,bazel-bin,.bazel-cache,bazel-testlogs,bazel-daml,bazel-s,node_modules,dev-env,result-* \
          --detect.blackduck.signature.scanner.exclusion.name.patterns=.bazel-out,bazel-bin,.bazel-cache,bazel-testlogs,bazel-daml,bazel-s,node_modules,dev-env,result-*,*_bazel_vsts* \
          --detect.detector.search.exclusion.paths=bazel-out,bazel-bin,.bazel-cache,bazel-testlogs,bazel-daml,bazel-s,node_modules,dev-env,result-*,*_bazel_vsts* \
          --detect.notices.report=true \
          --detect.code.location.name=digital-asset_daml_python \
          --detect.timeout=1500          
        displayName: 'Blackduck Python Scan'
        env:
          BLACKDUCK_HUBDETECT_TOKEN: $(BLACKDUCK_HUBDETECT_TOKEN)
      - bash: |
          set -euo pipefail
          eval "$(dev-env/bin/dade-assist)"

          bash <(curl -s https://raw.githubusercontent.com/DACH-NY/security-blackduck/master/synopsys-detect) \
          ci-build digital-asset_daml $(Build.SourceBranchName) \
          --logging.level.com.synopsys.integration=DEBUG \
          --detect.tools=DETECTOR \
          --detect.included.detector.types=GIT,GO_MOD,GO_DEP,GO_VNDR,GO_VENDOR,GO_GRADLE \
          --detect.follow.symbolic.links=false \
          --detect.go.path=$(bazel info execution_root)/external/go_sdk/bin/go \
          --detect.excluded.directories=bazel-out,bazel-bin,.bazel-cache,bazel-testlogs,bazel-daml,bazel-s,node_modules,dev-env,result-* \
          --detect.detector.search.exclusion.paths=bazel-out,bazel-bin,.bazel-cache,bazel-testlogs,bazel-daml,bazel-s,node_modules,dev-env,result-*\
          --detect.policy.check.fail.on.severities=MAJOR,CRITICAL,BLOCKER \
          --detect.code.location.name=digital-asset_daml_go \
          --detect.timeout=1500          
        displayName: 'Blackduck Go Scan'
        env:
          BLACKDUCK_HUBDETECT_TOKEN: $(BLACKDUCK_HUBDETECT_TOKEN)
      - template: ../bash-lib.yml
        parameters:
          var_name: bash_lib
      - bash: |
          set -euo pipefail
          eval "$(./dev-env/bin/dade-assist)"
          source $(bash_lib)

          branch="notices-update-$(Build.BuildId)"

          tr -d '\015' <*_Black_Duck_Notices_Report.txt | grep -v digital-asset_daml >NOTICES
          if git diff --exit-code -- NOTICES; then
              echo "NOTICES file already up-to-date."
              setvar need_to_build false
          else
              git add NOTICES
              open_pr "$branch" "update NOTICES file"
              setvar need_to_build true
          fi          
        displayName: open PR
        name: out
        condition: and(succeeded(),
                       eq(variables['Build.SourceBranchName'], 'main'))

  - job: run_notices_pr_build
    timeoutInMinutes: 60
    dependsOn: ["blackduck_scan"]
    condition: and(succeeded(),
                   eq(variables['Build.SourceBranchName'], 'main'))
    pool:
      name: 'ubuntu_20_04'
      demands: assignment -equals default
    variables:
      need_to_build: $[ dependencies.blackduck_scan.outputs['out.need_to_build'] ]
    steps:
      - bash: |
          eval "$(./dev-env/bin/dade-assist)"
          if [ "$(need_to_build)" == "true" ]; then
              branch="notices-update-$(Build.BuildId)"
              az extension add --name azure-devops
              trap "az devops logout" EXIT
              echo "$(System.AccessToken)" | az devops login --org "https://dev.azure.com/digitalasset"
              az pipelines build queue --branch "$branch" \
                                       --definition-name "PRs" \
                                       --org "https://dev.azure.com/digitalasset" \
                                       --project daml
          fi          

  - job: report
    dependsOn: [compatibility_ts_libs, compatibility, compatibility_windows,
                perf_speedy, perf_http_json, perf_http_json_querystore, check_releases,
                blackduck_scan, run_notices_pr_build]
    condition: and(succeededOrFailed(),
                   eq(variables['Build.SourceBranchName'], 'main'))
    pool:
      name: ubuntu_20_04
      demands: assignment -equals default
    variables:
      compatibility_ts_libs: $[ dependencies.compatibility_ts_libs.result ]
      compatibility: $[ dependencies.compatibility.result ]
      compatibility_windows: $[ dependencies.compatibility_windows.result ]
      perf_speedy: $[ dependencies.perf_speedy.result ]
      speedy_perf: $[ dependencies.perf_speedy.outputs['out.speedy_perf'] ]
      perf_http_json: $[ dependencies.perf_http_json.result ]
      perf_http_json_querystore: $[ dependencies.perf_http_json_querystore.result ]
      oracle_perf_results: $[ dependencies.perf_http_json_querystore.outputs['oracle.out.oracle_perf_results'] ]
      postgres_perf_results: $[ dependencies.perf_http_json_querystore.outputs['postgres.out.postgres_perf_results'] ]
      check_releases: $[ dependencies.check_releases.result ]
      blackduck_scan: $[ dependencies.blackduck_scan.result ]
      run_notices_pr_build: $[ dependencies.run_notices_pr_build.result ]
    steps:
    - template: ../bash-lib.yml
      parameters:
        var_name: bash_lib
    - bash: |
        set -euo pipefail

        eval "$(./dev-env/bin/dade-assist)"
        source $(bash_lib)

        COMMIT_TITLE="$(escape_slack "$(git log --pretty=format:%s -n1)")"
        COMMIT_LINK="<https://dev.azure.com/digitalasset/daml/_build/results?buildId=$(Build.BuildId)|${COMMIT_TITLE}>"

        if [[ "$(compatibility_ts_libs)" == "Succeeded"
           && "$(compatibility)" == "Succeeded"
           && "$(compatibility_windows)" == "Succeeded"
           && "$(perf_speedy)" == "Succeeded"
           && "$(perf_http_json)" == "Succeeded"
           && "$(perf_http_json_querystore)" == "Succeeded"
           && "$(check_releases)" == "Succeeded"
           && ("$(blackduck_scan)" == "Succeeded" || "$(blackduck_scan)" == "Skipped")
           && ("$(run_notices_pr_build)" == "Succeeded" || "$(run_notices_pr_build)" == "Skipped") ]]; then
            MSG="Daily tests passed: $COMMIT_LINK"
            REPORT='```
        speedy_perf: $(speedy_perf)
        $(oracle_perf_results)
        $(postgres_perf_results)
        ```
        '
            tell_slack "$(echo -e "$MSG\n$REPORT")" "$(Slack.ci-failures-daml)"
        else
            tell_slack "Daily tests failed: $COMMIT_LINK." "$(Slack.ci-failures-daml)"
        fi