diff --git a/ci/cron/compare-perf.sh b/ci/cron/compare-perf.sh new file mode 100755 index 00000000000..81310b3ebc0 --- /dev/null +++ b/ci/cron/compare-perf.sh @@ -0,0 +1,33 @@ +# Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +#!/usr/bin/env bash + +set -euo pipefail + +BASELINE=$1 + +measure() { + local treeish=$1 + local out=$(mktemp -d)/out.json + git checkout $treeish >&2 + bazel run daml-lf/scenario-interpreter:scenario-perf -- -rf json -rff $out >&2 + cat $out | jq '.[0].primaryMetric.score' +} + +main() { + local current=$(git rev-parse HEAD) + + local baseline_perf=$(measure $BASELINE) + local current_perf=$(measure $current) + + git checkout $current >&2 + + local factor=$(echo "$current_perf / $baseline_perf" | bc -l) + local speedup=$(printf "%.1f" $(echo "($factor - 1) * 100" | bc -l)) + local progress=$(printf "%05.2f%%" $(echo "100 / (l(5) / l($factor))" | bc -l)) + + echo '{"current-perf": '$current_perf', "baseline-perf": '$baseline_perf', "improvement": '$speedup', "progress_towards_5x": "'$progress'", "current-sha": "'$current'", "baseline-sha": "'$BASELINE'"}' +} + +main diff --git a/ci/cron/daily-compat.yml b/ci/cron/daily-compat.yml index d2d38d3cb4c..ce40253e2b2 100644 --- a/ci/cron/daily-compat.yml +++ b/ci/cron/daily-compat.yml @@ -16,7 +16,7 @@ trigger: none # Targeting 6AM UTC seems to fit that. schedules: - cron: "0 6 * * *" - displayName: daily compat check + displayName: daily checks and reporting branches: include: - master @@ -80,3 +80,49 @@ jobs: $(Slack.team-daml) displayName: report condition: always() + - job: performance_report + timeoutInMinutes: 120 + pool: + name: "linux-pool" + steps: + - checkout: self + - bash: ci/dev-env-install.sh + displayName: 'Build/Install the Developer Environment' + - bash: ci/configure-bazel.sh + displayName: 'Configure Bazel for root workspace' + env: + IS_FORK: $(System.PullRequest.IsFork) + # to upload to the bazel cache + GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT) + - bash: | + set -euo pipefail + eval "$(dev-env/bin/dade assist)" + BASELINE="cebc26af88efef4a7c81c62b0c14353f829b755e" + OUT="$(Build.StagingDirectory)/perf-results.json" + if git diff --exit-code $BASELINE -- daml-lf/scenario-interpreter/src/perf >&2; then + # no changes, all good + ci/cron/compare-perf.sh $BASELINE > $OUT + else + # the tests have changed, we need to figure out what to do with + # the baseline. + echo "Baseline no longer valid, needs manual correction." > $OUT + fi + displayName: measure perf + - bash: | + set -euo pipefail + eval "$(dev-env/bin/dade assist)" + cat $(Build.StagingDirectory)/perf-results.json + COMMIT_TITLE=$(git log --pretty=format:%s -n1) + COMMIT_LINK="" + if [ "$(Agent.JobStatus)" != "Succeeded" ]; then + MESSAGE="{\"text\": \":fire: :fire: :fire:\nperf *FAILED*: $COMMIT_LINK\n:fire: :fire:\"}" + else + MESSAGE=$(cat $(Build.StagingDirectory)/perf-results.json | jq . | jq -sR '{"text": "perf for '"$COMMIT_LINK"':```\(.)```"}') + fi + curl -XPOST \ + -i \ + -H 'Content-type: application/json' \ + --data "$MESSAGE" \ + $(Slack.team-daml) + displayName: report + condition: always() diff --git a/dev-env/bin/bc b/dev-env/bin/bc new file mode 120000 index 00000000000..943ba2d88c9 --- /dev/null +++ b/dev-env/bin/bc @@ -0,0 +1 @@ +../lib/dade-exec-nix-tool \ No newline at end of file diff --git a/nix/default.nix b/nix/default.nix index 421bac1ab34..278d66a3b34 100644 --- a/nix/default.nix +++ b/nix/default.nix @@ -203,6 +203,7 @@ in rec { sha1sum = pkgs.coreutils; xmlstarlet = pkgs.xmlstarlet; grep = pkgs.gnugrep; + bc = pkgs.bc; # Cryptography tooling gnupg = pkgs.gnupg;