add daily perf report (#5843)

This PR adds a simple daily job that runs the performance test on a
chosen "baseline" commit and then runs the same benchmark on latest
master. This should allow us to track overall performance improvements.

CHANGELOG_BEGIN
CHANGELOG_END

ci/cron/compare-perf.sh

@@ -0,0 +1,33 @@
+# Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+BASELINE=$1
+
+measure() {
+    local treeish=$1
+    local out=$(mktemp -d)/out.json
+    git checkout $treeish >&2
+    bazel run daml-lf/scenario-interpreter:scenario-perf -- -rf json -rff $out >&2
+    cat $out | jq '.[0].primaryMetric.score'
+}
+
+main() {
+  local current=$(git rev-parse HEAD)
+
+  local baseline_perf=$(measure $BASELINE)
+  local current_perf=$(measure $current)
+
+  git checkout $current >&2
+
+  local factor=$(echo "$current_perf / $baseline_perf" | bc -l)
+  local speedup=$(printf "%.1f" $(echo "($factor - 1) * 100" | bc -l))
+  local progress=$(printf "%05.2f%%" $(echo "100 / (l(5) / l($factor))" | bc -l))
+
+  echo '{"current-perf": '$current_perf', "baseline-perf": '$baseline_perf', "improvement": '$speedup', "progress_towards_5x": "'$progress'", "current-sha": "'$current'", "baseline-sha": "'$BASELINE'"}'
+}
+
+main

ci/cron/daily-compat.yml

@@ -16,7 +16,7 @@ trigger: none
 # Targeting 6AM UTC seems to fit that.
 schedules:
 - cron: "0 6 * * *"
-  displayName: daily compat check
+  displayName: daily checks and reporting
   branches:
     include:
     - master
@@ -80,3 +80,49 @@ jobs:
                $(Slack.team-daml)
         displayName: report
         condition: always()
+  - job: performance_report
+    timeoutInMinutes: 120
+    pool:
+      name: "linux-pool"
+    steps:
+      - checkout: self
+      - bash: ci/dev-env-install.sh
+        displayName: 'Build/Install the Developer Environment'
+      - bash: ci/configure-bazel.sh
+        displayName: 'Configure Bazel for root workspace'
+        env:
+          IS_FORK: $(System.PullRequest.IsFork)
+          # to upload to the bazel cache
+          GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
+      - bash: |
+          set -euo pipefail
+          eval "$(dev-env/bin/dade assist)"
+          BASELINE="cebc26af88efef4a7c81c62b0c14353f829b755e"
+          OUT="$(Build.StagingDirectory)/perf-results.json"
+          if git diff --exit-code $BASELINE -- daml-lf/scenario-interpreter/src/perf >&2; then
+              # no changes, all good
+              ci/cron/compare-perf.sh $BASELINE > $OUT
+          else
+              # the tests have changed, we need to figure out what to do with
+              # the baseline.
+              echo "Baseline no longer valid, needs manual correction." > $OUT
+          fi
+        displayName: measure perf
+      - bash: |
+          set -euo pipefail
+          eval "$(dev-env/bin/dade assist)"
+          cat $(Build.StagingDirectory)/perf-results.json
+          COMMIT_TITLE=$(git log --pretty=format:%s -n1)
+          COMMIT_LINK="<https://dev.azure.com/digitalasset/daml/_build/results?buildId=$(Build.BuildId)|$COMMIT_TITLE>"
+          if [ "$(Agent.JobStatus)" != "Succeeded" ]; then
+              MESSAGE="{\"text\": \":fire: <!here> :fire: :fire:\nperf *FAILED*: $COMMIT_LINK\n:fire: :fire:\"}"
+          else
+              MESSAGE=$(cat $(Build.StagingDirectory)/perf-results.json | jq . | jq -sR '{"text": "perf for '"$COMMIT_LINK"':```\(.)```"}')
+          fi
+          curl -XPOST \
+               -i \
+               -H 'Content-type: application/json' \
+               --data "$MESSAGE" \
+               $(Slack.team-daml)
+        displayName: report
+        condition: always()

dev-env/bin/bc

@@ -0,0 +1 @@
+../lib/dade-exec-nix-tool

nix/default.nix

@@ -203,6 +203,7 @@ in rec {
     sha1sum = pkgs.coreutils;
     xmlstarlet = pkgs.xmlstarlet;
     grep = pkgs.gnugrep;
+    bc = pkgs.bc;
 
     # Cryptography tooling
     gnupg = pkgs.gnupg;