better release triggers (#6859)

Based on feedback from @nickchapman-da, this PR aims at making the
release process easier by:

- Automatically opening a release PR on Wednesday morning. The goal here
  is that by the time we start working, there is a release already
  built, so we save about an hour on waiting for that. This obviously
  doesn't help with ad-hoc releases.
- On a release PR build, posting to Slack when the release is ready to
  merge.
- On a release master build, posting to Slack when a release is ready to
  be tested.

My hope is that this makes the release process less tedious. This is not
trying to address the actual release testing, but hopefully should
reduce the annoyance of having to constantly go and check if the release
is ready.

CHANGELOG_BEGIN
CHANGELOG_END

azure-pipelines.yml

@@ -373,7 +373,7 @@ jobs:
       - checkout: self
         persistCredentials: true
       - bash: |
-          set -euxo pipefail
+          set -euo pipefail
           if git tag v$(release_tag) $(release_sha); then
             git push origin v$(release_tag)
             mkdir $(Build.StagingDirectory)/release
@@ -478,6 +478,17 @@ jobs:
         env:
           GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
         condition: not(eq(variables['skip-github'], 'TRUE'))
+      - bash: |
+          set -euo pipefail
+          eval "$(dev-env/bin/dade-assist)"
+
+          curl -XPOST \
+               -i \
+               -H 'Content-Type: application/json' \
+               --data "{\"text\":\"Release \`$(release_tag)\` is ready for testing. (<https://dev.azure.com/digitalasset/daml/_build/results?buildId=$(Build.BuildId)|build>, <https://github.com/digital-asset/daml/commit/$(trigger_sha)|trigger commit>, <https://github.com/digital-asset/daml/commit/$(release_sha)|target commit>)\"}" \
+               $(Slack.team-daml)
+          }
+
       - template: ci/tell-slack-failed.yml
         parameters:
           trigger_sha: '$(trigger_sha)'
@@ -536,7 +547,7 @@ jobs:
         git -c user.name="Azure Pipelines DAML Build" \
             -c user.email="support@digitalasset.com" \
             commit \
-            -m "$(printf "update compat versions for $(release_tag)\n\nCHANGELOG_BEGIN\nCHANGELOG_END\n"))"
+            -m "$(printf "update compat versions for $(release_tag)\n\nCHANGELOG_BEGIN\nCHANGELOG_END\n")"
         git push origin $BRANCH:$BRANCH
         curl -H "Content-Type: application/json" \
              -u $AUTH \
@@ -777,6 +788,7 @@ jobs:
     dependsOn:
       - git_sha
       - collect_build_data
+      - check_for_release
     pool:
       name: 'linux-pool'
       demands: assignment -equals default
@@ -784,6 +796,7 @@ jobs:
       pr.num: $[ variables['System.PullRequest.PullRequestNumber'] ]
       branch_sha: $[ dependencies.git_sha.outputs['out.branch'] ]
       status: $[ dependencies.collect_build_data.result ]
+      is_release: $[ dependencies.check_for_release.outputs['out.is_release'] ]
     steps:
       - bash: |
           set -euo pipefail
@@ -798,6 +811,22 @@ jobs:
                    $(Slack.team-daml-ci)
           }
 
+          tell_daml() {
+              local message
+              message=$1
+              curl -XPOST \
+                   -i \
+                   -H 'Content-Type: application/json' \
+                   --data "{\"text\":\"<https://dev.azure.com/digitalasset/daml/_build/results?buildId=$(Build.BuildId)|Build $(Build.BuildId)> for release PR <https://github.com/digital-asset/daml/pull/$(pr.num)|#$(pr.num)> has completed with status ${message}.\"}" \
+                   $(Slack.team-daml)
+          }
+
+          if [ "$(is_release)" == "true" ]; then
+              # releases matter to everyone
+              tell_daml "$(status)"
+              exit 0
+          fi
+
           EMAIL=$(git log -n 1 --format=%ae $(branch_sha))
           user_registered() {
               cat ci/slack_user_ids | grep $EMAIL

ci/cron/wednesday.yml

@@ -0,0 +1,48 @@
+# Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+pr: none
+trigger: none
+
+schedules:
+- cron: "0 6 * * Wed"
+  displayName: weekly snapshot
+  branches:
+    include:
+    - master
+  always: true
+
+jobs:
+- job: open_release_pr
+  timeoutInMinutes: 60
+  pool:
+    name: linux-pool
+    demands: assignment -equals default
+  steps:
+  - checkout: self
+    persistCredentials: true
+  - bash: ci/dev-env-install.sh
+  - bash: |
+      set -euo pipefail
+      eval "$(./dev-env/bin/dade-assist)"
+
+      AUTH="$(git config remote.origin.url | grep -o "://.*:.*@" | cut -c4- | rev | cut -c2- | rev)"
+
+      BRANCH=auto-release-pr-$(date -I)
+      git branch -D $BRANCH || true
+      git checkout -b $BRANCH
+      ./release.sh new snapshot
+      git add LATEST
+      RELEASE=$(head -1 LATEST | awk '{print $2}')
+      git -c user.name="Azure Pipelines DAML Build" \
+          -c user.email="support@digitalasset.com" \
+          commit \
+          -m "$(printf "release $RELEASE\n\nCHANGELOG_BEGIN\nCHANGELOG_END\n")"
+      git push origin $BRANCH:$BRANCH
+      curl -H "Content-Type: application/json" \
+           -u $AUTH \
+           --silent \
+           --include \
+           --location \
+           -d "{\"title\": \"release $RELEASE\", \"head\": \"$BRANCH\", \"base\": \"master\", \"body\": \"This PR has been created by a script, which is not very smart and does not have all the context. Please do double-check that the version prefix is correct before merging.\"}" \
+           https://api.github.com/repos/digital-asset/daml/pulls

release.sh

@@ -74,6 +74,11 @@ $0 snapshot SHA PREFIX
 
         Any non-ambiguous git commit reference can be given as SHA.
 
+$0 new snapshot
+        Updates LATEST to add current commit as a new snapshot. Figures out
+        prefix version by keeping the same if the first line in LATEST is a
+        snapshot, and incrementing minor if the first line is stable.
+
 $0 check
         Checks that each line of the LATEST file is well-formed.
 
@@ -98,6 +103,43 @@ commit_belongs_to_release_branch() {
     return 1
 }
 
+new_snapshot () {
+    local sha latest latest_prefix prefix new_line tmp
+    sha=$(git rev-parse HEAD)
+    if ! commit_belongs_to_release_branch $sha; then
+        echo "WARNING: Commit does not belong to a release branch."
+    fi
+    latest=$(head -1 LATEST | awk '{print $2}')
+    latest_prefix=$(echo $latest | grep -o -P "$STABLE_REGEX" | head -1)
+    if is_stable $latest; then
+        # As part of our normal processes, this case should never happen.
+        # Versions in the LATEST file are supposed to be ordered (highest
+        # version at the top), and we're supposed to start creating 1.5
+        # snapshots before we have fully settled on the 1.4 stable. So the
+        # normal case for the top two lines of the LATEST file is either two
+        # snapshots or one snapshot then one stable.
+        #
+        # Still, if it does encounter that case, the only sensible thing the
+        # script can do is bump the version. (Well, or exit 1, I guess.)
+        prefix=$(echo $latest_prefix | jq -Rr '. | split(".") | [.[0], (.[1] | tonumber + 1 | tostring), "0"] | join(".")')
+    else
+        prefix=$latest_prefix
+    fi
+    new_line=$(make_snapshot $sha $prefix)
+    tmp=$(mktemp)
+    cp LATEST $tmp
+    if is_stable $latest; then
+        # This case should not happen (see above), but if it does, we need to
+        # add the new snapshot while keeping the existing stable.
+        cat <(echo $new_line) $tmp > LATEST
+    else
+        # This is the only case consistent with all of our other processes
+        # working as expected: top of LATEST is a snapshot and we replace it
+        # with the new one.
+        cat <(echo $new_line) <(tail -n +2 $tmp) > LATEST
+    fi
+}
+
 case $1 in
     snapshot)
         if [ -n "${2+x}" ] && [ -n "${3+x}" ]; then
@@ -109,6 +151,13 @@ case $1 in
             display_help
         fi
     ;;
+    new)
+        if [ $# -eq 2 ] && [ "$2" == 'snapshot' ]; then
+            new_snapshot
+        else
+            display_help
+        fi
+    ;;
     check)
         check
     ;;