[sync] any canton we want (#18780)

This PR moves the whole "update canton" logic to a script anyone can run
locally, and changes it to be able to get any canton commit we want, or
possibly any dirty local workdir.

.gitignore

@@ -80,3 +80,4 @@ out
 !.vscode/settings.json.default
 
 /.tmp-pg/
+/.canton

ci/cron/daily-compat.yml

@@ -184,48 +184,7 @@ jobs:
           git fetch
           git checkout origin/$(Build.SourceBranchName)
 
-          get_canton_version_for_major_version() {
-            major=$1
-            curl -u $AUTH \
-                 --fail \
-                 --location \
-                 --silent \
-                 https://digitalasset.jfrog.io/artifactory/api/storage/assembly/canton \
-            | jq -r '.children[].uri' \
-            | sed -e 's/^\///' \
-            | grep -P "^${major}\.\d+\.\d+" \
-            | sort -V \
-            | tail -1
-          }
-
-          canton_version_to_commitish() {
-            canton_version=$1
-            STABLE_REGEX="\d+\.\d+\.\d+"
-            SNAPSHOT_REGEX="^${STABLE_REGEX}-snapshot(\.\d+)+\.v[0-9a-f]{8}$"
-            if (echo "$1" | grep -q -P "${SNAPSHOT_REGEX}"); then
-              # Extracting commit (short) sha from snapshot version
-              echo "${canton_version: -8}"
-            else
-              # relying on stable version tag
-              echo "v$canton_version"
-            fi
-          }
-
-          canton3_version=$(get_canton_version_for_major_version 3)
-
-          sed -i 's|SKIP_DEV_CANTON_TESTS=.*|SKIP_DEV_CANTON_TESTS=false|' build.sh
-
-          ### code drop ###
-
-          tmp=$(mktemp -d)
-          trap "rm -rf ${tmp}" EXIT
-
-          git clone https://$GITHUB_TOKEN@github.com/DACH-NY/canton $tmp
-          git -C $tmp checkout $(canton_version_to_commitish $canton3_version)
-
-          ci/copy-canton.sh $tmp
-
-          ### create PR ###
+          ci/refresh-canton.sh
 
           branch="main-canton-update-$canton3_version"
 
@@ -233,7 +192,7 @@ jobs:
               echo "Already up-to-date with latest Canton source & snapshot."
           else
               if [ "main" = "$(Build.SourceBranchName)" ]; then
-                  git add build.sh
+                  git add build.sh canton
                   open_pr "$branch" "update canton to $canton3_version" "tell-slack: canton" "" "$(Build.SourceBranchName)"
                   az extension add --name azure-devops
                   trap "az devops logout" EXIT

ci/refresh-canton.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+# Copyright (c) 2024 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+set -euo pipefail
+
+DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+
+cd "$DIR/.."
+
+if [ -z "${GITHUB_TOKEN:-}" ]; then
+  echo "This script requires GITHUB_TOKEN to be a valid GitHub token with read access to DACH-NY/canton."
+  exit 1
+fi
+
+LOG=$(mktemp)
+
+trap "cat $LOG" EXIT
+
+CANTON_DIR=${1:-//unset}
+
+if [ "//unset" = "$CANTON_DIR" ]; then
+  CANTON_DIR=$(realpath "$DIR/../.canton")
+  echo "Using '$CANTON_DIR' as '\$1' was not provided." >&2
+  if ! [ -d "$CANTON_DIR" ]; then
+    echo "Cloning canton for the first time, this may take a while..." >&2
+    git clone git@github.com:DACH-NY/canton.git "$CANTON_DIR" >$LOG 2>&1
+  fi
+  (
+    cd "$CANTON_DIR"
+    git checkout main >$LOG 2>&1
+    git pull >$LOG 2>&1
+  )
+fi
+
+if ! [ -d "$CANTON_DIR" ]; then
+  echo "CANTON_DIR '$CANTON_DIR' does not seem to exist."
+  exit 1
+fi
+
+sed -i 's|SKIP_DEV_CANTON_TESTS=.*|SKIP_DEV_CANTON_TESTS=false|' "$DIR/../build.sh"
+
+CODE_DROP_DIR="$DIR"/../canton
+for path in community daml-common-staging README.md; do
+  rm -rf "$CODE_DROP_DIR/$path"
+  for f in  $(git -C "$CANTON_DIR" ls-files "$path"); do
+    # we're only interested in copying files, not directories, as git-ls has
+    # explicitly expanded all directories
+    if [[ -f "$CANTON_DIR/$f" ]]; then
+      # we create the parent directories of f under canton/ if they don't exist
+      mkdir -p "$CODE_DROP_DIR/$(dirname $f)"
+      cp "$CANTON_DIR/$f" "$CODE_DROP_DIR/$f"
+    fi
+  done
+done
+
+trap - EXIT