Run costly tests after only after merging (#17956)

* do not run pr-only tests on main, do not run main-only tests on prs

* split data dep tests into main-only and pr-only

* run non-dev conformance tests on main only

azure-pipelines.yml

@@ -18,6 +18,8 @@ pr: none
 
 jobs:
 - template: ci/build.yml
+  parameters:
+    test_mode: main
 - template: ci/check-for-release-job.yml
 
 - job: release

build.ps1

@@ -74,6 +74,14 @@ if ($env:SKIP_TESTS -ceq "False") {
       | Out-File -Encoding UTF8 -NoNewline scala-test-suite-name-map.json
 
     $tag_filter = "-dev-canton-test"
+    switch ($env:TEST_MODE) {
+      'main' { $tag_filter = "$tag_filter,-pr-only" }
+      'pr'   { $tag_filter = "$tag_filter,-main-only" }
+      Default {
+        Write-Output "<< unknown test mode: $env:TEST_MODE)"
+        throw ("Was given an unknown test mode: $env:TEST_MODE")
+      }
+    }
 
     bazel test //... `
       `-`-build_tag_filters "$tag_filter,-canton-ee" `

build.sh

@@ -13,6 +13,21 @@ export LC_ALL=en_US.UTF-8
 ARTIFACT_DIRS="${BUILD_ARTIFACTSTAGINGDIRECTORY:-$PWD}"
 mkdir -p "${ARTIFACT_DIRS}/logs"
 
+case $3 in
+  # When running against main, exclude "pr-only" tests
+  main)
+    tag_filter="-pr-only"
+    ;;
+  # When running against a PR, exclude "main-only" tests
+  pr)
+    tag_filter="-main-only"
+    ;;
+  *)
+    echo "unknown test mode: $3"
+    exit 1
+    ;;
+esac
+
 tag_filter=""
 if [[ "$(uname)" == "Darwin" ]]; then
   tag_filter="$tag_filter,-dont-run-on-darwin,-scaladoc,-pdfdocs"

ci/build-unix.yml

@@ -6,6 +6,7 @@ parameters:
   name: ''
   release_tag: ''
   scala_version: ''
+  test_mode: ''
 
 steps:
   - bash: ci/dev-env-install.sh
@@ -52,10 +53,11 @@ steps:
       set -euo pipefail
       p="_${{parameters.name}}"
       a="_$(Build.BuildNumber)_$(System.JobAttempt)"
+      t="${{parameters.test_mode}}"
       if [ "$p" == "_m1" ]; then
-          ./build.sh $p $a || ./build.sh $p $a || ./build.sh $p $a || ./build.sh $p $a
+          ./build.sh $p $a $t || ./build.sh $p $a $t || ./build.sh $p $a $t || ./build.sh $p $a $t 
       else
-          ./build.sh $p $a
+          ./build.sh $p $a $t 
       fi
     displayName: 'Build'
     env:

ci/build-windows.yml

@@ -6,6 +6,7 @@ parameters:
   is_release: ''
   is_split_release: 'false'
   skip_tests: ''
+  test_mode: ''
 
 steps:
   - bash: ci/configure-bazel.sh
@@ -41,6 +42,7 @@ steps:
       #   part of the release PR that triggered the build (with correct version
       #   number).
       SKIP_TESTS: ${{parameters.skip_tests}}
+      TEST_MODE: ${{parameters.test_mode}}
 
   - template: publish-platform-independence-dar.yml
     parameters:

ci/build.yml

@@ -1,8 +1,10 @@
 # Copyright (c) 2023 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-jobs:
+parameters:
+  test_mode: ''
 
+jobs:
 - job: git_sha
   pool:
     name: 'ubuntu_20_04'
@@ -60,6 +62,7 @@ jobs:
         release_tag: $(release_tag)
         name: 'linux'
         is_release: variables.is_release
+        test_mode: ${{parameters.test_mode}}
     - template: upload-bazel-metrics.yml
     - bash: |
         set -euo pipefail
@@ -77,12 +80,14 @@ jobs:
     job_name: macOS
     name: macos
     assignment: default
+    test_mode: ${{parameters.test_mode}}
 
 - template: macOS.yml
   parameters:
     job_name: m1
     name: m1
     assignment: m1-builds
+    test_mode: ${{parameters.test_mode}}
 
 - template: blackduck.yml
 
@@ -141,6 +146,7 @@ jobs:
         skip_tests: $(skip_tests)
         is_release: variables.is_release
         is_split_release: $(is_split_release)
+        test_mode: ${{parameters.test_mode}}
     - template: upload-bazel-metrics.yml
     - template: tell-slack-failed.yml
       parameters:

ci/daily-snapshot.yml

@@ -75,6 +75,8 @@ jobs:
       env:
         AUTH: $(ARTIFACTORY_USERNAME):$(ARTIFACTORY_PASSWORD)
 - template: build.yml
+  parameters:
+    test_mode: main
 - template: split-release-job.yml
 - job: release
   condition: false

ci/macOS.yml

@@ -5,6 +5,7 @@ parameters:
 - name: name
 - name: assignment
 - name: job_name
+- name: test_mode
 
 jobs:
 - job: ${{parameters.job_name}}
@@ -45,6 +46,7 @@ jobs:
         release_tag: $(release_tag)
         name: ${{parameters.name}}
         is_release: variables.is_release
+        test_mode: ${{parameters.test_mode}}
     # Do not upload metrics for m1
     - ${{ if eq(parameters.name, 'macos') }}:
       - template: upload-bazel-metrics.yml

ci/prs.yml

@@ -12,6 +12,8 @@ pr:
 
 jobs:
 - template: build.yml
+  parameters:
+    test_mode: pr
 - template: check-for-release-job.yml
 
 

compiler/damlc/tests/BUILD.bazel

@@ -516,10 +516,12 @@ daml_compile(
 )
 
 # Tests for data-dependencies
-da_haskell_test(
-    name = "data-dependencies",
+[
+    da_haskell_test(
+        name = "data-dependencies-{}".format(suffix),
         timeout = "long",
         srcs = ["src/DA/Test/DataDependencies.hs"],
+        args = test_args,
         data = [
             "//compiler/damlc",
             "@damlc_legacy",
@@ -552,7 +554,7 @@ da_haskell_test(
         src_strip_prefix = "src",
         tags = [
             "cpu:3",
-    ],
+        ] + extra_tags,
         visibility = ["//visibility:public"],
         deps = [
             "//:sdk-version-hs-lib",
@@ -564,7 +566,27 @@ da_haskell_test(
             "//libs-haskell/da-hs-base",
             "//libs-haskell/test-utils",
         ],
-)
+    )
+    for (suffix, extra_tags, test_args) in [
+        # Test all LF versions on the main branch, after merging
+        (
+            "all",
+            ["main-only"],
+            [],
+        ),
+        # Don't test LF v2 when running the CI on a PR
+        (
+            "no-v2",
+            ["pr-only"],
+            [
+                "-p",
+                # Test paths for these tests are of the form "foo.{lf-version}.bar"
+                # This pattern excludes tests whose lf-version starts with "2."
+                "'$$2 !~ /2./'",
+            ],
+        ),
+    ]
+]
 
 [
     [

ledger-test-tool/tool/BUILD.bazel

@@ -202,6 +202,7 @@ conformance_test(
             "_shared.participant_ledger-api_tls.minimum-server-protocol-version=TLSv1.3",
         ]),
     ],
+    tags = ["main-only"],
     test_tool_args = conformance_test_tool_args_base + [
         "--exclude=" + ",".join(conformance_test_excluded_test),
         "--additional=TLSOnePointThreeIT",
@@ -223,6 +224,7 @@ conformance_test(
             "dev-mode=no",
         ]),
     ],
+    tags = ["main-only"],
     test_tool_args = conformance_test_tool_args_base + ["--include=TLSAtLeastOnePointTwoIT"],
 ) if not is_windows else None