Run costly tests after only after merging (#17956)

* do not run pr-only tests on main, do not run main-only tests on prs

* split data dep tests into main-only and pr-only

* run non-dev conformance tests on main only

azure-pipelines.yml

@@ -18,6 +18,8 @@ pr: none
 
 jobs:
 - template: ci/build.yml
+  parameters:
+    test_mode: main
 - template: ci/check-for-release-job.yml
 
 - job: release

build.ps1

@@ -74,6 +74,14 @@ if ($env:SKIP_TESTS -ceq "False") {
       | Out-File -Encoding UTF8 -NoNewline scala-test-suite-name-map.json
 
     $tag_filter = "-dev-canton-test"
+    switch ($env:TEST_MODE) {
+      'main' { $tag_filter = "$tag_filter,-pr-only" }
+      'pr'   { $tag_filter = "$tag_filter,-main-only" }
+      Default {
+        Write-Output "<< unknown test mode: $env:TEST_MODE)"
+        throw ("Was given an unknown test mode: $env:TEST_MODE")
+      }
+    }
 
     bazel test //... `
       `-`-build_tag_filters "$tag_filter,-canton-ee" `

build.sh

@@ -13,6 +13,21 @@ export LC_ALL=en_US.UTF-8
 ARTIFACT_DIRS="${BUILD_ARTIFACTSTAGINGDIRECTORY:-$PWD}"
 mkdir -p "${ARTIFACT_DIRS}/logs"
 
+case $3 in
+  # When running against main, exclude "pr-only" tests
+  main)
+    tag_filter="-pr-only"
+    ;;
+  # When running against a PR, exclude "main-only" tests
+  pr)
+    tag_filter="-main-only"
+    ;;
+  *)
+    echo "unknown test mode: $3"
+    exit 1
+    ;;
+esac
+
 tag_filter=""
 if [[ "$(uname)" == "Darwin" ]]; then
   tag_filter="$tag_filter,-dont-run-on-darwin,-scaladoc,-pdfdocs"

ci/build-unix.yml

@@ -6,6 +6,7 @@ parameters:
   name: ''
   release_tag: ''
   scala_version: ''
+  test_mode: ''
 
 steps:
   - bash: ci/dev-env-install.sh
@@ -52,10 +53,11 @@ steps:
       set -euo pipefail
       p="_${{parameters.name}}"
       a="_$(Build.BuildNumber)_$(System.JobAttempt)"
+      t="${{parameters.test_mode}}"
       if [ "$p" == "_m1" ]; then
-          ./build.sh $p $a || ./build.sh $p $a || ./build.sh $p $a || ./build.sh $p $a
+          ./build.sh $p $a $t || ./build.sh $p $a $t || ./build.sh $p $a $t || ./build.sh $p $a $t 
       else
-          ./build.sh $p $a
+          ./build.sh $p $a $t 
       fi
     displayName: 'Build'
     env:

ci/build-windows.yml

@@ -6,6 +6,7 @@ parameters:
   is_release: ''
   is_split_release: 'false'
   skip_tests: ''
+  test_mode: ''
 
 steps:
   - bash: ci/configure-bazel.sh
@@ -41,6 +42,7 @@ steps:
       #   part of the release PR that triggered the build (with correct version
       #   number).
       SKIP_TESTS: ${{parameters.skip_tests}}
+      TEST_MODE: ${{parameters.test_mode}}
 
   - template: publish-platform-independence-dar.yml
     parameters:

ci/build.yml

@@ -1,8 +1,10 @@
 # Copyright (c) 2023 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-jobs:
+parameters:
+  test_mode: ''
 
+jobs:
 - job: git_sha
   pool:
     name: 'ubuntu_20_04'
@@ -60,6 +62,7 @@ jobs:
         release_tag: $(release_tag)
         name: 'linux'
         is_release: variables.is_release
+        test_mode: ${{parameters.test_mode}}
     - template: upload-bazel-metrics.yml
     - bash: |
         set -euo pipefail
@@ -77,12 +80,14 @@ jobs:
     job_name: macOS
     name: macos
     assignment: default
+    test_mode: ${{parameters.test_mode}}
 
 - template: macOS.yml
   parameters:
     job_name: m1
     name: m1
     assignment: m1-builds
+    test_mode: ${{parameters.test_mode}}
 
 - template: blackduck.yml
 
@@ -141,6 +146,7 @@ jobs:
         skip_tests: $(skip_tests)
         is_release: variables.is_release
         is_split_release: $(is_split_release)
+        test_mode: ${{parameters.test_mode}}
     - template: upload-bazel-metrics.yml
     - template: tell-slack-failed.yml
       parameters:

ci/daily-snapshot.yml

@@ -75,6 +75,8 @@ jobs:
       env:
         AUTH: $(ARTIFACTORY_USERNAME):$(ARTIFACTORY_PASSWORD)
 - template: build.yml
+  parameters:
+    test_mode: main
 - template: split-release-job.yml
 - job: release
   condition: false

ci/macOS.yml

@@ -5,6 +5,7 @@ parameters:
 - name: name
 - name: assignment
 - name: job_name
+- name: test_mode
 
 jobs:
 - job: ${{parameters.job_name}}
@@ -45,6 +46,7 @@ jobs:
         release_tag: $(release_tag)
         name: ${{parameters.name}}
         is_release: variables.is_release
+        test_mode: ${{parameters.test_mode}}
     # Do not upload metrics for m1
     - ${{ if eq(parameters.name, 'macos') }}:
       - template: upload-bazel-metrics.yml

ci/prs.yml

@@ -12,6 +12,8 @@ pr:
 
 jobs:
 - template: build.yml
+  parameters:
+    test_mode: pr
 - template: check-for-release-job.yml
 
 

compiler/damlc/tests/BUILD.bazel

@@ -516,55 +516,77 @@ daml_compile(
 )
 
 # Tests for data-dependencies
-da_haskell_test(
-    name = "data-dependencies",
-    timeout = "long",
-    srcs = ["src/DA/Test/DataDependencies.hs"],
-    data = [
-        "//compiler/damlc",
-        "@damlc_legacy",
-        "//compiler/damlc/tests:generate-simple-dalf",
-        "//daml-script/daml:daml-script-1.dev.dar",
-        "//daml-script/daml3:daml3-script-2.dev.dar",
-        # Feel free to update this to 0.13.55 once that is frozen.
-        ":dars/old-proj-0.13.55-snapshot.20200309.3401.0.6f8c3ad8-1.8.dar",
+[
+    da_haskell_test(
+        name = "data-dependencies-{}".format(suffix),
+        timeout = "long",
+        srcs = ["src/DA/Test/DataDependencies.hs"],
+        args = test_args,
+        data = [
+            "//compiler/damlc",
+            "@damlc_legacy",
+            "//compiler/damlc/tests:generate-simple-dalf",
+            "//daml-script/daml:daml-script-1.dev.dar",
+            "//daml-script/daml3:daml3-script-2.dev.dar",
+            # Feel free to update this to 0.13.55 once that is frozen.
+            ":dars/old-proj-0.13.55-snapshot.20200309.3401.0.6f8c3ad8-1.8.dar",
 
-        # TODO(https://github.com/digital-asset/daml/issues/17366)
-        # During development of LF 2.x this is generated using the HEAD compiler.
-        # Once we release LF 2.1, we should replace this with a dar file
-        # produced by the compiler corresponding to that release.
-        ":old-proj-2.1.dar",
-    ],
-    hackage_deps = [
-        "base",
-        "bytestring",
-        "extra",
-        "ghc-lib-parser",
-        "filepath",
-        "process",
-        "safe",
-        "safe-exceptions",
-        "tasty",
-        "tasty-hunit",
-        "zip-archive",
-    ],
-    main_function = "DA.Test.DataDependencies.main",
-    src_strip_prefix = "src",
-    tags = [
-        "cpu:3",
-    ],
-    visibility = ["//visibility:public"],
-    deps = [
-        "//:sdk-version-hs-lib",
-        "//compiler/daml-lf-ast",
-        "//compiler/daml-lf-proto",
-        "//compiler/daml-lf-reader",
-        "//compiler/damlc/stable-packages:stable-packages-lib",
-        "//libs-haskell/bazel-runfiles",
-        "//libs-haskell/da-hs-base",
-        "//libs-haskell/test-utils",
-    ],
-)
+            # TODO(https://github.com/digital-asset/daml/issues/17366)
+            # During development of LF 2.x this is generated using the HEAD compiler.
+            # Once we release LF 2.1, we should replace this with a dar file
+            # produced by the compiler corresponding to that release.
+            ":old-proj-2.1.dar",
+        ],
+        hackage_deps = [
+            "base",
+            "bytestring",
+            "extra",
+            "ghc-lib-parser",
+            "filepath",
+            "process",
+            "safe",
+            "safe-exceptions",
+            "tasty",
+            "tasty-hunit",
+            "zip-archive",
+        ],
+        main_function = "DA.Test.DataDependencies.main",
+        src_strip_prefix = "src",
+        tags = [
+            "cpu:3",
+        ] + extra_tags,
+        visibility = ["//visibility:public"],
+        deps = [
+            "//:sdk-version-hs-lib",
+            "//compiler/daml-lf-ast",
+            "//compiler/daml-lf-proto",
+            "//compiler/daml-lf-reader",
+            "//compiler/damlc/stable-packages:stable-packages-lib",
+            "//libs-haskell/bazel-runfiles",
+            "//libs-haskell/da-hs-base",
+            "//libs-haskell/test-utils",
+        ],
+    )
+    for (suffix, extra_tags, test_args) in [
+        # Test all LF versions on the main branch, after merging
+        (
+            "all",
+            ["main-only"],
+            [],
+        ),
+        # Don't test LF v2 when running the CI on a PR
+        (
+            "no-v2",
+            ["pr-only"],
+            [
+                "-p",
+                # Test paths for these tests are of the form "foo.{lf-version}.bar"
+                # This pattern excludes tests whose lf-version starts with "2."
+                "'$$2 !~ /2./'",
+            ],
+        ),
+    ]
+]
 
 [
     [

ledger-test-tool/tool/BUILD.bazel

@@ -202,6 +202,7 @@ conformance_test(
             "_shared.participant_ledger-api_tls.minimum-server-protocol-version=TLSv1.3",
         ]),
     ],
+    tags = ["main-only"],
     test_tool_args = conformance_test_tool_args_base + [
         "--exclude=" + ",".join(conformance_test_excluded_test),
         "--additional=TLSOnePointThreeIT",
@@ -223,6 +224,7 @@ conformance_test(
             "dev-mode=no",
         ]),
     ],
+    tags = ["main-only"],
     test_tool_args = conformance_test_tool_args_base + ["--include=TLSAtLeastOnePointTwoIT"],
 ) if not is_windows else None