Extend compatibility tests to test against HEAD (#5714)

fixes #5691

changelog_begin
changelog_end

azure-pipelines.yml

@@ -183,25 +183,9 @@ jobs:
     steps:
       - template: ci/report-start.yml
       - checkout: self
-      - bash: ci/dev-env-install.sh
-        displayName: 'Build/Install the Developer Environment'
-      - bash: ci/configure-bazel.sh
-        displayName: 'Configure Bazel'
-        env:
-          BAZEL_CONFIG_DIR: compatibility
-          IS_FORK: $(System.PullRequest.IsFork)
-          # to upload to the bazel cache
-          GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
-      - bash: |
-          set -euo pipefail
-          eval "$(./dev-env/bin/dade-assist)"
-          cd compatibility
-          bazel build //...
-          # Run one test as a sanity check. We do not run all of them
-          # since the test matrix will get large.
-          bazel test \
-            //:ledger-api-test-tool-1.0.1-snapshot.20200417.3908.1.722bac90-platform-1.0.0 \
-            //:ledger-api-test-tool-1.0.1-snapshot.20200417.3908.1.722bac90-platform-1.0.0-postgresql
+      - template: ci/compatibility.yml
+        parameters:
+          test_flags: '--quick'
       - template: ci/tell-slack-failed.yml
       - template: ci/report-end.yml
 

ci/compatibility.yml

@@ -0,0 +1,53 @@
+# Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+parameters:
+  - name: test_flags
+    type: string
+    default: ''
+
+steps:
+  - bash: ci/dev-env-install.sh
+    displayName: 'Build/Install the Developer Environment'
+  - bash: ci/configure-bazel.sh
+    displayName: 'Configure Bazel for root workspace'
+    env:
+      IS_FORK: $(System.PullRequest.IsFork)
+      # to upload to the bazel cache
+      GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
+  - bash: ci/configure-bazel.sh
+    displayName: 'Configure Bazel for compatibility workspace'
+    env:
+      BAZEL_CONFIG_DIR: compatibility
+      IS_FORK: $(System.PullRequest.IsFork)
+      # to upload to the bazel cache
+      GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
+  - bash: |
+      set -euo pipefail
+      ./compatibility/build-release-artifacts.sh $(Build.StagingDirectory)
+    displayName: 'Build release artifacts'
+  # At the moment the indirection of first publishing and then downloading
+  # artifacts is not strictly necessary. However, we have some artifacts
+  # that are platform independent where this could come in handy
+  # (e.g. build the typescript libraries on Linux and fetch them on
+  # windows) so we use this from the start.
+  - task: PublishPipelineArtifact@0
+    inputs:
+      targetPath: $(Build.StagingDirectory)/sdk-release-tarball.tar.gz
+      artifactName: sdk-release-tarball.tar.gz
+  - task: PublishPipelineArtifact@0
+    inputs:
+      targetPath: $(Build.StagingDirectory)/ledger-api-test-tool_deploy.jar
+      artifactName: ledger-api-test-tool_deploy.jar
+  - task: DownloadPipelineArtifact@0
+    inputs:
+      artifactName: sdk-release-tarball.tar.gz
+      targetPath: compatibility/head_sdk
+  - task: DownloadPipelineArtifact@0
+    inputs:
+      artifactName: ledger-api-test-tool_deploy.jar
+      targetPath: compatibility/head_sdk
+  - bash: |
+      set -eou pipefail
+      ./compatibility/test.sh ${{ parameters.test_flags }}
+    displayName: 'Run tests'

ci/cron/daily-compat.yml

@@ -37,20 +37,5 @@ jobs:
       name: $(pool)
     steps:
       - checkout: self
-      - bash: ci/dev-env-install.sh
-        displayName: 'Build/Install the Developer Environment'
-      - bash: |
-          set -euo pipefail
-
-          eval "$(./dev-env/bin/dade-assist)"
-
-          cd compatibility
-          bazel build //...
-          bazel test //...
-
-          # FIXME: fill in the gaps.
-          exit 0
-      # For whatever reason, templates are searched from the path of the
-      # current YML file. I could not find any reference to that in the
-      # documentation.
+      - template: ../compatibility.yml
       - template: ../tell-slack-failed.yml

compatibility/.bazelignore

@@ -0,0 +1 @@
+head-sdk

compatibility/BUILD

@@ -19,12 +19,14 @@ sdk_versions = [
     "1.0.0",
     "1.0.1-snapshot.20200417.3908.1.722bac90",
     "1.1.0-snapshot.20200422.3991.0.6391ee9f",
+    "0.0.0",
 ]
 
 platform_versions = [
     "1.0.0",
     "1.0.1-snapshot.20200417.3908.1.722bac90",
     "1.1.0-snapshot.20200422.3991.0.6391ee9f",
+    "0.0.0",
 ]
 
 [
@@ -48,3 +50,8 @@ platform_versions = [
     for sdk_version in sdk_versions
     for platform_version in platform_versions
 ] if is_linux else None
+
+test_suite(
+    name = "head-quick",
+    tags = ["head-quick"],
+)

compatibility/WORKSPACE

@@ -129,7 +129,17 @@ load("//:bazel-haskell-deps.bzl", "daml_haskell_deps")
 
 daml_haskell_deps()
 
-load("//bazel_tools:daml_sdk.bzl", "daml_sdk")
+load("//bazel_tools:daml_sdk.bzl", "daml_sdk", "daml_sdk_head")
+
+local_repository(
+    name = "head_sdk",
+    path = "head_sdk",
+)
+
+daml_sdk_head(
+    ledger_api_test_tool = "@head_sdk//:ledger-api-test-tool_deploy.jar",
+    sdk_tarball = "@head_sdk//:sdk-release-tarball.tar.gz",
+)
 
 daml_sdk(
     sdk_sha256 = "ee7e2f50394d44fb3490068de64d37f4f87534e802717bd7e07e8767df2e4e05",

compatibility/bazel_tools/daml_sdk.bzl

@@ -14,19 +14,35 @@ source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \
 """
 
 def _daml_sdk_impl(ctx):
-    ctx.download_and_extract(
-        output = "extracted-sdk",
-        # TODO (MK) Make this work on other platforms.
-        url =
-            "https://github.com/digital-asset/daml/releases/download/v{}/daml-sdk-{}-linux.tar.gz".format(ctx.attr.version, ctx.attr.version),
-        sha256 = ctx.attr.sdk_sha256,
-        stripPrefix = "sdk-{}".format(ctx.attr.version),
-    )
-    ctx.download(
-        output = "ledger-api-test-tool.jar",
-        url = "https://repo1.maven.org/maven2/com/daml/ledger-api-test-tool/{}/ledger-api-test-tool-{}.jar".format(ctx.attr.version, ctx.attr.version),
-        sha256 = ctx.attr.test_tool_sha256,
-    )
+    if ctx.attr.sdk_tarball:
+        ctx.extract(
+            ctx.attr.sdk_tarball,
+            output = "extracted-sdk",
+            stripPrefix = "sdk-{}".format(ctx.attr.version),
+        )
+    elif ctx.attr.sdk_sha256:
+        ctx.download_and_extract(
+            output = "extracted-sdk",
+            # TODO (MK) Make this work on other platforms.
+            url =
+                "https://github.com/digital-asset/daml/releases/download/v{}/daml-sdk-{}-linux.tar.gz".format(ctx.attr.version, ctx.attr.version),
+            sha256 = ctx.attr.sdk_sha256,
+            stripPrefix = "sdk-{}".format(ctx.attr.version),
+        )
+    else:
+        fail("Must specify either sdk_tarball or sdk_sha256")
+
+    if ctx.attr.test_tool:
+        ctx.symlink(ctx.attr.test_tool, "ledger-api-test-tool.jar")
+    elif ctx.attr.test_tool_sha256:
+        ctx.download(
+            output = "ledger-api-test-tool.jar",
+            url = "https://repo1.maven.org/maven2/com/daml/ledger-api-test-tool/{}/ledger-api-test-tool-{}.jar".format(ctx.attr.version, ctx.attr.version),
+            sha256 = ctx.attr.test_tool_sha256,
+        )
+    else:
+        fail("Must specify either test_tool or test_tool_sha256")
+
     ctx.extract(
         "ledger-api-test-tool.jar",
         output = "extracted-test-tool",
@@ -102,8 +118,10 @@ _daml_sdk = repository_rule(
     implementation = _daml_sdk_impl,
     attrs = {
         "version": attr.string(mandatory = True),
-        "sdk_sha256": attr.string(mandatory = True),
-        "test_tool_sha256": attr.string(mandatory = True),
+        "sdk_sha256": attr.string(mandatory = False),
+        "sdk_tarball": attr.label(allow_single_file = True, mandatory = False),
+        "test_tool_sha256": attr.string(mandatory = False),
+        "test_tool": attr.label(allow_single_file = True, mandatory = False),
     },
 )
 
@@ -113,3 +131,13 @@ def daml_sdk(version, **kwargs):
         version = version,
         **kwargs
     )
+
+def daml_sdk_head(sdk_tarball, ledger_api_test_tool, **kwargs):
+    version = "0.0.0"
+    _daml_sdk(
+        name = "daml-sdk-{}".format(version),
+        version = version,
+        sdk_tarball = sdk_tarball,
+        test_tool = ledger_api_test_tool,
+        **kwargs
+    )

compatibility/bazel_tools/testing.bzl

@@ -6,6 +6,64 @@ load(
     "client_server_test",
 )
 
+latest_stable = "1.0.0"
+
+# Indexed first by test tool version and then by sandbox version.
+# Note that at this point the granularity for disabling tests
+# is sadly quite coarse. See
+# https://discuss.daml.com/t/can-i-disable-individual-tests-in-the-ledger-api-test-tool/226
+# for details.
+excluded_test_tool_tests = {
+    "1.0.0": {
+        "0.0.0": [
+            # This restriction has been removed in https://github.com/digital-asset/daml/pull/5611.
+            "ContractKeysSubmitterIsMaintainerIT",
+            # Fix for https://github.com/digital-asset/daml/issues/5562
+            "ContractKeysIT",
+        ],
+    },
+    "1.0.1-snapshot.20200417.3908.1.722bac90": {
+        "0.0.0": [
+            # This restriction has been removed in https://github.com/digital-asset/daml/pull/5611.
+            "ContractKeysSubmitterIsMaintainerIT",
+            # Fix for https://github.com/digital-asset/daml/issues/5562
+            "ContractKeysIT",
+        ],
+    },
+    "1.1.0-snapshot.20200422.3991.0.6391ee9f": {
+        "0.0.0": [
+            # This restriction has been removed in https://github.com/digital-asset/daml/pull/5611.
+            "ContractKeysSubmitterIsMaintainerIT",
+            # Fix for https://github.com/digital-asset/daml/issues/5562
+            "ContractKeysIT",
+        ],
+    },
+    "0.0.0": {
+        "1.0.0": [
+            # Fix for https://github.com/digital-asset/daml/issues/5562
+            "ContractKeysIT",
+        ],
+        "1.0.1-snapshot.20200417.3908.1.722bac90": [
+            # Fix for https://github.com/digital-asset/daml/issues/5562
+            "ContractKeysIT",
+        ],
+        "1.1.0-snapshot.20200422.3991.0.6391ee9f": [
+            # Fix for https://github.com/digital-asset/daml/issues/5562
+            "ContractKeysIT",
+        ],
+    },
+}
+
+def get_excluded_tests(test_tool_version, sandbox_version):
+    return excluded_test_tool_tests.get(test_tool_version, default = {}).get(sandbox_version, default = [])
+
+def extra_tags(sdk_version, platform_version):
+    if sorted([sdk_version, platform_version]) == sorted(["0.0.0", latest_stable]):
+        # These tests are the ones that we check on each PR since they
+        # are the most useful ones and hopefully fast enough.
+        return ["head-quick"]
+    return []
+
 def sdk_platform_test(sdk_version, platform_version):
     # SDK components
     ledger_api_test_tool = "@daml-sdk-{sdk_version}//:ledger-api-test-tool".format(
@@ -26,6 +84,7 @@ def sdk_platform_test(sdk_version, platform_version):
         sdk_version = sdk_version,
         platform_version = platform_version,
     )
+    exclusions = ["--exclude=" + test for test in get_excluded_tests(test_tool_version = sdk_version, sandbox_version = platform_version)]
     client_server_test(
         name = name,
         client = ledger_api_test_tool,
@@ -33,7 +92,7 @@ def sdk_platform_test(sdk_version, platform_version):
             "localhost:6865",
             "--open-world",
             "--exclude=ClosedWorldIT",
-        ],
+        ] + exclusions,
         data = [dar_files],
         runner = "@//bazel_tools/client_server:runner",
         runner_args = ["6865"],
@@ -42,7 +101,7 @@ def sdk_platform_test(sdk_version, platform_version):
         server_files = ["$(rootpaths {dar_files})".format(
             dar_files = dar_files,
         )],
-        tags = ["exclusive"],
+        tags = ["exclusive"] + extra_tags(sdk_version, platform_version),
     )
 
     client_server_test(
@@ -52,7 +111,7 @@ def sdk_platform_test(sdk_version, platform_version):
             "localhost:6865",
             "--open-world",
             "--exclude=ClosedWorldIT",
-        ],
+        ] + exclusions,
         data = [dar_files],
         runner = "@//bazel_tools/client_server:runner",
         runner_args = ["6865"],
@@ -61,5 +120,5 @@ def sdk_platform_test(sdk_version, platform_version):
         server_files = ["$(rootpaths {dar_files})".format(
             dar_files = dar_files,
         )],
-        tags = ["exclusive"],
+        tags = ["exclusive"] + extra_tags(sdk_version, platform_version),
     )

compatibility/build-release-artifacts.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+
+# Build the release artifacts required for running the compatibility
+# tests against HEAD. At the moment this includes the SDK release tarball
+# and the ledger-api-test-tool fat JAR.
+
+set -eou pipefail
+
+cd "$(dirname "$0")/.."
+
+eval "$(./dev-env/bin/dade-assist)"
+
+# We allow overwriting this since on CI we build this in a separate step and upload it first
+# before fetching it in another step.
+HEAD_TARGET_DIR=${1:-compatibility/head_sdk}
+
+bazel build \
+  //release:sdk-release-tarball \
+  //ledger/ledger-api-test-tool:ledger-api-test-tool_deploy.jar
+
+cp -f bazel-bin/release/sdk-release-tarball.tar.gz "$HEAD_TARGET_DIR"
+cp -f bazel-bin/ledger/ledger-api-test-tool/ledger-api-test-tool_deploy.jar "$HEAD_TARGET_DIR"
+

compatibility/head_sdk/.gitignore

@@ -0,0 +1,2 @@
+sdk-release-tarball.tar.gz
+ledger-api-test-tool_deploy.jar

compatibility/head_sdk/BUILD

@@ -0,0 +1,4 @@
+exports_files([
+    "sdk-release-tarball.tar.gz",
+    "ledger-api-test-tool_deploy.jar",
+])

compatibility/head_sdk/WORKSPACE

@@ -0,0 +1 @@
+workspace(name = "sdk_head")

compatibility/test.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+# Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+
+# Build the release artifacts required for running the compatibility
+# tests against HEAD. At the moment this includes the SDK release tarball
+# and the ledger-api-test-tool fat JAR.
+
+set -eou pipefail
+
+cd "$(dirname "$0")"
+
+eval "$(../dev-env/bin/dade-assist)"
+
+bazel build //...
+if [ "${1:-}" = "--quick" ]; then
+    bazel test //:head-quick
+else
+    bazel test //...
+fi
+