From 0d1f21e4a25843058342beef1391c86b98c50a2e Mon Sep 17 00:00:00 2001 From: Moritz Kiefer Date: Fri, 24 Apr 2020 14:43:35 +0200 Subject: [PATCH] Extend compatibility tests to test against HEAD (#5714) fixes #5691 changelog_begin changelog_end --- azure-pipelines.yml | 22 ++------ ci/compatibility.yml | 53 +++++++++++++++++++ ci/cron/daily-compat.yml | 17 +----- compatibility/.bazelignore | 1 + compatibility/BUILD | 7 +++ compatibility/WORKSPACE | 12 ++++- compatibility/bazel_tools/daml_sdk.bzl | 58 ++++++++++++++------ compatibility/bazel_tools/testing.bzl | 67 ++++++++++++++++++++++-- compatibility/build-release-artifacts.sh | 26 +++++++++ compatibility/head_sdk/.gitignore | 2 + compatibility/head_sdk/BUILD | 4 ++ compatibility/head_sdk/WORKSPACE | 1 + compatibility/test.sh | 22 ++++++++ 13 files changed, 237 insertions(+), 55 deletions(-) create mode 100644 ci/compatibility.yml create mode 100644 compatibility/.bazelignore create mode 100755 compatibility/build-release-artifacts.sh create mode 100644 compatibility/head_sdk/.gitignore create mode 100644 compatibility/head_sdk/BUILD create mode 100644 compatibility/head_sdk/WORKSPACE create mode 100755 compatibility/test.sh diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ba9dc467b1..5aeda9c12d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -183,25 +183,9 @@ jobs: steps: - template: ci/report-start.yml - checkout: self - - bash: ci/dev-env-install.sh - displayName: 'Build/Install the Developer Environment' - - bash: ci/configure-bazel.sh - displayName: 'Configure Bazel' - env: - BAZEL_CONFIG_DIR: compatibility - IS_FORK: $(System.PullRequest.IsFork) - # to upload to the bazel cache - GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT) - - bash: | - set -euo pipefail - eval "$(./dev-env/bin/dade-assist)" - cd compatibility - bazel build //... - # Run one test as a sanity check. We do not run all of them - # since the test matrix will get large. - bazel test \ - //:ledger-api-test-tool-1.0.1-snapshot.20200417.3908.1.722bac90-platform-1.0.0 \ - //:ledger-api-test-tool-1.0.1-snapshot.20200417.3908.1.722bac90-platform-1.0.0-postgresql + - template: ci/compatibility.yml + parameters: + test_flags: '--quick' - template: ci/tell-slack-failed.yml - template: ci/report-end.yml diff --git a/ci/compatibility.yml b/ci/compatibility.yml new file mode 100644 index 0000000000..514177af9d --- /dev/null +++ b/ci/compatibility.yml @@ -0,0 +1,53 @@ +# Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +parameters: + - name: test_flags + type: string + default: '' + +steps: + - bash: ci/dev-env-install.sh + displayName: 'Build/Install the Developer Environment' + - bash: ci/configure-bazel.sh + displayName: 'Configure Bazel for root workspace' + env: + IS_FORK: $(System.PullRequest.IsFork) + # to upload to the bazel cache + GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT) + - bash: ci/configure-bazel.sh + displayName: 'Configure Bazel for compatibility workspace' + env: + BAZEL_CONFIG_DIR: compatibility + IS_FORK: $(System.PullRequest.IsFork) + # to upload to the bazel cache + GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT) + - bash: | + set -euo pipefail + ./compatibility/build-release-artifacts.sh $(Build.StagingDirectory) + displayName: 'Build release artifacts' + # At the moment the indirection of first publishing and then downloading + # artifacts is not strictly necessary. However, we have some artifacts + # that are platform independent where this could come in handy + # (e.g. build the typescript libraries on Linux and fetch them on + # windows) so we use this from the start. + - task: PublishPipelineArtifact@0 + inputs: + targetPath: $(Build.StagingDirectory)/sdk-release-tarball.tar.gz + artifactName: sdk-release-tarball.tar.gz + - task: PublishPipelineArtifact@0 + inputs: + targetPath: $(Build.StagingDirectory)/ledger-api-test-tool_deploy.jar + artifactName: ledger-api-test-tool_deploy.jar + - task: DownloadPipelineArtifact@0 + inputs: + artifactName: sdk-release-tarball.tar.gz + targetPath: compatibility/head_sdk + - task: DownloadPipelineArtifact@0 + inputs: + artifactName: ledger-api-test-tool_deploy.jar + targetPath: compatibility/head_sdk + - bash: | + set -eou pipefail + ./compatibility/test.sh ${{ parameters.test_flags }} + displayName: 'Run tests' diff --git a/ci/cron/daily-compat.yml b/ci/cron/daily-compat.yml index 36034087d6..6724880553 100644 --- a/ci/cron/daily-compat.yml +++ b/ci/cron/daily-compat.yml @@ -37,20 +37,5 @@ jobs: name: $(pool) steps: - checkout: self - - bash: ci/dev-env-install.sh - displayName: 'Build/Install the Developer Environment' - - bash: | - set -euo pipefail - - eval "$(./dev-env/bin/dade-assist)" - - cd compatibility - bazel build //... - bazel test //... - - # FIXME: fill in the gaps. - exit 0 - # For whatever reason, templates are searched from the path of the - # current YML file. I could not find any reference to that in the - # documentation. + - template: ../compatibility.yml - template: ../tell-slack-failed.yml diff --git a/compatibility/.bazelignore b/compatibility/.bazelignore new file mode 100644 index 0000000000..9c10a71873 --- /dev/null +++ b/compatibility/.bazelignore @@ -0,0 +1 @@ +head-sdk diff --git a/compatibility/BUILD b/compatibility/BUILD index 15c43ac124..556c6abf73 100644 --- a/compatibility/BUILD +++ b/compatibility/BUILD @@ -19,12 +19,14 @@ sdk_versions = [ "1.0.0", "1.0.1-snapshot.20200417.3908.1.722bac90", "1.1.0-snapshot.20200422.3991.0.6391ee9f", + "0.0.0", ] platform_versions = [ "1.0.0", "1.0.1-snapshot.20200417.3908.1.722bac90", "1.1.0-snapshot.20200422.3991.0.6391ee9f", + "0.0.0", ] [ @@ -48,3 +50,8 @@ platform_versions = [ for sdk_version in sdk_versions for platform_version in platform_versions ] if is_linux else None + +test_suite( + name = "head-quick", + tags = ["head-quick"], +) diff --git a/compatibility/WORKSPACE b/compatibility/WORKSPACE index d938d4569e..6a57390810 100644 --- a/compatibility/WORKSPACE +++ b/compatibility/WORKSPACE @@ -129,7 +129,17 @@ load("//:bazel-haskell-deps.bzl", "daml_haskell_deps") daml_haskell_deps() -load("//bazel_tools:daml_sdk.bzl", "daml_sdk") +load("//bazel_tools:daml_sdk.bzl", "daml_sdk", "daml_sdk_head") + +local_repository( + name = "head_sdk", + path = "head_sdk", +) + +daml_sdk_head( + ledger_api_test_tool = "@head_sdk//:ledger-api-test-tool_deploy.jar", + sdk_tarball = "@head_sdk//:sdk-release-tarball.tar.gz", +) daml_sdk( sdk_sha256 = "ee7e2f50394d44fb3490068de64d37f4f87534e802717bd7e07e8767df2e4e05", diff --git a/compatibility/bazel_tools/daml_sdk.bzl b/compatibility/bazel_tools/daml_sdk.bzl index 4f8d2aa27d..5b30e43605 100644 --- a/compatibility/bazel_tools/daml_sdk.bzl +++ b/compatibility/bazel_tools/daml_sdk.bzl @@ -14,19 +14,35 @@ source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \ """ def _daml_sdk_impl(ctx): - ctx.download_and_extract( - output = "extracted-sdk", - # TODO (MK) Make this work on other platforms. - url = - "https://github.com/digital-asset/daml/releases/download/v{}/daml-sdk-{}-linux.tar.gz".format(ctx.attr.version, ctx.attr.version), - sha256 = ctx.attr.sdk_sha256, - stripPrefix = "sdk-{}".format(ctx.attr.version), - ) - ctx.download( - output = "ledger-api-test-tool.jar", - url = "https://repo1.maven.org/maven2/com/daml/ledger-api-test-tool/{}/ledger-api-test-tool-{}.jar".format(ctx.attr.version, ctx.attr.version), - sha256 = ctx.attr.test_tool_sha256, - ) + if ctx.attr.sdk_tarball: + ctx.extract( + ctx.attr.sdk_tarball, + output = "extracted-sdk", + stripPrefix = "sdk-{}".format(ctx.attr.version), + ) + elif ctx.attr.sdk_sha256: + ctx.download_and_extract( + output = "extracted-sdk", + # TODO (MK) Make this work on other platforms. + url = + "https://github.com/digital-asset/daml/releases/download/v{}/daml-sdk-{}-linux.tar.gz".format(ctx.attr.version, ctx.attr.version), + sha256 = ctx.attr.sdk_sha256, + stripPrefix = "sdk-{}".format(ctx.attr.version), + ) + else: + fail("Must specify either sdk_tarball or sdk_sha256") + + if ctx.attr.test_tool: + ctx.symlink(ctx.attr.test_tool, "ledger-api-test-tool.jar") + elif ctx.attr.test_tool_sha256: + ctx.download( + output = "ledger-api-test-tool.jar", + url = "https://repo1.maven.org/maven2/com/daml/ledger-api-test-tool/{}/ledger-api-test-tool-{}.jar".format(ctx.attr.version, ctx.attr.version), + sha256 = ctx.attr.test_tool_sha256, + ) + else: + fail("Must specify either test_tool or test_tool_sha256") + ctx.extract( "ledger-api-test-tool.jar", output = "extracted-test-tool", @@ -102,8 +118,10 @@ _daml_sdk = repository_rule( implementation = _daml_sdk_impl, attrs = { "version": attr.string(mandatory = True), - "sdk_sha256": attr.string(mandatory = True), - "test_tool_sha256": attr.string(mandatory = True), + "sdk_sha256": attr.string(mandatory = False), + "sdk_tarball": attr.label(allow_single_file = True, mandatory = False), + "test_tool_sha256": attr.string(mandatory = False), + "test_tool": attr.label(allow_single_file = True, mandatory = False), }, ) @@ -113,3 +131,13 @@ def daml_sdk(version, **kwargs): version = version, **kwargs ) + +def daml_sdk_head(sdk_tarball, ledger_api_test_tool, **kwargs): + version = "0.0.0" + _daml_sdk( + name = "daml-sdk-{}".format(version), + version = version, + sdk_tarball = sdk_tarball, + test_tool = ledger_api_test_tool, + **kwargs + ) diff --git a/compatibility/bazel_tools/testing.bzl b/compatibility/bazel_tools/testing.bzl index 5c775a96d1..7cadf41528 100644 --- a/compatibility/bazel_tools/testing.bzl +++ b/compatibility/bazel_tools/testing.bzl @@ -6,6 +6,64 @@ load( "client_server_test", ) +latest_stable = "1.0.0" + +# Indexed first by test tool version and then by sandbox version. +# Note that at this point the granularity for disabling tests +# is sadly quite coarse. See +# https://discuss.daml.com/t/can-i-disable-individual-tests-in-the-ledger-api-test-tool/226 +# for details. +excluded_test_tool_tests = { + "1.0.0": { + "0.0.0": [ + # This restriction has been removed in https://github.com/digital-asset/daml/pull/5611. + "ContractKeysSubmitterIsMaintainerIT", + # Fix for https://github.com/digital-asset/daml/issues/5562 + "ContractKeysIT", + ], + }, + "1.0.1-snapshot.20200417.3908.1.722bac90": { + "0.0.0": [ + # This restriction has been removed in https://github.com/digital-asset/daml/pull/5611. + "ContractKeysSubmitterIsMaintainerIT", + # Fix for https://github.com/digital-asset/daml/issues/5562 + "ContractKeysIT", + ], + }, + "1.1.0-snapshot.20200422.3991.0.6391ee9f": { + "0.0.0": [ + # This restriction has been removed in https://github.com/digital-asset/daml/pull/5611. + "ContractKeysSubmitterIsMaintainerIT", + # Fix for https://github.com/digital-asset/daml/issues/5562 + "ContractKeysIT", + ], + }, + "0.0.0": { + "1.0.0": [ + # Fix for https://github.com/digital-asset/daml/issues/5562 + "ContractKeysIT", + ], + "1.0.1-snapshot.20200417.3908.1.722bac90": [ + # Fix for https://github.com/digital-asset/daml/issues/5562 + "ContractKeysIT", + ], + "1.1.0-snapshot.20200422.3991.0.6391ee9f": [ + # Fix for https://github.com/digital-asset/daml/issues/5562 + "ContractKeysIT", + ], + }, +} + +def get_excluded_tests(test_tool_version, sandbox_version): + return excluded_test_tool_tests.get(test_tool_version, default = {}).get(sandbox_version, default = []) + +def extra_tags(sdk_version, platform_version): + if sorted([sdk_version, platform_version]) == sorted(["0.0.0", latest_stable]): + # These tests are the ones that we check on each PR since they + # are the most useful ones and hopefully fast enough. + return ["head-quick"] + return [] + def sdk_platform_test(sdk_version, platform_version): # SDK components ledger_api_test_tool = "@daml-sdk-{sdk_version}//:ledger-api-test-tool".format( @@ -26,6 +84,7 @@ def sdk_platform_test(sdk_version, platform_version): sdk_version = sdk_version, platform_version = platform_version, ) + exclusions = ["--exclude=" + test for test in get_excluded_tests(test_tool_version = sdk_version, sandbox_version = platform_version)] client_server_test( name = name, client = ledger_api_test_tool, @@ -33,7 +92,7 @@ def sdk_platform_test(sdk_version, platform_version): "localhost:6865", "--open-world", "--exclude=ClosedWorldIT", - ], + ] + exclusions, data = [dar_files], runner = "@//bazel_tools/client_server:runner", runner_args = ["6865"], @@ -42,7 +101,7 @@ def sdk_platform_test(sdk_version, platform_version): server_files = ["$(rootpaths {dar_files})".format( dar_files = dar_files, )], - tags = ["exclusive"], + tags = ["exclusive"] + extra_tags(sdk_version, platform_version), ) client_server_test( @@ -52,7 +111,7 @@ def sdk_platform_test(sdk_version, platform_version): "localhost:6865", "--open-world", "--exclude=ClosedWorldIT", - ], + ] + exclusions, data = [dar_files], runner = "@//bazel_tools/client_server:runner", runner_args = ["6865"], @@ -61,5 +120,5 @@ def sdk_platform_test(sdk_version, platform_version): server_files = ["$(rootpaths {dar_files})".format( dar_files = dar_files, )], - tags = ["exclusive"], + tags = ["exclusive"] + extra_tags(sdk_version, platform_version), ) diff --git a/compatibility/build-release-artifacts.sh b/compatibility/build-release-artifacts.sh new file mode 100755 index 0000000000..2d0b53c579 --- /dev/null +++ b/compatibility/build-release-artifacts.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + + +# Build the release artifacts required for running the compatibility +# tests against HEAD. At the moment this includes the SDK release tarball +# and the ledger-api-test-tool fat JAR. + +set -eou pipefail + +cd "$(dirname "$0")/.." + +eval "$(./dev-env/bin/dade-assist)" + +# We allow overwriting this since on CI we build this in a separate step and upload it first +# before fetching it in another step. +HEAD_TARGET_DIR=${1:-compatibility/head_sdk} + +bazel build \ + //release:sdk-release-tarball \ + //ledger/ledger-api-test-tool:ledger-api-test-tool_deploy.jar + +cp -f bazel-bin/release/sdk-release-tarball.tar.gz "$HEAD_TARGET_DIR" +cp -f bazel-bin/ledger/ledger-api-test-tool/ledger-api-test-tool_deploy.jar "$HEAD_TARGET_DIR" + diff --git a/compatibility/head_sdk/.gitignore b/compatibility/head_sdk/.gitignore new file mode 100644 index 0000000000..bf63fb3a0a --- /dev/null +++ b/compatibility/head_sdk/.gitignore @@ -0,0 +1,2 @@ +sdk-release-tarball.tar.gz +ledger-api-test-tool_deploy.jar diff --git a/compatibility/head_sdk/BUILD b/compatibility/head_sdk/BUILD new file mode 100644 index 0000000000..2e81330858 --- /dev/null +++ b/compatibility/head_sdk/BUILD @@ -0,0 +1,4 @@ +exports_files([ + "sdk-release-tarball.tar.gz", + "ledger-api-test-tool_deploy.jar", +]) diff --git a/compatibility/head_sdk/WORKSPACE b/compatibility/head_sdk/WORKSPACE new file mode 100644 index 0000000000..57116114d6 --- /dev/null +++ b/compatibility/head_sdk/WORKSPACE @@ -0,0 +1 @@ +workspace(name = "sdk_head") diff --git a/compatibility/test.sh b/compatibility/test.sh new file mode 100755 index 0000000000..80e65446b0 --- /dev/null +++ b/compatibility/test.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + + +# Build the release artifacts required for running the compatibility +# tests against HEAD. At the moment this includes the SDK release tarball +# and the ledger-api-test-tool fat JAR. + +set -eou pipefail + +cd "$(dirname "$0")" + +eval "$(../dev-env/bin/dade-assist)" + +bazel build //... +if [ "${1:-}" = "--quick" ]; then + bazel test //:head-quick +else + bazel test //... +fi +