From 7ceda5678a64ec1d473b517340db653e9bd39cbc Mon Sep 17 00:00:00 2001 From: Gary Verhaegen Date: Mon, 27 Apr 2020 14:55:16 +0200 Subject: [PATCH] run compatibility tests on macos (#5723) This PR extends the existing Linux compatibility tests to run on macOS too. Fixes #5692. CHANGELOG_BEGIN CHANGELOG_END Co-authored-by: Moritz Kiefer --- azure-pipelines.yml | 41 +++++++++++++++++-- bazel_tools/os_info.bzl | 1 + ci/compatibility.yml | 21 ---------- ci/cron/daily-compat.yml | 4 +- compatibility/BUILD | 4 +- compatibility/WORKSPACE | 21 ++++++++-- .../bazel_tools/client_server/WithPostgres.hs | 7 +++- compatibility/bazel_tools/daml_sdk.bzl | 12 ++++-- 8 files changed, 74 insertions(+), 37 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5aeda9c12d..472b5996e7 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -174,12 +174,27 @@ jobs: trigger_sha: '$(trigger_sha)' - template: ci/report-end.yml - - job: compatibility + - job: compatibility_linux dependsOn: - check_for_release timeoutInMinutes: 60 pool: - name: 'linux-pool' + name: linux-pool + steps: + - template: ci/report-start.yml + - checkout: self + - template: ci/compatibility.yml + parameters: + test_flags: '--quick' + - template: ci/tell-slack-failed.yml + - template: ci/report-end.yml + + - job: compatibility_macos + dependsOn: + - check_for_release + timeoutInMinutes: 60 + pool: + name: macos-pool steps: - template: ci/report-start.yml - checkout: self @@ -361,6 +376,8 @@ jobs: - release - write_ledger_dump - git_sha + - compatibility_macos + - compatibility_linux pool: name: "linux-pool" variables: @@ -384,6 +401,14 @@ jobs: dump.machine: $[ dependencies.write_ledger_dump.outputs['start.machine'] ] dump.end: $[ dependencies.write_ledger_dump.outputs['end.time'] ] dump.status: $[ dependencies.write_ledger_dump.result ] + compatibility_linux.start: $[ dependencies.compatibility_linux.outputs['start.time'] ] + compatibility_linux.machine: $[ dependencies.compatibility_linux.outputs['start.machine'] ] + compatibility_linux.end: $[ dependencies.compatibility_linux.outputs['end.time'] ] + compatibility_linux.status: $[ dependencies.compatibility_linux.result ] + compatibility_macos.start: $[ dependencies.compatibility_macos.outputs['start.time'] ] + compatibility_macos.machine: $[ dependencies.compatibility_macos.outputs['start.machine'] ] + compatibility_macos.end: $[ dependencies.compatibility_macos.outputs['end.time'] ] + compatibility_macos.status: $[ dependencies.compatibility_macos.result ] branch_sha: $[ dependencies.git_sha.outputs['out.branch'] ] master_sha: $[ dependencies.git_sha.outputs['out.master'] ] @@ -422,7 +447,15 @@ jobs: "release": {"start": "$(release.start)", "machine": "$(release.machine)", "end": "$(release.end)", - "status": "$(release.status)"}}, + "status": "$(release.status)"}, + "compatibility_linux": {"start": "$(compatibility_linux.start)", + "machine": "$(compatibility_linux.machine)", + "end": "$(compatibility_linux.end)", + "status": "$(compatibility_linux.status)"}, + "compatibility_macos": {"start": "$(compatibility_macos.start)", + "machine": "$(compatibility_macos.machine)", + "end": "$(compatibility_macos.end)", + "status": "$(compatibility_macos.status)"}}, "id": "$(Build.BuildId)", "url": "https://dev.azure.com/digitalasset/daml/_build/results?buildId=$(Build.BuildId)", "name": "$(Build.DefinitionName)", @@ -470,6 +503,8 @@ jobs: if [[ "$(Linux.status)" != "Succeeded" || "$(macOS.status)" != "Succeeded" || "$(Windows.status)" != "Succeeded" + || "$(compatibility_linux.status)" != "Succeeded" + || "$(compatibility_macos.status)" != "Succeeded" || "$(dump.status)" == "Canceled" || "$(release.status)" == "Canceled" ]]; then exit 1 diff --git a/bazel_tools/os_info.bzl b/bazel_tools/os_info.bzl index 9729f46ebf..729c5a5b89 100644 --- a/bazel_tools/os_info.bzl +++ b/bazel_tools/os_info.bzl @@ -8,6 +8,7 @@ cpu_value = "{CPU_VALUE}" is_darwin = cpu_value == "darwin" is_linux = cpu_value == "k8" is_windows = cpu_value == "x64_windows" +os_name = "macos" if is_darwin else "linux" if is_linux else "windows" """ def _os_info_impl(repository_ctx): diff --git a/ci/compatibility.yml b/ci/compatibility.yml index 514177af9d..63235265f4 100644 --- a/ci/compatibility.yml +++ b/ci/compatibility.yml @@ -26,27 +26,6 @@ steps: set -euo pipefail ./compatibility/build-release-artifacts.sh $(Build.StagingDirectory) displayName: 'Build release artifacts' - # At the moment the indirection of first publishing and then downloading - # artifacts is not strictly necessary. However, we have some artifacts - # that are platform independent where this could come in handy - # (e.g. build the typescript libraries on Linux and fetch them on - # windows) so we use this from the start. - - task: PublishPipelineArtifact@0 - inputs: - targetPath: $(Build.StagingDirectory)/sdk-release-tarball.tar.gz - artifactName: sdk-release-tarball.tar.gz - - task: PublishPipelineArtifact@0 - inputs: - targetPath: $(Build.StagingDirectory)/ledger-api-test-tool_deploy.jar - artifactName: ledger-api-test-tool_deploy.jar - - task: DownloadPipelineArtifact@0 - inputs: - artifactName: sdk-release-tarball.tar.gz - targetPath: compatibility/head_sdk - - task: DownloadPipelineArtifact@0 - inputs: - artifactName: ledger-api-test-tool_deploy.jar - targetPath: compatibility/head_sdk - bash: | set -eou pipefail ./compatibility/test.sh ${{ parameters.test_flags }} diff --git a/ci/cron/daily-compat.yml b/ci/cron/daily-compat.yml index 6724880553..562cfa0479 100644 --- a/ci/cron/daily-compat.yml +++ b/ci/cron/daily-compat.yml @@ -29,8 +29,8 @@ jobs: matrix: linux: pool: 'linux-pool' - # macos: - # pool: 'macOS-pool' + macos: + pool: 'macOS-pool' # windows: # pool: 'windows-pool' pool: diff --git a/compatibility/BUILD b/compatibility/BUILD index 556c6abf73..106afb4e6c 100644 --- a/compatibility/BUILD +++ b/compatibility/BUILD @@ -5,7 +5,7 @@ load( "@daml//bazel_tools/client_server:client_server_test.bzl", "client_server_test", ) -load("@os_info//:os_info.bzl", "is_linux") +load("@os_info//:os_info.bzl", "is_windows") load("//bazel_tools:testing.bzl", "sdk_platform_test") config_setting( @@ -49,7 +49,7 @@ platform_versions = [ ) for sdk_version in sdk_versions for platform_version in platform_versions -] if is_linux else None +] if not is_windows else None test_suite( name = "head-quick", diff --git a/compatibility/WORKSPACE b/compatibility/WORKSPACE index 6a57390810..8aca5e6cbe 100644 --- a/compatibility/WORKSPACE +++ b/compatibility/WORKSPACE @@ -12,7 +12,7 @@ load("@daml//bazel_tools:os_info.bzl", "os_info") os_info(name = "os_info") -load("@os_info//:os_info.bzl", "is_darwin", "is_linux", "is_windows") +load("@os_info//:os_info.bzl", "is_linux", "os_name") load("@daml//bazel_tools:build_environment.bzl", "build_environment") build_environment(name = "build_environment") @@ -138,23 +138,36 @@ local_repository( daml_sdk_head( ledger_api_test_tool = "@head_sdk//:ledger-api-test-tool_deploy.jar", + os_name = os_name, sdk_tarball = "@head_sdk//:sdk-release-tarball.tar.gz", ) daml_sdk( - sdk_sha256 = "ee7e2f50394d44fb3490068de64d37f4f87534e802717bd7e07e8767df2e4e05", + os_name = os_name, + sdk_sha256 = { + "linux": "ee7e2f50394d44fb3490068de64d37f4f87534e802717bd7e07e8767df2e4e05", + "macos": "feb2086a9a01048300270c71eb212c8541cdec1082f541408250d6124bc307a8", + }, test_tool_sha256 = "cf66efafd9490e1256e825f377b208b8ae90151f56e411b596fbaaef91353e14", version = "1.0.0", ) daml_sdk( - sdk_sha256 = "aaf832ceda1a66a8469460d5a4b6c14f681ce692d4e9ef6010896febbaf4b6e1", + os_name = os_name, + sdk_sha256 = { + "linux": "aaf832ceda1a66a8469460d5a4b6c14f681ce692d4e9ef6010896febbaf4b6e1", + "macos": "c2f89e394332b6ff19f547ccb399bacd3cd50d2493249c2d3a1ecaad0b87ac8b", + }, test_tool_sha256 = "762cd4836a8359dca0fb3271ba2e1d0629138f7d8d914298324418a174c5d22a", version = "1.0.1-snapshot.20200417.3908.1.722bac90", ) daml_sdk( - sdk_sha256 = "d3bddaa903ebaebb6f714a3ac39598ba7fd71e8b80636c1275054ed7b883a0d9", + os_name = os_name, + sdk_sha256 = { + "linux": "d3bddaa903ebaebb6f714a3ac39598ba7fd71e8b80636c1275054ed7b883a0d9", + "macos": "66f1713057800ed75db1b967a8ea2d9b6c18da1a76b8224abdec0d33ed5533ae", + }, test_tool_sha256 = "649ecf7a7e98caef7dac20082526444b33d85dfe79b4b9b66d069ad67aac74fa", version = "1.1.0-snapshot.20200422.3991.0.6391ee9f", ) diff --git a/compatibility/bazel_tools/client_server/WithPostgres.hs b/compatibility/bazel_tools/client_server/WithPostgres.hs index f04f4fb20f..e40e072d44 100644 --- a/compatibility/bazel_tools/client_server/WithPostgres.hs +++ b/compatibility/bazel_tools/client_server/WithPostgres.hs @@ -54,7 +54,12 @@ withPostgres f = -- com.daml.testing.postgresql.Tool. callProcess "external/postgresql_nix/bin/initdb" - ["--username=" <> T.unpack dbUser, dataDir] + [ "--username=" <> T.unpack dbUser + , dataDir + , "--locale=en_US.UTF-8" + , "-E", "UNICODE" + , "-A", "trust" + ] writeFileUTF8 (dataDir "postgresql.conf") (T.unpack $ postgresConfig dbPort) bracket_ (startPostgres dataDir logFile) (stopPostgres dataDir) $ do createDatabase diff --git a/compatibility/bazel_tools/daml_sdk.bzl b/compatibility/bazel_tools/daml_sdk.bzl index 5b30e43605..2c9c2b177c 100644 --- a/compatibility/bazel_tools/daml_sdk.bzl +++ b/compatibility/bazel_tools/daml_sdk.bzl @@ -24,9 +24,12 @@ def _daml_sdk_impl(ctx): ctx.download_and_extract( output = "extracted-sdk", # TODO (MK) Make this work on other platforms. - url = - "https://github.com/digital-asset/daml/releases/download/v{}/daml-sdk-{}-linux.tar.gz".format(ctx.attr.version, ctx.attr.version), - sha256 = ctx.attr.sdk_sha256, + url = "https://github.com/digital-asset/daml/releases/download/v{version}/daml-sdk-{version}-{os}.tar.gz" + .format( + version = ctx.attr.version, + os = ctx.attr.os_name, + ), + sha256 = ctx.attr.sdk_sha256[ctx.attr.os_name], stripPrefix = "sdk-{}".format(ctx.attr.version), ) else: @@ -118,10 +121,11 @@ _daml_sdk = repository_rule( implementation = _daml_sdk_impl, attrs = { "version": attr.string(mandatory = True), - "sdk_sha256": attr.string(mandatory = False), + "sdk_sha256": attr.string_dict(mandatory = False), "sdk_tarball": attr.label(allow_single_file = True, mandatory = False), "test_tool_sha256": attr.string(mandatory = False), "test_tool": attr.label(allow_single_file = True, mandatory = False), + "os_name": attr.string(mandatory = True), }, )