Add platform-version field to daml.yaml (#6736)

* Add `platform-version` field to `daml.yaml`

This PR adds the `platform-version` field to `daml.yaml`. Based on the
approach agreed upon in #6558, the logic for this all sits in
`daml-helper` and there are no changes to the assistant.

The details of how the logic work are in a comment so I’m not going to
repeat them here but the commands that are affected are:

- `daml sandbox`
- `daml sandbox-classic`
- `daml json-api`
- `daml start` (but only for sandbox and the JSON API, not for
  Navigator or anything else)

For tests, I’ve added a test to the compat workspace that installs two
SDKs simultaneously and tries out various combinations and verifies
that we get the correct version. Open to other ideas for testing this
but that seemed like the most sensible option that actually tests what
we run.

changelog_begin

- [DAML Assistant] You can now specify the version of Sandbox and the
  JSON API independently of your SDK version by setting
  ``platform-version`` in your ``daml.yaml``. This is useful if you
  are deploying to a ledger that is running components from a
  different SDK version. See
  https://docs.daml.com/tools/assistant.html#project-config-file-daml-yaml
  for details.

changelog_end

* Run platform-version tests

changelog_begin
changelog_end

* Fix tag globbing

changelog_begin
changelog_end

* fmt

changelog_begin
changelog_end

* .

changelog_begin
changelog_end

* Try to fix env vars

changelog_begin
changelog_end

* Remove hardcoded references to 1.2.0

changelog_begin
changelog_end

* Rephrase doc comment

changelog_begin
changelog_end

* get things to compile

changelog_begin
changelog_end

* maybe fix things for realz

changelog_begin
changelog_end

* Remove debugging output

changelog_begin
changelog_end

* Get angry at windows

changelog_begin
changelog_end

* why is windows

changelog_begin
changelog_end

* .

changelog_begin
changelog_end

compatibility/BUILD

@@ -97,11 +97,6 @@ config_setting(
     if versions.is_at_least(sdk_version, platform_version)
 ]
 
-test_suite(
-    name = "head-quick",
-    tags = ["head-quick"],
-)
-
 # We have two migration tests: migration-stable runs through all stable releases
 # including current HEAD. migration-all includes snapshot releases.
 

compatibility/assistant/BUILD

@@ -0,0 +1,42 @@
+# Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+load("@daml//bazel_tools:haskell.bzl", "da_haskell_test")
+
+da_haskell_test(
+    name = "assistant-platform-version",
+    srcs = glob(["src/**/*.hs"]),
+    data = [
+        "@daml-sdk-tarball-latest-stable//file:downloaded",
+        "@head_sdk//:sdk-release-tarball.tar.gz",
+    ],
+    hackage_deps = [
+        "base",
+        "tar-conduit",
+        "conduit",
+        "conduit-extra",
+        "text",
+        "filepath",
+        "directory",
+        "extra",
+        "process",
+        "typed-process",
+        "safe-exceptions",
+        "tasty",
+        "tasty-hunit",
+        "utf8-string",
+        "stm",
+    ],
+    main_function = "DA.Test.PlatformVersion.main",
+    tags = [
+        "exclusive",
+        "head-quick",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//bazel_tools:versions-hs-lib",
+        "//bazel_tools/daml_ledger:sandbox-helper",
+        "//bazel_tools/test_utils",
+        "@rules_haskell//tools/runfiles",
+    ],
+)

compatibility/assistant/src/DA/Test/PlatformVersion.hs

@@ -0,0 +1,152 @@
+-- Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+-- SPDX-License-Identifier: Apache-2.0
+
+module DA.Test.PlatformVersion (main) where
+
+import qualified Bazel.Runfiles
+import Control.Concurrent.STM
+import Control.Exception.Safe
+import Control.Monad
+import Data.ByteString.Lazy.UTF8 (ByteString, toString)
+import Data.Conduit ((.|), runConduitRes)
+import qualified Data.Conduit.Combinators as Conduit
+import qualified Data.Conduit.Tar as Tar
+import qualified Data.Conduit.Zlib as Zlib
+import Data.List
+import qualified Data.Text as T
+import DA.Test.Tar
+import System.Directory
+import System.Environment.Blank
+import System.FilePath
+import System.Info (os)
+import System.IO.Extra
+import qualified System.Process.Typed as Proc
+import Test.Tasty
+import Test.Tasty.HUnit
+import Data.Maybe
+import Sandbox (readPortFile, maxRetries)
+import Versions (latestStableVersion)
+import System.Process (interruptProcessGroupOf)
+
+main :: IO ()
+main = do
+    setEnv "TASTY_NUM_THREADS" "1" True
+    javaHome <- fromJust <$> getEnv "JAVA_HOME"
+    oldPath <- getSearchPath
+    setEnv "PATH" (intercalate [searchPathSeparator] ((javaHome </> "bin") : oldPath)) True
+    defaultMain $ withSdkResource $ \_getSdkPath ->
+        testGroup "platform-version"
+            [ testCase "no project, no DAML_PLATFORM_VERSION" $ do
+                  env <- extendEnv [("DAML_SDK_VERSION", "0.0.0")]
+                  out <- Proc.readProcessStdout_ (Proc.setEnv env $ Proc.shell "daml sandbox --help")
+                  assertInfixOf "Sandbox version 0.0.0" out
+                  out <- Proc.readProcessStdout_ (Proc.setEnv env $ Proc.shell "daml sandbox-classic --help")
+                  assertInfixOf "Sandbox version 0.0.0" out
+            , testCase "no project, DAML_PLATFORM_VERSION" $ do
+                  env <- extendEnv [("DAML_SDK_VERSION", "0.0.0"), ("DAML_PLATFORM_VERSION", latestStableVersion)]
+                  out <- Proc.readProcessStdout_
+                      (Proc.setEnv env $ Proc.shell "daml sandbox --help")
+                  assertInfixOf ("Sandbox version " <> latestStableVersion) out
+                  out <- Proc.readProcessStdout_
+                      (Proc.setEnv env $ Proc.shell "daml sandbox-classic --help")
+                  assertInfixOf ("Sandbox version " <> latestStableVersion) out
+            , testCase "no project, platform-version" $ withTempDir $ \tempDir -> do
+                  writeFileUTF8 (tempDir </> "daml.yaml") $ unlines
+                    [ "sdk-version: 0.0.0"
+                    , "platform-version: " <> latestStableVersion
+                    ]
+                  out <- Proc.readProcessStdout_ (Proc.setWorkingDir tempDir (Proc.shell "daml sandbox --help"))
+                  assertInfixOf ("Sandbox version " <> latestStableVersion) out
+                  -- Env var takes precedence
+                  env <- extendEnv [("DAML_PLATFORM_VERSION", "0.0.0")]
+                  out <- Proc.readProcessStdout_
+                      (Proc.setWorkingDir tempDir $ Proc.setEnv env $ Proc.shell "daml sandbox --help")
+                  assertInfixOf "Sandbox version 0.0.0" out
+            , testCase "daml start" $ withTempDir $ \tempDir -> do
+                  writeFileUTF8 (tempDir </> "daml.yaml") $ unlines
+                    [ "sdk-version: 0.0.0"
+                    , "platform-version: " <> latestStableVersion
+                    , "name: foobar"
+                    , "version: 0.0.1"
+                    , "dependencies: [daml-prim, daml-stdlib]"
+                    , "source: ."
+                    , "parties: []"
+                    ]
+                  let conf =
+                          Proc.setCreateGroup True $
+                          Proc.setStdin Proc.createPipe $
+                          Proc.setStdout Proc.byteStringOutput $
+                          Proc.setWorkingDir tempDir $
+                          Proc.shell $
+                            "daml start --shutdown-stdin-close --open-browser=no --json-api-option --port-file --json-api-option " <> show (tempDir </> "portfile")
+                  getOut <- Proc.withProcessWait conf $ \ph -> do
+                      -- Wait for the port file as a sign that the JSON API has started.
+                      _ <- readPortFile maxRetries (tempDir </> "portfile")
+                      hClose (Proc.getStdin ph)
+                      -- Process management on Windows is a nightmare so we opt
+                      -- for the safest option of creating a group and killing everything
+                      -- in that group.
+                      interruptProcessGroupOf (Proc.unsafeProcessHandle ph)
+                      pure $ Proc.getStdout ph
+                  out <- atomically getOut
+                  putStrLn "got stdout"
+                  assertInfixOf ("sandbox version " <> latestStableVersion) out
+                  -- Navigator, sadly not prefixed with Navigator
+                  assertInfixOf "Version 0.0.0" out
+                  -- JSON API, doesn’t even print a version but let’s at least check
+                  -- that something started.
+                  assertInfixOf "httpPort=7575" out
+
+            ]
+
+extendEnv :: [(String, String)] -> IO [(String, String)]
+extendEnv xs = do
+    oldEnv <- getEnvironment
+    pure (xs ++ filter (\(k, _) -> k `notElem` map fst xs) oldEnv)
+
+withSdkResource :: (IO FilePath -> TestTree) -> TestTree
+withSdkResource f =
+    withTempDirResource $ \getDir ->
+    withResource (installSdk =<< getDir) (const $ pure ()) (const $ f getDir)
+  where installSdk targetDir = do
+            runfiles <- Bazel.Runfiles.create
+            let headSdk = Bazel.Runfiles.rlocation runfiles "head_sdk/sdk-release-tarball.tar.gz"
+            let latestStableSdk = Bazel.Runfiles.rlocation runfiles "daml-sdk-tarball-latest-stable/file/downloaded"
+            setEnv "DAML_HOME" targetDir True
+            withTempDir $ \extractDir -> do
+                runConduitRes
+                    $ Conduit.sourceFileBS headSdk
+                    .| Zlib.ungzip
+                    .| Tar.untar (restoreFile (\a b -> fail (T.unpack $ a <> b)) extractDir)
+                Proc.runProcess_ $
+                    Proc.proc
+                        (extractDir </> "daml" </> exe "daml")
+                        ["install", "--install-assistant=yes", "--set-path=no", extractDir, "--quiet"]
+                pure ()
+            oldPath <- getSearchPath
+            setEnv "PATH" (intercalate [searchPathSeparator] ((targetDir </> "bin") : oldPath)) True
+            Proc.runProcess_ (Proc.shell $ "daml install --quiet " <> latestStableSdk)
+            pure ()
+
+withTempDirResource :: (IO FilePath -> TestTree) -> TestTree
+withTempDirResource f = withResource newTempDir delete (f . fmap fst)
+    -- The delete action provided by `newTempDir` calls `removeDirectoryRecursively`
+    -- and silently swallows errors. SDK installations are marked read-only
+    -- which means that they don’t end up being removed which is obviously
+    -- not what we intend.
+    -- As usual Windows is terrible and doesn’t let you remove the SDK
+    -- if there is a process running. Simultaneously it is also terrible
+    -- at process management so we end up with running processes
+    -- since child processes aren’t torn down properly
+    -- (Bazel will kill them later when the test finishes). Therefore,
+    -- we ignore exceptions and hope for the best. On Windows that
+    -- means we still leak directories :(
+    where delete (d, _delete) = void $ tryIO $ removePathForcibly d
+
+exe :: FilePath -> FilePath
+exe | os == "mingw32" = (<.> "exe")
+    | otherwise       = id
+
+assertInfixOf :: String -> ByteString -> Assertion
+assertInfixOf needle haystack = assertBool ("Expected " <> show needle <> " in output but but got " <> show haystack) (needle `isInfixOf` toString haystack)
+

compatibility/bazel-haskell-deps.bzl

@@ -67,14 +67,17 @@ def daml_haskell_deps():
             "safe-exceptions",
             "semver",
             "split",
+            "stm",
             "tagged",
             "tar-conduit",
             "tasty",
             "tasty-hunit",
             "text",
+            "typed-process",
             "optparse-applicative",
             "unix-compat",
             "unordered-containers",
+            "utf8-string",
             "uuid",
         ] + (["unix"] if not is_windows else ["Win32"]),
         stack = "@stack_windows//:stack.exe" if is_windows else None,

compatibility/bazel_tools/BUILD

@@ -1,6 +1,30 @@
+load("@daml//bazel_tools:haskell.bzl", "da_haskell_library")
+load("//:versions.bzl", "latest_stable_version")
+
 exports_files([
     "daml_ledger_test.sh",
     "create_daml_app_test.sh",
     "sandbox-with-postgres.sh",
     "daml.cc",
 ])
+
+genrule(
+    name = "versions-hs-lib-gen",
+    srcs = [],
+    outs = ["Versions.hs"],
+    cmd = """
+        cat > $@ <<EOF
+module Versions where
+
+latestStableVersion :: String
+latestStableVersion = "{}"
+EOF
+""".format(latest_stable_version),
+)
+
+da_haskell_library(
+    name = "versions-hs-lib",
+    srcs = [":Versions.hs"],
+    hackage_deps = ["base"],
+    visibility = ["//visibility:public"],
+)

compatibility/deps.bzl

@@ -45,6 +45,8 @@ load(
     "zlib_sha256",
     "zlib_version",
 )
+load("//:versions.bzl", "latest_stable_version", "version_sha256s")
+load("@os_info//:os_info.bzl", "os_name")
 
 def daml_deps():
     if "rules_haskell" not in native.existing_rules():
@@ -128,3 +130,13 @@ def daml_deps():
             ],
             patch_args = ["-p1"],
         )
+
+    # The tests for the `platform-version` field need a proper assistant installation with
+    # multiple installed SDKs. Therefore, we fetch the (unextracted) installation
+    # tarball for the latest stable release. Together with the tarball for HEAD
+    # that gives us two SDK version to test with.
+    http_file(
+        name = "daml-sdk-tarball-latest-stable",
+        sha256 = version_sha256s[latest_stable_version][os_name],
+        urls = ["https://github.com/digital-asset/daml/releases/download/v{}/daml-sdk-{}-{}.tar.gz".format(latest_stable_version, latest_stable_version, os_name)],
+    )

compatibility/stackage_snapshot.json

@@ -1,6 +1,6 @@
 {
-  "__GENERATED_FILE_DO_NOT_MODIFY_MANUALLY": 1930796810,
+  "__GENERATED_FILE_DO_NOT_MODIFY_MANUALLY": 1145367801,
-  "all-cabal-hashes": "https://raw.githubusercontent.com/commercialhaskell/all-cabal-hashes/dd401bef3bed95bce26d3644994f35cb6394a40e",
+  "all-cabal-hashes": "https://raw.githubusercontent.com/commercialhaskell/all-cabal-hashes/fce569f07aff95333abf763f199b3fbe77f0499b",
   "resolved": {
     "Cabal": {"dependencies":["array","base","binary","bytestring","containers","deepseq","directory","filepath","mtl","parsec","pretty","process","text","time","transformers","unix"],"location":{"type":"core"},"name":"Cabal","version":"2.4.1.0"},
     "QuickCheck": {"dependencies":["base","containers","deepseq","random","splitmix","template-haskell","transformers"],"location":{"type":"hackage","url":"https://hackage.haskell.org/package/QuickCheck-2.13.2/QuickCheck-2.13.2.tar.gz"},"name":"QuickCheck","pinned":{"url":["https://hackage.haskell.org/package/QuickCheck-2.13.2/QuickCheck-2.13.2.tar.gz","https://s3.amazonaws.com/hackage.fpcomplete.com/package/QuickCheck-2.13.2.tar.gz"],"sha256":"7b560baa5853de777702dc23a6f2126ae4adbfdab163295bc56323a706914610","cabal-sha256":"636e7265bf75122e7e2f97627c47aad3b772ee3b35b134cafb6095116ce8d07a"},"version":"2.13.2"},
@@ -122,8 +122,9 @@
     "unix-compat": {"dependencies":["base","unix"],"location":{"type":"hackage","url":"https://hackage.haskell.org/package/unix-compat-0.5.2/unix-compat-0.5.2.tar.gz"},"name":"unix-compat","pinned":{"url":["https://hackage.haskell.org/package/unix-compat-0.5.2/unix-compat-0.5.2.tar.gz","https://s3.amazonaws.com/hackage.fpcomplete.com/package/unix-compat-0.5.2.tar.gz"],"sha256":"659a4e442f71505d45b0d0fb28a347aa1ac5e1f39feaea5d58e674e5d2ce0ba9","cabal-sha256":"5508ebcfaf1a862886cd217178e283d613d47803785d8ab4a3bd2bb9b542a3fb"},"version":"0.5.2"},
     "unliftio-core": {"dependencies":["base","transformers"],"location":{"type":"hackage","url":"https://hackage.haskell.org/package/unliftio-core-0.1.2.0/unliftio-core-0.1.2.0.tar.gz"},"name":"unliftio-core","pinned":{"url":["https://hackage.haskell.org/package/unliftio-core-0.1.2.0/unliftio-core-0.1.2.0.tar.gz","https://s3.amazonaws.com/hackage.fpcomplete.com/package/unliftio-core-0.1.2.0.tar.gz"],"sha256":"24c38b3d610ca2642ed496d1de3d7b6b398ce0410aa0a15f3c7ce636ba8f7a78","cabal-sha256":"7f9b48adef8e36da0202e6e70a733a5e210263ed4177c93e47a4b3f89694194b"},"version":"0.1.2.0"},
     "unordered-containers": {"dependencies":["base","deepseq","hashable"],"location":{"type":"hackage","url":"https://hackage.haskell.org/package/unordered-containers-0.2.10.0/unordered-containers-0.2.10.0.tar.gz"},"name":"unordered-containers","pinned":{"url":["https://hackage.haskell.org/package/unordered-containers-0.2.10.0/unordered-containers-0.2.10.0.tar.gz","https://s3.amazonaws.com/hackage.fpcomplete.com/package/unordered-containers-0.2.10.0.tar.gz"],"sha256":"65f117bdbdea9efc75fb9fd539873de7687e005d8898bb21821020a4b383c573","cabal-sha256":"5e9b095a9283d9e2f064fec73a81a6b6ea0b7fda3f219a8175785d2d2a3de204"},"version":"0.2.10.0"},
+    "utf8-string": {"dependencies":["base","bytestring"],"location":{"type":"hackage","url":"https://hackage.haskell.org/package/utf8-string-1.0.1.1/utf8-string-1.0.1.1.tar.gz"},"name":"utf8-string","pinned":{"url":["https://hackage.haskell.org/package/utf8-string-1.0.1.1/utf8-string-1.0.1.1.tar.gz","https://s3.amazonaws.com/hackage.fpcomplete.com/package/utf8-string-1.0.1.1.tar.gz"],"sha256":"fb0b9e3acbe0605bcd1c63e51f290a7bbbe6628dfa3294ff453e4235fbaef140","cabal-sha256":"68cc6cf665e7212334a51b63d6936daeaca023b2cfe8637d130acfe95f91700b"},"version":"1.0.1.1"},
     "uuid": {"dependencies":["base","binary","bytestring","cryptohash-md5","cryptohash-sha1","entropy","network-info","random","text","time","uuid-types"],"location":{"type":"hackage","url":"https://hackage.haskell.org/package/uuid-1.3.13/uuid-1.3.13.tar.gz"},"name":"uuid","pinned":{"url":["https://hackage.haskell.org/package/uuid-1.3.13/uuid-1.3.13.tar.gz","https://s3.amazonaws.com/hackage.fpcomplete.com/package/uuid-1.3.13.tar.gz"],"sha256":"dfac808a7026217d018b408eab18facc6a85c6183be308d4ac7877e80599b027","cabal-sha256":"380b45cfd1d2dfda0b52d414f0040cef3adfc02bf54bd9e61f2ef47755ce5adc"},"version":"1.3.13"},
-    "uuid-types": {"dependencies":["base","binary","bytestring","deepseq","hashable","random","text"],"location":{"type":"hackage","url":"https://hackage.haskell.org/package/uuid-types-1.0.3/uuid-types-1.0.3.tar.gz"},"name":"uuid-types","pinned":{"url":["https://hackage.haskell.org/package/uuid-types-1.0.3/uuid-types-1.0.3.tar.gz","https://s3.amazonaws.com/hackage.fpcomplete.com/package/uuid-types-1.0.3.tar.gz"],"sha256":"9276517ab24a9b06f39d6e3c33c6c2b4ace1fc2126dbc1cd9806866a6551b3fd","cabal-sha256":"2cab161f274309cfd9ef62694a6bd1b47525bc4c79a827293b055e4a9eadb4d2"},"version":"1.0.3"},
+    "uuid-types": {"dependencies":["base","binary","bytestring","deepseq","hashable","random","text"],"location":{"type":"hackage","url":"https://hackage.haskell.org/package/uuid-types-1.0.3/uuid-types-1.0.3.tar.gz"},"name":"uuid-types","pinned":{"url":["https://hackage.haskell.org/package/uuid-types-1.0.3/uuid-types-1.0.3.tar.gz","https://s3.amazonaws.com/hackage.fpcomplete.com/package/uuid-types-1.0.3.tar.gz"],"sha256":"9276517ab24a9b06f39d6e3c33c6c2b4ace1fc2126dbc1cd9806866a6551b3fd","cabal-sha256":"4e27aa142cb37f391e78f36d13139ba0a907675429a21e8f2cfae9223202dd7e"},"version":"1.0.3"},
     "vector": {"dependencies":["base","deepseq","ghc-prim","primitive"],"location":{"type":"hackage","url":"https://hackage.haskell.org/package/vector-0.12.0.3/vector-0.12.0.3.tar.gz"},"name":"vector","pinned":{"url":["https://hackage.haskell.org/package/vector-0.12.0.3/vector-0.12.0.3.tar.gz","https://s3.amazonaws.com/hackage.fpcomplete.com/package/vector-0.12.0.3.tar.gz"],"sha256":"b8a2bfbf9d22d34a28cde9b9e92bfb054e46797754154dd5883295c38936e5a8","cabal-sha256":"1422b0bcf4e7675116ca8d9f473bf239850c58c4518a56010e3bfebeac345ace"},"version":"0.12.0.3"},
     "vector-algorithms": {"dependencies":["base","bytestring","primitive","vector"],"location":{"type":"hackage","url":"https://hackage.haskell.org/package/vector-algorithms-0.8.0.1/vector-algorithms-0.8.0.1.tar.gz"},"name":"vector-algorithms","pinned":{"url":["https://hackage.haskell.org/package/vector-algorithms-0.8.0.1/vector-algorithms-0.8.0.1.tar.gz","https://s3.amazonaws.com/hackage.fpcomplete.com/package/vector-algorithms-0.8.0.1.tar.gz"],"sha256":"15bcde786dcf03861946885e030d3dbe3b683e1a6fc12d7317e115084f4637fe","cabal-sha256":"8496dd4b5e79c37c064e69ce20c5bb31048dfb4b59ce43de6a01b5d0fa384457"},"version":"0.8.0.1"},
     "void": {"dependencies":["base"],"location":{"type":"hackage","url":"https://hackage.haskell.org/package/void-0.7.3/void-0.7.3.tar.gz"},"name":"void","pinned":{"url":["https://hackage.haskell.org/package/void-0.7.3/void-0.7.3.tar.gz","https://s3.amazonaws.com/hackage.fpcomplete.com/package/void-0.7.3.tar.gz"],"sha256":"53af758ddc37dc63981671e503438d02c6f64a2d8744e9bec557a894431f7317","cabal-sha256":"13d30f62fcdf065e595d679d4ac8b4b0c1bb1a1b73db7b5b5a8f857cb5c8a546"},"version":"0.7.3"},

compatibility/test-windows.ps1

@@ -7,9 +7,9 @@ $ErrorActionPreference = 'Stop'
 # tests against HEAD. At the moment this includes the SDK release tarball
 # and the ledger-api-test-tool fat JAR.
 
-$test_args = "//..."
+$test_args = ""
 if (($args.length -ge 1) -and ($args[0] -eq "--quick")) {
-  $test_args = "//:head-quick"
+  $test_args = "--test_tag_filters=+head-quick"
 }
 write-output $test_args
 
@@ -49,4 +49,4 @@ bazel fetch @nodejs_dev_env//...
 bazel build //...
 bazel shutdown
 
-bazel test "$test_args"
+bazel test //... $test_args

compatibility/test.sh

@@ -59,12 +59,13 @@ start_postgresql
 
 bazel build //...
 
-BAZEL_TARGET="//..."
+BAZEL_ARGS=""
 if [ "${1:-}" = "--quick" ]; then
-    BAZEL_TARGET="//:head-quick"
+    BAZEL_ARGS="--test_tag_filters +head-quick"
 fi
 
-bazel test $BAZEL_TARGET \
+bazel test //... \
   --test_env "POSTGRESQL_HOST=${POSTGRESQL_HOST}" \
   --test_env "POSTGRESQL_PORT=${POSTGRESQL_PORT}" \
   --test_env "POSTGRESQL_USERNAME=${POSTGRESQL_USERNAME}" \
+  $BAZEL_ARGS

daml-assistant/daml-helper/src/DA/Daml/Helper/Main.hs

@@ -49,6 +49,11 @@ data Command
         , remainingArguments :: [String]
         , shutdownStdinClose :: Bool
         }
+    | RunPlatformJar
+        { args :: [String]
+        , logbackConfig :: FilePath
+        , shutdownStdinClose :: Bool
+        }
     | New { targetFolder :: FilePath, templateNameM :: Maybe String }
     | CreateDamlApp { targetFolder :: FilePath }
     -- ^ CreateDamlApp is sufficiently special that in addition to
@@ -89,6 +94,7 @@ commandParser = subparser $ fold
     , command "deploy" (info (deployCmd <**> helper) deployCmdInfo)
     , command "ledger" (info (ledgerCmd <**> helper) ledgerCmdInfo)
     , command "run-jar" (info runJarCmd forwardOptions)
+    , command "run-platform-jar" (info runPlatformJarCmd forwardOptions)
     , command "codegen" (info (codegenCmd <**> helper) forwardOptions)
     ]
   where
@@ -118,6 +124,11 @@ commandParser = subparser $ fold
         <*> many (argument str (metavar "ARG"))
         <*> stdinCloseOpt
 
+    runPlatformJarCmd = RunPlatformJar
+        <$> many (argument str (metavar "ARG"))
+        <*> strOption (long "logback-config")
+        <*> stdinCloseOpt
+
     newCmd = asum
         [ ListTemplates <$ flag' () (long "list" <> help "List the available project templates.")
         , New
@@ -332,6 +343,9 @@ runCommand = \case
     RunJar {..} ->
         (if shutdownStdinClose then withCloseOnStdin else id) $
         runJar jarPath mbLogbackConfig remainingArguments
+    RunPlatformJar {..} ->
+        (if shutdownStdinClose then withCloseOnStdin else id) $
+        runPlatformJar args logbackConfig
     New {..} -> runNew targetFolder templateNameM
     CreateDamlApp{..} -> runNew targetFolder (Just "create-daml-app")
     Init {..} -> runInit targetFolderM

daml-assistant/daml-helper/src/DA/Daml/Helper/Start.hs

@@ -3,6 +3,7 @@
 {-# LANGUAGE MultiWayIf #-}
 module DA.Daml.Helper.Start
     ( runStart
+    , runPlatformJar
 
     , withJar
     , withSandbox
@@ -26,6 +27,7 @@ module DA.Daml.Helper.Start
 
 import Control.Concurrent
 import Control.Concurrent.Async
+import Control.Exception
 import Control.Monad
 import Control.Monad.Extra hiding (fromMaybeM)
 import qualified Data.HashMap.Strict as HashMap
@@ -35,6 +37,7 @@ import DA.PortFile
 import qualified Data.Text as T
 import qualified Data.Text.Encoding as T
 import qualified Network.HTTP.Simple as HTTP
+import System.Environment (getEnvironment, getEnv, lookupEnv)
 import System.FilePath
 import System.Process.Typed
 import System.IO.Extra
@@ -45,6 +48,31 @@ import Data.Aeson
 import DA.Daml.Helper.Util
 import DA.Daml.Project.Config
 import DA.Daml.Project.Consts
+import DA.Daml.Project.Types
+
+-- [Note] The `platform-version` field:
+--
+-- Platform commands (at this point `daml sandbox`, `daml sandbox-classic` and `daml json-api`)
+-- are handled as follows:
+--
+-- 1. The assistant invokes `daml-helper` as usual. The assistant is not aware of the
+--    `platform-version` field or what is and what is not a platform command.
+-- 2. `daml-helper` reads the `DAML_PLATFORM_VERSION` env var falling back to
+--    the `platform-version` field from `daml.yaml`.
+-- 3.1. If the platform version is equal to the SDK version (from `DAML_SDK_VERSION`),
+--      `daml-helper` invokes the underlying tools (sandbox, JSON API, …) directly
+--      and we are finished.
+-- 3.2. If the platform version is different from the SDK version, `daml-helper`
+--      invokes the underlying tool via the assistant, setting both `DAML_SDK_VERSION`
+--      and `DAML_PLATFORM_VERSION` to the platform version.
+-- 4. The assistant will now invoke `daml-helper` from the platform version SDK.
+--    At this point, we are guaranteed to fall into 3.1 and daml-helper invokes the
+--    tool directly.
+--
+-- Note that this supports `platform-version`s for SDKs that are not
+-- aware of `platform-version`. In that case, `daml-helper` only has
+-- the codepath for invoking the underlying tool so we are also guaranteed
+-- to go to step 3.1.
 
 data SandboxPortSpec = FreePort | SpecifiedPort SandboxPort
 
@@ -69,10 +97,13 @@ navigatorURL :: NavigatorPort -> String
 navigatorURL (NavigatorPort p) = "http://localhost:" <> show p
 
 withSandbox :: SandboxClassic -> SandboxPortSpec -> [String] -> (Process () () () -> SandboxPort -> IO a) -> IO a
-withSandbox (SandboxClassic classic) portSpec args a = withTempFile $ \portFile -> do
+withSandbox (SandboxClassic classic) portSpec extraArgs a = withTempFile $ \portFile -> do
-    logbackArg <- getLogbackArg (damlSdkJarFolder </> "sandbox-logback.xml")
     let sandbox = if classic then "sandbox-classic" else "sandbox"
-    withJar damlSdkJar [logbackArg] ([sandbox, "--port", show (fromSandboxPortSpec portSpec), "--port-file", portFile] ++ args) $ \ph -> do
+    let args = [ sandbox
+               , "--port", show (fromSandboxPortSpec portSpec)
+               , "--port-file", portFile
+               ] ++ extraArgs
+    withPlatformJar args "sandbox-logback.xml" $ \ph -> do
         putStrLn "Waiting for sandbox to start: "
         port <- readPortFile maxRetries portFile
         a ph (SandboxPort port)
@@ -92,12 +123,15 @@ withNavigator (SandboxPort sandboxPort) navigatorPort args a = do
         a ph
 
 withJsonApi :: SandboxPort -> JsonApiPort -> [String] -> (Process () () () -> IO a) -> IO a
-withJsonApi (SandboxPort sandboxPort) (JsonApiPort jsonApiPort) args a = do
+withJsonApi (SandboxPort sandboxPort) (JsonApiPort jsonApiPort) extraArgs a = do
-    logbackArg <- getLogbackArg (damlSdkJarFolder </> "json-api-logback.xml")
+    let args =
-    let jsonApiArgs =
+            [ "json-api"
-            ["--ledger-host", "localhost", "--ledger-port", show sandboxPort,
+            , "--ledger-host", "localhost"
-             "--http-port", show jsonApiPort, "--allow-insecure-tokens"] <> args
+            , "--ledger-port", show sandboxPort
-    withJar damlSdkJar [logbackArg] ("json-api":jsonApiArgs) $ \ph -> do
+            , "--http-port", show jsonApiPort
+            , "--allow-insecure-tokens"
+            ] ++ extraArgs
+    withPlatformJar args "json-api-logback.xml" $ \ph -> do
         putStrLn "Waiting for JSON API to start: "
         -- The secret doesn’t matter here
         let token = JWT.encodeSigned (JWT.HMACSecret "secret") mempty mempty
@@ -225,3 +259,76 @@ runStart
             case mbJsonApiPort of
                 Nothing -> f sandboxPh
                 Just jsonApiPort -> withJsonApi sandboxPort jsonApiPort args f
+
+platformVersionEnvVar :: String
+platformVersionEnvVar = "DAML_PLATFORM_VERSION"
+
+-- | Returns the platform version determined as follows:
+--
+-- 1. If DAML_PLATFORM_VERSION is set return that.
+-- 2. If DAML_PROJECT is set and non-empty and `daml.yaml`
+--    has a `platform-version` field return that.
+-- 3. If `DAML_SDK_VERSION` is set return that.
+-- 4. Else we are invoked outside of the assistant and we throw an exception.
+getPlatformVersion :: IO String
+getPlatformVersion = do
+  mbPlatformVersion <- lookupEnv platformVersionEnvVar
+  case mbPlatformVersion of
+    Just platformVersion -> pure platformVersion
+    Nothing -> do
+      mbProjPath <- getProjectPath
+      case mbProjPath of
+        Just projPath -> do
+          project <- readProjectConfig (ProjectPath projPath)
+          case queryProjectConfig ["platform-version"] project of
+            Left err -> throwIO err
+            Right (Just ver) -> pure ver
+            Right Nothing -> getSdkVersion
+        Nothing -> getSdkVersion
+
+-- Convenience-wrapper around `withPlatformProcess` for commands that call the SDK
+-- JAR `daml-sdk.jar`.
+withPlatformJar
+    :: [String]
+    -- ^ Commands passed to the assistant and the platform JAR.
+    -> FilePath
+    -- ^ File name of the logback config.
+    -> (Process () () () -> IO a)
+    -> IO a
+withPlatformJar args logbackConf f = do
+    logbackArg <- getLogbackArg (damlSdkJarFolder </> logbackConf)
+    withPlatformProcess args (withJar damlSdkJar [logbackArg] args) f
+
+runPlatformJar :: [String] -> FilePath -> IO ()
+runPlatformJar args logback = do
+    withPlatformJar args logback (const $ pure ())
+
+withPlatformProcess
+    :: [String]
+    -- ^ List of commands passed to the assistant to invoke the command.
+    -> ((Process () () () -> IO a) -> IO a)
+    -- ^ Function to invoke the command in the current SDK.
+    -> (Process () () () -> IO a)
+    -> IO a
+withPlatformProcess args runInCurrentSdk f = do
+    platformVersion <- getPlatformVersion
+    sdkVersion <- getSdkVersion
+    if platformVersion == sdkVersion
+       then runInCurrentSdk f
+       else do
+        assistant <- getEnv damlAssistantEnvVar
+        env <- extendEnv
+            [ (platformVersionEnvVar, platformVersion)
+            , (sdkVersionEnvVar, platformVersion)
+            ]
+        withProcessWait_
+            ( setEnv env
+            $ proc assistant args
+            )
+            f
+
+extendEnv :: [(String, String)] -> IO [(String, String)]
+extendEnv xs = do
+    oldEnv <- getEnvironment
+    -- DAML_SDK is version specific so we need to filter it.
+    pure (xs ++ filter (\(k, _) -> k `notElem` (sdkPathEnvVar : map fst xs)) oldEnv)

docs/source/tools/assistant.rst

@@ -82,6 +82,7 @@ The existence of a ``daml.yaml`` file is what tells ``daml`` that this directory
 .. code-block:: yaml
 
     sdk-version: __VERSION__
+    platform-version: __VERSION__
     name: __PROJECT_NAME__
     source: daml
     scenario: Main:setup
@@ -111,6 +112,17 @@ Here is what each field means:
    external project that you want to build with a specific version.
 
    The assistant will warn you when it is time to update this setting (see the ``update-check`` setting in the global config  to control how often it checks, or to disable this check entirely).
+- ``platform-version``: Optional SDK version of platform components. Not setting this
+  is equivalent to setting it to the same version as ``sdk-version``. At the moment this includes
+  Sandbox, Sandbox classic and the HTTP JSON API both when invoked directly via ``daml sandbox``
+  as well as when invoked via ``daml start``. Changing the platform version is useful if you deploy
+  to a ledger that is running on a different SDK version than you use locally and you want to make
+  sure that you catch any issues during testing. E.g., you might compile your DAML code using
+  SDK 1.3.0 so you get improvements in DAML Studio but deploy to DABL which could still be running
+  a ledger and the JSON API from SDK 1.2.0. In that case, you can set ``sdk-version: 1.3.0``
+  and ``platform-version: 1.2.0``.
+  It is possible to override the platform version by setting the ``DAML_PLATFORM_VERSION``
+  environment variable.
 - ``name``: the name of the project. This determines the filename of the ``.dar`` file compiled by ``daml build``.
 - ``source``: the root folder of your DAML source code files relative to the project root.
 - ``scenario``: the name of the scenario to run when using ``daml start``.

release/sdk-config.yaml.tmpl

@@ -47,11 +47,11 @@ commands:
 - name: sandbox
   path: daml-helper/daml-helper
   desc: "Launch Sandbox"
-  args: ["run-jar", "--logback-config=daml-sdk/sandbox-logback.xml", "daml-sdk/daml-sdk.jar", "sandbox"]
+  args: ["run-platform-jar", "--logback-config=sandbox-logback.xml", "sandbox"]
 - name: sandbox-classic
   path: daml-helper/daml-helper
   desc: "Launch Sandbox Classic (the default Sandbox implementation for SDK <= 0.13.55)"
-  args: ["run-jar", "--logback-config=daml-sdk/sandbox-logback.xml", "daml-sdk/daml-sdk.jar", "sandbox-classic"]
+  args: ["run-platform-jar", "--logback-config=sandbox-logback.xml", "sandbox-classic"]
 - name: navigator
   path: daml-helper/daml-helper
   desc: "Launch the Navigator"
@@ -81,7 +81,7 @@ commands:
 - name: json-api
   path: daml-helper/daml-helper
   desc: "Launch the HTTP JSON API"
-  args: ["run-jar", "--logback-config=daml-sdk/json-api-logback.xml", "daml-sdk/daml-sdk.jar", "json-api"]
+  args: ["run-platform-jar", "--logback-config=json-api-logback.xml", "json-api"]
 - name: trigger-service
   path: daml-helper/daml-helper
   desc: "Launch the trigger service"