diff --git a/compatibility/BUILD b/compatibility/BUILD index e951de265e..f489ea4996 100644 --- a/compatibility/BUILD +++ b/compatibility/BUILD @@ -97,11 +97,6 @@ config_setting( if versions.is_at_least(sdk_version, platform_version) ] -test_suite( - name = "head-quick", - tags = ["head-quick"], -) - # We have two migration tests: migration-stable runs through all stable releases # including current HEAD. migration-all includes snapshot releases. diff --git a/compatibility/assistant/BUILD b/compatibility/assistant/BUILD new file mode 100644 index 0000000000..375505a3de --- /dev/null +++ b/compatibility/assistant/BUILD @@ -0,0 +1,42 @@ +# Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +load("@daml//bazel_tools:haskell.bzl", "da_haskell_test") + +da_haskell_test( + name = "assistant-platform-version", + srcs = glob(["src/**/*.hs"]), + data = [ + "@daml-sdk-tarball-latest-stable//file:downloaded", + "@head_sdk//:sdk-release-tarball.tar.gz", + ], + hackage_deps = [ + "base", + "tar-conduit", + "conduit", + "conduit-extra", + "text", + "filepath", + "directory", + "extra", + "process", + "typed-process", + "safe-exceptions", + "tasty", + "tasty-hunit", + "utf8-string", + "stm", + ], + main_function = "DA.Test.PlatformVersion.main", + tags = [ + "exclusive", + "head-quick", + ], + visibility = ["//visibility:public"], + deps = [ + "//bazel_tools:versions-hs-lib", + "//bazel_tools/daml_ledger:sandbox-helper", + "//bazel_tools/test_utils", + "@rules_haskell//tools/runfiles", + ], +) diff --git a/compatibility/assistant/src/DA/Test/PlatformVersion.hs b/compatibility/assistant/src/DA/Test/PlatformVersion.hs new file mode 100644 index 0000000000..05769a5c04 --- /dev/null +++ b/compatibility/assistant/src/DA/Test/PlatformVersion.hs @@ -0,0 +1,152 @@ +-- Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. +-- SPDX-License-Identifier: Apache-2.0 + +module DA.Test.PlatformVersion (main) where + +import qualified Bazel.Runfiles +import Control.Concurrent.STM +import Control.Exception.Safe +import Control.Monad +import Data.ByteString.Lazy.UTF8 (ByteString, toString) +import Data.Conduit ((.|), runConduitRes) +import qualified Data.Conduit.Combinators as Conduit +import qualified Data.Conduit.Tar as Tar +import qualified Data.Conduit.Zlib as Zlib +import Data.List +import qualified Data.Text as T +import DA.Test.Tar +import System.Directory +import System.Environment.Blank +import System.FilePath +import System.Info (os) +import System.IO.Extra +import qualified System.Process.Typed as Proc +import Test.Tasty +import Test.Tasty.HUnit +import Data.Maybe +import Sandbox (readPortFile, maxRetries) +import Versions (latestStableVersion) +import System.Process (interruptProcessGroupOf) + +main :: IO () +main = do + setEnv "TASTY_NUM_THREADS" "1" True + javaHome <- fromJust <$> getEnv "JAVA_HOME" + oldPath <- getSearchPath + setEnv "PATH" (intercalate [searchPathSeparator] ((javaHome "bin") : oldPath)) True + defaultMain $ withSdkResource $ \_getSdkPath -> + testGroup "platform-version" + [ testCase "no project, no DAML_PLATFORM_VERSION" $ do + env <- extendEnv [("DAML_SDK_VERSION", "0.0.0")] + out <- Proc.readProcessStdout_ (Proc.setEnv env $ Proc.shell "daml sandbox --help") + assertInfixOf "Sandbox version 0.0.0" out + out <- Proc.readProcessStdout_ (Proc.setEnv env $ Proc.shell "daml sandbox-classic --help") + assertInfixOf "Sandbox version 0.0.0" out + , testCase "no project, DAML_PLATFORM_VERSION" $ do + env <- extendEnv [("DAML_SDK_VERSION", "0.0.0"), ("DAML_PLATFORM_VERSION", latestStableVersion)] + out <- Proc.readProcessStdout_ + (Proc.setEnv env $ Proc.shell "daml sandbox --help") + assertInfixOf ("Sandbox version " <> latestStableVersion) out + out <- Proc.readProcessStdout_ + (Proc.setEnv env $ Proc.shell "daml sandbox-classic --help") + assertInfixOf ("Sandbox version " <> latestStableVersion) out + , testCase "no project, platform-version" $ withTempDir $ \tempDir -> do + writeFileUTF8 (tempDir "daml.yaml") $ unlines + [ "sdk-version: 0.0.0" + , "platform-version: " <> latestStableVersion + ] + out <- Proc.readProcessStdout_ (Proc.setWorkingDir tempDir (Proc.shell "daml sandbox --help")) + assertInfixOf ("Sandbox version " <> latestStableVersion) out + -- Env var takes precedence + env <- extendEnv [("DAML_PLATFORM_VERSION", "0.0.0")] + out <- Proc.readProcessStdout_ + (Proc.setWorkingDir tempDir $ Proc.setEnv env $ Proc.shell "daml sandbox --help") + assertInfixOf "Sandbox version 0.0.0" out + , testCase "daml start" $ withTempDir $ \tempDir -> do + writeFileUTF8 (tempDir "daml.yaml") $ unlines + [ "sdk-version: 0.0.0" + , "platform-version: " <> latestStableVersion + , "name: foobar" + , "version: 0.0.1" + , "dependencies: [daml-prim, daml-stdlib]" + , "source: ." + , "parties: []" + ] + let conf = + Proc.setCreateGroup True $ + Proc.setStdin Proc.createPipe $ + Proc.setStdout Proc.byteStringOutput $ + Proc.setWorkingDir tempDir $ + Proc.shell $ + "daml start --shutdown-stdin-close --open-browser=no --json-api-option --port-file --json-api-option " <> show (tempDir "portfile") + getOut <- Proc.withProcessWait conf $ \ph -> do + -- Wait for the port file as a sign that the JSON API has started. + _ <- readPortFile maxRetries (tempDir "portfile") + hClose (Proc.getStdin ph) + -- Process management on Windows is a nightmare so we opt + -- for the safest option of creating a group and killing everything + -- in that group. + interruptProcessGroupOf (Proc.unsafeProcessHandle ph) + pure $ Proc.getStdout ph + out <- atomically getOut + putStrLn "got stdout" + assertInfixOf ("sandbox version " <> latestStableVersion) out + -- Navigator, sadly not prefixed with Navigator + assertInfixOf "Version 0.0.0" out + -- JSON API, doesn’t even print a version but let’s at least check + -- that something started. + assertInfixOf "httpPort=7575" out + + ] + +extendEnv :: [(String, String)] -> IO [(String, String)] +extendEnv xs = do + oldEnv <- getEnvironment + pure (xs ++ filter (\(k, _) -> k `notElem` map fst xs) oldEnv) + +withSdkResource :: (IO FilePath -> TestTree) -> TestTree +withSdkResource f = + withTempDirResource $ \getDir -> + withResource (installSdk =<< getDir) (const $ pure ()) (const $ f getDir) + where installSdk targetDir = do + runfiles <- Bazel.Runfiles.create + let headSdk = Bazel.Runfiles.rlocation runfiles "head_sdk/sdk-release-tarball.tar.gz" + let latestStableSdk = Bazel.Runfiles.rlocation runfiles "daml-sdk-tarball-latest-stable/file/downloaded" + setEnv "DAML_HOME" targetDir True + withTempDir $ \extractDir -> do + runConduitRes + $ Conduit.sourceFileBS headSdk + .| Zlib.ungzip + .| Tar.untar (restoreFile (\a b -> fail (T.unpack $ a <> b)) extractDir) + Proc.runProcess_ $ + Proc.proc + (extractDir "daml" exe "daml") + ["install", "--install-assistant=yes", "--set-path=no", extractDir, "--quiet"] + pure () + oldPath <- getSearchPath + setEnv "PATH" (intercalate [searchPathSeparator] ((targetDir "bin") : oldPath)) True + Proc.runProcess_ (Proc.shell $ "daml install --quiet " <> latestStableSdk) + pure () + +withTempDirResource :: (IO FilePath -> TestTree) -> TestTree +withTempDirResource f = withResource newTempDir delete (f . fmap fst) + -- The delete action provided by `newTempDir` calls `removeDirectoryRecursively` + -- and silently swallows errors. SDK installations are marked read-only + -- which means that they don’t end up being removed which is obviously + -- not what we intend. + -- As usual Windows is terrible and doesn’t let you remove the SDK + -- if there is a process running. Simultaneously it is also terrible + -- at process management so we end up with running processes + -- since child processes aren’t torn down properly + -- (Bazel will kill them later when the test finishes). Therefore, + -- we ignore exceptions and hope for the best. On Windows that + -- means we still leak directories :( + where delete (d, _delete) = void $ tryIO $ removePathForcibly d + +exe :: FilePath -> FilePath +exe | os == "mingw32" = (<.> "exe") + | otherwise = id + +assertInfixOf :: String -> ByteString -> Assertion +assertInfixOf needle haystack = assertBool ("Expected " <> show needle <> " in output but but got " <> show haystack) (needle `isInfixOf` toString haystack) + diff --git a/compatibility/bazel-haskell-deps.bzl b/compatibility/bazel-haskell-deps.bzl index 3ac6b54a0d..f66cfe739a 100644 --- a/compatibility/bazel-haskell-deps.bzl +++ b/compatibility/bazel-haskell-deps.bzl @@ -67,14 +67,17 @@ def daml_haskell_deps(): "safe-exceptions", "semver", "split", + "stm", "tagged", "tar-conduit", "tasty", "tasty-hunit", "text", + "typed-process", "optparse-applicative", "unix-compat", "unordered-containers", + "utf8-string", "uuid", ] + (["unix"] if not is_windows else ["Win32"]), stack = "@stack_windows//:stack.exe" if is_windows else None, diff --git a/compatibility/bazel_tools/BUILD b/compatibility/bazel_tools/BUILD index 50ea39d00f..a0ca763854 100644 --- a/compatibility/bazel_tools/BUILD +++ b/compatibility/bazel_tools/BUILD @@ -1,6 +1,30 @@ +load("@daml//bazel_tools:haskell.bzl", "da_haskell_library") +load("//:versions.bzl", "latest_stable_version") + exports_files([ "daml_ledger_test.sh", "create_daml_app_test.sh", "sandbox-with-postgres.sh", "daml.cc", ]) + +genrule( + name = "versions-hs-lib-gen", + srcs = [], + outs = ["Versions.hs"], + cmd = """ + cat > $@ < helper) deployCmdInfo) , command "ledger" (info (ledgerCmd <**> helper) ledgerCmdInfo) , command "run-jar" (info runJarCmd forwardOptions) + , command "run-platform-jar" (info runPlatformJarCmd forwardOptions) , command "codegen" (info (codegenCmd <**> helper) forwardOptions) ] where @@ -118,6 +124,11 @@ commandParser = subparser $ fold <*> many (argument str (metavar "ARG")) <*> stdinCloseOpt + runPlatformJarCmd = RunPlatformJar + <$> many (argument str (metavar "ARG")) + <*> strOption (long "logback-config") + <*> stdinCloseOpt + newCmd = asum [ ListTemplates <$ flag' () (long "list" <> help "List the available project templates.") , New @@ -332,6 +343,9 @@ runCommand = \case RunJar {..} -> (if shutdownStdinClose then withCloseOnStdin else id) $ runJar jarPath mbLogbackConfig remainingArguments + RunPlatformJar {..} -> + (if shutdownStdinClose then withCloseOnStdin else id) $ + runPlatformJar args logbackConfig New {..} -> runNew targetFolder templateNameM CreateDamlApp{..} -> runNew targetFolder (Just "create-daml-app") Init {..} -> runInit targetFolderM diff --git a/daml-assistant/daml-helper/src/DA/Daml/Helper/Start.hs b/daml-assistant/daml-helper/src/DA/Daml/Helper/Start.hs index 4756d4d7ce..8222966eaf 100644 --- a/daml-assistant/daml-helper/src/DA/Daml/Helper/Start.hs +++ b/daml-assistant/daml-helper/src/DA/Daml/Helper/Start.hs @@ -3,6 +3,7 @@ {-# LANGUAGE MultiWayIf #-} module DA.Daml.Helper.Start ( runStart + , runPlatformJar , withJar , withSandbox @@ -26,6 +27,7 @@ module DA.Daml.Helper.Start import Control.Concurrent import Control.Concurrent.Async +import Control.Exception import Control.Monad import Control.Monad.Extra hiding (fromMaybeM) import qualified Data.HashMap.Strict as HashMap @@ -35,6 +37,7 @@ import DA.PortFile import qualified Data.Text as T import qualified Data.Text.Encoding as T import qualified Network.HTTP.Simple as HTTP +import System.Environment (getEnvironment, getEnv, lookupEnv) import System.FilePath import System.Process.Typed import System.IO.Extra @@ -45,6 +48,31 @@ import Data.Aeson import DA.Daml.Helper.Util import DA.Daml.Project.Config import DA.Daml.Project.Consts +import DA.Daml.Project.Types + +-- [Note] The `platform-version` field: +-- +-- Platform commands (at this point `daml sandbox`, `daml sandbox-classic` and `daml json-api`) +-- are handled as follows: +-- +-- 1. The assistant invokes `daml-helper` as usual. The assistant is not aware of the +-- `platform-version` field or what is and what is not a platform command. +-- 2. `daml-helper` reads the `DAML_PLATFORM_VERSION` env var falling back to +-- the `platform-version` field from `daml.yaml`. +-- 3.1. If the platform version is equal to the SDK version (from `DAML_SDK_VERSION`), +-- `daml-helper` invokes the underlying tools (sandbox, JSON API, …) directly +-- and we are finished. +-- 3.2. If the platform version is different from the SDK version, `daml-helper` +-- invokes the underlying tool via the assistant, setting both `DAML_SDK_VERSION` +-- and `DAML_PLATFORM_VERSION` to the platform version. +-- 4. The assistant will now invoke `daml-helper` from the platform version SDK. +-- At this point, we are guaranteed to fall into 3.1 and daml-helper invokes the +-- tool directly. +-- +-- Note that this supports `platform-version`s for SDKs that are not +-- aware of `platform-version`. In that case, `daml-helper` only has +-- the codepath for invoking the underlying tool so we are also guaranteed +-- to go to step 3.1. data SandboxPortSpec = FreePort | SpecifiedPort SandboxPort @@ -69,10 +97,13 @@ navigatorURL :: NavigatorPort -> String navigatorURL (NavigatorPort p) = "http://localhost:" <> show p withSandbox :: SandboxClassic -> SandboxPortSpec -> [String] -> (Process () () () -> SandboxPort -> IO a) -> IO a -withSandbox (SandboxClassic classic) portSpec args a = withTempFile $ \portFile -> do - logbackArg <- getLogbackArg (damlSdkJarFolder "sandbox-logback.xml") +withSandbox (SandboxClassic classic) portSpec extraArgs a = withTempFile $ \portFile -> do let sandbox = if classic then "sandbox-classic" else "sandbox" - withJar damlSdkJar [logbackArg] ([sandbox, "--port", show (fromSandboxPortSpec portSpec), "--port-file", portFile] ++ args) $ \ph -> do + let args = [ sandbox + , "--port", show (fromSandboxPortSpec portSpec) + , "--port-file", portFile + ] ++ extraArgs + withPlatformJar args "sandbox-logback.xml" $ \ph -> do putStrLn "Waiting for sandbox to start: " port <- readPortFile maxRetries portFile a ph (SandboxPort port) @@ -92,12 +123,15 @@ withNavigator (SandboxPort sandboxPort) navigatorPort args a = do a ph withJsonApi :: SandboxPort -> JsonApiPort -> [String] -> (Process () () () -> IO a) -> IO a -withJsonApi (SandboxPort sandboxPort) (JsonApiPort jsonApiPort) args a = do - logbackArg <- getLogbackArg (damlSdkJarFolder "json-api-logback.xml") - let jsonApiArgs = - ["--ledger-host", "localhost", "--ledger-port", show sandboxPort, - "--http-port", show jsonApiPort, "--allow-insecure-tokens"] <> args - withJar damlSdkJar [logbackArg] ("json-api":jsonApiArgs) $ \ph -> do +withJsonApi (SandboxPort sandboxPort) (JsonApiPort jsonApiPort) extraArgs a = do + let args = + [ "json-api" + , "--ledger-host", "localhost" + , "--ledger-port", show sandboxPort + , "--http-port", show jsonApiPort + , "--allow-insecure-tokens" + ] ++ extraArgs + withPlatformJar args "json-api-logback.xml" $ \ph -> do putStrLn "Waiting for JSON API to start: " -- The secret doesn’t matter here let token = JWT.encodeSigned (JWT.HMACSecret "secret") mempty mempty @@ -225,3 +259,76 @@ runStart case mbJsonApiPort of Nothing -> f sandboxPh Just jsonApiPort -> withJsonApi sandboxPort jsonApiPort args f + +platformVersionEnvVar :: String +platformVersionEnvVar = "DAML_PLATFORM_VERSION" + +-- | Returns the platform version determined as follows: +-- +-- 1. If DAML_PLATFORM_VERSION is set return that. +-- 2. If DAML_PROJECT is set and non-empty and `daml.yaml` +-- has a `platform-version` field return that. +-- 3. If `DAML_SDK_VERSION` is set return that. +-- 4. Else we are invoked outside of the assistant and we throw an exception. +getPlatformVersion :: IO String +getPlatformVersion = do + mbPlatformVersion <- lookupEnv platformVersionEnvVar + case mbPlatformVersion of + Just platformVersion -> pure platformVersion + Nothing -> do + mbProjPath <- getProjectPath + case mbProjPath of + Just projPath -> do + project <- readProjectConfig (ProjectPath projPath) + case queryProjectConfig ["platform-version"] project of + Left err -> throwIO err + Right (Just ver) -> pure ver + Right Nothing -> getSdkVersion + Nothing -> getSdkVersion + +-- Convenience-wrapper around `withPlatformProcess` for commands that call the SDK +-- JAR `daml-sdk.jar`. +withPlatformJar + :: [String] + -- ^ Commands passed to the assistant and the platform JAR. + -> FilePath + -- ^ File name of the logback config. + -> (Process () () () -> IO a) + -> IO a +withPlatformJar args logbackConf f = do + logbackArg <- getLogbackArg (damlSdkJarFolder logbackConf) + withPlatformProcess args (withJar damlSdkJar [logbackArg] args) f + +runPlatformJar :: [String] -> FilePath -> IO () +runPlatformJar args logback = do + withPlatformJar args logback (const $ pure ()) + +withPlatformProcess + :: [String] + -- ^ List of commands passed to the assistant to invoke the command. + -> ((Process () () () -> IO a) -> IO a) + -- ^ Function to invoke the command in the current SDK. + -> (Process () () () -> IO a) + -> IO a +withPlatformProcess args runInCurrentSdk f = do + platformVersion <- getPlatformVersion + sdkVersion <- getSdkVersion + if platformVersion == sdkVersion + then runInCurrentSdk f + else do + assistant <- getEnv damlAssistantEnvVar + env <- extendEnv + [ (platformVersionEnvVar, platformVersion) + , (sdkVersionEnvVar, platformVersion) + ] + withProcessWait_ + ( setEnv env + $ proc assistant args + ) + f + +extendEnv :: [(String, String)] -> IO [(String, String)] +extendEnv xs = do + oldEnv <- getEnvironment + -- DAML_SDK is version specific so we need to filter it. + pure (xs ++ filter (\(k, _) -> k `notElem` (sdkPathEnvVar : map fst xs)) oldEnv) diff --git a/docs/source/tools/assistant.rst b/docs/source/tools/assistant.rst index 2efeebc2ff..1b04dd38b6 100644 --- a/docs/source/tools/assistant.rst +++ b/docs/source/tools/assistant.rst @@ -82,6 +82,7 @@ The existence of a ``daml.yaml`` file is what tells ``daml`` that this directory .. code-block:: yaml sdk-version: __VERSION__ + platform-version: __VERSION__ name: __PROJECT_NAME__ source: daml scenario: Main:setup @@ -111,6 +112,17 @@ Here is what each field means: external project that you want to build with a specific version. The assistant will warn you when it is time to update this setting (see the ``update-check`` setting in the global config to control how often it checks, or to disable this check entirely). +- ``platform-version``: Optional SDK version of platform components. Not setting this + is equivalent to setting it to the same version as ``sdk-version``. At the moment this includes + Sandbox, Sandbox classic and the HTTP JSON API both when invoked directly via ``daml sandbox`` + as well as when invoked via ``daml start``. Changing the platform version is useful if you deploy + to a ledger that is running on a different SDK version than you use locally and you want to make + sure that you catch any issues during testing. E.g., you might compile your DAML code using + SDK 1.3.0 so you get improvements in DAML Studio but deploy to DABL which could still be running + a ledger and the JSON API from SDK 1.2.0. In that case, you can set ``sdk-version: 1.3.0`` + and ``platform-version: 1.2.0``. + It is possible to override the platform version by setting the ``DAML_PLATFORM_VERSION`` + environment variable. - ``name``: the name of the project. This determines the filename of the ``.dar`` file compiled by ``daml build``. - ``source``: the root folder of your DAML source code files relative to the project root. - ``scenario``: the name of the scenario to run when using ``daml start``. diff --git a/release/sdk-config.yaml.tmpl b/release/sdk-config.yaml.tmpl index 52d17040da..8233c3ed6b 100644 --- a/release/sdk-config.yaml.tmpl +++ b/release/sdk-config.yaml.tmpl @@ -47,11 +47,11 @@ commands: - name: sandbox path: daml-helper/daml-helper desc: "Launch Sandbox" - args: ["run-jar", "--logback-config=daml-sdk/sandbox-logback.xml", "daml-sdk/daml-sdk.jar", "sandbox"] + args: ["run-platform-jar", "--logback-config=sandbox-logback.xml", "sandbox"] - name: sandbox-classic path: daml-helper/daml-helper desc: "Launch Sandbox Classic (the default Sandbox implementation for SDK <= 0.13.55)" - args: ["run-jar", "--logback-config=daml-sdk/sandbox-logback.xml", "daml-sdk/daml-sdk.jar", "sandbox-classic"] + args: ["run-platform-jar", "--logback-config=sandbox-logback.xml", "sandbox-classic"] - name: navigator path: daml-helper/daml-helper desc: "Launch the Navigator" @@ -81,7 +81,7 @@ commands: - name: json-api path: daml-helper/daml-helper desc: "Launch the HTTP JSON API" - args: ["run-jar", "--logback-config=daml-sdk/json-api-logback.xml", "daml-sdk/daml-sdk.jar", "json-api"] + args: ["run-platform-jar", "--logback-config=json-api-logback.xml", "json-api"] - name: trigger-service path: daml-helper/daml-helper desc: "Launch the trigger service"