Prototype to accelerate Windows development (#411)

* Initial rattle prototype * Build the IDE core * Ignore the Rattle directory * Clean up the dependencies * Require stack-1.10, since that does much better extra-dep caching * Streamline the ghc-lib dependencies * Compile more pieces * Add a build.bat * Make the Windows build use the correct stack.yaml to bootstrap * Fix up enough to build on Windows * Generate the dylib's on Mac * Remove accidental -v * Make the Haskell build driven by the Bazel metadata * Get proto3-suite building * Delete the unneeded haskell-dependencies * Allow generating the proto files and compiling them * Fix metadata to deal with """ syntax * Fix metadata to deal with a list of globs * More work in the direction of daml-ghc * Use correct daml_lf proto version * Tell GHC to use shared objects in TH * Specify needed packages * wip * wip * Switch to the fork of gRPC-haskell * Build executables with rattle * setup build.sbt in daml-lf * Build binaries with rattle * rattle-sbt, move scala build scripts out of daml-lf subdir, and into rattle subdir * convert scala-build.sh into MainScala.hs * Clean up rattle build * Pre-merge clean up * Switch to the newer version of ghc-lib-parser * remove dev ls from MainScala.hs * compile java generated from protos as separate projects * Add copyright headers * HLint fixes * Uscrewup an HLint fix * fix scala formatting of rattle/build.sbt
2024-11-04 00:36:58 +03:00 · 2019-04-12 10:45:11 +01:00 · 2019-04-12 10:45:11 +01:00 · bfa019997a
commit bfa019997a
parent 5989549081
13 changed files with 685 additions and 0 deletions
--- a/.gitignore
+++ b/.gitignore
@ -84,3 +84,7 @@ ledger-api/.bin
 .ijwb
 .bazelproject
 .bazel-cache
+.rattle
+daml-lf/project
+daml-lf/gen
+daml-lf/build.sbt
--- a/rattle/Main.hs
+++ b/rattle/Main.hs
@ -0,0 +1,291 @@
+-- Copyright (c) 2019 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+-- SPDX-License-Identifier: Apache-2.0
+
+{-# LANGUAGE ViewPatterns, RecordWildCards #-}
+{-# OPTIONS_GHC -Wall #-}
+{-# OPTIONS_GHC -Wno-name-shadowing #-}
+{-# OPTIONS_GHC -Wno-missing-signatures #-}
+
+{- HLINT ignore "Avoid restricted extensions" -}
+{- HLINT ignore "Avoid restricted flags" -}
+
+module Main(main) where
+
+import Rattle
+import Metadata
+import Util
+import Data.Char
+import Data.Maybe
+import System.Info.Extra
+import System.Process.Extra
+import System.FilePattern.Directory
+import System.FilePath
+import System.Directory
+import System.Environment
+import System.IO.Unsafe
+import Control.Monad.Extra
+import Data.List.Extra
+
+targets =
+    ["libs-haskell/prettyprinter-syntax"
+    ,"daml-assistant"
+    ,"daml-foundations/daml-tools/da-hs-damlc-app"
+    ,"daml-foundations/daml-tools/da-hs-daml-cli"
+    ,"compiler/haskell-ide-core"
+    ,"libs-haskell/da-hs-base"
+    ,"libs-haskell/da-hs-pretty"
+    ,"libs-haskell/da-hs-language-server"
+    ,"libs-haskell/da-hs-json-rpc"
+    ,"compiler/daml-lf-ast"
+    ,"compiler/daml-lf-tools"
+    ,"compiler/daml-lf-proto"
+    ,"daml-lf/archive"
+    ,"daml-foundations/daml-ghc"
+    ,"libs-haskell/bazel-runfiles"
+    ,"compiler/scenario-service/client"
+    ,"compiler/scenario-service/protos"
+    ]
+
+haskellExts =
+    ["BangPatterns"
+    ,"DeriveDataTypeable"
+    ,"DeriveFoldable"
+    ,"DeriveFunctor"
+    ,"DeriveGeneric"
+    ,"DeriveTraversable"
+    ,"FlexibleContexts"
+    ,"GeneralizedNewtypeDeriving"
+    ,"LambdaCase"
+    ,"NamedFieldPuns"
+    ,"NumericUnderscores"
+    ,"PackageImports"
+    ,"RecordWildCards"
+    ,"ScopedTypeVariables"
+    ,"StandaloneDeriving"
+    ,"TupleSections"
+    ,"TypeApplications"
+    ,"ViewPatterns"
+    ]
+
+haskellFlags =
+    ["-Wall", "-Werror", "-Wincomplete-uni-patterns", "-Wno-name-shadowing"
+    ,"-fno-omit-yields"
+    ,"-threaded", "-rtsopts"
+    -- run on two cores, disable idle & parallel GC
+    ,"-with-rtsopts=-N2 -qg -I0"
+    ]
+
+main = rattle $ do
+    putStrLn "  ---- Starting rattle build ----"
+    metadata <- concatMapM (\x -> readMetadata $ x </> "BUILD.bazel") targets
+    metadata <- return [x{dhl_deps = dhl_deps x `intersect` map dhl_name metadata} | x <- metadata]
+    metadata <- return $ topSort [(dhl_name, dhl_deps, x) | x@Metadata{..} <- metadata]
+
+    -- Figure out what libraries we need
+    let neededDeps = ["grpc-haskell","proto3-suite"] ++ nubSort (concatMap dhl_hazel_deps metadata)
+
+    installDependencies neededDeps
+
+    -- generate the LF protobuf output
+    let lfMajorVersions = ["0", "1", "dev"]
+    forM_ ("":lfMajorVersions) $ \v -> do
+        cmd_ (Cwd "daml-lf/archive")
+            "compile-proto-file --proto" ["da/daml_lf" ++ ['_' | v /= ""] ++ v ++ ".proto"]
+            "--out ../../.rattle/generated/daml-lf/haskell"
+
+    -- generate the scenario protobuf output
+    cmd_ (Cwd "compiler/scenario-service/protos")
+            "compile-proto-file --proto scenario_service.proto"
+            "--out ../../../.rattle/generated/scenario-service/haskell"
+
+    -- build all the Haskell projects
+    let trans = transitive [(dhl_name, dhl_deps) | Metadata{..} <- metadata]
+    let patch x | dhl_name x == "daml_lf_haskell_proto" = x{dhl_dir = ".rattle/generated/daml-lf/haskell", dhl_srcs = ["**/*.hs"]}
+                | dhl_name x == "scenario_service_haskell_proto" = x{dhl_dir = ".rattle/generated/scenario-service/haskell", dhl_srcs = ["**/*.hs"]}
+                | otherwise = x
+
+    putStrLn $ intercalate "\n - " $ "Building targets:" : (prettyName <$> metadata)
+
+    forM_ metadata $ \m -> buildHaskellPackage $ patch m{dhl_deps = nubSort $ concatMap trans $ dhl_deps m}
+
+    putStrLn $ intercalate "\n - " $ "Successfully built targets:" : (prettyName <$> metadata)
+
+-- | Builds a haskell package.
+--
+-- If the package is an executable, builds all modules and links them to an
+-- executable.
+--
+-- If the package is a library, builds all modules and creates a package DB
+-- with both a static and a dynamic lib.
+buildHaskellPackage :: Metadata -> IO ()
+buildHaskellPackage (fixNames -> o@Metadata{..}) = do
+    putStrLn "-----------"
+    putStrLn $ "  " <> prettyName o
+    putStrLn "-----------"
+    print ("buildHaskellPackage",o)
+
+    files <- map (drop $ if null dhl_src_strip_prefix then 0 else length dhl_src_strip_prefix + 1) <$>
+             getDirectoryFiles dhl_dir dhl_srcs
+
+    -- Work around the fact that SdkVersion is bazel-generated
+    files <- pure $ files <> (case dhl_name of
+      "da-hs-daml-cli" -> ["SdkVersion.hs"]
+      _ -> [])
+
+    let modules = map (intercalate "." . splitDirectories . dropExtension) files
+
+    let dirs = [dhl_dir </> dhl_src_strip_prefix] <> (case dhl_name of
+            -- Work around the fact that SdkVersion is bazel-generated
+            "da-hs-daml-cli" -> ["rattle" </> "hacks"]
+            _ -> [])
+
+    let commonFlags =
+            -- set the correct output directories for objects, headers, etc
+            [ flag ++ "=.rattle/haskell" </> dhl_name
+            | flag <- ["-outputdir","-odir","-hidir","-stubdir"]
+            ] <>
+
+            -- specify which directories to look into for modules
+            ["-i" ++ dir | dir <- dirs] <>
+
+            -- makes sure GHC uses shared objects in RTS linker
+            -- https://github.com/ghc/ghc/blob/cf9e1837adc647c90cfa176669d14e0d413c043d/compiler/main/DynFlags.hs#L2087
+            ["-fexternal-interpreter"] <>
+
+            -- Ensure GHC finds the packages
+            concat [["-package", d] | d <- dhl_deps ] <>
+            concat [["-package", d] | d <- dhl_hazel_deps ] <>
+            ["-package-db=.rattle/haskell" </> d </> "pkg.db" | d <- dhl_deps] <>
+
+            -- All extra extensions and flags
+            map ("-X"++) haskellExts <> haskellFlags
+
+    case dhl_main_is of
+      Just main_ -> do
+
+        let mainMod =
+                -- get the module name by dropping the function name
+                -- Foo.Bar.main -> Foo.Bar
+                intercalate "." .
+                (reverse . drop 1 . reverse . wordsBy (== '.')) $ main_
+
+        -- Include the main module name, if it's missing
+        modules <- pure $ nubSort $ mainMod : modules
+
+        rpath <- ghcLibPath
+
+        -- XXX: is this going to work on windows?
+        cmd_ "mkdir" "-p" (".rattle/haskell" </> dhl_name)
+
+        cmd_ "ghc" commonFlags
+            modules
+            "-o" [".rattle/haskell" </> dhl_name </> dhl_name]
+            ["-main-is", main_]
+            -- Set the rpath so that the executable uses the proper libffi so
+            ["-optl-Wl,-rpath," <> rpath]
+
+      Nothing -> do
+        cmd_ "ghc" commonFlags
+            "-dynosuf=dyn_o -dynhisuf=dyn_hi"
+            ["-static"] ["-dynamic-too" | not isWindows]
+            modules
+            ["-this-unit-id=" ++ dhl_name]
+
+        cmd_ "ar -r -s" [".rattle/haskell" </> dhl_name </> "libHS" ++ dhl_name ++ ".a"]
+            [".rattle/haskell" </> dhl_name </> x -<.> "o" | x <- files]
+        if isWindows then
+            cmd_ "ld -x -r -o" [".rattle/haskell" </> dhl_name </> "HS" ++ dhl_name ++ ".o"]
+                [".rattle/haskell" </> dhl_name </> x -<.> "o" | x <- files]
+        else
+            cmd_ "ghc -shared -dynamic" ["-dynload deploy"] "-o" [".rattle/haskell" </> dhl_name </> "libHS" ++ dhl_name ++ ".dylib"]
+                [".rattle/haskell" </> dhl_name </> x -<.> "dyn_o" | x <- files]
+        unlessM (doesDirectoryExist $ ".rattle/haskell" </> dhl_name </> "pkg.db") $
+            cmd_ "ghc-pkg init" [".rattle/haskell" </> dhl_name </> "pkg.db"]
+        keys <- map (drop 4) . filter (not . isInfixOf "haskeline") . lines <$>
+            systemOutput_ "ghc-pkg field \"*\" key"
+        writeFile (".rattle/haskell" </> dhl_name </> "pkg.db" </> dhl_name <.> "conf") $ unlines $
+            ["name: " ++ dhl_name
+            ,"version: 0"
+            ,"id: " ++ dhl_name
+            ,"key: " ++ dhl_name
+            ,"hs-libraries: HS" ++ dhl_name
+            ,"import-dirs: ${pkgroot}"
+            ,"library-dirs: ${pkgroot}"
+            ,"dynamic-library-dirs: ${pkgroot}"
+            ,"exposed: True"
+            ,"exposed-modules:"] ++
+            map (" "++) modules ++
+            ["depends:"] ++
+            keys ++ map (" " ++) dhl_deps
+        cmd_ "ghc-pkg recache" ["--package-db=" ++ ".rattle/haskell" </> dhl_name </> "pkg.db"]
+
+-- | Installs all the specified packages in the stack database.
+installDependencies :: [String] -> IO ()
+installDependencies neededDeps = do
+
+    putStrLn $ "Found " <> show (length neededDeps) <> " needed dependencies"
+
+    -- Diff the installed dependencies to figure out what's missing
+    Stdout sout <- cmd stack "exec" "--" ["ghc-pkg", "list"]
+    installedDeps <- pure $ nubSort $ parseLibs $ lines sout
+    missingDeps <- pure $ neededDeps \\ installedDeps
+
+    unless (null missingDeps) $
+      putStrLn $ unwords $ "Installing missing deps:" : missingDeps
+
+    -- 'stack build' sometimes gets confused if we ask for several libs to be
+    -- installed at once, so we install them one by one
+    forM_ missingDeps $ \dep -> do
+      putStrLn $ "Installing " <> dep
+      cmd_ stack "build" dep
+
+    -- Double check that everything's available, otherwise fail
+    Stdout sout <- cmd stack "exec" " --" ["ghc-pkg", "list"]
+    installedDeps <- pure $ nubSort $ parseLibs $ lines sout
+    missingDeps <- pure $ neededDeps \\ installedDeps
+    unless (null missingDeps) $ do
+        error $ unwords $ "Could not find missing deps, try adding to extra-deps? :" : missingDeps
+
+{-# NOINLINE stack #-}
+stack :: String
+stack = unsafePerformIO $ do -- assume LD_LIBRARY_PATH won't change during a run
+    -- XXX: We may have to use ';' on Windows
+    dirs <- maybe [] (wordsBy (== ':')) <$> lookupEnv "LD_LIBRARY_PATH"
+    pure $ unwords $
+      [ "stack" ] <>
+      ((\d -> ["--extra-lib-dirs", d]) `concatMap` dirs) <>
+      ["--stack-yaml=rattle/stack.yaml"]
+
+-- | Ask stack for the path of GHC's bundled libs
+-- XXX: Path gymnastics happening here, let's hope stack is somewhat consistent
+ghcLibPath :: IO String
+ghcLibPath = do
+    Stdout str <- cmd stack "path" "--compiler-bin"
+    case reverse . splitPath <$> lines str of
+      ["bin" : ver : rest] -> do
+        pure $ joinPath $ reverse $ "rts" : ver : "lib": ver : rest
+      xs -> error $ "Couldn't parse ghc path: " <> show xs
+
+-- | Parse the output of ghc-pkg list into a list of package names, e.g.
+--  [ "lens", "conduit", ... ]
+parseLibs :: [String] -> [String]
+parseLibs = mapMaybe $ fmap parseLib . parseLine
+  where
+    parseLib :: String -> String
+    parseLib = reverse . drop 1 . dropWhile (\c -> isDigit c || (c == '.')) . reverse
+    parseLine :: String -> Maybe String
+    parseLine = stripPrefix "    "
+
+prettyName :: Metadata -> String
+prettyName Metadata{..} =
+    dhl_name <> maybe " (lib)" (const " (bin)") dhl_main_is
+
+-- some packages (e.g. daml_lf_haskell_proto) use names which are unsuitable for GHC
+-- so we switch them and the dependencies here
+fixNames :: Metadata -> Metadata
+fixNames o = o
+    { dhl_name = fixName (dhl_name o)
+    , dhl_deps = map fixName (dhl_deps o)
+    }
+  where
+    fixName = replace "_" "-"
--- a/rattle/MainScala.hs
+++ b/rattle/MainScala.hs
@ -0,0 +1,37 @@
+-- Copyright (c) 2019 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+-- SPDX-License-Identifier: Apache-2.0
+
+module MainScala(main) where
+
+import Rattle
+import Metadata
+import Util
+import Data.Char
+import Data.Maybe
+import System.IO.Extra
+import System.Info.Extra
+import System.Process.Extra
+import System.FilePattern.Directory
+import System.FilePath
+import System.Directory
+import System.Environment
+import System.IO.Unsafe
+import Control.Monad.Extra
+import Data.List.Extra
+
+bash :: String -> IO ()
+bash s = cmd_ "bash" ["-v", "-c", s]
+
+main :: IO ()
+main = rattle $ do
+    putStrLn "  ---- Starting SCALA rattle build ----"
+
+    bash "mkdir -p daml-lf/gen/for-archive/src/main/scala"
+    bash "(cd daml-lf/archive; protoc da/*.proto --java_out=../gen/for-archive/src/main/scala)"
+
+    bash "mkdir -p daml-lf/gen/for-transaction/src/main/scala"
+    bash "(cd daml-lf/transaction/src/main/protobuf;  protoc com/digitalasset/daml/lf/*.proto --java_out=../../../../gen/for-transaction/src/main/scala)"
+
+    bash "cp rattle/build.sbt daml-lf"
+    bash "(cd daml-lf; sbt compile)"
+    --bash "(cd daml-lf; sbt test)"
--- a/rattle/Metadata.hs
+++ b/rattle/Metadata.hs
@ -0,0 +1,111 @@
+-- Copyright (c) 2019 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+-- SPDX-License-Identifier: Apache-2.0
+
+{-# LANGUAGE ViewPatterns, LambdaCase #-}
+{- HLINT ignore "Avoid restricted extensions" -}
+
+module Metadata(
+    readMetadata, Metadata(..)
+    ) where
+
+import System.IO.Extra
+import Data.Tuple.Extra
+import Data.Char
+import System.FilePath
+import Data.List.Extra
+
+
+data Metadata = Metadata
+    {dhl_dir :: FilePath
+    ,dhl_name :: String
+    ,dhl_src_strip_prefix :: String
+    ,dhl_srcs :: [String]
+    ,dhl_deps :: [String]
+    ,dhl_hazel_deps :: [String]
+    ,dhl_main_is :: Maybe String
+    } deriving Show
+
+readMetadata :: FilePath -> IO [Metadata]
+readMetadata file = do
+    src <- readFile' file
+    return $ map (\x -> x{dhl_dir = takeDirectory file}) $ search $ lexPython src
+
+lexPython :: String -> [String]
+lexPython (dropWhile isSpace -> ('\'':xs)) | (inner,'\'':xs) <- break (== '\'') xs = ("\"" ++ inner ++ "\"") : lexPython xs
+lexPython (dropWhile isSpace -> ('\"':'\"':'\"':xs)) | (inner,_:_:_:xs) <- breakOn "\"\"\"" xs = ("\"" ++ inner ++ "\"") : lexPython xs
+lexPython x = case lex x of
+    [("#",x)] -> lexPython $ drop 1 $ dropWhile (/= '\n') x
+    [("","")] -> []
+    [(x,y)] -> x : lexPython y
+    [] -> []
+
+search :: [String] -> [Metadata]
+search (x:xs)
+    | Just md <- defaultMetadata x
+    , Just (fields, rest) <- paren xs
+    = f md fields : search rest
+    where
+        f r ("name":"=":name:xs) = f r{dhl_name = read name} xs
+        f r ("src_strip_prefix":"=":name:xs) = f r{dhl_src_strip_prefix = read name} xs
+        f r ("srcs":"=":"glob":"(":(square -> Just (glob, ")":xs))) = f r{dhl_srcs = map read $ filter (/= ",") glob} xs
+        f r ("srcs":"=":"native":".":"glob":"(":"[":glob:"]":")":xs) = f r{dhl_srcs = [read glob]} xs
+        f r ("srcs":"=":(square -> Just (srcs, xs))) = f r{dhl_srcs = map read $ filter (/= ",") srcs} xs
+        f r ("hazel_deps":"=":(square -> Just (names, xs))) = f r{dhl_hazel_deps = map read $ filter (/= ",") names} xs
+        f r ("deps":"=":(square -> Just (names, xs))) = f r{dhl_deps = delete "" $ map (last . wordsBy (`elem` "/:") . read) $ filter (/= ",") names} xs
+        f r ("main_function":"=":main_:xs) = f r{dhl_main_is = Just $ read main_} xs
+        f r (x:xs) = f r xs
+        f r [] = r
+search (x:xs) = search xs
+search [] = []
+
+defaultMetadata :: String -> Maybe Metadata
+defaultMetadata = \case
+    "da_haskell_library" -> Just $ Metadata [] [] [] [] [] [] Nothing
+    "da_haskell_binary" -> Just $ Metadata [] [] [] [] [] [] (Just "Main.main")
+    _ -> Nothing
+
+
+paren = bracketed "(" ")"
+square = bracketed "[" "]"
+
+bracketed :: String -> String -> [String] -> Maybe ([String], [String])
+bracketed open close (x:xs) | x == open = f 1 xs
+    where
+        f _ [] = Nothing
+        f 1 (x:xs) | x == close = Just ([], xs)
+        f i (x:xs) = first (x:) <$> f i2 xs
+            where i2 | x == close = i-1
+                     | x == open = i+1
+                     | otherwise = i
+bracketed _ _ _ = Nothing
+
+
+{-
+da_haskell_library (
+    name = "daml-lf-proto",
+    src_strip_prefix = "src",
+    srcs = glob (["src/**/*.hs"]),
+    extra_srcs = ["//daml-lf/archive:da/daml_lf_dev.proto"],
+    deps = [
+        "//compiler/daml-lf-ast",
+        "//libs-haskell/da-hs-base",
+        "//libs-haskell/da-hs-pretty",
+        "//daml-lf/archive:daml_lf_haskell_proto",
+        "//nix/third-party/proto3-suite:proto3-suite",
+    ],
+    hazel_deps = [
+        "base",
+        "bytestring",
+        "containers",
+        "cryptonite",
+        "either",
+        "lens",
+        "memory",
+        "scientific",
+        "template-haskell",
+        "text",
+        "vector",
+    ],
+    visibility = ["//visibility:public"],
+)
+-}
--- a/rattle/Rattle.hs
+++ b/rattle/Rattle.hs
@ -0,0 +1,10 @@
+-- Copyright (c) 2019 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+-- SPDX-License-Identifier: Apache-2.0
+
+
+module Rattle(rattle, CmdOption(..), cmd, cmd_, Stdout(..), Exit(..), Stderr(..)) where
+
+import Development.Shake.Command
+
+rattle :: IO () -> IO ()
+rattle act = act
--- a/rattle/Util.hs
+++ b/rattle/Util.hs
@ -0,0 +1,26 @@
+-- Copyright (c) 2019 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+-- SPDX-License-Identifier: Apache-2.0
+
+
+module Util(topSort, transitive) where
+
+import Data.Tuple.Extra
+import Data.List
+import Data.Maybe
+
+
+topSort :: (Show k, Eq k) => [(k, [k], v)] -> [v]
+topSort [] = []
+topSort xs
+    | null now = error $ "topSort failed to order things\n" ++ unlines (map (\(a,b,_) -> show (a,b)) xs)
+    | otherwise = map thd3 now ++ topSort [(k, ks \\ map fst3 now, v) | (k, ks, v) <- later]
+    where (now,later) = partition (null . snd3) xs
+
+
+transitive :: Ord k => [(k, [k])] -> k -> [k]
+transitive xs k = f [] [k]
+    where
+        f seen [] = seen
+        f seen (t:odo)
+            | t `elem` seen = f seen odo
+            | otherwise = f (t:seen) (fromMaybe [] (lookup t xs) ++ odo)
--- a/rattle/build.bat
+++ b/rattle/build.bat
@ -0,0 +1,7 @@
+:: Copyright (c) 2019 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+:: SPDX-License-Identifier: Apache-2.0
+
+@echo off
+setlocal
+cd %~dp0\..
+stack exec --stack-yaml=rattle/stack.yaml sh rattle/build.sh
--- a/rattle/build.sbt
+++ b/rattle/build.sbt
@ -0,0 +1,153 @@
+name := "daml-lf"
+scalaVersion := "2.12.6"
+
+//TODO(nic): These options came from the build.sbt which was deleted back in March
+// check they are all still required
+
+lazy val commonSettings = Seq(
+  javacOptions ++= Seq("-source", "1.8", "-target", "1.8", "-parameters"),
+  scalacOptions ++= Seq(
+    "-feature", // doesn't allow advance features of the language without explict import (higherkinds, implicits)
+    "-target:jvm-1.8",
+    "-encoding",
+    "UTF-8",
+    "-unchecked", // more detailed information about type-erasure related warnings
+    "-deprecation", // warn if using deprecated stuff
+    "-Xfuture",
+    "-Xlint:_,-unused",
+    "-Xmacro-settings:materialize-derivations", // better error reporting for pureconfig
+    "-Xfatal-warnings",
+    "-Yno-adapted-args", // adapted args is a deprecated feature: `def foo(a: (A, B))` can be called with `foo(a, b)`. properly it should be `foo((a,b))`
+    "-Ywarn-dead-code",
+    "-Ywarn-numeric-widen", // Warn about implicit conversion between numerical types
+    "-Ywarn-value-discard", // Gives a warning for functions declared as returning Unit, but the body returns a value
+    "-Ywarn-unused-import",
+    "-Ywarn-unused",
+  ),
+  scalacOptions in (Compile, console) ~= (_.filterNot(_ == "-Xfatal-warnings")),
+  credentials += Credentials(Path.userHome / ".sbt" / ".credentials")
+)
+
+// Versions are derived from ../dependencies.yaml // TODO(nic): this should be automated!
+
+lazy val protobuf = "com.google.protobuf" % "protobuf-java" % "3.6.1"
+lazy val scalatest = "org.scalatest" %% "scalatest" % "3.0.5" % "test"
+lazy val scalacheck = "org.scalacheck" %% "scalacheck" % "1.14.0" % "test"
+lazy val scalaz = "org.scalaz" %% "scalaz-core" % "7.2.24"
+lazy val scalaz_scalacheck_binding = "org.scalaz" %% "scalaz-scalacheck-binding" % "7.2.24-scalacheck-1.14"
+lazy val scalapb_runtime = "com.thesamet.scalapb" %% "scalapb-runtime" % "0.8.4"
+lazy val parser_combinators = "org.scala-lang.modules" %% "scala-parser-combinators" % "1.0.4"
+lazy val paiges = "org.typelevel" %% "paiges-core" % "0.2.1"
+lazy val scalameter_core = "com.storm-enroute" %% "scalameter-core" % "0.10.1" % "test"
+
+lazy val data_deps = Seq(scalatest, scalacheck, scalaz, scalaz_scalacheck_binding)
+lazy val transaction_deps = Seq(scalaz, protobuf, scalatest, scalacheck, scalapb_runtime)
+lazy val lfpackage_deps = scalatest +: data_deps
+lazy val interp_deps = Seq(paiges, scalatest, scalacheck, scalapb_runtime)
+lazy val validation_deps = Seq(scalatest, scalacheck)
+lazy val engine_deps = Seq(protobuf, scalatest, scalacheck, scalameter_core)
+lazy val testing_deps = engine_deps
+
+lazy val repl_deps = Seq(
+  // featurful line reader for the repl
+  "org.jline" % "jline" % "3.7.1",
+  "org.jline" % "jline-reader" % "3.7.1",
+  // parser combinators for parsing values in the repl
+  parser_combinators,
+  // colourful pretty printing of arbitrary values
+  "com.lihaoyi" %% "pprint" % "0.5.3",
+  // scala implementation of wadler's pretty printer
+  paiges
+)
+
+//TODO(nic): DarReaderTest fails
+
+lazy val archive = (project in file("archive"))
+  .dependsOn(data, archiveGen)
+  .settings(
+    libraryDependencies ++= Seq(protobuf, scalatest)
+  )
+
+lazy val archiveGen = (project in file("gen/for-archive"))
+  .settings(
+    libraryDependencies ++= Seq(protobuf, scalatest)
+  )
+
+lazy val data = (project in file("data"))
+  .settings(
+    libraryDependencies ++= data_deps
+  )
+
+lazy val engine = (project in file("engine"))
+  .dependsOn(data, transaction, lfpackage, interpreter, validation, archive)
+  .settings(
+    libraryDependencies ++= engine_deps
+  )
+
+lazy val interface = (project in file("interface"))
+  .dependsOn(archive, parser % "test->test")
+  .settings(
+    libraryDependencies ++= Seq(scalaz, scalatest)
+  )
+
+lazy val interpreter = (project in file("interpreter"))
+  .dependsOn(data, transaction, lfpackage, archive, validation, parser % "test->test")
+  .settings(
+    libraryDependencies ++= interp_deps
+  )
+
+lazy val lfpackage = (project in file("lfpackage"))
+  .dependsOn(data, archive)
+  .settings(
+    libraryDependencies ++= lfpackage_deps
+  )
+
+lazy val parser = (project in file("parser"))
+  .dependsOn(lfpackage)
+  .settings(
+    libraryDependencies ++= Seq(parser_combinators, scalatest, scalacheck)
+  )
+
+lazy val repl = (project in file("repl"))
+  .dependsOn(interpreter, `scenario-interpreter`, validation)
+  .settings(
+    libraryDependencies ++= repl_deps
+  )
+
+lazy val `scenario-interpreter` = (project in file("scenario-interpreter"))
+  .dependsOn(interpreter)
+  .settings(
+    libraryDependencies ++= interp_deps
+  )
+
+lazy val testingTools = (project in file("testing-tools"))
+  .dependsOn(archive, interpreter, `scenario-interpreter`, engine)
+  .settings(
+    libraryDependencies ++= testing_deps
+  )
+
+// TODO(nic): transaction/test fails to compile -- needs ValueGenerators
+// which is in transactionScalacheck - but that causes a recursive dep!
+
+lazy val transaction = (project in file("transaction"))
+  .dependsOn(data, archive, transactionGen) //, transactionScalacheck)
+  .settings(
+    libraryDependencies ++= transaction_deps
+  )
+
+lazy val transactionGen = (project in file("gen/for-transaction"))
+  .settings(
+    libraryDependencies ++= transaction_deps
+  )
+
+lazy val transactionScalacheck = (project in file("transaction-scalacheck"))
+  .dependsOn(data, archive, transaction)
+  .settings(
+    libraryDependencies ++= transaction_deps
+  )
+
+lazy val validation = (project in file("validation"))
+  .dependsOn(lfpackage, parser % "test->test")
+  .settings(
+    libraryDependencies ++= validation_deps
+  )
--- a/rattle/build.sh
+++ b/rattle/build.sh
@ -0,0 +1,6 @@
+#!/bin/bash
+# Copyright (c) 2019 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+cd "$(dirname "$0")/.."
+stack exec --package=shake --package=filepattern --stack-yaml=rattle/stack.yaml -- runhaskell -irattle rattle/Main.hs
--- a/rattle/ghcid.sh
+++ b/rattle/ghcid.sh
@ -0,0 +1,6 @@
+#!/bin/bash
+# Copyright (c) 2019 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+cd "$(dirname "$0")/.."
+stack exec --package=shake --package=filepattern --stack-yaml=rattle/stack.yaml -- ghcid -c "ghci -irattle rattle/Main.hs"
--- a/rattle/hacks/SdkVersion.hs
+++ b/rattle/hacks/SdkVersion.hs
@ -0,0 +1,7 @@
+-- Copyright (c) 2019 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+-- SPDX-License-Identifier: Apache-2.0
+
+module SdkVersion where
+sdkVersion, componentVersion :: String
+sdkVersion = "<rattle-dummy>"
+componentVersion = "<rattle-dummy>"
--- a/rattle/scala-build.sh
+++ b/rattle/scala-build.sh
@ -0,0 +1,6 @@
+#!/bin/bash
+# Copyright (c) 2019 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+cd "$(dirname "$0")/.."
+stack exec --package=shake --package=filepattern --stack-yaml=rattle/stack.yaml -- runhaskell -irattle rattle/MainScala.hs
--- a/rattle/stack.yaml
+++ b/rattle/stack.yaml
@ -0,0 +1,21 @@
+require-stack-version: ">= 1.10"
+resolver: nightly-2019-04-05
+packages:
+- ../nix/third-party/proto3-suite
+- ../nix/third-party/proto3-wire
+- ../nix/third-party/gRPC-haskell
+- ../nix/third-party/gRPC-haskell/core
+
+extra-deps:
+  - name: ghc-lib-parser
+    archive: https://digitalassetsdk.bintray.com/ghc-lib/ghc-lib-parser-0.20190411.tar.gz
+    sha256: ebf7c72ffaeabdc8d81bace1b49b57d7c5eedf9b28adb4b24dd52438177a5cbd
+    size: 1632233
+  - name: ghc-lib
+    archive: https://digitalassetsdk.bintray.com/ghc-lib/ghc-lib-0.20190411.tar.gz
+    sha256: 2fb69e40601e49e2f312e784e5c72d3724c49865e38379a2ec1f787f14282577
+    size: 2849748
+  - terminal-progress-bar-0.4.0.1
+
+# Turtle causes issues: https://github.com/Gabriel439/Haskell-Turtle-Library/issues/323
+allow-newer: true