on master help sign in
Remove needless Text <-> String round trips, and log more
commit0be51b39a431736ddc2b1a61c60bd95a3d379150 authorJohn Ericson <John.Ericson@Obsidian.Systems> dateWed 17 Oct 2018, 13:38 (8 years ago) parent6983037
8 files changed+61−33
lib/cliapp/src/Obelisk/CliApp.hs+1 lib/cliapp/src/Obelisk/CliApp/Demo.hs+1−1 lib/cliapp/src/Obelisk/CliApp/Process.hs+45−16 lib/command/src/Obelisk/Command/Nix.hs+1−1 lib/command/src/Obelisk/Command/Thunk.hs+4−5 lib/command/src/Obelisk/Command/Upgrade/Hash.hs+4−5 lib/command/src/Obelisk/Command/Utils.hs+3−3 lib/command/src/Obelisk/Command/VmBuilder.hs+2−2
lib/cliapp/src/Obelisk/CliApp.hs+1
@@ -34,6 +34,7 @@ module Obelisk.CliApp
3434 -- .Process
3535 , ProcessFailed (..)
3636 , readProcessAndLogStderr
37+ , readProcessAndLogOutput
3738 , callProcessAndLogOutput
3839 , createProcess_
3940 , callProcess
lib/cliapp/src/Obelisk/CliApp/Demo.hs+1−1
@@ -40,7 +40,7 @@ cliDemo = withSpinner "CLI Demo" $ do
4040 withSpinner "Looking around" $ do
4141 delay
4242 output <- readProcessAndLogStderr Notice $ proc "ls" ["-l", "/"]
43− putLog Notice $ "Output was: " <> T.pack output
43+ putLog Notice $ "Output was: " <> output
4444 delay
4545 callProcessAndLogOutput (Notice, Error) $ proc "ls" ["-l", "/does-not-exist"]
4646 delay
lib/cliapp/src/Obelisk/CliApp/Process.hs+45−16
@@ -11,6 +11,7 @@
1111module Obelisk.CliApp.Process
1212 ( ProcessFailed (..)
1313 , readProcessAndLogStderr
14+ , readProcessAndLogOutput
1415 , callProcessAndLogOutput
1516 , createProcess
1617 , createProcess_
@@ -18,16 +19,18 @@ module Obelisk.CliApp.Process
1819 , callCommand
1920 ) where
2021
21−import Control.Applicative (liftA2)
22−import Control.Monad (void, (<=<))
22+import Control.Monad ((<=<), join, void)
2323import Control.Monad.Catch (Exception(..), MonadMask, throwM)
24+import Control.Monad.Except (throwError)
2425import Control.Monad.IO.Class (MonadIO, liftIO)
26+import qualified Data.ByteString as BS
2527import qualified Data.ByteString.Char8 as BSC
2628import Data.Function (fix)
2729import Data.Monoid ((<>))
30+import Data.Text (Text)
2831import qualified Data.Text as T
29−import Data.Text.Encoding (decodeUtf8)
30−import qualified Data.Text.IO as T
32+import qualified Data.Text.Encoding as T
33+--import qualified Data.Text.IO as T
3134import System.Exit (ExitCode (..))
3235import System.IO (Handle)
3336import System.IO.Streams (InputStream, handleToInputStream)
@@ -50,26 +53,52 @@ instance Exception ProcessFailed where
5053
5154readProcessAndLogStderr
5255 :: (MonadIO m, MonadMask m, Cli m)
53− => Severity -> CreateProcess -> m String
56+ => Severity -> CreateProcess -> m Text
5457readProcessAndLogStderr sev process = do
5558 (out, _err) <- withProcess process $ \_out err -> do
5659 streamToLog =<< liftIO (streamHandle sev err)
57− -- TODO: Why are we using Text here?
58− liftIO $ T.unpack . T.strip <$> T.hGetContents out
60+ liftIO $ T.decodeUtf8 <$> BS.hGetContents out
5961
60−-- | Like `System.Process.callProcess` but logs the combined output--stdout and stderr with the corresponding
61−-- severity.
62+-- | Like `System.Process.readProcess` but logs the combined output (stdout and stderr)
63+-- with the corresponding severity.
6264--
63−-- Usually this function is called as `callProcessAndLogOutput (Debug, Error)`. However some processes
64−-- are known to spit out diagnostic or informative messages in stderr, in which case it is advisable to call
65−-- it with a non-Error severity for stderr, like `callProcessAndLogOutput (Debug, Debug)`.
65+-- Usually this function is called as `callProcessAndLogOutput (Debug, Error)`. However
66+-- some processes are known to spit out diagnostic or informative messages in stderr, in
67+-- which case it is advisable to call it with a non-Error severity for stderr, like
68+-- `callProcessAndLogOutput (Debug, Debug)`.
69+
70+readProcessAndLogOutput
71+ :: (MonadIO m, MonadMask m, Cli m)
72+ => (Severity, Severity) -> CreateProcess -> m Text
73+readProcessAndLogOutput (sev_out, sev_err) process = do
74+ (_, Just out, Just err, p) <- createProcess $ process
75+ { std_out = CreatePipe , std_err = CreatePipe }
76+
77+ -- TODO interleave stdout and stderr in log correctly
78+ streamToLog =<< liftIO (streamHandle sev_err err)
79+ outText <- liftIO $ T.decodeUtf8 <$> BS.hGetContents out
80+ putLogRaw sev_out outText
81+
82+ liftIO (waitForProcess p) >>= \case
83+ ExitSuccess -> pure outText
84+ ExitFailure code -> throwError $ T.pack $ show $ ProcessFailed (cmdspec process) code
85+
86+-- | Like `System.Process.callProcess` but logs the combined output (stdout and stderr)
87+-- with the corresponding severity.
88+--
89+-- Usually this function is called as `callProcessAndLogOutput (Debug, Error)`. However
90+-- some processes are known to spit out diagnostic or informative messages in stderr, in
91+-- which case it is advisable to call it with a non-Error severity for stderr, like
92+-- `callProcessAndLogOutput (Debug, Debug)`.
6693callProcessAndLogOutput
94+
6795 :: (MonadIO m, MonadMask m, Cli m)
6896 => (Severity, Severity) -> CreateProcess -> m ()
69−callProcessAndLogOutput (sev_out, sev_err) process = do
97+callProcessAndLogOutput (sev_out, sev_err) process =
7098 void $ withProcess process $ \out err -> do
71− stream <- liftIO $ do
72− uncurry combineStream =<< liftA2 (,) (streamHandle sev_out out) (streamHandle sev_err err)
99+ stream <- liftIO $ join $ combineStream
100+ <$> streamHandle sev_out out
101+ <*> streamHandle sev_err err
73102 streamToLog stream
74103 where
75104 combineStream s1 s2 = concurrentMerge [s1, s2]
@@ -131,7 +160,7 @@ streamToLog
131160streamToLog stream = fix $ \loop -> do
132161 liftIO (Streams.read stream) >>= \case
133162 Nothing -> return ()
134− Just (sev, line) -> putLogRaw sev (decodeUtf8 line) >> loop
163+ Just (sev, line) -> putLogRaw sev (T.decodeUtf8 line) >> loop
135164
136165reconstructCommand :: Process.CmdSpec -> String
137166reconstructCommand (Process.ShellCommand str) = str
lib/command/src/Obelisk/Command/Nix.hs+1−1
@@ -81,7 +81,7 @@ instance Default NixBuildConfig where
8181
8282nixBuild :: MonadObelisk m => NixBuildConfig -> m FilePath
8383nixBuild cfg = withSpinner' ("Running nix-build on " <> desc) (Just $ const $ "Built " <> desc) $ do
84− withNixRemoteCheck $ readProcessAndLogStderr Debug $ proc "nix-build" $ mconcat
84+ withNixRemoteCheck $ fmap T.unpack $ readProcessAndLogStderr Debug $ proc "nix-build" $ mconcat
8585 [[path], attrArg, args, outLink, buildersArg]
8686 where
8787 path = _target_path $ _nixBuildConfig_target cfg
lib/command/src/Obelisk/Command/Thunk.hs+4−5
@@ -39,7 +39,6 @@ import qualified Data.Aeson as Aeson
3939import Data.Aeson.Encode.Pretty
4040import qualified Data.Aeson.Types as Aeson
4141import qualified Data.ByteString.Lazy as LBS
42−import qualified Data.ByteString.Lazy.Char8 as LBSC
4342import Data.Default
4443import Data.Either.Combinators (rightToMaybe)
4544import Data.Foldable
@@ -135,7 +134,7 @@ commitNameToRef (N c) = Ref.fromHex $ encodeUtf8 c
135134getNixSha256ForUriUnpacked :: MonadObelisk m => URI -> m NixSha256
136135getNixSha256ForUriUnpacked uri =
137136 withExitFailMessage ("nix-prefetch-url: Failed to determine sha256 hash of URL " <> T.pack (show uri)) $ do
138− fmap T.pack $ withNixRemoteCheck $ readProcessAndLogStderr Debug $
137+ withNixRemoteCheck $ readProcessAndLogStderr Debug $
139138 proc "nix-prefetch-url" ["--unpack" , "--type" , "sha256" , show uri]
140139
141140nixPrefetchGit :: MonadObelisk m => URI -> Text -> Bool -> m NixSha256
@@ -149,14 +148,14 @@ nixPrefetchGit uri rev fetchSubmodules =
149148 , "--quiet"
150149 ]
151150
152− case parseMaybe (Aeson..: "sha256") =<< Aeson.decode (LBSC.pack out) of
153− Nothing -> failWith $ "nix-prefetch-git: unrecognized output " <> T.pack out
151+ case parseMaybe (Aeson..: "sha256") =<< Aeson.decodeStrict (encodeUtf8 out) of
152+ Nothing -> failWith $ "nix-prefetch-git: unrecognized output " <> out
154153 Just x -> pure x
155154
156155getLatestGitRev :: MonadObelisk m => GitSource -> m ThunkRev
157156getLatestGitRev s = withSystemTempDirectory "git-clone" $ \tmpDir -> do
158157 withExitFailMessage ("Unable to determine latest git revision for branch " <> branch) $ do
159− let git = fmap T.pack . readProcessAndLogStderr Debug . gitProc tmpDir
158+ let git = readProcessAndLogStderr Debug . gitProc tmpDir
160159 out <- liftA2 (<>)
161160 (git ["clone", "--no-checkout", show $ _gitSource_url s]) -- Might be able to use --bare here instead to go faster.
162161 (git ["log", T.unpack branch, "--max-count=1", "--format=%H"])
lib/command/src/Obelisk/Command/Upgrade/Hash.hs+4−5
@@ -102,15 +102,14 @@ tidyUpGitWorkingCopy :: MonadObelisk m => FilePath -> m ()
102102tidyUpGitWorkingCopy dir = withSpinnerNoTrail "Tidying up git working copy" $ do
103103 ignored <- gitLsFiles dir ["--ignored", "--exclude-standard", "--others"]
104104 untracked <- gitLsFiles dir ["--exclude-standard", "--others"]
105− putLog Debug $ T.pack $ "Ignored: " <> show (length ignored) <> " files."
106− putLog Debug $ T.pack $ "Untracked files:\n" <> unlines untracked
105+ putLog Debug $ "Ignored: " <> T.pack (show $ length ignored) <> " files."
106+ putLog Debug $ "Untracked files:\n" <> T.unlines untracked
107107 withSpinnerNoTrail "Removing untracked and ignored files" $ do
108− forM_ (fmap (dir </>) $ ignored <> untracked) $
108+ forM_ (fmap ((dir </>) . T.unpack) $ ignored <> untracked) $
109109 liftIO . removePathForcibly
110110 -- Empty directories won't be included in these lists. Git doesn't track them
111111 -- So we must delete these separately.
112112 runProc $ proc "find" [dir, "-depth", "-empty", "-type", "d", "-delete"]
113113 where
114− gitLsFiles pwd opts = fmap lines $ readProcessAndLogStderr Error $
114+ gitLsFiles pwd opts = fmap T.lines $ readProc $
115115 (proc "git" $ ["ls-files", "."] <> opts) { cwd = Just pwd }
116−
lib/command/src/Obelisk/Command/Utils.hs+3−3
@@ -25,7 +25,7 @@ checkGitCleanStatus repo withIgnored = do
2525 runGit = readProcessAndLogStderr Debug . gitProc repo
2626 gitStatus = runGit $ ["status", "--porcelain"] <> bool [] ["--ignored"] withIgnored
2727 gitDiff = runGit ["diff"]
28− null <$> liftA2 (<>) gitStatus gitDiff
28+ T.null <$> liftA2 (<>) gitStatus gitDiff
2929
3030-- | Ensure that git repo is clean
3131ensureCleanGitRepo :: MonadObelisk m => FilePath -> Bool -> Text -> m ()
@@ -60,7 +60,7 @@ copyDir src dest =
6060 (P.proc "cp" ["-a", ".", dest]) { P.cwd = Just src }
6161
6262readGitProcess :: MonadObelisk m => FilePath -> [String] -> m Text
63−readGitProcess repo = fmap T.pack . readProcessAndLogStderr Notice . gitProc repo
63+readGitProcess repo = readProcessAndLogStderr Notice . gitProc repo
6464
6565processToShellString :: FilePath -> [String] -> String
6666processToShellString cmd args = unwords $ map quoteAndEscape (cmd : args)
@@ -76,7 +76,7 @@ runProcSilently = callProcessAndLogOutput (Debug, Debug)
7676
7777-- | A simpler wrapper for CliApp's readProcessAndLogStderr with sensible defaults.
7878readProc :: MonadObelisk m => P.CreateProcess -> m Text
79−readProc = fmap T.pack . readProcessAndLogStderr Error
79+readProc = readProcessAndLogStderr Error
8080
8181tshow :: Show a => a -> Text
8282tshow = T.pack . show
lib/command/src/Obelisk/Command/VmBuilder.hs+2−2
@@ -43,7 +43,7 @@ containerName = "obelisk-docker-nix-builder"
4343-- | Check to see if the Docker container exists. This will exit with a helpful message if Docker is not installed.
4444containerExists :: MonadObelisk m => m Bool
4545containerExists = withExitFailMessage needDockerMsg $ do
46− containerNames <- fmap (map T.strip . T.lines . T.pack) $
46+ containerNames <- fmap (map T.strip . T.lines) $
4747 readProcessAndLogStderr Error $
4848 proc "docker" ["container", "list", "--all", "--format", "{{.Names}}"]
4949 pure $ containerName `elem` containerNames
@@ -81,7 +81,7 @@ setupNixDocker stateDir = withSpinner ("Creating Docker container named " <> con
8181 , "--restart", "always"
8282 , "--detach"
8383 , "--publish", show containerSshPort <> ":22"
84− , "--name", containerName, containerId
84+ , "--name", T.unpack containerName, T.unpack containerId
8585 ]
8686 exists <- containerExists
8787 unless exists $