@@ -5,74 +5,58 @@
5{-# LANGUAGE MultiWayIf #-}
5{-# LANGUAGE MultiWayIf #-}
6{-# LANGUAGE OverloadedStrings #-}
6{-# LANGUAGE OverloadedStrings #-}
7{-# LANGUAGE PatternGuards #-}
7{-# LANGUAGE PatternGuards #-}
8+{-# LANGUAGE QuasiQuotes #-}
8{-# LANGUAGE ScopedTypeVariables #-}
9{-# LANGUAGE ScopedTypeVariables #-}
9{-# LANGUAGE TypeApplications #-}
10{-# LANGUAGE TypeApplications #-}
10−module Obelisk.Command.Thunk
11+module Obelisk.Command.Thunk where
13− , ReadThunkError (..)
16− , ThunkPackConfig (..)
20− , ThunkUpdateConfig (..)
22− , createThunkWithLatest
27− , nixBuildAttrWithCache
28− , nixBuildThunkAttrWithCache
34− , updateThunkToLatest
38import Control.Applicative
13import Control.Applicative
39import Control.Exception (displayException, try)
14import Control.Exception (displayException, try)
40−import qualified Control.Lens as Lens
15+import Control.Lens (ifor, ifor_, (.~))
41−import Control.Lens.Indexed hiding ((<.>))
42import Control.Monad
16import Control.Monad
43−import Control.Monad.Catch (handle)
17+import Control.Monad.Extra (findM)
18+import Control.Monad.Catch (MonadCatch, handle)
44import Control.Monad.Except
19import Control.Monad.Except
45import Data.Aeson ((.=))
20import Data.Aeson ((.=))
46import qualified Data.Aeson as Aeson
21import qualified Data.Aeson as Aeson
47import Data.Aeson.Encode.Pretty
22import Data.Aeson.Encode.Pretty
48import qualified Data.Aeson.Types as Aeson
23import qualified Data.Aeson.Types as Aeson
24+import Data.Bifunctor (first)
49import qualified Data.ByteString.Lazy as LBS
25import qualified Data.ByteString.Lazy as LBS
26+import Data.Containers.ListUtils (nubOrd)
50import Data.Default
27import Data.Default
51import Data.Either.Combinators (fromRight', rightToMaybe)
28import Data.Either.Combinators (fromRight', rightToMaybe)
29+import Data.Foldable (toList)
30+import Data.Function ((&))
31+import Data.Functor ((<&>))
52import Data.Git.Ref (Ref)
32import Data.Git.Ref (Ref)
53import qualified Data.Git.Ref as Ref
33import qualified Data.Git.Ref as Ref
54import qualified Data.List as L
34import qualified Data.List as L
55−import Data.List.NonEmpty (NonEmpty (..))
35+import Data.List.NonEmpty (NonEmpty (..), nonEmpty)
56import qualified Data.List.NonEmpty as NonEmpty
36import qualified Data.List.NonEmpty as NonEmpty
57import Data.Map (Map)
37import Data.Map (Map)
58import qualified Data.Map as Map
38import qualified Data.Map as Map
59import Data.Maybe
39import Data.Maybe
60import Data.Set (Set)
40import Data.Set (Set)
61import qualified Data.Set as Set
41import qualified Data.Set as Set
42+import Data.String.Here.Interpolated (i)
43+import Data.String.Here.Uninterpolated (here)
62import Data.Text (Text)
44import Data.Text (Text)
63import qualified Data.Text as T
45import qualified Data.Text as T
64import Data.Text.Encoding
46import Data.Text.Encoding
65import qualified Data.Text.IO as T
47import qualified Data.Text.IO as T
66import Data.Time.Clock.POSIX (posixSecondsToUTCTime)
48import Data.Time.Clock.POSIX (posixSecondsToUTCTime)
49+import Data.Traversable (for)
67import Data.Yaml (parseMaybe)
50import Data.Yaml (parseMaybe)
68import GitHub
51import GitHub
69import GitHub.Data.Name
52import GitHub.Data.Name
70import Obelisk.Command.Nix
53import Obelisk.Command.Nix
71import System.Directory
54import System.Directory
72import System.FilePath
56import System.FilePath
73import System.IO.Error
57import System.IO.Error
74import System.IO.Temp
58import System.IO.Temp
75−import System.Posix (getSymbolicLinkStatus, modificationTime)
59+import System.PosixCompat.Files (getSymbolicLinkStatus, modificationTime)
76import qualified Text.URI as URI
60import qualified Text.URI as URI
78import Obelisk.App (MonadObelisk)
62import Obelisk.App (MonadObelisk)
@@ -81,11 +65,10 @@ import Obelisk.Command.Utils
82--TODO: Support symlinked thunk data
66--TODO: Support symlinked thunk data
83data ThunkData
67data ThunkData
84− = ThunkData_Packed ThunkPtr
68+ = ThunkData_Packed ThunkSpec ThunkPtr
85 -- ^ Packed thunk
69 -- ^ Packed thunk
86− | ThunkData_Checkout (Maybe ThunkPtr)
70+ | ThunkData_Checkout
87 -- ^ Checked out thunk that was unpacked from this pointer
71 -- ^ Checked out thunk that was unpacked from this pointer
88− deriving (Show, Eq, Ord)
90-- | A reference to the exact data that a thunk should translate into
73-- | A reference to the exact data that a thunk should translate into
91data ThunkPtr = ThunkPtr
74data ThunkPtr = ThunkPtr
@@ -111,6 +94,11 @@ data ThunkSource
111 | ThunkSource_Git GitSource
94 | ThunkSource_Git GitSource
112 deriving (Show, Eq, Ord)
95 deriving (Show, Eq, Ord)
97+thunkSourceToGitSource :: ThunkSource -> GitSource
98+thunkSourceToGitSource = \case
99+ ThunkSource_GitHub s -> forgetGithub False s
100+ ThunkSource_Git s -> s
114data GitHubSource = GitHubSource
102data GitHubSource = GitHubSource
115 { _gitHubSource_owner :: Name Owner
103 { _gitHubSource_owner :: Name Owner
116 , _gitHubSource_repo :: Name Repo
104 , _gitHubSource_repo :: Name Repo
@@ -119,7 +107,7 @@ data GitHubSource = GitHubSource
120 deriving (Show, Eq, Ord)
108 deriving (Show, Eq, Ord)
122−newtype GitUri = GitUri URI.URI deriving (Eq, Ord, Show)
110+newtype GitUri = GitUri { unGitUri :: URI.URI } deriving (Eq, Ord, Show)
124gitUriToText :: GitUri -> Text
112gitUriToText :: GitUri -> Text
125gitUriToText (GitUri uri)
113gitUriToText (GitUri uri)
@@ -190,14 +178,14 @@ getNixSha256ForUriUnpacked
190getNixSha256ForUriUnpacked uri =
178getNixSha256ForUriUnpacked uri =
191 withExitFailMessage ("nix-prefetch-url: Failed to determine sha256 hash of URL " <> gitUriToText uri) $ do
179 withExitFailMessage ("nix-prefetch-url: Failed to determine sha256 hash of URL " <> gitUriToText uri) $ do
192 [hash] <- fmap T.lines $ readProcessAndLogOutput (Debug, Debug) $
180 [hash] <- fmap T.lines $ readProcessAndLogOutput (Debug, Debug) $
193− proc "nix-prefetch-url" ["--unpack", "--type", "sha256", T.unpack $ gitUriToText uri]
181+ proc nixPrefetchUrlPath ["--unpack", "--type", "sha256", T.unpack $ gitUriToText uri]
194 pure hash
182 pure hash
196nixPrefetchGit :: MonadObelisk m => GitUri -> Text -> Bool -> m NixSha256
184nixPrefetchGit :: MonadObelisk m => GitUri -> Text -> Bool -> m NixSha256
197nixPrefetchGit uri rev fetchSubmodules =
185nixPrefetchGit uri rev fetchSubmodules =
198 withExitFailMessage ("nix-prefetch-git: Failed to determine sha256 hash of Git repo " <> gitUriToText uri <> " at " <> rev) $ do
186 withExitFailMessage ("nix-prefetch-git: Failed to determine sha256 hash of Git repo " <> gitUriToText uri <> " at " <> rev) $ do
199 out <- readProcessAndLogStderr Debug $
187 out <- readProcessAndLogStderr Debug $
200− proc "nix-prefetch-git" $ filter (/="")
188+ proc nixPrefetchGitPath $ filter (/="")
201 [ "--url", T.unpack $ gitUriToText uri
189 [ "--url", T.unpack $ gitUriToText uri
202 , "--rev", T.unpack rev
190 , "--rev", T.unpack rev
203 , if fetchSubmodules then "--fetch-submodules" else ""
191 , if fetchSubmodules then "--fetch-submodules" else ""
@@ -210,93 +198,117 @@ nixPrefetchGit uri rev fetchSubmodules =
211--TODO: Pretty print these
199--TODO: Pretty print these
212data ReadThunkError
200data ReadThunkError
213− = ReadThunkError_UnrecognizedFiles
201+ = ReadThunkError_UnrecognizedThunk
214− | ReadThunkError_AmbiguousFiles
202+ | ReadThunkError_UnrecognizedPaths (NonEmpty FilePath)
215− | ReadThunkError_UnrecognizedLoader Text
203+ | ReadThunkError_MissingPaths (NonEmpty FilePath)
216− | ReadThunkError_UnparseablePtr LBS.ByteString
204+ | ReadThunkError_UnparseablePtr FilePath String
217− deriving (Show)
205+ | ReadThunkError_FileError IOError
218−
206+ | ReadThunkError_FileDoesNotMatch FilePath Text
219−-- | Read a thunk and validate that it is only a thunk, either packed or unpacked.
207+ | ReadThunkError_UnrecognizedState String
220−-- If the thunk is packed and additional data is present, fail.
208+ | ReadThunkError_AmbiguousPackedState ThunkPtr ThunkPtr
221−readThunk :: MonadObelisk m => FilePath -> m (Either ReadThunkError ThunkData)
209+ deriving (Show)
222−readThunk thunkDir = do
210+
223− files <- liftIO $ listDirectory thunkDir
211+unpackedDirName :: FilePath
224− case ".git" `elem` files of
212+unpackedDirName = "."
225− True -> fmap ThunkData_Checkout <$> do
213+
226− let origThunkPath = thunkDir </> ".git" </> "obelisk" </> "orig-thunk"
214+attrCacheFileName :: FilePath
227− liftIO $ doesDirectoryExist origThunkPath >>= \case
215+attrCacheFileName = ".attr-cache"
228− True -> fmap Just <$> readPackedThunk origThunkPath
216+
229− False -> return $ return Nothing
217+-- | Specification for how a file in a thunk version works.
230− False -> liftIO $ fmap ThunkData_Packed <$> readPackedThunk thunkDir
218+data ThunkFileSpec
231−
219+ = ThunkFileSpec_Ptr (LBS.ByteString -> Either String ThunkPtr) -- ^ This file specifies 'ThunkPtr' data
232−data ThunkType = ThunkType
220+ | ThunkFileSpec_FileMatches Text -- ^ This file must match the given content exactly
233− { _thunkType_loader :: FilePath
221+ | ThunkFileSpec_CheckoutIndicator -- ^ Existence of this directory indicates that the thunk is unpacked
234− , _thunkType_json :: FilePath
222+ | ThunkFileSpec_AttrCache -- ^ This directory is an attribute cache
235− , _thunkType_optional :: Set FilePath
223+
236− , _thunkType_loaderVersions :: NonEmpty Text
224+-- | Specification for how a set of files in a thunk version work.
237− , _thunkType_parser :: Aeson.Object -> Aeson.Parser ThunkPtr
225+data ThunkSpec = ThunkSpec
238− }
226+ { _thunkSpec_name :: !Text
239−
227+ , _thunkSpec_files :: !(Map FilePath ThunkFileSpec)
240−gitHubThunkType :: ThunkType
241−gitHubThunkType = ThunkType
242− { _thunkType_loader = "default.nix"
243− , _thunkType_json = "github.json"
244− , _thunkType_optional = Set.fromList [".attr-cache"]
245− , _thunkType_loaderVersions = gitHubStandaloneLoaders
246− , _thunkType_parser = parseThunkPtr $ \v ->
247− ThunkSource_GitHub <$> parseGitHubSource v <|> ThunkSource_Git <$> parseGitSource v
250−gitThunkType :: ThunkType
230+thunkSpecTypes :: NonEmpty (NonEmpty ThunkSpec)
251−gitThunkType = ThunkType
231+thunkSpecTypes = gitThunkSpecs :| [gitHubThunkSpecs]
252− { _thunkType_loader = "default.nix"
232+
253− , _thunkType_json = "git.json"
233+-- | Attempts to match a 'ThunkSpec' to a given directory.
254− , _thunkType_optional = Set.fromList [".attr-cache"]
234+matchThunkSpecToDir
255− , _thunkType_loaderVersions = plainGitStandaloneLoaders
235+ :: (MonadError ReadThunkError m, MonadIO m, MonadCatch m)
256− , _thunkType_parser = parseThunkPtr $ fmap ThunkSource_Git . parseGitSource
236+ => ThunkSpec -- ^ 'ThunkSpec' to match against the given files/directory
257− }
237+ -> FilePath -- ^ Path to directory
258−
238+ -> Set FilePath -- ^ Set of file paths relative to the given directory
259−thunkTypes :: [ThunkType]
239+ -> m ThunkData
260−thunkTypes = [gitThunkType, gitHubThunkType]
240+matchThunkSpecToDir thunkSpec dir dirFiles = do
261−
241+ case nonEmpty (toList $ dirFiles `Set.difference` expectedPaths) of
262−findThunkType :: [ThunkType] -> FilePath -> IO (Either ReadThunkError ThunkType)
242+ Just fs -> throwError $ ReadThunkError_UnrecognizedPaths $ (dir </>) <$> fs
263−findThunkType types thunkDir = do
243+ Nothing -> pure ()
264− matches <- fmap catMaybes $ forM types $ \thunkType -> do
244+ case nonEmpty (toList $ requiredPaths `Set.difference` dirFiles) of
265− let
245+ Just fs -> throwError $ ReadThunkError_MissingPaths $ (dir </>) <$> fs
266− expectedContents = Set.fromList $ (thunkDir </>) <$>
246+ Nothing -> pure ()
267− [ _thunkType_loader thunkType
247+ datas <- fmap toList $ flip Map.traverseMaybeWithKey (_thunkSpec_files thunkSpec) $ \expectedPath -> \case
268− , _thunkType_json thunkType
248+ ThunkFileSpec_AttrCache -> Nothing <$ dirMayExist expectedPath
269− ]
249+ ThunkFileSpec_CheckoutIndicator -> liftIO (doesDirectoryExist (dir </> expectedPath)) <&> \case
270− optionalContents = Set.map (thunkDir </>) (_thunkType_optional thunkType)
250+ False -> Nothing
271−
251+ True -> Just ThunkData_Checkout
272− -- Ensure that there aren't any other files in the thunk
252+ ThunkFileSpec_FileMatches expectedContents -> handle (\(e :: IOError) -> throwError $ ReadThunkError_FileError e) $ do
273− -- NB: System.Directory.listDirectory returns the contents without the directory path
253+ actualContents <- liftIO (T.readFile $ dir </> expectedPath)
274− files <- liftIO $ Set.fromList . fmap (thunkDir </>) <$> listDirectory thunkDir
254+ case T.strip expectedContents == T.strip actualContents of
275− let unexpectedContents = files `Set.difference` (expectedContents `Set.union` optionalContents)
255+ True -> pure Nothing
276− missingContents = expectedContents `Set.difference` files
256+ False -> throwError $ ReadThunkError_FileDoesNotMatch (dir </> expectedPath) expectedContents
277− pure $ if Set.null unexpectedContents && Set.null missingContents
257+ ThunkFileSpec_Ptr parser -> handle (\(e :: IOError) -> throwError $ ReadThunkError_FileError e) $ do
278− then Just thunkType
258+ let path = dir </> expectedPath
279− else Nothing
259+ liftIO (doesFileExist path) >>= \case
280−
260+ False -> pure Nothing
281− pure $ case matches of
261+ True -> do
282− [singleMatch] -> Right singleMatch
262+ actualContents <- liftIO $ LBS.readFile path
283− [] -> Left ReadThunkError_UnrecognizedFiles
263+ case parser actualContents of
284− _ -> Left ReadThunkError_AmbiguousFiles
264+ Right v -> pure $ Just (ThunkData_Packed thunkSpec v)
265+ Left e -> throwError $ ReadThunkError_UnparseablePtr (dir </> expectedPath) e
267+ case nonEmpty datas of
268+ Nothing -> throwError ReadThunkError_UnrecognizedThunk
269+ Just xs -> fold1WithM xs $ \a b -> either throwError pure (mergeThunkData a b)
271+ rootPathsOnly = Set.fromList . mapMaybe takeRootDir . Map.keys
272+ takeRootDir = fmap NonEmpty.head . nonEmpty . splitPath
274+ expectedPaths = rootPathsOnly $ _thunkSpec_files thunkSpec
276+ requiredPaths = rootPathsOnly $ Map.filter isRequiredFileSpec $ _thunkSpec_files thunkSpec
277+ isRequiredFileSpec = \case
278+ ThunkFileSpec_FileMatches _ -> True
281+ dirMayExist expectedPath = liftIO (doesFileExist (dir </> expectedPath)) >>= \case
282+ True -> throwError $ ReadThunkError_UnrecognizedPaths $ expectedPath :| []
285+ -- Combine 'ThunkData' from different files, preferring "Checkout" over "Packed"
286+ mergeThunkData ThunkData_Checkout ThunkData_Checkout = Right ThunkData_Checkout
287+ mergeThunkData ThunkData_Checkout ThunkData_Packed{} = Left bothPackedAndUnpacked
288+ mergeThunkData ThunkData_Packed{} ThunkData_Checkout = Left bothPackedAndUnpacked
289+ mergeThunkData a@(ThunkData_Packed _ ptrA) (ThunkData_Packed _ ptrB) =
290+ if ptrA == ptrB then Right a else Left $ ReadThunkError_AmbiguousPackedState ptrA ptrB
292+ bothPackedAndUnpacked = ReadThunkError_UnrecognizedState "Both packed data and checkout present"
294+ fold1WithM (x :| xs) f = foldM f x xs
298+ => NonEmpty (NonEmpty ThunkSpec) -> FilePath -> m (Either ReadThunkError ThunkData)
299+readThunkWith specTypes dir = do
300+ dirFiles <- Set.fromList <$> liftIO (listDirectory dir)
301+ let specs = concatMap toList $ toList $ NonEmpty.transpose specTypes -- Interleave spec types so we try each one in a "fair" ordering
302+ flip fix specs $ \loop -> \case
303+ [] -> pure $ Left ReadThunkError_UnrecognizedThunk
304+ spec:rest -> runExceptT (matchThunkSpecToDir spec dir dirFiles) >>= \case
305+ Left e -> putLog Debug [i|Thunk specification ${_thunkSpec_name spec} did not match ${dir}: ${e}|] *> loop rest
306+ x@(Right _) -> x <$ putLog Debug [i|Thunk specification ${_thunkSpec_name spec} matched ${dir}|]
286-- | Read a thunk and validate that it is exactly a packed thunk.
308-- | Read a thunk and validate that it is exactly a packed thunk.
287-- If additional data is present, fail.
309-- If additional data is present, fail.
288−readPackedThunk :: FilePath -> IO (Either ReadThunkError ThunkPtr)
310+readThunk :: (MonadObelisk m) => FilePath -> m (Either ReadThunkError ThunkData)
289−readPackedThunk thunkDir = runExceptT $ do
311+readThunk = readThunkWith thunkSpecTypes
290− thunkType <- ExceptT $ findThunkType thunkTypes thunkDir
291− -- Ensure that we recognize the thunk loader
292− loader <- liftIO $ T.readFile $ thunkDir </> _thunkType_loader thunkType
293− unless (loader `elem` _thunkType_loaderVersions thunkType) $ do
294− throwError $ ReadThunkError_UnrecognizedLoader loader
296− txt <- liftIO $ LBS.readFile $ thunkDir </> _thunkType_json thunkType
297− case parseMaybe (_thunkType_parser thunkType) =<< Aeson.decode txt of
298− Nothing -> throwError $ ReadThunkError_UnparseablePtr txt
299− Just ptr -> return ptr
301parseThunkPtr :: (Aeson.Object -> Aeson.Parser ThunkSource) -> Aeson.Object -> Aeson.Parser ThunkPtr
313parseThunkPtr :: (Aeson.Object -> Aeson.Parser ThunkSource) -> Aeson.Object -> Aeson.Parser ThunkPtr
302parseThunkPtr parseSrc v = do
314parseThunkPtr parseSrc v = do
@@ -340,16 +352,18 @@ parseGitSource v = do
340overwriteThunk :: MonadObelisk m => FilePath -> ThunkPtr -> m ()
352overwriteThunk :: MonadObelisk m => FilePath -> ThunkPtr -> m ()
341overwriteThunk target thunk = do
353overwriteThunk target thunk = do
342 -- Ensure that this directory is a valid thunk (i.e. so we aren't losing any data)
354 -- Ensure that this directory is a valid thunk (i.e. so we aren't losing any data)
343− Right _ <- readThunk target
355+ readThunk target >>= \case
356+ Left e -> failWith [i|Invalid thunk at ${target}: ${e}|]
345 --TODO: Is there a safer way to do this overwriting?
359 --TODO: Is there a safer way to do this overwriting?
346− liftIO $ removeDirectoryRecursive target
360+ liftIO $ removePathForcibly target
347− liftIO $ createThunk target thunk
361+ createThunk target $ Right thunk
349−thunkPtrLoader :: ThunkPtr -> Text
363+thunkPtrToSpec :: ThunkPtr -> ThunkSpec
350−thunkPtrLoader thunk = case _thunkPtr_source thunk of
364+thunkPtrToSpec thunk = case _thunkPtr_source thunk of
351− ThunkSource_GitHub _ -> NonEmpty.head gitHubStandaloneLoaders
365+ ThunkSource_GitHub _ -> NonEmpty.head gitHubThunkSpecs
352− ThunkSource_Git _ -> NonEmpty.head plainGitStandaloneLoaders
366+ ThunkSource_Git _ -> NonEmpty.head gitThunkSpecs
354-- It's important that formatting be very consistent here, because
368-- It's important that formatting be very consistent here, because
355-- otherwise when people update thunks, their patches will be messy
369-- otherwise when people update thunks, their patches will be messy
@@ -396,35 +410,41 @@ encodeThunkPtrData (ThunkPtr rev src) = case src of
396 , confTrailingNewline = True
410 , confTrailingNewline = True
399−createThunk :: MonadIO m => FilePath -> ThunkPtr -> m ()
413+createThunk :: MonadObelisk m => FilePath -> Either ThunkSpec ThunkPtr -> m ()
400−createThunk target thunk = liftIO $ do
414+createThunk target ptrInfo =
401− createDirectoryIfMissing True (target </> ".attr-cache")
415+ ifor_ (_thunkSpec_files spec) $ \path -> \case
402− T.writeFile (target </> "default.nix") (thunkPtrLoader thunk)
416+ ThunkFileSpec_FileMatches content -> withReadyPath path $ \p -> liftIO $ T.writeFile p content
403− let
417+ ThunkFileSpec_Ptr _ -> case ptrInfo of
404− jsonFileName = case _thunkPtr_source thunk of
418+ Left _ -> pure () -- We can't write the ptr without it
405− ThunkSource_GitHub _ -> "github"
406− ThunkSource_Git _ -> "git"
407− LBS.writeFile (target </> jsonFileName <.> "json") (encodeThunkPtrData thunk)
419+ Right ptr -> withReadyPath path $ \p -> liftIO $ LBS.writeFile p (encodeThunkPtrData ptr)
422+ spec = either id thunkPtrToSpec ptrInfo
423+ withReadyPath path f = do
424+ let fullPath = target </> path
425+ putLog Debug $ "Writing thunk file " <> T.pack fullPath
426+ liftIO $ createDirectoryIfMissing True $ takeDirectory fullPath
409createThunkWithLatest :: MonadObelisk m => FilePath -> ThunkSource -> m ()
429createThunkWithLatest :: MonadObelisk m => FilePath -> ThunkSource -> m ()
410createThunkWithLatest target s = do
430createThunkWithLatest target s = do
411 rev <- getLatestRev s
431 rev <- getLatestRev s
412− createThunk target $ ThunkPtr
432+ createThunk target $ Right $ ThunkPtr
413 { _thunkPtr_source = s
433 { _thunkPtr_source = s
414 , _thunkPtr_rev = rev
434 , _thunkPtr_rev = rev
417updateThunkToLatest :: MonadObelisk m => ThunkUpdateConfig -> FilePath -> m ()
437updateThunkToLatest :: MonadObelisk m => ThunkUpdateConfig -> FilePath -> m ()
418−updateThunkToLatest (ThunkUpdateConfig mBranch thunkConfig) target = withSpinner' ("Updating thunk " <> T.pack target <> " to latest") (pure $ const $ "Thunk " <> T.pack target <> " updated to latest") $ do
438+updateThunkToLatest (ThunkUpdateConfig mBranch thunkConfig) target = spinner $ do
419− checkThunkDirectory "ob thunk update directory cannot be '.'" target
439+ checkThunkDirectory target
420 -- check to see if thunk should be updated to a specific branch or just update it's current branch
440 -- check to see if thunk should be updated to a specific branch or just update it's current branch
421 case mBranch of
441 case mBranch of
422 Nothing -> do
442 Nothing -> do
423 (overwrite, ptr) <- readThunk target >>= \case
443 (overwrite, ptr) <- readThunk target >>= \case
424− Left err -> failWith $ T.pack $ "thunk update: " <> show err
444+ Left err -> failWith [i|Thunk update: ${err}|]
425 Right c -> case c of
445 Right c -> case c of
426− ThunkData_Packed t -> return (target, t)
446+ ThunkData_Packed _ t -> return (target, t)
427− ThunkData_Checkout _ -> failWith "cannot update an unpacked thunk"
447+ ThunkData_Checkout -> failWith "cannot update an unpacked thunk"
428 let src = _thunkPtr_source ptr
448 let src = _thunkPtr_source ptr
429 rev <- getLatestRev src
449 rev <- getLatestRev src
430 overwriteThunk overwrite $ modifyThunkPtrByConfig thunkConfig $ ThunkPtr
450 overwriteThunk overwrite $ modifyThunkPtrByConfig thunkConfig $ ThunkPtr
@@ -432,38 +452,36 @@ updateThunkToLatest (ThunkUpdateConfig mBranch thunkConfig) target = withSpinner
432 , _thunkPtr_rev = rev
452 , _thunkPtr_rev = rev
434 Just branch -> readThunk target >>= \case
454 Just branch -> readThunk target >>= \case
435− Left err -> failWith $ T.pack $ "thunk update: " <> show err
455+ Left err -> failWith [i|Thunk update: ${err}|]
436 Right c -> case c of
456 Right c -> case c of
437− ThunkData_Packed t -> case _thunkPtr_source t of
457+ ThunkData_Packed _ t -> setThunk thunkConfig target (thunkSourceToGitSource $ _thunkPtr_source t) branch
438− ThunkSource_Git tsg -> setThunk thunkConfig target tsg branch
439− ThunkSource_GitHub tsgh -> do
440− let tsg = forgetGithub False tsgh
441− setThunk thunkConfig target tsg branch
442− ThunkData_Checkout _ -> failWith $ T.pack $ "thunk located at " <> show target <> " is unpacked. Use ob thunk pack on the desired directory and then try ob thunk update again."
458+ ThunkData_Checkout -> failWith [i|Thunk located at ${target} is unpacked. Use 'ob thunk pack' on the desired directory and then try 'ob thunk update' again.|]
460+ spinner = withSpinner' ("Updating thunk " <> T.pack target <> " to latest") (pure $ const $ "Thunk " <> T.pack target <> " updated to latest")
444setThunk :: MonadObelisk m => ThunkConfig -> FilePath -> GitSource -> String -> m ()
462setThunk :: MonadObelisk m => ThunkConfig -> FilePath -> GitSource -> String -> m ()
445setThunk thunkConfig target gs branch = do
463setThunk thunkConfig target gs branch = do
446− newThunkPtr <- uriThunkPtr (_gitSource_url gs) (Just $ T.pack branch) Nothing
464+ newThunkPtr <- uriThunkPtr (_gitSource_url gs) (_thunkConfig_private thunkConfig) (Just $ T.pack branch) Nothing
447 overwriteThunk target newThunkPtr
465 overwriteThunk target newThunkPtr
448 updateThunkToLatest (ThunkUpdateConfig Nothing thunkConfig) target
466 updateThunkToLatest (ThunkUpdateConfig Nothing thunkConfig) target
450-- | All recognized github standalone loaders, ordered from newest to oldest.
468-- | All recognized github standalone loaders, ordered from newest to oldest.
451-- This tool will only ever produce the newest one when it writes a thunk.
469-- This tool will only ever produce the newest one when it writes a thunk.
452−gitHubStandaloneLoaders :: NonEmpty Text
470+gitHubThunkSpecs :: NonEmpty ThunkSpec
453−gitHubStandaloneLoaders =
471+gitHubThunkSpecs =
454− gitHubStandaloneLoaderV4 :|
472+ gitHubThunkSpecV5 :|
455− [ gitHubStandaloneLoaderV3
473+ [ gitHubThunkSpecV4
456− , gitHubStandaloneLoaderV2
474+ , gitHubThunkSpecV3
457− , gitHubStandaloneLoaderV1
475+ , gitHubThunkSpecV2
460−gitHubStandaloneLoaderV1 :: Text
479+gitHubThunkSpecV1 :: ThunkSpec
461−gitHubStandaloneLoaderV1 = T.unlines
480+gitHubThunkSpecV1 = legacyGitHubThunkSpec "github-v1"
462− [ "import ((import <nixpkgs> {}).fetchFromGitHub (builtins.fromJSON (builtins.readFile ./github.json)))"
481+ "import ((import <nixpkgs> {}).fetchFromGitHub (builtins.fromJSON (builtins.readFile ./github.json)))"
465−gitHubStandaloneLoaderV2 :: Text
483+gitHubThunkSpecV2 :: ThunkSpec
466−gitHubStandaloneLoaderV2 = T.unlines
484+gitHubThunkSpecV2 = legacyGitHubThunkSpec "github-v2" $ T.unlines
467 [ "# DO NOT HAND-EDIT THIS FILE" --TODO: Add something about how to get more info on Obelisk, etc.
485 [ "# DO NOT HAND-EDIT THIS FILE" --TODO: Add something about how to get more info on Obelisk, etc.
468 , "import ((import <nixpkgs> {}).fetchFromGitHub ("
486 , "import ((import <nixpkgs> {}).fetchFromGitHub ("
469 , " let json = builtins.fromJSON (builtins.readFile ./github.json);"
487 , " let json = builtins.fromJSON (builtins.readFile ./github.json);"
@@ -473,8 +491,8 @@ gitHubStandaloneLoaderV2 = T.unlines
476−gitHubStandaloneLoaderV3 :: Text
494+gitHubThunkSpecV3 :: ThunkSpec
477−gitHubStandaloneLoaderV3 = T.unlines
495+gitHubThunkSpecV3 = legacyGitHubThunkSpec "github-v3" $ T.unlines
478 [ "# DO NOT HAND-EDIT THIS FILE"
496 [ "# DO NOT HAND-EDIT THIS FILE"
480 , " fetch = { private ? false, ... }@args: if private && builtins.hasAttr \"fetchGit\" builtins"
498 , " fetch = { private ? false, ... }@args: if private && builtins.hasAttr \"fetchGit\" builtins"
@@ -493,8 +511,8 @@ gitHubStandaloneLoaderV3 = T.unlines
493 , "in import (fetch (builtins.fromJSON (builtins.readFile ./github.json)))"
511 , "in import (fetch (builtins.fromJSON (builtins.readFile ./github.json)))"
496−gitHubStandaloneLoaderV4 :: Text
514+gitHubThunkSpecV4 :: ThunkSpec
497−gitHubStandaloneLoaderV4 = T.unlines
515+gitHubThunkSpecV4 = legacyGitHubThunkSpec "github-v4" $ T.unlines
498 [ "# DO NOT HAND-EDIT THIS FILE"
516 [ "# DO NOT HAND-EDIT THIS FILE"
499 , "let fetch = { private ? false, fetchSubmodules ? false, owner, repo, rev, sha256, ... }:"
517 , "let fetch = { private ? false, fetchSubmodules ? false, owner, repo, rev, sha256, ... }:"
500 , " if !fetchSubmodules && !private then builtins.fetchTarball {"
518 , " if !fetchSubmodules && !private then builtins.fetchTarball {"
@@ -505,23 +523,50 @@ gitHubStandaloneLoaderV4 = T.unlines
505 , "in import (fetch (builtins.fromJSON (builtins.readFile ./github.json)))"
523 , "in import (fetch (builtins.fromJSON (builtins.readFile ./github.json)))"
508−plainGitStandaloneLoaders :: NonEmpty Text
526+legacyGitHubThunkSpec :: Text -> Text -> ThunkSpec
509−plainGitStandaloneLoaders =
527+legacyGitHubThunkSpec name loader = ThunkSpec name $ Map.fromList
510− plainGitStandaloneLoaderV3 :|
528+ [ ("default.nix", ThunkFileSpec_FileMatches $ T.strip loader)
511− [ plainGitStandaloneLoaderV1
529+ , ("github.json" , ThunkFileSpec_Ptr parseGitHubJsonBytes)
512− , plainGitStandaloneLoaderV2
530+ , (attrCacheFileName, ThunkFileSpec_AttrCache)
531+ , (".git", ThunkFileSpec_CheckoutIndicator)
534+gitHubThunkSpecV5 :: ThunkSpec
535+gitHubThunkSpecV5 = mkThunkSpec "github-v5" "github.json" parseGitHubJsonBytes [here|
536+# DO NOT HAND-EDIT THIS FILE
537+let fetch = { private ? false, fetchSubmodules ? false, owner, repo, rev, sha256, ... }:
538+ if !fetchSubmodules && !private then builtins.fetchTarball {
539+ url = "https://github.com/${owner}/${repo}/archive/${rev}.tar.gz"; inherit sha256;
540+ } else (import <nixpkgs> {}).fetchFromGitHub {
541+ inherit owner repo rev sha256 fetchSubmodules private;
543+ json = builtins.fromJSON (builtins.readFile ./github.json);
547+parseGitHubJsonBytes :: LBS.ByteString -> Either String ThunkPtr
548+parseGitHubJsonBytes = parseJsonObject $ parseThunkPtr $ \v ->
549+ ThunkSource_GitHub <$> parseGitHubSource v <|> ThunkSource_Git <$> parseGitSource v
551+gitThunkSpecs :: NonEmpty ThunkSpec
515−plainGitStandaloneLoaderV1 :: Text
560+gitThunkSpecV1 :: ThunkSpec
516−plainGitStandaloneLoaderV1 = T.unlines
561+gitThunkSpecV1 = legacyGitThunkSpec "git-v1" $ T.unlines
517 [ "# DO NOT HAND-EDIT THIS FILE"
562 [ "# DO NOT HAND-EDIT THIS FILE"
518 , "let fetchGit = {url, rev, ref ? null, branch ? null, sha256 ? null, fetchSubmodules ? null}:"
563 , "let fetchGit = {url, rev, ref ? null, branch ? null, sha256 ? null, fetchSubmodules ? null}:"
519 , " assert !fetchSubmodules; (import <nixpkgs> {}).fetchgit { inherit url rev sha256; };"
564 , " assert !fetchSubmodules; (import <nixpkgs> {}).fetchgit { inherit url rev sha256; };"
520 , "in import (fetchGit (builtins.fromJSON (builtins.readFile ./git.json)))"
565 , "in import (fetchGit (builtins.fromJSON (builtins.readFile ./git.json)))"
523−plainGitStandaloneLoaderV2 :: Text
568+gitThunkSpecV2 :: ThunkSpec
524−plainGitStandaloneLoaderV2 = T.unlines
569+gitThunkSpecV2 = legacyGitThunkSpec "git-v2" $ T.unlines
525 [ "# DO NOT HAND-EDIT THIS FILE"
570 [ "# DO NOT HAND-EDIT THIS FILE"
526 , "let fetchGit = {url, rev, ref ? null, branch ? null, sha256 ? null, fetchSubmodules ? null}:"
571 , "let fetchGit = {url, rev, ref ? null, branch ? null, sha256 ? null, fetchSubmodules ? null}:"
527 , " if builtins.hasAttr \"fetchGit\" builtins"
572 , " if builtins.hasAttr \"fetchGit\" builtins"
@@ -530,8 +575,10 @@ plainGitStandaloneLoaderV2 = T.unlines
530 , "in import (fetchGit (builtins.fromJSON (builtins.readFile ./git.json)))"
575 , "in import (fetchGit (builtins.fromJSON (builtins.readFile ./git.json)))"
533−plainGitStandaloneLoaderV3 :: Text
578+-- This loader has a bug because @builtins.fetchGit@ is not given a @ref@
534−plainGitStandaloneLoaderV3 = T.unlines
579+-- and will fail to find commits without this because it does shallow clones.
580+gitThunkSpecV3 :: ThunkSpec
581+gitThunkSpecV3 = legacyGitThunkSpec "git-v3" $ T.unlines
535 [ "# DO NOT HAND-EDIT THIS FILE"
582 [ "# DO NOT HAND-EDIT THIS FILE"
536 , "let fetch = {url, rev, ref ? null, sha256 ? null, fetchSubmodules ? false, private ? false, ...}:"
583 , "let fetch = {url, rev, ref ? null, sha256 ? null, fetchSubmodules ? false, private ? false, ...}:"
537 , " let realUrl = let firstChar = builtins.substring 0 1 url; in"
584 , " let realUrl = let firstChar = builtins.substring 0 1 url; in"
@@ -546,10 +593,69 @@ plainGitStandaloneLoaderV3 = T.unlines
546 , "in import (fetch (builtins.fromJSON (builtins.readFile ./git.json)))"
593 , "in import (fetch (builtins.fromJSON (builtins.readFile ./git.json)))"
549−thunkFileNames :: [FilePath]
596+gitThunkSpecV4 :: ThunkSpec
550−thunkFileNames = ["default.nix", "github.json", "git.json"]
597+gitThunkSpecV4 = legacyGitThunkSpec "git-v4" $ T.unlines
598+ [ "# DO NOT HAND-EDIT THIS FILE"
599+ , "let fetch = {url, rev, branch ? null, sha256 ? null, fetchSubmodules ? false, private ? false, ...}:"
600+ , " let realUrl = let firstChar = builtins.substring 0 1 url; in"
601+ , " if firstChar == \"/\" then /. + url"
602+ , " else if firstChar == \".\" then ./. + url"
604+ , " in if !fetchSubmodules && private then builtins.fetchGit {"
605+ , " url = realUrl; inherit rev;"
606+ , " ${if branch == null then null else \"ref\"} = branch;"
607+ , " } else (import <nixpkgs> {}).fetchgit {"
608+ , " url = realUrl; inherit rev sha256;"
610+ , "in import (fetch (builtins.fromJSON (builtins.readFile ./git.json)))"
613+legacyGitThunkSpec :: Text -> Text -> ThunkSpec
614+legacyGitThunkSpec name loader = ThunkSpec name $ Map.fromList
615+ [ ("default.nix", ThunkFileSpec_FileMatches $ T.strip loader)
616+ , ("git.json" , ThunkFileSpec_Ptr parseGitJsonBytes)
617+ , (attrCacheFileName, ThunkFileSpec_AttrCache)
618+ , (".git", ThunkFileSpec_CheckoutIndicator)
621+gitThunkSpecV5 :: ThunkSpec
622+gitThunkSpecV5 = mkThunkSpec "git-v5" "git.json" parseGitJsonBytes [here|
623+# DO NOT HAND-EDIT THIS FILE
624+let fetch = {url, rev, branch ? null, sha256 ? null, fetchSubmodules ? false, private ? false, ...}:
625+ let realUrl = let firstChar = builtins.substring 0 1 url; in
626+ if firstChar == "/" then /. + url
627+ else if firstChar == "." then ./. + url
629+ in if !fetchSubmodules && private then builtins.fetchGit {
630+ url = realUrl; inherit rev;
631+ ${if branch == null then null else "ref"} = branch;
632+ } else (import <nixpkgs> {}).fetchgit {
633+ url = realUrl; inherit rev sha256;
635+ json = builtins.fromJSON (builtins.readFile ./git.json);
639+parseGitJsonBytes :: LBS.ByteString -> Either String ThunkPtr
640+parseGitJsonBytes = parseJsonObject $ parseThunkPtr $ fmap ThunkSource_Git . parseGitSource
642+mkThunkSpec :: Text -> FilePath -> (LBS.ByteString -> Either String ThunkPtr) -> Text -> ThunkSpec
643+mkThunkSpec name jsonFileName parser srcNix = ThunkSpec name $ Map.fromList
644+ [ ("default.nix", ThunkFileSpec_FileMatches defaultNixViaSrc)
645+ , ("thunk.nix", ThunkFileSpec_FileMatches srcNix)
646+ , (jsonFileName, ThunkFileSpec_Ptr parser)
647+ , (attrCacheFileName, ThunkFileSpec_AttrCache)
648+ , (normalise $ unpackedDirName </> ".git", ThunkFileSpec_CheckoutIndicator)
651+ defaultNixViaSrc = [here|
652+# DO NOT HAND-EDIT THIS FILE
653+import (import ./thunk.nix)
552−--TODO: when checking something out, make a shallow clone
657+parseJsonObject :: (Aeson.Object -> Aeson.Parser a) -> LBS.ByteString -> Either String a
658+parseJsonObject p bytes = Aeson.parseEither p =<< Aeson.eitherDecode bytes
554-- | Checks a cache directory to see if there is a fresh symlink
660-- | Checks a cache directory to see if there is a fresh symlink
555-- to the result of building an attribute of a thunk.
661-- to the result of building an attribute of a thunk.
@@ -557,44 +663,49 @@ thunkFileNames = ["default.nix", "github.json", "git.json"]
557-- and the result is symlinked into the cache.
663-- and the result is symlinked into the cache.
558nixBuildThunkAttrWithCache
664nixBuildThunkAttrWithCache
559 :: MonadObelisk m
665 :: MonadObelisk m
560− => FilePath
667+ -> FilePath
561 -- ^ Path to directory containing Thunk
668 -- ^ Path to directory containing Thunk
562 -> String
669 -> String
563 -- ^ Attribute to build
670 -- ^ Attribute to build
564− -> m FilePath
671+ -> m (Maybe FilePath)
565 -- ^ Symlink to cached or built nix output
672 -- ^ Symlink to cached or built nix output
566-- WARNING: If the thunk uses an impure reference such as '<nixpkgs>'
673-- WARNING: If the thunk uses an impure reference such as '<nixpkgs>'
567-- the caching mechanism will fail as it merely measures the modification
674-- the caching mechanism will fail as it merely measures the modification
568-- time of the cache link and the expression to build.
675-- time of the cache link and the expression to build.
569−nixBuildThunkAttrWithCache thunkDir attr = do
676+nixBuildThunkAttrWithCache thunkSpec thunkDir attr = do
570− --NB: Expects thunkDir to be normalised with no trailing path separator.
571− --This should be guaranteed by the command argument parser.
572− let cacheErrHandler e
573− | isDoesNotExistError e = return Nothing -- expected from a cache miss
574− | otherwise = putLog Error (T.pack $ displayException e) >> return Nothing
575− cacheDir = thunkDir </> ".attr-cache"
576− cachePath = cacheDir </> attr <.> "out"
577 latestChange <- liftIO $ do
677 latestChange <- liftIO $ do
578− createDirectoryIfMissing False cacheDir
678+ let
579− let getModificationTimeMaybe = fmap rightToMaybe . try @IOError . getModificationTime
679+ getModificationTimeMaybe = fmap rightToMaybe . try @IOError . getModificationTime
680+ thunkFileNames = Map.keys $ _thunkSpec_files thunkSpec
580− maximum . catMaybes <$> mapM (getModificationTimeMaybe . (thunkDir </>)) thunkFileNames
681+ maximum . catMaybes <$> traverse (getModificationTimeMaybe . (thunkDir </>)) thunkFileNames
683+ let cachePaths' = nonEmpty $ Map.keys $ Map.filter (\case ThunkFileSpec_AttrCache -> True; _ -> False) $
684+ _thunkSpec_files thunkSpec
685+ for cachePaths' $ \cachePaths ->
686+ fmap NonEmpty.head $ for cachePaths $ \cacheDir -> do
688+ cachePath = thunkDir </> cacheDir </> attr <.> "out"
690+ | isDoesNotExistError e = pure Nothing -- expected from a cache miss
691+ | otherwise = Nothing <$ putLog Error (T.pack $ displayException e)
581− cacheHit <- handle cacheErrHandler $ do
692+ cacheHit <- handle cacheErrHandler $ do
582− cacheTime <- liftIO $ posixSecondsToUTCTime . realToFrac . modificationTime <$> getSymbolicLinkStatus cachePath
693+ cacheTime <- liftIO $ posixSecondsToUTCTime . realToFrac . modificationTime <$> getSymbolicLinkStatus cachePath
583− return $ if latestChange <= cacheTime
694+ pure $ if latestChange <= cacheTime
584− then Just cachePath
695+ then Just cachePath
585− else Nothing
696+ else Nothing
586− case cacheHit of
697+ case cacheHit of
587− Just c -> return c
698+ Just c -> pure c
588− Nothing -> do
699+ Nothing -> do
589− putLog Warning $ T.pack $ mconcat [thunkDir, ": ", attr, " not cached, building ..."]
700+ putLog Warning $ T.pack $ mconcat [thunkDir, ": ", attr, " not cached, building ..."]
701+ liftIO $ createDirectoryIfMissing True (takeDirectory cachePath)
590− _ <- nixCmd $ NixCmd_Build $ def
702+ (cachePath <$) $ nixCmd $ NixCmd_Build $ def
591− Lens.& nixBuildConfig_outLink Lens..~ OutLink_IndirectRoot cachePath
703+ & nixBuildConfig_outLink .~ OutLink_IndirectRoot cachePath
592− Lens.& nixCmdConfig_target Lens..~ Target
704+ & nixCmdConfig_target .~ Target
593− { _target_path = Just thunkDir
705+ { _target_path = Just thunkDir
594− , _target_attr = Just attr
706+ , _target_attr = Just attr
595− , _target_expr = Nothing
707+ , _target_expr = Nothing
599-- | Build a nix attribute, and cache the result if possible
710-- | Build a nix attribute, and cache the result if possible
600nixBuildAttrWithCache
711nixBuildAttrWithCache
@@ -605,13 +716,15 @@ nixBuildAttrWithCache
605 -- ^ Attribute to build
716 -- ^ Attribute to build
606 -> m FilePath
717 -> m FilePath
607 -- ^ Symlink to cached or built nix output
718 -- ^ Symlink to cached or built nix output
608−nixBuildAttrWithCache exprPath attr = do
719+nixBuildAttrWithCache exprPath attr = readThunk exprPath >>= \case
609− readThunk exprPath >>= \case
610− -- Only packed thunks are cached. in particular, checkouts are not
720+ -- Only packed thunks are cached. In particular, checkouts are not.
721+ Right (ThunkData_Packed spec _) ->
611− Right (ThunkData_Packed _) -> nixBuildThunkAttrWithCache exprPath attr
722+ maybe build pure =<< nixBuildThunkAttrWithCache spec exprPath attr
612− _ -> nixCmd $ NixCmd_Build $ def
725+ build = nixCmd $ NixCmd_Build $ def
613− Lens.& nixBuildConfig_outLink Lens..~ OutLink_None
726+ & nixBuildConfig_outLink .~ OutLink_None
614− Lens.& nixCmdConfig_target Lens..~ Target
727+ & nixCmdConfig_target .~ Target
615 { _target_path = Just exprPath
728 { _target_path = Just exprPath
616 , _target_attr = Just attr
729 , _target_attr = Just attr
617 , _target_expr = Nothing
730 , _target_expr = Nothing
@@ -632,7 +745,7 @@ updateThunk p f = withSystemTempDirectory "obelisk-thunkptr-" $ \tmpDir -> do
633 copyThunkToTmp tmpDir thunkDir = readThunk thunkDir >>= \case
746 copyThunkToTmp tmpDir thunkDir = readThunk thunkDir >>= \case
634 Left err -> failWith $ "withThunkUnpacked: " <> T.pack (show err)
747 Left err -> failWith $ "withThunkUnpacked: " <> T.pack (show err)
635− Right (ThunkData_Packed _) -> do
748+ Right ThunkData_Packed{} -> do
636 let tmpThunk = tmpDir </> "thunk"
749 let tmpThunk = tmpDir </> "thunk"
637 callProcessAndLogOutput (Notice, Error) $
750 callProcessAndLogOutput (Notice, Error) $
638 proc cp ["-r", "-T", thunkDir, tmpThunk]
751 proc cp ["-r", "-T", thunkDir, tmpThunk]
@@ -646,62 +759,81 @@ updateThunk p f = withSystemTempDirectory "obelisk-thunkptr-" $ \tmpDir -> do
646finalMsg :: Bool -> (a -> Text) -> Maybe (a -> Text)
759finalMsg :: Bool -> (a -> Text) -> Maybe (a -> Text)
647finalMsg noTrail s = if noTrail then Nothing else Just s
760finalMsg noTrail s = if noTrail then Nothing else Just s
649−unpackThunk :: MonadObelisk m => FilePath -> m ()
650−unpackThunk = unpackThunk' False
652-- | Check that we are not somewhere inside the thunk directory
762-- | Check that we are not somewhere inside the thunk directory
653−checkThunkDirectory :: MonadObelisk m => Text -> FilePath -> m ()
763+checkThunkDirectory :: MonadObelisk m => FilePath -> m ()
654−checkThunkDirectory msg thunkDir = do
764+checkThunkDirectory thunkDir = do
655 currentDir <- liftIO getCurrentDirectory
765 currentDir <- liftIO getCurrentDirectory
656 thunkDir' <- liftIO $ canonicalizePath thunkDir
766 thunkDir' <- liftIO $ canonicalizePath thunkDir
657− when (thunkDir' `L.isInfixOf` currentDir) $ failWith msg
767+ when (thunkDir' `L.isInfixOf` currentDir) $
768+ failWith [i|Can't perform thunk operations from within the thunk directory: ${thunkDir}|]
770+ -- Don't let thunk commands work when directly given an unpacked repo
771+ when (takeFileName thunkDir == unpackedDirName) $
772+ readThunk (takeDirectory thunkDir) >>= \case
773+ Right _ -> failWith [i|Refusing to perform thunk operation on ${thunkDir} because it is a thunk's unpacked source|]
776+unpackThunk :: MonadObelisk m => FilePath -> m ()
777+unpackThunk = unpackThunk' False
659unpackThunk' :: MonadObelisk m => Bool -> FilePath -> m ()
779unpackThunk' :: MonadObelisk m => Bool -> FilePath -> m ()
660−unpackThunk' noTrail thunkDir = checkThunkDirectory "Can't pack/unpack from within the thunk directory" thunkDir >> readThunk thunkDir >>= \case
780+unpackThunk' noTrail thunkDir = checkThunkDirectory thunkDir *> readThunk thunkDir >>= \case
661− Left err -> failWith $ "thunk unpack: " <> T.pack (show err)
781+ Left err -> failWith [i|Invalid thunk at ${thunkDir}: ${err}|]
662 --TODO: Overwrite option that rechecks out thunk; force option to do so even if working directory is dirty
782 --TODO: Overwrite option that rechecks out thunk; force option to do so even if working directory is dirty
663− Right (ThunkData_Checkout _) -> failWith "thunk unpack: thunk is already unpacked"
783+ Right ThunkData_Checkout -> failWith [i|Thunk at ${thunkDir} is already unpacked|]
664− Right (ThunkData_Packed tptr) -> do
784+ Right (ThunkData_Packed _ tptr) -> do
665 let (thunkParent, thunkName) = splitFileName thunkDir
785 let (thunkParent, thunkName) = splitFileName thunkDir
666− withTempDirectory thunkParent thunkName $ \tmpRepo -> do
786+ withTempDirectory thunkParent thunkName $ \tmpThunk -> do
667− let obGitDir = tmpRepo </> ".git" </> "obelisk"
787+ let
788+ gitSrc = thunkSourceToGitSource $ _thunkPtr_source tptr
668− s = case _thunkPtr_source tptr of
789+ newSpec = case _thunkPtr_source tptr of
669− ThunkSource_GitHub s' -> forgetGithub False s'
790+ ThunkSource_GitHub _ -> NonEmpty.head gitHubThunkSpecs
670− ThunkSource_Git s' -> s'
791+ ThunkSource_Git _ -> NonEmpty.head gitThunkSpecs
671 withSpinner' ("Fetching thunk " <> T.pack thunkName)
792 withSpinner' ("Fetching thunk " <> T.pack thunkName)
672 (finalMsg noTrail $ const $ "Fetched thunk " <> T.pack thunkName) $ do
793 (finalMsg noTrail $ const $ "Fetched thunk " <> T.pack thunkName) $ do
794+ let unpackedPath = tmpThunk </> unpackedDirName
795+ gitCloneForThunkUnpack gitSrc (_thunkRev_commit $ _thunkPtr_rev tptr) unpackedPath
797+ let normalizeMore = dropTrailingPathSeparator . normalise
798+ when (normalizeMore unpackedPath /= normalizeMore tmpThunk) $ -- Only write meta data if the checkout is not inplace
799+ createThunk tmpThunk $ Left newSpec
802+ removePathForcibly thunkDir
803+ renameDirectory tmpThunk thunkDir
805+gitCloneForThunkUnpack
807+ => GitSource -- ^ Git source to use
808+ -> Ref hash -- ^ Commit hash to reset to
809+ -> FilePath -- ^ Directory to clone into
811+gitCloneForThunkUnpack gitSrc commit dir = do
673− let git = callProcessAndLogOutput (Notice, Notice) . gitProc tmpRepo
812+ let git = callProcessAndLogOutput (Notice, Notice) . gitProc dir
674− git $ [ "clone" ]
813+ git $ [ "clone" ]
675− ++ ("--recursive" <$ guard (_gitSource_fetchSubmodules s))
814+ ++ ["--recursive" | _gitSource_fetchSubmodules gitSrc]
676− ++ [ T.unpack $ gitUriToText $ _gitSource_url s ]
815+ ++ [ T.unpack $ gitUriToText $ _gitSource_url gitSrc ]
677− ++ do branch <- maybeToList $ _gitSource_branch s
816+ ++ do branch <- maybeToList $ _gitSource_branch gitSrc
678− [ "--branch", T.unpack $ untagName branch ]
817+ [ "--branch", T.unpack $ untagName branch ]
679− git ["reset", "--hard", Ref.toHexString $ _thunkRev_commit $ _thunkPtr_rev tptr]
818+ git ["reset", "--hard", Ref.toHexString commit]
680− when (_gitSource_fetchSubmodules s) $
819+ when (_gitSource_fetchSubmodules gitSrc) $
681− git ["submodule", "update", "--recursive", "--init"]
820+ git ["submodule", "update", "--recursive", "--init"]
683− liftIO $ createDirectory obGitDir
684− callProcessAndLogOutput (Notice, Error) $
685− proc cp ["-r", "-T", thunkDir </> ".", obGitDir </> "orig-thunk"]
686− callProcessAndLogOutput (Notice, Error) $
687− proc "rm" ["-r", thunkDir]
688− callProcessAndLogOutput (Notice, Error) $
689− proc "mv" ["-T", tmpRepo, thunkDir]
691--TODO: add a rollback mode to pack to the original thunk
822--TODO: add a rollback mode to pack to the original thunk
692packThunk :: MonadObelisk m => ThunkPackConfig -> FilePath -> m ThunkPtr
823packThunk :: MonadObelisk m => ThunkPackConfig -> FilePath -> m ThunkPtr
693packThunk = packThunk' False
824packThunk = packThunk' False
695packThunk' :: MonadObelisk m => Bool -> ThunkPackConfig -> FilePath -> m ThunkPtr
826packThunk' :: MonadObelisk m => Bool -> ThunkPackConfig -> FilePath -> m ThunkPtr
696−packThunk' noTrail (ThunkPackConfig force thunkConfig) thunkDir = checkThunkDirectory "Can't pack/unpack from within the thunk directory" thunkDir >> readThunk thunkDir >>= \case
827+packThunk' noTrail (ThunkPackConfig force thunkConfig) thunkDir = checkThunkDirectory thunkDir *> readThunk thunkDir >>= \case
697− Left err -> failWith $ T.pack $ "thunk pack: " <> show err
698− Right (ThunkData_Packed _) -> failWith "pack: thunk is already packed"
828+ Right ThunkData_Packed{} -> failWith [i|Thunk at ${thunkDir} is is already packed|]
699− Right (ThunkData_Checkout _) -> do
829+ _ -> withSpinner'
700− withSpinner' ("Packing thunk " <> T.pack thunkDir)
830+ ("Packing thunk " <> T.pack thunkDir)
701− (finalMsg noTrail $ const $ "Packed thunk " <> T.pack thunkDir) $ do
831+ (finalMsg noTrail $ const $ "Packed thunk " <> T.pack thunkDir) $
833+ let checkClean = if force then CheckClean_NoCheck else CheckClean_FullCheck
702− thunkPtr <- modifyThunkPtrByConfig thunkConfig <$> getThunkPtr (not force) thunkDir
834+ thunkPtr <- modifyThunkPtrByConfig thunkConfig <$> getThunkPtr checkClean thunkDir (_thunkConfig_private thunkConfig)
703− callProcessAndLogOutput (Debug, Error) $ proc "rm" ["-rf", thunkDir]
835+ liftIO $ removePathForcibly thunkDir
704− liftIO $ createThunk thunkDir thunkPtr
836+ createThunk thunkDir $ Right thunkPtr
705 pure thunkPtr
837 pure thunkPtr
707modifyThunkPtrByConfig :: ThunkConfig -> ThunkPtr -> ThunkPtr
839modifyThunkPtrByConfig :: ThunkConfig -> ThunkPtr -> ThunkPtr
@@ -712,29 +844,43 @@ modifyThunkPtrByConfig (ThunkConfig markPrivate') ptr = case markPrivate' of
712 ThunkSource_GitHub s -> ThunkSource_GitHub $ s { _gitHubSource_private = markPrivate }
844 ThunkSource_GitHub s -> ThunkSource_GitHub $ s { _gitHubSource_private = markPrivate }
848+ = CheckClean_FullCheck
849+ -- ^ Check that the repo is clean, including .gitignored files
850+ | CheckClean_NotIgnored
851+ -- ^ Check that the repo is clean, not including .gitignored files
852+ | CheckClean_NoCheck
853+ -- ^ Don't check that the repo is clean
715−getThunkPtr :: forall m. MonadObelisk m => Bool -> FilePath -> m ThunkPtr
855+getThunkPtr :: forall m. MonadObelisk m => CheckClean -> FilePath -> Maybe Bool -> m ThunkPtr
716−getThunkPtr checkClean thunkDir = do
856+getThunkPtr gitCheckClean dir mPrivate = do
857+ let repoLocations = nubOrd $ map (first normalise)
858+ [(".git", "."), (unpackedDirName </> ".git", unpackedDirName)]
859+ repoLocation' <- liftIO $ flip findM repoLocations $ doesDirectoryExist . (dir </>) . fst
860+ thunkDir <- case repoLocation' of
861+ Nothing -> failWith [i|Can't find an unpacked thunk in ${dir}|]
862+ Just (_, path) -> pure $ normalise $ dir </> path
864+ let (checkClean, checkIgnored) = case gitCheckClean of
865+ CheckClean_FullCheck -> (True, True)
866+ CheckClean_NotIgnored -> (True, False)
867+ CheckClean_NoCheck -> (False, False)
717− when checkClean $ ensureCleanGitRepo thunkDir True
868+ when checkClean $ ensureCleanGitRepo thunkDir checkIgnored
718 "thunk pack: thunk checkout contains unsaved modifications"
869 "thunk pack: thunk checkout contains unsaved modifications"
720 -- Check whether there are any stashes
871 -- Check whether there are any stashes
872+ when checkClean $ do
721− stashOutput <- readGitProcess thunkDir ["stash", "list"]
873+ stashOutput <- readGitProcess thunkDir ["stash", "list"]
722− when checkClean $ case T.null stashOutput of
874+ unless (T.null stashOutput) $
724 failWith $ T.unlines $
875 failWith $ T.unlines $
725 [ "thunk pack: thunk checkout has stashes"
876 [ "thunk pack: thunk checkout has stashes"
726 , "git stash list:"
877 , "git stash list:"
727 ] ++ T.lines stashOutput
878 ] ++ T.lines stashOutput
730− -- Get current branch ``
880+ -- Get current branch
731 (mCurrentBranch, mCurrentCommit) <- do
881 (mCurrentBranch, mCurrentCommit) <- do
734− <$> readGitProcess thunkDir ["rev-parse", "--abbrev-ref", "HEAD"]
882+ b <- listToMaybe . T.lines <$> readGitProcess thunkDir ["rev-parse", "--abbrev-ref", "HEAD"]
737− <$> readGitProcess thunkDir ["rev-parse", "HEAD"]
883+ c <- listToMaybe . T.lines <$> readGitProcess thunkDir ["rev-parse", "HEAD"]
738 case b of
884 case b of
739 (Just "HEAD") -> failWith $ T.unlines
885 (Just "HEAD") -> failWith $ T.unlines
740 [ "thunk pack: You are in 'detached HEAD' state."
886 [ "thunk pack: You are in 'detached HEAD' state."
@@ -745,7 +891,7 @@ getThunkPtr checkClean thunkDir = do
746 -- Get information on all branches and their (optional) designated upstream
892 -- Get information on all branches and their (optional) designated upstream
747 -- correspondents
893 -- correspondents
748− (headDump :: [Text]) <- T.lines <$> readGitProcess thunkDir
894+ headDump :: [Text] <- T.lines <$> readGitProcess thunkDir
749 [ "for-each-ref"
895 [ "for-each-ref"
750 , "--format=%(refname:short) %(upstream:short) %(upstream:remotename)"
896 , "--format=%(refname:short) %(upstream:short) %(upstream:remotename)"
751 , "refs/heads/"
897 , "refs/heads/"
@@ -791,7 +937,7 @@ getThunkPtr checkClean thunkDir = do
793 -- loosely by https://stackoverflow.com/questions/7773939/show-git-ahead-and-behind-info-for-all-branches-including-remotes
939 -- loosely by https://stackoverflow.com/questions/7773939/show-git-ahead-and-behind-info-for-all-branches-including-remotes
794− stats <- iforM headUpstream $ \branch (upstream, _remote) -> do
940+ stats <- ifor headUpstream $ \branch (upstream, _remote) -> do
795 (stat :: [Text]) <- T.lines <$> readGitProcess thunkDir
941 (stat :: [Text]) <- T.lines <$> readGitProcess thunkDir
796 [ "rev-list", "--left-right"
942 [ "rev-list", "--left-right"
797 , T.unpack branch <> "..." <> T.unpack upstream
943 , T.unpack branch <> "..." <> T.unpack upstream
@@ -815,8 +961,9 @@ getThunkPtr checkClean thunkDir = do
815 \pushed but this repo's remote tracking branches don't know it.)"
961 \pushed but this repo's remote tracking branches don't know it.)"
964+ when checkClean $ do
818− -- We assume it's safe to pack the thunk at this point
965+ -- We assume it's safe to pack the thunk at this point
819− putLog Informational "All changes safe in git remotes. OK to pack thunk."
966+ putLog Informational "All changes safe in git remotes. OK to pack thunk."
821 let remote = maybe "origin" snd $ flip Map.lookup headUpstream =<< mCurrentBranch
968 let remote = maybe "origin" snd $ flip Map.lookup headUpstream =<< mCurrentBranch
@@ -829,14 +976,12 @@ getThunkPtr checkClean thunkDir = do
829 remoteUri <- case parseGitUri remoteUri' of
976 remoteUri <- case parseGitUri remoteUri' of
830 Nothing -> failWith $ "Could not identify git remote: " <> remoteUri'
977 Nothing -> failWith $ "Could not identify git remote: " <> remoteUri'
831 Just uri -> pure uri
978 Just uri -> pure uri
832− uriThunkPtr remoteUri mCurrentBranch mCurrentCommit
979+ uriThunkPtr remoteUri mPrivate mCurrentBranch mCurrentCommit
834-- | Get the latest revision available from the given source
981-- | Get the latest revision available from the given source
835getLatestRev :: MonadObelisk m => ThunkSource -> m ThunkRev
982getLatestRev :: MonadObelisk m => ThunkSource -> m ThunkRev
836getLatestRev os = do
983getLatestRev os = do
837− let gitS = case os of
984+ let gitS = thunkSourceToGitSource os
838− ThunkSource_GitHub s -> forgetGithub False s
839− ThunkSource_Git s -> s
840 (_, commit) <- gitGetCommitBranch (_gitSource_url gitS) (untagName <$> _gitSource_branch gitS)
985 (_, commit) <- gitGetCommitBranch (_gitSource_url gitS) (untagName <$> _gitSource_branch gitS)
841 case os of
986 case os of
842 ThunkSource_GitHub s -> githubThunkRev s commit
987 ThunkSource_GitHub s -> githubThunkRev s commit
@@ -848,12 +993,12 @@ getLatestRev os = do
848-- performance. If that doesn't work (e.g. authentication issue), we fall back
993-- performance. If that doesn't work (e.g. authentication issue), we fall back
849-- on just doing things the normal way for git repos in general, and save it as
994-- on just doing things the normal way for git repos in general, and save it as
850-- a regular git thunk.
995-- a regular git thunk.
851−uriThunkPtr :: MonadObelisk m => GitUri -> Maybe Text -> Maybe Text -> m ThunkPtr
996+uriThunkPtr :: MonadObelisk m => GitUri -> Maybe Bool -> Maybe Text -> Maybe Text -> m ThunkPtr
852−uriThunkPtr uri mbranch mcommit = do
997+uriThunkPtr uri mPrivate mbranch mcommit = do
853 commit <- case mcommit of
998 commit <- case mcommit of
854 Nothing -> gitGetCommitBranch uri mbranch >>= return . snd
999 Nothing -> gitGetCommitBranch uri mbranch >>= return . snd
855 (Just c) -> return c
1000 (Just c) -> return c
856− (src, rev) <- case uriToThunkSource uri mbranch of
1001+ (src, rev) <- uriToThunkSource uri mPrivate mbranch >>= \case
857 ThunkSource_GitHub s -> do
1002 ThunkSource_GitHub s -> do
858 rev <- runExceptT $ githubThunkRev s commit
1003 rev <- runExceptT $ githubThunkRev s commit
859 case rev of
1004 case rev of
@@ -876,8 +1021,8 @@ uriThunkPtr uri mbranch mcommit = do
876-- If the thunk is a GitHub thunk and fails, we do *not* fall back like with
1021-- If the thunk is a GitHub thunk and fails, we do *not* fall back like with
877-- `uriThunkPtr`. Unlike a plain URL, a thunk src explicitly states which method
1022-- `uriThunkPtr`. Unlike a plain URL, a thunk src explicitly states which method
878-- should be employed, and so we respect that.
1023-- should be employed, and so we respect that.
879−uriToThunkSource :: GitUri -> Maybe Text -> ThunkSource
1024+uriToThunkSource :: MonadObelisk m => GitUri -> Maybe Bool -> Maybe Text -> m ThunkSource
880−uriToThunkSource (GitUri u)
1025+uriToThunkSource (GitUri u) mPrivate
881 | Right uriAuth <- URI.uriAuthority u
1026 | Right uriAuth <- URI.uriAuthority u
882 , Just scheme <- URI.unRText <$> URI.uriScheme u
1027 , Just scheme <- URI.unRText <$> URI.uriScheme u
883 , case scheme of
1028 , case scheme of
@@ -889,21 +1034,53 @@ uriToThunkSource (GitUri u)
889 s -> s `L.elem` [ "git", "https", "http" ] -- "http:" just redirects to "https:"
1034 s -> s `L.elem` [ "git", "https", "http" ] -- "http:" just redirects to "https:"
890 && URI.unRText (URI.authHost uriAuth) == "github.com"
1035 && URI.unRText (URI.authHost uriAuth) == "github.com"
891 , Just (_, owner :| [repoish]) <- URI.uriPath u
1036 , Just (_, owner :| [repoish]) <- URI.uriPath u
1038+ isPrivate <- getIsPrivate
892− = \mbranch -> ThunkSource_GitHub $ GitHubSource
1039+ pure $ ThunkSource_GitHub $ GitHubSource
893− { _gitHubSource_owner = N $ URI.unRText owner
1040+ { _gitHubSource_owner = N $ URI.unRText owner
894− , _gitHubSource_repo = N $ let
1041+ , _gitHubSource_repo = N $ let
895− repoish' = URI.unRText repoish
1042+ repoish' = URI.unRText repoish
896− in fromMaybe repoish' $ T.stripSuffix ".git" repoish'
1043+ in fromMaybe repoish' $ T.stripSuffix ".git" repoish'
897− , _gitHubSource_branch = N <$> mbranch
1044+ , _gitHubSource_branch = N <$> mbranch
898− , _gitHubSource_private = False -- TODO: Can we try something to infer this?
1045+ , _gitHubSource_private = isPrivate
1048+ | otherwise = \mbranch -> do
1049+ isPrivate <- getIsPrivate
901− | otherwise = \mbranch -> ThunkSource_Git $ GitSource
1050+ pure $ ThunkSource_Git $ GitSource
902− { _gitSource_url = GitUri u
1051+ { _gitSource_url = GitUri u
903− , _gitSource_branch = N <$> mbranch
1052+ , _gitSource_branch = N <$> mbranch
904− , _gitSource_fetchSubmodules = False -- TODO: How do we determine if this should be true?
1053+ , _gitSource_fetchSubmodules = False -- TODO: How do we determine if this should be true?
905− , _gitSource_private = False -- TODO: Can we try something to infer this?
1054+ , _gitSource_private = isPrivate
1057+ getIsPrivate = maybe (guessGitRepoIsPrivate $ GitUri u) pure mPrivate
1059+guessGitRepoIsPrivate :: MonadObelisk m => GitUri -> m Bool
1060+guessGitRepoIsPrivate uri = flip fix urisToTry $ \loop -> \case
1062+ uriAttempt:xs -> do
1063+ result <- readCreateProcessWithExitCode $
1070+ , T.unpack $ gitUriToText uriAttempt
1073+ (ExitSuccess, _, _) -> pure False -- Must be a public repo
1076+ urisToTry = nubOrd $
1077+ -- Include the original URI if it isn't using SSH because SSH will certainly fail.
1078+ [uri | fmap URI.unRText (URI.uriScheme (unGitUri uri)) /= Just "ssh"] <>
1079+ [changeScheme "https" uri, changeScheme "http" uri]
1080+ changeScheme scheme (GitUri u) = GitUri $ u
1081+ { URI.uriScheme = URI.mkScheme scheme
1082+ , URI.uriAuthority = (\x -> x { URI.authUserInfo = Nothing }) <$> URI.uriAuthority u
908-- Funny signature indicates no effects depend on the optional branch name.
1085-- Funny signature indicates no effects depend on the optional branch name.
909githubThunkRev
1086githubThunkRev
@@ -963,6 +1140,7 @@ gitThunkRev s commit = do
964-- If the branch name is passed in, it is returned exactly as-is. If it is not
1141-- If the branch name is passed in, it is returned exactly as-is. If it is not
965-- passed it, the default branch of the repo is used instead.
1142-- passed it, the default branch of the repo is used instead.
966gitGetCommitBranch
1144gitGetCommitBranch
967 :: MonadObelisk m => GitUri -> Maybe Text -> m (Text, CommitId)
1145 :: MonadObelisk m => GitUri -> Maybe Text -> m (Text, CommitId)
968gitGetCommitBranch uri mbranch = withExitFailMessage ("Failure for git remote " <> uriMsg) $ do
1146gitGetCommitBranch uri mbranch = withExitFailMessage ("Failure for git remote " <> uriMsg) $ do