@@ -5,86 +5,70 @@
5{-# LANGUAGE MultiWayIf #-}
5{-# LANGUAGE MultiWayIf #-}
6{-# LANGUAGE OverloadedStrings #-}
6{-# LANGUAGE OverloadedStrings #-}
7{-# LANGUAGE PatternGuards #-}
7{-# LANGUAGE PatternGuards #-}
8+{-# LANGUAGE QuasiQuotes #-}
8{-# LANGUAGE ScopedTypeVariables #-}
9{-# LANGUAGE ScopedTypeVariables #-}
9{-# LANGUAGE TypeApplications #-}
10{-# LANGUAGE TypeApplications #-}
10−module Obelisk.Command.Thunk
11+module Obelisk.Command.Thunk where
15− , ReadThunkError (..)
19− , updateThunkToLatest
21− , createThunkWithLatest
22− , nixBuildAttrWithCache
23− , nixBuildThunkAttrWithCache
34import Control.Applicative
13import Control.Applicative
35−import Control.Exception (displayException, throwIO, try)
14+import Control.Exception (displayException, try)
36−import Control.Lens.Indexed hiding ((<.>))
15+import Control.Lens (ifor, ifor_, (.~))
37import Control.Monad
16import Control.Monad
38−import Control.Monad.Catch (handle)
17+import Control.Monad.Extra (findM)
18+import Control.Monad.Catch (MonadCatch, handle)
39import Control.Monad.Except
19import Control.Monad.Except
40import Data.Aeson ((.=))
20import Data.Aeson ((.=))
41import qualified Data.Aeson as Aeson
21import qualified Data.Aeson as Aeson
42import Data.Aeson.Encode.Pretty
22import Data.Aeson.Encode.Pretty
43import qualified Data.Aeson.Types as Aeson
23import qualified Data.Aeson.Types as Aeson
24+import Data.Bifunctor (first)
44import qualified Data.ByteString.Lazy as LBS
25import qualified Data.ByteString.Lazy as LBS
26+import Data.Containers.ListUtils (nubOrd)
45import Data.Default
27import Data.Default
46import Data.Either.Combinators (fromRight', rightToMaybe)
28import Data.Either.Combinators (fromRight', rightToMaybe)
47−import Data.Foldable
29+import Data.Foldable (toList)
30+import Data.Function ((&))
31+import Data.Functor ((<&>))
48import Data.Git.Ref (Ref)
32import Data.Git.Ref (Ref)
49import qualified Data.Git.Ref as Ref
33import qualified Data.Git.Ref as Ref
50import qualified Data.List as L
34import qualified Data.List as L
51−import Data.List.NonEmpty (NonEmpty (..))
35+import Data.List.NonEmpty (NonEmpty (..), nonEmpty)
52import qualified Data.List.NonEmpty as NonEmpty
36import qualified Data.List.NonEmpty as NonEmpty
53import Data.Map (Map)
37import Data.Map (Map)
54import qualified Data.Map as Map
38import qualified Data.Map as Map
55import Data.Maybe
39import Data.Maybe
56import Data.Set (Set)
40import Data.Set (Set)
57import qualified Data.Set as Set
41import qualified Data.Set as Set
42+import Data.String.Here.Interpolated (i)
43+import Data.String.Here.Uninterpolated (here)
58import Data.Text (Text)
44import Data.Text (Text)
59import qualified Data.Text as T
45import qualified Data.Text as T
60import Data.Text.Encoding
46import Data.Text.Encoding
61import qualified Data.Text.IO as T
47import qualified Data.Text.IO as T
62import Data.Time.Clock.POSIX (posixSecondsToUTCTime)
48import Data.Time.Clock.POSIX (posixSecondsToUTCTime)
63−import Data.Yaml (parseMaybe, (.:))
49+import Data.Traversable (for)
64−import qualified Data.Yaml as Yaml
50+import Data.Yaml (parseMaybe)
65import GitHub
51import GitHub
66import GitHub.Data.Name
52import GitHub.Data.Name
67import Obelisk.Command.Nix
53import Obelisk.Command.Nix
68import System.Directory
54import System.Directory
69import System.FilePath
56import System.FilePath
70import System.IO.Error
57import System.IO.Error
71import System.IO.Temp
58import System.IO.Temp
72−import System.Posix (getSymbolicLinkStatus, modificationTime)
59+import System.PosixCompat.Files (getSymbolicLinkStatus, modificationTime)
73−import System.Process (proc)
60+import qualified Text.URI as URI
76import Obelisk.App (MonadObelisk)
62import Obelisk.App (MonadObelisk)
77import Obelisk.CliApp
63import Obelisk.CliApp
78−import Obelisk.Command.Nix (withNixRemoteCheck)
79import Obelisk.Command.Utils
64import Obelisk.Command.Utils
81--TODO: Support symlinked thunk data
66--TODO: Support symlinked thunk data
82data ThunkData
67data ThunkData
83− = ThunkData_Packed ThunkPtr
68+ = ThunkData_Packed ThunkSpec ThunkPtr
84 -- ^ Packed thunk
69 -- ^ Packed thunk
85− | ThunkData_Checkout (Maybe ThunkPtr)
70+ | ThunkData_Checkout
86 -- ^ Checked out thunk that was unpacked from this pointer
71 -- ^ Checked out thunk that was unpacked from this pointer
87− deriving (Show, Eq, Ord)
89-- | A reference to the exact data that a thunk should translate into
73-- | A reference to the exact data that a thunk should translate into
90data ThunkPtr = ThunkPtr
74data ThunkPtr = ThunkPtr
@@ -97,7 +81,7 @@ type NixSha256 = Text --TODO: Use a smart constructor and make this actually ver
98-- | A specific revision of data; it may be available from multiple sources
82-- | A specific revision of data; it may be available from multiple sources
99data ThunkRev = ThunkRev
83data ThunkRev = ThunkRev
100− { _thunkRev_commit :: Ref
84+ { _thunkRev_commit :: Ref Ref.SHA1
101 , _thunkRev_nixSha256 :: NixSha256
85 , _thunkRev_nixSha256 :: NixSha256
103 deriving (Show, Eq, Ord)
87 deriving (Show, Eq, Ord)
@@ -110,40 +94,72 @@ data ThunkSource
110 | ThunkSource_Git GitSource
94 | ThunkSource_Git GitSource
111 deriving (Show, Eq, Ord)
95 deriving (Show, Eq, Ord)
97+thunkSourceToGitSource :: ThunkSource -> GitSource
98+thunkSourceToGitSource = \case
99+ ThunkSource_GitHub s -> forgetGithub False s
100+ ThunkSource_Git s -> s
113data GitHubSource = GitHubSource
102data GitHubSource = GitHubSource
114 { _gitHubSource_owner :: Name Owner
103 { _gitHubSource_owner :: Name Owner
115 , _gitHubSource_repo :: Name Repo
104 , _gitHubSource_repo :: Name Repo
116 , _gitHubSource_branch :: Maybe (Name Branch)
105 , _gitHubSource_branch :: Maybe (Name Branch)
106+ , _gitHubSource_private :: Bool
118 deriving (Show, Eq, Ord)
108 deriving (Show, Eq, Ord)
110+newtype GitUri = GitUri { unGitUri :: URI.URI } deriving (Eq, Ord, Show)
112+gitUriToText :: GitUri -> Text
113+gitUriToText (GitUri uri)
114+ | (T.toLower . URI.unRText <$> URI.uriScheme uri) == Just "file"
115+ , Just (_, path) <- URI.uriPath uri
116+ = "/" <> T.intercalate "/" (map URI.unRText $ NonEmpty.toList path)
117+ | otherwise = URI.render uri
120data GitSource = GitSource
119data GitSource = GitSource
121− { _gitSource_url :: URI
120+ { _gitSource_url :: GitUri
122 , _gitSource_branch :: Maybe (Name Branch)
121 , _gitSource_branch :: Maybe (Name Branch)
123 , _gitSource_fetchSubmodules :: Bool
122 , _gitSource_fetchSubmodules :: Bool
123+ , _gitSource_private :: Bool
125 deriving (Show, Eq, Ord)
125 deriving (Show, Eq, Ord)
127+newtype ThunkConfig = ThunkConfig
128+ { _thunkConfig_private :: Maybe Bool
131+data ThunkUpdateConfig = ThunkUpdateConfig
132+ { _thunkUpdateConfig_branch :: Maybe String
133+ , _thunkUpdateConfig_config :: ThunkConfig
136+data ThunkPackConfig = ThunkPackConfig
137+ { _thunkPackConfig_force :: Bool
138+ , _thunkPackConfig_config :: ThunkConfig
127-- | Convert a GitHub source to a regular Git source. Assumes no submodules.
141-- | Convert a GitHub source to a regular Git source. Assumes no submodules.
128−forgetGithub :: GitHubSource -> GitSource
142+forgetGithub :: Bool -> GitHubSource -> GitSource
129−forgetGithub s = GitSource
143+forgetGithub useSsh s = GitSource
130− { _gitSource_url = URI
144+ { _gitSource_url = GitUri $ URI.URI
131− { uriScheme = Just $ fromRight' $ mkScheme "https"
145+ { URI.uriScheme = Just $ fromRight' $ URI.mkScheme $ if useSsh then "ssh" else "https"
132− , uriAuthority = Right $ Authority
146+ , URI.uriAuthority = Right $ URI.Authority
133− { authUserInfo = Nothing
147+ { URI.authUserInfo = URI.UserInfo (fromRight' $ URI.mkUsername "git") Nothing
134− , authHost = fromRight' $ mkHost "github.com"
149+ , URI.authHost = fromRight' $ URI.mkHost "github.com"
135− , authPort = Nothing
150+ , URI.authPort = Nothing
137− , uriPath = Just ( False
152+ , URI.uriPath = Just ( False
138− , fromRight' . mkPathPiece <$>
153+ , fromRight' . URI.mkPathPiece <$>
139 untagName (_gitHubSource_owner s)
154 untagName (_gitHubSource_owner s)
140 :| [ untagName (_gitHubSource_repo s) <> ".git" ]
155 :| [ untagName (_gitHubSource_repo s) <> ".git" ]
142− , uriQuery = []
157+ , URI.uriQuery = []
143− , uriFragment = Nothing
158+ , URI.uriFragment = Nothing
145 , _gitSource_branch = _gitHubSource_branch s
160 , _gitSource_branch = _gitHubSource_branch s
146 , _gitSource_fetchSubmodules = False
161 , _gitSource_fetchSubmodules = False
162+ , _gitSource_private = _gitHubSource_private s
149getThunkGitBranch :: ThunkPtr -> Maybe Text
165getThunkGitBranch :: ThunkPtr -> Maybe Text
@@ -151,25 +167,26 @@ getThunkGitBranch (ThunkPtr _ src) = fmap untagName $ case src of
151 ThunkSource_GitHub s -> _gitHubSource_branch s
167 ThunkSource_GitHub s -> _gitHubSource_branch s
152 ThunkSource_Git s -> _gitSource_branch s
168 ThunkSource_Git s -> _gitSource_branch s
154−commitNameToRef :: Name Commit -> Ref
170+commitNameToRef :: Name Commit -> Ref Ref.SHA1
155commitNameToRef (N c) = Ref.fromHex $ encodeUtf8 c
171commitNameToRef (N c) = Ref.fromHex $ encodeUtf8 c
157-- TODO: Use spinner here.
173-- TODO: Use spinner here.
158getNixSha256ForUriUnpacked
174getNixSha256ForUriUnpacked
159 :: MonadObelisk m
175 :: MonadObelisk m
160− => URI
176+ => GitUri
161 -> m NixSha256
177 -> m NixSha256
162getNixSha256ForUriUnpacked uri =
178getNixSha256ForUriUnpacked uri =
163− withExitFailMessage ("nix-prefetch-url: Failed to determine sha256 hash of URL " <> T.pack (show uri)) $ do
179+ withExitFailMessage ("nix-prefetch-url: Failed to determine sha256 hash of URL " <> gitUriToText uri) $ do
164− withNixRemoteCheck $ readProcessAndLogStderr Debug $
180+ [hash] <- fmap T.lines $ readProcessAndLogOutput (Debug, Debug) $
165− proc "nix-prefetch-url" ["--unpack" , "--type" , "sha256" , show uri]
181+ proc nixPrefetchUrlPath ["--unpack", "--type", "sha256", T.unpack $ gitUriToText uri]
167−nixPrefetchGit :: MonadObelisk m => URI -> Text -> Bool -> m NixSha256
184+nixPrefetchGit :: MonadObelisk m => GitUri -> Text -> Bool -> m NixSha256
168nixPrefetchGit uri rev fetchSubmodules =
185nixPrefetchGit uri rev fetchSubmodules =
169− withExitFailMessage ("nix-prefetch-git: Failed to determine sha256 hash of Git repo " <> T.pack (show uri) <> " at " <> rev) $ do
186+ withExitFailMessage ("nix-prefetch-git: Failed to determine sha256 hash of Git repo " <> gitUriToText uri <> " at " <> rev) $ do
170− out <- withNixRemoteCheck $ readProcessAndLogStderr Debug $
187+ out <- readProcessAndLogStderr Debug $
171− proc "nix-prefetch-git" $ filter (/="")
188+ proc nixPrefetchGitPath $ filter (/="")
172− [ "--url", show uri
189+ [ "--url", T.unpack $ gitUriToText uri
173 , "--rev", T.unpack rev
190 , "--rev", T.unpack rev
174 , if fetchSubmodules then "--fetch-submodules" else ""
191 , if fetchSubmodules then "--fetch-submodules" else ""
175 , "--quiet"
192 , "--quiet"
@@ -181,93 +198,117 @@ nixPrefetchGit uri rev fetchSubmodules =
182--TODO: Pretty print these
199--TODO: Pretty print these
183data ReadThunkError
200data ReadThunkError
184− = ReadThunkError_UnrecognizedFiles
201+ = ReadThunkError_UnrecognizedThunk
185− | ReadThunkError_AmbiguousFiles
202+ | ReadThunkError_UnrecognizedPaths (NonEmpty FilePath)
186− | ReadThunkError_UnrecognizedLoader Text
203+ | ReadThunkError_MissingPaths (NonEmpty FilePath)
187− | ReadThunkError_UnparseablePtr LBS.ByteString
204+ | ReadThunkError_UnparseablePtr FilePath String
205+ | ReadThunkError_FileError IOError
206+ | ReadThunkError_FileDoesNotMatch FilePath Text
207+ | ReadThunkError_UnrecognizedState String
208+ | ReadThunkError_AmbiguousPackedState ThunkPtr ThunkPtr
188− deriving (Show)
209+ deriving (Show)
190−-- | Read a thunk and validate that it is only a thunk, either packed or unpacked.
211+unpackedDirName :: FilePath
191−-- If the thunk is packed and additional data is present, fail.
212+unpackedDirName = "."
192−readThunk :: MonadObelisk m => FilePath -> m (Either ReadThunkError ThunkData)
213+
193−readThunk thunkDir = do
214+attrCacheFileName :: FilePath
194− files <- liftIO $ listDirectory thunkDir
215+attrCacheFileName = ".attr-cache"
195− case ".git" `elem` files of
216+
196− True -> fmap ThunkData_Checkout <$> do
217+-- | Specification for how a file in a thunk version works.
197− let origThunkPath = thunkDir </> ".git" </> "obelisk" </> "orig-thunk"
218+data ThunkFileSpec
198− liftIO $ doesDirectoryExist origThunkPath >>= \case
219+ = ThunkFileSpec_Ptr (LBS.ByteString -> Either String ThunkPtr) -- ^ This file specifies 'ThunkPtr' data
199− True -> fmap Just <$> readPackedThunk origThunkPath
220+ | ThunkFileSpec_FileMatches Text -- ^ This file must match the given content exactly
200− False -> return $ return Nothing
221+ | ThunkFileSpec_CheckoutIndicator -- ^ Existence of this directory indicates that the thunk is unpacked
201− False -> liftIO $ fmap ThunkData_Packed <$> readPackedThunk thunkDir
222+ | ThunkFileSpec_AttrCache -- ^ This directory is an attribute cache
203−data ThunkType = ThunkType
224+-- | Specification for how a set of files in a thunk version work.
204− { _thunkType_loader :: FilePath
225+data ThunkSpec = ThunkSpec
205− , _thunkType_json :: FilePath
226+ { _thunkSpec_name :: !Text
206− , _thunkType_optional :: Set FilePath
227+ , _thunkSpec_files :: !(Map FilePath ThunkFileSpec)
207− , _thunkType_loaderVersions :: NonEmpty Text
208− , _thunkType_parser :: Aeson.Object -> Aeson.Parser ThunkPtr
211−gitHubThunkType :: ThunkType
212−gitHubThunkType = ThunkType
213− { _thunkType_loader = "default.nix"
214− , _thunkType_json = "github.json"
215− , _thunkType_optional = Set.fromList [".attr-cache"]
216− , _thunkType_loaderVersions = gitHubStandaloneLoaders
217− , _thunkType_parser = parseThunkPtr $ \v ->
218− ThunkSource_GitHub <$> parseGitHubSource v <|> ThunkSource_Git <$> parseGitSource v
221−gitThunkType :: ThunkType
222−gitThunkType = ThunkType
223− { _thunkType_loader = "default.nix"
224− , _thunkType_json = "git.json"
225− , _thunkType_optional = Set.fromList [".attr-cache"]
226− , _thunkType_loaderVersions = plainGitStandaloneLoaders
227− , _thunkType_parser = parseThunkPtr $ fmap ThunkSource_Git . parseGitSource
230−thunkTypes :: [ThunkType]
230+thunkSpecTypes :: NonEmpty (NonEmpty ThunkSpec)
231−thunkTypes = [gitThunkType, gitHubThunkType]
231+thunkSpecTypes = gitThunkSpecs :| [gitHubThunkSpecs]
233−findThunkType :: [ThunkType] -> FilePath -> IO (Either ReadThunkError ThunkType)
233+-- | Attempts to match a 'ThunkSpec' to a given directory.
234−findThunkType types thunkDir = do
234+matchThunkSpecToDir
235− matches <- fmap catMaybes $ forM types $ \thunkType -> do
235+ :: (MonadError ReadThunkError m, MonadIO m, MonadCatch m)
236− let
236+ => ThunkSpec -- ^ 'ThunkSpec' to match against the given files/directory
237− expectedContents = Set.fromList $ (thunkDir </>) <$>
237+ -> FilePath -- ^ Path to directory
238− [ _thunkType_loader thunkType
238+ -> Set FilePath -- ^ Set of file paths relative to the given directory
239− , _thunkType_json thunkType
239+ -> m ThunkData
240− ]
240+matchThunkSpecToDir thunkSpec dir dirFiles = do
241− optionalContents = Set.map (thunkDir </>) (_thunkType_optional thunkType)
241+ case nonEmpty (toList $ dirFiles `Set.difference` expectedPaths) of
242−
242+ Just fs -> throwError $ ReadThunkError_UnrecognizedPaths $ (dir </>) <$> fs
243− -- Ensure that there aren't any other files in the thunk
243+ Nothing -> pure ()
244− -- NB: System.Directory.listDirectory returns the contents without the directory path
244+ case nonEmpty (toList $ requiredPaths `Set.difference` dirFiles) of
245− files <- liftIO $ Set.fromList . fmap (thunkDir </>) <$> listDirectory thunkDir
245+ Just fs -> throwError $ ReadThunkError_MissingPaths $ (dir </>) <$> fs
246− let unexpectedContents = files `Set.difference` (expectedContents `Set.union` optionalContents)
246+ Nothing -> pure ()
247− missingContents = expectedContents `Set.difference` files
247+ datas <- fmap toList $ flip Map.traverseMaybeWithKey (_thunkSpec_files thunkSpec) $ \expectedPath -> \case
248− pure $ if Set.null unexpectedContents && Set.null missingContents
248+ ThunkFileSpec_AttrCache -> Nothing <$ dirMayExist expectedPath
249− then Just thunkType
249+ ThunkFileSpec_CheckoutIndicator -> liftIO (doesDirectoryExist (dir </> expectedPath)) <&> \case
250− else Nothing
250+ False -> Nothing
251−
251+ True -> Just ThunkData_Checkout
252− pure $ case matches of
252+ ThunkFileSpec_FileMatches expectedContents -> handle (\(e :: IOError) -> throwError $ ReadThunkError_FileError e) $ do
253− [singleMatch] -> Right singleMatch
253+ actualContents <- liftIO (T.readFile $ dir </> expectedPath)
254− [] -> Left ReadThunkError_UnrecognizedFiles
254+ case T.strip expectedContents == T.strip actualContents of
255− _ -> Left ReadThunkError_AmbiguousFiles
255+ True -> pure Nothing
256+ False -> throwError $ ReadThunkError_FileDoesNotMatch (dir </> expectedPath) expectedContents
257+ ThunkFileSpec_Ptr parser -> handle (\(e :: IOError) -> throwError $ ReadThunkError_FileError e) $ do
258+ let path = dir </> expectedPath
259+ liftIO (doesFileExist path) >>= \case
260+ False -> pure Nothing
262+ actualContents <- liftIO $ LBS.readFile path
263+ case parser actualContents of
264+ Right v -> pure $ Just (ThunkData_Packed thunkSpec v)
265+ Left e -> throwError $ ReadThunkError_UnparseablePtr (dir </> expectedPath) e
267+ case nonEmpty datas of
268+ Nothing -> throwError ReadThunkError_UnrecognizedThunk
269+ Just xs -> fold1WithM xs $ \a b -> either throwError pure (mergeThunkData a b)
271+ rootPathsOnly = Set.fromList . mapMaybe takeRootDir . Map.keys
272+ takeRootDir = fmap NonEmpty.head . nonEmpty . splitPath
274+ expectedPaths = rootPathsOnly $ _thunkSpec_files thunkSpec
276+ requiredPaths = rootPathsOnly $ Map.filter isRequiredFileSpec $ _thunkSpec_files thunkSpec
277+ isRequiredFileSpec = \case
278+ ThunkFileSpec_FileMatches _ -> True
281+ dirMayExist expectedPath = liftIO (doesFileExist (dir </> expectedPath)) >>= \case
282+ True -> throwError $ ReadThunkError_UnrecognizedPaths $ expectedPath :| []
285+ -- Combine 'ThunkData' from different files, preferring "Checkout" over "Packed"
286+ mergeThunkData ThunkData_Checkout ThunkData_Checkout = Right ThunkData_Checkout
287+ mergeThunkData ThunkData_Checkout ThunkData_Packed{} = Left bothPackedAndUnpacked
288+ mergeThunkData ThunkData_Packed{} ThunkData_Checkout = Left bothPackedAndUnpacked
289+ mergeThunkData a@(ThunkData_Packed _ ptrA) (ThunkData_Packed _ ptrB) =
290+ if ptrA == ptrB then Right a else Left $ ReadThunkError_AmbiguousPackedState ptrA ptrB
292+ bothPackedAndUnpacked = ReadThunkError_UnrecognizedState "Both packed data and checkout present"
294+ fold1WithM (x :| xs) f = foldM f x xs
298+ => NonEmpty (NonEmpty ThunkSpec) -> FilePath -> m (Either ReadThunkError ThunkData)
299+readThunkWith specTypes dir = do
300+ dirFiles <- Set.fromList <$> liftIO (listDirectory dir)
301+ let specs = concatMap toList $ toList $ NonEmpty.transpose specTypes -- Interleave spec types so we try each one in a "fair" ordering
302+ flip fix specs $ \loop -> \case
303+ [] -> pure $ Left ReadThunkError_UnrecognizedThunk
304+ spec:rest -> runExceptT (matchThunkSpecToDir spec dir dirFiles) >>= \case
305+ Left e -> putLog Debug [i|Thunk specification ${_thunkSpec_name spec} did not match ${dir}: ${e}|] *> loop rest
306+ x@(Right _) -> x <$ putLog Debug [i|Thunk specification ${_thunkSpec_name spec} matched ${dir}|]
257-- | Read a thunk and validate that it is exactly a packed thunk.
308-- | Read a thunk and validate that it is exactly a packed thunk.
258-- If additional data is present, fail.
309-- If additional data is present, fail.
259−readPackedThunk :: FilePath -> IO (Either ReadThunkError ThunkPtr)
310+readThunk :: (MonadObelisk m) => FilePath -> m (Either ReadThunkError ThunkData)
260−readPackedThunk thunkDir = runExceptT $ do
311+readThunk = readThunkWith thunkSpecTypes
261− thunkType <- ExceptT $ findThunkType thunkTypes thunkDir
262− -- Ensure that we recognize the thunk loader
263− loader <- liftIO $ T.readFile $ thunkDir </> _thunkType_loader thunkType
264− unless (loader `elem` _thunkType_loaderVersions thunkType) $ do
265− throwError $ ReadThunkError_UnrecognizedLoader loader
267− txt <- liftIO $ LBS.readFile $ thunkDir </> _thunkType_json thunkType
268− case parseMaybe (_thunkType_parser thunkType) =<< Aeson.decode txt of
269− Nothing -> throwError $ ReadThunkError_UnparseablePtr txt
270− Just ptr -> return ptr
272parseThunkPtr :: (Aeson.Object -> Aeson.Parser ThunkSource) -> Aeson.Object -> Aeson.Parser ThunkPtr
313parseThunkPtr :: (Aeson.Object -> Aeson.Parser ThunkSource) -> Aeson.Object -> Aeson.Parser ThunkPtr
273parseThunkPtr parseSrc v = do
314parseThunkPtr parseSrc v = do
@@ -287,10 +328,12 @@ parseGitHubSource v = do
287 owner <- v Aeson..: "owner"
328 owner <- v Aeson..: "owner"
288 repo <- v Aeson..: "repo"
329 repo <- v Aeson..: "repo"
289 branch <- v Aeson..:! "branch"
330 branch <- v Aeson..:! "branch"
331+ private <- v Aeson..:? "private"
290 pure $ GitHubSource
332 pure $ GitHubSource
291 { _gitHubSource_owner = owner
333 { _gitHubSource_owner = owner
292 , _gitHubSource_repo = repo
334 , _gitHubSource_repo = repo
293 , _gitHubSource_branch = branch
335 , _gitHubSource_branch = branch
336+ , _gitHubSource_private = fromMaybe False private
296parseGitSource :: Aeson.Object -> Aeson.Parser GitSource
339parseGitSource :: Aeson.Object -> Aeson.Parser GitSource
@@ -298,25 +341,29 @@ parseGitSource v = do
298 Just url <- parseGitUri <$> v Aeson..: "url"
341 Just url <- parseGitUri <$> v Aeson..: "url"
299 branch <- v Aeson..:! "branch"
342 branch <- v Aeson..:! "branch"
300 fetchSubmodules <- v Aeson..:! "fetchSubmodules"
343 fetchSubmodules <- v Aeson..:! "fetchSubmodules"
344+ private <- v Aeson..:? "private"
301 pure $ GitSource
345 pure $ GitSource
302 { _gitSource_url = url
346 { _gitSource_url = url
303 , _gitSource_branch = branch
347 , _gitSource_branch = branch
304 , _gitSource_fetchSubmodules = fromMaybe False fetchSubmodules
348 , _gitSource_fetchSubmodules = fromMaybe False fetchSubmodules
349+ , _gitSource_private = fromMaybe False private
307overwriteThunk :: MonadObelisk m => FilePath -> ThunkPtr -> m ()
352overwriteThunk :: MonadObelisk m => FilePath -> ThunkPtr -> m ()
308overwriteThunk target thunk = do
353overwriteThunk target thunk = do
309 -- Ensure that this directory is a valid thunk (i.e. so we aren't losing any data)
354 -- Ensure that this directory is a valid thunk (i.e. so we aren't losing any data)
310− Right _ <- readThunk target
355+ readThunk target >>= \case
356+ Left e -> failWith [i|Invalid thunk at ${target}: ${e}|]
312 --TODO: Is there a safer way to do this overwriting?
359 --TODO: Is there a safer way to do this overwriting?
313− liftIO $ removeDirectoryRecursive target
360+ liftIO $ removePathForcibly target
314− liftIO $ createThunk target thunk
361+ createThunk target $ Right thunk
316−thunkPtrLoader :: ThunkPtr -> Text
363+thunkPtrToSpec :: ThunkPtr -> ThunkSpec
317−thunkPtrLoader thunk = case _thunkPtr_source thunk of
364+thunkPtrToSpec thunk = case _thunkPtr_source thunk of
318− ThunkSource_GitHub _ -> NonEmpty.head gitHubStandaloneLoaders
365+ ThunkSource_GitHub _ -> NonEmpty.head gitHubThunkSpecs
319− ThunkSource_Git _ -> NonEmpty.head plainGitStandaloneLoaders
366+ ThunkSource_Git _ -> NonEmpty.head gitThunkSpecs
321-- It's important that formatting be very consistent here, because
368-- It's important that formatting be very consistent here, because
322-- otherwise when people update thunks, their patches will be messy
369-- otherwise when people update thunks, their patches will be messy
@@ -328,14 +375,15 @@ encodeThunkPtrData (ThunkPtr rev src) = case src of
328 , ("branch" .=) <$> _gitHubSource_branch s
375 , ("branch" .=) <$> _gitHubSource_branch s
329 , Just $ "rev" .= Ref.toHexString (_thunkRev_commit rev)
376 , Just $ "rev" .= Ref.toHexString (_thunkRev_commit rev)
330 , Just $ "sha256" .= _thunkRev_nixSha256 rev
377 , Just $ "sha256" .= _thunkRev_nixSha256 rev
378+ , Just $ "private" .= _gitHubSource_private s
332 ThunkSource_Git s -> encodePretty' plainGitCfg $ Aeson.object $ catMaybes
380 ThunkSource_Git s -> encodePretty' plainGitCfg $ Aeson.object $ catMaybes
333− --TODO: Is this safe? read/show is jank for `URI`
334− [ Just $ "url" .= show (_gitSource_url s)
381+ [ Just $ "url" .= gitUriToText (_gitSource_url s)
335 , Just $ "rev" .= Ref.toHexString (_thunkRev_commit rev)
382 , Just $ "rev" .= Ref.toHexString (_thunkRev_commit rev)
336 , ("branch" .=) <$> _gitSource_branch s
383 , ("branch" .=) <$> _gitSource_branch s
337 , Just $ "sha256" .= _thunkRev_nixSha256 rev
384 , Just $ "sha256" .= _thunkRev_nixSha256 rev
338 , Just $ "fetchSubmodules" .= _gitSource_fetchSubmodules s
385 , Just $ "fetchSubmodules" .= _gitSource_fetchSubmodules s
386+ , Just $ "private" .= _gitSource_private s
341 githubCfg = defConfig
389 githubCfg = defConfig
@@ -356,59 +404,84 @@ encodeThunkPtrData (ThunkPtr rev src) = case src of
358 , "sha256"
406 , "sha256"
359 , "fetchSubmodules"
408 , "fetchSubmodules"
360 ] <> compare
409 ] <> compare
361 , confTrailingNewline = True
410 , confTrailingNewline = True
364−createThunk :: MonadIO m => FilePath -> ThunkPtr -> m ()
413+createThunk :: MonadObelisk m => FilePath -> Either ThunkSpec ThunkPtr -> m ()
365−createThunk target thunk = liftIO $ do
414+createThunk target ptrInfo =
366− createDirectoryIfMissing True (target </> ".attr-cache")
415+ ifor_ (_thunkSpec_files spec) $ \path -> \case
367− T.writeFile (target </> "default.nix") (thunkPtrLoader thunk)
416+ ThunkFileSpec_FileMatches content -> withReadyPath path $ \p -> liftIO $ T.writeFile p content
368− let
417+ ThunkFileSpec_Ptr _ -> case ptrInfo of
369− jsonFileName = case _thunkPtr_source thunk of
418+ Left _ -> pure () -- We can't write the ptr without it
370− ThunkSource_GitHub _ -> "github"
371− ThunkSource_Git _ -> "git"
372− LBS.writeFile (target </> jsonFileName <.> "json") (encodeThunkPtrData thunk)
419+ Right ptr -> withReadyPath path $ \p -> liftIO $ LBS.writeFile p (encodeThunkPtrData ptr)
422+ spec = either id thunkPtrToSpec ptrInfo
423+ withReadyPath path f = do
424+ let fullPath = target </> path
425+ putLog Debug $ "Writing thunk file " <> T.pack fullPath
426+ liftIO $ createDirectoryIfMissing True $ takeDirectory fullPath
374createThunkWithLatest :: MonadObelisk m => FilePath -> ThunkSource -> m ()
429createThunkWithLatest :: MonadObelisk m => FilePath -> ThunkSource -> m ()
375createThunkWithLatest target s = do
430createThunkWithLatest target s = do
376 rev <- getLatestRev s
431 rev <- getLatestRev s
377− createThunk target $ ThunkPtr
432+ createThunk target $ Right $ ThunkPtr
378 { _thunkPtr_source = s
433 { _thunkPtr_source = s
379 , _thunkPtr_rev = rev
434 , _thunkPtr_rev = rev
382−updateThunkToLatest :: MonadObelisk m => FilePath -> m ()
437+updateThunkToLatest :: MonadObelisk m => ThunkUpdateConfig -> FilePath -> m ()
383−updateThunkToLatest target = withSpinner' ("Updating thunk " <> T.pack target <> " to latest") (pure $ const $ "Thunk " <> T.pack target <> " updated to latest") $ do
438+updateThunkToLatest (ThunkUpdateConfig mBranch thunkConfig) target = spinner $ do
439+ checkThunkDirectory target
440+ -- check to see if thunk should be updated to a specific branch or just update it's current branch
384− (overwrite, ptr) <- readThunk target >>= \case
443+ (overwrite, ptr) <- readThunk target >>= \case
385− Left err -> failWith $ T.pack $ "thunk update: " <> show err
444+ Left err -> failWith [i|Thunk update: ${err}|]
386− Right c -> case c of
445+ Right c -> case c of
387− ThunkData_Packed t -> return (target, t)
446+ ThunkData_Packed _ t -> return (target, t)
388− ThunkData_Checkout _ -> failWith "cannot update an unpacked thunk"
447+ ThunkData_Checkout -> failWith "cannot update an unpacked thunk"
389− let src = _thunkPtr_source ptr
448+ let src = _thunkPtr_source ptr
390− rev <- getLatestRev src
449+ rev <- getLatestRev src
391− overwriteThunk overwrite $ ThunkPtr
450+ overwriteThunk overwrite $ modifyThunkPtrByConfig thunkConfig $ ThunkPtr
392− { _thunkPtr_source = src
451+ { _thunkPtr_source = src
393− , _thunkPtr_rev = rev
452+ , _thunkPtr_rev = rev
454+ Just branch -> readThunk target >>= \case
455+ Left err -> failWith [i|Thunk update: ${err}|]
456+ Right c -> case c of
457+ ThunkData_Packed _ t -> setThunk thunkConfig target (thunkSourceToGitSource $ _thunkPtr_source t) branch
458+ ThunkData_Checkout -> failWith [i|Thunk located at ${target} is unpacked. Use 'ob thunk pack' on the desired directory and then try 'ob thunk update' again.|]
460+ spinner = withSpinner' ("Updating thunk " <> T.pack target <> " to latest") (pure $ const $ "Thunk " <> T.pack target <> " updated to latest")
462+setThunk :: MonadObelisk m => ThunkConfig -> FilePath -> GitSource -> String -> m ()
463+setThunk thunkConfig target gs branch = do
464+ newThunkPtr <- uriThunkPtr (_gitSource_url gs) (_thunkConfig_private thunkConfig) (Just $ T.pack branch) Nothing
465+ overwriteThunk target newThunkPtr
466+ updateThunkToLatest (ThunkUpdateConfig Nothing thunkConfig) target
396-- | All recognized github standalone loaders, ordered from newest to oldest.
468-- | All recognized github standalone loaders, ordered from newest to oldest.
397-- This tool will only ever produce the newest one when it writes a thunk.
469-- This tool will only ever produce the newest one when it writes a thunk.
398−gitHubStandaloneLoaders :: NonEmpty Text
470+gitHubThunkSpecs :: NonEmpty ThunkSpec
399−gitHubStandaloneLoaders =
471+gitHubThunkSpecs =
400− gitHubStandaloneLoaderV2 :|
472+ gitHubThunkSpecV5 :|
401− [ gitHubStandaloneLoaderV3
473+ [ gitHubThunkSpecV4
402− , gitHubStandaloneLoaderV1
474+ , gitHubThunkSpecV3
405−gitHubStandaloneLoaderV1 :: Text
479+gitHubThunkSpecV1 :: ThunkSpec
406−gitHubStandaloneLoaderV1 = T.unlines
480+gitHubThunkSpecV1 = legacyGitHubThunkSpec "github-v1"
407− [ "import ((import <nixpkgs> {}).fetchFromGitHub (builtins.fromJSON (builtins.readFile ./github.json)))"
481+ "import ((import <nixpkgs> {}).fetchFromGitHub (builtins.fromJSON (builtins.readFile ./github.json)))"
410−gitHubStandaloneLoaderV2 :: Text
483+gitHubThunkSpecV2 :: ThunkSpec
411−gitHubStandaloneLoaderV2 = T.unlines
484+gitHubThunkSpecV2 = legacyGitHubThunkSpec "github-v2" $ T.unlines
412 [ "# DO NOT HAND-EDIT THIS FILE" --TODO: Add something about how to get more info on Obelisk, etc.
485 [ "# DO NOT HAND-EDIT THIS FILE" --TODO: Add something about how to get more info on Obelisk, etc.
413 , "import ((import <nixpkgs> {}).fetchFromGitHub ("
486 , "import ((import <nixpkgs> {}).fetchFromGitHub ("
414 , " let json = builtins.fromJSON (builtins.readFile ./github.json);"
487 , " let json = builtins.fromJSON (builtins.readFile ./github.json);"
@@ -418,8 +491,8 @@ gitHubStandaloneLoaderV2 = T.unlines
421−gitHubStandaloneLoaderV3 :: Text
494+gitHubThunkSpecV3 :: ThunkSpec
422−gitHubStandaloneLoaderV3 = T.unlines
495+gitHubThunkSpecV3 = legacyGitHubThunkSpec "github-v3" $ T.unlines
423 [ "# DO NOT HAND-EDIT THIS FILE"
496 [ "# DO NOT HAND-EDIT THIS FILE"
425 , " fetch = { private ? false, ... }@args: if private && builtins.hasAttr \"fetchGit\" builtins"
498 , " fetch = { private ? false, ... }@args: if private && builtins.hasAttr \"fetchGit\" builtins"
@@ -438,22 +511,62 @@ gitHubStandaloneLoaderV3 = T.unlines
438 , "in import (fetch (builtins.fromJSON (builtins.readFile ./github.json)))"
511 , "in import (fetch (builtins.fromJSON (builtins.readFile ./github.json)))"
441−plainGitStandaloneLoaders :: NonEmpty Text
514+gitHubThunkSpecV4 :: ThunkSpec
442−plainGitStandaloneLoaders =
515+gitHubThunkSpecV4 = legacyGitHubThunkSpec "github-v4" $ T.unlines
443− plainGitStandaloneLoaderV1 :|
516+ [ "# DO NOT HAND-EDIT THIS FILE"
444− [ plainGitStandaloneLoaderV2
517+ , "let fetch = { private ? false, fetchSubmodules ? false, owner, repo, rev, sha256, ... }:"
518+ , " if !fetchSubmodules && !private then builtins.fetchTarball {"
519+ , " url = \"https://github.com/${owner}/${repo}/archive/${rev}.tar.gz\"; inherit sha256;"
520+ , " } else (import <nixpkgs> {}).fetchFromGitHub {"
521+ , " inherit owner repo rev sha256 fetchSubmodules private;"
523+ , "in import (fetch (builtins.fromJSON (builtins.readFile ./github.json)))"
447−plainGitStandaloneLoaderV1 :: Text
526+legacyGitHubThunkSpec :: Text -> Text -> ThunkSpec
448−plainGitStandaloneLoaderV1 = T.unlines
527+legacyGitHubThunkSpec name loader = ThunkSpec name $ Map.fromList
528+ [ ("default.nix", ThunkFileSpec_FileMatches $ T.strip loader)
529+ , ("github.json" , ThunkFileSpec_Ptr parseGitHubJsonBytes)
530+ , (attrCacheFileName, ThunkFileSpec_AttrCache)
531+ , (".git", ThunkFileSpec_CheckoutIndicator)
534+gitHubThunkSpecV5 :: ThunkSpec
535+gitHubThunkSpecV5 = mkThunkSpec "github-v5" "github.json" parseGitHubJsonBytes [here|
536+# DO NOT HAND-EDIT THIS FILE
537+let fetch = { private ? false, fetchSubmodules ? false, owner, repo, rev, sha256, ... }:
538+ if !fetchSubmodules && !private then builtins.fetchTarball {
539+ url = "https://github.com/${owner}/${repo}/archive/${rev}.tar.gz"; inherit sha256;
540+ } else (import <nixpkgs> {}).fetchFromGitHub {
541+ inherit owner repo rev sha256 fetchSubmodules private;
543+ json = builtins.fromJSON (builtins.readFile ./github.json);
547+parseGitHubJsonBytes :: LBS.ByteString -> Either String ThunkPtr
548+parseGitHubJsonBytes = parseJsonObject $ parseThunkPtr $ \v ->
549+ ThunkSource_GitHub <$> parseGitHubSource v <|> ThunkSource_Git <$> parseGitSource v
551+gitThunkSpecs :: NonEmpty ThunkSpec
560+gitThunkSpecV1 :: ThunkSpec
561+gitThunkSpecV1 = legacyGitThunkSpec "git-v1" $ T.unlines
449 [ "# DO NOT HAND-EDIT THIS FILE"
562 [ "# DO NOT HAND-EDIT THIS FILE"
450 , "let fetchGit = {url, rev, ref ? null, branch ? null, sha256 ? null, fetchSubmodules ? null}:"
563 , "let fetchGit = {url, rev, ref ? null, branch ? null, sha256 ? null, fetchSubmodules ? null}:"
451 , " assert !fetchSubmodules; (import <nixpkgs> {}).fetchgit { inherit url rev sha256; };"
564 , " assert !fetchSubmodules; (import <nixpkgs> {}).fetchgit { inherit url rev sha256; };"
452 , "in import (fetchGit (builtins.fromJSON (builtins.readFile ./git.json)))"
565 , "in import (fetchGit (builtins.fromJSON (builtins.readFile ./git.json)))"
455−plainGitStandaloneLoaderV2 :: Text
568+gitThunkSpecV2 :: ThunkSpec
456−plainGitStandaloneLoaderV2 = T.unlines
569+gitThunkSpecV2 = legacyGitThunkSpec "git-v2" $ T.unlines
457 [ "# DO NOT HAND-EDIT THIS FILE"
570 [ "# DO NOT HAND-EDIT THIS FILE"
458 , "let fetchGit = {url, rev, ref ? null, branch ? null, sha256 ? null, fetchSubmodules ? null}:"
571 , "let fetchGit = {url, rev, ref ? null, branch ? null, sha256 ? null, fetchSubmodules ? null}:"
459 , " if builtins.hasAttr \"fetchGit\" builtins"
572 , " if builtins.hasAttr \"fetchGit\" builtins"
@@ -462,24 +575,87 @@ plainGitStandaloneLoaderV2 = T.unlines
462 , "in import (fetchGit (builtins.fromJSON (builtins.readFile ./git.json)))"
575 , "in import (fetchGit (builtins.fromJSON (builtins.readFile ./git.json)))"
465−thunkFileNames :: [FilePath]
578+-- This loader has a bug because @builtins.fetchGit@ is not given a @ref@
466−thunkFileNames = ["default.nix", "github.json", "git.json"]
579+-- and will fail to find commits without this because it does shallow clones.
467−
580+gitThunkSpecV3 :: ThunkSpec
468−-- | Look for authentication info from the 'hub' command
581+gitThunkSpecV3 = legacyGitThunkSpec "git-v3" $ T.unlines
469−getHubAuth
582+ [ "# DO NOT HAND-EDIT THIS FILE"
470− :: Text -- ^ Domain name
583+ , "let fetch = {url, rev, ref ? null, sha256 ? null, fetchSubmodules ? false, private ? false, ...}:"
471− -> IO (Maybe Auth)
584+ , " let realUrl = let firstChar = builtins.substring 0 1 url; in"
472−getHubAuth domain = do
585+ , " if firstChar == \"/\" then /. + url"
473− hubConfig <- getXdgDirectory XdgConfig "hub"
586+ , " else if firstChar == \".\" then ./. + url"
474− Yaml.decodeFileEither hubConfig >>= \case
587+ , " else url;"
475− Left _ -> return Nothing
588+ , " in if !fetchSubmodules && private then builtins.fetchGit {"
476− Right v -> return $ flip parseMaybe v $ \v' -> do
589+ , " url = realUrl; inherit rev;"
477− Yaml.Array domainConfigs <- v' .: domain --TODO: Determine what multiple domainConfigs means
590+ , " } else (import <nixpkgs> {}).fetchgit {"
478− [Yaml.Object domainConfig] <- return $ toList domainConfigs
591+ , " url = realUrl; inherit rev sha256;"
479− token <- domainConfig .: "oauth_token"
592+ , " };"
480− return $ OAuth $ encodeUtf8 token
593+ , "in import (fetch (builtins.fromJSON (builtins.readFile ./git.json)))"
482−--TODO: when checking something out, make a shallow clone
595+
596+gitThunkSpecV4 :: ThunkSpec
597+gitThunkSpecV4 = legacyGitThunkSpec "git-v4" $ T.unlines
598+ [ "# DO NOT HAND-EDIT THIS FILE"
599+ , "let fetch = {url, rev, branch ? null, sha256 ? null, fetchSubmodules ? false, private ? false, ...}:"
600+ , " let realUrl = let firstChar = builtins.substring 0 1 url; in"
601+ , " if firstChar == \"/\" then /. + url"
602+ , " else if firstChar == \".\" then ./. + url"
604+ , " in if !fetchSubmodules && private then builtins.fetchGit {"
605+ , " url = realUrl; inherit rev;"
606+ , " ${if branch == null then null else \"ref\"} = branch;"
607+ , " } else (import <nixpkgs> {}).fetchgit {"
608+ , " url = realUrl; inherit rev sha256;"
610+ , "in import (fetch (builtins.fromJSON (builtins.readFile ./git.json)))"
613+legacyGitThunkSpec :: Text -> Text -> ThunkSpec
614+legacyGitThunkSpec name loader = ThunkSpec name $ Map.fromList
615+ [ ("default.nix", ThunkFileSpec_FileMatches $ T.strip loader)
616+ , ("git.json" , ThunkFileSpec_Ptr parseGitJsonBytes)
617+ , (attrCacheFileName, ThunkFileSpec_AttrCache)
618+ , (".git", ThunkFileSpec_CheckoutIndicator)
621+gitThunkSpecV5 :: ThunkSpec
622+gitThunkSpecV5 = mkThunkSpec "git-v5" "git.json" parseGitJsonBytes [here|
623+# DO NOT HAND-EDIT THIS FILE
624+let fetch = {url, rev, branch ? null, sha256 ? null, fetchSubmodules ? false, private ? false, ...}:
625+ let realUrl = let firstChar = builtins.substring 0 1 url; in
626+ if firstChar == "/" then /. + url
627+ else if firstChar == "." then ./. + url
629+ in if !fetchSubmodules && private then builtins.fetchGit {
630+ url = realUrl; inherit rev;
631+ ${if branch == null then null else "ref"} = branch;
632+ } else (import <nixpkgs> {}).fetchgit {
633+ url = realUrl; inherit rev sha256;
635+ json = builtins.fromJSON (builtins.readFile ./git.json);
639+parseGitJsonBytes :: LBS.ByteString -> Either String ThunkPtr
640+parseGitJsonBytes = parseJsonObject $ parseThunkPtr $ fmap ThunkSource_Git . parseGitSource
642+mkThunkSpec :: Text -> FilePath -> (LBS.ByteString -> Either String ThunkPtr) -> Text -> ThunkSpec
643+mkThunkSpec name jsonFileName parser srcNix = ThunkSpec name $ Map.fromList
644+ [ ("default.nix", ThunkFileSpec_FileMatches defaultNixViaSrc)
645+ , ("thunk.nix", ThunkFileSpec_FileMatches srcNix)
646+ , (jsonFileName, ThunkFileSpec_Ptr parser)
647+ , (attrCacheFileName, ThunkFileSpec_AttrCache)
648+ , (normalise $ unpackedDirName </> ".git", ThunkFileSpec_CheckoutIndicator)
651+ defaultNixViaSrc = [here|
652+# DO NOT HAND-EDIT THIS FILE
653+import (import ./thunk.nix)
657+parseJsonObject :: (Aeson.Object -> Aeson.Parser a) -> LBS.ByteString -> Either String a
658+parseJsonObject p bytes = Aeson.parseEither p =<< Aeson.eitherDecode bytes
484-- | Checks a cache directory to see if there is a fresh symlink
660-- | Checks a cache directory to see if there is a fresh symlink
485-- to the result of building an attribute of a thunk.
661-- to the result of building an attribute of a thunk.
@@ -487,44 +663,49 @@ getHubAuth domain = do
487-- and the result is symlinked into the cache.
663-- and the result is symlinked into the cache.
488nixBuildThunkAttrWithCache
664nixBuildThunkAttrWithCache
489 :: MonadObelisk m
665 :: MonadObelisk m
490− => FilePath
667+ -> FilePath
491 -- ^ Path to directory containing Thunk
668 -- ^ Path to directory containing Thunk
492 -> String
669 -> String
493 -- ^ Attribute to build
670 -- ^ Attribute to build
494− -> m FilePath
671+ -> m (Maybe FilePath)
495 -- ^ Symlink to cached or built nix output
672 -- ^ Symlink to cached or built nix output
496-- WARNING: If the thunk uses an impure reference such as '<nixpkgs>'
673-- WARNING: If the thunk uses an impure reference such as '<nixpkgs>'
497-- the caching mechanism will fail as it merely measures the modification
674-- the caching mechanism will fail as it merely measures the modification
498-- time of the cache link and the expression to build.
675-- time of the cache link and the expression to build.
499−nixBuildThunkAttrWithCache thunkDir attr = do
676+nixBuildThunkAttrWithCache thunkSpec thunkDir attr = do
500− --NB: Expects thunkDir to be normalised with no trailing path separator.
501− --This should be guaranteed by the command argument parser.
502− let cacheErrHandler e
503− | isDoesNotExistError e = return Nothing -- expected from a cache miss
504− | otherwise = putLog Error (T.pack $ displayException e) >> return Nothing
505− cacheDir = thunkDir </> ".attr-cache"
506− cachePath = cacheDir </> attr <.> "out"
507 latestChange <- liftIO $ do
677 latestChange <- liftIO $ do
508− createDirectoryIfMissing False cacheDir
678+ let
509− let getModificationTimeMaybe = fmap rightToMaybe . try @IOError . getModificationTime
679+ getModificationTimeMaybe = fmap rightToMaybe . try @IOError . getModificationTime
680+ thunkFileNames = Map.keys $ _thunkSpec_files thunkSpec
510− maximum . catMaybes <$> mapM (getModificationTimeMaybe . (thunkDir </>)) thunkFileNames
681+ maximum . catMaybes <$> traverse (getModificationTimeMaybe . (thunkDir </>)) thunkFileNames
683+ let cachePaths' = nonEmpty $ Map.keys $ Map.filter (\case ThunkFileSpec_AttrCache -> True; _ -> False) $
684+ _thunkSpec_files thunkSpec
685+ for cachePaths' $ \cachePaths ->
686+ fmap NonEmpty.head $ for cachePaths $ \cacheDir -> do
688+ cachePath = thunkDir </> cacheDir </> attr <.> "out"
690+ | isDoesNotExistError e = pure Nothing -- expected from a cache miss
691+ | otherwise = Nothing <$ putLog Error (T.pack $ displayException e)
511− cacheHit <- handle cacheErrHandler $ do
692+ cacheHit <- handle cacheErrHandler $ do
512− cacheTime <- liftIO $ posixSecondsToUTCTime . realToFrac . modificationTime <$> getSymbolicLinkStatus cachePath
693+ cacheTime <- liftIO $ posixSecondsToUTCTime . realToFrac . modificationTime <$> getSymbolicLinkStatus cachePath
513− return $ if latestChange <= cacheTime
694+ pure $ if latestChange <= cacheTime
514− then Just cachePath
695+ then Just cachePath
515− else Nothing
696+ else Nothing
516− case cacheHit of
697+ case cacheHit of
517− Just c -> return c
698+ Just c -> pure c
518− Nothing -> do
699+ Nothing -> do
519− putLog Warning $ T.pack $ mconcat [thunkDir, ": ", attr, " not cached, building ..."]
700+ putLog Warning $ T.pack $ mconcat [thunkDir, ": ", attr, " not cached, building ..."]
520− _ <- nixBuild $ def
701+ liftIO $ createDirectoryIfMissing True (takeDirectory cachePath)
521− { _nixBuildConfig_target = Target
702+ (cachePath <$) $ nixCmd $ NixCmd_Build $ def
703+ & nixBuildConfig_outLink .~ OutLink_IndirectRoot cachePath
704+ & nixCmdConfig_target .~ Target
522− { _target_path = thunkDir
705+ { _target_path = Just thunkDir
523− , _target_attr = Just attr
706+ , _target_attr = Just attr
524− }
707+ , _target_expr = Nothing
525− , _nixBuildConfig_outLink = OutLink_IndirectRoot cachePath
529-- | Build a nix attribute, and cache the result if possible
710-- | Build a nix attribute, and cache the result if possible
530nixBuildAttrWithCache
711nixBuildAttrWithCache
@@ -535,17 +716,19 @@ nixBuildAttrWithCache
535 -- ^ Attribute to build
716 -- ^ Attribute to build
536 -> m FilePath
717 -> m FilePath
537 -- ^ Symlink to cached or built nix output
718 -- ^ Symlink to cached or built nix output
538−nixBuildAttrWithCache exprPath attr = do
719+nixBuildAttrWithCache exprPath attr = readThunk exprPath >>= \case
539− readThunk exprPath >>= \case
540− -- Only packed thunks are cached. in particular, checkouts are not
720+ -- Only packed thunks are cached. In particular, checkouts are not.
721+ Right (ThunkData_Packed spec _) ->
541− Right (ThunkData_Packed _) -> nixBuildThunkAttrWithCache exprPath attr
722+ maybe build pure =<< nixBuildThunkAttrWithCache spec exprPath attr
542− _ -> nixBuild $ def
723+ _ -> build
543− { _nixBuildConfig_target = Target
724+ where
725+ build = nixCmd $ NixCmd_Build $ def
726+ & nixBuildConfig_outLink .~ OutLink_None
727+ & nixCmdConfig_target .~ Target
544− { _target_path = exprPath
728+ { _target_path = Just exprPath
545 , _target_attr = Just attr
729 , _target_attr = Just attr
730+ , _target_expr = Nothing
547− , _nixBuildConfig_outLink = OutLink_None
550-- | Safely update thunk using a custom action
733-- | Safely update thunk using a custom action
@@ -562,147 +745,184 @@ updateThunk p f = withSystemTempDirectory "obelisk-thunkptr-" $ \tmpDir -> do
563 copyThunkToTmp tmpDir thunkDir = readThunk thunkDir >>= \case
746 copyThunkToTmp tmpDir thunkDir = readThunk thunkDir >>= \case
564 Left err -> failWith $ "withThunkUnpacked: " <> T.pack (show err)
747 Left err -> failWith $ "withThunkUnpacked: " <> T.pack (show err)
565− Right (ThunkData_Packed _) -> do
748+ Right ThunkData_Packed{} -> do
566 let tmpThunk = tmpDir </> "thunk"
749 let tmpThunk = tmpDir </> "thunk"
567 callProcessAndLogOutput (Notice, Error) $
750 callProcessAndLogOutput (Notice, Error) $
568− proc "cp" ["-r", "-T", thunkDir, tmpThunk]
751+ proc cp ["-r", "-T", thunkDir, tmpThunk]
569 return tmpThunk
752 return tmpThunk
570− Right _ -> failWith $ "Thunk is not packed"
753+ Right _ -> failWith "Thunk is not packed"
571 updateThunkFromTmp p' = do
754 updateThunkFromTmp p' = do
572− packThunk' True p'
755+ _ <- packThunk' True (ThunkPackConfig False (ThunkConfig Nothing)) p'
573 callProcessAndLogOutput (Notice, Error) $
756 callProcessAndLogOutput (Notice, Error) $
574− proc "cp" ["-r", "-T", p', p]
757+ proc cp ["-r", "-T", p', p]
576finalMsg :: Bool -> (a -> Text) -> Maybe (a -> Text)
759finalMsg :: Bool -> (a -> Text) -> Maybe (a -> Text)
577finalMsg noTrail s = if noTrail then Nothing else Just s
760finalMsg noTrail s = if noTrail then Nothing else Just s
762+-- | Check that we are not somewhere inside the thunk directory
763+checkThunkDirectory :: MonadObelisk m => FilePath -> m ()
764+checkThunkDirectory thunkDir = do
765+ currentDir <- liftIO getCurrentDirectory
766+ thunkDir' <- liftIO $ canonicalizePath thunkDir
767+ when (thunkDir' `L.isInfixOf` currentDir) $
768+ failWith [i|Can't perform thunk operations from within the thunk directory: ${thunkDir}|]
770+ -- Don't let thunk commands work when directly given an unpacked repo
771+ when (takeFileName thunkDir == unpackedDirName) $
772+ readThunk (takeDirectory thunkDir) >>= \case
773+ Right _ -> failWith [i|Refusing to perform thunk operation on ${thunkDir} because it is a thunk's unpacked source|]
579unpackThunk :: MonadObelisk m => FilePath -> m ()
776unpackThunk :: MonadObelisk m => FilePath -> m ()
580unpackThunk = unpackThunk' False
777unpackThunk = unpackThunk' False
582unpackThunk' :: MonadObelisk m => Bool -> FilePath -> m ()
779unpackThunk' :: MonadObelisk m => Bool -> FilePath -> m ()
583−unpackThunk' noTrail thunkDir = readThunk thunkDir >>= \case
780+unpackThunk' noTrail thunkDir = checkThunkDirectory thunkDir *> readThunk thunkDir >>= \case
584− Left err -> failWith $ "thunk unpack: " <> T.pack (show err)
781+ Left err -> failWith [i|Invalid thunk at ${thunkDir}: ${err}|]
585 --TODO: Overwrite option that rechecks out thunk; force option to do so even if working directory is dirty
782 --TODO: Overwrite option that rechecks out thunk; force option to do so even if working directory is dirty
586− Right (ThunkData_Checkout _) -> failWith "thunk unpack: thunk is already unpacked"
783+ Right ThunkData_Checkout -> failWith [i|Thunk at ${thunkDir} is already unpacked|]
587− Right (ThunkData_Packed tptr) -> do
784+ Right (ThunkData_Packed _ tptr) -> do
588 let (thunkParent, thunkName) = splitFileName thunkDir
785 let (thunkParent, thunkName) = splitFileName thunkDir
589− withTempDirectory thunkParent thunkName $ \tmpRepo -> do
786+ withTempDirectory thunkParent thunkName $ \tmpThunk -> do
590− let obGitDir = tmpRepo </> ".git" </> "obelisk"
787+ let
788+ gitSrc = thunkSourceToGitSource $ _thunkPtr_source tptr
591− case _thunkPtr_source tptr of
789+ newSpec = case _thunkPtr_source tptr of
592− ThunkSource_GitHub s -> do
790+ ThunkSource_GitHub _ -> NonEmpty.head gitHubThunkSpecs
593− mauth <- liftIO $ getHubAuth "github.com"
594− repoResult <- liftIO $ executeRequestMaybe mauth $ repositoryR (_gitHubSource_owner s) (_gitHubSource_repo s)
595− githubURI <- either (liftIO . throwIO) (return . repoSshUrl) repoResult >>= \case
596− Nothing -> failWith "Cannot determine clone URI for thunk source"
597− Just c -> return $ T.unpack $ getUrl c
599− withSpinner' ("Fetching thunk " <> T.pack thunkDir)
600− (finalMsg noTrail $ const $ "Fetched thunk " <> T.pack thunkDir) $ do
601− callProcessAndLogOutput (Debug, Debug) $
602− proc "hub" ["clone", "-n", githubURI, tmpRepo]
603− --If this directory already exists then something is weird and we should fail
604− liftIO $ createDirectory obGitDir
605− callProcessAndLogOutput (Debug, Error) $
606− proc "cp" ["-r", "-T", thunkDir </> ".", obGitDir </> "orig-thunk"]
609− putLog Debug $ "Checking out " <> T.pack (show $ maybe "" untagName $ _gitHubSource_branch s)
610− callProcessAndLogOutput (Debug, Debug) $
614− , maybe [] (\n -> ["-B", T.unpack (untagName n)]) (_gitHubSource_branch s)
615− , pure $ Ref.toHexString $ _thunkRev_commit $ _thunkPtr_rev tptr
617− -- Set upstream branch
618− forM_ (_gitHubSource_branch s) $ \branch ->
619− callProcessAndLogOutput (Debug, Error) $
620− proc "hub" ["-C", tmpRepo, "branch", "-u", "origin/" <> T.unpack (untagName branch)]
621− callProcessAndLogOutput (Notice, Error) $ proc "rm" ["-r", thunkDir]
622− callProcessAndLogOutput (Notice, Error) $ proc "mv" ["-T", tmpRepo, thunkDir]
624− ThunkSource_Git s -> do
791+ ThunkSource_Git _ -> NonEmpty.head gitThunkSpecs
625− withSpinner' ("Fetching thunk " <> T.pack thunkName)
792+ withSpinner' ("Fetching thunk " <> T.pack thunkName)
626− (finalMsg noTrail $ const $ "Fetched thunk " <> T.pack thunkName) $ do
793+ (finalMsg noTrail $ const $ "Fetched thunk " <> T.pack thunkName) $ do
794+ let unpackedPath = tmpThunk </> unpackedDirName
795+ gitCloneForThunkUnpack gitSrc (_thunkRev_commit $ _thunkPtr_rev tptr) unpackedPath
797+ let normalizeMore = dropTrailingPathSeparator . normalise
798+ when (normalizeMore unpackedPath /= normalizeMore tmpThunk) $ -- Only write meta data if the checkout is not inplace
799+ createThunk tmpThunk $ Left newSpec
802+ removePathForcibly thunkDir
803+ renameDirectory tmpThunk thunkDir
805+gitCloneForThunkUnpack
807+ => GitSource -- ^ Git source to use
808+ -> Ref hash -- ^ Commit hash to reset to
809+ -> FilePath -- ^ Directory to clone into
811+gitCloneForThunkUnpack gitSrc commit dir = do
627− let git = callProcessAndLogOutput (Notice, Notice) . gitProc tmpRepo
812+ let git = callProcessAndLogOutput (Notice, Notice) . gitProc dir
628− git ["clone", if _gitSource_fetchSubmodules s then "--recursive" else "", show (_gitSource_url s)]
813+ git $ [ "clone" ]
814+ ++ ["--recursive" | _gitSource_fetchSubmodules gitSrc]
815+ ++ [ T.unpack $ gitUriToText $ _gitSource_url gitSrc ]
816+ ++ do branch <- maybeToList $ _gitSource_branch gitSrc
817+ [ "--branch", T.unpack $ untagName branch ]
629− git ["reset", "--hard", Ref.toHexString $ _thunkRev_commit $ _thunkPtr_rev tptr]
818+ git ["reset", "--hard", Ref.toHexString commit]
630− when (_gitSource_fetchSubmodules s) $
819+ when (_gitSource_fetchSubmodules gitSrc) $
631− git ["submodule", "update", "--recursive", "--init"]
820+ git ["submodule", "update", "--recursive", "--init"]
632− case _gitSource_branch s of
821+
633− Just b -> git ["branch", "-u", "origin/" <> T.unpack (untagName $ b)]
636− liftIO $ createDirectory obGitDir
637− callProcessAndLogOutput (Notice, Error) $
638− proc "cp" ["-r", "-T", thunkDir </> ".", obGitDir </> "orig-thunk"]
639− callProcessAndLogOutput (Notice, Error) $
640− proc "rm" ["-r", thunkDir]
641− callProcessAndLogOutput (Notice, Error) $
642− proc "mv" ["-T", tmpRepo, thunkDir]
644−--TODO: add force mode to pack even if changes are present
645--TODO: add a rollback mode to pack to the original thunk
822--TODO: add a rollback mode to pack to the original thunk
646−packThunk :: MonadObelisk m => FilePath -> m ()
823+packThunk :: MonadObelisk m => ThunkPackConfig -> FilePath -> m ThunkPtr
647packThunk = packThunk' False
824packThunk = packThunk' False
649−packThunk' :: MonadObelisk m => Bool -> FilePath -> m ()
826+packThunk' :: MonadObelisk m => Bool -> ThunkPackConfig -> FilePath -> m ThunkPtr
650−packThunk' noTrail thunkDir = readThunk thunkDir >>= \case
827+packThunk' noTrail (ThunkPackConfig force thunkConfig) thunkDir = checkThunkDirectory thunkDir *> readThunk thunkDir >>= \case
651− Left err -> failWith $ T.pack $ "thunk pack: " <> show err
652− Right (ThunkData_Packed _) -> failWith "pack: thunk is already packed"
828+ Right ThunkData_Packed{} -> failWith [i|Thunk at ${thunkDir} is is already packed|]
653− Right (ThunkData_Checkout _) -> do
829+ _ -> withSpinner'
654− withSpinner' ("Packing thunk " <> T.pack thunkDir)
830+ ("Packing thunk " <> T.pack thunkDir)
655− (finalMsg noTrail $ const $ "Packed thunk " <> T.pack thunkDir) $ do
831+ (finalMsg noTrail $ const $ "Packed thunk " <> T.pack thunkDir) $
656− thunkPtr <- getThunkPtr thunkDir
832+ do
657− callProcessAndLogOutput (Debug, Error) $ proc "rm" ["-rf", thunkDir]
833+ let checkClean = if force then CheckClean_NoCheck else CheckClean_FullCheck
834+ thunkPtr <- modifyThunkPtrByConfig thunkConfig <$> getThunkPtr checkClean thunkDir (_thunkConfig_private thunkConfig)
835+ liftIO $ removePathForcibly thunkDir
658− liftIO $ createThunk thunkDir thunkPtr
836+ createThunk thunkDir $ Right thunkPtr
660−getThunkPtr :: MonadObelisk m => FilePath -> m ThunkPtr
838+
661−getThunkPtr = getThunkPtr' True
839+modifyThunkPtrByConfig :: ThunkConfig -> ThunkPtr -> ThunkPtr
662−
840+modifyThunkPtrByConfig (ThunkConfig markPrivate') ptr = case markPrivate' of
842+ Just markPrivate -> ptr { _thunkPtr_source = case _thunkPtr_source ptr of
843+ ThunkSource_Git s -> ThunkSource_Git $ s { _gitSource_private = markPrivate }
844+ ThunkSource_GitHub s -> ThunkSource_GitHub $ s { _gitHubSource_private = markPrivate }
848+ = CheckClean_FullCheck
849+ -- ^ Check that the repo is clean, including .gitignored files
850+ | CheckClean_NotIgnored
851+ -- ^ Check that the repo is clean, not including .gitignored files
852+ | CheckClean_NoCheck
853+ -- ^ Don't check that the repo is clean
663−getThunkPtr' :: forall m. MonadObelisk m => Bool -> FilePath -> m ThunkPtr
855+getThunkPtr :: forall m. MonadObelisk m => CheckClean -> FilePath -> Maybe Bool -> m ThunkPtr
664−getThunkPtr' checkClean thunkDir = do
856+getThunkPtr gitCheckClean dir mPrivate = do
857+ let repoLocations = nubOrd $ map (first normalise)
858+ [(".git", "."), (unpackedDirName </> ".git", unpackedDirName)]
859+ repoLocation' <- liftIO $ flip findM repoLocations $ doesDirectoryExist . (dir </>) . fst
860+ thunkDir <- case repoLocation' of
861+ Nothing -> failWith [i|Can't find an unpacked thunk in ${dir}|]
862+ Just (_, path) -> pure $ normalise $ dir </> path
864+ let (checkClean, checkIgnored) = case gitCheckClean of
865+ CheckClean_FullCheck -> (True, True)
866+ CheckClean_NotIgnored -> (True, False)
867+ CheckClean_NoCheck -> (False, False)
665− when checkClean $ ensureCleanGitRepo thunkDir True $
868+ when checkClean $ ensureCleanGitRepo thunkDir checkIgnored
666 "thunk pack: thunk checkout contains unsaved modifications"
869 "thunk pack: thunk checkout contains unsaved modifications"
668 -- Check whether there are any stashes
871 -- Check whether there are any stashes
872+ when checkClean $ do
669− stashOutput <- readGitProcess thunkDir ["stash", "list"]
873+ stashOutput <- readGitProcess thunkDir ["stash", "list"]
670− when checkClean $ case T.null stashOutput of
874+ unless (T.null stashOutput) $
672 failWith $ T.unlines $
875 failWith $ T.unlines $
673 [ "thunk pack: thunk checkout has stashes"
876 [ "thunk pack: thunk checkout has stashes"
674 , "git stash list:"
877 , "git stash list:"
675 ] ++ T.lines stashOutput
878 ] ++ T.lines stashOutput
880+ -- Get current branch
881+ (mCurrentBranch, mCurrentCommit) <- do
882+ b <- listToMaybe . T.lines <$> readGitProcess thunkDir ["rev-parse", "--abbrev-ref", "HEAD"]
883+ c <- listToMaybe . T.lines <$> readGitProcess thunkDir ["rev-parse", "HEAD"]
885+ (Just "HEAD") -> failWith $ T.unlines
886+ [ "thunk pack: You are in 'detached HEAD' state."
887+ , "If you want to pack at the current ref \
888+ \then please create a new branch with 'git checkout -b <new-branch-name>' and push this upstream."
676− True -> return ()
890+ _ -> return (b, c)
678 -- Get information on all branches and their (optional) designated upstream
892 -- Get information on all branches and their (optional) designated upstream
679 -- correspondents
893 -- correspondents
680− (headDump :: [Text]) <- T.lines <$> readGitProcess thunkDir
894+ headDump :: [Text] <- T.lines <$> readGitProcess thunkDir
681 [ "for-each-ref"
895 [ "for-each-ref"
682− , "--format=%(refname:short) %(upstream:short)"
896+ , "--format=%(refname:short) %(upstream:short) %(upstream:remotename)"
683 , "refs/heads/"
897 , "refs/heads/"
900+ (headInfo :: Map Text (Maybe (Text, Text)))
685− (headInfo :: Map Text (Maybe Text)) <- fmap Map.fromList $ forM headDump $ \line -> do
901+ <- fmap Map.fromList $ forM headDump $ \line -> do
686− (branch : restOfLine) <- pure $ T.words line
902+ (branch : restOfLine) <- pure $ T.words line
687− mUpstream <- case restOfLine of
903+ mUpstream <- case restOfLine of
688− [] -> pure Nothing
904+ [] -> pure Nothing
689− [u] -> pure $ Just u
905+ [u, r] -> pure $ Just (u, r)
690− (_:_) -> failWith "git for-each-ref invalid output"
906+ (_:_) -> failWith "git for-each-ref invalid output"
691− pure (branch, mUpstream)
907+ pure (branch, mUpstream)
909+ putLog Debug $ "branches: " <> T.pack (show headInfo)
693 let errorMap :: Map Text ()
911 let errorMap :: Map Text ()
694− headUpstream :: Map Text Text
912+ headUpstream :: Map Text (Text, Text)
695 (errorMap, headUpstream) = flip Map.mapEither headInfo $ \case
913 (errorMap, headUpstream) = flip Map.mapEither headInfo $ \case
696 Nothing -> Left ()
914 Nothing -> Left ()
697 Just b -> Right b
915 Just b -> Right b
917+ putLog Debug $ "branches with upstream branch set: " <> T.pack (show headUpstream)
699 -- Check that every branch has a remote equivalent
919 -- Check that every branch has a remote equivalent
700 when checkClean $ do
920 when checkClean $ do
701 let untrackedBranches = Map.keys errorMap
921 let untrackedBranches = Map.keys errorMap
702 when (not $ L.null untrackedBranches) $ failWith $ T.unlines $
922 when (not $ L.null untrackedBranches) $ failWith $ T.unlines $
703 [ "thunk pack: Certain branches in the thunk have no upstream branch \
923 [ "thunk pack: Certain branches in the thunk have no upstream branch \
704− \ set. This means don't know to check whether all your work is \
924+ \set. This means we don't know to check whether all your work is \
705− \ saved. The offending branches are:"
925+ \saved. The offending branches are:"
707 , T.unwords untrackedBranches
927 , T.unwords untrackedBranches
@@ -712,12 +932,12 @@ getThunkPtr' checkClean thunkDir = do
712 ((\branch -> "git push -u origin " <> branch) <$> untrackedBranches) ++
932 ((\branch -> "git push -u origin " <> branch) <$> untrackedBranches) ++
714 , "These will push the branches to the default remote under the same \
934 , "These will push the branches to the default remote under the same \
715− \ name, and (thanks to the `-u`) remember that choice so you don't \
935+ \name, and (thanks to the `-u`) remember that choice so you don't \
716− \ get this error again."
936+ \get this error again."
719 -- loosely by https://stackoverflow.com/questions/7773939/show-git-ahead-and-behind-info-for-all-branches-including-remotes
939 -- loosely by https://stackoverflow.com/questions/7773939/show-git-ahead-and-behind-info-for-all-branches-including-remotes
720− stats <- iforM headUpstream $ \branch upstream -> do
940+ stats <- ifor headUpstream $ \branch (upstream, _remote) -> do
721 (stat :: [Text]) <- T.lines <$> readGitProcess thunkDir
941 (stat :: [Text]) <- T.lines <$> readGitProcess thunkDir
722 [ "rev-list", "--left-right"
942 [ "rev-list", "--left-right"
723 , T.unpack branch <> "..." <> T.unpack upstream
943 , T.unpack branch <> "..." <> T.unpack upstream
@@ -734,21 +954,18 @@ getThunkPtr' checkClean thunkDir = do
734 [ "thunk pack: Certain branches in the thunk have commits not yet pushed upstream:"
954 [ "thunk pack: Certain branches in the thunk have commits not yet pushed upstream:"
737− (flip map (Map.toList nonGood) $ \(branch, (upstream, (ahead, behind))) -> mconcat
957+ flip map (Map.toList nonGood) (\(branch, (upstream, (ahead, behind))) -> mconcat
738 [" ", branch, " ahead: ", T.pack (show ahead), " behind: ", T.pack (show behind), " remote branch ", upstream]) ++
958 [" ", branch, " ahead: ", T.pack (show ahead), " behind: ", T.pack (show behind), " remote branch ", upstream]) ++
740− , "Please push these upstream and try again."
960+ , "Please push these upstream and try again. (Or just fetch, if they are somehow \
961+ \pushed but this repo's remote tracking branches don't know it.)"
964+ when checkClean $ do
743− -- We assume it's safe to pack the thunk at this point
965+ -- We assume it's safe to pack the thunk at this point
744−
966+ putLog Informational "All changes safe in git remotes. OK to pack thunk."
745− -- Get current branch `git rev-parse --abbrev-ref HEAD`
746− mCurrentBranch <- listToMaybe
748− <$> readGitProcess thunkDir ["rev-parse", "--abbrev-ref", "HEAD"]
750− let remote = maybe "origin" id $
751− flip Map.lookup headUpstream =<< mCurrentBranch
968+ let remote = maybe "origin" snd $ flip Map.lookup headUpstream =<< mCurrentBranch
753 [remoteUri'] <- fmap T.lines $ readGitProcess thunkDir
970 [remoteUri'] <- fmap T.lines $ readGitProcess thunkDir
754 [ "config"
971 [ "config"
@@ -759,60 +976,111 @@ getThunkPtr' checkClean thunkDir = do
759 remoteUri <- case parseGitUri remoteUri' of
976 remoteUri <- case parseGitUri remoteUri' of
760 Nothing -> failWith $ "Could not identify git remote: " <> remoteUri'
977 Nothing -> failWith $ "Could not identify git remote: " <> remoteUri'
761 Just uri -> pure uri
978 Just uri -> pure uri
762− uriThunkPtr remoteUri mCurrentBranch
979+ uriThunkPtr remoteUri mPrivate mCurrentBranch mCurrentCommit
764-- | Get the latest revision available from the given source
981-- | Get the latest revision available from the given source
765getLatestRev :: MonadObelisk m => ThunkSource -> m ThunkRev
982getLatestRev :: MonadObelisk m => ThunkSource -> m ThunkRev
766getLatestRev os = do
983getLatestRev os = do
767− let gitS = case os of
984+ let gitS = thunkSourceToGitSource os
768− ThunkSource_GitHub s -> forgetGithub s
769− ThunkSource_Git s -> s
770 (_, commit) <- gitGetCommitBranch (_gitSource_url gitS) (untagName <$> _gitSource_branch gitS)
985 (_, commit) <- gitGetCommitBranch (_gitSource_url gitS) (untagName <$> _gitSource_branch gitS)
771 case os of
986 case os of
772 ThunkSource_GitHub s -> githubThunkRev s commit
987 ThunkSource_GitHub s -> githubThunkRev s commit
773 ThunkSource_Git s -> gitThunkRev s commit
988 ThunkSource_Git s -> gitThunkRev s commit
990+-- | Convert a URI to a thunk
992+-- If the URL is a github URL, we try to just download an archive for
993+-- performance. If that doesn't work (e.g. authentication issue), we fall back
994+-- on just doing things the normal way for git repos in general, and save it as
995+-- a regular git thunk.
775−uriThunkPtr :: MonadObelisk m => URI -> Maybe Text -> m ThunkPtr
996+uriThunkPtr :: MonadObelisk m => GitUri -> Maybe Bool -> Maybe Text -> Maybe Text -> m ThunkPtr
776−uriThunkPtr uri mbranch = do
997+uriThunkPtr uri mPrivate mbranch mcommit = do
998+ commit <- case mcommit of
777− (_, commit) <- gitGetCommitBranch uri mbranch
999+ Nothing -> gitGetCommitBranch uri mbranch >>= return . snd
1000+ (Just c) -> return c
778− let src = uriToThunkSource uri mbranch
1001+ (src, rev) <- uriToThunkSource uri mPrivate mbranch >>= \case
780− ThunkSource_GitHub s -> githubThunkRev s commit
1002+ ThunkSource_GitHub s -> do
1003+ rev <- runExceptT $ githubThunkRev s commit
1005+ Right r -> pure (ThunkSource_GitHub s, r)
1008+\Failed to fetch archive from GitHub. This is probably a private repo. \
1009+\Falling back on normal fetchgit. Original failure:"
1011+ let s' = forgetGithub True s
1012+ (,) (ThunkSource_Git s') <$> gitThunkRev s' commit
781− ThunkSource_Git s -> gitThunkRev s commit
1013+ ThunkSource_Git s -> (,) (ThunkSource_Git s) <$> gitThunkRev s commit
782 pure $ ThunkPtr
1014 pure $ ThunkPtr
783 { _thunkPtr_rev = rev
1015 { _thunkPtr_rev = rev
784 , _thunkPtr_source = src
1016 , _thunkPtr_source = src
787-- | N.B. Cannot infer all fields.
1019-- | N.B. Cannot infer all fields.
1021+-- If the thunk is a GitHub thunk and fails, we do *not* fall back like with
1022+-- `uriThunkPtr`. Unlike a plain URL, a thunk src explicitly states which method
1023+-- should be employed, and so we respect that.
788−uriToThunkSource :: URI -> Maybe Text -> ThunkSource
1024+uriToThunkSource :: MonadObelisk m => GitUri -> Maybe Bool -> Maybe Text -> m ThunkSource
789−uriToThunkSource u
1025+uriToThunkSource (GitUri u) mPrivate
790− | Right uriAuth <- uriAuthority u
1026+ | Right uriAuth <- URI.uriAuthority u
791− , Just scheme <- unRText <$> uriScheme u
1027+ , Just scheme <- URI.unRText <$> URI.uriScheme u
792 , case scheme of
1028 , case scheme of
793− "ssh" -> uriAuth == Authority
1029+ "ssh" -> uriAuth == URI.Authority
794− { authUserInfo = Just $ UserInfo (fromRight' $ mkUsername "git") Nothing
1030+ { URI.authUserInfo = Just $ URI.UserInfo (fromRight' $ URI.mkUsername "git") Nothing
795− , authHost = fromRight' $ mkHost "github.com"
1031+ , URI.authHost = fromRight' $ URI.mkHost "github.com"
796− , authPort = Nothing
1032+ , URI.authPort = Nothing
798 s -> s `L.elem` [ "git", "https", "http" ] -- "http:" just redirects to "https:"
1034 s -> s `L.elem` [ "git", "https", "http" ] -- "http:" just redirects to "https:"
799− && unRText (authHost uriAuth) == "github.com"
1035+ && URI.unRText (URI.authHost uriAuth) == "github.com"
800− , Just (_, owner :| [repoish]) <- uriPath u
1036+ , Just (_, owner :| [repoish]) <- URI.uriPath u
1038+ isPrivate <- getIsPrivate
801− = \mbranch -> ThunkSource_GitHub $ GitHubSource
1039+ pure $ ThunkSource_GitHub $ GitHubSource
802− { _gitHubSource_owner = N $ unRText owner
1040+ { _gitHubSource_owner = N $ URI.unRText owner
803− , _gitHubSource_repo = N $ let
1041+ , _gitHubSource_repo = N $ let
804− repoish' = unRText repoish
1042+ repoish' = URI.unRText repoish
805− in case T.stripSuffix ".git" repoish' of
1043+ in fromMaybe repoish' $ T.stripSuffix ".git" repoish'
808− , _gitHubSource_branch = N <$> mbranch
1044+ , _gitHubSource_branch = N <$> mbranch
1045+ , _gitHubSource_private = isPrivate
1048+ | otherwise = \mbranch -> do
1049+ isPrivate <- getIsPrivate
811− | otherwise = \mbranch -> ThunkSource_Git $ GitSource
1050+ pure $ ThunkSource_Git $ GitSource
812− { _gitSource_url = u
1051+ { _gitSource_url = GitUri u
813− , _gitSource_branch = N <$> mbranch
1052+ , _gitSource_branch = N <$> mbranch
814− , _gitSource_fetchSubmodules = False -- TODO: How do we determine if this should be true?
1053+ , _gitSource_fetchSubmodules = False -- TODO: How do we determine if this should be true?
1054+ , _gitSource_private = isPrivate
1057+ getIsPrivate = maybe (guessGitRepoIsPrivate $ GitUri u) pure mPrivate
1059+guessGitRepoIsPrivate :: MonadObelisk m => GitUri -> m Bool
1060+guessGitRepoIsPrivate uri = flip fix urisToTry $ \loop -> \case
1062+ uriAttempt:xs -> do
1063+ result <- readCreateProcessWithExitCode $
1070+ , T.unpack $ gitUriToText uriAttempt
1073+ (ExitSuccess, _, _) -> pure False -- Must be a public repo
1076+ urisToTry = nubOrd $
1077+ -- Include the original URI if it isn't using SSH because SSH will certainly fail.
1078+ [uri | fmap URI.unRText (URI.uriScheme (unGitUri uri)) /= Just "ssh"] <>
1079+ [changeScheme "https" uri, changeScheme "http" uri]
1080+ changeScheme scheme (GitUri u) = GitUri $ u
1081+ { URI.uriScheme = URI.mkScheme scheme
1082+ , URI.uriAuthority = (\x -> x { URI.authUserInfo = Nothing }) <$> URI.uriAuthority u
817-- Funny signature indicates no effects depend on the optional branch name.
1085-- Funny signature indicates no effects depend on the optional branch name.
818githubThunkRev
1086githubThunkRev
@@ -824,19 +1092,19 @@ githubThunkRev
824githubThunkRev s commit = do
1092githubThunkRev s commit = do
825 owner <- forcePP $ _gitHubSource_owner s
1093 owner <- forcePP $ _gitHubSource_owner s
826 repo <- forcePP $ _gitHubSource_repo s
1094 repo <- forcePP $ _gitHubSource_repo s
827− revTarball <- mkPathPiece $ commit <> ".tar.gz"
1095+ revTarball <- URI.mkPathPiece $ commit <> ".tar.gz"
828− let archiveUri = URI
1096+ let archiveUri = GitUri $ URI.URI
829− { uriScheme = Just $ fromRight' $ mkScheme "https"
1097+ { URI.uriScheme = Just $ fromRight' $ URI.mkScheme "https"
830− , uriAuthority = Right $ Authority
1098+ , URI.uriAuthority = Right $ URI.Authority
831− { authUserInfo = Nothing
1099+ { URI.authUserInfo = Nothing
832− , authHost = fromRight' $ mkHost "github.com"
1100+ , URI.authHost = fromRight' $ URI.mkHost "github.com"
833− , authPort = Nothing
1101+ , URI.authPort = Nothing
835− , uriPath = Just ( False
1103+ , URI.uriPath = Just ( False
836− , owner :| [ repo, fromRight' $ mkPathPiece "archive", revTarball ]
1104+ , owner :| [ repo, fromRight' $ URI.mkPathPiece "archive", revTarball ]
838− , uriQuery = []
1106+ , URI.uriQuery = []
839− , uriFragment = Nothing
1107+ , URI.uriFragment = Nothing
841 hash <- getNixSha256ForUriUnpacked archiveUri
1109 hash <- getNixSha256ForUriUnpacked archiveUri
842 putLog Debug $ "Nix sha256 is " <> hash
1110 putLog Debug $ "Nix sha256 is " <> hash
@@ -845,8 +1113,8 @@ githubThunkRev s commit = do
845 , _thunkRev_nixSha256 = hash
1113 , _thunkRev_nixSha256 = hash
848− forcePP :: Name entity -> m (RText 'PathPiece)
1116+ forcePP :: Name entity -> m (URI.RText 'URI.PathPiece)
849− forcePP = mkPathPiece . untagName
1117+ forcePP = URI.mkPathPiece . untagName
851gitThunkRev
1119gitThunkRev
852 :: MonadObelisk m
1120 :: MonadObelisk m
@@ -855,13 +1123,13 @@ gitThunkRev
855 -> m ThunkRev
1123 -> m ThunkRev
856gitThunkRev s commit = do
1124gitThunkRev s commit = do
857 let u = _gitSource_url s
1125 let u = _gitSource_url s
858− protocols = ["https", "ssh", "git"]
1126+ protocols = ["file", "https", "ssh", "git"]
859− Just scheme <- pure $ unRText <$> uriScheme u
1127+ scheme = maybe "file" URI.unRText $ URI.uriScheme $ (\(GitUri x) -> x) u
860 unless (T.toLower scheme `elem` protocols) $
1128 unless (T.toLower scheme `elem` protocols) $
861 failWith $ "obelisk currently only supports "
1129 failWith $ "obelisk currently only supports "
862− <> T.intercalate ", " protocols <> " protocols for non-GitHub remotes"
1130+ <> T.intercalate ", " protocols <> " protocols for plain Git remotes"
863 hash <- nixPrefetchGit u commit $ _gitSource_fetchSubmodules s
1131 hash <- nixPrefetchGit u commit $ _gitSource_fetchSubmodules s
864− putLog Debug $ "Nix sha256 is " <> hash
1132+ putLog Informational $ "Nix sha256 is " <> hash
865 pure $ ThunkRev
1133 pure $ ThunkRev
866 { _thunkRev_commit = commitNameToRef (N commit)
1134 { _thunkRev_commit = commitNameToRef (N commit)
867 , _thunkRev_nixSha256 = hash
1135 , _thunkRev_nixSha256 = hash
@@ -872,10 +1140,14 @@ gitThunkRev s commit = do
873-- If the branch name is passed in, it is returned exactly as-is. If it is not
1141-- If the branch name is passed in, it is returned exactly as-is. If it is not
874-- passed it, the default branch of the repo is used instead.
1142-- passed it, the default branch of the repo is used instead.
875gitGetCommitBranch
1144gitGetCommitBranch
876− :: MonadObelisk m => URI -> Maybe Text -> m (Text, CommitId)
1145+ :: MonadObelisk m => GitUri -> Maybe Text -> m (Text, CommitId)
877gitGetCommitBranch uri mbranch = withExitFailMessage ("Failure for git remote " <> uriMsg) $ do
1146gitGetCommitBranch uri mbranch = withExitFailMessage ("Failure for git remote " <> uriMsg) $ do
1147+ (_, bothMaps) <- gitLsRemote
1148+ (T.unpack $ gitUriToText uri)
878− bothMaps <- rethrowE =<< liftIO (gitLsRemote (show uri) (GitRef_Branch <$> mbranch))
1149+ (GitRef_Branch <$> mbranch)
879 branch <- case mbranch of
1151 branch <- case mbranch of
880 Nothing -> withExitFailMessage "Failed to find default branch" $ do
1152 Nothing -> withExitFailMessage "Failed to find default branch" $ do
881 b <- rethrowE $ gitLookupDefaultBranch bothMaps
1153 b <- rethrowE $ gitLookupDefaultBranch bothMaps
@@ -883,18 +1155,27 @@ gitGetCommitBranch uri mbranch = withExitFailMessage ("Failure for git remote "
884 Just b -> pure b
1156 Just b -> pure b
885 commit <- rethrowE $ gitLookupCommitForRef bothMaps (GitRef_Branch branch)
1157 commit <- rethrowE $ gitLookupCommitForRef bothMaps (GitRef_Branch branch)
886− putLog Debug $ "Latest commit in branch " <> branch
1158+ putLog Informational $ "Latest commit in branch " <> branch
887 <> " from remote repo " <> uriMsg
1159 <> " from remote repo " <> uriMsg
888 <> " is " <> commit
1160 <> " is " <> commit
889 pure (branch, commit)
1161 pure (branch, commit)
891 rethrowE = either failWith pure
1163 rethrowE = either failWith pure
892− uriMsg = T.pack $ show uri
1164+ uriMsg = gitUriToText uri
1166+parseGitUri :: Text -> Maybe GitUri
1167+parseGitUri x = GitUri <$> (parseFileURI x <|> parseAbsoluteURI x <|> parseSshShorthand x)
1169+parseFileURI :: Text -> Maybe URI.URI
1170+parseFileURI uri = if "/" `T.isPrefixOf` uri then parseAbsoluteURI ("file://" <> uri) else Nothing
894−parseGitUri :: Text -> Maybe URI
1172+parseAbsoluteURI :: Text -> Maybe URI.URI
895−parseGitUri x = mkURI x <|> parseSshShorthand x
1173+parseAbsoluteURI uri = do
1174+ parsedUri <- URI.mkURI uri
1175+ guard $ URI.isPathAbsolute parsedUri
897−parseSshShorthand :: Text -> Maybe URI
1178+parseSshShorthand :: Text -> Maybe URI.URI
898parseSshShorthand uri = do
1179parseSshShorthand uri = do
899 -- This is what git does to check that the remote
1180 -- This is what git does to check that the remote
900 -- is not a local file path when parsing shorthand.
1181 -- is not a local file path when parsing shorthand.
@@ -907,4 +1188,4 @@ parseSshShorthand uri = do
907 -- This check is used to disambiguate a filepath containing a colon from shorthand
1188 -- This check is used to disambiguate a filepath containing a colon from shorthand
908 guard $ isNothing (T.findIndex (=='/') authAndHostname)
1189 guard $ isNothing (T.findIndex (=='/') authAndHostname)
909 && not (T.null colonAndPath)
1190 && not (T.null colonAndPath)
910− mkURI properUri
1191+ URI.mkURI properUri