@@ -5,71 +5,59 @@
5{-# LANGUAGE MultiWayIf #-}
5{-# LANGUAGE MultiWayIf #-}
6{-# LANGUAGE OverloadedStrings #-}
6{-# LANGUAGE OverloadedStrings #-}
7{-# LANGUAGE PatternGuards #-}
7{-# LANGUAGE PatternGuards #-}
8+{-# LANGUAGE QuasiQuotes #-}
8{-# LANGUAGE ScopedTypeVariables #-}
9{-# LANGUAGE ScopedTypeVariables #-}
9{-# LANGUAGE TypeApplications #-}
10{-# LANGUAGE TypeApplications #-}
10−module Obelisk.Command.Thunk
11+module Obelisk.Command.Thunk where
15− , ReadThunkError (..)
19− , updateThunkToLatest
21− , createThunkWithLatest
22− , nixBuildAttrWithCache
23− , nixBuildThunkAttrWithCache
33import Control.Applicative
13import Control.Applicative
34import Control.Exception (displayException, try)
14import Control.Exception (displayException, try)
35−import qualified Control.Lens as Lens
15+import Control.Lens (ifor, ifor_, (.~))
36−import Control.Lens.Indexed hiding ((<.>))
37import Control.Monad
16import Control.Monad
38−import Control.Monad.Catch (handle)
17+import Control.Monad.Extra (findM)
18+import Control.Monad.Catch (MonadCatch, handle)
39import Control.Monad.Except
19import Control.Monad.Except
40import Data.Aeson ((.=))
20import Data.Aeson ((.=))
41import qualified Data.Aeson as Aeson
21import qualified Data.Aeson as Aeson
42import Data.Aeson.Encode.Pretty
22import Data.Aeson.Encode.Pretty
43import qualified Data.Aeson.Types as Aeson
23import qualified Data.Aeson.Types as Aeson
24+import Data.Bifunctor (first)
44import qualified Data.ByteString.Lazy as LBS
25import qualified Data.ByteString.Lazy as LBS
26+import Data.Containers.ListUtils (nubOrd)
45import Data.Default
27import Data.Default
46import Data.Either.Combinators (fromRight', rightToMaybe)
28import Data.Either.Combinators (fromRight', rightToMaybe)
29+import Data.Foldable (toList)
30+import Data.Function ((&))
31+import Data.Functor ((<&>))
47import Data.Git.Ref (Ref)
32import Data.Git.Ref (Ref)
48import qualified Data.Git.Ref as Ref
33import qualified Data.Git.Ref as Ref
49import qualified Data.List as L
34import qualified Data.List as L
50−import Data.List.NonEmpty (NonEmpty (..))
35+import Data.List.NonEmpty (NonEmpty (..), nonEmpty)
51import qualified Data.List.NonEmpty as NonEmpty
36import qualified Data.List.NonEmpty as NonEmpty
52import Data.Map (Map)
37import Data.Map (Map)
53import qualified Data.Map as Map
38import qualified Data.Map as Map
54import Data.Maybe
39import Data.Maybe
55import Data.Set (Set)
40import Data.Set (Set)
56import qualified Data.Set as Set
41import qualified Data.Set as Set
42+import Data.String.Here.Interpolated (i)
43+import Data.String.Here.Uninterpolated (here)
57import Data.Text (Text)
44import Data.Text (Text)
58import qualified Data.Text as T
45import qualified Data.Text as T
59import Data.Text.Encoding
46import Data.Text.Encoding
60import qualified Data.Text.IO as T
47import qualified Data.Text.IO as T
61import Data.Time.Clock.POSIX (posixSecondsToUTCTime)
48import Data.Time.Clock.POSIX (posixSecondsToUTCTime)
49+import Data.Traversable (for)
62import Data.Yaml (parseMaybe)
50import Data.Yaml (parseMaybe)
63import GitHub
51import GitHub
64import GitHub.Data.Name
52import GitHub.Data.Name
65import Obelisk.Command.Nix
53import Obelisk.Command.Nix
66import System.Directory
54import System.Directory
67import System.FilePath
56import System.FilePath
68import System.IO.Error
57import System.IO.Error
69import System.IO.Temp
58import System.IO.Temp
70−import System.Posix (getSymbolicLinkStatus, modificationTime)
59+import System.PosixCompat.Files (getSymbolicLinkStatus, modificationTime)
71−import System.Process (proc)
60+import qualified Text.URI as URI
74import Obelisk.App (MonadObelisk)
62import Obelisk.App (MonadObelisk)
75import Obelisk.CliApp
63import Obelisk.CliApp
@@ -77,11 +65,10 @@ import Obelisk.Command.Utils
78--TODO: Support symlinked thunk data
66--TODO: Support symlinked thunk data
79data ThunkData
67data ThunkData
80− = ThunkData_Packed ThunkPtr
68+ = ThunkData_Packed ThunkSpec ThunkPtr
81 -- ^ Packed thunk
69 -- ^ Packed thunk
82− | ThunkData_Checkout (Maybe ThunkPtr)
70+ | ThunkData_Checkout
83 -- ^ Checked out thunk that was unpacked from this pointer
71 -- ^ Checked out thunk that was unpacked from this pointer
84− deriving (Show, Eq, Ord)
86-- | A reference to the exact data that a thunk should translate into
73-- | A reference to the exact data that a thunk should translate into
87data ThunkPtr = ThunkPtr
74data ThunkPtr = ThunkPtr
@@ -94,7 +81,7 @@ type NixSha256 = Text --TODO: Use a smart constructor and make this actually ver
95-- | A specific revision of data; it may be available from multiple sources
82-- | A specific revision of data; it may be available from multiple sources
96data ThunkRev = ThunkRev
83data ThunkRev = ThunkRev
97− { _thunkRev_commit :: Ref
84+ { _thunkRev_commit :: Ref Ref.SHA1
98 , _thunkRev_nixSha256 :: NixSha256
85 , _thunkRev_nixSha256 :: NixSha256
100 deriving (Show, Eq, Ord)
87 deriving (Show, Eq, Ord)
@@ -107,41 +94,72 @@ data ThunkSource
107 | ThunkSource_Git GitSource
94 | ThunkSource_Git GitSource
108 deriving (Show, Eq, Ord)
95 deriving (Show, Eq, Ord)
97+thunkSourceToGitSource :: ThunkSource -> GitSource
98+thunkSourceToGitSource = \case
99+ ThunkSource_GitHub s -> forgetGithub False s
100+ ThunkSource_Git s -> s
110data GitHubSource = GitHubSource
102data GitHubSource = GitHubSource
111 { _gitHubSource_owner :: Name Owner
103 { _gitHubSource_owner :: Name Owner
112 , _gitHubSource_repo :: Name Repo
104 , _gitHubSource_repo :: Name Repo
113 , _gitHubSource_branch :: Maybe (Name Branch)
105 , _gitHubSource_branch :: Maybe (Name Branch)
106+ , _gitHubSource_private :: Bool
115 deriving (Show, Eq, Ord)
108 deriving (Show, Eq, Ord)
110+newtype GitUri = GitUri { unGitUri :: URI.URI } deriving (Eq, Ord, Show)
112+gitUriToText :: GitUri -> Text
113+gitUriToText (GitUri uri)
114+ | (T.toLower . URI.unRText <$> URI.uriScheme uri) == Just "file"
115+ , Just (_, path) <- URI.uriPath uri
116+ = "/" <> T.intercalate "/" (map URI.unRText $ NonEmpty.toList path)
117+ | otherwise = URI.render uri
117data GitSource = GitSource
119data GitSource = GitSource
118− { _gitSource_url :: URI
120+ { _gitSource_url :: GitUri
119 , _gitSource_branch :: Maybe (Name Branch)
121 , _gitSource_branch :: Maybe (Name Branch)
120 , _gitSource_fetchSubmodules :: Bool
122 , _gitSource_fetchSubmodules :: Bool
123+ , _gitSource_private :: Bool
122 deriving (Show, Eq, Ord)
125 deriving (Show, Eq, Ord)
127+newtype ThunkConfig = ThunkConfig
128+ { _thunkConfig_private :: Maybe Bool
131+data ThunkUpdateConfig = ThunkUpdateConfig
132+ { _thunkUpdateConfig_branch :: Maybe String
133+ , _thunkUpdateConfig_config :: ThunkConfig
136+data ThunkPackConfig = ThunkPackConfig
137+ { _thunkPackConfig_force :: Bool
138+ , _thunkPackConfig_config :: ThunkConfig
124-- | Convert a GitHub source to a regular Git source. Assumes no submodules.
141-- | Convert a GitHub source to a regular Git source. Assumes no submodules.
125forgetGithub :: Bool -> GitHubSource -> GitSource
142forgetGithub :: Bool -> GitHubSource -> GitSource
126forgetGithub useSsh s = GitSource
143forgetGithub useSsh s = GitSource
127− { _gitSource_url = URI
144+ { _gitSource_url = GitUri $ URI.URI
128− { uriScheme = Just $ fromRight' $ mkScheme $ if useSsh then "ssh" else "https"
145+ { URI.uriScheme = Just $ fromRight' $ URI.mkScheme $ if useSsh then "ssh" else "https"
129− , uriAuthority = Right $ Authority
146+ , URI.uriAuthority = Right $ URI.Authority
130− { authUserInfo = UserInfo (fromRight' $ mkUsername "git") Nothing
147+ { URI.authUserInfo = URI.UserInfo (fromRight' $ URI.mkUsername "git") Nothing
131 <$ guard useSsh
148 <$ guard useSsh
132− , authHost = fromRight' $ mkHost "github.com"
149+ , URI.authHost = fromRight' $ URI.mkHost "github.com"
133− , authPort = Nothing
150+ , URI.authPort = Nothing
135− , uriPath = Just ( False
152+ , URI.uriPath = Just ( False
136− , fromRight' . mkPathPiece <$>
153+ , fromRight' . URI.mkPathPiece <$>
137 untagName (_gitHubSource_owner s)
154 untagName (_gitHubSource_owner s)
138 :| [ untagName (_gitHubSource_repo s) <> ".git" ]
155 :| [ untagName (_gitHubSource_repo s) <> ".git" ]
140− , uriQuery = []
157+ , URI.uriQuery = []
141− , uriFragment = Nothing
158+ , URI.uriFragment = Nothing
143 , _gitSource_branch = _gitHubSource_branch s
160 , _gitSource_branch = _gitHubSource_branch s
144 , _gitSource_fetchSubmodules = False
161 , _gitSource_fetchSubmodules = False
162+ , _gitSource_private = _gitHubSource_private s
147getThunkGitBranch :: ThunkPtr -> Maybe Text
165getThunkGitBranch :: ThunkPtr -> Maybe Text
@@ -149,26 +167,26 @@ getThunkGitBranch (ThunkPtr _ src) = fmap untagName $ case src of
149 ThunkSource_GitHub s -> _gitHubSource_branch s
167 ThunkSource_GitHub s -> _gitHubSource_branch s
150 ThunkSource_Git s -> _gitSource_branch s
168 ThunkSource_Git s -> _gitSource_branch s
152−commitNameToRef :: Name Commit -> Ref
170+commitNameToRef :: Name Commit -> Ref Ref.SHA1
153commitNameToRef (N c) = Ref.fromHex $ encodeUtf8 c
171commitNameToRef (N c) = Ref.fromHex $ encodeUtf8 c
155-- TODO: Use spinner here.
173-- TODO: Use spinner here.
156getNixSha256ForUriUnpacked
174getNixSha256ForUriUnpacked
157 :: MonadObelisk m
175 :: MonadObelisk m
158− => URI
176+ => GitUri
159 -> m NixSha256
177 -> m NixSha256
160getNixSha256ForUriUnpacked uri =
178getNixSha256ForUriUnpacked uri =
161− withExitFailMessage ("nix-prefetch-url: Failed to determine sha256 hash of URL " <> render uri) $ do
179+ withExitFailMessage ("nix-prefetch-url: Failed to determine sha256 hash of URL " <> gitUriToText uri) $ do
162 [hash] <- fmap T.lines $ readProcessAndLogOutput (Debug, Debug) $
180 [hash] <- fmap T.lines $ readProcessAndLogOutput (Debug, Debug) $
163− proc "nix-prefetch-url" ["--unpack" , "--type" , "sha256" , T.unpack $ render uri]
181+ proc nixPrefetchUrlPath ["--unpack", "--type", "sha256", T.unpack $ gitUriToText uri]
164 pure hash
182 pure hash
166−nixPrefetchGit :: MonadObelisk m => URI -> Text -> Bool -> m NixSha256
184+nixPrefetchGit :: MonadObelisk m => GitUri -> Text -> Bool -> m NixSha256
167nixPrefetchGit uri rev fetchSubmodules =
185nixPrefetchGit uri rev fetchSubmodules =
168− withExitFailMessage ("nix-prefetch-git: Failed to determine sha256 hash of Git repo " <> render uri <> " at " <> rev) $ do
186+ withExitFailMessage ("nix-prefetch-git: Failed to determine sha256 hash of Git repo " <> gitUriToText uri <> " at " <> rev) $ do
169 out <- readProcessAndLogStderr Debug $
187 out <- readProcessAndLogStderr Debug $
170− proc "nix-prefetch-git" $ filter (/="")
188+ proc nixPrefetchGitPath $ filter (/="")
171− [ "--url", T.unpack $ render uri
189+ [ "--url", T.unpack $ gitUriToText uri
172 , "--rev", T.unpack rev
190 , "--rev", T.unpack rev
173 , if fetchSubmodules then "--fetch-submodules" else ""
191 , if fetchSubmodules then "--fetch-submodules" else ""
174 , "--quiet"
192 , "--quiet"
@@ -180,93 +198,117 @@ nixPrefetchGit uri rev fetchSubmodules =
181--TODO: Pretty print these
199--TODO: Pretty print these
182data ReadThunkError
200data ReadThunkError
183− = ReadThunkError_UnrecognizedFiles
201+ = ReadThunkError_UnrecognizedThunk
184− | ReadThunkError_AmbiguousFiles
202+ | ReadThunkError_UnrecognizedPaths (NonEmpty FilePath)
185− | ReadThunkError_UnrecognizedLoader Text
203+ | ReadThunkError_MissingPaths (NonEmpty FilePath)
186− | ReadThunkError_UnparseablePtr LBS.ByteString
204+ | ReadThunkError_UnparseablePtr FilePath String
187− deriving (Show)
205+ | ReadThunkError_FileError IOError
188−
206+ | ReadThunkError_FileDoesNotMatch FilePath Text
189−-- | Read a thunk and validate that it is only a thunk, either packed or unpacked.
207+ | ReadThunkError_UnrecognizedState String
190−-- If the thunk is packed and additional data is present, fail.
208+ | ReadThunkError_AmbiguousPackedState ThunkPtr ThunkPtr
191−readThunk :: MonadObelisk m => FilePath -> m (Either ReadThunkError ThunkData)
209+ deriving (Show)
192−readThunk thunkDir = do
210+
193− files <- liftIO $ listDirectory thunkDir
211+unpackedDirName :: FilePath
194− case ".git" `elem` files of
212+unpackedDirName = "."
195− True -> fmap ThunkData_Checkout <$> do
213+
196− let origThunkPath = thunkDir </> ".git" </> "obelisk" </> "orig-thunk"
214+attrCacheFileName :: FilePath
197− liftIO $ doesDirectoryExist origThunkPath >>= \case
215+attrCacheFileName = ".attr-cache"
198− True -> fmap Just <$> readPackedThunk origThunkPath
216+
199− False -> return $ return Nothing
217+-- | Specification for how a file in a thunk version works.
200− False -> liftIO $ fmap ThunkData_Packed <$> readPackedThunk thunkDir
218+data ThunkFileSpec
201−
219+ = ThunkFileSpec_Ptr (LBS.ByteString -> Either String ThunkPtr) -- ^ This file specifies 'ThunkPtr' data
202−data ThunkType = ThunkType
220+ | ThunkFileSpec_FileMatches Text -- ^ This file must match the given content exactly
203− { _thunkType_loader :: FilePath
221+ | ThunkFileSpec_CheckoutIndicator -- ^ Existence of this directory indicates that the thunk is unpacked
204− , _thunkType_json :: FilePath
222+ | ThunkFileSpec_AttrCache -- ^ This directory is an attribute cache
205− , _thunkType_optional :: Set FilePath
223+
206− , _thunkType_loaderVersions :: NonEmpty Text
224+-- | Specification for how a set of files in a thunk version work.
207− , _thunkType_parser :: Aeson.Object -> Aeson.Parser ThunkPtr
225+data ThunkSpec = ThunkSpec
208− }
226+ { _thunkSpec_name :: !Text
209−
227+ , _thunkSpec_files :: !(Map FilePath ThunkFileSpec)
210−gitHubThunkType :: ThunkType
211−gitHubThunkType = ThunkType
212− { _thunkType_loader = "default.nix"
213− , _thunkType_json = "github.json"
214− , _thunkType_optional = Set.fromList [".attr-cache"]
215− , _thunkType_loaderVersions = gitHubStandaloneLoaders
216− , _thunkType_parser = parseThunkPtr $ \v ->
217− ThunkSource_GitHub <$> parseGitHubSource v <|> ThunkSource_Git <$> parseGitSource v
220−gitThunkType :: ThunkType
230+thunkSpecTypes :: NonEmpty (NonEmpty ThunkSpec)
221−gitThunkType = ThunkType
231+thunkSpecTypes = gitThunkSpecs :| [gitHubThunkSpecs]
222− { _thunkType_loader = "default.nix"
232+
223− , _thunkType_json = "git.json"
233+-- | Attempts to match a 'ThunkSpec' to a given directory.
224− , _thunkType_optional = Set.fromList [".attr-cache"]
234+matchThunkSpecToDir
225− , _thunkType_loaderVersions = plainGitStandaloneLoaders
235+ :: (MonadError ReadThunkError m, MonadIO m, MonadCatch m)
226− , _thunkType_parser = parseThunkPtr $ fmap ThunkSource_Git . parseGitSource
236+ => ThunkSpec -- ^ 'ThunkSpec' to match against the given files/directory
227− }
237+ -> FilePath -- ^ Path to directory
228−
238+ -> Set FilePath -- ^ Set of file paths relative to the given directory
229−thunkTypes :: [ThunkType]
239+ -> m ThunkData
230−thunkTypes = [gitThunkType, gitHubThunkType]
240+matchThunkSpecToDir thunkSpec dir dirFiles = do
231−
241+ case nonEmpty (toList $ dirFiles `Set.difference` expectedPaths) of
232−findThunkType :: [ThunkType] -> FilePath -> IO (Either ReadThunkError ThunkType)
242+ Just fs -> throwError $ ReadThunkError_UnrecognizedPaths $ (dir </>) <$> fs
233−findThunkType types thunkDir = do
243+ Nothing -> pure ()
234− matches <- fmap catMaybes $ forM types $ \thunkType -> do
244+ case nonEmpty (toList $ requiredPaths `Set.difference` dirFiles) of
235− let
245+ Just fs -> throwError $ ReadThunkError_MissingPaths $ (dir </>) <$> fs
236− expectedContents = Set.fromList $ (thunkDir </>) <$>
246+ Nothing -> pure ()
237− [ _thunkType_loader thunkType
247+ datas <- fmap toList $ flip Map.traverseMaybeWithKey (_thunkSpec_files thunkSpec) $ \expectedPath -> \case
238− , _thunkType_json thunkType
248+ ThunkFileSpec_AttrCache -> Nothing <$ dirMayExist expectedPath
239− ]
249+ ThunkFileSpec_CheckoutIndicator -> liftIO (doesDirectoryExist (dir </> expectedPath)) <&> \case
240− optionalContents = Set.map (thunkDir </>) (_thunkType_optional thunkType)
250+ False -> Nothing
241−
251+ True -> Just ThunkData_Checkout
242− -- Ensure that there aren't any other files in the thunk
252+ ThunkFileSpec_FileMatches expectedContents -> handle (\(e :: IOError) -> throwError $ ReadThunkError_FileError e) $ do
243− -- NB: System.Directory.listDirectory returns the contents without the directory path
253+ actualContents <- liftIO (T.readFile $ dir </> expectedPath)
244− files <- liftIO $ Set.fromList . fmap (thunkDir </>) <$> listDirectory thunkDir
254+ case T.strip expectedContents == T.strip actualContents of
245− let unexpectedContents = files `Set.difference` (expectedContents `Set.union` optionalContents)
255+ True -> pure Nothing
246− missingContents = expectedContents `Set.difference` files
256+ False -> throwError $ ReadThunkError_FileDoesNotMatch (dir </> expectedPath) expectedContents
247− pure $ if Set.null unexpectedContents && Set.null missingContents
257+ ThunkFileSpec_Ptr parser -> handle (\(e :: IOError) -> throwError $ ReadThunkError_FileError e) $ do
248− then Just thunkType
258+ let path = dir </> expectedPath
249− else Nothing
259+ liftIO (doesFileExist path) >>= \case
250−
260+ False -> pure Nothing
251− pure $ case matches of
261+ True -> do
252− [singleMatch] -> Right singleMatch
262+ actualContents <- liftIO $ LBS.readFile path
253− [] -> Left ReadThunkError_UnrecognizedFiles
263+ case parser actualContents of
254− _ -> Left ReadThunkError_AmbiguousFiles
264+ Right v -> pure $ Just (ThunkData_Packed thunkSpec v)
265+ Left e -> throwError $ ReadThunkError_UnparseablePtr (dir </> expectedPath) e
267+ case nonEmpty datas of
268+ Nothing -> throwError ReadThunkError_UnrecognizedThunk
269+ Just xs -> fold1WithM xs $ \a b -> either throwError pure (mergeThunkData a b)
271+ rootPathsOnly = Set.fromList . mapMaybe takeRootDir . Map.keys
272+ takeRootDir = fmap NonEmpty.head . nonEmpty . splitPath
274+ expectedPaths = rootPathsOnly $ _thunkSpec_files thunkSpec
276+ requiredPaths = rootPathsOnly $ Map.filter isRequiredFileSpec $ _thunkSpec_files thunkSpec
277+ isRequiredFileSpec = \case
278+ ThunkFileSpec_FileMatches _ -> True
281+ dirMayExist expectedPath = liftIO (doesFileExist (dir </> expectedPath)) >>= \case
282+ True -> throwError $ ReadThunkError_UnrecognizedPaths $ expectedPath :| []
285+ -- Combine 'ThunkData' from different files, preferring "Checkout" over "Packed"
286+ mergeThunkData ThunkData_Checkout ThunkData_Checkout = Right ThunkData_Checkout
287+ mergeThunkData ThunkData_Checkout ThunkData_Packed{} = Left bothPackedAndUnpacked
288+ mergeThunkData ThunkData_Packed{} ThunkData_Checkout = Left bothPackedAndUnpacked
289+ mergeThunkData a@(ThunkData_Packed _ ptrA) (ThunkData_Packed _ ptrB) =
290+ if ptrA == ptrB then Right a else Left $ ReadThunkError_AmbiguousPackedState ptrA ptrB
292+ bothPackedAndUnpacked = ReadThunkError_UnrecognizedState "Both packed data and checkout present"
294+ fold1WithM (x :| xs) f = foldM f x xs
298+ => NonEmpty (NonEmpty ThunkSpec) -> FilePath -> m (Either ReadThunkError ThunkData)
299+readThunkWith specTypes dir = do
300+ dirFiles <- Set.fromList <$> liftIO (listDirectory dir)
301+ let specs = concatMap toList $ toList $ NonEmpty.transpose specTypes -- Interleave spec types so we try each one in a "fair" ordering
302+ flip fix specs $ \loop -> \case
303+ [] -> pure $ Left ReadThunkError_UnrecognizedThunk
304+ spec:rest -> runExceptT (matchThunkSpecToDir spec dir dirFiles) >>= \case
305+ Left e -> putLog Debug [i|Thunk specification ${_thunkSpec_name spec} did not match ${dir}: ${e}|] *> loop rest
306+ x@(Right _) -> x <$ putLog Debug [i|Thunk specification ${_thunkSpec_name spec} matched ${dir}|]
256-- | Read a thunk and validate that it is exactly a packed thunk.
308-- | Read a thunk and validate that it is exactly a packed thunk.
257-- If additional data is present, fail.
309-- If additional data is present, fail.
258−readPackedThunk :: FilePath -> IO (Either ReadThunkError ThunkPtr)
310+readThunk :: (MonadObelisk m) => FilePath -> m (Either ReadThunkError ThunkData)
259−readPackedThunk thunkDir = runExceptT $ do
311+readThunk = readThunkWith thunkSpecTypes
260− thunkType <- ExceptT $ findThunkType thunkTypes thunkDir
261− -- Ensure that we recognize the thunk loader
262− loader <- liftIO $ T.readFile $ thunkDir </> _thunkType_loader thunkType
263− unless (loader `elem` _thunkType_loaderVersions thunkType) $ do
264− throwError $ ReadThunkError_UnrecognizedLoader loader
266− txt <- liftIO $ LBS.readFile $ thunkDir </> _thunkType_json thunkType
267− case parseMaybe (_thunkType_parser thunkType) =<< Aeson.decode txt of
268− Nothing -> throwError $ ReadThunkError_UnparseablePtr txt
269− Just ptr -> return ptr
271parseThunkPtr :: (Aeson.Object -> Aeson.Parser ThunkSource) -> Aeson.Object -> Aeson.Parser ThunkPtr
313parseThunkPtr :: (Aeson.Object -> Aeson.Parser ThunkSource) -> Aeson.Object -> Aeson.Parser ThunkPtr
272parseThunkPtr parseSrc v = do
314parseThunkPtr parseSrc v = do
@@ -286,10 +328,12 @@ parseGitHubSource v = do
286 owner <- v Aeson..: "owner"
328 owner <- v Aeson..: "owner"
287 repo <- v Aeson..: "repo"
329 repo <- v Aeson..: "repo"
288 branch <- v Aeson..:! "branch"
330 branch <- v Aeson..:! "branch"
331+ private <- v Aeson..:? "private"
289 pure $ GitHubSource
332 pure $ GitHubSource
290 { _gitHubSource_owner = owner
333 { _gitHubSource_owner = owner
291 , _gitHubSource_repo = repo
334 , _gitHubSource_repo = repo
292 , _gitHubSource_branch = branch
335 , _gitHubSource_branch = branch
336+ , _gitHubSource_private = fromMaybe False private
295parseGitSource :: Aeson.Object -> Aeson.Parser GitSource
339parseGitSource :: Aeson.Object -> Aeson.Parser GitSource
@@ -297,25 +341,29 @@ parseGitSource v = do
297 Just url <- parseGitUri <$> v Aeson..: "url"
341 Just url <- parseGitUri <$> v Aeson..: "url"
298 branch <- v Aeson..:! "branch"
342 branch <- v Aeson..:! "branch"
299 fetchSubmodules <- v Aeson..:! "fetchSubmodules"
343 fetchSubmodules <- v Aeson..:! "fetchSubmodules"
344+ private <- v Aeson..:? "private"
300 pure $ GitSource
345 pure $ GitSource
301 { _gitSource_url = url
346 { _gitSource_url = url
302 , _gitSource_branch = branch
347 , _gitSource_branch = branch
303 , _gitSource_fetchSubmodules = fromMaybe False fetchSubmodules
348 , _gitSource_fetchSubmodules = fromMaybe False fetchSubmodules
349+ , _gitSource_private = fromMaybe False private
306overwriteThunk :: MonadObelisk m => FilePath -> ThunkPtr -> m ()
352overwriteThunk :: MonadObelisk m => FilePath -> ThunkPtr -> m ()
307overwriteThunk target thunk = do
353overwriteThunk target thunk = do
308 -- Ensure that this directory is a valid thunk (i.e. so we aren't losing any data)
354 -- Ensure that this directory is a valid thunk (i.e. so we aren't losing any data)
309− Right _ <- readThunk target
355+ readThunk target >>= \case
356+ Left e -> failWith [i|Invalid thunk at ${target}: ${e}|]
311 --TODO: Is there a safer way to do this overwriting?
359 --TODO: Is there a safer way to do this overwriting?
312− liftIO $ removeDirectoryRecursive target
360+ liftIO $ removePathForcibly target
313− liftIO $ createThunk target thunk
361+ createThunk target $ Right thunk
315−thunkPtrLoader :: ThunkPtr -> Text
363+thunkPtrToSpec :: ThunkPtr -> ThunkSpec
316−thunkPtrLoader thunk = case _thunkPtr_source thunk of
364+thunkPtrToSpec thunk = case _thunkPtr_source thunk of
317− ThunkSource_GitHub _ -> NonEmpty.head gitHubStandaloneLoaders
365+ ThunkSource_GitHub _ -> NonEmpty.head gitHubThunkSpecs
318− ThunkSource_Git _ -> NonEmpty.head plainGitStandaloneLoaders
366+ ThunkSource_Git _ -> NonEmpty.head gitThunkSpecs
320-- It's important that formatting be very consistent here, because
368-- It's important that formatting be very consistent here, because
321-- otherwise when people update thunks, their patches will be messy
369-- otherwise when people update thunks, their patches will be messy
@@ -327,13 +375,15 @@ encodeThunkPtrData (ThunkPtr rev src) = case src of
327 , ("branch" .=) <$> _gitHubSource_branch s
375 , ("branch" .=) <$> _gitHubSource_branch s
328 , Just $ "rev" .= Ref.toHexString (_thunkRev_commit rev)
376 , Just $ "rev" .= Ref.toHexString (_thunkRev_commit rev)
329 , Just $ "sha256" .= _thunkRev_nixSha256 rev
377 , Just $ "sha256" .= _thunkRev_nixSha256 rev
378+ , Just $ "private" .= _gitHubSource_private s
331 ThunkSource_Git s -> encodePretty' plainGitCfg $ Aeson.object $ catMaybes
380 ThunkSource_Git s -> encodePretty' plainGitCfg $ Aeson.object $ catMaybes
332− [ Just $ "url" .= render (_gitSource_url s)
381+ [ Just $ "url" .= gitUriToText (_gitSource_url s)
333 , Just $ "rev" .= Ref.toHexString (_thunkRev_commit rev)
382 , Just $ "rev" .= Ref.toHexString (_thunkRev_commit rev)
334 , ("branch" .=) <$> _gitSource_branch s
383 , ("branch" .=) <$> _gitSource_branch s
335 , Just $ "sha256" .= _thunkRev_nixSha256 rev
384 , Just $ "sha256" .= _thunkRev_nixSha256 rev
336 , Just $ "fetchSubmodules" .= _gitSource_fetchSubmodules s
385 , Just $ "fetchSubmodules" .= _gitSource_fetchSubmodules s
386+ , Just $ "private" .= _gitSource_private s
339 githubCfg = defConfig
389 githubCfg = defConfig
@@ -354,78 +404,84 @@ encodeThunkPtrData (ThunkPtr rev src) = case src of
356 , "sha256"
406 , "sha256"
357 , "fetchSubmodules"
408 , "fetchSubmodules"
358 ] <> compare
409 ] <> compare
359 , confTrailingNewline = True
410 , confTrailingNewline = True
362−createThunk :: MonadIO m => FilePath -> ThunkPtr -> m ()
413+createThunk :: MonadObelisk m => FilePath -> Either ThunkSpec ThunkPtr -> m ()
363−createThunk target thunk = liftIO $ do
414+createThunk target ptrInfo =
364− createDirectoryIfMissing True (target </> ".attr-cache")
415+ ifor_ (_thunkSpec_files spec) $ \path -> \case
365− T.writeFile (target </> "default.nix") (thunkPtrLoader thunk)
416+ ThunkFileSpec_FileMatches content -> withReadyPath path $ \p -> liftIO $ T.writeFile p content
366− let
417+ ThunkFileSpec_Ptr _ -> case ptrInfo of
367− jsonFileName = case _thunkPtr_source thunk of
418+ Left _ -> pure () -- We can't write the ptr without it
368− ThunkSource_GitHub _ -> "github"
419+ Right ptr -> withReadyPath path $ \p -> liftIO $ LBS.writeFile p (encodeThunkPtrData ptr)
369− ThunkSource_Git _ -> "git"
420+ _ -> pure ()
370− LBS.writeFile (target </> jsonFileName <.> "json") (encodeThunkPtrData thunk)
421+ where
422+ spec = either id thunkPtrToSpec ptrInfo
423+ withReadyPath path f = do
424+ let fullPath = target </> path
425+ putLog Debug $ "Writing thunk file " <> T.pack fullPath
426+ liftIO $ createDirectoryIfMissing True $ takeDirectory fullPath
372createThunkWithLatest :: MonadObelisk m => FilePath -> ThunkSource -> m ()
429createThunkWithLatest :: MonadObelisk m => FilePath -> ThunkSource -> m ()
373createThunkWithLatest target s = do
430createThunkWithLatest target s = do
374 rev <- getLatestRev s
431 rev <- getLatestRev s
375− createThunk target $ ThunkPtr
432+ createThunk target $ Right $ ThunkPtr
376 { _thunkPtr_source = s
433 { _thunkPtr_source = s
377 , _thunkPtr_rev = rev
434 , _thunkPtr_rev = rev
380−updateThunkToLatest :: MonadObelisk m => FilePath -> Maybe String -> m ()
437+updateThunkToLatest :: MonadObelisk m => ThunkUpdateConfig -> FilePath -> m ()
381−updateThunkToLatest target mBranch = withSpinner' ("Updating thunk " <> T.pack target <> " to latest") (pure $ const $ "Thunk " <> T.pack target <> " updated to latest") $ do
438+updateThunkToLatest (ThunkUpdateConfig mBranch thunkConfig) target = spinner $ do
382− checkThunkDirectory "ob thunk update directory cannot be '.'" target
439+ checkThunkDirectory target
383 -- check to see if thunk should be updated to a specific branch or just update it's current branch
440 -- check to see if thunk should be updated to a specific branch or just update it's current branch
384 case mBranch of
441 case mBranch of
385 Nothing -> do
442 Nothing -> do
386 (overwrite, ptr) <- readThunk target >>= \case
443 (overwrite, ptr) <- readThunk target >>= \case
387− Left err -> failWith $ T.pack $ "thunk update: " <> show err
444+ Left err -> failWith [i|Thunk update: ${err}|]
388 Right c -> case c of
445 Right c -> case c of
389− ThunkData_Packed t -> return (target, t)
446+ ThunkData_Packed _ t -> return (target, t)
390− ThunkData_Checkout _ -> failWith "cannot update an unpacked thunk"
447+ ThunkData_Checkout -> failWith "cannot update an unpacked thunk"
391 let src = _thunkPtr_source ptr
448 let src = _thunkPtr_source ptr
392 rev <- getLatestRev src
449 rev <- getLatestRev src
393− overwriteThunk overwrite $ ThunkPtr
450+ overwriteThunk overwrite $ modifyThunkPtrByConfig thunkConfig $ ThunkPtr
394 { _thunkPtr_source = src
451 { _thunkPtr_source = src
395 , _thunkPtr_rev = rev
452 , _thunkPtr_rev = rev
397 Just branch -> readThunk target >>= \case
454 Just branch -> readThunk target >>= \case
398− Left err -> failWith $ T.pack $ "thunk update: " <> show err
455+ Left err -> failWith [i|Thunk update: ${err}|]
399 Right c -> case c of
456 Right c -> case c of
400− ThunkData_Packed t -> case _thunkPtr_source t of
457+ ThunkData_Packed _ t -> setThunk thunkConfig target (thunkSourceToGitSource $ _thunkPtr_source t) branch
401− ThunkSource_Git tsg -> setThunk target tsg branch
458+ ThunkData_Checkout -> failWith [i|Thunk located at ${target} is unpacked. Use 'ob thunk pack' on the desired directory and then try 'ob thunk update' again.|]
402− ThunkSource_GitHub tsgh -> do
459+ where
403− let tsg = forgetGithub False tsgh
460+ spinner = withSpinner' ("Updating thunk " <> T.pack target <> " to latest") (pure $ const $ "Thunk " <> T.pack target <> " updated to latest")
404− setThunk target tsg branch
461+
405− ThunkData_Checkout _ -> failWith $ T.pack $ "thunk located at " <> (show target) <> " is unpacked. Use ob thunk pack on the desired directory and then try ob thunk update again."
462+setThunk :: MonadObelisk m => ThunkConfig -> FilePath -> GitSource -> String -> m ()
406−
463+setThunk thunkConfig target gs branch = do
407−setThunk :: MonadObelisk m => FilePath -> GitSource -> String -> m ()
464+ newThunkPtr <- uriThunkPtr (_gitSource_url gs) (_thunkConfig_private thunkConfig) (Just $ T.pack branch) Nothing
408−setThunk target gs branch = do
409− newThunkPtr <- uriThunkPtr (_gitSource_url gs) (Just $ T.pack branch) Nothing
410 overwriteThunk target newThunkPtr
465 overwriteThunk target newThunkPtr
411− updateThunkToLatest target Nothing
466+ updateThunkToLatest (ThunkUpdateConfig Nothing thunkConfig) target
413-- | All recognized github standalone loaders, ordered from newest to oldest.
468-- | All recognized github standalone loaders, ordered from newest to oldest.
414-- This tool will only ever produce the newest one when it writes a thunk.
469-- This tool will only ever produce the newest one when it writes a thunk.
415−gitHubStandaloneLoaders :: NonEmpty Text
470+gitHubThunkSpecs :: NonEmpty ThunkSpec
416−gitHubStandaloneLoaders =
471+gitHubThunkSpecs =
417− gitHubStandaloneLoaderV2 :|
472+ gitHubThunkSpecV5 :|
418− [ gitHubStandaloneLoaderV3
473+ [ gitHubThunkSpecV4
419− , gitHubStandaloneLoaderV1
474+ , gitHubThunkSpecV3
422−gitHubStandaloneLoaderV1 :: Text
479+gitHubThunkSpecV1 :: ThunkSpec
423−gitHubStandaloneLoaderV1 = T.unlines
480+gitHubThunkSpecV1 = legacyGitHubThunkSpec "github-v1"
424− [ "import ((import <nixpkgs> {}).fetchFromGitHub (builtins.fromJSON (builtins.readFile ./github.json)))"
481+ "import ((import <nixpkgs> {}).fetchFromGitHub (builtins.fromJSON (builtins.readFile ./github.json)))"
427−gitHubStandaloneLoaderV2 :: Text
483+gitHubThunkSpecV2 :: ThunkSpec
428−gitHubStandaloneLoaderV2 = T.unlines
484+gitHubThunkSpecV2 = legacyGitHubThunkSpec "github-v2" $ T.unlines
429 [ "# DO NOT HAND-EDIT THIS FILE" --TODO: Add something about how to get more info on Obelisk, etc.
485 [ "# DO NOT HAND-EDIT THIS FILE" --TODO: Add something about how to get more info on Obelisk, etc.
430 , "import ((import <nixpkgs> {}).fetchFromGitHub ("
486 , "import ((import <nixpkgs> {}).fetchFromGitHub ("
431 , " let json = builtins.fromJSON (builtins.readFile ./github.json);"
487 , " let json = builtins.fromJSON (builtins.readFile ./github.json);"
@@ -435,8 +491,8 @@ gitHubStandaloneLoaderV2 = T.unlines
438−gitHubStandaloneLoaderV3 :: Text
494+gitHubThunkSpecV3 :: ThunkSpec
439−gitHubStandaloneLoaderV3 = T.unlines
495+gitHubThunkSpecV3 = legacyGitHubThunkSpec "github-v3" $ T.unlines
440 [ "# DO NOT HAND-EDIT THIS FILE"
496 [ "# DO NOT HAND-EDIT THIS FILE"
442 , " fetch = { private ? false, ... }@args: if private && builtins.hasAttr \"fetchGit\" builtins"
498 , " fetch = { private ? false, ... }@args: if private && builtins.hasAttr \"fetchGit\" builtins"
@@ -455,22 +511,62 @@ gitHubStandaloneLoaderV3 = T.unlines
455 , "in import (fetch (builtins.fromJSON (builtins.readFile ./github.json)))"
511 , "in import (fetch (builtins.fromJSON (builtins.readFile ./github.json)))"
458−plainGitStandaloneLoaders :: NonEmpty Text
514+gitHubThunkSpecV4 :: ThunkSpec
459−plainGitStandaloneLoaders =
515+gitHubThunkSpecV4 = legacyGitHubThunkSpec "github-v4" $ T.unlines
460− plainGitStandaloneLoaderV1 :|
516+ [ "# DO NOT HAND-EDIT THIS FILE"
461− [ plainGitStandaloneLoaderV2
517+ , "let fetch = { private ? false, fetchSubmodules ? false, owner, repo, rev, sha256, ... }:"
518+ , " if !fetchSubmodules && !private then builtins.fetchTarball {"
519+ , " url = \"https://github.com/${owner}/${repo}/archive/${rev}.tar.gz\"; inherit sha256;"
520+ , " } else (import <nixpkgs> {}).fetchFromGitHub {"
521+ , " inherit owner repo rev sha256 fetchSubmodules private;"
523+ , "in import (fetch (builtins.fromJSON (builtins.readFile ./github.json)))"
526+legacyGitHubThunkSpec :: Text -> Text -> ThunkSpec
527+legacyGitHubThunkSpec name loader = ThunkSpec name $ Map.fromList
528+ [ ("default.nix", ThunkFileSpec_FileMatches $ T.strip loader)
529+ , ("github.json" , ThunkFileSpec_Ptr parseGitHubJsonBytes)
530+ , (attrCacheFileName, ThunkFileSpec_AttrCache)
531+ , (".git", ThunkFileSpec_CheckoutIndicator)
534+gitHubThunkSpecV5 :: ThunkSpec
535+gitHubThunkSpecV5 = mkThunkSpec "github-v5" "github.json" parseGitHubJsonBytes [here|
536+# DO NOT HAND-EDIT THIS FILE
537+let fetch = { private ? false, fetchSubmodules ? false, owner, repo, rev, sha256, ... }:
538+ if !fetchSubmodules && !private then builtins.fetchTarball {
539+ url = "https://github.com/${owner}/${repo}/archive/${rev}.tar.gz"; inherit sha256;
540+ } else (import <nixpkgs> {}).fetchFromGitHub {
541+ inherit owner repo rev sha256 fetchSubmodules private;
543+ json = builtins.fromJSON (builtins.readFile ./github.json);
547+parseGitHubJsonBytes :: LBS.ByteString -> Either String ThunkPtr
548+parseGitHubJsonBytes = parseJsonObject $ parseThunkPtr $ \v ->
549+ ThunkSource_GitHub <$> parseGitHubSource v <|> ThunkSource_Git <$> parseGitSource v
551+gitThunkSpecs :: NonEmpty ThunkSpec
464−plainGitStandaloneLoaderV1 :: Text
560+gitThunkSpecV1 :: ThunkSpec
465−plainGitStandaloneLoaderV1 = T.unlines
561+gitThunkSpecV1 = legacyGitThunkSpec "git-v1" $ T.unlines
466 [ "# DO NOT HAND-EDIT THIS FILE"
562 [ "# DO NOT HAND-EDIT THIS FILE"
467 , "let fetchGit = {url, rev, ref ? null, branch ? null, sha256 ? null, fetchSubmodules ? null}:"
563 , "let fetchGit = {url, rev, ref ? null, branch ? null, sha256 ? null, fetchSubmodules ? null}:"
468 , " assert !fetchSubmodules; (import <nixpkgs> {}).fetchgit { inherit url rev sha256; };"
564 , " assert !fetchSubmodules; (import <nixpkgs> {}).fetchgit { inherit url rev sha256; };"
469 , "in import (fetchGit (builtins.fromJSON (builtins.readFile ./git.json)))"
565 , "in import (fetchGit (builtins.fromJSON (builtins.readFile ./git.json)))"
472−plainGitStandaloneLoaderV2 :: Text
568+gitThunkSpecV2 :: ThunkSpec
473−plainGitStandaloneLoaderV2 = T.unlines
569+gitThunkSpecV2 = legacyGitThunkSpec "git-v2" $ T.unlines
474 [ "# DO NOT HAND-EDIT THIS FILE"
570 [ "# DO NOT HAND-EDIT THIS FILE"
475 , "let fetchGit = {url, rev, ref ? null, branch ? null, sha256 ? null, fetchSubmodules ? null}:"
571 , "let fetchGit = {url, rev, ref ? null, branch ? null, sha256 ? null, fetchSubmodules ? null}:"
476 , " if builtins.hasAttr \"fetchGit\" builtins"
572 , " if builtins.hasAttr \"fetchGit\" builtins"
@@ -479,10 +575,87 @@ plainGitStandaloneLoaderV2 = T.unlines
479 , "in import (fetchGit (builtins.fromJSON (builtins.readFile ./git.json)))"
575 , "in import (fetchGit (builtins.fromJSON (builtins.readFile ./git.json)))"
482−thunkFileNames :: [FilePath]
578+-- This loader has a bug because @builtins.fetchGit@ is not given a @ref@
483−thunkFileNames = ["default.nix", "github.json", "git.json"]
579+-- and will fail to find commits without this because it does shallow clones.
580+gitThunkSpecV3 :: ThunkSpec
581+gitThunkSpecV3 = legacyGitThunkSpec "git-v3" $ T.unlines
582+ [ "# DO NOT HAND-EDIT THIS FILE"
583+ , "let fetch = {url, rev, ref ? null, sha256 ? null, fetchSubmodules ? false, private ? false, ...}:"
584+ , " let realUrl = let firstChar = builtins.substring 0 1 url; in"
585+ , " if firstChar == \"/\" then /. + url"
586+ , " else if firstChar == \".\" then ./. + url"
588+ , " in if !fetchSubmodules && private then builtins.fetchGit {"
589+ , " url = realUrl; inherit rev;"
590+ , " } else (import <nixpkgs> {}).fetchgit {"
591+ , " url = realUrl; inherit rev sha256;"
593+ , "in import (fetch (builtins.fromJSON (builtins.readFile ./git.json)))"
596+gitThunkSpecV4 :: ThunkSpec
597+gitThunkSpecV4 = legacyGitThunkSpec "git-v4" $ T.unlines
598+ [ "# DO NOT HAND-EDIT THIS FILE"
599+ , "let fetch = {url, rev, branch ? null, sha256 ? null, fetchSubmodules ? false, private ? false, ...}:"
600+ , " let realUrl = let firstChar = builtins.substring 0 1 url; in"
601+ , " if firstChar == \"/\" then /. + url"
602+ , " else if firstChar == \".\" then ./. + url"
604+ , " in if !fetchSubmodules && private then builtins.fetchGit {"
605+ , " url = realUrl; inherit rev;"
606+ , " ${if branch == null then null else \"ref\"} = branch;"
607+ , " } else (import <nixpkgs> {}).fetchgit {"
608+ , " url = realUrl; inherit rev sha256;"
610+ , "in import (fetch (builtins.fromJSON (builtins.readFile ./git.json)))"
485−--TODO: when checking something out, make a shallow clone
613+legacyGitThunkSpec :: Text -> Text -> ThunkSpec
614+legacyGitThunkSpec name loader = ThunkSpec name $ Map.fromList
615+ [ ("default.nix", ThunkFileSpec_FileMatches $ T.strip loader)
616+ , ("git.json" , ThunkFileSpec_Ptr parseGitJsonBytes)
617+ , (attrCacheFileName, ThunkFileSpec_AttrCache)
618+ , (".git", ThunkFileSpec_CheckoutIndicator)
621+gitThunkSpecV5 :: ThunkSpec
622+gitThunkSpecV5 = mkThunkSpec "git-v5" "git.json" parseGitJsonBytes [here|
623+# DO NOT HAND-EDIT THIS FILE
624+let fetch = {url, rev, branch ? null, sha256 ? null, fetchSubmodules ? false, private ? false, ...}:
625+ let realUrl = let firstChar = builtins.substring 0 1 url; in
626+ if firstChar == "/" then /. + url
627+ else if firstChar == "." then ./. + url
629+ in if !fetchSubmodules && private then builtins.fetchGit {
630+ url = realUrl; inherit rev;
631+ ${if branch == null then null else "ref"} = branch;
632+ } else (import <nixpkgs> {}).fetchgit {
633+ url = realUrl; inherit rev sha256;
635+ json = builtins.fromJSON (builtins.readFile ./git.json);
639+parseGitJsonBytes :: LBS.ByteString -> Either String ThunkPtr
640+parseGitJsonBytes = parseJsonObject $ parseThunkPtr $ fmap ThunkSource_Git . parseGitSource
642+mkThunkSpec :: Text -> FilePath -> (LBS.ByteString -> Either String ThunkPtr) -> Text -> ThunkSpec
643+mkThunkSpec name jsonFileName parser srcNix = ThunkSpec name $ Map.fromList
644+ [ ("default.nix", ThunkFileSpec_FileMatches defaultNixViaSrc)
645+ , ("thunk.nix", ThunkFileSpec_FileMatches srcNix)
646+ , (jsonFileName, ThunkFileSpec_Ptr parser)
647+ , (attrCacheFileName, ThunkFileSpec_AttrCache)
648+ , (normalise $ unpackedDirName </> ".git", ThunkFileSpec_CheckoutIndicator)
651+ defaultNixViaSrc = [here|
652+# DO NOT HAND-EDIT THIS FILE
653+import (import ./thunk.nix)
657+parseJsonObject :: (Aeson.Object -> Aeson.Parser a) -> LBS.ByteString -> Either String a
658+parseJsonObject p bytes = Aeson.parseEither p =<< Aeson.eitherDecode bytes
487-- | Checks a cache directory to see if there is a fresh symlink
660-- | Checks a cache directory to see if there is a fresh symlink
488-- to the result of building an attribute of a thunk.
661-- to the result of building an attribute of a thunk.
@@ -490,44 +663,49 @@ thunkFileNames = ["default.nix", "github.json", "git.json"]
490-- and the result is symlinked into the cache.
663-- and the result is symlinked into the cache.
491nixBuildThunkAttrWithCache
664nixBuildThunkAttrWithCache
492 :: MonadObelisk m
665 :: MonadObelisk m
493− => FilePath
666+ => ThunkSpec
494 -- ^ Path to directory containing Thunk
668 -- ^ Path to directory containing Thunk
495 -> String
669 -> String
496 -- ^ Attribute to build
670 -- ^ Attribute to build
497− -> m FilePath
671+ -> m (Maybe FilePath)
498 -- ^ Symlink to cached or built nix output
672 -- ^ Symlink to cached or built nix output
499-- WARNING: If the thunk uses an impure reference such as '<nixpkgs>'
673-- WARNING: If the thunk uses an impure reference such as '<nixpkgs>'
500-- the caching mechanism will fail as it merely measures the modification
674-- the caching mechanism will fail as it merely measures the modification
501-- time of the cache link and the expression to build.
675-- time of the cache link and the expression to build.
502−nixBuildThunkAttrWithCache thunkDir attr = do
676+nixBuildThunkAttrWithCache thunkSpec thunkDir attr = do
503− --NB: Expects thunkDir to be normalised with no trailing path separator.
504− --This should be guaranteed by the command argument parser.
505− let cacheErrHandler e
506− | isDoesNotExistError e = return Nothing -- expected from a cache miss
507− | otherwise = putLog Error (T.pack $ displayException e) >> return Nothing
508− cacheDir = thunkDir </> ".attr-cache"
509− cachePath = cacheDir </> attr <.> "out"
510 latestChange <- liftIO $ do
677 latestChange <- liftIO $ do
511− createDirectoryIfMissing False cacheDir
678+ let
512− let getModificationTimeMaybe = fmap rightToMaybe . try @IOError . getModificationTime
679+ getModificationTimeMaybe = fmap rightToMaybe . try @IOError . getModificationTime
513− maximum . catMaybes <$> mapM (getModificationTimeMaybe . (thunkDir </>)) thunkFileNames
680+ thunkFileNames = Map.keys $ _thunkSpec_files thunkSpec
514− cacheHit <- handle cacheErrHandler $ do
681+ maximum . catMaybes <$> traverse (getModificationTimeMaybe . (thunkDir </>)) thunkFileNames
515− cacheTime <- liftIO $ posixSecondsToUTCTime . realToFrac . modificationTime <$> getSymbolicLinkStatus cachePath
682+
516− return $ if latestChange <= cacheTime
683+ let cachePaths' = nonEmpty $ Map.keys $ Map.filter (\case ThunkFileSpec_AttrCache -> True; _ -> False) $
517− then Just cachePath
684+ _thunkSpec_files thunkSpec
518− else Nothing
685+ for cachePaths' $ \cachePaths ->
519− case cacheHit of
686+ fmap NonEmpty.head $ for cachePaths $ \cacheDir -> do
520− Just c -> return c
687+ let
521− Nothing -> do
688+ cachePath = thunkDir </> cacheDir </> attr <.> "out"
522− putLog Warning $ T.pack $ mconcat [thunkDir, ": ", attr, " not cached, building ..."]
689+ cacheErrHandler e
523− _ <- nixCmd $ NixCmd_Build$ def
690+ | isDoesNotExistError e = pure Nothing -- expected from a cache miss
524− Lens.& nixBuildConfig_outLink Lens..~ OutLink_IndirectRoot cachePath
691+ | otherwise = Nothing <$ putLog Error (T.pack $ displayException e)
525− Lens.& nixCmdConfig_target Lens..~ Target
692+ cacheHit <- handle cacheErrHandler $ do
526− { _target_path = Just thunkDir
693+ cacheTime <- liftIO $ posixSecondsToUTCTime . realToFrac . modificationTime <$> getSymbolicLinkStatus cachePath
527− , _target_attr = Just attr
694+ pure $ if latestChange <= cacheTime
528− , _target_expr = Nothing
695+ then Just cachePath
530− return cachePath
697+ case cacheHit of
700+ putLog Warning $ T.pack $ mconcat [thunkDir, ": ", attr, " not cached, building ..."]
701+ liftIO $ createDirectoryIfMissing True (takeDirectory cachePath)
702+ (cachePath <$) $ nixCmd $ NixCmd_Build $ def
703+ & nixBuildConfig_outLink .~ OutLink_IndirectRoot cachePath
704+ & nixCmdConfig_target .~ Target
705+ { _target_path = Just thunkDir
706+ , _target_attr = Just attr
707+ , _target_expr = Nothing
532-- | Build a nix attribute, and cache the result if possible
710-- | Build a nix attribute, and cache the result if possible
533nixBuildAttrWithCache
711nixBuildAttrWithCache
@@ -538,13 +716,15 @@ nixBuildAttrWithCache
538 -- ^ Attribute to build
716 -- ^ Attribute to build
539 -> m FilePath
717 -> m FilePath
540 -- ^ Symlink to cached or built nix output
718 -- ^ Symlink to cached or built nix output
541−nixBuildAttrWithCache exprPath attr = do
719+nixBuildAttrWithCache exprPath attr = readThunk exprPath >>= \case
542− readThunk exprPath >>= \case
720+ -- Only packed thunks are cached. In particular, checkouts are not.
543− -- Only packed thunks are cached. in particular, checkouts are not
721+ Right (ThunkData_Packed spec _) ->
544− Right (ThunkData_Packed _) -> nixBuildThunkAttrWithCache exprPath attr
722+ maybe build pure =<< nixBuildThunkAttrWithCache spec exprPath attr
545− _ -> nixCmd $ NixCmd_Build $ def
723+ _ -> build
546− Lens.& nixBuildConfig_outLink Lens..~ OutLink_None
724+ where
547− Lens.& nixCmdConfig_target Lens..~ Target
725+ build = nixCmd $ NixCmd_Build $ def
726+ & nixBuildConfig_outLink .~ OutLink_None
727+ & nixCmdConfig_target .~ Target
548 { _target_path = Just exprPath
728 { _target_path = Just exprPath
549 , _target_attr = Just attr
729 , _target_attr = Just attr
550 , _target_expr = Nothing
730 , _target_expr = Nothing
@@ -565,103 +745,144 @@ updateThunk p f = withSystemTempDirectory "obelisk-thunkptr-" $ \tmpDir -> do
566 copyThunkToTmp tmpDir thunkDir = readThunk thunkDir >>= \case
746 copyThunkToTmp tmpDir thunkDir = readThunk thunkDir >>= \case
567 Left err -> failWith $ "withThunkUnpacked: " <> T.pack (show err)
747 Left err -> failWith $ "withThunkUnpacked: " <> T.pack (show err)
568− Right (ThunkData_Packed _) -> do
748+ Right ThunkData_Packed{} -> do
569 let tmpThunk = tmpDir </> "thunk"
749 let tmpThunk = tmpDir </> "thunk"
570 callProcessAndLogOutput (Notice, Error) $
750 callProcessAndLogOutput (Notice, Error) $
571− proc "cp" ["-r", "-T", thunkDir, tmpThunk]
751+ proc cp ["-r", "-T", thunkDir, tmpThunk]
572 return tmpThunk
752 return tmpThunk
573− Right _ -> failWith $ "Thunk is not packed"
753+ Right _ -> failWith "Thunk is not packed"
574 updateThunkFromTmp p' = do
754 updateThunkFromTmp p' = do
575− _ <- packThunk' True False p'
755+ _ <- packThunk' True (ThunkPackConfig False (ThunkConfig Nothing)) p'
576 callProcessAndLogOutput (Notice, Error) $
756 callProcessAndLogOutput (Notice, Error) $
577− proc "cp" ["-r", "-T", p', p]
757+ proc cp ["-r", "-T", p', p]
579finalMsg :: Bool -> (a -> Text) -> Maybe (a -> Text)
759finalMsg :: Bool -> (a -> Text) -> Maybe (a -> Text)
580finalMsg noTrail s = if noTrail then Nothing else Just s
760finalMsg noTrail s = if noTrail then Nothing else Just s
582−unpackThunk :: MonadObelisk m => FilePath -> m ()
583−unpackThunk = unpackThunk' False
585-- | Check that we are not somewhere inside the thunk directory
762-- | Check that we are not somewhere inside the thunk directory
586−checkThunkDirectory :: MonadObelisk m => Text -> FilePath -> m ()
763+checkThunkDirectory :: MonadObelisk m => FilePath -> m ()
587−checkThunkDirectory msg thunkDir = do
764+checkThunkDirectory thunkDir = do
588 currentDir <- liftIO getCurrentDirectory
765 currentDir <- liftIO getCurrentDirectory
589 thunkDir' <- liftIO $ canonicalizePath thunkDir
766 thunkDir' <- liftIO $ canonicalizePath thunkDir
590− when (thunkDir' `L.isInfixOf` currentDir) $ failWith msg
767+ when (thunkDir' `L.isInfixOf` currentDir) $
768+ failWith [i|Can't perform thunk operations from within the thunk directory: ${thunkDir}|]
770+ -- Don't let thunk commands work when directly given an unpacked repo
771+ when (takeFileName thunkDir == unpackedDirName) $
772+ readThunk (takeDirectory thunkDir) >>= \case
773+ Right _ -> failWith [i|Refusing to perform thunk operation on ${thunkDir} because it is a thunk's unpacked source|]
776+unpackThunk :: MonadObelisk m => FilePath -> m ()
777+unpackThunk = unpackThunk' False
592unpackThunk' :: MonadObelisk m => Bool -> FilePath -> m ()
779unpackThunk' :: MonadObelisk m => Bool -> FilePath -> m ()
593−unpackThunk' noTrail thunkDir = checkThunkDirectory "Can't pack/unpack from within the thunk directory" thunkDir >> readThunk thunkDir >>= \case
780+unpackThunk' noTrail thunkDir = checkThunkDirectory thunkDir *> readThunk thunkDir >>= \case
594− Left err -> failWith $ "thunk unpack: " <> T.pack (show err)
781+ Left err -> failWith [i|Invalid thunk at ${thunkDir}: ${err}|]
595 --TODO: Overwrite option that rechecks out thunk; force option to do so even if working directory is dirty
782 --TODO: Overwrite option that rechecks out thunk; force option to do so even if working directory is dirty
596− Right (ThunkData_Checkout _) -> failWith "thunk unpack: thunk is already unpacked"
783+ Right ThunkData_Checkout -> failWith [i|Thunk at ${thunkDir} is already unpacked|]
597− Right (ThunkData_Packed tptr) -> do
784+ Right (ThunkData_Packed _ tptr) -> do
598 let (thunkParent, thunkName) = splitFileName thunkDir
785 let (thunkParent, thunkName) = splitFileName thunkDir
599− withTempDirectory thunkParent thunkName $ \tmpRepo -> do
786+ withTempDirectory thunkParent thunkName $ \tmpThunk -> do
600− let obGitDir = tmpRepo </> ".git" </> "obelisk"
787+ let
601− s = case _thunkPtr_source tptr of
788+ gitSrc = thunkSourceToGitSource $ _thunkPtr_source tptr
602− ThunkSource_GitHub s' -> forgetGithub False s'
789+ newSpec = case _thunkPtr_source tptr of
603− ThunkSource_Git s' -> s'
790+ ThunkSource_GitHub _ -> NonEmpty.head gitHubThunkSpecs
791+ ThunkSource_Git _ -> NonEmpty.head gitThunkSpecs
604 withSpinner' ("Fetching thunk " <> T.pack thunkName)
792 withSpinner' ("Fetching thunk " <> T.pack thunkName)
605 (finalMsg noTrail $ const $ "Fetched thunk " <> T.pack thunkName) $ do
793 (finalMsg noTrail $ const $ "Fetched thunk " <> T.pack thunkName) $ do
606− let git = callProcessAndLogOutput (Notice, Notice) . gitProc tmpRepo
794+ let unpackedPath = tmpThunk </> unpackedDirName
607− git $ [ "clone" ]
795+ gitCloneForThunkUnpack gitSrc (_thunkRev_commit $ _thunkPtr_rev tptr) unpackedPath
608− ++ ("--recursive" <$ guard (_gitSource_fetchSubmodules s))
796+
609− ++ [ T.unpack $ render $ _gitSource_url s ]
797+ let normalizeMore = dropTrailingPathSeparator . normalise
610− ++ do branch <- maybeToList $ _gitSource_branch s
798+ when (normalizeMore unpackedPath /= normalizeMore tmpThunk) $ -- Only write meta data if the checkout is not inplace
611− [ "--branch", T.unpack $ untagName branch ]
799+ createThunk tmpThunk $ Left newSpec
612− git ["reset", "--hard", Ref.toHexString $ _thunkRev_commit $ _thunkPtr_rev tptr]
800+
613− when (_gitSource_fetchSubmodules s) $
801+ liftIO $ do
614− git ["submodule", "update", "--recursive", "--init"]
802+ removePathForcibly thunkDir
615−
803+ renameDirectory tmpThunk thunkDir
616− liftIO $ createDirectory obGitDir
804+
617− callProcessAndLogOutput (Notice, Error) $
805+gitCloneForThunkUnpack
618− proc "cp" ["-r", "-T", thunkDir </> ".", obGitDir </> "orig-thunk"]
806+ :: MonadObelisk m
619− callProcessAndLogOutput (Notice, Error) $
807+ => GitSource -- ^ Git source to use
620− proc "rm" ["-r", thunkDir]
808+ -> Ref hash -- ^ Commit hash to reset to
621− callProcessAndLogOutput (Notice, Error) $
809+ -> FilePath -- ^ Directory to clone into
622− proc "mv" ["-T", tmpRepo, thunkDir]
810+ -> m ()
811+gitCloneForThunkUnpack gitSrc commit dir = do
812+ let git = callProcessAndLogOutput (Notice, Notice) . gitProc dir
814+ ++ ["--recursive" | _gitSource_fetchSubmodules gitSrc]
815+ ++ [ T.unpack $ gitUriToText $ _gitSource_url gitSrc ]
816+ ++ do branch <- maybeToList $ _gitSource_branch gitSrc
817+ [ "--branch", T.unpack $ untagName branch ]
818+ git ["reset", "--hard", Ref.toHexString commit]
819+ when (_gitSource_fetchSubmodules gitSrc) $
820+ git ["submodule", "update", "--recursive", "--init"]
624--TODO: add a rollback mode to pack to the original thunk
822--TODO: add a rollback mode to pack to the original thunk
625−packThunk :: MonadObelisk m => Bool -> FilePath -> m ThunkPtr
823+packThunk :: MonadObelisk m => ThunkPackConfig -> FilePath -> m ThunkPtr
626packThunk = packThunk' False
824packThunk = packThunk' False
628−packThunk' :: MonadObelisk m => Bool -> Bool -> FilePath -> m ThunkPtr
826+packThunk' :: MonadObelisk m => Bool -> ThunkPackConfig -> FilePath -> m ThunkPtr
629−packThunk' noTrail force thunkDir = checkThunkDirectory "Can't pack/unpack from within the thunk directory" thunkDir >> readThunk thunkDir >>= \case
827+packThunk' noTrail (ThunkPackConfig force thunkConfig) thunkDir = checkThunkDirectory thunkDir *> readThunk thunkDir >>= \case
630− Left err -> failWith $ T.pack $ "thunk pack: " <> show err
828+ Right ThunkData_Packed{} -> failWith [i|Thunk at ${thunkDir} is is already packed|]
631− Right (ThunkData_Packed _) -> failWith "pack: thunk is already packed"
829+ _ -> withSpinner'
632− Right (ThunkData_Checkout _) -> do
830+ ("Packing thunk " <> T.pack thunkDir)
633− withSpinner' ("Packing thunk " <> T.pack thunkDir)
831+ (finalMsg noTrail $ const $ "Packed thunk " <> T.pack thunkDir) $
634− (finalMsg noTrail $ const $ "Packed thunk " <> T.pack thunkDir) $ do
832+ do
635− thunkPtr <- getThunkPtr (not force) thunkDir
833+ let checkClean = if force then CheckClean_NoCheck else CheckClean_FullCheck
636− callProcessAndLogOutput (Debug, Error) $ proc "rm" ["-rf", thunkDir]
834+ thunkPtr <- modifyThunkPtrByConfig thunkConfig <$> getThunkPtr checkClean thunkDir (_thunkConfig_private thunkConfig)
637− liftIO $ createThunk thunkDir thunkPtr
835+ liftIO $ removePathForcibly thunkDir
836+ createThunk thunkDir $ Right thunkPtr
638 pure thunkPtr
837 pure thunkPtr
640−getThunkPtr :: forall m. MonadObelisk m => Bool -> FilePath -> m ThunkPtr
839+modifyThunkPtrByConfig :: ThunkConfig -> ThunkPtr -> ThunkPtr
641−getThunkPtr checkClean thunkDir = do
840+modifyThunkPtrByConfig (ThunkConfig markPrivate') ptr = case markPrivate' of
642− when checkClean $ ensureCleanGitRepo thunkDir True $
841+ Nothing -> ptr
842+ Just markPrivate -> ptr { _thunkPtr_source = case _thunkPtr_source ptr of
843+ ThunkSource_Git s -> ThunkSource_Git $ s { _gitSource_private = markPrivate }
844+ ThunkSource_GitHub s -> ThunkSource_GitHub $ s { _gitHubSource_private = markPrivate }
848+ = CheckClean_FullCheck
849+ -- ^ Check that the repo is clean, including .gitignored files
850+ | CheckClean_NotIgnored
851+ -- ^ Check that the repo is clean, not including .gitignored files
852+ | CheckClean_NoCheck
853+ -- ^ Don't check that the repo is clean
855+getThunkPtr :: forall m. MonadObelisk m => CheckClean -> FilePath -> Maybe Bool -> m ThunkPtr
856+getThunkPtr gitCheckClean dir mPrivate = do
857+ let repoLocations = nubOrd $ map (first normalise)
858+ [(".git", "."), (unpackedDirName </> ".git", unpackedDirName)]
859+ repoLocation' <- liftIO $ flip findM repoLocations $ doesDirectoryExist . (dir </>) . fst
860+ thunkDir <- case repoLocation' of
861+ Nothing -> failWith [i|Can't find an unpacked thunk in ${dir}|]
862+ Just (_, path) -> pure $ normalise $ dir </> path
864+ let (checkClean, checkIgnored) = case gitCheckClean of
865+ CheckClean_FullCheck -> (True, True)
866+ CheckClean_NotIgnored -> (True, False)
867+ CheckClean_NoCheck -> (False, False)
868+ when checkClean $ ensureCleanGitRepo thunkDir checkIgnored
643 "thunk pack: thunk checkout contains unsaved modifications"
869 "thunk pack: thunk checkout contains unsaved modifications"
645 -- Check whether there are any stashes
871 -- Check whether there are any stashes
646− stashOutput <- readGitProcess thunkDir ["stash", "list"]
872+ when checkClean $ do
647− when checkClean $ case T.null stashOutput of
873+ stashOutput <- readGitProcess thunkDir ["stash", "list"]
648− False -> do
874+ unless (T.null stashOutput) $
649 failWith $ T.unlines $
875 failWith $ T.unlines $
650 [ "thunk pack: thunk checkout has stashes"
876 [ "thunk pack: thunk checkout has stashes"
651 , "git stash list:"
877 , "git stash list:"
652 ] ++ T.lines stashOutput
878 ] ++ T.lines stashOutput
655− -- Get current branch ``
880+ -- Get current branch
656 (mCurrentBranch, mCurrentCommit) <- do
881 (mCurrentBranch, mCurrentCommit) <- do
657− b <- listToMaybe
882+ b <- listToMaybe . T.lines <$> readGitProcess thunkDir ["rev-parse", "--abbrev-ref", "HEAD"]
658− <$> T.lines
883+ c <- listToMaybe . T.lines <$> readGitProcess thunkDir ["rev-parse", "HEAD"]
659− <$> readGitProcess thunkDir ["rev-parse", "--abbrev-ref", "HEAD"]
662− <$> readGitProcess thunkDir ["rev-parse", "HEAD"]
663 case b of
884 case b of
664− (Just "HEAD") -> failWith $ T.unlines $
885+ (Just "HEAD") -> failWith $ T.unlines
665 [ "thunk pack: You are in 'detached HEAD' state."
886 [ "thunk pack: You are in 'detached HEAD' state."
666 , "If you want to pack at the current ref \
887 , "If you want to pack at the current ref \
667 \then please create a new branch with 'git checkout -b <new-branch-name>' and push this upstream."
888 \then please create a new branch with 'git checkout -b <new-branch-name>' and push this upstream."
@@ -670,7 +891,7 @@ getThunkPtr checkClean thunkDir = do
671 -- Get information on all branches and their (optional) designated upstream
892 -- Get information on all branches and their (optional) designated upstream
672 -- correspondents
893 -- correspondents
673− (headDump :: [Text]) <- T.lines <$> readGitProcess thunkDir
894+ headDump :: [Text] <- T.lines <$> readGitProcess thunkDir
674 [ "for-each-ref"
895 [ "for-each-ref"
675 , "--format=%(refname:short) %(upstream:short) %(upstream:remotename)"
896 , "--format=%(refname:short) %(upstream:short) %(upstream:remotename)"
676 , "refs/heads/"
897 , "refs/heads/"
@@ -716,7 +937,7 @@ getThunkPtr checkClean thunkDir = do
718 -- loosely by https://stackoverflow.com/questions/7773939/show-git-ahead-and-behind-info-for-all-branches-including-remotes
939 -- loosely by https://stackoverflow.com/questions/7773939/show-git-ahead-and-behind-info-for-all-branches-including-remotes
719− stats <- iforM headUpstream $ \branch (upstream, _remote) -> do
940+ stats <- ifor headUpstream $ \branch (upstream, _remote) -> do
720 (stat :: [Text]) <- T.lines <$> readGitProcess thunkDir
941 (stat :: [Text]) <- T.lines <$> readGitProcess thunkDir
721 [ "rev-list", "--left-right"
942 [ "rev-list", "--left-right"
722 , T.unpack branch <> "..." <> T.unpack upstream
943 , T.unpack branch <> "..." <> T.unpack upstream
@@ -733,15 +954,16 @@ getThunkPtr checkClean thunkDir = do
733 [ "thunk pack: Certain branches in the thunk have commits not yet pushed upstream:"
954 [ "thunk pack: Certain branches in the thunk have commits not yet pushed upstream:"
736− (flip map (Map.toList nonGood) $ \(branch, (upstream, (ahead, behind))) -> mconcat
957+ flip map (Map.toList nonGood) (\(branch, (upstream, (ahead, behind))) -> mconcat
737 [" ", branch, " ahead: ", T.pack (show ahead), " behind: ", T.pack (show behind), " remote branch ", upstream]) ++
958 [" ", branch, " ahead: ", T.pack (show ahead), " behind: ", T.pack (show behind), " remote branch ", upstream]) ++
739 , "Please push these upstream and try again. (Or just fetch, if they are somehow \
960 , "Please push these upstream and try again. (Or just fetch, if they are somehow \
740 \pushed but this repo's remote tracking branches don't know it.)"
961 \pushed but this repo's remote tracking branches don't know it.)"
743− -- We assume it's safe to pack the thunk at this point
964+ when checkClean $ do
744− putLog Informational $ "All changes safe in git remotes. OK to pack thunk."
965+ -- We assume it's safe to pack the thunk at this point
966+ putLog Informational "All changes safe in git remotes. OK to pack thunk."
746 let remote = maybe "origin" snd $ flip Map.lookup headUpstream =<< mCurrentBranch
968 let remote = maybe "origin" snd $ flip Map.lookup headUpstream =<< mCurrentBranch
@@ -754,14 +976,12 @@ getThunkPtr checkClean thunkDir = do
754 remoteUri <- case parseGitUri remoteUri' of
976 remoteUri <- case parseGitUri remoteUri' of
755 Nothing -> failWith $ "Could not identify git remote: " <> remoteUri'
977 Nothing -> failWith $ "Could not identify git remote: " <> remoteUri'
756 Just uri -> pure uri
978 Just uri -> pure uri
757− uriThunkPtr remoteUri mCurrentBranch mCurrentCommit
979+ uriThunkPtr remoteUri mPrivate mCurrentBranch mCurrentCommit
759-- | Get the latest revision available from the given source
981-- | Get the latest revision available from the given source
760getLatestRev :: MonadObelisk m => ThunkSource -> m ThunkRev
982getLatestRev :: MonadObelisk m => ThunkSource -> m ThunkRev
761getLatestRev os = do
983getLatestRev os = do
762− let gitS = case os of
984+ let gitS = thunkSourceToGitSource os
763− ThunkSource_GitHub s -> forgetGithub False s
764− ThunkSource_Git s -> s
765 (_, commit) <- gitGetCommitBranch (_gitSource_url gitS) (untagName <$> _gitSource_branch gitS)
985 (_, commit) <- gitGetCommitBranch (_gitSource_url gitS) (untagName <$> _gitSource_branch gitS)
766 case os of
986 case os of
767 ThunkSource_GitHub s -> githubThunkRev s commit
987 ThunkSource_GitHub s -> githubThunkRev s commit
@@ -773,18 +993,18 @@ getLatestRev os = do
773-- performance. If that doesn't work (e.g. authentication issue), we fall back
993-- performance. If that doesn't work (e.g. authentication issue), we fall back
774-- on just doing things the normal way for git repos in general, and save it as
994-- on just doing things the normal way for git repos in general, and save it as
775-- a regular git thunk.
995-- a regular git thunk.
776−uriThunkPtr :: MonadObelisk m => URI -> Maybe Text -> Maybe Text -> m ThunkPtr
996+uriThunkPtr :: MonadObelisk m => GitUri -> Maybe Bool -> Maybe Text -> Maybe Text -> m ThunkPtr
777−uriThunkPtr uri mbranch mcommit = do
997+uriThunkPtr uri mPrivate mbranch mcommit = do
778 commit <- case mcommit of
998 commit <- case mcommit of
779 Nothing -> gitGetCommitBranch uri mbranch >>= return . snd
999 Nothing -> gitGetCommitBranch uri mbranch >>= return . snd
780 (Just c) -> return c
1000 (Just c) -> return c
781− (src, rev) <- case uriToThunkSource uri mbranch of
1001+ (src, rev) <- uriToThunkSource uri mPrivate mbranch >>= \case
782 ThunkSource_GitHub s -> do
1002 ThunkSource_GitHub s -> do
783 rev <- runExceptT $ githubThunkRev s commit
1003 rev <- runExceptT $ githubThunkRev s commit
784 case rev of
1004 case rev of
785 Right r -> pure (ThunkSource_GitHub s, r)
1005 Right r -> pure (ThunkSource_GitHub s, r)
786 Left e -> do
1006 Left e -> do
787− putLog Warning $ "\
1007+ putLog Warning "\
788\Failed to fetch archive from GitHub. This is probably a private repo. \
1008\Failed to fetch archive from GitHub. This is probably a private repo. \
789\Falling back on normal fetchgit. Original failure:"
1009\Falling back on normal fetchgit. Original failure:"
790 errorToWarning e
1010 errorToWarning e
@@ -801,34 +1021,66 @@ uriThunkPtr uri mbranch mcommit = do
801-- If the thunk is a GitHub thunk and fails, we do *not* fall back like with
1021-- If the thunk is a GitHub thunk and fails, we do *not* fall back like with
802-- `uriThunkPtr`. Unlike a plain URL, a thunk src explicitly states which method
1022-- `uriThunkPtr`. Unlike a plain URL, a thunk src explicitly states which method
803-- should be employed, and so we respect that.
1023-- should be employed, and so we respect that.
804−uriToThunkSource :: URI -> Maybe Text -> ThunkSource
1024+uriToThunkSource :: MonadObelisk m => GitUri -> Maybe Bool -> Maybe Text -> m ThunkSource
805−uriToThunkSource u
1025+uriToThunkSource (GitUri u) mPrivate
806− | Right uriAuth <- uriAuthority u
1026+ | Right uriAuth <- URI.uriAuthority u
807− , Just scheme <- unRText <$> uriScheme u
1027+ , Just scheme <- URI.unRText <$> URI.uriScheme u
808 , case scheme of
1028 , case scheme of
809− "ssh" -> uriAuth == Authority
1029+ "ssh" -> uriAuth == URI.Authority
810− { authUserInfo = Just $ UserInfo (fromRight' $ mkUsername "git") Nothing
1030+ { URI.authUserInfo = Just $ URI.UserInfo (fromRight' $ URI.mkUsername "git") Nothing
811− , authHost = fromRight' $ mkHost "github.com"
1031+ , URI.authHost = fromRight' $ URI.mkHost "github.com"
812− , authPort = Nothing
1032+ , URI.authPort = Nothing
814 s -> s `L.elem` [ "git", "https", "http" ] -- "http:" just redirects to "https:"
1034 s -> s `L.elem` [ "git", "https", "http" ] -- "http:" just redirects to "https:"
815− && unRText (authHost uriAuth) == "github.com"
1035+ && URI.unRText (URI.authHost uriAuth) == "github.com"
816− , Just (_, owner :| [repoish]) <- uriPath u
1036+ , Just (_, owner :| [repoish]) <- URI.uriPath u
817− = \mbranch -> ThunkSource_GitHub $ GitHubSource
1037+ = \mbranch -> do
818− { _gitHubSource_owner = N $ unRText owner
1038+ isPrivate <- getIsPrivate
819− , _gitHubSource_repo = N $ let
1039+ pure $ ThunkSource_GitHub $ GitHubSource
820− repoish' = unRText repoish
1040+ { _gitHubSource_owner = N $ URI.unRText owner
821− in case T.stripSuffix ".git" repoish' of
1041+ , _gitHubSource_repo = N $ let
822− Just repo -> repo
1042+ repoish' = URI.unRText repoish
823− Nothing -> repoish'
1043+ in fromMaybe repoish' $ T.stripSuffix ".git" repoish'
824− , _gitHubSource_branch = N <$> mbranch
1044+ , _gitHubSource_branch = N <$> mbranch
825− }
1045+ , _gitHubSource_private = isPrivate
827− | otherwise = \mbranch -> ThunkSource_Git $ GitSource
1048+ | otherwise = \mbranch -> do
828− { _gitSource_url = u
1049+ isPrivate <- getIsPrivate
829− , _gitSource_branch = N <$> mbranch
1050+ pure $ ThunkSource_Git $ GitSource
830− , _gitSource_fetchSubmodules = False -- TODO: How do we determine if this should be true?
1051+ { _gitSource_url = GitUri u
831− }
1052+ , _gitSource_branch = N <$> mbranch
1053+ , _gitSource_fetchSubmodules = False -- TODO: How do we determine if this should be true?
1054+ , _gitSource_private = isPrivate
1057+ getIsPrivate = maybe (guessGitRepoIsPrivate $ GitUri u) pure mPrivate
1059+guessGitRepoIsPrivate :: MonadObelisk m => GitUri -> m Bool
1060+guessGitRepoIsPrivate uri = flip fix urisToTry $ \loop -> \case
1062+ uriAttempt:xs -> do
1063+ result <- readCreateProcessWithExitCode $
1070+ , T.unpack $ gitUriToText uriAttempt
1073+ (ExitSuccess, _, _) -> pure False -- Must be a public repo
1076+ urisToTry = nubOrd $
1077+ -- Include the original URI if it isn't using SSH because SSH will certainly fail.
1078+ [uri | fmap URI.unRText (URI.uriScheme (unGitUri uri)) /= Just "ssh"] <>
1079+ [changeScheme "https" uri, changeScheme "http" uri]
1080+ changeScheme scheme (GitUri u) = GitUri $ u
1081+ { URI.uriScheme = URI.mkScheme scheme
1082+ , URI.uriAuthority = (\x -> x { URI.authUserInfo = Nothing }) <$> URI.uriAuthority u
833-- Funny signature indicates no effects depend on the optional branch name.
1085-- Funny signature indicates no effects depend on the optional branch name.
834githubThunkRev
1086githubThunkRev
@@ -840,19 +1092,19 @@ githubThunkRev
840githubThunkRev s commit = do
1092githubThunkRev s commit = do
841 owner <- forcePP $ _gitHubSource_owner s
1093 owner <- forcePP $ _gitHubSource_owner s
842 repo <- forcePP $ _gitHubSource_repo s
1094 repo <- forcePP $ _gitHubSource_repo s
843− revTarball <- mkPathPiece $ commit <> ".tar.gz"
1095+ revTarball <- URI.mkPathPiece $ commit <> ".tar.gz"
844− let archiveUri = URI
1096+ let archiveUri = GitUri $ URI.URI
845− { uriScheme = Just $ fromRight' $ mkScheme "https"
1097+ { URI.uriScheme = Just $ fromRight' $ URI.mkScheme "https"
846− , uriAuthority = Right $ Authority
1098+ , URI.uriAuthority = Right $ URI.Authority
847− { authUserInfo = Nothing
1099+ { URI.authUserInfo = Nothing
848− , authHost = fromRight' $ mkHost "github.com"
1100+ , URI.authHost = fromRight' $ URI.mkHost "github.com"
849− , authPort = Nothing
1101+ , URI.authPort = Nothing
851− , uriPath = Just ( False
1103+ , URI.uriPath = Just ( False
852− , owner :| [ repo, fromRight' $ mkPathPiece "archive", revTarball ]
1104+ , owner :| [ repo, fromRight' $ URI.mkPathPiece "archive", revTarball ]
854− , uriQuery = []
1106+ , URI.uriQuery = []
855− , uriFragment = Nothing
1107+ , URI.uriFragment = Nothing
857 hash <- getNixSha256ForUriUnpacked archiveUri
1109 hash <- getNixSha256ForUriUnpacked archiveUri
858 putLog Debug $ "Nix sha256 is " <> hash
1110 putLog Debug $ "Nix sha256 is " <> hash
@@ -861,8 +1113,8 @@ githubThunkRev s commit = do
861 , _thunkRev_nixSha256 = hash
1113 , _thunkRev_nixSha256 = hash
864− forcePP :: Name entity -> m (RText 'PathPiece)
1116+ forcePP :: Name entity -> m (URI.RText 'URI.PathPiece)
865− forcePP = mkPathPiece . untagName
1117+ forcePP = URI.mkPathPiece . untagName
867gitThunkRev
1119gitThunkRev
868 :: MonadObelisk m
1120 :: MonadObelisk m
@@ -871,11 +1123,11 @@ gitThunkRev
871 -> m ThunkRev
1123 -> m ThunkRev
872gitThunkRev s commit = do
1124gitThunkRev s commit = do
873 let u = _gitSource_url s
1125 let u = _gitSource_url s
874− protocols = ["https", "ssh", "git"]
1126+ protocols = ["file", "https", "ssh", "git"]
875− Just scheme <- pure $ unRText <$> uriScheme u
1127+ scheme = maybe "file" URI.unRText $ URI.uriScheme $ (\(GitUri x) -> x) u
876 unless (T.toLower scheme `elem` protocols) $
1128 unless (T.toLower scheme `elem` protocols) $
877 failWith $ "obelisk currently only supports "
1129 failWith $ "obelisk currently only supports "
878− <> T.intercalate ", " protocols <> " protocols for non-GitHub remotes"
1130+ <> T.intercalate ", " protocols <> " protocols for plain Git remotes"
879 hash <- nixPrefetchGit u commit $ _gitSource_fetchSubmodules s
1131 hash <- nixPrefetchGit u commit $ _gitSource_fetchSubmodules s
880 putLog Informational $ "Nix sha256 is " <> hash
1132 putLog Informational $ "Nix sha256 is " <> hash
881 pure $ ThunkRev
1133 pure $ ThunkRev
@@ -888,11 +1140,12 @@ gitThunkRev s commit = do
889-- If the branch name is passed in, it is returned exactly as-is. If it is not
1141-- If the branch name is passed in, it is returned exactly as-is. If it is not
890-- passed it, the default branch of the repo is used instead.
1142-- passed it, the default branch of the repo is used instead.
891gitGetCommitBranch
1144gitGetCommitBranch
892− :: MonadObelisk m => URI -> Maybe Text -> m (Text, CommitId)
1145+ :: MonadObelisk m => GitUri -> Maybe Text -> m (Text, CommitId)
893gitGetCommitBranch uri mbranch = withExitFailMessage ("Failure for git remote " <> uriMsg) $ do
1146gitGetCommitBranch uri mbranch = withExitFailMessage ("Failure for git remote " <> uriMsg) $ do
894 (_, bothMaps) <- gitLsRemote
1147 (_, bothMaps) <- gitLsRemote
895− (T.unpack $ render uri)
1148+ (T.unpack $ gitUriToText uri)
896 (GitRef_Branch <$> mbranch)
1149 (GitRef_Branch <$> mbranch)
898 branch <- case mbranch of
1151 branch <- case mbranch of
@@ -908,18 +1161,21 @@ gitGetCommitBranch uri mbranch = withExitFailMessage ("Failure for git remote "
908 pure (branch, commit)
1161 pure (branch, commit)
910 rethrowE = either failWith pure
1163 rethrowE = either failWith pure
911− uriMsg = render uri
1164+ uriMsg = gitUriToText uri
1166+parseGitUri :: Text -> Maybe GitUri
1167+parseGitUri x = GitUri <$> (parseFileURI x <|> parseAbsoluteURI x <|> parseSshShorthand x)
913−parseGitUri :: Text -> Maybe URI
1169+parseFileURI :: Text -> Maybe URI.URI
914−parseGitUri x = parseAbsoluteURI x <|> parseSshShorthand x
1170+parseFileURI uri = if "/" `T.isPrefixOf` uri then parseAbsoluteURI ("file://" <> uri) else Nothing
916−parseAbsoluteURI :: Text -> Maybe URI
1172+parseAbsoluteURI :: Text -> Maybe URI.URI
917parseAbsoluteURI uri = do
1173parseAbsoluteURI uri = do
918− parsedUri <- mkURI uri
1174+ parsedUri <- URI.mkURI uri
919− guard $ isPathAbsolute parsedUri
1175+ guard $ URI.isPathAbsolute parsedUri
920− pure $ parsedUri
1176+ pure parsedUri
922−parseSshShorthand :: Text -> Maybe URI
1178+parseSshShorthand :: Text -> Maybe URI.URI
923parseSshShorthand uri = do
1179parseSshShorthand uri = do
924 -- This is what git does to check that the remote
1180 -- This is what git does to check that the remote
925 -- is not a local file path when parsing shorthand.
1181 -- is not a local file path when parsing shorthand.
@@ -932,4 +1188,4 @@ parseSshShorthand uri = do
932 -- This check is used to disambiguate a filepath containing a colon from shorthand
1188 -- This check is used to disambiguate a filepath containing a colon from shorthand
933 guard $ isNothing (T.findIndex (=='/') authAndHostname)
1189 guard $ isNothing (T.findIndex (=='/') authAndHostname)
934 && not (T.null colonAndPath)
1190 && not (T.null colonAndPath)
935− mkURI properUri
1191+ URI.mkURI properUri