External package dependencies (#2272)

This PR adds external git dependency support to the Juvix package
format.

## New dependency Git item

You can now add a `git` block to the dependencies list:

```yaml
name: HelloWorld
main: HelloWorld.juvix
dependencies:
  - .juvix-build/stdlib
  - git:
      url: https://my.git.repo
      name: myGitRepo
      ref: main
version: 0.1.0
```

Git block required fields:
* `url`: The URL of the git repository
* `ref`: The git reference that should be checked out
* `name`: The name for the dependency. This is used to name the
directory of the clone, it is required. Perhaps we could come up with a
way to automatically name the clone directory. Current ideas are to
somehow encode the URL / ref combination or use a UUID. However there's
some value in having the clone directory named in a friendly way.

NB:
* The values of the `name` fields must be unique among the git blocks in
the dependencies list.

## Behaviour

When dependencies for a package are registered, at the beginning of the
compiler pipeline, all remote dependencies are processed:

1. If it doesn't already exist, the remote dependency is cloned to
`.juvix-build/deps/$name`
2. `git fetch` is run in the clone
3. `git checkout` at the specified `ref` is run in the clone

The clone is then processed by the PathResolver in the same way as path
dependencies.

NB:
* Remote dependencies of transitive dependencies are also processed.
* The `git fetch` step is required for the case where the remote is
updated. In this case we want the user to be able to update the `ref`
field.

## Errors

1. Missing fields in the Git dependency block are YAML parse errors
2. Duplicate `name` values in the dependencies list is an error thrown
when the package file is processed
3. The `ref` doesn't exist in the clone or the clone directory is
otherwise corrupt. An error with a suggestion to `juvix clean` is given.
The package file path is used as the location in the error message.
4. Other `git` command errors (command not found, etc.), a more verbose
error is given with the arguments that were passed to the git command.

## Future work

1. Add an offline mode
2. Add a lock file mechanism that resolves branch/tag git refs to commit
hashes

* closes https://github.com/anoma/juvix/issues/2083

---------

Co-authored-by: Jan Mas Rovira <janmasrovira@gmail.com>

.github/workflows/ci.yml

@@ -151,6 +151,14 @@ jobs:
           extension-matching: disable
           cache: enable
 
+      # Smoke tests make git commits
+      - name: Setup git
+        shell: bash
+        run: |
+          git config --global user.email "tara-juvix@heliax.dev"
+          git config --global user.name "Tara"
+          git config --global init.defaultBranch main
+
       - name: Smoke testing
         id: smoke-linux
         if: ${{ success() }}
@@ -287,6 +295,14 @@ jobs:
           cd main
           make check-format-juvix-files && make typecheck-juvix-examples
 
+      # Smoke tests make git commits
+      - name: Setup git
+        shell: bash
+        run: |
+          git config --global user.email "tara-juvix@heliax.dev"
+          git config --global user.name "Tara"
+          git config --global init.defaultBranch main
+
       - name: Smoke testing (macOS)
         id: smoke-macos
         if: ${{ success() }}

app/Commands/Init.hs

@@ -44,13 +44,15 @@ getPackage = do
   tproj <- getProjName
   say "Write the version of your project [leave empty for 0.0.0]"
   tversion :: SemVer <- getVersion
+  cwd <- getCurrentDir
   return
     Package
       { _packageName = tproj,
         _packageVersion = tversion,
         _packageBuildDir = Nothing,
         _packageMain = Nothing,
-        _packageDependencies = [defaultStdlibDep DefaultBuildDir]
+        _packageDependencies = [defaultStdlibDep DefaultBuildDir],
+        _packageFile = cwd <//> juvixYamlFile
       }
 
 getProjName :: forall r. (Members '[Embed IO] r) => Sem r Text

app/Commands/Repl.hs

@@ -21,6 +21,7 @@ import Juvix.Compiler.Concrete.Data.ScopedName qualified as Scoped
 import Juvix.Compiler.Concrete.Language qualified as Concrete
 import Juvix.Compiler.Concrete.Pretty qualified as Concrete
 import Juvix.Compiler.Concrete.Translation.FromParsed.Analysis.PathResolver (runPathResolver)
+import Juvix.Compiler.Concrete.Translation.FromParsed.Analysis.PathResolver.Error
 import Juvix.Compiler.Core qualified as Core
 import Juvix.Compiler.Core.Extra.Value
 import Juvix.Compiler.Core.Info qualified as Info
@@ -32,6 +33,8 @@ import Juvix.Compiler.Internal.Language qualified as Internal
 import Juvix.Compiler.Internal.Pretty qualified as Internal
 import Juvix.Compiler.Pipeline.Repl
 import Juvix.Compiler.Pipeline.Setup (entrySetup)
+import Juvix.Data.Effect.Git
+import Juvix.Data.Effect.Process
 import Juvix.Data.Error.GenericError qualified as Error
 import Juvix.Data.NameKind
 import Juvix.Extra.Paths
@@ -143,6 +146,11 @@ loadDefaultPrelude = whenJustM defaultPreludeEntryPoint $ \e -> do
     . runFilesIO
     . runError @Text
     . runReader e
+    . runLogIO
+    . runProcessIO
+    . runError @GitProcessError
+    . runGitProcess
+    . runError @DependencyError
     . runPathResolver root
     $ entrySetup
   loadEntryPoint e

package.yaml

@@ -80,6 +80,7 @@ dependencies:
   - th-utilities == 0.2.*
   - time == 1.12.*
   - transformers == 0.5.*
+  - typed-process == 0.2.*
   - unicode-show == 0.1.*
   - uniplate == 1.6.*
   - unix-compat == 0.7.*

src/Juvix/Compiler/Concrete/Translation/FromParsed/Analysis/PathResolver.hs

@@ -20,11 +20,13 @@ where
 
 import Data.HashMap.Strict qualified as HashMap
 import Data.HashSet qualified as HashSet
+import Data.Text qualified as T
 import Juvix.Compiler.Concrete.Data.Name
 import Juvix.Compiler.Concrete.Translation.FromParsed.Analysis.PathResolver.Base
 import Juvix.Compiler.Concrete.Translation.FromParsed.Analysis.PathResolver.Error
 import Juvix.Compiler.Concrete.Translation.FromParsed.Analysis.PathResolver.PackageInfo
 import Juvix.Compiler.Pipeline.EntryPoint
+import Juvix.Data.Effect.Git
 import Juvix.Extra.Paths
 import Juvix.Extra.Stdlib (ensureStdlib)
 import Juvix.Prelude
@@ -66,21 +68,30 @@ iniResolverState =
 withEnvRoot :: (Members '[Reader ResolverEnv] r) => Path Abs Dir -> Sem r a -> Sem r a
 withEnvRoot root' = local (set envRoot root')
 
+mkPackage ::
+  forall r.
+  (Members '[Files, Error Text, Reader ResolverEnv, GitClone] r) =>
+  Maybe EntryPoint ->
+  Path Abs Dir ->
+  Sem r Package
+mkPackage mpackageEntry _packageRoot = do
+  let buildDir :: Path Abs Dir = maybe (rootBuildDir _packageRoot) (someBaseToAbs _packageRoot . (^. entryPointBuildDir)) mpackageEntry
+      buildDirDep :: BuildDir
+        | isJust mpackageEntry = CustomBuildDir (Abs buildDir)
+        | otherwise = DefaultBuildDir
+  maybe (readPackage _packageRoot buildDirDep) (return . (^. entryPointPackage)) mpackageEntry
+
 mkPackageInfo ::
   forall r.
-  (Members '[Files, Error Text, Reader ResolverEnv] r) =>
+  (Members '[Files, Error Text, Reader ResolverEnv, Error DependencyError, GitClone] r) =>
   Maybe EntryPoint ->
   Path Abs Dir ->
   Sem r PackageInfo
 mkPackageInfo mpackageEntry _packageRoot = do
   let buildDir :: Path Abs Dir = maybe (rootBuildDir _packageRoot) (someBaseToAbs _packageRoot . (^. entryPointBuildDir)) mpackageEntry
-      buildDirDep :: BuildDir
-        | isJust mpackageEntry = CustomBuildDir (Abs buildDir)
-        | otherwise = DefaultBuildDir
-
-  _packagePackage <- maybe (readPackage _packageRoot buildDirDep) (return . (^. entryPointPackage)) mpackageEntry
+  _packagePackage <- mkPackage mpackageEntry _packageRoot
   let deps :: [Dependency] = _packagePackage ^. packageDependencies
-  depsPaths <- mapM getDependencyPath deps
+  depsPaths <- mapM (getDependencyPath . mkPackageDependencyInfo (_packagePackage ^. packageFile)) deps
   ensureStdlib _packageRoot buildDir deps
   files :: [Path Rel File] <-
     map (fromJust . stripProperPrefix _packageRoot) <$> walkDirRelAccum juvixAccum _packageRoot []
@@ -95,21 +106,42 @@ mkPackageInfo mpackageEntry _packageRoot = do
         newJuvixFiles :: [Path Abs File]
         newJuvixFiles = [cd <//> f | f <- files, isJuvixFile f]
 
-dependencyCached :: (Members '[State ResolverState, Reader ResolverEnv, Files] r) => Dependency -> Sem r Bool
-dependencyCached d = do
-  p <- getDependencyPath d
-  HashMap.member p <$> gets (^. resolverPackages)
+dependencyCached :: (Members '[State ResolverState, Reader ResolverEnv, Files, GitClone] r) => Path Abs Dir -> Sem r Bool
+dependencyCached p = HashMap.member p <$> gets (^. resolverPackages)
 
 withPathFile :: (Members '[PathResolver] r) => TopModulePath -> (Either PathResolverError (Path Abs File) -> Sem r a) -> Sem r a
 withPathFile m f = withPath m (f . mapRight (uncurry (<//>)))
 
-getDependencyPath :: (Members '[Reader ResolverEnv, Files] r) => Dependency -> Sem r (Path Abs Dir)
-getDependencyPath (Dependency p) = do
-  r <- asks (^. envRoot)
-  canonicalDir r p
+getDependencyPath :: forall r. (Members '[Reader ResolverEnv, Files, Error DependencyError, GitClone] r) => PackageDependencyInfo -> Sem r (Path Abs Dir)
+getDependencyPath i = case i ^. packageDepdendencyInfoDependency of
+  DependencyPath p -> do
+    r <- asks (^. envRoot)
+    canonicalDir r (p ^. pathDependencyPath)
+  DependencyGit g -> do
+    r <- rootBuildDir <$> asks (^. envRoot)
+    let cloneDir = r <//> relDependenciesDir <//> relDir (T.unpack (g ^. gitDependencyName))
+        cloneArgs = CloneArgs {_cloneArgsCloneDir = cloneDir, _cloneArgsRepoUrl = g ^. gitDependencyUrl}
+        errorHandler' = errorHandler cloneDir
+    scoped @CloneArgs @Git cloneArgs $ do
+      fetch errorHandler'
+      checkout errorHandler' (g ^. gitDependencyRef)
+      return cloneDir
+    where
+      errorHandler :: Path Abs Dir -> GitError -> Sem (Git ': r) a
+      errorHandler p c =
+        throw
+          DependencyError
+            { _dependencyErrorCause =
+                GitDependencyError
+                  DependencyErrorGit
+                    { _dependencyErrorGitCloneDir = p,
+                      _dependencyErrorGitError = c
+                    },
+              _dependencyErrorPackageFile = i ^. packageDependencyInfoPackageFile
+            }
 
 registerDependencies' ::
-  (Members '[Reader EntryPoint, State ResolverState, Reader ResolverEnv, Files, Error Text] r) =>
+  (Members '[Reader EntryPoint, State ResolverState, Reader ResolverEnv, Files, Error Text, Error DependencyError, GitClone] r) =>
   Sem r ()
 registerDependencies' = do
   e <- ask @EntryPoint
@@ -117,30 +149,34 @@ registerDependencies' = do
   if
       | isGlobal -> do
           glob <- globalRoot
-          let globDep = Dependency (mkPrepath (toFilePath glob))
-          addDependency' (Just e) globDep
-      | otherwise -> addDependency' (Just e) (Dependency (mkPrepath (toFilePath (e ^. entryPointRoot))))
+          let globDep = mkPathDependency (toFilePath glob)
+              globalPackageFile = mkPackageFilePath glob
+          addDependency' (Just e) (mkPackageDependencyInfo globalPackageFile globDep)
+      | otherwise -> do
+          let f = mkPackageFilePath (e ^. entryPointRoot)
+          addDependency' (Just e) (mkPackageDependencyInfo f (mkPathDependency (toFilePath (e ^. entryPointRoot))))
 
 addDependency' ::
-  (Members '[State ResolverState, Reader ResolverEnv, Files, Error Text] r) =>
+  (Members '[State ResolverState, Reader ResolverEnv, Files, Error Text, Error DependencyError, GitClone] r) =>
   Maybe EntryPoint ->
-  Dependency ->
+  PackageDependencyInfo ->
   Sem r ()
 addDependency' me = addDependencyHelper me
 
 addDependencyHelper ::
-  (Members '[State ResolverState, Reader ResolverEnv, Files, Error Text] r) =>
+  (Members '[State ResolverState, Reader ResolverEnv, Files, Error Text, Error DependencyError, GitClone] r) =>
   Maybe EntryPoint ->
-  Dependency ->
+  PackageDependencyInfo ->
   Sem r ()
 addDependencyHelper me d = do
   p <- getDependencyPath d
-  unlessM (dependencyCached d) $ withEnvRoot p $ do
+  unlessM (dependencyCached p) $ withEnvRoot p $ do
     pkgInfo <- mkPackageInfo me p
     modify' (set (resolverPackages . at p) (Just pkgInfo))
     forM_ (pkgInfo ^. packageRelativeFiles) $ \f -> do
       modify' (over resolverFiles (HashMap.insertWith (<>) f (pure pkgInfo)))
-    forM_ (pkgInfo ^. packagePackage . packageDependencies) (addDependency' Nothing)
+    let packagePath = pkgInfo ^. packagePackage . packageFile
+    forM_ (pkgInfo ^. packagePackage . packageDependencies) (addDependency' Nothing . mkPackageDependencyInfo packagePath)
 
 currentPackage :: (Members '[State ResolverState, Reader ResolverEnv] r) => Sem r PackageInfo
 currentPackage = do
@@ -186,7 +222,7 @@ expectedPath' actualPath m = do
 
 re ::
   forall r a.
-  (Members '[Reader EntryPoint, Files, Error Text] r) =>
+  (Members '[Reader EntryPoint, Files, Error Text, Error DependencyError, GitClone] r) =>
   Sem (PathResolver ': r) a ->
   Sem (Reader ResolverEnv ': State ResolverState ': r) a
 re = reinterpret2H helper
@@ -209,13 +245,13 @@ re = reinterpret2H helper
               Right (r, _) -> r
         raise (evalPathResolver' st' root' (a' x'))
 
-evalPathResolver' :: (Members '[Reader EntryPoint, Files, Error Text] r) => ResolverState -> Path Abs Dir -> Sem (PathResolver ': r) a -> Sem r a
+evalPathResolver' :: (Members '[Reader EntryPoint, Files, Error Text, Error DependencyError, GitClone] r) => ResolverState -> Path Abs Dir -> Sem (PathResolver ': r) a -> Sem r a
 evalPathResolver' st root = fmap snd . runPathResolver' st root
 
-runPathResolver :: (Members '[Reader EntryPoint, Files, Error Text] r) => Path Abs Dir -> Sem (PathResolver ': r) a -> Sem r (ResolverState, a)
+runPathResolver :: (Members '[Reader EntryPoint, Files, Error Text, Error DependencyError, GitClone] r) => Path Abs Dir -> Sem (PathResolver ': r) a -> Sem r (ResolverState, a)
 runPathResolver = runPathResolver' iniResolverState
 
-runPathResolver' :: (Members '[Reader EntryPoint, Files, Error Text] r) => ResolverState -> Path Abs Dir -> Sem (PathResolver ': r) a -> Sem r (ResolverState, a)
+runPathResolver' :: (Members '[Reader EntryPoint, Files, Error Text, Error DependencyError, GitClone] r) => ResolverState -> Path Abs Dir -> Sem (PathResolver ': r) a -> Sem r (ResolverState, a)
 runPathResolver' st root x = do
   e <- ask
   let _envSingleFile :: Maybe (Path Abs File)
@@ -230,15 +266,15 @@ runPathResolver' st root x = do
           }
   runState st (runReader env (re x))
 
-runPathResolverPipe' :: (Members '[Files, Reader EntryPoint] r) => ResolverState -> Sem (PathResolver ': r) a -> Sem r (ResolverState, a)
+runPathResolverPipe' :: (Members '[Files, Reader EntryPoint, Error DependencyError, GitClone] r) => ResolverState -> Sem (PathResolver ': r) a -> Sem r (ResolverState, a)
 runPathResolverPipe' iniState a = do
   r <- asks (^. entryPointResolverRoot)
   runError (runPathResolver' iniState r (raiseUnder a)) >>= either error return
 
-runPathResolverPipe :: (Members '[Files, Reader EntryPoint] r) => Sem (PathResolver ': r) a -> Sem r (ResolverState, a)
+runPathResolverPipe :: (Members '[Files, Reader EntryPoint, Error DependencyError, GitClone] r) => Sem (PathResolver ': r) a -> Sem r (ResolverState, a)
 runPathResolverPipe a = do
   r <- asks (^. entryPointResolverRoot)
   runError (runPathResolver r (raiseUnder a)) >>= either error return
 
-evalPathResolverPipe :: (Members '[Files, Reader EntryPoint] r) => Sem (PathResolver ': r) a -> Sem r a
+evalPathResolverPipe :: (Members '[Files, Reader EntryPoint, Error DependencyError, GitClone] r) => Sem (PathResolver ': r) a -> Sem r a
 evalPathResolverPipe = fmap snd . runPathResolverPipe

src/Juvix/Compiler/Concrete/Translation/FromParsed/Analysis/PathResolver/Error.hs

@@ -5,8 +5,64 @@ import Juvix.Compiler.Concrete.Translation.FromParsed.Analysis.PathResolver.Base
 import Juvix.Compiler.Concrete.Translation.FromParsed.Analysis.PathResolver.PackageInfo
 import Juvix.Compiler.Pipeline.Package
 import Juvix.Data.CodeAnn
+import Juvix.Data.Effect.Git
 import Juvix.Prelude
 
+data DependencyErrorGit = DependencyErrorGit
+  { _dependencyErrorGitError :: GitError,
+    _dependencyErrorGitCloneDir :: Path Abs Dir
+  }
+
+newtype DependencyErrorCause = GitDependencyError DependencyErrorGit
+
+data DependencyError = DependencyError
+  { _dependencyErrorPackageFile :: Path Abs File,
+    _dependencyErrorCause :: DependencyErrorCause
+  }
+
+makeLenses ''DependencyError
+makeLenses ''DependencyErrorGit
+
+instance ToGenericError DependencyError where
+  genericError e = do
+    let msg = ppCodeAnn (e ^. dependencyErrorCause)
+    return
+      ( GenericError
+          { _genericErrorMessage = mkAnsiText msg,
+            _genericErrorLoc = i,
+            _genericErrorIntervals = [i]
+          }
+      )
+    where
+      i = getLoc e
+
+instance HasLoc DependencyError where
+  getLoc e = singletonInterval (mkInitialLoc (e ^. dependencyErrorPackageFile))
+
+instance PrettyCodeAnn DependencyErrorCause where
+  ppCodeAnn = \case
+    GitDependencyError e -> ppCodeAnn e
+
+instance PrettyCodeAnn DependencyErrorGit where
+  ppCodeAnn d = case d ^. dependencyErrorGitError of
+    NotAClone ->
+      prefix
+        <> "The directory"
+        <+> code (pretty (d ^. dependencyErrorGitCloneDir))
+        <+> "is not a valid git clone."
+          <> line
+          <> "Try running"
+        <+> code "juvix clean"
+    NoSuchRef ref ->
+      prefix
+        <> "The git ref:"
+        <+> code (pretty ref)
+        <+> "does not exist in the clone:"
+        <+> code (pretty (d ^. dependencyErrorGitCloneDir))
+    where
+      prefix :: Doc CodeAnn
+      prefix = pretty @Text "Failed to obtain remote dependencies" <> line
+
 data PathResolverError
   = ErrDependencyConflict DependencyConflict
   | ErrMissingModule MissingModule

src/Juvix/Compiler/Concrete/Translation/FromParsed/Analysis/Scoping.hs

@@ -29,7 +29,7 @@ import Juvix.Compiler.Pipeline.EntryPoint
 import Juvix.Data.FixityInfo qualified as FI
 import Juvix.Data.IteratorAttribs
 import Juvix.Data.NameKind
-import Juvix.Prelude
+import Juvix.Prelude hiding (scoped)
 
 iniScoperState :: ScoperState
 iniScoperState =

src/Juvix/Compiler/Pipeline.hs

@@ -37,84 +37,86 @@ import Juvix.Compiler.Pipeline.Root
 import Juvix.Compiler.Pipeline.Setup
 import Juvix.Compiler.Reg.Data.InfoTable qualified as Reg
 import Juvix.Compiler.Reg.Translation.FromAsm qualified as Reg
+import Juvix.Data.Effect.Git
+import Juvix.Data.Effect.Process
 import Juvix.Prelude
 
-type PipelineEff = '[PathResolver, Reader EntryPoint, Files, NameIdGen, Builtins, Error JuvixError, HighlightBuilder, Embed IO]
+type PipelineEff = '[PathResolver, Error DependencyError, GitClone, Error GitProcessError, Process, Log, Reader EntryPoint, Files, NameIdGen, Builtins, Error JuvixError, HighlightBuilder, Embed IO]
 
-type TopPipelineEff = '[PathResolver, Reader EntryPoint, Files, NameIdGen, Builtins, State Artifacts, Error JuvixError, HighlightBuilder, Embed IO]
+type TopPipelineEff = '[PathResolver, Error DependencyError, GitClone, Error GitProcessError, Process, Log, Reader EntryPoint, Files, NameIdGen, Builtins, State Artifacts, Error JuvixError, HighlightBuilder, Embed IO]
 
 --------------------------------------------------------------------------------
 -- Workflows
 --------------------------------------------------------------------------------
 
 upToParsing ::
-  (Members '[HighlightBuilder, Reader EntryPoint, Files, Error JuvixError, NameIdGen, PathResolver] r) =>
+  (Members '[HighlightBuilder, Reader EntryPoint, Files, Error JuvixError, NameIdGen, GitClone, PathResolver] r) =>
   Sem r Parser.ParserResult
 upToParsing = entrySetup >> ask >>= Parser.fromSource
 
 upToScoping ::
-  (Members '[HighlightBuilder, Reader EntryPoint, Files, NameIdGen, Error JuvixError, PathResolver] r) =>
+  (Members '[HighlightBuilder, Reader EntryPoint, Files, NameIdGen, Error JuvixError, GitClone, PathResolver] r) =>
   Sem r Scoper.ScoperResult
 upToScoping = upToParsing >>= Scoper.fromParsed
 
 upToInternal ::
-  (Members '[HighlightBuilder, Reader EntryPoint, Files, NameIdGen, Builtins, Error JuvixError, PathResolver, Termination] r) =>
+  (Members '[HighlightBuilder, Reader EntryPoint, Files, NameIdGen, Builtins, Error JuvixError, GitClone, PathResolver, Termination] r) =>
   Sem r Internal.InternalResult
 upToInternal = upToScoping >>= Internal.fromConcrete
 
 upToInternalArity ::
-  (Members '[HighlightBuilder, Reader EntryPoint, Files, NameIdGen, Builtins, Error JuvixError, PathResolver, Termination] r) =>
+  (Members '[HighlightBuilder, Reader EntryPoint, Files, NameIdGen, Builtins, Error JuvixError, GitClone, PathResolver, Termination] r) =>
   Sem r Internal.InternalArityResult
 upToInternalArity = upToInternal >>= Internal.arityChecking
 
 upToInternalTyped ::
-  (Members '[HighlightBuilder, Reader EntryPoint, Files, NameIdGen, Error JuvixError, Builtins, PathResolver] r) =>
+  (Members '[HighlightBuilder, Reader EntryPoint, Files, NameIdGen, Error JuvixError, Builtins, GitClone, PathResolver] r) =>
   Sem r Internal.InternalTypedResult
 upToInternalTyped = Internal.typeChecking upToInternalArity
 
 upToInternalReachability ::
-  (Members '[HighlightBuilder, Reader EntryPoint, Files, NameIdGen, Error JuvixError, Builtins, PathResolver] r) =>
+  (Members '[HighlightBuilder, Reader EntryPoint, Files, NameIdGen, Error JuvixError, Builtins, GitClone, PathResolver] r) =>
   Sem r Internal.InternalTypedResult
 upToInternalReachability =
   upToInternalTyped >>= Internal.filterUnreachable
 
 upToCore ::
-  (Members '[HighlightBuilder, Reader EntryPoint, Files, NameIdGen, Error JuvixError, Builtins, PathResolver] r) =>
+  (Members '[HighlightBuilder, Reader EntryPoint, Files, NameIdGen, Error JuvixError, Builtins, GitClone, PathResolver] r) =>
   Sem r Core.CoreResult
 upToCore = upToInternalReachability >>= Core.fromInternal
 
 upToAsm ::
-  (Members '[HighlightBuilder, Reader EntryPoint, Files, NameIdGen, Error JuvixError, Builtins, PathResolver] r) =>
+  (Members '[HighlightBuilder, Reader EntryPoint, Files, NameIdGen, Error JuvixError, Builtins, GitClone, PathResolver] r) =>
   Sem r Asm.InfoTable
 upToAsm =
   upToCore >>= \Core.CoreResult {..} -> coreToAsm _coreResultTable
 
 upToMiniC ::
-  (Members '[HighlightBuilder, Reader EntryPoint, Files, NameIdGen, Error JuvixError, Builtins, PathResolver] r) =>
+  (Members '[HighlightBuilder, Reader EntryPoint, Files, NameIdGen, Error JuvixError, Builtins, GitClone, PathResolver] r) =>
   Sem r C.MiniCResult
 upToMiniC = upToAsm >>= asmToMiniC
 
 upToVampIR ::
-  (Members '[HighlightBuilder, Reader EntryPoint, Files, NameIdGen, Error JuvixError, Builtins, PathResolver] r) =>
+  (Members '[HighlightBuilder, Reader EntryPoint, Files, NameIdGen, Error JuvixError, Builtins, GitClone, PathResolver] r) =>
   Sem r VampIR.Result
 upToVampIR =
   upToCore >>= \Core.CoreResult {..} -> coreToVampIR _coreResultTable
 
 upToGeb ::
-  (Members '[HighlightBuilder, Reader EntryPoint, Files, NameIdGen, Error JuvixError, Builtins, PathResolver] r) =>
+  (Members '[HighlightBuilder, Reader EntryPoint, Files, NameIdGen, Error JuvixError, Builtins, GitClone, PathResolver] r) =>
   Geb.ResultSpec ->
   Sem r Geb.Result
 upToGeb spec =
   upToCore >>= \Core.CoreResult {..} -> coreToGeb spec _coreResultTable
 
 upToCoreTypecheck ::
-  (Members '[HighlightBuilder, Reader EntryPoint, Files, NameIdGen, Error JuvixError, Builtins, PathResolver] r) =>
+  (Members '[HighlightBuilder, Reader EntryPoint, Files, NameIdGen, Error JuvixError, Builtins, GitClone, PathResolver] r) =>
   Sem r Core.CoreResult
 upToCoreTypecheck =
   upToCore >>= \r -> Core.toTypechecked (r ^. Core.coreResultTable) >>= \tab -> return r {Core._coreResultTable = tab}
 
 upToEval ::
-  (Members '[HighlightBuilder, Reader EntryPoint, Files, NameIdGen, Error JuvixError, Builtins, PathResolver] r) =>
+  (Members '[HighlightBuilder, Reader EntryPoint, Files, NameIdGen, Error JuvixError, Builtins, GitClone, PathResolver] r) =>
   Sem r Core.CoreResult
 upToEval =
   upToCore >>= \r -> Core.toEval (r ^. Core.coreResultTable) >>= \tab -> return r {Core._coreResultTable = tab}
@@ -178,6 +180,11 @@ runIOEitherHelper entry =
     . evalTopNameIdGen
     . runFilesIO
     . runReader entry
+    . runLogIO
+    . runProcessIO
+    . mapError (JuvixError @GitProcessError)
+    . runGitProcess
+    . mapError (JuvixError @DependencyError)
     . runPathResolverPipe
 
 runIO :: GenericOptions -> EntryPoint -> Sem PipelineEff a -> IO (ResolverState, a)
@@ -215,6 +222,11 @@ corePipelineIOEither entry = do
       . runNameIdGenArtifacts
       . runFilesIO
       . runReader entry
+      . runLogIO
+      . mapError (JuvixError @GitProcessError)
+      . runProcessIO
+      . runGitProcess
+      . mapError (JuvixError @DependencyError)
       . runPathResolverArtifacts
       $ upToCore
   return $ case eith of

src/Juvix/Compiler/Pipeline/Artifacts.hs

@@ -22,6 +22,7 @@ import Juvix.Compiler.Internal.Translation.FromConcrete qualified as Internal
 import Juvix.Compiler.Internal.Translation.FromInternal.Analysis.Termination.Checker
 import Juvix.Compiler.Internal.Translation.FromInternal.Analysis.TypeChecking.Data.Context
 import Juvix.Compiler.Pipeline.EntryPoint
+import Juvix.Data.Effect.Git
 import Juvix.Prelude
 
 -- | `Artifacts` contains enough information so that the pipeline can be
@@ -64,7 +65,7 @@ tmpCoreInfoTableBuilderArtifacts m = do
   modify' (set artifactCoreTable tbl)
   return a
 
-runPathResolverArtifacts :: (Members '[Files, Reader EntryPoint, State Artifacts] r) => Sem (PathResolver ': r) a -> Sem r a
+runPathResolverArtifacts :: (Members '[Files, Reader EntryPoint, State Artifacts, Error DependencyError, GitClone] r) => Sem (PathResolver ': r) a -> Sem r a
 runPathResolverArtifacts = runStateLikeArtifacts runPathResolverPipe' artifactResolver
 
 runBuiltinsArtifacts :: (Members '[Error JuvixError, State Artifacts] r) => Sem (Builtins ': r) a -> Sem r a

src/Juvix/Compiler/Pipeline/Package.hs

@@ -11,6 +11,7 @@ module Juvix.Compiler.Pipeline.Package
     packageVersion,
     packageDependencies,
     packageMain,
+    packageFile,
     rawPackage,
     readPackage,
     readPackageIO,
@@ -18,6 +19,7 @@ module Juvix.Compiler.Pipeline.Package
     globalPackage,
     emptyPackage,
     readGlobalPackage,
+    mkPackageFilePath,
   )
 where
 
@@ -25,6 +27,7 @@ import Data.Aeson (genericToEncoding, genericToJSON)
 import Data.Aeson.BetterErrors
 import Data.Aeson.TH
 import Data.ByteString qualified as ByteString
+import Data.HashSet qualified as HashSet
 import Data.Kind qualified as GHC
 import Data.Versions
 import Data.Yaml
@@ -48,12 +51,18 @@ type family DependenciesType s = res | res -> s where
   DependenciesType 'Raw = Maybe [Dependency]
   DependenciesType 'Processed = [Dependency]
 
+type PackageFileType :: IsProcessed -> GHC.Type
+type family PackageFileType s = res | res -> s where
+  PackageFileType 'Raw = ()
+  PackageFileType 'Processed = Path Abs File
+
 data Package' (s :: IsProcessed) = Package
   { _packageName :: NameType s,
     _packageVersion :: VersionType s,
     _packageDependencies :: DependenciesType s,
     _packageBuildDir :: Maybe (SomeBase Dir),
-    _packageMain :: Maybe (Prepath File)
+    _packageMain :: Maybe (Prepath File),
+    _packageFile :: PackageFileType s
   }
   deriving stock (Generic)
 
@@ -93,6 +102,7 @@ instance FromJSON RawPackage where
         _packageDependencies <- keyMay "dependencies" fromAesonParser
         _packageBuildDir <- keyMay "build-dir" fromAesonParser
         _packageMain <- keyMay "main" fromAesonParser
+        let _packageFile = ()
         return Package {..}
       err :: a
       err = error "Failed to parse juvix.yaml"
@@ -107,14 +117,15 @@ resolveBuildDir = \case
   CustomBuildDir d -> d
 
 -- | This is used when juvix.yaml exists but it is empty
-emptyPackage :: BuildDir -> Package
-emptyPackage buildDir =
+emptyPackage :: BuildDir -> Path Abs File -> Package
+emptyPackage buildDir yamlPath =
   Package
     { _packageName = defaultPackageName,
       _packageVersion = defaultVersion,
       _packageDependencies = [defaultStdlibDep buildDir],
       _packageMain = Nothing,
-      _packageBuildDir = Nothing
+      _packageBuildDir = Nothing,
+      _packageFile = yamlPath
     }
 
 rawPackage :: Package -> RawPackage
@@ -124,15 +135,17 @@ rawPackage pkg =
       _packageVersion = Just (prettySemVer (pkg ^. packageVersion)),
       _packageDependencies = Just (pkg ^. packageDependencies),
       _packageBuildDir = pkg ^. packageBuildDir,
-      _packageMain = pkg ^. packageMain
+      _packageMain = pkg ^. packageMain,
+      _packageFile = ()
     }
 
-processPackage :: forall r. (Members '[Error Text] r) => BuildDir -> RawPackage -> Sem r Package
-processPackage buildDir pkg = do
+processPackage :: forall r. (Members '[Error Text] r) => Path Abs File -> BuildDir -> RawPackage -> Sem r Package
+processPackage _packageFile buildDir pkg = do
   let _packageName = fromMaybe defaultPackageName (pkg ^. packageName)
       base :: SomeBase Dir = (resolveBuildDir buildDir) <///> relStdlibDir
-      stdlib = Dependency (mkPrepath (fromSomeDir base))
+      stdlib = mkPathDependency (fromSomeDir base)
       _packageDependencies = fromMaybe [stdlib] (pkg ^. packageDependencies)
+  checkNoDuplicateDepNames _packageDependencies
   _packageVersion <- getVersion
   return
     Package
@@ -148,8 +161,24 @@ processPackage buildDir pkg = do
         Right v -> return v
         Left err -> throw (pack (errorBundlePretty err))
 
+    checkNoDuplicateDepNames :: [Dependency] -> Sem r ()
+    checkNoDuplicateDepNames deps = go HashSet.empty (deps ^.. traversed . _GitDependency . gitDependencyName)
+      where
+        go :: HashSet Text -> [Text] -> Sem r ()
+        go _ [] = return ()
+        go s (x : xs)
+          | x `HashSet.member` s = throw (errMsg x)
+          | otherwise = go (HashSet.insert x s) xs
+          where
+            errMsg :: Text -> Text
+            errMsg dupName =
+              "Juvix package file at: "
+                <> pack (toFilePath _packageFile)
+                <> " contains the duplicate dependency name: "
+                <> dupName
+
 defaultStdlibDep :: BuildDir -> Dependency
-defaultStdlibDep buildDir = Dependency (mkPrepath (fromSomeDir (resolveBuildDir buildDir <///> relStdlibDir)))
+defaultStdlibDep buildDir = mkPathDependency (fromSomeDir (resolveBuildDir buildDir <///> relStdlibDir))
 
 defaultPackageName :: Text
 defaultPackageName = "my-project"
@@ -157,16 +186,20 @@ defaultPackageName = "my-project"
 defaultVersion :: SemVer
 defaultVersion = SemVer 0 0 0 Nothing Nothing
 
-globalPackage :: Package
+globalPackage :: RawPackage
 globalPackage =
   Package
-    { _packageDependencies = [defaultStdlibDep DefaultBuildDir],
-      _packageName = "global-juvix-package",
-      _packageVersion = defaultVersion,
+    { _packageDependencies = Just [defaultStdlibDep DefaultBuildDir],
+      _packageName = Just "global-juvix-package",
+      _packageVersion = Just (prettySemVer defaultVersion),
       _packageMain = Nothing,
-      _packageBuildDir = Nothing
+      _packageBuildDir = Nothing,
+      _packageFile = ()
     }
 
+mkPackageFilePath :: Path Abs Dir -> Path Abs File
+mkPackageFilePath = (<//> juvixYamlFile)
+
 -- | Given some directory d it tries to read the file d/juvix.yaml and parse its contents
 readPackage ::
   forall r.
@@ -177,10 +210,10 @@ readPackage ::
 readPackage root buildDir = do
   bs <- readFileBS' yamlPath
   if
-      | ByteString.null bs -> return (emptyPackage buildDir)
-      | otherwise -> either (throw . pack . prettyPrintParseException) (processPackage buildDir) (decodeEither' bs)
+      | ByteString.null bs -> return (emptyPackage buildDir yamlPath)
+      | otherwise -> either (throw . pack . prettyPrintParseException) (processPackage yamlPath buildDir) (decodeEither' bs)
   where
-    yamlPath = root <//> juvixYamlFile
+    yamlPath = mkPackageFilePath root
 
 readPackageIO :: Path Abs Dir -> BuildDir -> IO Package
 readPackageIO root buildDir = do
@@ -208,4 +241,4 @@ writeGlobalPackage :: (Members '[Files] r) => Sem r ()
 writeGlobalPackage = do
   yamlPath <- globalYaml
   ensureDir' (parent yamlPath)
-  writeFileBS yamlPath (encode (rawPackage globalPackage))
+  writeFileBS yamlPath (encode globalPackage)

src/Juvix/Compiler/Pipeline/Package/Dependency.hs

@@ -1,26 +1,103 @@
-module Juvix.Compiler.Pipeline.Package.Dependency
-  ( Dependency (..),
-    dependencyPath,
-  )
-where
+module Juvix.Compiler.Pipeline.Package.Dependency where
 
+import Data.Aeson (genericToEncoding, genericToJSON)
+import Data.Aeson.BetterErrors
+import Data.Aeson.BetterErrors qualified as Aeson
+import Data.Aeson.TH
+import Data.Text.Encoding.Error (lenientDecode)
 import Data.Yaml
 import Juvix.Prelude
 import Juvix.Prelude.Pretty
+import Lens.Micro.Platform qualified as Lens
 
-newtype Dependency = Dependency
-  { _dependencyPath :: Prepath Dir
+data Dependency
+  = DependencyPath PathDependency
+  | DependencyGit GitDependency
+  deriving stock (Generic, Eq, Show)
+
+newtype PathDependency = PathDependency
+  { _pathDependencyPath :: Prepath Dir
   }
   deriving stock (Generic, Eq, Show)
 
-instance ToJSON Dependency where
-  toJSON (Dependency p) = toJSON p
-  toEncoding (Dependency p) = toEncoding p
+mkPathDependency :: FilePath -> Dependency
+mkPathDependency = DependencyPath . PathDependency . mkPrepath
 
-instance FromJSON Dependency where
-  parseJSON = fmap Dependency . parseJSON
+data GitDependency = GitDependency
+  { _gitDependencyUrl :: Text,
+    _gitDependencyRef :: Text,
+    _gitDependencyName :: Text
+  }
+  deriving stock (Generic, Eq, Show)
 
+data PackageDependencyInfo = PackageDependencyInfo
+  { _packageDependencyInfoPackageFile :: Path Abs File,
+    _packageDepdendencyInfoDependency :: Dependency
+  }
+
+makeLenses ''PackageDependencyInfo
 makeLenses ''Dependency
+makeLenses ''GitDependency
+makeLenses ''PathDependency
+
+mkPackageDependencyInfo :: Path Abs File -> Dependency -> PackageDependencyInfo
+mkPackageDependencyInfo = PackageDependencyInfo
+
+_GitDependency :: Traversal' Dependency GitDependency
+_GitDependency f = \case
+  DependencyGit g -> DependencyGit <$> f g
+  x@DependencyPath {} -> pure x
+
+instance Pretty PathDependency where
+  pretty (PathDependency p) = pretty p
+
+instance Pretty GitDependency where
+  pretty g = pretty (decodeUtf8With lenientDecode (encode g))
 
 instance Pretty Dependency where
-  pretty (Dependency i) = pretty i
+  pretty = \case
+    DependencyPath i -> pretty i
+    DependencyGit g -> pretty g
+
+instance ToJSON Dependency where
+  toJSON = \case
+    DependencyPath p -> toJSON p
+    DependencyGit g -> toJSON g
+  toEncoding = \case
+    DependencyPath p -> toEncoding p
+    DependencyGit g -> toEncoding g
+
+instance FromJSON Dependency where
+  parseJSON = toAesonParser' p
+    where
+      p :: Parse' Dependency
+      p = (DependencyPath <$> fromAesonParser) Aeson.<|> (DependencyGit <$> fromAesonParser)
+
+instance ToJSON PathDependency where
+  toJSON (PathDependency p) = toJSON p
+  toEncoding (PathDependency p) = toEncoding p
+
+instance FromJSON PathDependency where
+  parseJSON = fmap PathDependency . parseJSON
+
+gitDependencyOptions :: Options
+gitDependencyOptions =
+  defaultOptions
+    { fieldLabelModifier = over Lens._head toLower . dropPrefix "_gitDependency",
+      rejectUnknownFields = True,
+      omitNothingFields = True
+    }
+
+instance ToJSON GitDependency where
+  toJSON = genericToJSON gitDependencyOptions
+  toEncoding = genericToEncoding gitDependencyOptions
+
+instance FromJSON GitDependency where
+  parseJSON = toAesonParser' (key "git" p)
+    where
+      p :: Parse' GitDependency
+      p = do
+        _gitDependencyUrl <- key "url" asText
+        _gitDependencyRef <- key "ref" asText
+        _gitDependencyName <- key "name" asText
+        return GitDependency {..}

src/Juvix/Compiler/Pipeline/Repl.hs

@@ -14,6 +14,9 @@ import Juvix.Compiler.Internal.Translation.FromConcrete qualified as FromConcret
 import Juvix.Compiler.Internal.Translation.FromInternal.Analysis.Termination.Checker
 import Juvix.Compiler.Pipeline.Artifacts
 import Juvix.Compiler.Pipeline.EntryPoint
+import Juvix.Data.Effect.Git.Process
+import Juvix.Data.Effect.Git.Process.Error
+import Juvix.Data.Effect.Process (runProcessIO)
 import Juvix.Prelude
 
 arityCheckExpression ::
@@ -203,7 +206,12 @@ compileReplInputIO ::
   Sem r (Either JuvixError ReplPipelineResult)
 compileReplInputIO fp txt =
   runError
+    . runLogIO
     . runFilesIO
+    . mapError (JuvixError @GitProcessError)
+    . runProcessIO
+    . runGitProcess
+    . mapError (JuvixError @DependencyError)
     . runPathResolverArtifacts
     $ do
       p <- parseReplInput fp txt

src/Juvix/Compiler/Pipeline/Setup.hs

@@ -2,9 +2,10 @@ module Juvix.Compiler.Pipeline.Setup where
 
 import Juvix.Compiler.Concrete.Translation.FromParsed.Analysis.PathResolver
 import Juvix.Compiler.Pipeline.EntryPoint
+import Juvix.Data.Effect.Git
 import Juvix.Prelude
 
 entrySetup ::
-  (Members '[Reader EntryPoint, Files, PathResolver] r) =>
+  (Members '[Reader EntryPoint, Files, GitClone, PathResolver] r) =>
   Sem r ()
 entrySetup = registerDependencies

src/Juvix/Data/Effect/Git.hs

@@ -0,0 +1,10 @@
+module Juvix.Data.Effect.Git
+  ( module Juvix.Data.Effect.Git.Base,
+    module Juvix.Data.Effect.Git.Process,
+    module Juvix.Data.Effect.Git.Process.Error,
+  )
+where
+
+import Juvix.Data.Effect.Git.Base
+import Juvix.Data.Effect.Git.Process
+import Juvix.Data.Effect.Git.Process.Error

src/Juvix/Data/Effect/Git/Base.hs

@@ -0,0 +1,29 @@
+module Juvix.Data.Effect.Git.Base where
+
+import Juvix.Prelude
+
+type GitRef = Text
+
+type CloneDir = Path Abs Dir
+
+type RepoUrl = Text
+
+data CloneArgs = CloneArgs
+  { _cloneArgsCloneDir :: CloneDir,
+    _cloneArgsRepoUrl :: RepoUrl
+  }
+
+makeLenses ''CloneArgs
+
+data GitError
+  = NotAClone
+  | NoSuchRef GitRef
+
+data Git m a where
+  Fetch :: (GitError -> m ()) -> Git m ()
+  Checkout :: (GitError -> m ()) -> GitRef -> Git m ()
+  HeadRef :: (GitError -> m GitRef) -> Git m GitRef
+
+makeSem ''Git
+
+type GitClone = Scoped CloneArgs Git

src/Juvix/Data/Effect/Git/Process.hs

@@ -0,0 +1,102 @@
+module Juvix.Data.Effect.Git.Process where
+
+import Data.Text qualified as T
+import Juvix.Data.Effect.Git.Base
+import Juvix.Data.Effect.Git.Process.Error
+import Juvix.Data.Effect.Process
+import Juvix.Prelude
+import Polysemy.Opaque
+
+-- | Run a git command in the current working direcotory of the parent process.
+runGitCmd :: (Members '[Process, Error GitProcessError] r) => [Text] -> Sem r Text
+runGitCmd args = do
+  mcmd <- findExecutable' $(mkRelFile "git")
+  case mcmd of
+    Nothing -> throw GitCmdNotFound
+    Just cmd -> do
+      res <- readProcess' (ProcessCall {_processCallPath = cmd, _processCallArgs = args})
+      case res ^. processResultExitCode of
+        ExitFailure {} ->
+          throw
+            ( GitCmdError
+                ( GitCmdErrorDetails
+                    { _gitCmdErrorDetailsCmdPath = cmd,
+                      _gitCmdErrorDetailsArgs = args,
+                      _gitCmdErrorDetailsExitCode = res ^. processResultExitCode,
+                      _gitCmdErrorDetailsMessage = res ^. processResultStderr
+                    }
+                )
+            )
+        ExitSuccess -> return (res ^. processResultStdout)
+
+-- | Run a git command within a directory, throws an error if the directory is not a valid clone
+runGitCmdInDir :: (Members '[Process, Error GitProcessError, Reader CloneDir] r) => [Text] -> Sem r Text
+runGitCmdInDir args = do
+  checkValidGitClone
+  runGitCmdInDir' args
+
+-- | Run a git command within a directory
+runGitCmdInDir' :: (Members '[Process, Error GitProcessError, Reader CloneDir] r) => [Text] -> Sem r Text
+runGitCmdInDir' args = do
+  p :: Path Abs Dir <- ask
+  runGitCmd (["--git-dir", ".git", "-C", T.pack (toFilePath p)] <> args)
+
+-- | Throws an error if the directory is not a valid git clone
+checkValidGitClone :: (Members '[Process, Error GitProcessError, Reader CloneDir] r) => Sem r ()
+checkValidGitClone = void gitHeadRef
+
+isValidGitClone :: (Members '[Process, Reader CloneDir] r) => Sem r Bool
+isValidGitClone = isRight <$> runError @GitProcessError checkValidGitClone
+
+-- | Return the HEAD ref of the clone
+gitHeadRef :: (Members '[Process, Error GitProcessError, Reader CloneDir] r) => Sem r Text
+gitHeadRef = T.strip <$> runGitCmdInDir' ["rev-parse", "HEAD"]
+
+-- | Checkout the clone at a particular ref
+gitCheckout :: (Members '[Process, Error GitProcessError, Reader CloneDir] r) => Text -> Sem r ()
+gitCheckout ref = void (runGitCmdInDir ["checkout", ref])
+
+-- | Fetch in the clone
+gitFetch :: (Members '[Process, Error GitProcessError, Reader CloneDir] r) => Sem r ()
+gitFetch = void (runGitCmdInDir ["fetch"])
+
+cloneGitRepo :: (Members '[Log, Files, Process, Error GitProcessError, Reader CloneDir] r) => Text -> Sem r ()
+cloneGitRepo url = do
+  p :: Path Abs Dir <- ask
+  log ("cloning " <> url <> " to " <> pack (toFilePath p))
+  void (runGitCmd ["clone", url, T.pack (toFilePath p)])
+
+initGitRepo :: (Members '[Log, Files, Process, Error GitProcessError, Reader CloneDir] r) => Text -> Sem r (Path Abs Dir)
+initGitRepo url = do
+  p <- ask
+  unlessM (directoryExists' p) (cloneGitRepo url)
+  return p
+
+handleNotACloneError :: (Member (Error GitProcessError) r, Monad m) => (GitError -> m x) -> Tactical e m r x -> Tactical e m r x
+handleNotACloneError errorHandler eff = catch @GitProcessError eff $ \case
+  GitCmdError GitCmdErrorDetails {_gitCmdErrorDetailsExitCode = ExitFailure 128} -> runTSimple (return NotAClone) >>= bindTSimple errorHandler
+  e -> throw e
+
+handleNoSuchRefError :: (Member (Error GitProcessError) r, Monad m) => (GitError -> m x) -> GitRef -> Tactical e m r x -> Tactical e m r x
+handleNoSuchRefError errorHandler ref eff = catch @GitProcessError eff $ \case
+  GitCmdError GitCmdErrorDetails {_gitCmdErrorDetailsExitCode = ExitFailure 1} -> runTSimple (return (NoSuchRef ref)) >>= bindTSimple errorHandler
+  e -> throw e
+
+handleCheckoutError :: (Member (Error GitProcessError) r, Monad m) => (GitError -> m x) -> GitRef -> Tactical e m r x -> Tactical e m r x
+handleCheckoutError errorHandler ref eff = handleNoSuchRefError errorHandler ref (handleNotACloneError errorHandler eff)
+
+runGitProcess ::
+  forall r a.
+  (Members '[Log, Files, Process, Error GitProcessError] r) =>
+  Sem (Scoped CloneArgs Git ': r) a ->
+  Sem r a
+runGitProcess = interpretScopedH allocator handler
+  where
+    allocator :: forall q x. CloneArgs -> (Path Abs Dir -> Sem (Opaque q ': r) x) -> Sem (Opaque q ': r) x
+    allocator a use' = use' =<< runReader (a ^. cloneArgsCloneDir) (initGitRepo (a ^. cloneArgsRepoUrl))
+
+    handler :: forall q r0 x. Path Abs Dir -> Git (Sem r0) x -> Tactical Git (Sem r0) (Opaque q ': r) x
+    handler p eff = case eff of
+      HeadRef errorHandler -> handleNotACloneError errorHandler (runReader p gitHeadRef >>= pureT)
+      Fetch errorHandler -> handleNotACloneError errorHandler (runReader p gitFetch >>= pureT)
+      Checkout errorHandler ref -> handleCheckoutError errorHandler ref (runReader p (gitCheckout ref) >>= pureT)

src/Juvix/Data/Effect/Git/Process/Error.hs

@@ -0,0 +1,41 @@
+module Juvix.Data.Effect.Git.Process.Error where
+
+import Juvix.Data.PPOutput
+import Juvix.Prelude
+
+data GitCmdErrorDetails = GitCmdErrorDetails
+  { _gitCmdErrorDetailsCmdPath :: Path Abs File,
+    _gitCmdErrorDetailsArgs :: [Text],
+    _gitCmdErrorDetailsExitCode :: ExitCode,
+    _gitCmdErrorDetailsMessage :: Text
+  }
+
+data GitProcessError
+  = GitCmdError GitCmdErrorDetails
+  | GitCmdNotFound
+
+makeLenses ''GitCmdErrorDetails
+makeLenses ''GitProcessError
+
+instance Pretty GitCmdErrorDetails where
+  pretty d = pretty msg
+    where
+      msg :: Text
+      msg = "error when executing the git command with arguments: " <> show (d ^. gitCmdErrorDetailsArgs)
+
+instance Pretty GitProcessError where
+  pretty = \case
+    GitCmdError ce -> pretty ce
+    GitCmdNotFound {} -> "git command not found"
+
+instance ToGenericError GitProcessError where
+  genericError e =
+    return
+      ( GenericError
+          { _genericErrorMessage = ppOutput (pretty e),
+            _genericErrorLoc = i,
+            _genericErrorIntervals = [i]
+          }
+      )
+    where
+      i = singletonInterval (mkInitialLoc $(mkAbsFile "/<git>"))

src/Juvix/Data/Effect/Log.hs

@@ -18,3 +18,7 @@ runLogIO sem = do
         Log txt -> embed (Text.hPutStrLn stdout txt)
     )
     sem
+
+ignoreLog :: InterpreterFor Log r
+ignoreLog = interpret $ \case
+  Log _ -> return ()

src/Juvix/Data/Effect/Process.hs

@@ -0,0 +1,8 @@
+module Juvix.Data.Effect.Process
+  ( module Juvix.Data.Effect.Process.Base,
+    module Juvix.Data.Effect.Process.IO,
+  )
+where
+
+import Juvix.Data.Effect.Process.Base
+import Juvix.Data.Effect.Process.IO

src/Juvix/Data/Effect/Process/Base.hs

@@ -0,0 +1,22 @@
+module Juvix.Data.Effect.Process.Base where
+
+import Juvix.Prelude
+
+data ProcessResult = ProcessResult
+  { _processResultExitCode :: ExitCode,
+    _processResultStdout :: Text,
+    _processResultStderr :: Text
+  }
+
+data ProcessCall = ProcessCall
+  { _processCallPath :: Path Abs File,
+    _processCallArgs :: [Text]
+  }
+
+data Process m a where
+  FindExecutable' :: Path Rel File -> Process m (Maybe (Path Abs File))
+  ReadProcess' :: ProcessCall -> Process m ProcessResult
+
+makeSem ''Process
+makeLenses ''ProcessResult
+makeLenses ''ProcessCall

src/Juvix/Data/Effect/Process/IO.hs

@@ -0,0 +1,28 @@
+module Juvix.Data.Effect.Process.IO where
+
+import Data.ByteString.Lazy qualified as LBS
+import Data.Text qualified as T
+import Data.Text.Encoding.Error
+import Juvix.Data.Effect.Process.Base
+import Juvix.Prelude
+import System.Process.Typed qualified as P
+
+runProcessIO ::
+  forall r a.
+  (Members '[Embed IO] r) =>
+  Sem (Process ': r) a ->
+  Sem r a
+runProcessIO = interpret $ \case
+  FindExecutable' n -> findExecutable n
+  ReadProcess' call -> do
+    let p = P.proc (toFilePath (call ^. processCallPath)) (T.unpack <$> call ^. processCallArgs)
+    (exitCode, stdoutRes, stderrRes) <- P.readProcess p
+    return
+      ProcessResult
+        { _processResultExitCode = exitCode,
+          _processResultStdout = toText stdoutRes,
+          _processResultStderr = toText stderrRes
+        }
+  where
+    toText :: LBS.ByteString -> Text
+    toText lbs = decodeUtf8With lenientDecode (LBS.toStrict lbs)

src/Juvix/Extra/Paths/Base.hs

@@ -35,6 +35,9 @@ relBuildDir = $(mkRelDir ".juvix-build")
 relStdlibDir :: Path Rel Dir
 relStdlibDir = $(mkRelDir "stdlib")
 
+relDependenciesDir :: Path Rel Dir
+relDependenciesDir = $(mkRelDir "deps")
+
 rootBuildDir :: Path Abs Dir -> Path Abs Dir
 rootBuildDir root = root <//> relBuildDir
 

src/Juvix/Extra/Stdlib.hs

@@ -32,16 +32,18 @@ packageStdlib :: forall r. (Members '[Files] r) => Path Abs Dir -> Path Abs Dir
 packageStdlib rootDir buildDir = firstJustM isStdLib
   where
     isStdLib :: Dependency -> Sem r (Maybe (Path Abs Dir))
-    isStdLib (Dependency dep) = do
-      adir <- canonicalDir rootDir dep
-      let mstdlib :: Maybe (Path Rel Dir) = stripProperPrefix buildDir adir
-      return $
-        if
-            | mstdlib == Just relStdlibDir -> Just stdLibBuildDir
-            | otherwise -> Nothing
-      where
-        stdLibBuildDir :: Path Abs Dir
-        stdLibBuildDir = juvixStdlibDir buildDir
+    isStdLib = \case
+      DependencyPath dep -> do
+        adir <- canonicalDir rootDir (dep ^. pathDependencyPath)
+        let mstdlib :: Maybe (Path Rel Dir) = stripProperPrefix buildDir adir
+        return $
+          if
+              | mstdlib == Just relStdlibDir -> Just stdLibBuildDir
+              | otherwise -> Nothing
+        where
+          stdLibBuildDir :: Path Abs Dir
+          stdLibBuildDir = juvixStdlibDir buildDir
+      DependencyGit {} -> return Nothing
 
 writeStdlib :: forall r. (Members '[Reader StdlibRoot, Files] r) => Sem r ()
 writeStdlib = do

src/Juvix/Prelude/Base.hs

@@ -60,6 +60,7 @@ module Juvix.Prelude.Base
     module Polysemy.Reader,
     module Polysemy.Tagged,
     module Polysemy.Resource,
+    module Polysemy.Scoped,
     module Polysemy.State,
     module Language.Haskell.TH.Syntax,
     module Prettyprinter,
@@ -165,6 +166,7 @@ import Polysemy.Fixpoint
 import Polysemy.Output
 import Polysemy.Reader
 import Polysemy.Resource
+import Polysemy.Scoped
 import Polysemy.State
 import Polysemy.Tagged hiding (tag)
 import Prettyprinter (Doc, (<+>))

test/Scope/Positive.hs

@@ -11,6 +11,8 @@ import Juvix.Compiler.Concrete.Translation.FromParsed.Analysis.PathResolver
 import Juvix.Compiler.Concrete.Translation.FromParsed.Analysis.Scoping qualified as Scoper
 import Juvix.Compiler.Concrete.Translation.FromSource qualified as Parser
 import Juvix.Compiler.Pipeline.Setup
+import Juvix.Data.Effect.Git
+import Juvix.Data.Effect.Process
 import Juvix.Prelude.Aeson
 import Juvix.Prelude.Pretty
 
@@ -49,6 +51,11 @@ testDescr PosTest {..} = helper renderCodeNew
                         . evalTopNameIdGen
                         . runFilesPure files tRoot
                         . runReader entryPoint
+                        . ignoreLog
+                        . runProcessIO
+                        . mapError (JuvixError @GitProcessError)
+                        . runGitProcess
+                        . mapError (JuvixError @DependencyError)
                         . runPathResolverPipe
                     evalHelper :: HashMap (Path Abs File) Text -> Sem PipelineEff a -> IO a
                     evalHelper files = fmap snd . runHelper files

tests/smoke/Commands/compile-dependencies.smoke.yaml

@@ -0,0 +1,358 @@
+working-directory: ./../../../
+
+tests:
+  - name: git-dependencies-success
+    command:
+      shell:
+        - bash
+      script: |
+        temp=$(mktemp -d)
+        trap 'rm -rf -- "$temp"' EXIT
+
+        # create dependency
+        mkdir $temp/dep
+        cd $temp/dep
+        git init
+
+        cat <<-EOF > HelloDep.juvix
+        module HelloDep;
+        import Stdlib.Prelude open;
+        main : IO := printStringLn "Hello from dep";
+        EOF
+        touch juvix.yaml
+
+        git add -A
+        git commit -m "commit1"
+
+        dep1hash=$(git rev-parse HEAD)
+
+        # create project that uses dependency
+        mkdir $temp/base
+        cd $temp/base
+
+        cat <<-EOF > juvix.yaml
+        name: HelloWorld
+        main: HelloWorld.juvix
+        dependencies:
+          - .juvix-build/stdlib
+          - git:
+              url: $temp/dep
+              name: dep1
+              ref: $dep1hash
+        version: 0.1.0
+        EOF
+
+        cat <<-EOF > HelloWorld.juvix
+        -- HelloWorld.juvix
+        module HelloWorld;
+
+        import Stdlib.Prelude open;
+        import HelloDep;
+
+        main : IO := HelloDep.main;
+        EOF
+
+        # compile project
+        juvix compile HelloWorld.juvix
+        ./HelloWorld
+    stdout:
+      contains: Hello from dep
+    exit-status: 0
+
+  - name: git-dependencies-fetch-new-commits
+    command:
+      shell:
+        - bash
+      script: |
+        temp=$(mktemp -d)
+        trap 'rm -rf -- "$temp"' EXIT
+
+        # create dependency
+        mkdir $temp/dep
+        cd $temp/dep
+        git init
+
+        cat <<-EOF > HelloDep.juvix
+        module HelloDep;
+        import Stdlib.Prelude open;
+        main : IO := printStringLn "Hello from dep";
+        EOF
+        touch juvix.yaml
+
+        git add -A
+        git commit -m "commit1"
+
+        dep1hash=$(git rev-parse HEAD)
+
+        # create project that uses dependency
+        mkdir $temp/base
+        cd $temp/base
+
+        cat <<-EOF > juvix.yaml
+        name: HelloWorld
+        main: HelloWorld.juvix
+        dependencies:
+          - .juvix-build/stdlib
+          - git:
+              url: $temp/dep
+              name: dep1
+              ref: $dep1hash
+        version: 0.1.0
+        EOF
+
+        cat <<-EOF > HelloWorld.juvix
+        -- HelloWorld.juvix
+        module HelloWorld;
+
+        import Stdlib.Prelude open;
+        import HelloDep;
+
+        main : IO := HelloDep.main;
+        EOF
+
+        # compile project the first time
+        juvix compile HelloWorld.juvix
+
+        # update the dependency
+        cd $temp/dep
+
+        cat <<-EOF > HelloDep.juvix
+        module HelloDep;
+        import Stdlib.Prelude open;
+        main : IO := printStringLn "This is from the second commit";
+        EOF
+
+        git add -A
+        git commit -m "commit2"
+
+        dep1hash=$(git rev-parse HEAD)
+
+        # use the new hash
+        cd $temp/base
+
+        cat <<-EOF > juvix.yaml
+        name: HelloWorld
+        main: HelloWorld.juvix
+        dependencies:
+          - .juvix-build/stdlib
+          - git:
+              url: $temp/dep
+              name: dep1
+              ref: $dep1hash
+        version: 0.1.0
+        EOF
+
+        # compile with the new hash
+        juvix compile HelloWorld.juvix
+        ./HelloWorld
+    stdout:
+      contains: This is from the second commit
+    exit-status: 0
+
+  - name: git-dependencies-invalid-ref
+    command:
+      shell:
+        - bash
+      script: |
+        temp=$(mktemp -d)
+        trap 'rm -rf -- "$temp"' EXIT
+
+        # create dependency
+        mkdir $temp/dep
+        cd $temp/dep
+        git init
+
+        cat <<-EOF > HelloDep.juvix
+        module HelloDep;
+        import Stdlib.Prelude open;
+        main : IO := printStringLn "Hello from dep";
+        EOF
+        touch juvix.yaml
+
+        git add -A
+        git commit -m "commit1"
+
+        # create project that uses dependency
+        mkdir $temp/base
+        cd $temp/base
+
+        cat <<-EOF > juvix.yaml
+        name: HelloWorld
+        main: HelloWorld.juvix
+        dependencies:
+          - .juvix-build/stdlib
+          - git:
+              url: $temp/dep
+              name: dep1
+              ref: invalid-ref
+        version: 0.1.0
+        EOF
+
+        cat <<-EOF > HelloWorld.juvix
+        -- HelloWorld.juvix
+        module HelloWorld;
+
+        import Stdlib.Prelude open;
+        import HelloDep;
+
+        main : IO := HelloDep.main;
+        EOF
+
+        # compile project
+        juvix compile HelloWorld.juvix
+    stderr:
+      contains: invalid-ref
+    stdout:
+      contains: cloning
+    exit-status: 1
+
+  - name: git-dependencies-invalid-url
+    command:
+      shell:
+        - bash
+      script: |
+        temp=$(mktemp -d)
+        trap 'rm -rf -- "$temp"' EXIT
+
+        # create project that uses dependency
+        mkdir $temp/base
+        cd $temp/base
+
+        cat <<-EOF > juvix.yaml
+        name: HelloWorld
+        main: HelloWorld.juvix
+        dependencies:
+          - .juvix-build/stdlib
+          - git:
+              url: $temp/dep
+              name: dep1
+              ref: main
+        version: 0.1.0
+        EOF
+
+        cat <<-EOF > HelloWorld.juvix
+        -- HelloWorld.juvix
+        module HelloWorld;
+
+        import Stdlib.Prelude open;
+        import HelloDep;
+
+        main : IO := HelloDep.main;
+        EOF
+
+        # compile project
+        juvix compile HelloWorld.juvix
+    stderr:
+      contains: error
+    stdout:
+      contains: cloning
+    exit-status: 1
+
+  - name: git-dependencies-corrupt-clone
+    command:
+      shell:
+        - bash
+      script: |
+        temp=$(mktemp -d)
+        trap 'rm -rf -- "$temp"' EXIT
+
+        # create dependency
+        mkdir $temp/dep
+        cd $temp/dep
+        git init
+
+        cat <<-EOF > HelloDep.juvix
+        module HelloDep;
+        import Stdlib.Prelude open;
+        main : IO := printStringLn "Hello from dep";
+        EOF
+        touch juvix.yaml
+
+        git add -A
+        git commit -m "commit1"
+
+        dep1hash=$(git rev-parse HEAD)
+
+        # create project that uses dependency
+        mkdir $temp/base
+        cd $temp/base
+
+        cat <<-EOF > juvix.yaml
+        name: HelloWorld
+        main: HelloWorld.juvix
+        dependencies:
+          - .juvix-build/stdlib
+          - git:
+              url: $temp/dep
+              name: dep1
+              ref: $dep1hash
+        version: 0.1.0
+        EOF
+
+        cat <<-EOF > HelloWorld.juvix
+        -- HelloWorld.juvix
+        module HelloWorld;
+
+        import Stdlib.Prelude open;
+        import HelloDep;
+
+        main : IO := HelloDep.main;
+        EOF
+
+        # compile project
+        juvix compile HelloWorld.juvix
+
+        # corrupt the clone
+        rm -rf ./.juvix-build/deps/dep1/.git
+
+        # compile project
+        juvix compile HelloWorld.juvix
+    stderr:
+      contains: juvix clean
+    stdout:
+      contains: cloning
+    exit-status: 1
+
+  - name: git-dependencies-duplicate-names
+    command:
+      shell:
+        - bash
+      script: |
+        temp=$(mktemp -d)
+        trap 'rm -rf -- "$temp"' EXIT
+
+        # create project that uses dependency
+        mkdir $temp/base
+        cd $temp/base
+
+        cat <<-EOF > juvix.yaml
+        name: HelloWorld
+        main: HelloWorld.juvix
+        dependencies:
+          - .juvix-build/stdlib
+          - git:
+              url: $temp/dep
+              name: dep1
+              ref: main
+          - git:
+              url: $temp/dep2
+              name: dep1
+              ref: main
+        version: 0.1.0
+        EOF
+
+        cat <<-EOF > HelloWorld.juvix
+        -- HelloWorld.juvix
+        module HelloWorld;
+
+        import Stdlib.Prelude open;
+        import HelloDep;
+
+        main : IO := HelloDep.main;
+        EOF
+
+        # compile project
+        juvix compile HelloWorld.juvix
+    stdout:
+      contains: duplicate
+    exit-status: 1