Update the Juvix lock file when the Package file changes (#2522)

Adds a new version of the lock file that stores the hash (sha256 digest)
of the package file (Package.juvix, juvix.yaml) it was generated from as
a field:

```
# This file was autogenerated by Juvix version 0.5.4.
# Do not edit this file manually.

version: 2
checksum: d05940a4d3dc0e15451d02e1294819c875ba486ee54e26865ba8d190ac7c27c3
dependencies:
- git:
    name: stdlib
    ref: f68b0614ad695eaa13ead42f3466e0a78219f826
    url: https://github.com/anoma/juvix-stdlib.git
  dependencies: []
```

The lock file is regenerated when the hash of the package file doesn't
match the value of the `checksum` field, i.e when the user updates the
package file.

Existing lock files are automatically migrated to version 2.

* Closes https://github.com/anoma/juvix/issues/2464

.github/workflows/ci.yml

@@ -198,6 +198,14 @@ jobs:
         run: |
           sed --version
 
+      - name: Install coreutils
+        run: |
+          brew install coreutils
+
+      - name: Test sha256sum (used by smoke)
+        run: |
+          sha256sum --version
+
       - name: Download and extract wasi-sysroot
         run: >
           curl

Makefile

@@ -240,10 +240,12 @@ fast-test-skip-slow:
 	@${STACK} test --fast ${STACKFLAGS} ${STACKTESTFLAGS} --ta '-p "! /slow tests/"'
 
 SMOKE := $(shell command -v smoke 2> /dev/null)
+SHA256SUM := $(shell command -v sha256sum 2> /dev/null)
 
 .PHONY : smoke-only
 smoke-only:
 	@$(if $(SMOKE),, $(error "Smoke not found, please install it from https://github.com/jonaprieto/smoke"))
+	@$(if $(SHA256SUM),, $(error "sha256sum not found, please install the GNU coreutils package (e.g {apt, brew} install coreutils)"))
 	@smoke $(shell find tests -name '*.smoke.yaml')
 
 .PHONY : smoke

package.yaml

@@ -45,9 +45,11 @@ dependencies:
   - aeson == 2.1.*
   - ansi-terminal == 0.11.*
   - base == 4.17.*
+  - base16-bytestring == 1.0.*
   - blaze-html == 0.9.*
   - bytestring == 0.11.*
   - containers == 0.6.*
+  - cryptohash-sha256 == 0.11.*
   - directory == 1.3.*
   - dlist == 1.0.*
   - edit-distance == 0.2.*

src/Juvix/Compiler/Concrete/Translation/FromParsed/Analysis/PathResolver.hs

@@ -26,6 +26,7 @@ import Juvix.Compiler.Pipeline.Package
 import Juvix.Compiler.Pipeline.Package.Loader.EvalEff
 import Juvix.Data.Effect.Git
 import Juvix.Data.Effect.TaggedLock
+import Juvix.Data.SHA256 qualified as SHA256
 import Juvix.Extra.Paths
 import Juvix.Extra.Stdlib (ensureStdlib)
 import Juvix.Prelude
@@ -175,15 +176,26 @@ registerDependencies' conf = do
           void (addRootDependency conf e glob)
       | otherwise -> do
           lockfile <- addRootDependency conf e (e ^. entryPointRoot)
-          root <- asks (^. envRoot)
-          whenM shouldWriteLockfile (writeLockfile root lockfile)
+          whenM shouldWriteLockfile $ do
+            packageFileChecksum <- SHA256.digestFile (e ^. entryPointPackage . packageFile)
+            lockfilePath' <- lockfilePath
+            writeLockfile lockfilePath' packageFileChecksum lockfile
   where
     shouldWriteLockfile :: Sem r Bool
     shouldWriteLockfile = do
+      lockfileExists <- lockfilePath >>= fileExists'
+      hasRemoteDependencies <- gets (^. resolverHasRemoteDependencies)
+      shouldUpdateLockfile' <- gets (^. resolverShouldUpdateLockfile)
+
+      let shouldForce = conf ^. dependenciesConfigForceUpdateLockfile
+          shouldWriteInitialLockfile = not lockfileExists && hasRemoteDependencies
+          shouldUpdateLockfile = lockfileExists && shouldUpdateLockfile'
+      return (shouldForce || shouldWriteInitialLockfile || shouldUpdateLockfile)
+
+    lockfilePath :: Sem r (Path Abs File)
+    lockfilePath = do
       root <- asks (^. envRoot)
-      lockfileExists <- fileExists' (mkPackageLockfilePath root)
-      depsShouldWriteLockfile <- gets (^. resolverShouldWriteLockfile)
-      return (conf ^. dependenciesConfigForceUpdateLockfile || not lockfileExists && depsShouldWriteLockfile)
+      return (mkPackageLockfilePath root)
 
 addRootDependency ::
   forall r.
@@ -196,15 +208,23 @@ addRootDependency conf e root = do
   let pf = mkPackageFilePath root
       d = mkPackageDependencyInfo pf (mkPathDependency (toFilePath root))
   resolvedDependency <- resolveDependency d
-  checkShouldWriteLockfile resolvedDependency
+  checkRemoteDependency resolvedDependency
   let p = resolvedDependency ^. resolvedDependencyPath
   withEnvRoot p $ do
     pkg <- mkPackage (Just e) p
+    shouldUpdateLockfile' <- shouldUpdateLockfile pkg
+    when shouldUpdateLockfile' setShouldUpdateLockfile
     let resolvedPkg :: Package
-          | conf ^. dependenciesConfigForceUpdateLockfile = unsetPackageLockfile pkg
+          | shouldUpdateLockfile' = unsetPackageLockfile pkg
           | otherwise = pkg
     deps <- addDependency' resolvedPkg (Just e) resolvedDependency
     return Lockfile {_lockfileDependencies = deps ^. lockfileDependencyDependencies}
+  where
+    shouldUpdateLockfile :: Package -> Sem r Bool
+    shouldUpdateLockfile pkg = do
+      let checksumMay :: Maybe Text = pkg ^? packageLockfile . _Just . lockfileInfoChecksum . _Just
+      packageFileChecksum <- SHA256.digestFile (pkg ^. packageFile)
+      return (conf ^. dependenciesConfigForceUpdateLockfile || Just packageFileChecksum /= checksumMay)
 
 addDependency ::
   forall r.
@@ -214,7 +234,7 @@ addDependency ::
   Sem r LockfileDependency
 addDependency me d = do
   resolvedDependency <- resolveDependency d
-  checkShouldWriteLockfile resolvedDependency
+  checkRemoteDependency resolvedDependency
   let p = resolvedDependency ^. resolvedDependencyPath
   cached <- lookupCachedDependency p
   case cached of

src/Juvix/Compiler/Concrete/Translation/FromParsed/Analysis/PathResolver/Data.hs

@@ -23,7 +23,8 @@ data ResolverState = ResolverState
     _resolverFiles :: HashMap (Path Rel File) (NonEmpty PackageInfo),
     -- | PackageInfos indexed by root
     _resolverCache :: HashMap (Path Abs Dir) ResolverCacheItem,
-    _resolverShouldWriteLockfile :: Bool
+    _resolverHasRemoteDependencies :: Bool,
+    _resolverShouldUpdateLockfile :: Bool
   }
   deriving stock (Show)
 
@@ -42,12 +43,19 @@ iniResolverState =
   ResolverState
     { _resolverCache = mempty,
       _resolverFiles = mempty,
-      _resolverShouldWriteLockfile = False
+      _resolverHasRemoteDependencies = False,
+      _resolverShouldUpdateLockfile = False
     }
 
-checkShouldWriteLockfile :: (Member (State ResolverState) r) => ResolvedDependency -> Sem r ()
-checkShouldWriteLockfile d = case d ^. resolvedDependencyDependency of
-  DependencyGit {} -> modify' (set resolverShouldWriteLockfile True)
+setHasRemoteDependencies :: (Member (State ResolverState) r) => Sem r ()
+setHasRemoteDependencies = modify' (set resolverHasRemoteDependencies True)
+
+setShouldUpdateLockfile :: (Member (State ResolverState) r) => Sem r ()
+setShouldUpdateLockfile = modify' (set resolverShouldUpdateLockfile True)
+
+checkRemoteDependency :: (Member (State ResolverState) r) => ResolvedDependency -> Sem r ()
+checkRemoteDependency d = case d ^. resolvedDependencyDependency of
+  DependencyGit {} -> setHasRemoteDependencies
   DependencyPath {} -> return ()
 
 withEnvRoot :: (Members '[Reader ResolverEnv] r) => Path Abs Dir -> Sem r a -> Sem r a

src/Juvix/Compiler/Pipeline/Lockfile.hs

@@ -7,6 +7,7 @@ import Data.Aeson.BetterErrors qualified as Aeson
 import Data.Aeson.Encoding (pair)
 import Data.Aeson.TH
 import Data.String.Interpolate (i)
+import Data.Text qualified as T
 import Data.Yaml
 import Data.Yaml.Pretty
 import Juvix.Compiler.Pipeline.Package.Dependency
@@ -29,16 +30,59 @@ newtype Lockfile = Lockfile
   }
   deriving stock (Generic, Show, Eq)
 
+type LockfileV1 = Lockfile
+
+data LockfileV2 = LockfileV2
+  { _lockfileV2Version :: Int,
+    _lockfileV2Checksum :: Text,
+    _lockfileV2Dependencies :: [LockfileDependency]
+  }
+  deriving stock (Generic, Show, Eq)
+
+data VersionedLockfile
+  = V1Lockfile LockfileV1
+  | V2Lockfile LockfileV2
+  deriving stock (Show, Eq)
+
+data LockfileVersionTag
+  = LockfileV1Tag
+  | LockfileV2Tag
+  deriving stock (Show, Eq)
+
+allVersionedLockfiles :: [LockfileVersionTag]
+allVersionedLockfiles = [LockfileV1Tag, LockfileV2Tag]
+
+lockfileVersionNumber :: LockfileVersionTag -> Int
+lockfileVersionNumber = \case
+  LockfileV1Tag -> 1
+  LockfileV2Tag -> 2
+
 data LockfileInfo = LockfileInfo
   { _lockfileInfoPath :: Path Abs File,
+    _lockfileInfoChecksum :: Maybe Text,
     _lockfileInfoLockfile :: Lockfile
   }
   deriving stock (Eq, Show)
 
 makeLenses ''LockfileDependency
 makeLenses ''Lockfile
+makeLenses ''LockfileV2
 makeLenses ''LockfileInfo
 
+mkLockfileV2 :: Text -> [LockfileDependency] -> LockfileV2
+mkLockfileV2 _lockfileV2Checksum _lockfileV2Dependencies =
+  LockfileV2 {_lockfileV2Version = lockfileVersionNumber LockfileV2Tag, ..}
+
+lockfile' :: SimpleGetter VersionedLockfile Lockfile
+lockfile' = to $ \case
+  V1Lockfile l -> l
+  V2Lockfile l -> Lockfile {_lockfileDependencies = l ^. lockfileV2Dependencies}
+
+checksum :: SimpleGetter VersionedLockfile (Maybe Text)
+checksum = to $ \case
+  V1Lockfile {} -> Nothing
+  V2Lockfile l -> Just (l ^. lockfileV2Checksum)
+
 instance ToJSON LockfileDependency where
   toJSON d = object [dep, Str.dependencies .= toJSON (d ^. lockfileDependencyDependencies)]
     where
@@ -65,20 +109,50 @@ instance FromJSON LockfileDependency where
       p' :: Parse' Dependency
       p' = DependencyPath <$> (key Str.path_ fromAesonParser) Aeson.<|> DependencyGit <$> (key Str.git fromAesonParser)
 
-lockfileOptions :: Options
-lockfileOptions =
+lockfileV2Options :: Options
+lockfileV2Options =
   defaultOptions
-    { fieldLabelModifier = over Lens._head toLower . dropPrefix "_lockfile",
+    { fieldLabelModifier = over Lens._head toLower . dropPrefix "_lockfileV2",
       rejectUnknownFields = True,
       omitNothingFields = True
     }
 
-instance ToJSON Lockfile where
-  toJSON = genericToJSON lockfileOptions
-  toEncoding = genericToEncoding lockfileOptions
+instance ToJSON LockfileV2 where
+  toJSON = genericToJSON lockfileV2Options
+  toEncoding = genericToEncoding lockfileV2Options
 
-instance FromJSON Lockfile where
-  parseJSON = toAesonParser' (Lockfile <$> (key Str.dependencies fromAesonParser))
+data LockfileParseErr = LockfileUnsupportedVersion
+
+instance FromJSON VersionedLockfile where
+  parseJSON = toAesonParser displayErr $ do
+    v <- parseVersion
+    case v of
+      LockfileV1Tag -> V1Lockfile <$> parseV1
+      LockfileV2Tag -> V2Lockfile <$> parseV2
+    where
+      parseV1 :: Parse LockfileParseErr Lockfile
+      parseV1 = Lockfile <$> key Str.dependencies fromAesonParser
+
+      parseV2 :: Parse LockfileParseErr LockfileV2
+      parseV2 = do
+        checksum' <- key "checksum" asText
+        deps <- key Str.dependencies fromAesonParser
+        return (mkLockfileV2 checksum' deps)
+
+      parseVersion :: Parse LockfileParseErr LockfileVersionTag
+      parseVersion = fromMaybeM (return LockfileV1Tag) (keyMay Str.version parseVersion')
+        where
+          parseVersion' :: Parse LockfileParseErr LockfileVersionTag
+          parseVersion' = do
+            n <- asIntegral
+            if
+                | lockfileVersionNumber LockfileV1Tag == n -> return LockfileV1Tag
+                | lockfileVersionNumber LockfileV2Tag == n -> return LockfileV2Tag
+                | otherwise -> throwCustomError LockfileUnsupportedVersion
+
+      displayErr :: LockfileParseErr -> Text
+      displayErr = \case
+        LockfileUnsupportedVersion -> "lockfile error: unsupported version. Supported versions: " <> T.intercalate ", " (show . lockfileVersionNumber <$> allVersionedLockfiles)
 
 mkPackageLockfilePath :: Path Abs Dir -> Path Abs File
 mkPackageLockfilePath = (<//> juvixLockfile)
@@ -93,11 +167,16 @@ mayReadLockfile root = do
   if
       | lockfileExists -> do
           bs <- readFileBS' lockfilePath
-          either (throwErr . pack . prettyPrintParseException) ((return . Just) . mkLockfileInfo lockfilePath) (decodeEither' @Lockfile bs)
+          either (throwErr . pack . prettyPrintParseException) ((return . Just) . mkLockfileInfo lockfilePath) (decodeEither' @VersionedLockfile bs)
       | otherwise -> return Nothing
   where
-    mkLockfileInfo :: Path Abs File -> Lockfile -> LockfileInfo
-    mkLockfileInfo _lockfileInfoPath _lockfileInfoLockfile = LockfileInfo {..}
+    mkLockfileInfo :: Path Abs File -> VersionedLockfile -> LockfileInfo
+    mkLockfileInfo _lockfileInfoPath vl =
+      LockfileInfo
+        { _lockfileInfoChecksum = vl ^. checksum,
+          _lockfileInfoLockfile = vl ^. lockfile',
+          ..
+        }
 
     lockfilePath :: Path Abs File
     lockfilePath = mkPackageLockfilePath root
@@ -115,21 +194,23 @@ mayReadLockfile root = do
 lockfileEncodeConfig :: Config
 lockfileEncodeConfig = setConfCompare keyCompare defConfig
   where
-    -- serialize the dependencies after all other keys
+    -- serialize the dependencies field after all other keys and the version
+    -- field before all other keys
     keyCompare :: Text -> Text -> Ordering
     keyCompare x y =
       if
-          | y == Str.dependencies || x == Str.dependencies -> GT
+          | y == Str.dependencies -> LT
+          | x == Str.dependencies -> GT
+          | x == Str.version -> LT
+          | y == Str.version -> GT
           | otherwise -> compare x y
 
-writeLockfile :: (Members '[Files] r) => Path Abs Dir -> Lockfile -> Sem r ()
-writeLockfile root lf = do
+writeLockfile :: (Members '[Files] r) => Path Abs File -> Text -> Lockfile -> Sem r ()
+writeLockfile lockfilePath checksum' lf = do
   ensureDir' (parent lockfilePath)
-  writeFileBS lockfilePath (header <> encodePretty lockfileEncodeConfig lf)
+  let v2lf = mkLockfileV2 checksum' (lf ^. lockfileDependencies)
+  writeFileBS lockfilePath (header <> encodePretty lockfileEncodeConfig v2lf)
   where
-    lockfilePath :: Path Abs File
-    lockfilePath = mkPackageLockfilePath root
-
     header :: ByteString
     header = [i|\# This file was autogenerated by Juvix version #{versionDoc}.\n\# Do not edit this file manually.\n\n|]
 

src/Juvix/Compiler/Pipeline/Package/Loader/EvalEff/IO.hs

@@ -27,7 +27,7 @@ data LoaderResource = LoaderResource
 
 makeLenses ''LoaderResource
 
-runEvalFileEffIO :: forall r a. (Members '[TaggedLock, Embed IO, Error PackageLoaderError] r) => Sem (EvalFileEff ': r) a -> Sem r a
+runEvalFileEffIO :: forall r a. (Members '[TaggedLock, Files, Embed IO, Error PackageLoaderError] r) => Sem (EvalFileEff ': r) a -> Sem r a
 runEvalFileEffIO = interpretScopedAs allocator handler
   where
     allocator :: Path Abs File -> Sem r LoaderResource
@@ -115,7 +115,7 @@ runEvalFileEffIO = interpretScopedAs allocator handler
                   Just l -> l ^. intervalFile == f
                   Nothing -> False
 
-loadPackage' :: (Members '[TaggedLock, Embed IO, Error PackageLoaderError] r) => Path Abs File -> Sem r CoreResult
+loadPackage' :: (Members '[TaggedLock, Files, Embed IO, Error PackageLoaderError] r) => Path Abs File -> Sem r CoreResult
 loadPackage' packagePath = do
   ( mapError
       ( \e ->
@@ -127,7 +127,6 @@ loadPackage' packagePath = do
       . evalInternetOffline
       . ignoreHighlightBuilder
       . runProcessIO
-      . runFilesIO
       . evalTopBuiltins
       . evalTopNameIdGen
       . evalTopBuiltins

src/Juvix/Data/SHA256.hs

@@ -0,0 +1,9 @@
+module Juvix.Data.SHA256 where
+
+import Crypto.Hash.SHA256 qualified as SHA256
+import Data.ByteString.Base16 qualified as Base16
+import Juvix.Prelude
+
+-- | Create a HEX encoded, SHA256 digest of the contents of a file.
+digestFile :: (Member Files r) => Path Abs File -> Sem r Text
+digestFile = fmap (decodeUtf8Lenient . Base16.encode . SHA256.hash) . readFileBS'

src/Juvix/Extra/Strings.hs

@@ -802,3 +802,6 @@ path_ = "path"
 
 package :: (IsString s) => s
 package = "package"
+
+version :: (IsString s) => s
+version = "version"

src/Juvix/Prelude/Aeson.hs

@@ -6,9 +6,15 @@ module Juvix.Prelude.Aeson
 where
 
 import Data.Aeson
+import Data.Aeson.KeyMap qualified as KeyMap
 import Data.Aeson.Text
 import Data.Text.Lazy qualified as Lazy
 import Juvix.Prelude.Base
 
 encodeToText :: (ToJSON a) => a -> Text
 encodeToText = Lazy.toStrict . encodeToLazyText
+
+appendFields :: [(Key, Value)] -> Value -> Value
+appendFields keyValues = \case
+  Object obj -> Object (KeyMap.fromList keyValues <> obj)
+  a -> a

test/Scope/Positive.hs

@@ -10,6 +10,7 @@ import Juvix.Compiler.Concrete.Print qualified as P
 import Juvix.Compiler.Concrete.Translation.FromParsed.Analysis.PathResolver
 import Juvix.Compiler.Concrete.Translation.FromParsed.Analysis.Scoping qualified as Scoper
 import Juvix.Compiler.Concrete.Translation.FromSource qualified as Parser
+import Juvix.Compiler.Pipeline.Package.Loader
 import Juvix.Compiler.Pipeline.Package.Loader.Error
 import Juvix.Compiler.Pipeline.Package.Loader.EvalEff.IO
 import Juvix.Compiler.Pipeline.Package.Loader.PathResolver
@@ -17,7 +18,6 @@ import Juvix.Compiler.Pipeline.Setup
 import Juvix.Data.Effect.Git
 import Juvix.Data.Effect.Process
 import Juvix.Data.Effect.TaggedLock
-import Juvix.Prelude.Aeson
 import Juvix.Prelude.Pretty
 
 data PathResolverMode
@@ -94,9 +94,9 @@ testDescr PosTest {..} = helper renderCodeNew
                         Concrete.fromParsed p
                     )
 
-                let yamlFiles :: [(Path Abs File, Text)]
-                    yamlFiles =
-                      [ (pkgi ^. packageRoot <//> juvixYamlFile, encodeToText (rawPackage (pkgi ^. packagePackage)))
+                let packageFiles' :: [(Path Abs File, Text)]
+                    packageFiles' =
+                      [ (pkgi ^. packagePackage . packageFile, renderPackageVersion PackageVersion1 (pkgi ^. packagePackage))
                         | pkgi <- (^. resolverCacheItemPackage) <$> toList (resolverState ^. resolverCache)
                       ]
                     fsScoped :: HashMap (Path Abs File) Text
@@ -105,14 +105,14 @@ testDescr PosTest {..} = helper renderCodeNew
                         [ (getModuleFilePath m, renderer m)
                           | m <- toList (s ^. Scoper.resultScoperTable . Scoper.infoModules)
                         ]
-                          <> yamlFiles
+                          <> packageFiles'
                     fsParsed :: HashMap (Path Abs File) Text
                     fsParsed =
                       HashMap.fromList $
                         [ (getModuleFilePath m, renderCodeNew m)
                           | m <- toList (p ^. Parser.resultTable . Parser.infoParsedModules)
                         ]
-                          <> yamlFiles
+                          <> packageFiles'
 
                 step "Parsing pretty scoped"
                 p' :: Parser.ParserResult <- evalHelper fsScoped upToParsing

tests/positive/Dependencies/.libs/Base/Package.juvix

@@ -1,9 +1,5 @@
 module Package;
 
-import PackageDescription.V1 open;
+import PackageDescription.Basic open;
 
-package : Package :=
-  defaultPackage
-    {name := "base";
-     version := mkVersion 1 0 0;
-     dependencies := []};
+package : Package := basicPackage;

tests/positive/Dependencies/Package.juvix

@@ -8,5 +8,5 @@ package : Package :=
      version := mkVersion 0 0 1;
      dependencies := [ path ".libs/Extra"
                      ; path ".libs/Base"
-                     ; path ".juvix-build/stdlib"
+                     ; defaultStdlib
                      ]};

tests/smoke/Commands/compile-dependencies-package-juvix.smoke.yaml

@@ -317,7 +317,11 @@ tests:
             main := just "HelloWorld.juvix"};
         EOF
 
+        checksum=$(sha256sum Package.juvix | cut -d " " -f 1)
+
         cat <<-EOF > juvix.lock.yaml
+        version: 2
+        checksum: $checksum
         dependencies:
           - path: .juvix-build/stdlib/
             dependencies: []
@@ -353,6 +357,7 @@ tests:
         cat <<-EOF > HelloDep.juvix
         module HelloDep;
         import HelloDep2 open;
+        import Stdlib.Prelude open;
         main : IO := printStringLn "This is from the second commit" >> printStringLn hello;
         EOF
 
@@ -372,7 +377,7 @@ tests:
         git add -A
         git commit -m "commit2"
 
-        # rebuild the project (should use the first commit
+        # rebuild the project (should use the first commit)
         cd $temp/base
 
         # compile with the new hash
@@ -388,8 +393,7 @@ tests:
         - bash
       script: |
         temp=$(mktemp -d)
-        echo $temp
-        # trap 'rm -rf -- "$temp"' EXIT
+        trap 'rm -rf -- "$temp"' EXIT
 
         # create dependency
         mkdir $temp/dep
@@ -425,7 +429,7 @@ tests:
             version := mkVersion 0 0 1;
             dependencies :=
                [path ".juvix-build/stdlib";
-                git "dep1" "$temp/dep" "$dep1hash"];
+                git "dep1" "$temp/dep" "main"];
             main := just "HelloWorld.juvix"};
         EOF
 
@@ -457,23 +461,7 @@ tests:
 
         # compile project with new ref - should use lockfile ref
         cd $temp/base
-
-        cat <<-EOF > Package.juvix
-        -- juvix-package-version:1
-        module Package;
-
-        import Stdlib.Prelude open;
-        import PackageDescription.V1 open;
-
-        package : Package :=
-          defaultPackage
-            {name := "HelloWorld";
-            version := mkVersion 0 0 1;
-            dependencies :=
-               [path ".juvix-build/stdlib";
-                git "dep1" "$temp/dep" "$dep1hash"];
-            main := just "HelloWorld.juvix"};
-        EOF
+        juvix clean
 
         juvix compile HelloWorld.juvix
         ./HelloWorld
@@ -703,78 +691,6 @@ tests:
       contains: "Hello from dep1\nHello from dep2"
     exit-status: 0
 
-  - name: git-dependencies-lockfile-missing-dependency
-    command:
-      shell:
-        - bash
-      script: |
-        temp=$(mktemp -d)
-        trap 'rm -rf -- "$temp"' EXIT
-
-        # create dependency
-        mkdir $temp/dep1
-        cd $temp/dep1
-        git init
-
-        cat <<-EOF > HelloDep.juvix
-        module HelloDep;
-        import Stdlib.Prelude open;
-        main : IO := printStringLn "Hello from dep";
-        EOF
-
-        touch juvix.yaml
-
-        git add -A
-        git commit -m "commit1"
-
-        dep1hash=$(git rev-parse HEAD)
-
-        # create project that uses dependency
-        mkdir $temp/base
-        cd $temp/base
-
-        cat <<-EOF > Package.juvix
-        -- juvix-package-version:1
-        module Package;
-
-        import Stdlib.Prelude open;
-        import PackageDescription.V1 open;
-
-        package : Package :=
-          defaultPackage
-            {name := "HelloWorld";
-            version := mkVersion 0 0 1;
-            dependencies :=
-               [path ".juvix-build/stdlib/";
-                git "dep1" "$temp/dep1" "$dep1hash"];
-            main := just "HelloWorld.juvix"};
-        EOF
-
-        cat <<-EOF > juvix.lock.yaml
-        dependencies:
-          - path: .juvix-build/stdlib/
-            dependencies: []
-        EOF
-
-        cat <<-EOF > HelloWorld.juvix
-        -- HelloWorld.juvix
-        module HelloWorld;
-
-        import Stdlib.Prelude open;
-        import HelloDep;
-
-        main : IO := HelloDep.main;
-        EOF
-
-        # compile and run the project
-        juvix compile HelloWorld.juvix
-    stderr:
-      contains: "dep1"
-    stdout:
-      matches:
-        regex: ".*"
-    exit-status: 1
-
   - name: git-dependencies-update-ref-offline
     command:
       shell:

tests/smoke/Commands/compile-dependencies.smoke.yaml

@@ -287,7 +287,11 @@ tests:
         version: 0.1.0
         EOF
 
+        checksum=$(sha256sum juvix.yaml | cut -d " " -f 1)
+
         cat <<-EOF > juvix.lock.yaml
+        version: 2
+        checksum: $checksum
         dependencies:
           - path: .juvix-build/stdlib/
             dependencies: []
@@ -323,6 +327,7 @@ tests:
         cat <<-EOF > HelloDep.juvix
         module HelloDep;
         import HelloDep2 open;
+        import Stdlib.Prelude open;
         main : IO := printStringLn "This is from the second commit" >> printStringLn hello;
         EOF
 
@@ -342,7 +347,7 @@ tests:
         git add -A
         git commit -m "commit2"
 
-        # rebuild the project (should use the first commit
+        # rebuild the project (should use the first commit)
         cd $temp/base
 
         # compile with the new hash
@@ -358,8 +363,7 @@ tests:
         - bash
       script: |
         temp=$(mktemp -d)
-        echo $temp
-        # trap 'rm -rf -- "$temp"' EXIT
+        trap 'rm -rf -- "$temp"' EXIT
 
         # create dependency
         mkdir $temp/dep
@@ -390,7 +394,7 @@ tests:
           - git:
               url: $temp/dep
               name: dep1
-              ref: $dep1hash
+              ref: main
         version: 0.1.0
         EOF
 
@@ -423,18 +427,7 @@ tests:
         # compile project with new ref - should use lockfile ref
         cd $temp/base
 
-        cat <<-EOF > juvix.yaml
-        name: HelloWorld
-        main: HelloWorld.juvix
-        dependencies:
-          - .juvix-build/stdlib
-          - git:
-              url: $temp/dep
-              name: dep1
-              ref: $dep1hash
-        version: 0.1.0
-        EOF
-
+        juvix clean
         juvix compile HelloWorld.juvix
         ./HelloWorld
     stdout:
@@ -643,73 +636,6 @@ tests:
       contains: "Hello from dep1\nHello from dep2"
     exit-status: 0
 
-  - name: git-dependencies-lockfile-missing-dependency
-    command:
-      shell:
-        - bash
-      script: |
-        temp=$(mktemp -d)
-        trap 'rm -rf -- "$temp"' EXIT
-
-        # create dependency
-        mkdir $temp/dep1
-        cd $temp/dep1
-        git init
-
-        cat <<-EOF > HelloDep.juvix
-        module HelloDep;
-        import Stdlib.Prelude open;
-        main : IO := printStringLn "Hello from dep";
-        EOF
-
-        touch juvix.yaml
-
-        git add -A
-        git commit -m "commit1"
-
-        dep1hash=$(git rev-parse HEAD)
-
-        # create project that uses dependency
-        mkdir $temp/base
-        cd $temp/base
-
-        cat <<-EOF > juvix.yaml
-        name: HelloWorld
-        main: HelloWorld.juvix
-        dependencies:
-          - .juvix-build/stdlib/
-          - git:
-              url: $temp/dep1
-              name: dep1
-              ref: $dep1hash
-        version: 0.1.0
-        EOF
-
-        cat <<-EOF > juvix.lock.yaml
-        dependencies:
-          - path: .juvix-build/stdlib/
-            dependencies: []
-        EOF
-
-        cat <<-EOF > HelloWorld.juvix
-        -- HelloWorld.juvix
-        module HelloWorld;
-
-        import Stdlib.Prelude open;
-        import HelloDep;
-
-        main : IO := HelloDep.main;
-        EOF
-
-        # compile and run the project
-        juvix compile HelloWorld.juvix
-    stderr:
-      contains: "dep1"
-    stdout:
-      matches:
-        regex: ".*"
-    exit-status: 1
-
   - name: git-dependencies-update-ref-offline
     command:
       shell:
@@ -1061,6 +987,88 @@ tests:
       contains: duplicate
     exit-status: 1
 
+  - name: lockfile-update-when-package-file-is-updated
+    command:
+      shell:
+        - bash
+      script: |
+        temp=$(mktemp -d)
+        # trap 'rm -rf -- "$temp"' EXIT
+        echo $temp
+
+        # create dependency
+        mkdir $temp/dep
+        cd $temp/dep
+        git init
+
+        cat <<-EOF > HelloDep.juvix
+        module HelloDep;
+        import Stdlib.Prelude open;
+        main : IO := printStringLn "Hello from dep";
+        EOF
+
+        cat <<-EOF > Package.juvix
+        module Package;
+        import PackageDescription.Basic open;
+        package : Package := basicPackage;
+        EOF
+
+        git add -A
+        git commit -m "commit1"
+
+        # create project that uses dependency
+        mkdir $temp/base
+        cd $temp/base
+
+        cat <<-EOF > Package.juvix
+        module Package;
+        import PackageDescription.V1 open;
+        package : Package := defaultPackage {dependencies := [defaultStdlib ; git "dep1" "$temp/dep" "main"]}
+        EOF
+
+        cat <<-EOF > HelloWorld.juvix
+        -- HelloWorld.juvix
+        module HelloWorld;
+
+        import Stdlib.Prelude open;
+        import HelloDep;
+
+        main : IO := HelloDep.main;
+        EOF
+
+        # compile project to create lock file
+        juvix compile HelloWorld.juvix
+
+        cd $temp/dep
+        cat <<-EOF > HelloDep.juvix
+        module HelloDep;
+        import Stdlib.Prelude open;
+        main : IO := printStringLn "Hello from commit2";
+        EOF
+
+        git add -A
+        git commit -m "commit2"
+
+        cd $temp/base
+        juvix clean
+        juvix compile HelloWorld.juvix
+        ./HelloWorld
+
+        # Update the Package file and recompile
+        # it should use the latest commit
+        echo "-- comment" >> Package.juvix
+        juvix clean
+        juvix compile HelloWorld.juvix
+        ./HelloWorld
+
+    stderr: ""
+    stdout:
+      matches:
+        regex: "Hello from dep\n(.*?)Hello from commit2"
+        options:
+          - dot-all
+    exit-status: 0
+
   - name: git-dependencies-no-fetch-if-ref-exists-in-clone
     command:
       shell: