refactor dream.lock sources structure:

- now accessed like sources.{name}.{version}

flake.nix

@@ -74,6 +74,7 @@
           shellHook = ''
             export NIX_PATH=nixpkgs=${nixpkgs}
             export d2nExternalSources=${externalSourcesFor."${system}"}
+            export dream2nixWithExternals=${dream2nixFor."${system}".dream2nixWithExternals}
           '';
         });
       };

src/apps/cli/commands/package.py

@@ -115,8 +115,10 @@ class PackageCommand(Command):
         print(f"fetching source defined via existing dream.lock")
         with open(source) as f:
           sourceDreamLock = json.load(f)
+        sourceMainPackageName = sourceDreamLock['generic']['mainPackageName']
+        sourceMainPackageVersion = sourceDreamLock['generic']['mainPackageVersion']
         sourceSpec =\
-          sourceDreamLock['sources'][sourceDreamLock['generic']['mainPackage']]
+          sourceDreamLock['sources'][sourceMainPackageName][sourceMainPackageVersion]
         source = \
           buildNixFunction("fetchers.fetchSource", source=sourceSpec, extract=True)
 
@@ -268,15 +270,17 @@ class PackageCommand(Command):
     ])
 
     # add main package source
-    mainPackage = lock['generic']['mainPackage']
+    mainPackageName = lock['generic']['mainPackageName']
-    if mainPackage:
+    mainPackageVersion = lock['generic']['mainPackageVersion']
-      mainSource = sourceSpec.copy()
+    mainSource = sourceSpec.copy()
-      if not mainSource:
+    if not mainSource:
-        mainSource = dict(
+      mainSource = dict(
-          type="unknown",
+        type="unknown",
-          version="unknown",
+        version="unknown",
-        )
+      )
-      lock['sources'][mainPackage] = mainSource
+    lock['sources'][mainPackageName] = {
+      mainPackageVersion: mainSource
+    }
 
     # clean up dependency graph
     # remove empty entries

src/apps/cli/commands/update.py

@@ -59,13 +59,15 @@ class UpdateCommand(Command):
 
     cli_py = os.path.abspath(f"{__file__}/../../cli.py")
     # delete the hash
-    mainPackageSource = lock['sources'][lock['generic']['mainPackage']]
+    mainpakcageName = lock['generic']['mainPackage']
+    mainpakcageVersion = lock['generic']['mainpackageVersion']
+    mainPackageSource = lock['sources'][mainPackageName][mainPackageVersion]
     updatedSourceSpec = callNixFunction(
       "fetchers.updateSource",
       source=mainPackageSource,
       newVersion=version,
     )
-    lock['sources'][lock['generic']['mainPackage']] = updatedSourceSpec
+    lock['sources'][mainpakcageName][mainPackageVersion] = updatedSourceSpec
     with tempfile.NamedTemporaryFile("w", suffix="dream.lock") as tmpDreamLock:
       json.dump(lock, tmpDreamLock, indent=2)
       tmpDreamLock.seek(0)  # flushes write cache

src/apps/cli/default.nix

@@ -33,18 +33,26 @@ in
       sourcePathRelative,
     }:
     let
-      mainPackage = dreamLock.generic.mainPackage;
+      mainPackageName = dreamLock.generic.mainPackageName;
+      mainPackageVersion = dreamLock.generic.mainPackageVersion;
     in
     ''
       {
-        dream2nix ? import ${dream2nixLocationRelative} {},
+        dream2nix ? import (
+          let
+            dream2nixWithExternals = (builtins.getEnv "dream2nixWithExternals");
+          in
+            if dream2nixWithExternals != "" then dream2nixWithExternals else
+              throw '''
+                This default.nix is for debugging purposes and can only be evaluated within the dream2nix devShell env.
+              ''') {},
       }:
 
       (dream2nix.riseAndShine {
         dreamLock = ./dream.lock;
-        ${lib.optionalString (dreamLock.sources."${mainPackage}".type == "unknown") ''
+        ${lib.optionalString (dreamLock.sources."${mainPackageName}"."${mainPackageVersion}".type == "unknown") ''
           sourceOverrides = oldSources: {
-              "${mainPackage}" = ./${sourcePathRelative};
+              "${mainPackageName}#${mainPackageVersion}" = ./${sourcePathRelative};
             };
         ''}
       }).package.overrideAttrs (old: {

src/apps/cli/templateDefault.nix

@@ -1,9 +0,0 @@
-{
-  dream2nix,
-}:
-
-(dream2nix.riseAndShine {
-  dreamLock = ./dream.lock;
-}).package.overrideAttrs (old: {
-
-})

src/builders/nodejs/granular/default.nix

@@ -42,20 +42,28 @@ let
       lib.genAttrs standaloneKeys
         (key:
           (builders.nodejs.node2nix (args // {
-            dreamLock = lib.recursiveUpdate dreamLock 
+            dreamLock =
-              {
+              let
-                generic.mainPackage = key;
+                nameVer = lib.splitString "#" key;
+                name = b.elemAt nameVer 0;
+                version = b.elemAt nameVer 1;
+              in
+                lib.recursiveUpdate dreamLock 
+                  {
+                    generic.mainPackagenName = name;
+                    generic.mainPackagenVersion = Version;
 
-                # re-introduce removed dependencies
+                    # re-introduce removed dependencies
-                generic.dependencyGraph."${key}" =
+                    generic.dependencyGraph."${key}" =
-                  # b.trace "re-introduce removed ${b.toString dreamLock.generic.dependenciesRemoved."${key}" or []}"
+                      # b.trace "re-introduce removed ${b.toString dreamLock.generic.dependenciesRemoved."${key}" or []}"
-                  dreamLock.generic.dependencyGraph."${key}"
+                      dreamLock.generic.dependencyGraph."${key}"
-                  ++ dreamLock.generic.dependenciesRemoved."${key}" or [];
+                      ++ dreamLock.generic.dependenciesRemoved."${key}" or [];
-              };
+                  };
           })).package
         );
 
-  mainPackageKey = dreamLock.generic.mainPackage;
+  mainPackageKey =
+    "${dreamLock.generic.mainPackageName}#${dreamLock.generic.mainPackageVersion}";
 
   nodejsVersion = dreamLock.buildSystem.nodejsVersion;
 
@@ -70,125 +78,126 @@ let
 
   allPackages = lib.genAttrs (lib.attrNames fetchedSources) makePackage;
 
-  makePackage = pkgKey:
+  makePackage = name: version:
-    standAlonePackages."${pkgKey}"
+    let
-    or
+      pkgKey = "${name}#${version}";
-    (stdenv.mkDerivation (applyOverrides rec {
+    in
+      standAlonePackages."${name}#${version}"
+      or
+      (stdenv.mkDerivation (applyOverrides rec {
 
-      packageName = (lib.head (lib.splitString "#" pkgKey));
+        packageName = name;
-    
+      
-      pname =
+        pname = lib.straings.sanitizeDerivationName name;
-        lib.replaceStrings [ "@" "/" ] [ "_at_" "__" ]
-          packageName;
 
-      version = dreamLock.sources."${pkgKey}".version;
+        inherit version;
 
-      src = fetchedSources."${pkgKey}";
+        src = fetchedSources."${pkgKey}";
 
-      buildInputs = [ nodejs ];
+        buildInputs = [ nodejs ];
 
-      nodeDeps =
+        nodeDeps =
-        lib.forEach
+          lib.forEach
-          (dependencyGraph."${pkgKey}" or [])
+            (dependencyGraph."${pkgKey}" or [])
-          (depKey:
+            (depKey:
-            allPackages."${depKey}"
+              allPackages."${depKey}"
-          )
+            )
-        ++
+          ++
-        lib.forEach (dreamLock.generic.dependenciesRemoved."${pkgKey}" or [])
+          lib.forEach (dreamLock.generic.dependenciesRemoved."${pkgKey}" or [])
-          (removedDep: standAlonePackages."${pkgKey}");
+            (removedDep: standAlonePackages."${pkgKey}");
 
-      dontUnpack = true;
+        dontUnpack = true;
 
-      installPhase = ''
+        installPhase = ''
-        runHook preInstall
+          runHook preInstall
 
-        nodeModules=$out/lib/node_modules
+          nodeModules=$out/lib/node_modules
 
-        mkdir -p $nodeModules
+          mkdir -p $nodeModules
 
-        cd $TMPDIR
+          cd $TMPDIR
 
-        unpackFile ${src}
+          unpackFile ${src}
 
-        # Make the base dir in which the target dependency resides first
+          # Make the base dir in which the target dependency resides first
-        mkdir -p "$(dirname "$nodeModules/${packageName}")"
+          mkdir -p "$(dirname "$nodeModules/${packageName}")"
 
-        # install source
+          # install source
-        if [ -f "${src}" ]
+          if [ -f "${src}" ]
-        then
-            # Figure out what directory has been unpacked
-            packageDir="$(find . -maxdepth 1 -type d | tail -1)"
-
-            # Restore write permissions
-            find "$packageDir" -type d -exec chmod u+x {} \;
-            chmod -R u+w "$packageDir"
-
-            # Move the extracted tarball into the output folder
-            mv "$packageDir" "$nodeModules/${packageName}"
-        elif [ -d "${src}" ]
-        then
-            strippedName="$(stripHash ${src})"
-
-            # Restore write permissions
-            chmod -R u+w "$strippedName"
-
-            # Move the extracted directory into the output folder
-            mv "$strippedName" "$nodeModules/${packageName}"
-        fi
-
-        # repair 'link:' -> 'file:'
-        mv $nodeModules/${packageName}/package.json $nodeModules/${packageName}/package.json.old
-        cat $nodeModules/${packageName}/package.json.old | sed 's!link:!file\:!g' > $nodeModules/${packageName}/package.json
-        rm $nodeModules/${packageName}/package.json.old
-
-        # symlink dependency packages into node_modules
-        for dep in $nodeDeps; do
-          if [ -e $dep/lib/node_modules ]; then
-            for module in $(ls $dep/lib/node_modules); do
-              if [[ $module == @* ]]; then
-                for submodule in $(ls $dep/lib/node_modules/$module); do
-                  mkdir -p $nodeModules/${packageName}/node_modules/$module
-                  echo "ln -s $dep/lib/node_modules/$module/$submodule $nodeModules/${packageName}/node_modules/$module/$submodule"
-                  ln -s $dep/lib/node_modules/$module/$submodule $nodeModules/${packageName}/node_modules/$module/$submodule
-                done
-              else
-                mkdir -p $nodeModules/${packageName}/node_modules/
-                echo "ln -s $dep/lib/node_modules/$module $nodeModules/${packageName}/node_modules/$module"
-                ln -s $dep/lib/node_modules/$module $nodeModules/${packageName}/node_modules/$module
-              fi
-            done
-          fi
-        done
-
-        cd "$nodeModules/${packageName}"
-
-        export HOME=$TMPDIR
-
-        npm --offline --production --nodedir=${nodeSources} install
-
-        # Create symlink to the deployed executable folder, if applicable
-          if [ -d "$nodeModules/.bin" ]
           then
-            ln -s $nodeModules/.bin $out/bin
+              # Figure out what directory has been unpacked
+              packageDir="$(find . -maxdepth 1 -type d | tail -1)"
+
+              # Restore write permissions
+              find "$packageDir" -type d -exec chmod u+x {} \;
+              chmod -R u+w "$packageDir"
+
+              # Move the extracted tarball into the output folder
+              mv "$packageDir" "$nodeModules/${packageName}"
+          elif [ -d "${src}" ]
+          then
+              strippedName="$(stripHash ${src})"
+
+              # Restore write permissions
+              chmod -R u+w "$strippedName"
+
+              # Move the extracted directory into the output folder
+              mv "$strippedName" "$nodeModules/${packageName}"
           fi
 
-          # Create symlinks to the deployed manual page folders, if applicable
+          # repair 'link:' -> 'file:'
-          if [ -d "$nodeModules/${packageName}/man" ]
+          mv $nodeModules/${packageName}/package.json $nodeModules/${packageName}/package.json.old
-          then
+          cat $nodeModules/${packageName}/package.json.old | sed 's!link:!file\:!g' > $nodeModules/${packageName}/package.json
-              mkdir -p $out/share
+          rm $nodeModules/${packageName}/package.json.old
-              for dir in "$nodeModules/${packageName}/man/"*
+
-              do
+          # symlink dependency packages into node_modules
-                  mkdir -p $out/share/man/$(basename "$dir")
+          for dep in $nodeDeps; do
-                  for page in "$dir"/*
+            if [ -e $dep/lib/node_modules ]; then
-                  do
+              for module in $(ls $dep/lib/node_modules); do
-                      ln -s $page $out/share/man/$(basename "$dir")
+                if [[ $module == @* ]]; then
+                  for submodule in $(ls $dep/lib/node_modules/$module); do
+                    mkdir -p $nodeModules/${packageName}/node_modules/$module
+                    echo "ln -s $dep/lib/node_modules/$module/$submodule $nodeModules/${packageName}/node_modules/$module/$submodule"
+                    ln -s $dep/lib/node_modules/$module/$submodule $nodeModules/${packageName}/node_modules/$module/$submodule
                   done
+                else
+                  mkdir -p $nodeModules/${packageName}/node_modules/
+                  echo "ln -s $dep/lib/node_modules/$module $nodeModules/${packageName}/node_modules/$module"
+                  ln -s $dep/lib/node_modules/$module $nodeModules/${packageName}/node_modules/$module
+                fi
               done
-          fi
+            fi
+          done
 
-        runHook postInstall
+          cd "$nodeModules/${packageName}"
-      '';
+
-    }));
+          export HOME=$TMPDIR
+
+          npm --offline --production --nodedir=${nodeSources} install
+
+          # Create symlink to the deployed executable folder, if applicable
+            if [ -d "$nodeModules/.bin" ]
+            then
+              ln -s $nodeModules/.bin $out/bin
+            fi
+
+            # Create symlinks to the deployed manual page folders, if applicable
+            if [ -d "$nodeModules/${packageName}/man" ]
+            then
+                mkdir -p $out/share
+                for dir in "$nodeModules/${packageName}/man/"*
+                do
+                    mkdir -p $out/share/man/$(basename "$dir")
+                    for page in "$dir"/*
+                    do
+                        ln -s $page $out/share/man/$(basename "$dir")
+                    done
+                done
+            fi
+
+          runHook postInstall
+        '';
+      }));
 
   package = makePackage mainPackageKey;
 

src/builders/nodejs/node2nix/default.nix

@@ -17,9 +17,13 @@
   ...
 }@args:
 let
+  b = builtins;
+
   dreamLock = utils.readDreamLock { inherit (args) dreamLock; };
 
-  mainPackageName = dreamLock.generic.mainPackage;
+  mainPackageName = dreamLock.generic.mainPackageName;
+  mainPackageVersion = dreamLock.generic.mainPackageVersion;
+  mainPackageKey = "${mainPackageName}#${mainPackageVersion}";
 
   nodejsVersion = dreamLock.buildSystem.nodejsVersion;
 
@@ -33,17 +37,18 @@ let
     let
       makeSource = name: 
         let
-          packageName = lib.head (lib.splitString "#" name);
+          nameVer = lib.splitString "#" name;
+          packageName = lib.elemAt nameVer 0;
+          version = lib.elemAt nameVer 1;
         in
           {
-            inherit packageName;
+            inherit packageName version;
             name = lib.strings.sanitizeDerivationName packageName;
-            version = dreamLock.sources."${name}".version;
             src = fetchedSources."${name}";
             dependencies =
               lib.forEach
                 (lib.filter
-                  (depName: ! builtins.elem depName dreamLock.generic.dependencyGraph."${mainPackageName}")
+                  (depName: ! builtins.elem depName dreamLock.generic.dependencyGraph."${mainPackageKey}")
                   (dreamLock.generic.dependencyGraph."${name}" or []))
                 (dependency:
                   makeSource dependency
@@ -51,14 +56,14 @@ let
           };
     in
       lib.forEach
-        dreamLock.generic.dependencyGraph."${mainPackageName}"
+        dreamLock.generic.dependencyGraph."${mainPackageKey}"
         (dependency: makeSource dependency);
 
   callNode2Nix = funcName: args:
     node2nixEnv."${funcName}" rec {
       name = lib.strings.sanitizeDerivationName packageName;
-      packageName = lib.head (lib.splitString "#" mainPackageName);
+      packageName = mainPackageName;
-      version = dreamLock.sources."${mainPackageName}".version;
+      version = mainPackageVersion;
       dependencies = node2nixDependencies;
       # buildInputs ? []
       # npmFlags ? ""
@@ -71,7 +76,7 @@ let
       production = true;
       bypassCache = true;
       reconstructLock = true;
-      src = fetchedSources."${dreamLock.generic.mainPackage}";
+      src = fetchedSources."${mainPackageKey}";
     }
     // args;
 

src/builders/python/simple-builder/default.nix

@@ -20,7 +20,7 @@ let
     else
       python.pkgs.buildPythonPackage;
 
-  mainPackageName = dreamLock.generic.mainPackage;
+  mainPackageName = dreamLock.generic.mainPackageName;
 
   packageName =
     if mainPackageName == null then

src/default.nix

@@ -163,7 +163,7 @@ rec {
       };
 
       dreamLock = lib.recursiveUpdate dreamLock' {
-        sources."${dreamLock'.generic.mainPackage}" = {
+        sources."${dreamLock'.generic.mainPackageName}"."${dreamLock'.generic.mainPackageVersion}" = {
           type = "path";
           path = source;
           version = "unknown";

src/fetchers/default-fetcher.nix

@@ -15,11 +15,24 @@
 
 {
   # attrset: pname -> path of downloaded source
-  fetchedSources = lib.mapAttrs (pname: source:
+  fetchedSources =
-    if source.type == "unknown" then
+    lib.listToAttrs
-      "unknown"
+      (lib.flatten
-    else if fetchers.fetchers ? "${source.type}" then
+        (lib.mapAttrsToList
-      fetchSource { inherit source; }
+          (pname: versions:
-    else throw "unsupported source type '${source.type}'"
+            # list of name value pairs
-  ) sources;
+            lib.mapAttrsToList
+              (version: source:
+                lib.nameValuePair
+                  "${pname}#${version}"
+                  (if source.type == "unknown" then
+                    "unknown"
+                  else if fetchers.fetchers ? "${source.type}" then
+                    fetchSource { inherit source; }
+                  else throw "unsupported source type '${source.type}'")
+              )
+              versions
+          )
+          sources))
+  ;
 }

src/specifications/dream-lock-example.json

@@ -3,7 +3,8 @@
     "buildSystem": "python",
     "translatedBy": "python.impure.pip",
     "translatorArgs": "",
-    "mainPackage": "requests",
+    "mainPackageName": "requests",
+    "mainPackageVersion": "1.2.3",
     "removedDependencies": true,
     "dependencyGraph": {
       "requests": [
@@ -21,17 +22,21 @@
   },
 
   "sources": {
-    "requests#1.2.3": {
+    "requests": {
-      "url": "https://download.pypi.org/requests/2.28.0",
+      "1.2.3": {
-      "hash": "000000000000000000000000000000000000000",
+        "url": "https://download.pypi.org/requests/2.28.0",
-      "version": "1.2.3",
+        "hash": "000000000000000000000000000000000000000",
-      "type": "fetchurl"
+        "version": "1.2.3",
+        "type": "fetchurl"
+      }
     },
-    "certifi#2.3.4": {
+    "certifi": {
-      "url": "https://download.pypi.org/certifi/2.0",
+      "2.3.4": {
-      "hash": "000000000000000000000000000000000000000",
+        "url": "https://download.pypi.org/certifi/2.0",
-      "version": "2.3.4",
+        "hash": "000000000000000000000000000000000000000",
-      "type": "fetchurl"
+        "version": "2.3.4",
+        "type": "fetchurl"
+      }
     }
   }
 }

src/templates/translators/pure.nix

@@ -26,7 +26,8 @@
       generic = {
         buildSystem = "nodejs";
         producedBy = translatorName;
-        mainPackage = parsed.name;
+        mainPackageName = "some_name";
+        mainPackageVersion = "some_version";
         dependencyGraph = ;
         sourcesCombinedHash = null;
       };

src/translators/python/impure/pip/default.nix

@@ -89,7 +89,8 @@ in
       # -r ''${inputFiles/$'\n'/$' -r '}
 
     # generate the generic lock from the downloaded list of files
-    MAIN=$(${jq}/bin/jq '.name' -c -r $tmpBuild/python.json) \
+    NAME=$(${jq}/bin/jq '.name' -c -r $tmpBuild/python.json) \
+        VERSION=$(${jq}/bin/jq '.version' -c -r $tmpBuild/python.json) \
       $tmpBuild/python/bin/python ${./generate-dream-lock.py} $tmp $jsonInput
 
     rm -rf $tmp $tmpBuild

src/translators/python/impure/pip/generate-dream-lock.py

@@ -36,7 +36,7 @@ def main():
     # example: requests-2.26.0.tar.gz
     else:
       format = 'sdist'
-      pname, _, _ = file.rpartition('-')
+      pname, version, _ = file.rpartition('-')
       pyver = 'source'
 
     url = f"https://files.pythonhosted.org/packages/{pyver}/{pname[0]}/{pname}/{file}"
@@ -45,6 +45,7 @@ def main():
       sha256 = f"sha256-{base64.b64encode(hashlib.sha256(f.read()).digest()).decode()}"
 
     packages[pname] = dict(
+      version=version,
       url=url,
       sha256=sha256,
       format=format
@@ -58,7 +59,8 @@ def main():
     sources={},
     generic={
       "buildSystem": "python",
-      "mainPackage": os.environ.get('MAIN'),
+      "mainPackageName": os.environ.get('NAME'),
+      "mainPackageVersion": os.environ.get('VERSION'),
 
       "sourcesCombinedHash": None,
     },
@@ -72,7 +74,9 @@ def main():
 
   # populate sources of generic lock
   for pname, data in packages.items():
-    dream_lock['sources'][pname] = dict(
+    if pname not in dream_lock['sources']:
+      dream_lock['sources'][pname] = {}
+    dream_lock['sources'][pname][data['version']] = dict(
       url=data['url'],
       hash=data['sha256'],
       type='fetchurl',

src/utils/dream-lock.nix

@@ -22,7 +22,9 @@ let
       lock;
 
   getMainPackageSource = dreamLock:
-    dreamLock.sources."${dreamLock.generic.mainPackage}";
+    dreamLock.sources
+      ."${dreamLock.generic.mainPackageName}"
+      ."${dreamLock.generic.mainPackageVersion}";
 
 
 in

src/utils/translator.nix

@@ -51,14 +51,16 @@ let
 
       sources = b.foldl'
         (result: pkgData: lib.recursiveUpdate result {
-          "${getName pkgData}#${getVersion pkgData}" =
+          "${getName pkgData}" = {
-            let
+            "${getVersion pkgData}" =
-              type = getSourceType pkgData;
+              let
-              constructedArgs = 
+                type = getSourceType pkgData;
-                (sourceConstructors."${type}" pkgData)
+                constructedArgs = 
-                // { inherit type; };
+                  (sourceConstructors."${type}" pkgData)
-            in
+                  // { inherit type; };
-              fetchers.constructSource constructedArgs;
+              in
+                fetchers.constructSource constructedArgs;
+          };
         })
         {}
         serializedPackagesList;
@@ -91,8 +93,8 @@ let
 
           generic =
             {
+              inherit mainPackageName mainPackageVersion;
               buildSystem = buildSystemName;
-              mainPackage = "${mainPackageName}#${mainPackageVersion}";
               sourcesCombinedHash = null;
               translator = translatorName;
             }