Add builder for nodejs

- python builder support application
- add version to dream.lock
- allowBuiltinsFetchers config option
- node2nix builder
- handle github sources without hash

flake.lock

@@ -15,6 +15,22 @@
         "type": "indirect"
       }
     },
+    "node2nix": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1613594272,
+        "narHash": "sha256-fcnPNexhowSkatLSl+0dat5oDaWKH53Pg+VKrE8+x+Q=",
+        "owner": "svanderburg",
+        "repo": "node2nix",
+        "rev": "0c94281ea98f1b17532176106f90f909aa133704",
+        "type": "github"
+      },
+      "original": {
+        "owner": "svanderburg",
+        "repo": "node2nix",
+        "type": "github"
+      }
+    },
     "npmlock2nix": {
       "flake": false,
       "locked": {
@@ -34,6 +50,7 @@
     "root": {
       "inputs": {
         "nixpkgs": "nixpkgs",
+        "node2nix": "node2nix",
         "npmlock2nix": "npmlock2nix"
       }
     }

flake.nix

@@ -3,10 +3,11 @@
 
   inputs = {
     nixpkgs.url = "nixpkgs/nixos-unstable";
+    node2nix = { url = "github:svanderburg/node2nix"; flake = false; };
     npmlock2nix = { url = "github:nix-community/npmlock2nix"; flake = false; };
   };
 
-  outputs = { self, nixpkgs, npmlock2nix }:
+  outputs = { self, nixpkgs, node2nix, npmlock2nix }:
     let
 
       lib = nixpkgs.lib;
@@ -20,13 +21,16 @@
         overlays = [ self.overlay ];
       });
 
+      externalSourcesFor = forAllSystems (system: nixpkgsFor."${system}".runCommand "dream2nix-vendored" {} ''
+        mkdir -p $out/{npmlock2nix,node2nix}
+        cp ${npmlock2nix}/{internal.nix,LICENSE} $out/npmlock2nix/
+        cp ${node2nix}/{nix/node-env.nix,LICENSE} $out/node2nix/
+      '');
+
       dream2nixFor = forAllSystems (system: import ./src rec {
         pkgs = nixpkgsFor."${system}";
+        externalSources = externalSourcesFor."${system}";
         inherit lib;
-        externalSources = pkgs.runCommand "dream2nix-imported" {} ''
-          mkdir -p $out/npmlock2nix
-          cp ${npmlock2nix}/{internal.nix,LICENSE} $out/npmlock2nix/
-        '';
       });
 
     in
@@ -54,6 +58,7 @@
           ];
           shellHook = ''
             export NIX_PATH=nixpkgs=${nixpkgs}
+            export d2nExternalSources=${externalSourcesFor."${system}"}
           '';
         });
       };

specifications/generic-lock-example.json

@@ -3,11 +3,13 @@
     "requests": {
       "url": "https://download.pypi.org/requests/2.28.0",
       "hash": "000000000000000000000000000000000000000",
+      "version": "1.2.3",
       "type": "fetchurl"
     },
     "certifi": {
       "url": "https://download.pypi.org/certifi/2.0",
       "hash": "000000000000000000000000000000000000000",
+      "version": "2.3.4",
       "type": "fetchurl"
     }
   },
@@ -32,4 +34,4 @@
       "certifi": "wheel"
     }
   }
-}
+}

specifications/subsystems/nodejs/dream.lock.example.json

@@ -0,0 +1,5 @@
+{
+  "buildSystem": {
+    "nodejsVersion": 14
+  }
+}

src/apps/cli.py

@@ -13,7 +13,8 @@ with open (os.environ.get("translatorsJsonFile")) as f:
 
 def strip_hashes_from_lock(lock):
   for source in lock['sources'].values():
-    del source['hash']
+    if 'hash' in source:
+      del source['hash']
 
 
 def order_dict(d):
@@ -75,7 +76,9 @@ def translate(args):
       raise Exception(f"Input path '{path}' does not exist")
 
   inputFiles = list(filter(lambda p: os.path.isfile(p), inputPaths))
+  inputFiles = list(map(lambda p:os.path.realpath(p), inputFiles))
   inputDirectories = list(filter(lambda p: os.path.isdir(p), inputPaths))
+  inputDirectories = list(map(lambda p:os.path.realpath(p), inputDirectories))
 
   # determine output directory
   if os.path.isdir(args.output):
@@ -162,6 +165,17 @@ def translate(args):
   lock['generic']['translatedBy'] = f"{subsystem}.{trans_type}.{trans_name}"
   lock['generic']['translatorParams'] = " ".join(sys.argv[2:])
 
+  # clean up dependency graph
+  # remove empty entries
+  if 'dependencyGraph' in lock['generic']:
+    for pname, deps in lock['generic']['dependencyGraph'].copy().items():
+      if not deps:
+        del lock['generic']['dependencyGraph'][pname]
+  
+  # re-write dream.lock
+  with open(output, 'w') as f:
+    json.dump(lock, f, indent=2)
+
   # calculate combined hash if --combined was specified
   if args.combined:
 

src/apps/default.nix

@@ -43,6 +43,9 @@ in
         mkdir $target/external
         cp -r ${externalSources}/* $target/external/
         chmod -R +w $target
+
+        echo "Installed dream2nix successfully to '$target'."
+        echo "Please check/modify settings in '$target/config.json'"
       ''
   ) {};
 }

src/builders/default.nix

@@ -9,5 +9,12 @@
 
     simpleBuilder = callPackage ./python/simple-builder {};
   };
+
+  nodejs =  rec {
+
+    default = node2nix;
+
+    node2nix = callPackage ./nodejs/node2nix {};
+  };
   
 }

src/builders/nodejs/node2nix/default.nix

@@ -0,0 +1,72 @@
+# builder imported from node2nix
+
+{
+  externals,
+  node2nix ? externals.node2nix,
+
+  lib,
+  pkgs,
+  ...
+}:
+
+{
+  fetchedSources,
+  dreamLock,
+}:
+let
+  mainPackageName = dreamLock.generic.mainPackage;
+
+  nodejsVersion = dreamLock.buildSystem.nodejsVersion;
+
+  nodejs =
+    pkgs."nodejs-${builtins.toString nodejsVersion}_x"
+    or (throw "Could not find nodejs version '${nodejsVersion}' in pkgs");
+
+  node2nixEnv = node2nix nodejs;
+
+  # make node2nix compatible sources
+  makeSource = name: {
+    name = lib.head (lib.splitString "#" name);
+    packageName = lib.head (lib.splitString "#" name);
+    version = dreamLock.sources."${name}".version;
+    src = fetchedSources."${name}";
+    dependencies = lib.forEach dreamLock.generic.dependencyGraph."${name}" or [] (dependency:
+      makeSource dependency
+    );
+  };
+
+  callNode2Nix = funcName: args:
+    node2nixEnv."${funcName}" rec {
+      name = mainPackageName;
+      packageName = name;
+      version = dreamLock.sources."${mainPackageName}".version;
+      dependencies =
+        lib.forEach
+          (lib.filter
+            (pname: pname != mainPackageName)
+            (lib.attrNames dreamLock.generic.dependencyGraph)
+          )
+          (dependency: makeSource dependency);
+      # buildInputs ? []
+      # npmFlags ? ""
+      # dontNpmInstall ? false
+      # preRebuild ? ""
+      # dontStrip ? true
+      # unpackPhase ? "true"
+      # buildPhase ? "true"
+      # meta ? {}
+      production = true;
+      bypassCache = true;
+      reconstructLock = true;
+      src = fetchedSources."${dreamLock.generic.mainPackage}";
+    }
+    // args;
+
+in
+{
+
+  package = callNode2Nix "buildNodePackage" {};
+
+  shell = callNode2Nix "buildNodeShell" {};
+    
+}

src/builders/python/simple-builder/default.nix

@@ -1,3 +1,5 @@
+# A very simple single derivation python builder
+
 {
   lib,
   pkgs,
@@ -11,12 +13,29 @@
 
 let
   python = pkgs."${dreamLock.buildSystem.pythonAttr}";
+
+  buildFunc =
+    if dreamLock.buildSystem.application then
+      python.pkgs.buildPythonApplication
+    else
+      python.pkgs.buildPythonPackage;
+
+  mainPackageName = dreamLock.generic.mainPackage;
+
+  packageName =
+    if mainPackageName == null then
+      if dreamLock.buildSystem.application then
+        "application"
+      else
+        "environment"
+    else
+      mainPackageName;
 in
 
-python.pkgs.buildPythonPackage {
-  name = "python-environment";
+buildFunc {
+  name = packageName;
   format = "";
-  src = lib.attrValues fetchedSources;
+  src = fetchedSources."${toString (mainPackageName)}" or null;
   buildInputs = pkgs.pythonManylinuxPackages.manylinux1;
   nativeBuildInputs = [ pkgs.autoPatchelfHook python.pkgs.wheelUnpackHook ];
   unpackPhase = ''
@@ -32,7 +51,7 @@ python.pkgs.buildPythonPackage {
     runHook preInstall
     mkdir -p "$out/${python.sitePackages}"
     export PYTHONPATH="$out/${python.sitePackages}:$PYTHONPATH"
-    ${python}/bin/python -m pip install ./dist/*.{whl,tar.gz,zip} \
+    ${python}/bin/python -m pip install ./dist/*.{whl,tar.gz,zip} $src \
       --no-index \
       --no-warn-script-location \
       --prefix="$out" \

src/config.json

@@ -0,0 +1,3 @@
+{
+  "allowBuiltinFetchers": true
+}

src/default.nix

@@ -2,8 +2,10 @@
   pkgs ? import <nixpkgs> {},
   lib ? pkgs.lib,
   externalSources ?
-    if builtins.getEnv "d2nExternalSources" != "" then
+    # if called via CLI, load externals via env
+    if builtins ? getEnv && builtins.getEnv "d2nExternalSources" != "" then
       builtins.getEnv "d2nExternalSources"
+    # load from default dircetory
     else
       ./external,
 }:
@@ -14,24 +16,35 @@ let
 
   callPackage = f: args: pkgs.callPackage f (args // {
     inherit callPackage;
+    inherit externals;
+    inherit externalSources;
     inherit utils;
   });
 
   externals = {
     npmlock2nix = pkgs.callPackage "${externalSources}/npmlock2nix/internal.nix" {};
+    node2nix = nodejs: pkgs.callPackage "${externalSources}/node2nix/node-env.nix" { inherit nodejs; };
   };
 
+  config = builtins.fromJSON (builtins.readFile ./config.json);
+
 in
 
 rec {
 
-  apps = callPackage ./apps { inherit externalSources location translators; };
+  # apps for CLI and installation
+  apps = callPackage ./apps { inherit location translators; };
 
+  # builder implementaitons for all subsystems
   builders = callPackage ./builders {};
 
-  fetchers = callPackage ./fetchers {};
+  # fetcher implementations
+  fetchers = callPackage ./fetchers {
+    inherit (config) allowBuiltinFetchers;
+  };
 
-  translators = callPackage ./translators { inherit externalSources externals location; };
+  # the translator modules and utils for all subsystems
+  translators = callPackage ./translators { inherit location; };
 
 
   # the location of the dream2nix framework for self references (update scripts, etc.)
@@ -43,7 +56,10 @@ rec {
     let
       buildSystem = dreamLock.generic.buildSystem;
     in
-      builders."${buildSystem}".default;
+      if ! builders ? "${buildSystem}" then
+        throw "Could not find any builder for subsystem '${buildSystem}'"
+      else
+        builders."${buildSystem}".default;
 
 
   # detect if granular or combined fetching must be used
@@ -54,24 +70,27 @@ rec {
         fetchers.defaultFetcher;
 
 
+  # automatically parse dream.lock if passed as file
   parseLock = lock:
     if builtins.isPath lock || builtins.isString lock then
       builtins.fromJSON (builtins.readFile lock)
     else
       lock;
 
-
+  # fetch only sources and do not build
   fetchSources =
     {
       dreamLock,
       builder ? findBuilder (parseLock dreamLock),
       fetcher ? findFetcher (parseLock dreamLock),
       sourceOverrides ? oldSources: {},
+      allowBuiltinFetchers ? true,
     }:
     let
       # if generic lock is a file, read and parse it
       dreamLock' = (parseLock dreamLock);
       fetched = fetcher {
+        inherit allowBuiltinFetchers;
         sources = dreamLock'.sources;
         sourcesCombinedHash = dreamLock'.generic.sourcesCombinedHash;
       };
@@ -98,13 +117,15 @@ rec {
       };
 
 
-  # automatically build package defined by generic lock
+  # build package defined by dream.lock
+  # TODO: rename to riseAndShine
   buildPackage = 
     {
       dreamLock,
       builder ? findBuilder (parseLock dreamLock),
       fetcher ? findFetcher (parseLock dreamLock),
       sourceOverrides ? oldSources: {},
+      allowBuiltinFetchers ? true,
     }@args:
     let
       # if generic lock is a file, read and parse it

src/fetchers/default-fetcher.nix

@@ -11,30 +11,74 @@
 {
   # sources attrset from generic lock
   sources,
+  allowBuiltinFetchers,
   ...
 }:
+
+let
+  githubMissingHashErrorText = pname: ''
+    Error: Cannot verify the integrity of the source of '${pname}'
+    It is a github reference with no hash providedand.
+    Solve this problem via any of the wollowing ways:
+
+    - (alternative 1): allow the use of builtin fetchers (which can verify using git rev).
+      ```
+        dream2nix.buildPackage {
+          ...
+          allowBuiltinFetchers = true;
+          ...
+        }
+      ```
+
+    - (alternative 2): add a hash to the source via override
+      ```
+        dream2nix.buildPackage {
+          ...
+          sourceOverrides = oldSources: {
+            "${pname}" = oldSources."${pname}".overrideAttrs (_:{
+              hash = "";
+            })
+          }
+          ...
+        }
+      ```
+
+  '';
+in
+
 {
   # attrset: pname -> path of downloaded source
   fetchedSources = lib.mapAttrs (pname: source:
     if source.type == "github" then
-      fetchFromGitHub {
-        inherit (source) url owner repo rev;
-        sha256 = source.hash or null;
-      }
+      # handle when no hash is provided
+      if ! source ? hash then
+        if allowBuiltinFetchers then
+          builtins.fetchGit {
+            inherit (source) rev;
+            allRefs = true;
+            url = "https://github.com/${source.owner}/${source.repo}";
+          }
+        else
+          throw githubMissingHashErrorText pname
+      else
+        fetchFromGitHub {
+          inherit (source) url owner repo rev;
+          hash = source.hash or null;
+        }
     else if source.type == "gitlab" then 
       fetchFromGitLab {
         inherit (source) url owner repo rev;
-        sha256 = source.hash or null;
+        hash = source.hash or null;
       }
     else if source.type == "git" then
       fetchgit {
         inherit (source) url rev;
-        sha256 = source.hash or null;
+        hash = source.hash or null;
       }
     else if source.type == "fetchurl" then
       fetchurl {
         inherit (source) url;
-        sha256 = source.hash or null;
+        hash = source.hash or null;
       }
     else if source.type == "unknown" then
       "unknown"

src/fetchers/default.nix

@@ -1,5 +1,8 @@
 {
   callPackage,
+
+  # config
+  allowBuiltinFetchers,
   ...
 }:
 rec {

src/translators/default.nix

@@ -26,7 +26,12 @@ let
           translateBin = wrapPureTranslator [ subsystem type name ];
         };
     in
-      translatorWithBin // { inherit subsystem type name; };
+      translatorWithBin // {
+        inherit subsystem type name;
+        translate = args:
+          translator.translate
+            ((getSpecialArgsDefaults translator.specialArgs or {}) // args);
+      };
       
 
   buildSystems = dirNames ./.;
@@ -45,7 +50,7 @@ let
       bin = pkgs.writeScriptBin "translate" ''
         #!${pkgs.bash}/bin/bash
 
-        jsonInputFile=$1
+        jsonInputFile=$(realpath $1)
         outputFile=$(${pkgs.jq}/bin/jq '.outputFile' -c -r $jsonInputFile)
         export d2nExternalSources=${externalSources}
 
@@ -165,19 +170,21 @@ let
         lib.head (lib.attrValues (lib.head (lib.attrValues (lib.head (lib.attrValues compatTranslators)))))
   );
 
+  getSpecialArgsDefaults = specialArgsDef:
+    lib.mapAttrs
+      (name: def:
+        if def.type == "flag" then
+          false
+        else
+          def.default
+      )
+      specialArgsDef;
+
   selectTranslatorJSON = args:
     let
       translator = (selectTranslator args);
       data = {
-        SpecialArgsDefaults =
-          lib.mapAttrs
-            (name: def:
-              if def.type == "flag" then
-                false
-              else
-                def.default
-            )
-            translator.specialArgs or {};
+        SpecialArgsDefaults = getSpecialArgsDefaults (translator.specialArgs or {});
         inherit (translator) subsystem type name;
       };
     in

src/translators/nodejs/pure/npmlock2nix/default.nix

@@ -12,23 +12,116 @@
     {
       inputDirectories,
       inputFiles,
+
+      dev,
       ...
     }:
     let
       parsed = externals.npmlock2nix.readLockfile (builtins.elemAt inputFiles 0);
+
+      parseGithubDepedency = dependency:
+        externals.npmlock2nix.parseGitHubRef dependency.version;
+
+      getVersion = dependency:
+        if dependency ? from && dependency ? version then
+          builtins.substring 0 8 (parseGithubDepedency dependency).rev
+        else
+          dependency.version;
+
+      pinVersions = dependencies: parentScopeDeps:
+        lib.mapAttrs
+          (pname: pdata:
+            let
+              selfScopeDeps = parentScopeDeps // (pdata.dependencies or {});
+            in
+              pdata // {
+                depsExact =
+                  if ! pdata ? requires then
+                    []
+                  else
+                    lib.forEach (lib.attrNames pdata.requires) (reqName:
+                      "${reqName}#${getVersion selfScopeDeps."${reqName}"}"
+                    );
+                dependencies = pinVersions (pdata.dependencies or {}) selfScopeDeps;
+              }
+          )
+          dependencies;
+      
+      packageLockWithPinnedVersions = pinVersions parsed.dependencies parsed.dependencies;
+
+      # recursively collect dependencies
+      parseDependencies = dependencies:
+        lib.mapAttrsToList  # returns list of lists
+          (pname: pdata:
+            if ! dev && pdata.dev or false then
+              []
+            else
+              # handle github dependency
+              if pdata ? from && pdata ? version then
+                let
+                  githubData = parseGithubDepedency pdata;
+                in
+                [ rec {
+                  name = "${pname}#${version}";
+                  version = builtins.substring 0 8 githubData.rev;
+                  owner = githubData.org;
+                  repo = githubData.repo;
+                  rev = githubData.rev;
+                  type = "github";
+                  depsExact = pdata.depsExact;
+                }]
+              # handle http(s) dependency
+              else
+                [rec {
+                  name = "${pname}#${version}";
+                  version = pdata.version;
+                  url = pdata.resolved;
+                  type = "fetchurl";
+                  hash = pdata.integrity;
+                  depsExact = pdata.depsExact;
+                }]
+            ++
+            (lib.optionals (pdata ? dependencies)
+              (lib.flatten (parseDependencies pdata.dependencies))
+            )
+          )
+          dependencies;
     in
-    {
-      sources = builtins.mapAttrs (pname: pdata:{
-        url = pdata.resolved;
-        type = "fetchurl";
-        hash = pdata.integrity;
-      }) parsed.dependencies;
+    rec {
+      sources =
+        let
+          lockedSources = lib.listToAttrs (
+            map
+              (dep: lib.nameValuePair
+                dep.name
+                (
+                  if dep.type == "github" then
+                    { inherit (dep) type version owner repo rev; }
+                  else
+                    { inherit (dep) type version url hash; }
+                )
+              )
+              (lib.flatten (parseDependencies packageLockWithPinnedVersions))
+          );
+        in
+          # if only a package-lock.json is given, the main source is missing
+          lockedSources // {
+            "${parsed.name}" = {
+              type = "unknown";
+              version = parsed.version;
+            };
+          };
 
       generic = {
         buildSystem = "nodejs";
         producedBy = translatorName;
-        mainPackage = null;
-        dependencyGraph = null;
+        mainPackage = parsed.name;
+        dependencyGraph =
+          lib.listToAttrs 
+          (map
+            (dep: lib.nameValuePair dep.name dep.depsExact )
+            (lib.flatten (parseDependencies packageLockWithPinnedVersions))
+          );
         sourcesCombinedHash = null;
       };
 
@@ -42,10 +135,18 @@
       inputDirectories,
       inputFiles,
     }@args:
-    builtins.trace (lib.attrValues args)
     {
       inputDirectories = [];
       inputFiles =
         lib.filter (f: builtins.match ".*(package-lock\\.json)" f != null) args.inputFiles;
     };
+
+  specialArgs = {
+
+    dev = {
+      description = "include dependencies for development";
+      type = "flag";
+    };
+
+  };
 }