Merge pull request #38 from DavHau/dev

flakes output schema + nodejs builder improvements

There are now 2 different entry points for dream2nix.
  1. The ./default.nix requires pkgs to be passed and produces outputs only for that specific system
  2. The ./lib.nix is a wrapper for the previous, allowing to pass a list of systems or pkgs, and will then produce outputs for several systems. The output schema is the same as with flakes.

flake.lock

@@ -62,29 +62,12 @@
         "type": "github"
       }
     },
-    "npmlock2nix": {
-      "flake": false,
-      "locked": {
-        "lastModified": 1631558099,
-        "narHash": "sha256-xguvgtrIQHPxpc4J6EhfBnYtQDGJpt6hK1dN7KEi8R4=",
-        "owner": "nix-community",
-        "repo": "npmlock2nix",
-        "rev": "33eb3300561d724da64a46ab8e9a05a9cfa9264b",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-community",
-        "repo": "npmlock2nix",
-        "type": "github"
-      }
-    },
     "root": {
       "inputs": {
         "mach-nix": "mach-nix",
         "nix-parsec": "nix-parsec",
         "nixpkgs": "nixpkgs",
-        "node2nix": "node2nix",
-        "npmlock2nix": "npmlock2nix"
+        "node2nix": "node2nix"
       }
     }
   },

flake.nix

@@ -12,14 +12,13 @@
 
     # required for builder nodejs/node2nix
     node2nix = { url = "github:svanderburg/node2nix"; flake = false; };
-
-    # required for translator nodejs/pure/npmlock2nix
-    npmlock2nix = { url = "github:nix-community/npmlock2nix"; flake = false; };
   };
 
-  outputs = { self, mach-nix, nix-parsec, nixpkgs, node2nix, npmlock2nix }:
+  outputs = { self, mach-nix, nix-parsec, nixpkgs, node2nix, }:
     let
 
+      b = builtins;
+
       lib = nixpkgs.lib;
 
       supportedSystems = [ "x86_64-linux" "x86_64-darwin" ];
@@ -28,14 +27,20 @@
         f system (import nixpkgs { inherit system; overlays = [ self.overlay ]; })
       );
 
-      externalSourcesFor = forAllSystems (system: pkgs: pkgs.runCommand "dream2nix-vendored" {} ''
-        mkdir -p $out/{mach-nix-lib,npmlock2nix,node2nix,nix-parsec}
+      # To use dream2nix in non-flake + non-IFD enabled repos, the source code of dream2nix
+      # must be installed into that repo (using nix run dream2nix#install).
+      # The problem is, we also need to install all of dream2nix' dependecies as well.
+      # Therefore 'externalSourcesFor' contains all relevant files of external projects we depend on.
+      makeExternalSources = pkgs: pkgs.runCommand "dream2nix-external" {} ''
+        mkdir -p $out/{mach-nix-lib,node2nix,nix-parsec}
         cp ${mach-nix}/{lib/extractor/{default.nix,distutils.patch,setuptools.patch},LICENSE} $out/mach-nix-lib/
-        cp ${npmlock2nix}/{internal.nix,LICENSE} $out/npmlock2nix/
         cp ${node2nix}/{nix/node-env.nix,LICENSE} $out/node2nix/
         cp ${nix-parsec}/{parsec,lexer}.nix $out/nix-parsec/
-      '');
+      '';
 
+      externalSourcesFor = forAllSystems (system: makeExternalSources);
+
+      # system specific dream2nix api
       dream2nixFor = forAllSystems (system: pkgs: import ./src rec {
         externalSources = externalSourcesFor."${system}";
         inherit pkgs;
@@ -44,31 +49,44 @@
 
     in
       {
+        # overlay with flakes enabled nix
+        # (all of dream2nix cli dependends on nix ^2.4)
         overlay = new: old: {
           nix = old.writeScriptBin "nix" ''
             ${new.nixUnstable}/bin/nix --option experimental-features "nix-command flakes" "$@"
           '';
         };
 
-        lib.dream2nix = dream2nixFor;
+        # System independent dream2nix api.
+        # Similar to drem2nixFor but will require 'system(s)' or 'pkgs' as an argument.
+        # Produces flake-like output schema.
+        lib.dream2nix = import ./src/lib.nix {
+          inherit makeExternalSources lib;
+          nixpkgsSrc = "${nixpkgs}";
+        };
 
+        # the dream2nix cli to be used with 'nix run dream2nix'
         defaultApp = forAllSystems (system: pkgs: self.apps."${system}".cli);
 
+        # all apps including cli, install, etc.
         apps = forAllSystems (system: pkgs:
           lib.mapAttrs (appName: app:
             {
               type = "app";
-              program = builtins.toString app.program;
+              program = b.toString app.program;
             }
           ) dream2nixFor."${system}".apps.apps
         );
 
+        # a dev shell for working on dream2nix
+        # use via 'nix develop . -c $SHELL'
         devShell = forAllSystems (system: pkgs: pkgs.mkShell {
 
           buildInputs = with pkgs;
             (with pkgs; [
               nixUnstable
             ])
+            # using linux is highly recommended as cntr is amazing for debugging builds
             ++ lib.optionals stdenv.isLinux [ cntr ];
 
           shellHook = ''

src/apps/cli/default.nix

@@ -55,7 +55,7 @@ in
               "${mainPackageName}#${mainPackageVersion}" = ./${sourcePathRelative};
             };
         ''}
-      }).package.overrideAttrs (old: {
+      }).defaultPackage.overrideAttrs (old: {
 
       })
   '';

src/builders/nodejs/granular/default.nix

@@ -1,4 +1,5 @@
 {
+  jq,
   lib,
   pkgs,
   runCommand,
@@ -59,7 +60,9 @@ let
     mv node-* $out
   '';
 
-  allPackages =
+  defaultPackage = packages."${mainPackageName}"."${mainPackageVersion}";
+
+  packages =
     lib.mapAttrs
       (name: versions:
         lib.genAttrs
@@ -94,7 +97,7 @@ let
       nodeDeps =
         lib.forEach
           deps
-          (dep: allPackages."${dep.name}"."${dep.version}" );
+          (dep: packages."${dep.name}"."${dep.version}" );
 
       dependenciesJson = b.toJSON 
         (lib.listToAttrs
@@ -113,13 +116,15 @@ let
 
           src = getSource name version;
 
-          buildInputs = [ nodejs nodejs.python ];
+          buildInputs = [ jq nodejs nodejs.python ];
 
           passAsFile = [ "dependenciesJson" "nodeDeps" ];
 
           ignoreScripts = true;
 
           dontUnpack = true;
+          dontConfigure = true;
+          dontBuild = true;
 
           dontNpmInstall = false;
 
@@ -177,6 +182,11 @@ let
 
             # symlink dependency packages into node_modules
             for dep in $(cat $nodeDepsPath); do
+              # add bin to PATH
+              if [ -d "$dep/bin" ]; then
+                export PATH="$PATH:$dep/bin"
+              fi
+
               if [ -e $dep/lib/node_modules ]; then
                 for module in $(ls $dep/lib/node_modules); do
                   if [[ $module == @* ]]; then
@@ -216,13 +226,6 @@ let
               exit 1
             fi
 
-            # set flags for npm install
-            flags=("--offline" "--production" "--nodedir=$nodeSources" "--no-package-lock")
-            if [ -n "$ignoreScripts" ]; then
-              flags+=("--ignore-scripts")
-            fi
-
-
             # execute installation command
             if [ -n "$installScript" ]; then
               if [ -f "$installScript" ]; then
@@ -231,12 +234,15 @@ let
                 echo "$installScript" | bash
               fi
             elif [ -z "$dontNpmInstall" ]; then
-              npm "''${flags[@]}" install
+              if [ "$(jq '.scripts.postinstall' ./package.json)" != "null" ]; then
+                npm --production --offline --nodedir=$nodeSources run postinstall
+              fi
             fi
 
             # Create symlink to the deployed executable folder, if applicable
             if [ -d "$nodeModules/.bin" ]
             then
+              chmod +x $nodeModules/.bin/*
               ln -s $nodeModules/.bin $out/bin
             fi
 
@@ -260,11 +266,8 @@ let
     in
       (utils.applyOverridesToPackage packageOverrides pkg name);
 
-  package = allPackages."${mainPackageName}"."${mainPackageVersion}";
 
 in
 {
-
-  inherit package allPackages;
-    
+  inherit defaultPackage packages;
 }

src/builders/nodejs/granular/fix-package.py

@@ -1,5 +1,6 @@
 import json
 import os
+import pathlib
 import sys
 
 
@@ -23,6 +24,8 @@ if 'os' in package_json:
     exit(3)
 
 # replace version
+# If it is a github dependency referred by revision,
+# we can not rely on the version inside the package.json
 version = os.environ.get("version")
 if package_json['version'] != version:
   print(
@@ -34,6 +37,7 @@ if package_json['version'] != version:
   package_json['version'] = version
 
 # delete devDependencies
+# We rely on dream.lock having the correct dependencies specified
 if 'devDependencies' in package_json:
   print(
     f"Removing devDependencies from package.json",
@@ -43,6 +47,7 @@ if 'devDependencies' in package_json:
   del package_json['devDependencies']
 
 # delete peerDependencies
+# We rely on dream.lock instead
 if 'peerDependencies' in package_json:
   print(
     f"Removing peerDependencies from package.json",
@@ -52,6 +57,8 @@ if 'peerDependencies' in package_json:
   del package_json['peerDependencies']
 
 # pinpoint exact versions
+# This is mostly needed to replace git references with exact versions,
+# as NPM install will otherwise re-fetch these
 if 'dependencies' in package_json:
   for pname, version in package_json['dependencies'].items():
     if actual_deps[pname] != package_json['dependencies'][pname]:
@@ -63,6 +70,34 @@ if 'dependencies' in package_json:
         file=sys.stderr
       )
 
+# create symlinks for executables (bin entries from package.json)
+if 'bin' in package_json:
+  out = os.environ.get('out')
+  bin = package_json['bin']
+
+  if isinstance(bin, str):
+    name = package_json['name']
+    if not os.path.isfile(bin):
+      raise Exception(f"binary specified in package.json doesn't exist: {bin}")
+    source = f'{out}/lib/node_modules/.bin/{name}'
+    sourceDir = os.path.dirname(source)
+    # create parent dir
+    pathlib.Path(sourceDir).mkdir(parents=True, exist_ok=True)
+    
+    dest = os.path.relpath(bin, sourceDir)
+    print(f"dest: {dest}; source: {source}")
+    os.symlink(dest, source)
+
+  else:
+    for bin_name, relpath in bin.items():
+      source = f'{out}/lib/node_modules/.bin/{bin_name}'
+      sourceDir = os.path.dirname(source)
+      # create parent dir
+      pathlib.Path(sourceDir).mkdir(parents=True, exist_ok=True)
+      dest = os.path.relpath(relpath, sourceDir)
+      os.symlink(dest, source)
+
+
 # write changes to package.json
 if changed:
   with open('package.json', 'w') as f:

src/builders/nodejs/node2nix/default.nix

@@ -112,16 +112,16 @@ let
     // args);
 
 in
-{
+rec {
 
-  package =
+  packages."${mainPackageName}"."${mainPackageVersion}" = defaultPackage;
+
+  defaultPackage =
     let
       pkg = callNode2Nix "buildNodePackage" {};
     in
       utils.applyOverridesToPackage packageOverrides pkg mainPackageName;
 
-  shell = callNode2Nix "buildNodeShell" {};
-
-  # inherit allSources;
+  devShell = callNode2Nix "buildNodeShell" {};
     
 }

src/builders/python/simple-builder/default.nix

@@ -31,7 +31,7 @@ let
     else
       mainPackageName;
 
-  package = buildFunc {
+  defaultPackage = buildFunc {
     name = packageName;
     format = "";
     buildInputs = pkgs.pythonManylinuxPackages.manylinux1;
@@ -60,5 +60,5 @@ let
 };
 
 in {
-  inherit package;
+  inherit defaultPackage;
 }

src/default.nix

@@ -1,3 +1,8 @@
+# this is the system specific api for dream2nix
+# it requires passing one specifi pkgs
+# If the intention is to generate output for several systems,
+# use ./lib.nix instead
+
 {
   pkgs ? import <nixpkgs> {},
   lib ? pkgs.lib,
@@ -32,7 +37,6 @@ let
   });
 
   externals = {
-    npmlock2nix = pkgs.callPackage "${externalSources}/npmlock2nix/internal.nix" {};
     node2nix = nodejs: pkgs.callPackage "${externalSources}/node2nix/node-env.nix" { inherit nodejs; };
     nix-parsec = rec {
       lexer = import "${externalSources}/nix-parsec/lexer.nix" { inherit parsec; };
@@ -132,12 +136,23 @@ rec {
       };
 
   
-  justBuild =
+  makeDreamLockForSource =
     {
       source,
     }@args:
     let
 
+      sourceSpec =
+        if b.isString args.source && ! lib.isStorePath args.source then
+          fetchers.translateShortcut { shortcut = args.source; }
+        else
+          {
+            type = "path";
+            path = args.source;
+          };
+
+      source = fetchers.fetchSource { source = sourceSpec; };
+
       translatorsForSource = translators.translatorsForInput {
         inputFiles = [];
         inputDirectories = [ source ];
@@ -158,17 +173,12 @@ rec {
       dreamLock = lib.recursiveUpdate dreamLock' {
         sources."${dreamLock'.generic.mainPackageName}"."${dreamLock'.generic.mainPackageVersion}" = {
           type = "path";
-          path = source;
-          version = "unknown";
+          path = "${source}";
         };
       };
 
-      argsForRise = b.removeAttrs args [ "source" ];
-
     in
-      (riseAndShine ({
-        inherit dreamLock;
-      } // argsForRise)).package;
+      dreamLock;
 
 
   # build a dream lock via a specific builder
@@ -237,20 +247,36 @@ rec {
         });
 
 
-  # build package defined by dream.lock
+  # produce outputs for a dream.lock or a source
   riseAndShine = 
     {
-      dreamLock,
+      dreamLock ? null,
       builder ? null,
       fetcher ? null,
       inject ? {},
+      source ? null,
       sourceOverrides ? oldSources: {},
       packageOverrides ? {},
       builderArgs ? {},
     }@args:
+
+    # ensure either dreamLock or source is used
+    if source != null && dreamLock != null then
+      throw "Define either 'dreamLock' or 'source', not both"
+    else if source == null && dreamLock == null then
+      throw "Define either 'dreamLock' or 'source'"
+    else
+
     let
-      # if generic lock is a file, read and parse it
-      dreamLockLoaded = utils.readDreamLock { inherit (args) dreamLock; };
+
+      dreamLock' =
+        if source != null then
+          makeDreamLockForSource { inherit source; }
+        else
+          args.dreamLock;
+
+      # parse dreamLock
+      dreamLockLoaded = utils.readDreamLock { dreamLock = dreamLock'; };
       dreamLock = dreamLockLoaded.lock;
       dreamLockInterface = dreamLockLoaded.interface;
 

src/fetchers/default-fetcher.nix

@@ -31,7 +31,11 @@ let
                   (if source.type == "unknown" then
                     "unknown"
                   else if source.type == "path" then
-                    "${fetchedSources."${mainPackageName}#${mainPackageVersion}"}/${source.path}"
+                    if lib.isStorePath source.path then
+                      source.path
+                    # assume path relative to main package source
+                    else
+                      "${fetchedSources."${mainPackageName}#${mainPackageVersion}"}/${source.path}"
                   else if fetchers.fetchers ? "${source.type}" then
                     fetchSource { inherit source; }
                   else throw "unsupported source type '${source.type}'")

src/lib.nix

@@ -0,0 +1,104 @@
+# like ./default.nix but system intependent
+# (allows to generate outputs for several systems)
+# follows flake output schema
+
+{
+  nixpkgsSrc ? <nixpkgs>,
+  lib ? (import nixpkgsSrc {}).lib,
+
+  # dream2nix
+  makeExternalSources,
+}:
+
+let
+
+  b = builtins;
+
+  # TODO: design output schema for cross compiled packages
+  makePkgsKey = pkgs:
+    let
+      build = pkgs.buildPlatform.system;
+      host = pkgs.hostPlatform.system;
+    in
+      if build == host then build
+      else throw "cross compiling currently not supported";
+
+  makeNixpkgs = pkgsList: systems:
+
+    # fail if neither pkgs nor systems are defined
+    if pkgsList == null && systems == [] then
+      throw "Either `systems` or `pkgs` must be defined"
+
+    # fail if pkgs and systems are both defined
+    else if pkgsList != null && systems != [] then
+      throw "Define either `systems` or `pkgs`, not both"
+    
+    # only pkgs is specified
+    else if pkgsList != null then
+      if b.isList pkgsList then
+        lib.listToAttrs
+          (pkgs: lib.nameValuePair (makePkgsKey pkgs) pkgs)
+          pkgsList
+      else
+        { "${makePkgsKey pkgsList}" = pkgsList; }
+    
+    # only systems is specified
+    else
+      lib.genAttrs systems
+        (system: import nixpkgsSrc { inherit system; });
+
+    
+    flakifyBuilderOutputs = system: outputs:
+      (lib.optionalAttrs (outputs ? "defaultPackage") {
+        defaultPackage."${system}" = outputs.defaultPackage;
+      })
+      //
+      (lib.optionalAttrs (outputs ? "packages") {
+        packages."${system}" = outputs.packages;
+      })
+      //
+      (lib.optionalAttrs (outputs ? "devShell") {
+        devShell."${system}" = outputs.devShell;
+      });
+
+in
+
+{
+  riseAndShine = 
+    {
+      pkgs ? null,
+      systems ? [],
+      ...
+    }@args:
+    let
+
+      argsForward = b.removeAttrs args [ "pkgs" "systems" ];
+
+      allPkgs = makeNixpkgs pkgs systems;
+
+      dream2nixFor =
+        lib.mapAttrs
+          (system: pkgs:
+            import ./default.nix {
+              inherit pkgs;
+              externalSources = makeExternalSources pkgs;
+            })
+          allPkgs;
+
+      allBuilderOutputs =
+        lib.mapAttrs
+          (system: pkgs:
+            dream2nixFor."${system}".riseAndShine argsForward)
+          allPkgs;
+
+      flakifiedOutputs =
+        lib.mapAttrsToList
+          (system: outputs: flakifyBuilderOutputs system outputs)
+          allBuilderOutputs;
+      
+    in
+      b.foldl'
+        (allOutputs: output: lib.recursiveUpdate allOutputs output)
+        {}
+        flakifiedOutputs;
+}

src/translators/nodejs/pure/yarn-lock/default.nix

@@ -217,7 +217,7 @@
   specialArgs = {
 
     name = {
-      description = "The name of hte main package";
+      description = "The name of the main package";
       examples = [
         "react"
         "@babel/code-frame"

src/utils/dream-lock.nix

@@ -16,7 +16,9 @@ let
     let
 
       lock =
-        if b.isPath dreamLock || b.isString dreamLock then
+        if b.isPath dreamLock
+            || b.isString dreamLock
+            || lib.isDerivation dreamLock then
           b.fromJSON (b.readFile dreamLock)
         else
           dreamLock;
@@ -54,7 +56,7 @@ let
                 lib.forEach removedKeys
                   (rKey: utils.keyToNameVersion rKey))
               versions)
-          lock.generic.dependenciesRemoved;
+          lock.generic.dependenciesRemoved or {};
 
       # Format:
       # {