multiple override dirs, nodejs improvements

- nodejs builder: symlink sub dependencies
  - allow multiple directories with overrides
  - rename app: cli -> dream2nix

flake.nix

@@ -75,12 +75,12 @@
           (lib.attrNames externalPaths)
           (inputName: inp."${inputName}");
 
-      overridesDir = "${./overrides}";
+      overridesDirs =  [ "${./overrides}" ];
 
       # system specific dream2nix api
       dream2nixFor = forAllSystems (system: pkgs: import ./src rec {
         externalDir = externalDirFor."${system}";
-        inherit externalSources lib overridesDir pkgs;
+        inherit externalSources lib overridesDirs pkgs;
       });
 
     in
@@ -97,7 +97,7 @@
         # Similar to drem2nixFor but will require 'system(s)' or 'pkgs' as an argument.
         # Produces flake-like output schema.
         lib = (import ./src/lib.nix {
-          inherit externalSources overridesDir lib;
+          inherit externalSources overridesDirs lib;
           nixpkgsSrc = "${nixpkgs}";
         })
         # system specific dream2nix library
@@ -106,14 +106,15 @@
             inherit
               externalSources
               lib
-              overridesDir
+              overridesDirs
               pkgs
             ;
           }
         ));
 
         # the dream2nix cli to be used with 'nix run dream2nix'
-        defaultApp = forAllSystems (system: pkgs: self.apps."${system}".cli);
+        defaultApp =
+          forAllSystems (system: pkgs: self.apps."${system}".dream2nix);
 
         # all apps including cli, install, etc.
         apps = forAllSystems (system: pkgs:
@@ -140,7 +141,7 @@
             export NIX_PATH=nixpkgs=${nixpkgs}
             export d2nExternalDir=${externalDirFor."${system}"}
             export dream2nixWithExternals=${dream2nixFor."${system}".dream2nixWithExternals}
-            export d2nOverridesDir=${./overrides}
+            export d2nOverridesDirs=${./overrides}
 
             echo -e "\nManually execute 'export dream2nixWithExternals={path to your dream2nix checkout}'"
           '';

src/apps/cli/commands/package.py

@@ -391,7 +391,7 @@ class PackageCommand(Command):
       sourcePathRelative = os.path.relpath(source, os.path.dirname(outputDefaultNix))
     )
     # with open(f"{dream2nix_src}/apps/cli2/templateDefault.nix") as template:
-    if not self.option('no-default-nix'):
+    if 'default.nix' in filesToCreate:
       with open(outputDefaultNix, 'w') as defaultNix:
         defaultNix.write(template)
         print(f"Created {output}/default.nix")

src/apps/default.nix

@@ -6,7 +6,10 @@
   ...
 }:
 rec {
-  apps = { inherit cli contribute install; };
+  apps = {
+    inherit cli contribute install;
+    dream2nix = cli;
+  };
 
   # the unified translator cli
   cli = callPackageDream (import ./cli) {};

src/builders/nodejs/granular/default.nix

@@ -254,6 +254,8 @@ let
               fi
             done
 
+            python ${./symlink-deps.py}
+
             export NODE_PATH="$NODE_PATH:$nodeModules/$packageName/node_modules"
 
             export HOME=$TMPDIR

src/builders/nodejs/granular/symlink-deps.py

@@ -0,0 +1,69 @@
+import json
+import os
+import pathlib
+import sys
+
+
+out = os.environ.get('out')
+pname = os.environ.get('packageName')
+version = os.environ.get('version')
+root = f"{out}/lib/node_modules/{pname}/node_modules"
+
+if not os.path.isdir(root):
+  exit()
+
+
+with open(os.environ.get("nodeDepsPath")) as f:
+  nodeDeps = f.read().split()
+
+def getDependencies(root, depth):
+  if not os.path.isdir(root):
+    return []
+  dirs = os.listdir(root)
+
+  currentDeps = []
+  for d in dirs:
+    if d.rpartition('/')[-1].startswith('@'):
+      subdirs = os.listdir(f"{root}/{d}")
+      for sd in subdirs:
+        cur_dir = f"{root}/{d}/{sd}"
+        currentDeps.append(f"{cur_dir}")
+    else:
+      cur_dir = f"{root}/{d}"
+      currentDeps.append(cur_dir)
+  
+  if depth == 0:
+    return currentDeps
+  else:
+    depsOfDeps =\
+      map(lambda dep: getDependencies(f"{dep}/node_modules", depth - 1), currentDeps)
+    result = []
+    for deps in depsOfDeps:
+      result += deps
+    return result
+
+deps = getDependencies(root, 1)
+
+# symlink deps non-colliding deps
+for dep in deps:
+
+  # compute module path
+  d1, d2 = dep.split('/')[-2:]
+  if d1.startswith('@'):
+    path = f"{root}/{d1}/{d2}"
+  else:
+    path = f"{root}/{d2}"
+
+  # check for collision
+  if os.path.isdir(path):
+    continue
+
+  # create parent dir
+  pathlib.Path(os.path.dirname(path)).mkdir(parents=True, exist_ok=True)
+
+  # symlink dependency
+  os.symlink(dep, path)
+
+  
+
+  

src/default.nix

@@ -28,13 +28,13 @@
       ./external,
 
   # dream2nix overrides
-  overridesDir ?
+  overridesDirs ?
      # if called via CLI, load externals via env
-    if builtins ? getEnv && builtins.getEnv "d2nOverridesDir" != "" then
-      builtins.getEnv "d2nOverridesDir"
+    if builtins ? getEnv && builtins.getEnv "d2nOverridesDirs" != "" then
+      lib.splitString ":" (builtins.getEnv "d2nOverridesDirs")
     # load from default directory
     else
-      ./overrides,
+      [ ./overrides ],
 }:
 
 let
@@ -75,18 +75,20 @@ let
   translators = callPackageDream ./translators {};
 
   externals = {
-    node2nix = nodejs: pkgs.callPackage "${externalSources.node2nix}/nix/node-env.nix" { inherit nodejs; };
+    node2nix = nodejs:
+      pkgs.callPackage "${externalSources.node2nix}/nix/node-env.nix" {
+        inherit nodejs;
+      };
     nix-parsec = rec {
-      lexer = import "${externalSources.nix-parsec}/lexer.nix" { inherit parsec; };
+      lexer = import "${externalSources.nix-parsec}/lexer.nix" {
+        inherit parsec;
+      };
       parsec = import "${externalSources.nix-parsec}/parsec.nix";
     };
   };
 
-  dreamOverrides = lib.genAttrs (utils.dirNames overridesDir) (name:
-    import (overridesDir + "/${name}") {
-      inherit lib pkgs;
-    }
-  );
+  dreamOverrides =
+    utils.loadOverridesDirs overridesDirs pkgs;
 
   # the location of the dream2nix framework for self references (update scripts, etc.)
   dream2nixWithExternals =

src/lib.nix

@@ -8,8 +8,17 @@
 
   # (if called impurely ./default.nix will handle externals and overrides)
   externalSources ? null,
-  overridesDir ? null,
-}:
+
+  # dream2nix overrides
+  overridesDirs ?
+     # if called via CLI, load externals via env
+    if builtins ? getEnv && builtins.getEnv "d2nOverridesDirs" != "" then
+      lib.splitString ":" (builtins.getEnv "d2nOverridesDirs")
+    # load from default directory
+    else
+      [ ./overrides ],
+
+}@args:
 
 let
 
@@ -62,9 +71,54 @@ let
         devShell."${system}" = outputs.devShell;
       });
 
-in
+  init =
+    {
+      pkgs ? null,
+      systems ? [],
+      overridesDirs ? [],
+    }@argsInit:
+    let
+
+      overridesDirs' = args.overridesDirs ++ argsInit.overridesDirs or [];
+
+      allPkgs = makeNixpkgs pkgs systems;
+
+      dream2nixFor =
+        lib.mapAttrs
+          (system: pkgs:
+            import ./default.nix
+              (
+                {
+                  inherit pkgs;
+                  overridesDirs = overridesDirs';
+                }
+                // (lib.optionalAttrs (externalSources != null) {
+                  inherit externalSources;
+                })
+              ))
+          allPkgs;
+    in
+      {
+        riseAndShine = riseAndShineArgs:
+          let
+            allBuilderOutputs =
+              lib.mapAttrs
+                (system: pkgs:
+                  dream2nixFor."${system}".riseAndShine riseAndShineArgs)
+                allPkgs;
+
+            flakifiedOutputs =
+              lib.mapAttrsToList
+                (system: outputs: flakifyBuilderOutputs system outputs)
+                allBuilderOutputs;
+
+          in
+            b.foldl'
+              (allOutputs: output: lib.recursiveUpdate allOutputs output)
+              {}
+              flakifiedOutputs;
+      };
 
-{
   riseAndShine = 
     {
       pkgs ? null,
@@ -85,8 +139,8 @@ in
               // (lib.optionalAttrs (externalSources != null) {
                 inherit externalSources;
               })
-              // (lib.optionalAttrs (overridesDir != null) {
-                inherit overridesDir;
+              // (lib.optionalAttrs (overridesDirs != null) {
+                inherit overridesDirs;
               })))
           allPkgs;
 
@@ -106,4 +160,8 @@ in
         (allOutputs: output: lib.recursiveUpdate allOutputs output)
         {}
         flakifiedOutputs;
+
+in
+{
+  inherit init riseAndShine;
 }

src/utils/default.nix

@@ -152,4 +152,8 @@ rec {
         (v1: v2: versionGreater v1 v2)
         versions);
 
+  # like nixpkgs recursiveUpdateUntil, but the depth of the 
+  recursiveUpdateUntilDepth = depth: lhs: rhs:
+    lib.recursiveUpdateUntil (path: l: r: (b.length path) > depth) lhs rhs;
+
 }

src/utils/override.nix

@@ -1,11 +1,28 @@
 {
   lib,
+
+  # dream2nix
+  utils,
   ...
 }:
 let
 
   b = builtins;
 
+  loadOverridesDirs = overridesDirs: pkgs:
+    let
+      loadOverrides = dir:
+        lib.genAttrs (utils.dirNames dir) (name:
+          import (dir + "/${name}") {
+            inherit lib pkgs;
+          });
+    in
+      b.foldl'
+        (loaded: nextDir:
+          utils.recursiveUpdateUntilDepth 3 loaded (loadOverrides nextDir))
+        {}
+        overridesDirs;
+
   throwErrorUnclearAttributeOverride = pname: overrideName: attrName:
     throw ''
       Error while applying override for ${pname}: '${overrideName}'
@@ -44,10 +61,10 @@ let
 
         # if condition is unset, it will be assumed true
         evalCondition = condOverride: pkg:
-          if condOverride ? _condition then
-            condOverride._condition pkg
-          else
-            true;
+            if condOverride ? _condition then
+              condOverride._condition pkg
+            else
+              true;
 
         # filter the overrides by the package name and conditions
         overridesToApply =
@@ -151,5 +168,5 @@ let
 
 in
 {
-  inherit applyOverridesToPackage;
+  inherit applyOverridesToPackage loadOverridesDirs;
 }