refactor python builder

src/subsystems/nodejs/builders/strict-builder/default.nix

@@ -116,6 +116,7 @@
       devShellNodeModules = mkNodeModules {
         isMain = true;
         installMethod = "copy";
+        reason = "devShell";
         inherit pname version depsTree nodeModulesTree;
       };
       # type: nodeModules :: Derivation
@@ -143,8 +144,6 @@
 
           outputs = ["out" "lib"];
 
-          deps = nodeModules;
-
           passthru = {
             inherit nodeModules;
             devShell = import ./lib/devShell.nix {
@@ -217,7 +216,7 @@
                 npm --production --offline --nodedir=$nodeSources run postinstall
               fi
             fi
-
+            export NODE_MODULES_PATH=${nodeModules}
             ${nodejsBuilder}/bin/d2nMakeOutputs
 
 

src/subsystems/nodejs/builders/strict-builder/lib/node-modules-tree.nix

@@ -122,6 +122,7 @@
     installPath ? "",
     depsTree,
     nodeModulesTree,
+    reason ? "default",
   }:
   # dependency tree as JSON needed to build node_modules
   let
@@ -140,11 +141,14 @@
 
       export isMain=${b.toString isMain}
       export installMethod=${installMethod}
-      export installPath=${installPath}
+      export REASON=${reason}
 
       ${nodeModulesBuilder}
 
-      cp -r /build/node_modules $out 2> /dev/null || mkdir $out
+      # make sure $out gets created every time, even if it is empty
+      if [ ! -d "$out" ]; then
+        mkdir -p $out
+      fi
     '';
 in {
   inherit nodeModulesTree mkNodeModules;

src/subsystems/nodejs/builders/strict-builder/python-builder/app/lib/config.py

@@ -1,6 +1,10 @@
 from pathlib import Path
-from .derivation import env
+from .derivation import get_env
+from .logger import logger
 
-root = Path("/build")
-node_modules = root / Path("node_modules")
-bin_dir = node_modules / Path(".bin")
+# root is used to create the node_modules structure
+# defaults to $out,
+# which will create the node_modules directly in
+# $out of the derivation, and saves copy time
+root = Path(get_env("out"))
+bin_dir = root / Path(".bin")

src/subsystems/nodejs/builders/strict-builder/python-builder/app/lib/dependencies.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass
-from typing import Any, Callable, Literal, Optional, TypedDict, Union
+from typing import Any, Callable, Literal, Optional, TypedDict
 
 from .logger import logger
 
@@ -130,7 +130,7 @@ def recurse_deps_tree(
                 )
             else:
                 logger.debug(
-                    f"stopped recursing the dependency tree at {dependency.repr()}\
+                    f"stopped recursing the dependency tree at {dependency}\
     -> because the predicate function returned 'False'"
                 )
                 return accumulator

src/subsystems/nodejs/builders/strict-builder/python-builder/app/lib/derivation.py

@@ -1,49 +1,57 @@
 """
-    some utility functions to reference the value of 
+    some utility functions to reference the value of
     variables from the overlaying derivation (via env)
 """
 import os
 from enum import Enum
-from typing import Any, Optional
+from typing import Optional
 from pathlib import Path
 from .logger import logger
 from dataclasses import dataclass
 
 env: dict[str, str] = os.environ.copy()
 
-node_modules_link = env.get("NODE_MODULES_LINK")
+
+@dataclass
+class Input:
+    node_modules: Path
 
 
 @dataclass
 class Output:
     out: Path
     lib: Path
-    deps: Path
 
 
 def get_outputs() -> Output:
     outputs = {
-        "out": Path(get_env().get("out")),
-        "lib": Path(get_env().get("lib")),
-        "deps": Path(get_env().get("deps")),
+        "out": Path(get_env("out")),
+        "lib": Path(get_env("lib")),
     }
-    if None in outputs.values():
-        logger.error(
-            f"\
-At least one out path uninitialized: {outputs}"
-        )
-        exit(1)
-    return Output(outputs["out"], outputs["lib"], outputs["deps"])
+    return Output(outputs["out"], outputs["lib"])
+
+
+def get_inputs() -> Input:
+    node_modules_path = Path(get_env("NODE_MODULES_PATH"))
+    return Input(node_modules_path)
 
 
 def is_main_package() -> bool:
     """Returns True or False depending on the 'isMain' env variable."""
-    return bool(get_env().get("isMain"))
+    return bool(get_env("isMain"))
 
 
-def get_env() -> dict[str, Any]:
-    """Returns a copy of alle the current env variables"""
-    return env
+def get_env(key: str) -> str:
+    """
+    Returns the value of the required env variable
+    Prints an error end exits execution if the env variable is not set
+    """
+    try:
+        value = env[key]
+    except KeyError:
+        logger.error(f"env variable ${key} is not set")
+        exit(1)
+    return value
 
 
 @dataclass
@@ -54,7 +62,7 @@ class Info:
 
 def get_self() -> Info:
     """ """
-    return Info(get_env().get("pname", "unknown"), get_env().get("version", "unknown"))
+    return Info(env.get("pname", "unknown"), env.get("version", "unknown"))
 
 
 class InstallMethod(Enum):
@@ -64,7 +72,7 @@ class InstallMethod(Enum):
 
 def get_install_method() -> InstallMethod:
     """Returns the value of 'installMethod'"""
-    install_method: Optional[str] = get_env().get("installMethod")
+    install_method: Optional[str] = env.get("installMethod")
     try:
         return InstallMethod(install_method)
     except ValueError:

src/subsystems/nodejs/builders/strict-builder/python-builder/app/lib/node_modules.py

@@ -6,7 +6,7 @@ from typing import Any, Optional, TypedDict
 from .config import root
 from .dependencies import Dependency, DepsTree, get_all_deps, recurse_deps_tree
 from .logger import logger
-from .derivation import InstallMethod, get_install_method, get_self, node_modules_link
+from .derivation import InstallMethod, get_install_method, get_self
 from .package import (
     NodeModulesPackage,
     NodeModulesTree,
@@ -55,22 +55,31 @@ def _create_package_from_derivation(
 
 
 class Passthrough(TypedDict):
+    """
+    Wrapper class
+    Holds global informations during recursion in <_make_folders_rec>
+    """
+
     all_deps: dict[str, Dependency]
     flat_deps: list[str]
 
 
 def _make_folders_rec(
-    node_modules_tree: NodeModulesPackage,
+    node_modules_tree: NodeModulesTree,
     passthrough: Passthrough,
-    path: Path = Path("node_modules"),
+    path: Path = Path(""),
 ):
+    """
+    Builds the node_modules folder structure
+    from the NodeModulesTree datastructures
+    """
     name: str
-    meta: NodeModulesTree
+    meta: NodeModulesPackage
 
     for name, meta in node_modules_tree.items():
 
         version = meta["version"]
-        dependencies: Optional[NodeModulesPackage] = meta.get("dependencies", None)
+        dependencies: Optional[NodeModulesTree] = meta.get("dependencies", None)
         found_dependency = passthrough["all_deps"].get(f"{name}@{version}")
 
         if found_dependency:
@@ -121,11 +130,3 @@ def create_node_modules():
         nm_tree,
         passthrough={"all_deps": collected, "flat_deps": flat_deps},
     )
-
-    if node_modules_link:
-        logger.debug(f"/build/node modules symlinked to '{node_modules_link}'.")
-        Path(node_modules_link).symlink_to(root / Path("node_modules"))
-    else:
-        logger.debug(
-            f"/build/node modules created but not exposed. $NODE_MODULES_LINK='{node_modules_link}'. Set it to valid path for automatic symlinks."
-        )

src/subsystems/nodejs/builders/strict-builder/python-builder/app/lib/package.py

@@ -22,7 +22,7 @@ def get_package_json(path: Path = Path("")) -> Union[dict[str, Any], None]:
 
 def has_scripts(
     package_json: dict[str, Any],
-    lifecycle_scripts: tuple[str] = (
+    lifecycle_scripts: tuple[str, str, str] = (
         "preinstall",
         "install",
         "postinstall",
@@ -56,14 +56,14 @@ def create_binary(target: Path, source: Path):
 
 def get_all_deps_tree() -> DepsTree:
     deps = {}
-    dependenciesJsonPath = get_env().get("depsTreeJSONPath")
+    dependenciesJsonPath = get_env("depsTreeJSONPath")
     if dependenciesJsonPath:
         with open(dependenciesJsonPath) as f:
             deps = json.load(f)
     return deps
 
 
-class NodeModulesTree(TypedDict):
+class NodeModulesPackage(TypedDict):
     version: str
     # mypy does not allow recursive types yet.
     # The real type is:
@@ -71,12 +71,12 @@ class NodeModulesTree(TypedDict):
     dependencies: Optional[dict[str, Any]]
 
 
-NodeModulesPackage = dict[str, NodeModulesTree]
+NodeModulesTree = dict[str, NodeModulesPackage]
 
 
 def get_node_modules_tree() -> dict[str, Any]:
     tree = {}
-    dependenciesJsonPath = get_env().get("nmTreeJSONPath")
+    dependenciesJsonPath = get_env("nmTreeJSONPath")
     if dependenciesJsonPath:
         with open(dependenciesJsonPath) as f:
             tree = json.load(f)

src/subsystems/nodejs/builders/strict-builder/python-builder/app/main.py

@@ -1,8 +1,8 @@
 from .lib.checks import check_platform
-from .lib.config import node_modules
 from .lib.derivation import (
     is_main_package,
     get_outputs,
+    get_inputs,
     get_self,
     InstallMethod,
     get_install_method,
@@ -62,48 +62,39 @@ def makeOutputs():
         ├── cli.js -> /nix/store/...-pname-1.0.0-lib/cli.js
         ├── ...
         ├── package.json -> /nix/store/...-pname-1.0.0-lib/package.json
-        └── node_modules -> /nix/store/...-pname-1.0.0-deps
+        └── node_modules -> /nix/store/...-pname-1.0.0-node_modules
     """
 
-    # get the outputs from env ($out, $lib, $deps)
     outputs = get_outputs()
+    inputs = get_inputs()
+    pkg = get_self()
 
-    # create empty deps path for packages without dependencies
-    if node_modules.exists():
-        # copy the tree and preserve symlinks
-        # copytree also checks for dangling symlinks and fails on broken links
-        shutil.copytree(node_modules, outputs.deps, symlinks=True)
-    else:
-        outputs.deps.mkdir(parents=True, exist_ok=True)
-
-    # copy package content only
-    # TODO: apply filter logic from npm ('files' entry in package-json)
-
-    # remove the leftover symlink from d2nNodeModules phase
-    Path("./node_modules").unlink(missing_ok=True)
-
-    # TODO: run scripts ? (pre-, post-, install scripts)
+    # create $lib output
     shutil.copytree(Path("."), outputs.lib)
 
+    # create $out output
     bin_out = outputs.out / Path("bin")
     lib_out = outputs.out / Path("lib")
     bin_out.mkdir(parents=True, exist_ok=True)
-
     install_method = get_install_method()
 
+    # create $out/lib
     if install_method == InstallMethod.copy:
         shutil.copytree(outputs.lib, lib_out, symlinks=True)
-        shutil.copytree(outputs.deps, lib_out / Path("node_modules"), symlinks=True)
+        shutil.copytree(
+            inputs.node_modules, lib_out / Path("node_modules"), symlinks=True
+        )
 
     elif install_method == InstallMethod.symlink:
         lib_out.mkdir(parents=True, exist_ok=True)
         for entry in os.listdir(outputs.lib):
             (lib_out / Path(entry)).symlink_to(outputs.lib / Path(entry))
 
-        (lib_out / Path("node_modules")).symlink_to(outputs.deps)
+        (lib_out / Path("node_modules")).symlink_to(inputs.node_modules)
 
-    # create binaries
-    pkg = get_self()
+    # create $out/bin
+    # collect all binaries declared from package
+    # and create symlinks to their sources e.g. $out/bin/cli -> $out/lib/cli.json
     dep = Dependency(name=pkg.name, version=pkg.version, derivation=outputs.lib)
     binaries = get_bins(dep)
     for name, rel_path in binaries.items():