diff --git a/src/subsystems/nodejs/builders/strict-builder/default.nix b/src/subsystems/nodejs/builders/strict-builder/default.nix index a29076b1..809c9e49 100644 --- a/src/subsystems/nodejs/builders/strict-builder/default.nix +++ b/src/subsystems/nodejs/builders/strict-builder/default.nix @@ -116,6 +116,7 @@ devShellNodeModules = mkNodeModules { isMain = true; installMethod = "copy"; + reason = "devShell"; inherit pname version depsTree nodeModulesTree; }; # type: nodeModules :: Derivation @@ -143,8 +144,6 @@ outputs = ["out" "lib"]; - deps = nodeModules; - passthru = { inherit nodeModules; devShell = import ./lib/devShell.nix { @@ -217,7 +216,7 @@ npm --production --offline --nodedir=$nodeSources run postinstall fi fi - + export NODE_MODULES_PATH=${nodeModules} ${nodejsBuilder}/bin/d2nMakeOutputs diff --git a/src/subsystems/nodejs/builders/strict-builder/lib/node-modules-tree.nix b/src/subsystems/nodejs/builders/strict-builder/lib/node-modules-tree.nix index 03541770..e427fffc 100644 --- a/src/subsystems/nodejs/builders/strict-builder/lib/node-modules-tree.nix +++ b/src/subsystems/nodejs/builders/strict-builder/lib/node-modules-tree.nix @@ -122,6 +122,7 @@ installPath ? "", depsTree, nodeModulesTree, + reason ? "default", }: # dependency tree as JSON needed to build node_modules let @@ -140,11 +141,14 @@ export isMain=${b.toString isMain} export installMethod=${installMethod} - export installPath=${installPath} + export REASON=${reason} ${nodeModulesBuilder} - cp -r /build/node_modules $out 2> /dev/null || mkdir $out + # make sure $out gets created every time, even if it is empty + if [ ! -d "$out" ]; then + mkdir -p $out + fi ''; in { inherit nodeModulesTree mkNodeModules; diff --git a/src/subsystems/nodejs/builders/strict-builder/python-builder/app/lib/config.py b/src/subsystems/nodejs/builders/strict-builder/python-builder/app/lib/config.py index a7032dc5..07fcd646 100644 --- a/src/subsystems/nodejs/builders/strict-builder/python-builder/app/lib/config.py +++ b/src/subsystems/nodejs/builders/strict-builder/python-builder/app/lib/config.py @@ -1,6 +1,10 @@ from pathlib import Path -from .derivation import env +from .derivation import get_env +from .logger import logger -root = Path("/build") -node_modules = root / Path("node_modules") -bin_dir = node_modules / Path(".bin") +# root is used to create the node_modules structure +# defaults to $out, +# which will create the node_modules directly in +# $out of the derivation, and saves copy time +root = Path(get_env("out")) +bin_dir = root / Path(".bin") diff --git a/src/subsystems/nodejs/builders/strict-builder/python-builder/app/lib/dependencies.py b/src/subsystems/nodejs/builders/strict-builder/python-builder/app/lib/dependencies.py index feec175f..4f0f1177 100644 --- a/src/subsystems/nodejs/builders/strict-builder/python-builder/app/lib/dependencies.py +++ b/src/subsystems/nodejs/builders/strict-builder/python-builder/app/lib/dependencies.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from typing import Any, Callable, Literal, Optional, TypedDict, Union +from typing import Any, Callable, Literal, Optional, TypedDict from .logger import logger @@ -130,7 +130,7 @@ def recurse_deps_tree( ) else: logger.debug( - f"stopped recursing the dependency tree at {dependency.repr()}\ + f"stopped recursing the dependency tree at {dependency}\ -> because the predicate function returned 'False'" ) return accumulator diff --git a/src/subsystems/nodejs/builders/strict-builder/python-builder/app/lib/derivation.py b/src/subsystems/nodejs/builders/strict-builder/python-builder/app/lib/derivation.py index 768a91ba..079598f2 100644 --- a/src/subsystems/nodejs/builders/strict-builder/python-builder/app/lib/derivation.py +++ b/src/subsystems/nodejs/builders/strict-builder/python-builder/app/lib/derivation.py @@ -1,49 +1,57 @@ """ - some utility functions to reference the value of + some utility functions to reference the value of variables from the overlaying derivation (via env) """ import os from enum import Enum -from typing import Any, Optional +from typing import Optional from pathlib import Path from .logger import logger from dataclasses import dataclass env: dict[str, str] = os.environ.copy() -node_modules_link = env.get("NODE_MODULES_LINK") + +@dataclass +class Input: + node_modules: Path @dataclass class Output: out: Path lib: Path - deps: Path def get_outputs() -> Output: outputs = { - "out": Path(get_env().get("out")), - "lib": Path(get_env().get("lib")), - "deps": Path(get_env().get("deps")), + "out": Path(get_env("out")), + "lib": Path(get_env("lib")), } - if None in outputs.values(): - logger.error( - f"\ -At least one out path uninitialized: {outputs}" - ) - exit(1) - return Output(outputs["out"], outputs["lib"], outputs["deps"]) + return Output(outputs["out"], outputs["lib"]) + + +def get_inputs() -> Input: + node_modules_path = Path(get_env("NODE_MODULES_PATH")) + return Input(node_modules_path) def is_main_package() -> bool: """Returns True or False depending on the 'isMain' env variable.""" - return bool(get_env().get("isMain")) + return bool(get_env("isMain")) -def get_env() -> dict[str, Any]: - """Returns a copy of alle the current env variables""" - return env +def get_env(key: str) -> str: + """ + Returns the value of the required env variable + Prints an error end exits execution if the env variable is not set + """ + try: + value = env[key] + except KeyError: + logger.error(f"env variable ${key} is not set") + exit(1) + return value @dataclass @@ -54,7 +62,7 @@ class Info: def get_self() -> Info: """ """ - return Info(get_env().get("pname", "unknown"), get_env().get("version", "unknown")) + return Info(env.get("pname", "unknown"), env.get("version", "unknown")) class InstallMethod(Enum): @@ -64,7 +72,7 @@ class InstallMethod(Enum): def get_install_method() -> InstallMethod: """Returns the value of 'installMethod'""" - install_method: Optional[str] = get_env().get("installMethod") + install_method: Optional[str] = env.get("installMethod") try: return InstallMethod(install_method) except ValueError: diff --git a/src/subsystems/nodejs/builders/strict-builder/python-builder/app/lib/node_modules.py b/src/subsystems/nodejs/builders/strict-builder/python-builder/app/lib/node_modules.py index 2c0fb749..03597ee2 100644 --- a/src/subsystems/nodejs/builders/strict-builder/python-builder/app/lib/node_modules.py +++ b/src/subsystems/nodejs/builders/strict-builder/python-builder/app/lib/node_modules.py @@ -6,7 +6,7 @@ from typing import Any, Optional, TypedDict from .config import root from .dependencies import Dependency, DepsTree, get_all_deps, recurse_deps_tree from .logger import logger -from .derivation import InstallMethod, get_install_method, get_self, node_modules_link +from .derivation import InstallMethod, get_install_method, get_self from .package import ( NodeModulesPackage, NodeModulesTree, @@ -55,22 +55,31 @@ def _create_package_from_derivation( class Passthrough(TypedDict): + """ + Wrapper class + Holds global informations during recursion in <_make_folders_rec> + """ + all_deps: dict[str, Dependency] flat_deps: list[str] def _make_folders_rec( - node_modules_tree: NodeModulesPackage, + node_modules_tree: NodeModulesTree, passthrough: Passthrough, - path: Path = Path("node_modules"), + path: Path = Path(""), ): + """ + Builds the node_modules folder structure + from the NodeModulesTree datastructures + """ name: str - meta: NodeModulesTree + meta: NodeModulesPackage for name, meta in node_modules_tree.items(): version = meta["version"] - dependencies: Optional[NodeModulesPackage] = meta.get("dependencies", None) + dependencies: Optional[NodeModulesTree] = meta.get("dependencies", None) found_dependency = passthrough["all_deps"].get(f"{name}@{version}") if found_dependency: @@ -121,11 +130,3 @@ def create_node_modules(): nm_tree, passthrough={"all_deps": collected, "flat_deps": flat_deps}, ) - - if node_modules_link: - logger.debug(f"/build/node modules symlinked to '{node_modules_link}'.") - Path(node_modules_link).symlink_to(root / Path("node_modules")) - else: - logger.debug( - f"/build/node modules created but not exposed. $NODE_MODULES_LINK='{node_modules_link}'. Set it to valid path for automatic symlinks." - ) diff --git a/src/subsystems/nodejs/builders/strict-builder/python-builder/app/lib/package.py b/src/subsystems/nodejs/builders/strict-builder/python-builder/app/lib/package.py index 87c81d95..4d8f1153 100644 --- a/src/subsystems/nodejs/builders/strict-builder/python-builder/app/lib/package.py +++ b/src/subsystems/nodejs/builders/strict-builder/python-builder/app/lib/package.py @@ -22,7 +22,7 @@ def get_package_json(path: Path = Path("")) -> Union[dict[str, Any], None]: def has_scripts( package_json: dict[str, Any], - lifecycle_scripts: tuple[str] = ( + lifecycle_scripts: tuple[str, str, str] = ( "preinstall", "install", "postinstall", @@ -56,14 +56,14 @@ def create_binary(target: Path, source: Path): def get_all_deps_tree() -> DepsTree: deps = {} - dependenciesJsonPath = get_env().get("depsTreeJSONPath") + dependenciesJsonPath = get_env("depsTreeJSONPath") if dependenciesJsonPath: with open(dependenciesJsonPath) as f: deps = json.load(f) return deps -class NodeModulesTree(TypedDict): +class NodeModulesPackage(TypedDict): version: str # mypy does not allow recursive types yet. # The real type is: @@ -71,12 +71,12 @@ class NodeModulesTree(TypedDict): dependencies: Optional[dict[str, Any]] -NodeModulesPackage = dict[str, NodeModulesTree] +NodeModulesTree = dict[str, NodeModulesPackage] def get_node_modules_tree() -> dict[str, Any]: tree = {} - dependenciesJsonPath = get_env().get("nmTreeJSONPath") + dependenciesJsonPath = get_env("nmTreeJSONPath") if dependenciesJsonPath: with open(dependenciesJsonPath) as f: tree = json.load(f) diff --git a/src/subsystems/nodejs/builders/strict-builder/python-builder/app/main.py b/src/subsystems/nodejs/builders/strict-builder/python-builder/app/main.py index d1016425..98eca10e 100644 --- a/src/subsystems/nodejs/builders/strict-builder/python-builder/app/main.py +++ b/src/subsystems/nodejs/builders/strict-builder/python-builder/app/main.py @@ -1,8 +1,8 @@ from .lib.checks import check_platform -from .lib.config import node_modules from .lib.derivation import ( is_main_package, get_outputs, + get_inputs, get_self, InstallMethod, get_install_method, @@ -62,48 +62,39 @@ def makeOutputs(): ├── cli.js -> /nix/store/...-pname-1.0.0-lib/cli.js ├── ... ├── package.json -> /nix/store/...-pname-1.0.0-lib/package.json - └── node_modules -> /nix/store/...-pname-1.0.0-deps + └── node_modules -> /nix/store/...-pname-1.0.0-node_modules """ - # get the outputs from env ($out, $lib, $deps) outputs = get_outputs() + inputs = get_inputs() + pkg = get_self() - # create empty deps path for packages without dependencies - if node_modules.exists(): - # copy the tree and preserve symlinks - # copytree also checks for dangling symlinks and fails on broken links - shutil.copytree(node_modules, outputs.deps, symlinks=True) - else: - outputs.deps.mkdir(parents=True, exist_ok=True) - - # copy package content only - # TODO: apply filter logic from npm ('files' entry in package-json) - - # remove the leftover symlink from d2nNodeModules phase - Path("./node_modules").unlink(missing_ok=True) - - # TODO: run scripts ? (pre-, post-, install scripts) + # create $lib output shutil.copytree(Path("."), outputs.lib) + # create $out output bin_out = outputs.out / Path("bin") lib_out = outputs.out / Path("lib") bin_out.mkdir(parents=True, exist_ok=True) - install_method = get_install_method() + # create $out/lib if install_method == InstallMethod.copy: shutil.copytree(outputs.lib, lib_out, symlinks=True) - shutil.copytree(outputs.deps, lib_out / Path("node_modules"), symlinks=True) + shutil.copytree( + inputs.node_modules, lib_out / Path("node_modules"), symlinks=True + ) elif install_method == InstallMethod.symlink: lib_out.mkdir(parents=True, exist_ok=True) for entry in os.listdir(outputs.lib): (lib_out / Path(entry)).symlink_to(outputs.lib / Path(entry)) - (lib_out / Path("node_modules")).symlink_to(outputs.deps) + (lib_out / Path("node_modules")).symlink_to(inputs.node_modules) - # create binaries - pkg = get_self() + # create $out/bin + # collect all binaries declared from package + # and create symlinks to their sources e.g. $out/bin/cli -> $out/lib/cli.json dep = Dependency(name=pkg.name, version=pkg.version, derivation=outputs.lib) binaries = get_bins(dep) for name, rel_path in binaries.items():