mirror of
https://github.com/nix-community/dream2nix.git
synced 2024-11-23 09:04:37 +03:00
refactor: split concerns in multiple files
This commit is contained in:
hsjobeki
parent
fa0d483e8e
commit
1a556e3812
@ -69,42 +69,16 @@ in `projects.toml` set the `builder` attribute to `'strict-builder'`
|
||||
{{#include ../../../examples/nodejs_alternative_builder/projects.toml}}
|
||||
```
|
||||
|
||||
#### Override on the `strict-builder` (e.g custom bundling / static pages)
|
||||
|
||||
As currently the builder has three outputs: `$out`, `$lib`, `$deps`.
|
||||
|
||||
Sometimes it may be necessary to override the installPhase to build e.g. static page with `webpack` or others like `react`, `vue`, `anuglar` etc.
|
||||
|
||||
When overriding the `installPhase` the user is required to create all three out-paths:
|
||||
|
||||
> next.js export (e.g. for static sites)
|
||||
|
||||
```nix
|
||||
installPhase = ''
|
||||
runHook preInstall
|
||||
|
||||
next export
|
||||
cp -r out $out
|
||||
|
||||
touch $deps
|
||||
touch $lib
|
||||
|
||||
runHook postInstall
|
||||
'';
|
||||
}
|
||||
```
|
||||
|
||||
#### Multiple outputs
|
||||
|
||||
##### deps
|
||||
##### passthru.nodeModules
|
||||
|
||||
- content of `node_modules`
|
||||
- consumable by `devShells`
|
||||
- empty if package has no dependencies
|
||||
|
||||
```bash
|
||||
$deps
|
||||
/nix/store/...-pname-1.0.0-deps
|
||||
/nix/store/...-pname-node_modules-1.0.0
|
||||
├── .bin
|
||||
├── @babel
|
||||
├── ...
|
||||
|
||||
@ -25,6 +25,10 @@
|
||||
# all existing package names and versions
|
||||
# attrset of pname -> versions,
|
||||
# where versions is a list of version strings
|
||||
# type:
|
||||
# packageVersions :: {
|
||||
# ${pname} :: [ ${version} ]
|
||||
# }
|
||||
packageVersions,
|
||||
# function which applies overrides to a package
|
||||
# It must be applied by the builder to each individual derivation
|
||||
@ -36,6 +40,8 @@
|
||||
l = lib // builtins;
|
||||
b = builtins;
|
||||
|
||||
inherit (import ./nodejs_builder {inherit pkgs;}) nodejsBuilder;
|
||||
|
||||
nodejsVersion = subsystemAttrs.nodejsVersion;
|
||||
|
||||
defaultNodejsVersion = l.versions.major pkgs.nodejs.version;
|
||||
@ -55,346 +61,169 @@
|
||||
mv node-* $out
|
||||
'';
|
||||
|
||||
# e.g.
|
||||
# {
|
||||
# "@babel/core": ["1.0.0","2.0.0"]
|
||||
# ...
|
||||
# }
|
||||
# is mapped to
|
||||
# allPackages = {
|
||||
# "@babel/core": {"1.0.0": pkg-derivation, "2.0.0": pkg-derivation }
|
||||
# ...
|
||||
# }
|
||||
# Every package is mapped to a derivation
|
||||
# type:
|
||||
# allPackages :: {
|
||||
# ${pname} :: {
|
||||
# ${version} :: Derivation
|
||||
# }
|
||||
# }
|
||||
allPackages =
|
||||
lib.mapAttrs
|
||||
(
|
||||
name: versions:
|
||||
# genAttrs takes ["1.0.0, 2.0.0"] returns -> {"1.0.0": makePackage name version}
|
||||
# makePackage: produceDerivation: name name (stdenv.mkDerivation {...})
|
||||
# returns {"1.0.0": pkg-derivation, "2.0.0": pkg-derivation }
|
||||
lib.genAttrs
|
||||
versions
|
||||
(version: (mkNodeModule name version))
|
||||
(version: (mkPackage {inherit name version;}))
|
||||
)
|
||||
packageVersions;
|
||||
|
||||
# our builder, written in python. Better handles the complexity with how npm
|
||||
# builds node_modules
|
||||
nodejsBuilder = pkgs.python310Packages.buildPythonApplication {
|
||||
name = "builder";
|
||||
src = ./nodejs_builder;
|
||||
format = "pyproject";
|
||||
nativeBuildInputs = with pkgs.python310Packages; [poetry mypy flake8 black];
|
||||
doCheck = false;
|
||||
};
|
||||
|
||||
/*
|
||||
type:
|
||||
resolveChildren :: {
|
||||
name :: String,
|
||||
version :: String,
|
||||
ancestorCandidates :: {
|
||||
${String} :: {String}
|
||||
}
|
||||
}
|
||||
-> {
|
||||
${name} :: {
|
||||
version :: String,
|
||||
dependencies :: Self,
|
||||
}
|
||||
}
|
||||
|
||||
Function that resolves local vs global dependencies.
|
||||
We copy dependencies into the global node_modules scope, if they don't have
|
||||
conflicts there.
|
||||
Otherwise we need to declare the package as 'private'.
|
||||
*/
|
||||
resolveChildren = {
|
||||
name, #a
|
||||
version, #1.1.2
|
||||
# {
|
||||
# "packageNameA": "1.0.0",
|
||||
# "packageNameB": "2.0.0"
|
||||
# }
|
||||
ancestorCandidates,
|
||||
# function that 'builds' a package's derivation.
|
||||
# type:
|
||||
# mkPackage :: {
|
||||
# name :: String,
|
||||
# version :: String,
|
||||
# } -> Derivation
|
||||
mkPackage = {
|
||||
name,
|
||||
version,
|
||||
}: let
|
||||
directDeps = getDependencies name version;
|
||||
|
||||
/*
|
||||
Determines if a dep needs to be installed as a local dep.
|
||||
Node modules automatically inherit all ancestors and their siblings as
|
||||
dependencies.
|
||||
Therefore, installation of a local dep can be omitted, if the same dep
|
||||
is already present as an ancestor or ancestor sibling.
|
||||
*/
|
||||
installLocally = name: version:
|
||||
!(ancestorCandidates ? ${name})
|
||||
|| (ancestorCandidates.${name} != version);
|
||||
|
||||
locallyRequiredDeps =
|
||||
b.filter (d: installLocally d.name d.version) directDeps;
|
||||
|
||||
localDepsAttrs = b.listToAttrs (
|
||||
l.map (dep: l.nameValuePair dep.name dep.version) locallyRequiredDeps
|
||||
);
|
||||
|
||||
newAncestorCandidates = ancestorCandidates // localDepsAttrs;
|
||||
|
||||
# creates entry for single dependency.
|
||||
mkDependency = name: version: {
|
||||
inherit version;
|
||||
dependencies = resolveChildren {
|
||||
inherit name version;
|
||||
ancestorCandidates = newAncestorCandidates;
|
||||
};
|
||||
};
|
||||
|
||||
# attrset of locally installed dependencies
|
||||
dependencies = l.mapAttrs mkDependency localDepsAttrs;
|
||||
in
|
||||
dependencies;
|
||||
|
||||
# function that 'builds' a package.
|
||||
# executes
|
||||
# type: mkNodeModule :: String -> String -> Derivation
|
||||
mkNodeModule = name: version: let
|
||||
pname = lib.replaceStrings ["@" "/"] ["__at__" "__slash__"] (name + "@" + version);
|
||||
src = getSource name version;
|
||||
pname = lib.replaceStrings ["@" "/"] ["__at__" "__slash__"] name;
|
||||
|
||||
# all direct dependencies of current package
|
||||
deps = getDependencies name version;
|
||||
|
||||
# in case of a conflict pick the highest semantic version as root. All other version must then be private if used.
|
||||
# TODO: pick the version that minimizes the tree
|
||||
pickVersion = name: versions: directDepsAttrs.${name} or (l.head (l.sort (a: b: l.compareVersions a b == 1) versions));
|
||||
rootPackages = l.mapAttrs (name: versions: pickVersion name versions) packageVersions;
|
||||
|
||||
# direct dependencies are all direct dependencies parsed from the lockfile at root level.
|
||||
# Type: deps :: [ { name :: String, version :: String } ]
|
||||
directDeps = getDependencies name version;
|
||||
|
||||
# type: { ${String} :: String } # e.g { "prettier" = "1.2.3"; }
|
||||
directDepsAttrs = l.listToAttrs (b.map (dep: l.nameValuePair dep.name dep.version) directDeps);
|
||||
inherit
|
||||
(import ./lib/node-modules-tree.nix {
|
||||
inherit pkgs lib getDependencies packageVersions name version;
|
||||
nodeModulesBuilder = "${nodejsBuilder}/bin/d2nNodeModules";
|
||||
})
|
||||
nodeModulesTree
|
||||
mkNodeModules
|
||||
;
|
||||
|
||||
# build the node_modules tree from all known rootPackages
|
||||
# type: { ${String} :: { version :: String, dependencies :: Self } }
|
||||
nodeModulesTree =
|
||||
l.mapAttrs (
|
||||
name: version: let
|
||||
dependencies = resolveChildren {
|
||||
inherit name version;
|
||||
ancestorCandidates = rootPackages;
|
||||
};
|
||||
in {
|
||||
inherit version dependencies;
|
||||
}
|
||||
)
|
||||
(l.filterAttrs (n: v: n != name) rootPackages);
|
||||
inherit
|
||||
(import ./lib/dependencies.nix {
|
||||
inherit lib getDependencies allPackages;
|
||||
deps = directDeps;
|
||||
})
|
||||
depsTree
|
||||
;
|
||||
|
||||
nmTreeJSON = b.toJSON nodeModulesTree;
|
||||
# type: devShellNodeModules :: Derivation
|
||||
devShellNodeModules = mkNodeModules {
|
||||
isMain = true;
|
||||
installMethod = "copy";
|
||||
inherit pname version depsTree nodeModulesTree;
|
||||
};
|
||||
# type: nodeModules :: Derivation
|
||||
nodeModules = mkNodeModules {
|
||||
inherit installMethod isMain depsTree nodeModulesTree;
|
||||
inherit pname version;
|
||||
};
|
||||
|
||||
# appends the given dependencyAttrs into the dependencyTree
|
||||
# at location `tree.${dep.name}.${dep.version}`
|
||||
#
|
||||
# type:
|
||||
# makeDepAttrs :: {
|
||||
# deps :: DependencyTree,
|
||||
# dep :: Dependency,
|
||||
# attributes :: DependencyAttrs
|
||||
# } -> DependencyTree
|
||||
#
|
||||
# Dependency :: { name :: String, version :: String }
|
||||
# DependencyAttrs :: { { deps :: DependencyTree, derivation :: Derivation } }
|
||||
# DependencyTree :: { ${name} :: { ${version} :: DependencyAttrs } }
|
||||
insertDependencyAttrs = {
|
||||
dep,
|
||||
dependencyTree,
|
||||
dependencyAttrs,
|
||||
}:
|
||||
dependencyTree
|
||||
// {
|
||||
${dep.name} =
|
||||
(dependencyTree.${dep.name} or {})
|
||||
// {
|
||||
${dep.version} =
|
||||
(dependencyTree.${dep.name}.${dep.version} or {})
|
||||
// dependencyAttrs;
|
||||
};
|
||||
};
|
||||
installMethod =
|
||||
if isMainPackage name version
|
||||
then "copy"
|
||||
else "symlink";
|
||||
|
||||
# The fully rendered dependency tree.
|
||||
# "Who depends on whom"
|
||||
# needed because nix needs to know the order in which derivations must be built.
|
||||
# "Dependencies must be built from bottom to top"
|
||||
#
|
||||
# type: depsTree :: DependencyTree
|
||||
# (see insertDependencyAttrs for declaration)
|
||||
depsTree = let
|
||||
getDeps = tree: (b.foldl'
|
||||
(
|
||||
dependencyTree: dep:
|
||||
insertDependencyAttrs {
|
||||
inherit dependencyTree dep;
|
||||
dependencyAttrs = {
|
||||
deps = getDeps (getDependencies dep.name dep.version);
|
||||
derivation = allPackages.${dep.name}.${dep.version}.lib;
|
||||
};
|
||||
}
|
||||
)
|
||||
{}
|
||||
tree);
|
||||
in (getDeps deps);
|
||||
|
||||
# dependency tree as JSON needed to build node_modules
|
||||
depsTreeJSON = b.toJSON depsTree;
|
||||
|
||||
# Type: src :: Derivation
|
||||
src = getSource name version;
|
||||
|
||||
# produceDerivation makes the mkDerivation overridable by the dream2nix users
|
||||
nodeModules =
|
||||
pkgs.runCommandLocal "node-modules" {
|
||||
pname = "${pname}-node_modules";
|
||||
inherit version;
|
||||
|
||||
buildInputs = with pkgs; [jq nodejs python3];
|
||||
|
||||
inherit nmTreeJSON depsTreeJSON;
|
||||
passAsFile = ["nmTreeJSON" "depsTreeJSON"];
|
||||
} ''
|
||||
export isMain=1
|
||||
export installMethod="copy"
|
||||
|
||||
${nodejsBuilder}/bin/d2nNodeModules
|
||||
|
||||
cp -r /build/node_modules $out
|
||||
'';
|
||||
isMain = isMainPackage name version;
|
||||
|
||||
pkg = produceDerivation name (
|
||||
with pkgs;
|
||||
stdenv.mkDerivation
|
||||
{
|
||||
inherit pname version src nodeSources;
|
||||
inherit nmTreeJSON depsTreeJSON;
|
||||
passAsFile = ["nmTreeJSON" "depsTreeJSON"];
|
||||
pkgs.stdenv.mkDerivation
|
||||
{
|
||||
inherit pname version src;
|
||||
inherit nodeSources installMethod isMain;
|
||||
|
||||
# needed for some current overrides
|
||||
nativeBuildInputs = [makeWrapper];
|
||||
# makeWrapper is needed for some current overrides
|
||||
nativeBuildInputs = with pkgs; [makeWrapper];
|
||||
buildInputs = with pkgs; [jq nodejs python3];
|
||||
|
||||
buildInputs = [jq nodejs python3];
|
||||
outputs = ["out" "lib" "deps"];
|
||||
outputs = ["out" "lib"];
|
||||
|
||||
packageName = pname;
|
||||
name = pname;
|
||||
deps = nodeModules;
|
||||
|
||||
installMethod =
|
||||
if isMainPackage name version
|
||||
then "copy"
|
||||
else "symlink";
|
||||
|
||||
passthru.nodeModules = nodeModules;
|
||||
passthru.devShell = import ./devShell.nix {
|
||||
inherit nodejs pkg pkgs;
|
||||
passthru = {
|
||||
inherit nodeModules;
|
||||
devShell = import ./lib/devShell.nix {
|
||||
inherit nodejs pkgs;
|
||||
nodeModules = devShellNodeModules;
|
||||
};
|
||||
};
|
||||
|
||||
unpackCmd =
|
||||
if lib.hasSuffix ".tgz" src
|
||||
then "tar --delay-directory-restore -xf $src"
|
||||
else null;
|
||||
unpackCmd =
|
||||
if lib.hasSuffix ".tgz" src
|
||||
then "tar --delay-directory-restore -xf $src"
|
||||
else null;
|
||||
|
||||
preConfigurePhases = ["skipForeignPlatform"];
|
||||
preConfigurePhases = ["skipForeignPlatform"];
|
||||
|
||||
unpackPhase = import ./unpackPhase.nix {};
|
||||
unpackPhase = import ./lib/unpackPhase.nix {};
|
||||
|
||||
# pre-checks:
|
||||
# - platform compatibility (os + arch must match)
|
||||
skipForeignPlatform = ''
|
||||
# exit code 3 -> the package is incompatible to the current platform
|
||||
# -> Let the build succeed, but don't create node_modules
|
||||
${nodejsBuilder}/bin/checkPlatform \
|
||||
|| \
|
||||
if [ "$?" == "3" ]; then
|
||||
mkdir -p $out
|
||||
mkdir -p $lib
|
||||
mkdir -p $deps
|
||||
echo "Not compatible with system $system" > $lib/error
|
||||
exit 0
|
||||
else
|
||||
exit 1
|
||||
# checks platform compatibility (os + arch must match)
|
||||
skipForeignPlatform = ''
|
||||
# exit code 3 -> the package is incompatible to the current platform
|
||||
# -> Let the build succeed, but don't create node_modules
|
||||
${nodejsBuilder}/bin/checkPlatform \
|
||||
|| \
|
||||
if [ "$?" == "3" ]; then
|
||||
mkdir -p $out
|
||||
mkdir -p $lib
|
||||
echo "Not compatible with system $system" > $lib/error
|
||||
exit 0
|
||||
else
|
||||
exit 1
|
||||
fi
|
||||
'';
|
||||
|
||||
# only build the main package
|
||||
# deps only get unpacked, installed, patched, etc
|
||||
dontBuild = ! isMain;
|
||||
|
||||
buildPhase = ''
|
||||
runHook preBuild
|
||||
|
||||
if [ "$(jq '.scripts.build' ./package.json)" != "null" ];
|
||||
then
|
||||
echo "running npm run build...."
|
||||
npm run build
|
||||
fi
|
||||
|
||||
runHook postBuild
|
||||
'';
|
||||
|
||||
# create package out-paths
|
||||
# $out
|
||||
# - $out/lib/... -> $lib ...(extracted tgz)
|
||||
# - $out/lib/node_modules -> $deps
|
||||
# - $out/bin
|
||||
|
||||
# $lib
|
||||
# - ... (extracted + install scripts runned)
|
||||
installPhase = ''
|
||||
runHook preInstall
|
||||
|
||||
if [ ! -n "$isMain" ];
|
||||
then
|
||||
if [ "$(jq '.scripts.preinstall' ./package.json)" != "null" ]; then
|
||||
npm --production --offline --nodedir=$nodeSources run preinstall
|
||||
fi
|
||||
'';
|
||||
|
||||
# create the node_modules folder
|
||||
# - uses symlinks as default
|
||||
# - symlink the .bin
|
||||
# - add PATH to .bin
|
||||
configurePhase = ''
|
||||
runHook preConfigure
|
||||
|
||||
export HOME=$TMPDIR
|
||||
|
||||
${nodejsBuilder}/bin/d2nNodeModules
|
||||
|
||||
export PATH="$PATH:node_modules/.bin"
|
||||
|
||||
runHook postConfigure
|
||||
'';
|
||||
|
||||
# only build the main package
|
||||
# deps only get unpacked, installed, patched, etc
|
||||
dontBuild = ! (isMainPackage name version);
|
||||
isMain = isMainPackage name version;
|
||||
# Build:
|
||||
# npm run build
|
||||
# custom build commands for:
|
||||
# - electron apps
|
||||
# fallback to npm lifecycle hooks, if no build script is present
|
||||
buildPhase = ''
|
||||
runHook preBuild
|
||||
|
||||
if [ "$(jq '.scripts.build' ./package.json)" != "null" ];
|
||||
then
|
||||
echo "running npm run build...."
|
||||
npm run build
|
||||
if [ "$(jq '.scripts.install' ./package.json)" != "null" ]; then
|
||||
npm --production --offline --nodedir=$nodeSources run install
|
||||
fi
|
||||
|
||||
runHook postBuild
|
||||
'';
|
||||
|
||||
# copy node_modules
|
||||
# - symlink .bin
|
||||
# - symlink manual pages
|
||||
# - dream2nix copies node_modules folder if it is the top-level package
|
||||
installPhase = ''
|
||||
runHook preInstall
|
||||
|
||||
if [ ! -n "$isMain" ];
|
||||
then
|
||||
if [ "$(jq '.scripts.preinstall' ./package.json)" != "null" ]; then
|
||||
npm --production --offline --nodedir=$nodeSources run preinstall
|
||||
fi
|
||||
if [ "$(jq '.scripts.install' ./package.json)" != "null" ]; then
|
||||
npm --production --offline --nodedir=$nodeSources run install
|
||||
fi
|
||||
if [ "$(jq '.scripts.postinstall' ./package.json)" != "null" ]; then
|
||||
npm --production --offline --nodedir=$nodeSources run postinstall
|
||||
fi
|
||||
if [ "$(jq '.scripts.postinstall' ./package.json)" != "null" ]; then
|
||||
npm --production --offline --nodedir=$nodeSources run postinstall
|
||||
fi
|
||||
fi
|
||||
|
||||
# $out
|
||||
# - $out/lib/... -> $lib ...(extracted tgz)
|
||||
# - $out/lib/node_modules -> $deps
|
||||
# - $out/bin
|
||||
|
||||
# $deps
|
||||
# - $deps/node_modules
|
||||
|
||||
# $lib
|
||||
# - ... (extracted + install scripts runned)
|
||||
${nodejsBuilder}/bin/d2nMakeOutputs
|
||||
${nodejsBuilder}/bin/d2nMakeOutputs
|
||||
|
||||
|
||||
runHook postInstall
|
||||
'';
|
||||
}
|
||||
runHook postInstall
|
||||
'';
|
||||
}
|
||||
);
|
||||
in
|
||||
pkg;
|
||||
|
||||
@ -1,35 +0,0 @@
|
||||
{
|
||||
nodejs,
|
||||
pkg,
|
||||
pkgs,
|
||||
}:
|
||||
with pkgs;
|
||||
mkShell {
|
||||
buildInputs = [
|
||||
nodejs
|
||||
];
|
||||
shellHook = let
|
||||
nodeModulesDir = pkg.deps;
|
||||
in ''
|
||||
# rsync the node_modules folder
|
||||
# - is way faster than copying everything again, because it only replaces updated files
|
||||
# - rsync can be restarted from any point, if failed or aborted mid execution.
|
||||
# Options:
|
||||
# -a -> all files recursive, preserve symlinks, etc.
|
||||
# -E -> preserve executables
|
||||
# --delete -> removes deleted files
|
||||
|
||||
ID=${nodeModulesDir}
|
||||
|
||||
mkdir -p .dream2nix
|
||||
if [[ "$ID" != "$(cat .dream2nix/.node_modules_id)" || ! -d "node_modules" ]];
|
||||
then
|
||||
${rsync}/bin/rsync -aE --chmod=ug+w --delete ${nodeModulesDir}/ ./node_modules/
|
||||
# chmod -R +w ./node_modules
|
||||
|
||||
echo $ID > .dream2nix/.node_modules_id
|
||||
fi
|
||||
|
||||
export PATH="$PATH:$(realpath ./node_modules)/.bin"
|
||||
'';
|
||||
}
|
||||
@ -0,0 +1,62 @@
|
||||
{
|
||||
lib,
|
||||
getDependencies,
|
||||
deps,
|
||||
allPackages,
|
||||
}: let
|
||||
# l = lib // builtins;
|
||||
b = builtins;
|
||||
# appends the given dependencyAttrs into the dependencyTree
|
||||
# at location `tree.${dep.name}.${dep.version}`
|
||||
#
|
||||
# type:
|
||||
# makeDepAttrs :: {
|
||||
# deps :: DependencyTree,
|
||||
# dep :: Dependency,
|
||||
# attributes :: DependencyAttrs
|
||||
# } -> DependencyTree
|
||||
#
|
||||
# Dependency :: { name :: String, version :: String }
|
||||
# DependencyAttrs :: { { deps :: DependencyTree, derivation :: Derivation } }
|
||||
# DependencyTree :: { ${name} :: { ${version} :: DependencyAttrs } }
|
||||
insertDependencyAttrs = {
|
||||
dep,
|
||||
dependencyTree,
|
||||
dependencyAttrs,
|
||||
}:
|
||||
dependencyTree
|
||||
// {
|
||||
${dep.name} =
|
||||
(dependencyTree.${dep.name} or {})
|
||||
// {
|
||||
${dep.version} =
|
||||
(dependencyTree.${dep.name}.${dep.version} or {})
|
||||
// dependencyAttrs;
|
||||
};
|
||||
};
|
||||
|
||||
# The fully rendered dependency tree.
|
||||
# "Who depends on whom"
|
||||
# needed because nix needs to know the order in which derivations must be built.
|
||||
# "Dependencies must be built from bottom to top"
|
||||
#
|
||||
# type: depsTree :: DependencyTree
|
||||
# (see insertDependencyAttrs for declaration)
|
||||
depsTree = let
|
||||
getDeps = tree: (b.foldl'
|
||||
(
|
||||
dependencyTree: dep:
|
||||
insertDependencyAttrs {
|
||||
inherit dependencyTree dep;
|
||||
dependencyAttrs = {
|
||||
deps = getDeps (getDependencies dep.name dep.version);
|
||||
derivation = allPackages.${dep.name}.${dep.version}.lib;
|
||||
};
|
||||
}
|
||||
)
|
||||
{}
|
||||
tree);
|
||||
in (getDeps deps);
|
||||
in {
|
||||
inherit depsTree;
|
||||
}
|
||||
@ -0,0 +1,33 @@
|
||||
{
|
||||
nodejs,
|
||||
nodeModules,
|
||||
pkgs,
|
||||
}:
|
||||
pkgs.mkShell {
|
||||
buildInputs = [
|
||||
nodejs
|
||||
];
|
||||
|
||||
shellHook = ''
|
||||
# rsync the node_modules folder
|
||||
# - way faster than copying everything again, because it only replaces updated files
|
||||
# - rsync can be restarted from any point, if failed or aborted mid execution.
|
||||
# Options:
|
||||
# -a -> all files recursive, preserve symlinks, etc.
|
||||
# --delete -> removes deleted files
|
||||
# --chmod=+ug+w -> make folder writeable by user+group
|
||||
|
||||
ID=${nodeModules}
|
||||
currID=$("$(cat .dream2nix/.node_modules_id)" 2> /dev/null)
|
||||
|
||||
mkdir -p .dream2nix
|
||||
if [[ "$ID" != "$currID" || ! -d "node_modules" ]];
|
||||
then
|
||||
${pkgs.rsync}/bin/rsync -a --chmod=ug+w --delete ${nodeModules}/ ./node_modules/
|
||||
echo $ID > .dream2nix/.node_modules_id
|
||||
echo "Ok: node_modules updated"
|
||||
fi
|
||||
|
||||
export PATH="$PATH:$(realpath ./node_modules)/.bin"
|
||||
'';
|
||||
}
|
||||
@ -0,0 +1,151 @@
|
||||
{
|
||||
lib,
|
||||
getDependencies,
|
||||
packageVersions,
|
||||
name,
|
||||
version,
|
||||
# resolveChildren,
|
||||
pkgs,
|
||||
nodeModulesBuilder,
|
||||
}: let
|
||||
l = lib // builtins;
|
||||
b = builtins;
|
||||
|
||||
/*
|
||||
Function that resolves local vs global dependencies.
|
||||
We copy dependencies into the global node_modules scope, if they don't have
|
||||
conflicts there.
|
||||
Otherwise we need to declare the package as 'private'.
|
||||
|
||||
type:
|
||||
resolveChildren :: {
|
||||
name :: String,
|
||||
version :: String,
|
||||
ancestorCandidates :: {
|
||||
${pname} :: String
|
||||
}
|
||||
}
|
||||
-> Dependencies
|
||||
|
||||
Dependencies :: {
|
||||
${pname} :: {
|
||||
version :: String,
|
||||
dependencies :: Dependencies,
|
||||
}
|
||||
}
|
||||
|
||||
*/
|
||||
resolveChildren = {
|
||||
name,
|
||||
version,
|
||||
ancestorCandidates,
|
||||
}: let
|
||||
directDeps = getDependencies name version;
|
||||
/*
|
||||
Determine if a dependency needs to be installed as a local dep.
|
||||
Node modules automatically inherits all ancestors and their siblings as
|
||||
dependencies.
|
||||
Therefore, installation of a local dep can be omitted, if the same dep
|
||||
is already present as an ancestor or ancestor sibling.
|
||||
*/
|
||||
installLocally = name: version:
|
||||
!(ancestorCandidates ? ${name})
|
||||
|| (ancestorCandidates.${name} != version);
|
||||
|
||||
locallyRequiredDeps =
|
||||
b.filter (d: installLocally d.name d.version) directDeps;
|
||||
|
||||
localDepsAttrs = b.listToAttrs (
|
||||
l.map (dep: l.nameValuePair dep.name dep.version) locallyRequiredDeps
|
||||
);
|
||||
|
||||
newAncestorCandidates = ancestorCandidates // localDepsAttrs;
|
||||
|
||||
# creates entry for single dependency.
|
||||
mkDependency = name: version: {
|
||||
inherit version;
|
||||
dependencies = resolveChildren {
|
||||
inherit name version;
|
||||
ancestorCandidates = newAncestorCandidates;
|
||||
};
|
||||
};
|
||||
|
||||
# attrset of locally installed dependencies
|
||||
dependencies = l.mapAttrs mkDependency localDepsAttrs;
|
||||
in
|
||||
dependencies;
|
||||
|
||||
# in case of a conflict pick the highest semantic version as root. All other version must then be private if used.
|
||||
# TODO: pick the version that minimizes the tree
|
||||
pickVersion = name: versions: directDepsAttrs.${name} or (l.head (l.sort (a: b: l.compareVersions a b == 1) versions));
|
||||
rootPackages = l.mapAttrs (name: versions: pickVersion name versions) packageVersions;
|
||||
|
||||
# direct dependencies are all direct dependencies parsed from the lockfile at root level.
|
||||
directDeps = getDependencies name version;
|
||||
|
||||
# type: { ${name} :: String } # e.g { "prettier" = "1.2.3"; }
|
||||
directDepsAttrs = l.listToAttrs (b.map (dep: l.nameValuePair dep.name dep.version) directDeps);
|
||||
|
||||
# build the node_modules tree from all known rootPackages
|
||||
# type: NodeModulesTree :: { ${name} :: { version :: String, dependencies :: NodeModulesTree } }
|
||||
nodeModulesTree =
|
||||
l.mapAttrs (
|
||||
name: version: let
|
||||
dependencies = resolveChildren {
|
||||
inherit name version;
|
||||
ancestorCandidates = rootPackages;
|
||||
};
|
||||
in {
|
||||
inherit version dependencies;
|
||||
}
|
||||
)
|
||||
(l.filterAttrs (n: v: n != name) rootPackages);
|
||||
|
||||
/*
|
||||
|
||||
Type:
|
||||
mkNodeModules :: {
|
||||
pname :: String,
|
||||
version :: String,
|
||||
isMain :: Bool,
|
||||
installMethod :: "copy" | "symlink",
|
||||
depsTree :: DependencyTree,
|
||||
nodeModulesTree :: NodeModulesTree,
|
||||
installPath :: ? String,
|
||||
}
|
||||
*/
|
||||
mkNodeModules = {
|
||||
isMain,
|
||||
installMethod,
|
||||
pname,
|
||||
version,
|
||||
installPath ? "",
|
||||
depsTree,
|
||||
nodeModulesTree,
|
||||
}:
|
||||
# dependency tree as JSON needed to build node_modules
|
||||
let
|
||||
depsTreeJSON = b.toJSON depsTree;
|
||||
nmTreeJSON = b.toJSON nodeModulesTree;
|
||||
in
|
||||
pkgs.runCommandLocal "node-modules" {
|
||||
pname = "${pname}-node_modules";
|
||||
inherit version;
|
||||
|
||||
buildInputs = with pkgs; [python3];
|
||||
|
||||
inherit nmTreeJSON depsTreeJSON;
|
||||
passAsFile = ["nmTreeJSON" "depsTreeJSON"];
|
||||
} ''
|
||||
|
||||
export isMain=${b.toString isMain}
|
||||
export installMethod=${installMethod}
|
||||
export installPath=${installPath}
|
||||
|
||||
${nodeModulesBuilder}
|
||||
|
||||
cp -r /build/node_modules $out 2> /dev/null || mkdir $out
|
||||
'';
|
||||
in {
|
||||
inherit nodeModulesTree mkNodeModules;
|
||||
}
|
||||
@ -5,7 +5,7 @@
|
||||
''
|
||||
runHook preUnpack
|
||||
|
||||
export sourceRoot="$packageName"
|
||||
export sourceRoot="$name"
|
||||
|
||||
# sometimes tarballs do not end with .tar.??
|
||||
unpackFallback(){
|
||||
@ -0,0 +1,11 @@
|
||||
# our builder, written in python. Better handles the complexity with how npm
|
||||
# builds node_modules
|
||||
{pkgs, ...}: {
|
||||
nodejsBuilder = pkgs.python310Packages.buildPythonApplication {
|
||||
name = "builder";
|
||||
src = ./.;
|
||||
format = "pyproject";
|
||||
nativeBuildInputs = with pkgs.python310Packages; [poetry mypy flake8 black];
|
||||
doCheck = false;
|
||||
};
|
||||
}
|
||||
@ -1,5 +1,5 @@
|
||||
from pathlib import Path
|
||||
from .module import env
|
||||
from .derivation import env
|
||||
|
||||
root = Path("/build")
|
||||
node_modules = root / Path("node_modules")
|
||||
|
||||
@ -9,10 +9,10 @@ class Dependency:
|
||||
name: str
|
||||
version: str
|
||||
derivation: str
|
||||
parent: Union["Dependency", None] = None
|
||||
dependencies: Union[dict[str, Any], None] = None
|
||||
parent: Optional["Dependency"] = None
|
||||
dependencies: Optional[dict[str, Any]] = None
|
||||
|
||||
def repr(self: "Dependency") -> str:
|
||||
def __str__(self: "Dependency") -> str:
|
||||
return f"{self.name}@{self.version}"
|
||||
|
||||
|
||||
@ -21,26 +21,32 @@ def get_all_deps(all_deps: dict[str, Any], name: str, version: str) -> list[str]
|
||||
Returns all dependencies. as flattened list
|
||||
"""
|
||||
|
||||
def is_found(acc: Any, dep: Dependency, dep_tree: Optional[DepsTree]) -> bool:
|
||||
return not bool(acc)
|
||||
def is_found(
|
||||
accumulator: Any, dep: Dependency, dep_tree: Optional[DepsTree]
|
||||
) -> bool:
|
||||
return not bool(accumulator)
|
||||
|
||||
def find_exact_dependency(
|
||||
acc: Any, dep: Dependency, dep_tree: Optional[DepsTree]
|
||||
accumulator: Any, dep: Dependency, dep_tree: Optional[DepsTree]
|
||||
) -> Any:
|
||||
if acc:
|
||||
return acc
|
||||
if accumulator:
|
||||
return accumulator
|
||||
|
||||
if dep.repr() == f"{name}@{version}":
|
||||
if str(dep) == f"{name}@{version}":
|
||||
return dep_tree
|
||||
return None
|
||||
|
||||
subtree = recurse_deps_tree(
|
||||
all_deps, find_exact_dependency, acc=None, pred=is_found, order="top-down"
|
||||
all_deps,
|
||||
find_exact_dependency,
|
||||
accumulator=None,
|
||||
pred=is_found,
|
||||
order="top-down",
|
||||
)
|
||||
|
||||
def flatten(acc: Any, dep: Dependency, dep_tree: Optional[DepsTree]) -> Any:
|
||||
acc.append(dep.repr())
|
||||
return acc
|
||||
def flatten(accumulator: Any, dep: Dependency, dep_tree: Optional[DepsTree]) -> Any:
|
||||
accumulator.append(str(dep))
|
||||
return accumulator
|
||||
|
||||
flattened: list[str] = []
|
||||
if subtree:
|
||||
@ -59,35 +65,35 @@ DepsTree = dict[str, dict[str, Meta]]
|
||||
|
||||
def recurse_deps_tree(
|
||||
deps: DepsTree,
|
||||
cb: Callable[[Any, Dependency, Optional[DepsTree]], Any],
|
||||
acc: Any,
|
||||
parent: Union[Dependency, None] = None,
|
||||
callback: Callable[[Any, Dependency, Optional[DepsTree]], Any],
|
||||
accumulator: Any,
|
||||
parent: Optional[Dependency] = None,
|
||||
order: Literal["bottom-up", "top-down"] = "bottom-up",
|
||||
pred: Optional[Callable[[Any, Dependency, Optional[DepsTree]], bool]] = None,
|
||||
):
|
||||
"""
|
||||
Generic function that traverses the dependency tree and calls
|
||||
'cb' on every node in the tree
|
||||
'callback' on every node in the tree
|
||||
|
||||
Parameters
|
||||
----------
|
||||
deps : DepsTree
|
||||
The nested tree of dependencies, that will be iterated through.
|
||||
cb : Callable[[Any, Dependency, Optional[DepsTree]], Any]
|
||||
The tree of dependencies, that will be iterated through.
|
||||
callback : Callable[[Any, Dependency, Optional[DepsTree]], Any]
|
||||
takes an accumulator (like 'fold' )
|
||||
acc : Any
|
||||
The initial value for the accumulator passed to 'cb'
|
||||
accumulator : Any
|
||||
The initial value for the accumulator passed to 'callback'
|
||||
parent : Dependency
|
||||
The parent dependency, defaults to None, is set automatically during recursion
|
||||
order : Literal["bottom-up", "top-down"]
|
||||
The order in which the callback gets called: "bottom-up" or "top-down"
|
||||
pred : Callable[[Any, Dependency, Optional[DepsTree]], bool]
|
||||
Like 'cb' but returns a bool that will stop recursion if False
|
||||
Like 'callback' but returns a bool that will stop recursion if False
|
||||
|
||||
Returns
|
||||
-------
|
||||
acc
|
||||
the last return value from 'cb: Callable'
|
||||
accumulator
|
||||
the last return value from 'callback: Callable'
|
||||
"""
|
||||
|
||||
dependencies: list[Dependency] = []
|
||||
@ -108,17 +114,17 @@ def recurse_deps_tree(
|
||||
for dependency in dependencies:
|
||||
|
||||
if order == "top-down":
|
||||
acc = cb(acc, dependency, dependency.dependencies)
|
||||
accumulator = callback(accumulator, dependency, dependency.dependencies)
|
||||
|
||||
if dependency.dependencies:
|
||||
stop = False
|
||||
if pred is not None:
|
||||
stop = not pred(acc, dependency, dependency.dependencies)
|
||||
stop = not pred(accumulator, dependency, dependency.dependencies)
|
||||
if not stop:
|
||||
acc = recurse_deps_tree(
|
||||
accumulator = recurse_deps_tree(
|
||||
dependency.dependencies,
|
||||
cb,
|
||||
acc=acc,
|
||||
callback,
|
||||
accumulator=accumulator,
|
||||
parent=dependency,
|
||||
order=order,
|
||||
)
|
||||
@ -127,9 +133,9 @@ def recurse_deps_tree(
|
||||
f"stopped recursing the dependency tree at {dependency.repr()}\
|
||||
-> because the predicate function returned 'False'"
|
||||
)
|
||||
return acc
|
||||
return accumulator
|
||||
|
||||
if order == "bottom-up":
|
||||
acc = cb(acc, dependency, dependency.dependencies)
|
||||
accumulator = callback(accumulator, dependency, dependency.dependencies)
|
||||
|
||||
return acc
|
||||
return accumulator
|
||||
|
||||
@ -1,3 +1,7 @@
|
||||
"""
|
||||
some utility functions to reference the value of
|
||||
variables from the overlaying derivation (via env)
|
||||
"""
|
||||
import os
|
||||
from enum import Enum
|
||||
from typing import Any, Optional
|
||||
@ -6,7 +6,7 @@ from typing import Any, Optional, TypedDict
|
||||
from .config import root
|
||||
from .dependencies import Dependency, DepsTree, get_all_deps, recurse_deps_tree
|
||||
from .logger import logger
|
||||
from .module import InstallMethod, get_install_method, get_self, node_modules_link
|
||||
from .derivation import InstallMethod, get_install_method, get_self, node_modules_link
|
||||
from .package import (
|
||||
NodeModulesPackage,
|
||||
NodeModulesTree,
|
||||
@ -30,7 +30,7 @@ def _create_package_from_derivation(
|
||||
install_method = get_install_method()
|
||||
|
||||
if not get_package_json(Path(dep.derivation)):
|
||||
logger.debug(f"{dep.repr()} is not a package. Skipping installation")
|
||||
logger.debug(f"{str(dep)} is not a package. Skipping installation")
|
||||
return
|
||||
# check if there is already the right package installed
|
||||
if not get_package_json(target):
|
||||
@ -101,18 +101,18 @@ def _make_folders_rec(
|
||||
|
||||
def create_node_modules():
|
||||
def collect_dependency(
|
||||
acc: Any, dep: Dependency, dep_tree: Optional[DepsTree]
|
||||
accumulator: Any, dep: Dependency, dep_tree: Optional[DepsTree]
|
||||
) -> Any:
|
||||
identifier = dep.repr()
|
||||
if identifier not in acc.keys():
|
||||
acc[identifier] = dep
|
||||
return acc
|
||||
identifier = str(dep)
|
||||
if identifier not in accumulator.keys():
|
||||
accumulator[identifier] = dep
|
||||
return accumulator
|
||||
|
||||
nm_tree = get_node_modules_tree()
|
||||
all_deps = get_all_deps_tree()
|
||||
|
||||
collected: dict[str, Dependency] = recurse_deps_tree(
|
||||
all_deps, collect_dependency, acc={}
|
||||
all_deps, collect_dependency, accumulator={}
|
||||
)
|
||||
root_pkg = get_self()
|
||||
flat_deps: list[str] = get_all_deps(all_deps, root_pkg.name, root_pkg.version)
|
||||
@ -123,4 +123,9 @@ def create_node_modules():
|
||||
)
|
||||
|
||||
if node_modules_link:
|
||||
logger.debug(f"/build/node modules symlinked to '{node_modules_link}'.")
|
||||
Path(node_modules_link).symlink_to(root / Path("node_modules"))
|
||||
else:
|
||||
logger.debug(
|
||||
f"/build/node modules created but not exposed. $NODE_MODULES_LINK='{node_modules_link}'. Set it to valid path for automatic symlinks."
|
||||
)
|
||||
|
||||
@ -4,7 +4,7 @@ from pathlib import Path
|
||||
from typing import Any, Optional, TypedDict, Union
|
||||
|
||||
from .dependencies import Dependency, DepsTree
|
||||
from .module import get_env
|
||||
from .derivation import get_env
|
||||
|
||||
package_json_cache = {}
|
||||
|
||||
@ -22,16 +22,14 @@ def get_package_json(path: Path = Path("")) -> Union[dict[str, Any], None]:
|
||||
|
||||
def has_scripts(
|
||||
package_json: dict[str, Any],
|
||||
lifecycle_scripts: list[str] = [
|
||||
lifecycle_scripts: tuple[str] = (
|
||||
"preinstall",
|
||||
"install",
|
||||
"postinstall",
|
||||
],
|
||||
),
|
||||
):
|
||||
return (
|
||||
package_json
|
||||
and package_json.get("scripts")
|
||||
and (set(package_json.get("scripts", {}).keys()) & set(lifecycle_scripts))
|
||||
return package_json and (
|
||||
package_json.get("scripts", {}).keys() & set(lifecycle_scripts)
|
||||
)
|
||||
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
from .lib.checks import check_platform
|
||||
from .lib.config import node_modules
|
||||
from .lib.module import (
|
||||
from .lib.derivation import (
|
||||
is_main_package,
|
||||
get_outputs,
|
||||
get_self,
|
||||
@ -40,21 +40,11 @@ def makeOutputs():
|
||||
build the outputs:
|
||||
- $lib
|
||||
- $out
|
||||
- $deps
|
||||
|
||||
> note: with installMethod == "copy" the symlinks sources are copied over.
|
||||
> note: binaries always reference their source.
|
||||
|
||||
The following three structured outputs are created.
|
||||
|
||||
# node_modules - consumable by devShell
|
||||
# empty if package has no dependencies
|
||||
$deps:
|
||||
/nix/store/...-pname-1.0.0-deps
|
||||
├── .bin
|
||||
├── @babel
|
||||
├── ....
|
||||
└── typescript
|
||||
The following structured outputs are created.
|
||||
|
||||
# package - consumable as bare package
|
||||
# containing all files from the source
|
||||
|
||||
Loading…
Reference in New Issue
Block a user