add new node builder mono-module (#748)

* add new node builder mono-module

* add example

* nodejs-builder-v3: rename to WIP-nodejs-builder-v3

* fix tests

---------

Co-authored-by: DavHau <hsngrmpf+github@gmail.com>
This commit is contained in:
Johannes Kirschbauer 2023-10-17 21:59:10 +02:00 committed by GitHub
parent b10f3495e5
commit 419d6f08c4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 794 additions and 1 deletions

View File

@ -0,0 +1,34 @@
{
lib,
config,
dream2nix,
...
}: {
imports = [
dream2nix.modules.dream2nix.WIP-nodejs-builder-v3
];
mkDerivation = {
src = config.deps.fetchFromGitHub {
owner = "DavHau";
repo = "cowsay";
rev = "package-lock-v3";
sha256 = "sha256-KuZkGWl5An78IFR5uT/2jVTXdm71oWB+p143svYVkqQ=";
};
};
deps = {nixpkgs, ...}: {
inherit
(nixpkgs)
fetchFromGitHub
stdenv
;
};
WIP-nodejs-builder-v3 = {
packageLockFile = "${config.mkDerivation.src}/package-lock.json";
};
name = "cowsay";
version = "1.5.0";
}

View File

@ -0,0 +1,73 @@
import fs from "fs";
import path from "path";
import { abort } from "process";
const { out, FILESYSTEM } = process.env;
/**
* A binary symlink called 'name' is created pointing to the executable in 'target'.
*
* @argument {[string, string]}
*/
function createBinEntry([name, target]) {
const binDir = path.join(out, path.dirname(name));
if (!fs.existsSync(binDir)) {
fs.mkdirSync(binDir, { recursive: true }, () => {
if (err) {
console.error(err);
abort();
}
console.log(`created dir: ${folder}`);
});
}
const relTarget = path.relative(path.dirname(name), target);
fs.chmod(target, fs.constants.S_IXUSR | fs.constants.S_IRUSR, () => {});
fs.symlink(relTarget, path.join(out, name), (err) => {
if (err) {
console.log(`could NOT symlink: ${name} -> ${target}`);
console.error(err);
abort();
}
console.log(`symlinked ${name} -> ${relTarget}`);
});
}
/**
* The source dist is copied to the target folder.
*
* @argument {[string, { source: string; bins: { [key: string]: string } } ] }
*/
function createEntry([folder, value]) {
const finalPath = path.join(out, folder);
fs.mkdirSync(finalPath, { recursive: true }, (err) => {
if (err) {
console.error(err);
abort();
}
console.log(`created dir: ${folder}`);
});
fs.cpSync(value.source, finalPath, { recursive: true }, (err) => {
if (err) {
console.error(err);
abort();
}
console.log(`copied: ${value.source} -> ${folder}`);
});
Object.entries(value.bins).forEach(createBinEntry);
}
Object.entries(JSON.parse(FILESYSTEM)).forEach(createEntry);
if (!fs.existsSync(out)) {
fs.mkdirSync(out, { recursive: true }, () => {
if (err) {
console.error(err);
abort();
}
console.log(`created empty out: ${out}`);
});
}

View File

@ -0,0 +1,234 @@
{
config,
lib,
dream2nix,
...
}: let
l = lib // builtins;
cfg = config.WIP-nodejs-builder-v3;
inherit (config.deps) fetchurl;
nodejsLockUtils = import ../../../lib/internal/nodejsLockUtils.nix {inherit lib;};
isLink = plent: plent ? link && plent.link;
parseSource = plent: name:
if isLink plent
then
# entry is local file
(builtins.dirOf cfg.packageLockFile) + "/${plent.resolved}"
else
config.deps.mkDerivation {
inherit name;
inherit (plent) version;
src = fetchurl {
url = plent.resolved;
hash = plent.integrity;
};
dontBuild = true;
installPhase = ''
cp -r . $out
'';
};
# Lock -> Pdefs
parse = lock:
builtins.foldl'
(acc: entry:
acc
// {
${entry.name} = acc.${entry.name} or {} // entry.value;
})
{}
# [{name=; value=;} ...]
(l.mapAttrsToList (parseEntry lock) lock.packages);
############################################################
pdefs = parse cfg.packageLock;
############################################################
# Utility functions #
# Type: lock.packages -> Info
getInfo = path: plent: {
initialPath = path;
initialState =
if
isLink plent
||
/*
IsRoot
*/
path == ""
then "source"
else "dist";
};
# Type: lock.packages -> Bins
getBins = path: plent:
if plent ? bin
then
if l.isAttrs plent.bin
then plent.bin
else if l.isList plent.bin
then {} l.foldl' (res: bin: res // {${bin} = bin;}) {} plent.bin
else throw ""
else {};
getDependencies = lock: path: plent:
l.mapAttrs (name: _descriptor: {
dev = plent.dev or false;
version = let
# Need this util as dependencies no explizitly locked version
# This findEntry is needed to find the exact locked version
packageIdent = nodejsLockUtils.findEntry lock path name;
in
# Read version from package-lock entry for the resolved package
lock.packages.${packageIdent}.version;
})
(plent.dependencies or {} // plent.devDependencies or {} // plent.optionalDependencies or {});
# Takes one entry of "package" from package-lock.json
parseEntry = lock: path: entry: let
info = getInfo path entry;
# TODO: Verify this is reasonable default;
source = builtins.dirOf cfg.packageLockFile;
makeNodeModules = ./build-node-modules.mjs;
in
if path == ""
then let
prepared-dev = config.deps.mkDerivation {
name = entry.name + "-node_modules-dev";
inherit (entry) version;
dontUnpack = true;
env = {
FILESYSTEM = builtins.toJSON (getFileSystem pdefs);
};
buildInputs = with config.deps; [nodejs];
buildPhase = ''
node ${makeNodeModules}
'';
};
dist = config.deps.mkDerivation {
inherit (entry) version;
name = entry.name + "-dist";
src = source;
buildInputs = with config.deps; [nodejs jq];
configurePhase = ''
cp -r ${prepared-dev}/node_modules node_modules
# TODO: run installScripts of trusted dependencies
'';
buildPhase = ''
echo "BUILDING... $name"
if [ -n "$runBuild" ] && [ "$(jq '.scripts.build' ./package.json)" != "null" ]; then
npm run build
fi;
'';
installPhase = ''
cp -r . $out
'';
};
in {
# Root level package
name = entry.name;
value = {
${entry.version} = {
dependencies = getDependencies lock path entry;
inherit info prepared-dev source dist;
};
};
}
else let
name = l.last (builtins.split "node_modules/" path);
source = parseSource entry name;
bins = getBins path entry;
version =
if isLink entry
then let
pjs = l.fromJSON (l.readFile (source + "/package.json"));
in
pjs.version
else entry.version;
in
# Every other package
{
inherit name;
value = {
${version} = {
inherit info bins;
dependencies = getDependencies lock path entry;
# We need to determine the initial state of every package and
# TODO: define dist and installed if they are in source form. We currently only do this for the root package.
${info.initialState} = source;
};
};
};
/*
Function that returns instructions to create the file system (aka. node_modules directory)
Every `source` entry here is created. Bins are symlinked to their target.
This behavior is implemented via the prepared-builder script.
@argument pdefs'
# The filtered and sanititized pdefs containing no cycles.
# Only pdefs required by the current root and environment.
# e.g. all buildtime dependencies of top-level package.
->
fileSystem :: {
"node_modules/typescript": {
source: <derivation typescript-dist>
bins: {
"node_modules/.bin/tsc": "node_modules/typescript/bin/tsc"
}
}
}
*/
getFileSystem = pdefs':
l.foldl' (
res: name:
res
// l.mapAttrs' (version: entry: {
name = entry.info.initialPath;
value = {
source = entry.dist;
bins =
l.mapAttrs' (name: target: {
name = (builtins.dirOf entry.info.initialPath) + "/.bin/" + name;
value = entry.info.initialPath + "/" + target;
})
pdefs'.${name}.${version}.bins;
};
}) (l.filterAttrs (n: v: v.info.initialState == "dist") pdefs'.${name})
) {} (l.attrNames pdefs');
in {
imports = [
./interface.nix
dream2nix.modules.dream2nix.mkDerivation
];
# declare external dependencies
deps = {nixpkgs, ...}: {
inherit
(nixpkgs)
fetchurl
jq
tree
;
nodejs = nixpkgs.nodejs_latest;
inherit
(nixpkgs.stdenv)
mkDerivation
;
};
package-func.result = l.mkForce (pdefs.${config.name}.${config.version}.dist);
# OUTPUTS
WIP-nodejs-builder-v3 = {
inherit pdefs;
packageLock =
lib.mkDefault
(builtins.fromJSON (builtins.readFile cfg.packageLockFile));
};
}

View File

@ -0,0 +1,34 @@
{
lib,
dream2nix,
specialArgs,
...
}: let
l = lib // builtins;
t = l.types;
in {
options.WIP-nodejs-builder-v3 = l.mapAttrs (_: l.mkOption) {
packageLockFile = {
type = t.nullOr t.path;
description = ''
The package-lock.json file to use.
'';
};
packageLock = {
type = t.attrs;
description = "The content of the package-lock.json";
};
inherit
(import ./types.nix {
inherit
lib
dream2nix
specialArgs
;
})
pdefs
fileSystem
;
};
}

View File

@ -0,0 +1,164 @@
/*
* Shared interface declarations
*/
{
lib,
dream2nix,
specialArgs,
...
}: let
l = lib // builtins;
t = l.types;
derivationType = t.oneOf [t.str t.path t.package];
# A stricter submodule type that prevents derivations from being
# detected as modules by accident. (derivations are attrs as well as modules)
drvPart = let
type = t.submoduleWith {
modules = [dream2nix.modules.dream2nix.core];
inherit specialArgs;
};
in
type
// {
# Ensure that derivations are never detected as modules by accident.
check = val: type.check val && (val.type or null != "derivation");
};
drvPartOrPackage = t.either derivationType drvPart;
# optPackage = l.mkOption {
# # type = t.raw;
# type = drvPartOrPackage;
# apply = drv: drv.public or drv;
# # default = null;
# };
optOptionalPackage = l.mkOption {
type = t.nullOr drvPartOrPackage;
apply = drv: drv.public or drv;
default = null;
};
depEntryType = t.submodule {
options.dev = l.mkOption {
type = t.bool;
# default = false;
};
options.version = l.mkOption {
type = t.str;
};
};
optBins = l.mkOption {
type = t.attrsOf t.str;
default = {};
};
# dependencies = {
# ${name} = {
# dev = boolean;
# version :: string;
# }
# }
dependenciesType = t.attrsOf depEntryType;
in {
/*
pdefs.${name}.${version} :: {
// [REQUIRED] all dependency entries of that package. (Might be empty)
// each dependency is guaranteed to have its own entry in 'pdef'
// A package without dependencies has `dependencies = {}` (So dependencies has a constant type)
dependencies = {
${name} = {
dev = boolean;
version :: string;
}
}
// [OPTIONAL] Pointing to the source of the package.
// SOURCE State of the package. Not every package is defined from source.
// The rawest form of a package. The plain source code.
// e.g. <fetch derivation> OR <filterSource derivation>
source :: Derivation | Path
// [OPTIONAL] PREPARED state of the packge
// noSource = true;
// Contains everything that is needed to build the package.
// Does NOT contain the source code!
// prod contains only the dependencies for runtime (needed in installed) dev contains everything needed for building (needed in dist and for the devShell).
// could contain e.g. node_modules, .svelte-kit
prepared-dev :: Derivation | Path
prepared-prod :: Derivation | Path
// [REQUIRED] BUILT State of the packge
// src = ./.;
// Ready to use as an dependency input for PREPARED state of another package.
// The equivalence of npm package from npmjs.org registry.
// In fact npm dependencies are in this state. Usually they don't even have SOURCE state.
dist :: Derivation | Path
// [REQUIRED] INSTALLED State of the packge
// Ready to be used via e.g. nix-shell
installed :: Derivation | Path
info :: {
initialState :: "source" | "dist"
initialPath :: "node_modules/"
}
}
*/
pdefs = {
type = t.attrsOf (t.attrsOf (t.submodule {
options.dependencies = l.mkOption {
type = dependenciesType;
};
options.source = optOptionalPackage;
options.prepared-dev = optOptionalPackage;
options.prepared-prod = optOptionalPackage;
options.dist = optOptionalPackage;
# options.installed = optOptionalPackage;
options.info.initialState = l.mkOption {
type = t.enum ["source" "dist"];
};
options.info.initialPath = l.mkOption {
type = t.str;
};
/*
"bin": {
"esparse": "bin/esparse.js",
"esvalidate": "bin/esvalidate.js"
}
*/
options.bins = optBins;
}));
};
/*
fileSystem :: {
"node_modules/babel" :: <derivation> ;
${nodePath} :: Derivation
}
*/
fileSystem = {
type = t.nullOr (t.attrsOf (t.submodule {
options.source = l.mkOption {
type = t.nullOr drvPartOrPackage;
};
options.bins = optBins;
}));
# options.bins = l.mkOption {
# type = t.attrsOf (t.submodule {
# options.name = l.mkOption {
# type = t.str;
# };
# options.target = l.mkOption {
# type = t.str;
# };
# });
# };
# }));
# default = null;
};
}

View File

@ -30,7 +30,6 @@ in {
The result of calling the final derivation function.
This is not necessarily the same as `final.package`. The function output might not be compatible to the interface of `final.package` and additional logic might be needed to create `final.package`.
'';
readOnly = true;
};
# add an option for each output, eg. out, bin, lib, etc...

View File

@ -0,0 +1,72 @@
{
pkgs ? import <nixpkgs> {},
lib ? import <nixpkgs/lib>,
dream2nix ? (import (../../../modules + "/flake.nix")).outputs {},
}: let
eval = module:
lib.evalModules {
modules = [module];
specialArgs = {
inherit dream2nix;
packageSets = {
nixpkgs = pkgs;
};
};
};
in {
test_nodejs_eval_dist = let
evaled = eval ({config, ...}: {
imports = [
dream2nix.modules.dream2nix.WIP-nodejs-builder-v3
];
WIP-nodejs-builder-v3.packageLockFile = ./package-lock.json;
});
config = evaled.config;
in {
expr = lib.generators.toPretty {} config.WIP-nodejs-builder-v3.pdefs."minimal"."1.0.0".dist;
expected = "<derivation minimal-dist>";
};
test_nodejs_eval_nodeModules = let
evaled = eval ({config, ...}: {
imports = [
dream2nix.modules.dream2nix.WIP-nodejs-builder-v3
];
WIP-nodejs-builder-v3.packageLockFile = ./package-lock.json;
});
config = evaled.config;
in {
expr = lib.generators.toPretty {} config.WIP-nodejs-builder-v3.pdefs."minimal"."1.0.0".prepared-dev;
expected = "<derivation minimal-node_modules-dev>";
};
test_nodejs_root_info = let
evaled = eval ({config, ...}: {
imports = [
dream2nix.modules.dream2nix.WIP-nodejs-builder-v3
];
WIP-nodejs-builder-v3.packageLockFile = ./package-lock.json;
});
config = evaled.config;
in {
expr = config.WIP-nodejs-builder-v3.pdefs."minimal"."1.0.0".info;
expected = {
initialPath = "";
initialState = "source";
};
};
# TODO: There is no prod node_modules yet.
# test_nodejs_eval_nodeModules_prod = let
# evaled = eval ({config, ...}: {
# imports = [
# dream2nix.modules.dream2nix.WIP-nodejs-builder-v3
# ];
# WIP-nodejs-builder-v3.packageLockFile = ./package-lock.json;
# });
# config = evaled.config;
# in {
# expr = lib.generators.toPretty {} config.WIP-nodejs-builder-v3.pdefs."minimal"."1.0.0".prepared-prod;
# expected = "<derivation minimal-node_modules-prod>";
# };
}

View File

@ -0,0 +1,12 @@
{
"name": "lib",
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"keywords": [],
"author": "",
"license": "ISC"
}

View File

@ -0,0 +1,64 @@
{
"name": "cowsay",
"version": "1.5.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"node_modules/chalk/node_modules/strip-ansi": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz",
"integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=",
"dev": true,
"dependencies": {
"ansi-regex": "^2.0.0"
},
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/cliui/node_modules/strip-ansi": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz",
"integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==",
"dependencies": {
"ansi-regex": "^5.0.0"
},
"engines": {
"node": ">=8"
}
},
"node_modules/strip-ansi": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz",
"integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=",
"dependencies": {
"ansi-regex": "^3.0.0"
},
"engines": {
"node": ">=4"
}
},
"node_modules/wrap-ansi/node_modules/strip-ansi": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz",
"integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==",
"dependencies": {
"ansi-regex": "^5.0.0"
},
"engines": {
"node": ">=8"
}
},
"node_modules/yargs/node_modules/strip-ansi": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz",
"integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==",
"dependencies": {
"ansi-regex": "^5.0.0"
},
"engines": {
"node": ">=8"
}
}
}
}

View File

@ -0,0 +1,97 @@
{
"name": "minimal",
"version": "1.0.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "minimal",
"version": "1.0.0",
"dependencies": {
"kitten": "^0.0.2"
}
},
"node_modules/@org/nested/node_modules/kitten": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/kitten/-/kitten-0.0.2.tgz",
"integrity": "sha512-OAxjUlTdnuVacqvwmjfz40qqVv4zc7Y8/l2p0FEpNKnEXdn4DlKaxPQRGyU1VdvORAIAaQqcBP5O33S5WhCeIQ==",
"extraneous": true
},
"node_modules/argparse": {
"version": "0.1.16",
"resolved": "https://registry.npmjs.org/argparse/-/argparse-0.1.16.tgz",
"integrity": "sha512-LjmC2dNpdn2L4UzyoaIr11ELYoLn37ZFy9zObrQFHsSuOepeUEMKnM8w5KL4Tnrp2gy88rRuQt6Ky8Bjml+Baw==",
"dependencies": {
"underscore": "~1.7.0",
"underscore.string": "~2.4.0"
}
},
"node_modules/async": {
"version": "0.2.10",
"resolved": "https://registry.npmjs.org/async/-/async-0.2.10.tgz",
"integrity": "sha512-eAkdoKxU6/LkKDBzLpT+t6Ff5EtfSF4wx1WfJiPEEV7WNLnDaRXk0oVysiEPm262roaachGexwUv94WhSgN5TQ=="
},
"node_modules/esprima": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/esprima/-/esprima-1.0.4.tgz",
"integrity": "sha512-rp5dMKN8zEs9dfi9g0X1ClLmV//WRyk/R15mppFNICIFRG5P92VP7Z04p8pk++gABo9W2tY+kHyu6P1mEHgmTA==",
"bin": {
"esparse": "bin/esparse.js",
"esvalidate": "bin/esvalidate.js"
},
"engines": {
"node": ">=0.4.0"
}
},
"node_modules/js-yaml": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-2.1.3.tgz",
"integrity": "sha512-2ElQ5tUBsI5GIjddfYGdudelD5+9JM9FfJXlrn+Mj3k72t4XrqBr3vf3+1sky0WKC3dSVhF0ZqIUpX9QFBmmfQ==",
"dependencies": {
"argparse": "~ 0.1.11",
"esprima": "~ 1.0.2"
},
"bin": {
"js-yaml": "bin/js-yaml.js"
},
"engines": {
"node": ">= 0.6.0"
}
},
"node_modules/kitten": {
"version": "0.0.2",
"resolved": "https://registry.npmjs.org/kitten/-/kitten-0.0.2.tgz",
"integrity": "sha512-OAxjUlTdnuVacqvwmjfz40qqVv4zc7Y8/l2p0FEpNKnEXdn4DlKaxPQRGyU1VdvORAIAaQqcBP5O33S5WhCeIQ==",
"dependencies": {
"async": "0.2.x",
"js-yaml": "2.1.x",
"lodash": "1.3.x"
},
"engines": {
"node": ">=0.8.x"
}
},
"node_modules/lodash": {
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-1.3.1.tgz",
"integrity": "sha512-F7AB8u+6d00CCgnbjWzq9fFLpzOMCgq6mPjOW4+8+dYbrnc0obRrC+IHctzfZ1KKTQxX0xo/punrlpOWcf4gpw==",
"engines": [
"node",
"rhino"
]
},
"node_modules/underscore": {
"version": "1.7.0",
"resolved": "https://registry.npmjs.org/underscore/-/underscore-1.7.0.tgz",
"integrity": "sha512-cp0oQQyZhUM1kpJDLdGO1jPZHgS/MpzoWYfe9+CM2h/QGDZlqwT2T3YGukuBdaNJ/CAPoeyAZRRHz8JFo176vA=="
},
"node_modules/underscore.string": {
"version": "2.4.0",
"resolved": "https://registry.npmjs.org/underscore.string/-/underscore.string-2.4.0.tgz",
"integrity": "sha512-yxkabuCaIBnzfIvX3kBxQqCs0ar/bfJwDnFEHJUm/ZrRVhT3IItdRF5cZjARLzEnyQYtIUhsZ2LG2j3HidFOFQ==",
"engines": {
"node": "*"
}
}
}
}

View File

@ -0,0 +1,10 @@
{
"name": "minimal",
"version": "1.0.0",
"dependencies": {
"kitten": "^0.0.2"
},
"scripts": {
"build": "node ./node_modules/.bin/esparse --version"
}
}