improve nodejs builders and translators

- add buildPackageWithOtherBuilder helper
  - do not delete cyclic dependencies by default
  - always delete devDependencies in package.json
  - fix handling of source type `path`
  - yarn-lock: handle missing name
  - update utils.dreamLock interface
  - add `createMissingSource` capability to simpleTranslator
This commit is contained in:
DavHau 2021-10-26 15:23:35 +07:00
parent 0e43d2dc7b
commit f49f05e943
14 changed files with 302 additions and 153 deletions

View File

@ -76,6 +76,8 @@
export d2nExternalSources=${externalSourcesFor."${system}"}
export dream2nixWithExternals=${dream2nixFor."${system}".dream2nixWithExternals}
export d2nExternalSources=$dream2nixWithExternals/external
echo "\nManually execute 'export dream2nixWithExternals={path to your dream2nix checkout}'"
'';
});
};

View File

@ -306,7 +306,6 @@ class PackageCommand(Command):
# remove_dependecy(indexed_pkgs, G, cycle[-1][0], cycle[-1][1])
node_from, node_to = cycle[-1][0], cycle[-1][1]
G.remove_edge(node_from, node_to)
depGraph[node_from].remove(node_to)
removed_edges.append((node_from, node_to))
except nx.NetworkXNoCycle:
continue

View File

@ -72,11 +72,19 @@ let
packageVersions;
makeCombinedPackage = name: version:
# b.trace name
(buildPackageWithOtherBuilder
builders.nodejs.node2nix
name
version).package;
let
built =
buildPackageWithOtherBuilder {
inherit name version;
builder = builders.nodejs.node2nix;
inject =
lib.optionalAttrs (dependenciesRemoved ? "${name}"."${version}") {
"${name}"."${version}" =
dependenciesRemoved."${name}"."${version}";
};
};
in
built.package;
makePackage = name: version:
let
@ -176,13 +184,18 @@ let
fi
done
export NODE_PATH="$NODE_PATH:$nodeModules/${packageName}/node_modules"
cd "$nodeModules/${packageName}"
# fix malformed dependency versions in package.json
python ${./fix-package-lock.py} $dependencies_json package.json
export HOME=$TMPDIR
# delete package-lock.json as it can lead to conflicts
rm -f package-lock.json
# fix malformed dependency versions in package.json
cp package.json package.json.bak
python ${./fix-package-lock.py} $dependencies_json package.json
flags=("--offline" "--production" "--nodedir=$nodeSources")
if [ -n "$ignoreScripts" ]; then
flags+=("--ignore-scripts")

View File

@ -9,13 +9,23 @@ with open(sys.argv[2]) as f:
package_json = json.load(f)
changed = False
# delete devDependencies
if 'devDependencies' in package_json:
print(
f"Removing devDependencies from package.json",
file=sys.stderr
)
changed = True
del package_json['devDependencies']
if 'dependencies' in package_json:
for pname, version in package_json['dependencies'].items():
if actual_deps[pname] != package_json['dependencies'][pname]:
package_json['dependencies'][pname] = actual_deps[pname]
changed = True
print(
f"WARNING: replacing malformed version '{version}' for dependency '{pname}' in package.json",
f"Replacing loose version '{version}' with '{actual_deps[pname]}' for dependency '{pname}' in package.json",
file=sys.stderr
)

View File

@ -18,16 +18,21 @@
mainPackageVersion,
getDependencies,
getSource,
packageVersions,
# overrides
packageOverrides ? {},
...
}@args:
let
b = builtins;
getDependencies = name: version:
(
if dependenciesRemoved ? "${name}" && dependenciesRemoved."${name}" ? "${version}" then
dependenciesRemoved."${name}"."${version}" ++ (args.getDependencies name version)
else
args.getDependencies name version;
args.getDependencies name version);
mainPackageKey = "${mainPackageName}#${mainPackageVersion}";
@ -40,30 +45,56 @@ let
or (throw "Could not find nodejs version '${nodejsVersion}' in pkgs");
node2nixEnv = node2nix nodejs;
# allSources =
# lib.mapAttrs
# (packageName: versions:
# lib.genAttrs versions
# (version: {
# inherit packageName version;
# name = utils.sanitizeDerivationName packageName;
# src = getSource packageName version;
# dependencies =
# # b.trace "current package: ${packageName}#${version}"
# lib.forEach
# (lib.filter
# (dep: (! builtins.elem dep mainPackageDependencies))
# (getDependencies packageName version))
# (dep:
# # b.trace "accessing allSources.${dep.name}.${dep.version}"
# b.trace "${dep.name}#${dep.version}"
# allSources."${dep.name}"."${dep.version}"
# );
# }))
# packageVersions;
makeSource = packageName: version: prevDeps:
rec {
inherit packageName version;
name = utils.sanitizeDerivationName packageName;
src = getSource packageName version;
dependencies =
let
parentDeps = prevDeps ++ depsFiltered;
depsFiltered =
(lib.filter
(dep:
! b.elem dep prevDeps)
(getDependencies packageName version));
in
lib.forEach
depsFiltered
(dep: makeSource dep.name dep.version parentDeps);
};
node2nixDependencies =
let
makeSource = packageName: version:
{
inherit packageName version;
name = utils.sanitizeDerivationName packageName;
src = getSource packageName version;
dependencies =
lib.forEach
(lib.filter
(dep: ! builtins.elem dep mainPackageDependencies)
(getDependencies packageName version))
(dep:
makeSource dep.name dep.version
);
};
in
lib.forEach
mainPackageDependencies
(dep: makeSource dep.name dep.version);
lib.forEach
mainPackageDependencies
(dep: makeSource dep.name dep.version mainPackageDependencies);
# (dep: allSources."${dep.name}"."${dep.version}");
callNode2Nix = funcName: args:
node2nixEnv."${funcName}" rec {
node2nixEnv."${funcName}" (rec {
name = utils.sanitizeDerivationName packageName;
packageName = mainPackageName;
version = mainPackageVersion;
@ -81,13 +112,19 @@ let
reconstructLock = true;
src = getSource mainPackageName mainPackageVersion;
}
// args;
// args);
in
{
package = callNode2Nix "buildNodePackage" {};
package =
let
pkg = callNode2Nix "buildNodePackage" {};
in
utils.applyOverridesToPackage packageOverrides pkg mainPackageName;
shell = callNode2Nix "buildNodeShell" {};
# inherit allSources;
}

View File

@ -112,6 +112,8 @@ rec {
args.fetcher;
fetched = fetcher {
mainPackageName = dreamLock.generic.mainPackageName;
mainPackageVersion = dreamLock.generic.mainPackageVersion;
sources = dreamLock'.sources;
sourcesCombinedHash = dreamLock'.generic.sourcesCombinedHash;
};
@ -176,18 +178,49 @@ rec {
builderArgs,
fetchedSources,
dreamLock,
dreamLockInterface,
inject,
sourceOverrides,
}@args:
let
# inject dependencies
dreamLock = utils.dreamLock.injectDependencies args.dreamLock inject;
dreamLockInterface = (utils.readDreamLock { inherit dreamLock; }).interface;
changedSources = sourceOverrides args.fetchedSources;
fetchedSources =
args.fetchedSources // (sourceOverrides args.fetchedSources);
args.fetchedSources // changedSources;
buildPackageWithOtherBuilder =
{
builder,
name,
version,
inject,
}@args2:
let
subDreamLockLoaded =
utils.readDreamLock {
dreamLock =
utils.dreamLock.getSubDreamLock dreamLock name version;
};
in
callBuilder {
inherit builder builderArgs inject sourceOverrides;
dreamLock =
subDreamLockLoaded.lock;
inherit fetchedSources;
};
in
builder {
builder ( builderArgs // {
inherit dreamLock;
inherit
buildPackageWithOtherBuilder
;
inherit (dreamLockInterface)
buildSystemAttrs
@ -200,30 +233,7 @@ rec {
getSource = utils.dreamLock.getSource fetchedSources;
buildPackageWithOtherBuilder = builder: name: version:
let
subDreamLockLoaded =
utils.readDreamLock {
dreamLock =
utils.dreamLock.getSubDreamLock dreamLock name version;
};
filteredFetchedSources =
utils.dreamLock.filterFetchedSources
args.fetchedSources subDreamLockLoaded.lock;
newFetchedSources =
filteredFetchedSources // (sourceOverrides filteredFetchedSources);
in
callBuilder rec {
inherit builder builderArgs sourceOverrides;
dreamLock =
subDreamLockLoaded.lock;
dreamLockInterface = subDreamLockLoaded.interface;
fetchedSources = newFetchedSources;
};
};
});
# build package defined by dream.lock
@ -232,6 +242,7 @@ rec {
dreamLock,
builder ? null,
fetcher ? null,
inject ? {},
sourceOverrides ? oldSources: {},
packageOverrides ? {},
builderArgs ? {},
@ -260,9 +271,14 @@ rec {
}).fetchedSources;
builderOutputs = callBuilder {
inherit dreamLock dreamLockInterface fetchedSources sourceOverrides;
inherit
dreamLock
fetchedSources
inject
sourceOverrides
;
builder = builder';
builderArgs = args.builderArgs // {
builderArgs = (args.builderArgs or {}) // {
inherit packageOverrides;
};
};

View File

@ -35,9 +35,13 @@ let
passthru.originalArgs = args;
})).originalArgs
# handle path sources
else if lib.isString fetched then
"ignore"
# handle unknown sources
else if fetched == "unknown" then
"unknown"
"ignore"
# error out on unknown source types
else
@ -48,7 +52,7 @@ let
)
defaultFetched;
in
lib.filterAttrs (pname: fetcherArgs: fetcherArgs != "unknown") FODArgsAll';
lib.filterAttrs (pname: fetcherArgs: fetcherArgs != "ignore") FODArgsAll';
# convert arbitrary types to string, like nix does with derivation arguments
toString = x:

View File

@ -8,6 +8,8 @@
}:
{
# sources attrset from generic lock
mainPackageName,
mainPackageVersion,
sources,
...
}:
@ -28,6 +30,8 @@ let
"${pname}#${version}"
(if source.type == "unknown" then
"unknown"
else if source.type == "path" then
"${fetchedSources."${mainPackageName}#${mainPackageVersion}"}/${source.path}"
else if fetchers.fetchers ? "${source.type}" then
fetchSource { inherit source; }
else throw "unsupported source type '${source.type}'")

View File

@ -21,17 +21,12 @@
"github",
"gitlab",
"path",
"pypi-sdist"
"pypi-sdist",
"unknown"
]
}
},
"allOf": [
{
"if": {
"properties": { "type": { "const": "unknown" } }
},
"then": { "properties": {} }
},
{
"if": {
"properties": { "type": { "const": "fetchurl" } }
@ -103,6 +98,18 @@
"required": ["type", "pname"],
"additionalProperties": false
}
},
{
"if": {
"properties": { "type": { "const": "unknown" } }
},
"then": {
"properties": {
"type": { "type": "string" }
},
"required": ["type"],
"additionalProperties": false
}
}
]
}

View File

@ -14,6 +14,8 @@
}:
let
b = builtins;
lib = pkgs.lib;
callTranslator = subsystem: type: name: file: args:
@ -111,7 +113,7 @@ let
if def.type == "flag" then
false
else
def.default
def.default or null
)
specialArgsDef;

View File

@ -14,6 +14,7 @@
inputFiles,
# extraArgs
name,
dev,
optional,
peer,
@ -39,8 +40,14 @@
utils.simpleTranslate translatorName rec {
inputData = parsedLock;
mainPackageName = packageJSON.name or "unknown";
mainPackageName =
packageJSON.name or
(if name != null then name else
throw "Could not identify package name. Please specify extra argument 'name'");
mainPackageVersion = packageJSON.version or "unknown";
buildSystemName = "nodejs";
buildSystemAttrs = {
@ -99,19 +106,20 @@
(dependency:
builtins.head (
lib.mapAttrsToList
(name: value:
(name: versionSpec:
let
yarnName = "${name}@${value}";
yarnName = "${name}@${versionSpec}";
depObject = dependenciesByOriginalID."${yarnName}";
version = depObject.version;
in
if ! dependenciesByOriginalID ? ${yarnName} then
# handle missing lock file entry
let
yarnNameSplit = lib.splitString "@" yarnName;
versionMatch = b.match ''.*\^([[:digit:]|\.]+)'' versionSpec;
in
{
name = b.elemAt yarnNameSplit 0;
version = b.elemAt yarnNameSplit 1;
inherit name;
version = b.elemAt versionMatch 0;
}
else
{ inherit name version; }
@ -174,6 +182,13 @@
};
};
# TODO: implement createMissingSource
# createMissingSource = name: version:
# let
# pname = lib.last (lib.splitString "/" name);
# in
};
@ -201,6 +216,15 @@
# String arguments contain a default value and examples. Flags do not.
specialArgs = {
name = {
description = "The name of hte main package";
examples = [
"react"
"@babel/code-frame"
];
type = "argument";
};
dev = {
description = "Whether to include development dependencies";
type = "flag";

View File

@ -108,6 +108,16 @@ rec {
sanitizeDerivationName = name:
lib.replaceStrings [ "@" "/" ] [ "__at__" "__slash__" ] name;
keyToNameVersion = key:
let
split = lib.splitString "#" key;
name = b.elemAt split 0;
version = b.elemAt split 1;
in
{ inherit name version; };
nameVersionToKey = nameVersion:
"${nameVersion.name}#${nameVersion.version}";
}

View File

@ -1,5 +1,8 @@
{
lib,
# dream2nix
utils,
...
}:
let
@ -25,9 +28,23 @@ let
dependencyGraph = lock.generic.dependencyGraph;
packageVersions =
lib.mapAttrs
(pname: versions: lib.attrNames versions)
sources;
let
allDependencyKeys =
lib.attrNames
(lib.genAttrs
(lib.flatten
((lib.attrValues dependencyGraph) ++ (lib.attrNames dependencyGraph)))
(x: null));
in
lib.foldl'
(packageVersions: dep:
packageVersions // {
"${dep.name}" = (packageVersions."${dep.name}" or []) ++ [
dep.version
];
})
{}
(b.map (utils.keyToNameVersion) allDependencyKeys);
dependenciesRemoved =
lib.mapAttrs
@ -35,13 +52,7 @@ let
lib.mapAttrs
(version: removedKeys:
lib.forEach removedKeys
(rKey:
let
split = lib.splitString "#" rKey;
name = b.elemAt split 0;
version = b.elemAt split 1;
in
{ inherit name version; }))
(rKey: utils.keyToNameVersion rKey))
versions)
lock.generic.dependenciesRemoved;
@ -56,18 +67,16 @@ let
lib.mapAttrs
(key: deps:
lib.forEach deps
(dep:
let
split = lib.splitString "#" dep;
name = b.elemAt split 0;
version = b.elemAt split 1;
in
{ inherit name version; }))
(dep: utils.keyToNameVersion dep))
dependencyGraph;
getDependencies = pname: version:
if dependenciesAttrs ? "${pname}#${version}" then
dependenciesAttrs."${pname}#${version}"
# filter out cyclicDependencies
lib.filter
(dep: ! b.elem dep (dependenciesRemoved."${pname}"."${version}" or []))
dependenciesAttrs."${pname}#${version}"
# assume no deps if package not found in dependencyGraph
else
[];
@ -120,79 +129,50 @@ let
getSubDreamLock = dreamLock: name: version:
let
lock = (readDreamLock { inherit dreamLock; }).lock;
allDependneciesOf = key:
let
next = lock.generic.dependencyGraph."${key}" or [];
in
next
++
lib.flatten (b.map allDependneciesOf next);
# use set for efficient lookups
allDependenciesOfMainPackage =
lib.genAttrs (allDependneciesOf "${name}#${version}") (x: null);
newSources =
lib.filterAttrs
(p_name: versions: versions != {})
(lib.mapAttrs
(p_name: versions:
lib.filterAttrs
(p_version: source:
allDependenciesOfMainPackage ? "${p_name}#${p_version}"
|| (name == p_name && version == p_version))
versions)
lock.sources);
newDependencyGraph =
lib.filterAttrs
(key: deps:
let
split = lib.splitString "#" key;
p_name = b.elemAt split 0;
p_version = b.elemAt split 1;
in
allDependenciesOfMainPackage ? "${key}"
||
(name == p_name && version == p_version))
lock.generic.dependencyGraph;
in
lock // {
generic = lock.generic // {
mainPackageName = name;
mainPackageVersion = version;
dependencyGraph = newDependencyGraph;
};
sources = newSources;
};
filterFetchedSources = fetchedSources: dreamLock:
injectDependencies = dreamLock: inject:
if inject == {} then dreamLock else
let
lockInterface = (readDreamLock { inherit dreamLock; }).interface;
allDependencyKeys =
lib.flatten
(lib.mapAttrsToList
(name: versions: lib.forEach versions
(v: "${name}#${v}"))
lockInterface.packageVersions);
lock = (readDreamLock { inherit dreamLock; }).lock;
allDependencyKeysSet = lib.genAttrs allDependencyKeys (x: null);
oldDependencyGraph = lock.generic.dependencyGraph;
newDependencyGraph =
lib.mapAttrs
(key: deps:
let
nameVer = utils.keyToNameVersion key;
in
deps
++
(if inject ? "${nameVer.name}"."${nameVer.version}" then
(b.map
utils.nameVersionToKey
inject."${nameVer.name}"."${nameVer.version}")
else
[]))
oldDependencyGraph;
in
lib.filterAttrs
(key: source: allDependencyKeysSet ? "${key}")
fetchedSources;
lib.recursiveUpdate lock {
generic.dependencyGraph = newDependencyGraph;
};
in
{
inherit
getMainPackageSource
filterFetchedSources
getSource
getSubDreamLock
readDreamLock
injectDependencies
;
}

View File

@ -25,6 +25,7 @@ let
getVersion,
getSourceType,
sourceConstructors,
createMissingSource ? (name: version: { type = "unknown"; }),
getDependencies ? null,
getOriginalID ? null,
mainPackageSource ? { type = "unknown"; },
@ -81,12 +82,52 @@ let
(depNameVer: "${depNameVer.name}#${depNameVer.version}")
(getDependencies pkgData getDepByNameVer dependenciesByOriginalID))));
allDependencyKeys =
lib.attrNames
(lib.genAttrs
(b.foldl'
(a: b: a ++ b)
[]
(lib.attrValues dependencyGraph))
(x: null));
missingDependencies =
lib.flatten
(lib.forEach allDependencyKeys
(depKey:
let
split = lib.splitString "#" depKey;
name = b.elemAt split 0;
version = b.elemAt split 1;
in
if sources ? "${name}" && sources."${name}" ? "${version}" then
[]
else
{ inherit name version; }));
generatedSources =
if missingDependencies == [] then
{}
else
lib.listToAttrs
(b.map
(dep: lib.nameValuePair
"${dep.name}"
{
"${dep.version}" =
createMissingSource dep.name dep.version;
})
missingDependencies);
allSources =
lib.recursiveUpdate sources generatedSources;
getDepByNameVer = name: version:
allDependencies."${name}"."${version}";
in
{
inherit sources;
sources = allSources;
generic =
{