nix-community/dream2nix
1
1
Fork 0
mirror of https://github.com/nix-community/dream2nix.git synced 2024-09-11 14:35:30 +03:00
Code Issues Projects Releases Wiki Activity

treewide: format with alejandra 1.0.0

Browse Source 
closes #84
This commit is contained in:
DavHau 2022-03-07 17:12:07 +07:00
parent 7641236307
commit 076a1c9aff
69 changed files with 5774 additions and 6468 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
ci.nix
View File

@ -4,18 +4,13 @@ let
flake = (import flakeCompatSrc {src = ./.;}).defaultNix;
pkgs = import flake.inputs.nixpkgs {};
recurseIntoAll = b.mapAttrs (name: val: pkgs.recurseIntoAttrs val);
in
# {
# inherit flake;
# }
# // (recurseIntoAll {
# checks = flake.checks.x86_64-linux;
# })
# hercules ci's nix version cannot fetch submodules and crashes
{
inherit (pkgs) hello;

81
flake.nix
View File

@ -14,22 +14,36 @@
flake-utils-pre-commit.url = "github:numtide/flake-utils";
pre-commit-hooks.inputs.flake-utils.follows = "flake-utils-pre-commit";
### framework dependencies
# required for builder go/gomod2nix
gomod2nix = { url = "github:tweag/gomod2nix"; flake = false; };
gomod2nix = {
url = "github:tweag/gomod2nix";
flake = false;
};
# required for translator pip
mach-nix = { url = "mach-nix"; flake = false; };
mach-nix = {
url = "mach-nix";
flake = false;
};
# required for builder nodejs/node2nix
node2nix = { url = "github:svanderburg/node2nix"; flake = false; };
node2nix = {
url = "github:svanderburg/node2nix";
flake = false;
};
# required for utils.satisfiesSemver
poetry2nix = { url = "github:nix-community/poetry2nix/1.21.0"; flake = false; };
poetry2nix = {
url = "github:nix-community/poetry2nix/1.21.0";
flake = false;
};
# required for builder rust/crane
crane = { url = "github:ipetkov/crane"; flake = false; };
crane = {
url = "github:ipetkov/crane";
flake = false;
};
};
outputs = {
@ -43,9 +57,7 @@
pre-commit-hooks,
crane,
...
}@inp:
let
} @ inp: let
b = builtins;
l = lib // builtins;
@ -56,7 +68,8 @@
supportedSystems = ["x86_64-linux" "x86_64-darwin" "aarch64-darwin"];
forSystems = systems: f: lib.genAttrs systems
forSystems = systems: f:
lib.genAttrs systems
(system: f system nixpkgs.legacyPackages.${system});
forAllSystems = forSystems supportedSystems;
@ -103,7 +116,8 @@
# create a directory containing the files listed in externalPaths
makeExternalDir = import ./src/utils/external-dir.nix;
externalDirFor = forAllSystems (system: pkgs: makeExternalDir {
externalDirFor = forAllSystems (system: pkgs:
makeExternalDir {
inherit externalPaths externalSources pkgs;
});
@ -120,7 +134,8 @@
overridesDirs = ["${./overrides}"];
# system specific dream2nix api
dream2nixFor = forAllSystems (system: pkgs: import ./src rec {
dream2nixFor = forAllSystems (system: pkgs:
import ./src rec {
externalDir = externalDirFor."${system}";
inherit dlib externalPaths externalSources lib pkgs;
config = {
@ -128,7 +143,8 @@
};
});
pre-commit-check = forAllSystems (system: pkgs:
pre-commit-check = forAllSystems (
system: pkgs:
pre-commit-hooks.lib.${system}.run {
src = ./.;
hooks = {
@ -145,13 +161,12 @@
};
}
);
in
{
in {
# System independent dream2nix api.
# Similar to drem2nixFor but will require 'system(s)' or 'pkgs' as an argument.
# Produces flake-like output schema.
lib = (import ./src/lib.nix {
lib =
(import ./src/lib.nix {
inherit dlib externalPaths externalSources overridesDirs lib;
nixpkgsSrc = "${nixpkgs}";
})
@ -159,30 +174,35 @@
// (forAllSystems (system: pkgs: dream2nixFor."${system}"));
# with project discovery enabled
lib2 = (import ./src/libV2.nix {
lib2 = import ./src/libV2.nix {
inherit dlib externalPaths externalSources overridesDirs lib;
nixpkgsSrc = "${nixpkgs}";
});
};
# the dream2nix cli to be used with 'nix run dream2nix'
defaultApp =
forAllSystems (system: pkgs: self.apps."${system}".dream2nix);
# all apps including cli, install, etc.
apps = forAllSystems (system: pkgs:
dream2nixFor."${system}".apps.flakeApps // {
apps = forAllSystems (
system: pkgs:
dream2nixFor."${system}".apps.flakeApps
// {
tests-impure.type = "app";
tests-impure.program = b.toString
tests-impure.program =
b.toString
(dream2nixFor."${system}".callPackageDream ./tests/impure {});
tests-unit.type = "app";
tests-unit.program = b.toString
tests-unit.program =
b.toString
(dream2nixFor."${system}".callPackageDream ./tests/unit {
inherit self;
});
tests-all.type = "app";
tests-all.program = l.toString
tests-all.program =
l.toString
(dream2nixFor.${system}.utils.writePureShellScript
[
alejandra.defaultPackage.${system}
@ -204,7 +224,8 @@
# passes through extra flags to treefmt
format.type = "app";
format.program = l.toString
format.program =
l.toString
(pkgs.writeScript "format" ''
export PATH="${alejandra.defaultPackage.${system}}/bin"
${pkgs.treefmt}/bin/treefmt "$@"
@ -214,8 +235,8 @@
# a dev shell for working on dream2nix
# use via 'nix develop . -c $SHELL'
devShell = forAllSystems (system: pkgs: pkgs.mkShell {
devShell = forAllSystems (system: pkgs:
pkgs.mkShell {
buildInputs =
(with pkgs; [
nix
@ -251,9 +272,9 @@
'';
});
checks = l.recursiveUpdate
(forAllSystems (system: pkgs:
(import ./tests/pure {
checks =
l.recursiveUpdate
(forAllSystems (system: pkgs: (import ./tests/pure {
inherit lib pkgs;
dream2nix = dream2nixFor."${system}";
})))

123
overrides/nodejs/default.nix
View File

@ -1,13 +1,10 @@
{
lib,
pkgs,
# dream2nix
satisfiesSemver,
...
}:
let
}: let
l = lib // builtins;
# include this into an override to enable cntr debugging
@ -29,12 +26,9 @@ let
exit 1
fi
'';
in
## OVERRIDES
{
atom = {
build = {
buildScript = ''
@ -61,16 +55,16 @@ in
cpu-features = {
add-inputs = {
nativeBuildInputs = old: old ++ [
nativeBuildInputs = old:
old
++ [
pkgs.cmake
];
};
};
css-loader = {
disable-source-map-v4-v5 = {
_condition = pkg:
satisfiesSemver "^4.0.0" pkg
|| satisfiesSemver "^5.0.0" pkg;
@ -84,17 +78,13 @@ in
};
cypress = {
add-binary = {
dontBuild = true;
};
};
"draw.io" = {
build = {
buildScript = ''
mkdir $out/bin
makeWrapper \
@ -107,9 +97,7 @@ in
};
dugite = {
add-git = {
buildScript = ''
ln -s ${pkgs.git} ./git
'';
@ -117,9 +105,7 @@ in
};
edex-ui = {
build = {
electronAppDir = "src";
preBuild = {outputs, ...}: ''
@ -138,9 +124,7 @@ in
};
};
electron =
let
electron = let
mkElectron =
pkgs.callPackage
./electron/generic.nix
@ -154,29 +138,29 @@ in
}))
hashes;
getElectronFor = version:
let
getElectronFor = version: let
semVerSpec = "~${version}";
filteredElectrons =
lib.filterAttrs
(electronVer: _: satisfiesSemver semVerSpec {
(electronVer: _:
satisfiesSemver semVerSpec {
version = electronVer;
})
nixpkgsElectrons;
electrons = l.attrValues filteredElectrons;
in
if l.length electrons == 0 then
if l.length electrons == 0
then
throw ''
Electron binary hashes are missing for required version ${version}
Please add the hashes in the override below the origin of this error.
To get the hashes, execute:
${./.}/electron/print-hashes.sh ${version}
''
else if l.length electrons > 1 then
let
else if l.length electrons > 1
then let
versionsSorted =
l.sort
(v1: v2: l.compareVersions v1 v2 == 1)
@ -189,8 +173,7 @@ in
Please delete the hashes for versions ${l.toString versionsToRemove}
in the override below the origin of this error.
''
else
l.head electrons;
else l.head electrons;
# TODO: generate more of these via the script in nixpkgs,
# once we feel confident about this approach
@ -302,13 +285,8 @@ in
headers = "1xnbzskvf8p5a07bha41qqnw1hb68f019qrda3z2jn96m3qnj46r";
};
};
in
{
in {
add-binary = {
overrideAttrs = old: {
postPatch = ''
cp -r ${getElectronFor "${old.version}"}/lib/electron ./dist
@ -328,7 +306,6 @@ in
# TODO: fix electron-builder call or find alternative
element-desktop = {
build = {
# TODO: build rust extensions to enable searching encrypted messages
# TODO: create lower case symlinks for all i18n strings
buildScript = {outputs, ...}: ''
@ -354,7 +331,6 @@ in
};
element-web = {
build = {
installMethod = "copy";
@ -378,8 +354,7 @@ in
esbuild = {
"add-binary-0.12.17" = {
_condition = pkg: pkg.version == "0.12.17";
ESBUILD_BINARY_PATH =
let
ESBUILD_BINARY_PATH = let
esbuild = pkgs.buildGoModule rec {
pname = "esbuild";
version = "0.12.17";
@ -393,14 +368,15 @@ in
vendorSha256 = "sha256-2ABWPqhK2Cf4ipQH7XvRrd+ZscJhYPc3SV2cGT0apdg=";
};
in
"${esbuild}/bin/esbuild";
in "${esbuild}/bin/esbuild";
};
};
fontmanager-redux = {
add-inputs = {
nativeBuildInputs = old: old ++ [
nativeBuildInputs = old:
old
++ [
pkgs.fontconfig
];
};
@ -413,9 +389,7 @@ in
};
enhanced-resolve = {
fix-resolution-v4 = {
_condition = satisfiesSemver "^4.0.0";
# respect node path
@ -430,7 +404,6 @@ in
};
fix-resolution-v5 = {
_condition = satisfiesSemver "^5.0.0";
patches = [
@ -452,7 +425,9 @@ in
keytar = {
add-pkg-config = {
nativeBuildInputs = old: old ++ [
nativeBuildInputs = old:
old
++ [
pkgs.libsecret
pkgs.pkg-config
];
@ -460,9 +435,7 @@ in
};
ledger-live-desktop = {
build = {
installMethod = "copy";
postPatch = ''
@ -474,9 +447,7 @@ in
};
mattermost-desktop = {
build = {
postPatch = ''
substituteInPlace webpack.config.base.js --replace \
"git rev-parse --short HEAD" \
@ -541,10 +512,10 @@ in
};
node-hid = {
build = {
nativeBuildInputs = old: old ++ [
nativeBuildInputs = old:
old
++ [
pkgs.pkg-config
pkgs.libusb
];
@ -605,10 +576,10 @@ in
};
sodium-native = {
build = {
nativeBuildInputs = old: old ++ [
nativeBuildInputs = old:
old
++ [
pkgs.autoconf
pkgs.automake
pkgs.libtool
@ -619,10 +590,11 @@ in
tabby = {
inherit cntr;
fix-build = {
electronAppDir = "./app";
nativeBuildInputs = old: old ++ [
nativeBuildInputs = old:
old
++ [
pkgs.fontconfig
pkgs.libsecret
pkgs.pkg-config
@ -650,14 +622,12 @@ in
otherModules=${pkgs.writeText "other-modules.json" (l.toJSON
(l.mapAttrs
(pname: subOutputs:
let
(pname: subOutputs: let
pkg = subOutputs.packages."${pname}".overrideAttrs (old: {
buildScript = "true";
installMethod = "copy";
});
in
"${pkg}/lib/node_modules/${pname}/node_modules")
in "${pkg}/lib/node_modules/${pname}/node_modules")
outputs.subPackages))}
symlinksToCopies() {
@ -707,10 +677,10 @@ in
};
usb-detection = {
build = {
nativeBuildInputs = old: old ++ [
nativeBuildInputs = old:
old
++ [
pkgs.libudev
];
};
@ -741,14 +711,16 @@ in
# TODO: Maybe should replace binaries with the ones from nixpkgs
"7zip-bin" = {
patch-binaries = {
nativeBuildInputs = old: old ++ [
nativeBuildInputs = old:
old
++ [
pkgs.autoPatchelfHook
];
buildInputs = old: old ++ [
buildInputs = old:
old
++ [
pkgs.gcc-unwrapped.lib
];
};
@ -763,9 +735,7 @@ in
};
"@ledgerhq/ledger-core" = {
build =
let
build = let
ledger-core-version = "4.2.0";
ledger-core = pkgs.stdenv.mkDerivation {
@ -787,9 +757,7 @@ in
url = "https://github.com/chfast/secp256k1/archive/ac8ccf29b8c6b2b793bc734661ce43d1f952977a.tar.gz";
hash = "sha256-7i61CGd+xFvPQkyN7CI7eEoTtko0S77eY+DXEbd3BE8=";
};
in
{
in {
buildInputs = [
ledger-core
];
@ -809,9 +777,7 @@ in
};
"@mattermost/webapp" = {
run-webpack = {
# custom webpack config
postPatch = ''
substituteInPlace webpack.config.js --replace \
@ -868,5 +834,4 @@ in
'';
};
};
}

19
src/apps/cli/default.nix
View File

@ -6,22 +6,16 @@
nix,
translators,
utils,
# from nixpkgs
gitMinimal,
lib,
python3,
...
}:
let
}: let
b = builtins;
cliPython = python3.withPackages (ps: [ps.networkx ps.cleo ps.jsonschema]);
in
{
in {
program =
utils.writePureShellScript
[
@ -39,17 +33,14 @@ in
${cliPython}/bin/python ${./.}/cli.py "$@"
'';
templateDefaultNix =
{
templateDefaultNix = {
dream2nixLocationRelative,
dreamLock,
sourcePathRelative,
}:
let
}: let
defaultPackage = dreamLock._generic.defaultPackage;
defaultPackageVersion = dreamLock._generic.packages."${defaultPackage}";
in
''
in ''
{
dream2nix ? import (
let

19
src/apps/cli2/default.nix
View File

@ -6,22 +6,16 @@
nix,
translators,
utils,
# from nixpkgs
gitMinimal,
lib,
python3,
...
}:
let
}: let
b = builtins;
cliPython = python3.withPackages (ps: [ps.networkx ps.cleo ps.jsonschema]);
in
{
in {
program =
utils.writePureShellScript
[
@ -39,17 +33,14 @@ in
${cliPython}/bin/python ${./.}/cli.py "$@"
'';
templateDefaultNix =
{
templateDefaultNix = {
dream2nixLocationRelative,
dreamLock,
sourcePathRelative,
}:
let
}: let
defaultPackage = dreamLock._generic.defaultPackage;
defaultPackageVersion = dreamLock._generic.packages."${defaultPackage}";
in
''
in ''
{
dream2nix ? import (
let

7
src/apps/contribute/default.nix
View File

@ -3,12 +3,9 @@
python3,
writeScript,
...
}:
let
}: let
cliPython = python3.withPackages (ps: [ps.cleo]);
in
{
in {
program = writeScript "contribute" ''
dream2nixSrc=${../../.} \
${cliPython}/bin/python ${./contribute.py} contribute "$@"

17
src/apps/default.nix
View File

@ -1,30 +1,25 @@
{
lib,
pkgs,
# dream2nix
callPackageDream,
translators,
...
}:
let
}: let
b = builtins;
in
rec {
in rec {
apps = {
inherit cli cli2 contribute install;
dream2nix = cli;
};
flakeApps =
lib.mapAttrs (appName: app:
{
flakeApps = lib.mapAttrs (
appName: app: {
type = "app";
program = b.toString app.program;
}
) apps;
)
apps;
# the dream2nix cli
cli = callPackageDream (import ./cli) {};

8
src/apps/install/default.nix
View File

@ -1,14 +1,12 @@
{
runCommand,
writeScript,
# dream2nix inputs
dream2nixWithExternals,
...
}:
{
program = writeScript "install"
}: {
program =
writeScript "install"
''
target="$1"
if [[ "$target" == "" ]]; then

5
src/builders/default.nix
View File

@ -2,17 +2,14 @@
builders,
callPackageDream,
...
}:
{
}: {
python = rec {
default = simpleBuilder;
simpleBuilder = callPackageDream ./python/simple-builder {};
};
nodejs = rec {
default = granular;
node2nix = callPackageDream ./nodejs/node2nix {};

10
src/builders/go/gomod2nix/default.nix
View File

@ -3,14 +3,12 @@
pkgs,
externals,
...
}:
{
}: {
fetchedSources,
dreamLock,
}:
let
gomod2nixTOML = fetchedSources.mapAttrs
}: let
gomod2nixTOML =
fetchedSources.mapAttrs
dependencyObject.goName;
in
externals.gomod2nixBuilder rec {

52
src/builders/nodejs/granular/default.nix
View File

@ -7,50 +7,38 @@
runCommand,
stdenv,
writeText,
# dream2nix inputs
builders,
externals,
utils,
...
}:
{
}: {
# Funcs
# AttrSet -> Bool) -> AttrSet -> [x]
getCyclicDependencies, # name: version: -> [ {name=; version=; } ]
getDependencies, # name: version: -> [ {name=; version=; } ]
getSource, # name: version: -> store-path
buildPackageWithOtherBuilder, # { builder, name, version }: -> drv
# Attributes
subsystemAttrs, # attrset
defaultPackageName, # string
defaultPackageVersion, # string
packages, # list
# attrset of pname -> versions,
# where versions is a list of version strings
packageVersions,
# function which applies overrides to a package
# It must be applied by the builder to each individual derivation
# Example:
# produceDerivation name (mkDerivation {...})
produceDerivation,
# Custom Options: (parametrize builder behavior)
# These can be passed by the user via `builderArgs`.
# All options must provide default
standalonePackageNames ? [],
nodejs ? null,
...
}@args:
let
} @ args: let
b = builtins;
nodejsVersion = subsystemAttrs.nodejsVersion;
@ -59,8 +47,8 @@ let
(args.packages."${name}" or null) == version;
nodejs =
if args ? nodejs then
args.nodejs
if args ? nodejs
then args.nodejs
else
pkgs."nodejs-${builtins.toString nodejsVersion}_x"
or (throw "Could not find nodejs version '${nodejsVersion}' in pkgs");
@ -159,9 +147,7 @@ let
'';
# Generates a derivation for a specific package name + version
makePackage = name: version:
let
makePackage = name: version: let
deps = getDependencies name version;
nodeDeps =
@ -169,15 +155,16 @@ let
deps
(dep: allPackages."${dep.name}"."${dep.version}");
dependenciesJson = b.toJSON
dependenciesJson =
b.toJSON
(lib.listToAttrs
(b.map
(dep: lib.nameValuePair dep.name dep.version)
deps));
electronDep =
if ! isMainPackage name version then
null
if ! isMainPackage name version
then null
else
lib.findFirst
(dep: dep.name == "electron")
@ -188,15 +175,11 @@ let
lib.versions.major electronDep.version;
electronHeaders =
if electronDep == null then
null
else
pkgs."electron_${electronVersionMajor}".headers;
pkg =
produceDerivation name (stdenv.mkDerivation rec {
if electronDep == null
then null
else pkgs."electron_${electronVersionMajor}".headers;
pkg = produceDerivation name (stdenv.mkDerivation rec {
inherit
dependenciesJson
electronHeaders
@ -269,10 +252,9 @@ let
# example which requires this:
# https://registry.npmjs.org/react-window-infinite-loader/-/react-window-infinite-loader-1.0.7.tgz
unpackCmd =
if lib.hasSuffix ".tgz" src then
"tar --delay-directory-restore -xf $src"
else
null;
if lib.hasSuffix ".tgz" src
then "tar --delay-directory-restore -xf $src"
else null;
unpackPhase = ''
runHook preUnpack
@ -454,7 +436,5 @@ let
});
in
pkg;
in
outputs

29
src/builders/nodejs/node2nix/default.nix
View File

@ -1,17 +1,13 @@
# builder imported from node2nix
{
lib,
pkgs,
# dream2nix inputs
externals,
node2nix ? externals.node2nix,
utils,
...
}:
{
}: {
subsystemAttrs,
defaultPackageName,
defaultPackageVersion,
@ -19,12 +15,10 @@
getDependencies,
getSource,
packageVersions,
# overrides
packageOverrides ? {},
...
}@args:
let
} @ args: let
b = builtins;
getAllDependencies = name: version:
@ -43,17 +37,15 @@ let
node2nixEnv = node2nix nodejs;
makeSource = packageName: version: prevDeps:
let
makeSource = packageName: version: prevDeps: let
depsFiltered =
(lib.filter
lib.filter
(dep:
! b.elem dep prevDeps)
(getAllDependencies packageName version));
(getAllDependencies packageName version);
parentDeps =
prevDeps ++ depsFiltered;
in
rec {
in rec {
inherit packageName version;
name = utils.sanitizeDerivationName packageName;
src = getSource packageName version;
@ -81,18 +73,13 @@ let
src = getSource defaultPackageName defaultPackageVersion;
}
// args);
in
rec {
in rec {
packages."${defaultPackageName}"."${defaultPackageVersion}" = defaultPackage;
defaultPackage =
let
defaultPackage = let
pkg = callNode2Nix "buildNodePackage" {};
in
utils.applyOverridesToPackage packageOverrides pkg defaultPackageName;
devShell = callNode2Nix "buildNodeShell" {};
}

30
src/builders/python/simple-builder/default.nix
View File

@ -1,35 +1,28 @@
# A very simple single derivation python builder
{
lib,
pkgs,
...
}:
{
}: {
fetchedSources,
dreamLock,
}:
let
}: let
python = pkgs."${dreamLock._subsystem.pythonAttr}";
buildFunc =
if dreamLock._subsystem.application then
python.pkgs.buildPythonApplication
else
python.pkgs.buildPythonPackage;
if dreamLock._subsystem.application
then python.pkgs.buildPythonApplication
else python.pkgs.buildPythonPackage;
defaultPackage = dreamLock._generic.defaultPackage;
packageName =
if defaultPackage == null then
if dreamLock._subsystem.application then
"application"
else
"environment"
else
defaultPackage;
if defaultPackage == null
then
if dreamLock._subsystem.application
then "application"
else "environment"
else defaultPackage;
defaultPackage = buildFunc {
name = packageName;
@ -58,7 +51,6 @@ let
runHook postInstall
'';
};
in {
inherit defaultPackage;
}

19
src/builders/rust/build-rust-package/default.nix
View File

@ -1,11 +1,8 @@
{
lib,
pkgs,
...
}:
{
}: {
subsystemAttrs,
defaultPackageName,
defaultPackageVersion,
@ -15,18 +12,14 @@
getSourceSpec,
packages,
produceDerivation,
...
}@args:
let
} @ args: let
l = lib // builtins;
utils = import ../utils.nix args;
vendoring = import ../vendor.nix (args // {inherit lib pkgs utils;});
buildPackage = pname: version:
let
buildPackage = pname: version: let
src = utils.getRootSource pname version;
vendorDir = vendoring.vendorDependencies pname version;
@ -51,12 +44,10 @@ let
${vendoring.writeGitVendorEntries "vendored-sources"}
'';
});
in
rec {
in rec {
packages =
l.mapAttrs
(name: version:
{ "${version}" = buildPackage name version; })
(name: version: {"${version}" = buildPackage name version;})
args.packages;
defaultPackage = packages."${defaultPackageName}"."${defaultPackageVersion}";

23
src/builders/rust/crane/default.nix
View File

@ -1,12 +1,9 @@
{
lib,
pkgs,
externals,
...
}:
{
}: {
subsystemAttrs,
defaultPackageName,
defaultPackageVersion,
@ -16,11 +13,8 @@
getSourceSpec,
packages,
produceDerivation,
...
}@args:
let
} @ args: let
l = lib // builtins;
utils = import ../utils.nix args;
@ -28,8 +22,7 @@ let
crane = externals.crane;
buildPackage = pname: version:
let
buildPackage = pname: version: let
src = utils.getRootSource pname version;
cargoVendorDir = vendoring.vendorDependencies pname version;
postUnpack = ''
@ -46,7 +39,9 @@ let
depsArgs = common // {pnameSuffix = depsNameSuffix;};
deps = produceDerivation "${pname}${depsNameSuffix}" (crane.buildDepsOnly depsArgs);
buildArgs = common // {
buildArgs =
common
// {
cargoArtifacts = deps;
# Make sure cargo only builds & tests the package we want
cargoBuildCommand = "cargo build --release --package ${pname}";
@ -54,12 +49,10 @@ let
};
in
produceDerivation pname (crane.buildPackage buildArgs);
in
rec {
in rec {
packages =
l.mapAttrs
(name: version:
{ "${version}" = buildPackage name version; })
(name: version: {"${version}" = buildPackage name version;})
args.packages;
defaultPackage = packages."${defaultPackageName}"."${defaultPackageVersion}";

9
src/builders/rust/utils.nix
View File

@ -2,11 +2,10 @@
getSourceSpec,
getSource,
getRoot,
...
}:
rec {
getRootSource = pname: version:
let root = getRoot pname version; in
}: rec {
getRootSource = pname: version: let
root = getRoot pname version;
in
getSource root.pname root.version;
}

28
src/builders/rust/vendor.nix
View File

@ -1,17 +1,14 @@
{
lib,
pkgs,
getRoot,
getSource,
getSourceSpec,
getDependencies,
getCyclicDependencies,
subsystemAttrs,
...
}:
let
}: let
l = lib // builtins;
isCyclic = cyclic: dep:
@ -44,10 +41,8 @@ let
in rec {
# Generates a shell script that writes git vendor entries to .cargo/config.
# `replaceWith` is the name of the vendored source(s) to use.
writeGitVendorEntries = replaceWith:
let
makeEntry = source:
''
writeGitVendorEntries = replaceWith: let
makeEntry = source: ''
[source."${source.url}${l.optionalString (source ? type) "?${source.type}=${source.value}"}"]
replace-with = "${replaceWith}"
git = "${source.url}"
@ -63,12 +58,10 @@ in rec {
# Vendor a package's dependencies like how `cargo vendor` would do,
# so we can use it with `cargo`.
vendorPackageDependencies = pname: version:
let
vendorPackageDependencies = pname: version: let
deps = getAllTransitiveDependencies pname version;
makeSource = dep:
let
makeSource = dep: let
path = getSource dep.name dep.version;
spec = getSourceSpec dep.name dep.version;
in {
@ -77,8 +70,7 @@ in rec {
};
sources = l.map makeSource deps;
findCrateSource = source:
let
findCrateSource = source: let
inherit (pkgs) cargo jq;
pkg = source.dep;
in ''
@ -103,8 +95,7 @@ in rec {
echo Found crate ${pkg.name} at $crateCargoTOML
tree="$(dirname $crateCargoTOML)"
'';
makeScript = source:
let
makeScript = source: let
isGit = source.spec.type == "git";
isPath = source.spec.type == "path";
in
@ -127,7 +118,8 @@ in rec {
'';
# Vendors a package's roots dependencies.
vendorDependencies = pname: version:
let root = getRoot pname version; in
vendorDependencies = pname: version: let
root = getRoot pname version;
in
vendorPackageDependencies root.pname root.version;
}

360
src/default.nix
View File

@ -2,49 +2,39 @@
# It requires passing one specific pkgs.
# If the intention is to generate output for several systems,
# use ./lib.nix instead.
{
pkgs ? import <nixpkgs> {},
dlib ? import ./lib {inherit config lib;},
lib ? pkgs.lib,
nix ? pkgs.nix,
# default to empty dream2nix config
config ?
# if called via CLI, load config via env
if builtins ? getEnv && builtins.getEnv "dream2nixConfig" != "" then
builtins.toPath (builtins.getEnv "dream2nixConfig")
if builtins ? getEnv && builtins.getEnv "dream2nixConfig" != ""
then builtins.toPath (builtins.getEnv "dream2nixConfig")
# load from default directory
else
{},
else {},
# dependencies of dream2nix
externalSources ?
lib.genAttrs
(lib.attrNames (builtins.readDir externalDir))
(inputName: "${/. + externalDir}/${inputName}"),
# will be defined if called via flake
externalPaths ? null,
# required for non-flake mode
externalDir ?
# if flake is used, construct external dir from flake inputs
if externalPaths != null then
if externalPaths != null
then
(import ./utils/external-dir.nix {
inherit externalPaths externalSources pkgs;
})
# if called via CLI, load externals via env
else if builtins ? getEnv && builtins.getEnv "d2nExternalDir" != "" then
/. + (builtins.getEnv "d2nExternalDir")
else if builtins ? getEnv && builtins.getEnv "d2nExternalDir" != ""
then /. + (builtins.getEnv "d2nExternalDir")
# load from default directory
else
./external,
}@args:
let
else ./external,
} @ args: let
b = builtins;
l = lib // builtins;
@ -54,7 +44,9 @@ let
configFile = pkgs.writeText "dream2nix-config.json" (b.toJSON config);
# like pkgs.callPackage, but includes all the dream2nix modules
callPackageDream = f: args: pkgs.callPackage f (args // {
callPackageDream = f: args:
pkgs.callPackage f (args
// {
inherit apps;
inherit builders;
inherit callPackageDream;
@ -70,7 +62,6 @@ let
inherit nix;
});
utils = callPackageDream ./utils {};
# apps for CLI and installation
@ -93,8 +84,7 @@ let
pkgs.callPackage "${externalSources.node2nix}/nix/node-env.nix" {
inherit nodejs;
};
crane =
let
crane = let
importLibFile = name: import "${externalSources.crane}/lib/${name}.nix";
makeHook = attrs: name:
@ -103,8 +93,7 @@ let
"${externalSources.crane}/pkgs/${name}.sh";
genHooks = names: attrs: lib.genAttrs names (makeHook attrs);
otherHooks =
genHooks [
otherHooks = genHooks [
"configureCargoCommonVarsHook"
"configureCargoVendoredDepsHook"
"remapSourcePathPrefixHook"
@ -146,16 +135,24 @@ let
mkCargoDerivation = importLibFile "mkCargoDerivation" ({
inherit (pkgs) cargo stdenv lib;
} // installHooks // otherHooks);
}
// installHooks
// otherHooks);
buildDepsOnly = importLibFile "buildDepsOnly" {
inherit
mkCargoDerivation crateNameFromCargoToml
vendorCargoDeps mkDummySrc;
mkCargoDerivation
crateNameFromCargoToml
vendorCargoDeps
mkDummySrc
;
};
cargoBuild = importLibFile "cargoBuild" {
inherit
mkCargoDerivation buildDepsOnly
crateNameFromCargoToml vendorCargoDeps;
mkCargoDerivation
buildDepsOnly
crateNameFromCargoToml
vendorCargoDeps
;
};
buildPackage = importLibFile "buildPackage" {
inherit (pkgs) lib;
@ -165,22 +162,19 @@ let
};
};
dreamOverrides =
let
dreamOverrides = let
overridesDirs =
config.overridesDirs
++
(lib.optionals (b ? getEnv && b.getEnv "d2nOverridesDir" != "") [
++ (lib.optionals (b ? getEnv && b.getEnv "d2nOverridesDir" != "") [
(b.getEnv "d2nOverridesDir")
]);
in
utils.loadOverridesDirs overridesDirs pkgs;
# the location of the dream2nix framework for self references (update scripts, etc.)
dream2nixWithExternals =
if b.pathExists (./. + "/external") then
./.
if b.pathExists (./. + "/external")
then ./.
else
pkgs.runCommand "dream2nix-full-src" {} ''
cp -r ${./.} $out
@ -191,40 +185,33 @@ let
'';
# automatically find a suitable builder for a given dream lock
findBuilder = dreamLock:
let
findBuilder = dreamLock: let
subsystem = dreamLock._generic.subsystem;
in
if ! builders ? "${subsystem}" then
throw "Could not find any builder for subsystem '${subsystem}'"
else
builders."${subsystem}".default;
if ! builders ? "${subsystem}"
then throw "Could not find any builder for subsystem '${subsystem}'"
else builders."${subsystem}".default;
# detect if granular or combined fetching must be used
findFetcher = dreamLock:
if null != dreamLock._generic.sourcesAggregatedHash then
fetchers.combinedFetcher
else
fetchers.defaultFetcher;
if null != dreamLock._generic.sourcesAggregatedHash
then fetchers.combinedFetcher
else fetchers.defaultFetcher;
# fetch only sources and do not build
fetchSources =
{
fetchSources = {
dreamLock,
fetcher ? null,
extract ? false,
sourceOverrides ? oldSources: {},
}@args:
let
} @ args: let
# if dream lock is a file, read and parse it
dreamLock' = (utils.readDreamLock {inherit dreamLock;}).lock;
fetcher =
if args.fetcher or null == null then
findFetcher dreamLock'
else
args.fetcher;
if args.fetcher or null == null
then findFetcher dreamLock'
else args.fetcher;
fetched = fetcher rec {
inherit sourceOverrides;
@ -235,48 +222,41 @@ let
};
fetchedSources = fetched.fetchedSources;
in
fetched // {
fetched
// {
fetchedSources =
if extract then
if extract
then
lib.mapAttrs
(key: source: utils.extractSource {inherit source;})
fetchedSources
else
fetchedSources;
else fetchedSources;
};
makeDreamLockForSource =
{
makeDreamLockForSource = {
source,
translator ? null,
translatorArgs ? {},
}@args:
let
} @ args: let
sourceSpec =
if b.isString args.source && ! lib.isStorePath args.source then
fetchers.translateShortcut { shortcut = args.source; }
else
{
if b.isString args.source && ! lib.isStorePath args.source
then fetchers.translateShortcut {shortcut = args.source;}
else {
type = "path";
path = args.source;
};
source = fetchers.fetchSource {source = sourceSpec;};
t =
let
t = let
translator = translators.findOneTranslator {
inherit source;
translatorName = args.translator or null;
};
in
if b.elem translator.type [ "pure" "ifd" ] then
translator
if b.elem translator.type ["pure" "ifd"]
then translator
else
throw ''
All comaptible translators are impure and therefore require
@ -285,13 +265,14 @@ let
nix run .# -- add ...
'';
dreamLock' = translators.translators."${t.subsystem}"."${t.type}"."${t.name}".translate
(translatorArgs // {
dreamLock' =
translators.translators."${t.subsystem}"."${t.type}"."${t.name}".translate
(translatorArgs
// {
inherit source;
});
dreamLock =
let
dreamLock = let
defaultPackage = dreamLock'._generic.defaultPackage;
defaultPackageVersion = dreamLock'._generic.packages."${defaultPackage}";
in
@ -301,14 +282,11 @@ let
path = "${source}";
};
};
in
dreamLock;
# build a dream lock via a specific builder
callBuilder =
{
callBuilder = {
builder,
builderArgs,
fetchedSources,
@ -317,9 +295,7 @@ let
sourceOverrides,
packageOverrides,
allOutputs,
}@args:
let
} @ args: let
# inject dependencies
dreamLock = utils.dreamLock.injectDependencies args.dreamLock inject;
@ -333,20 +309,16 @@ let
conditionalOverrides = packageOverrides;
};
buildPackageWithOtherBuilder =
{
buildPackageWithOtherBuilder = {
builder,
name,
version,
inject ? {},
}:
let
subDreamLockLoaded =
utils.readDreamLock {
}: let
subDreamLockLoaded = utils.readDreamLock {
dreamLock =
utils.dreamLock.getSubDreamLock dreamLock name version;
};
in
callBuilder {
inherit
@ -364,14 +336,15 @@ let
outputs = allOutputs;
};
outputs = builder ( builderArgs // {
outputs = builder (builderArgs
// {
inherit
buildPackageWithOtherBuilder
produceDerivation
;
inherit (dreamLockInterface)
inherit
(dreamLockInterface)
subsystemAttrs
getSourceSpec
getRoot
@ -384,44 +357,40 @@ let
;
getSource = utils.dreamLock.getSource fetchedSources;
});
# Makes the packages tree compatible with flakes schema.
# For each package the attr `{pname}` will link to the latest release.
# Other package versions will be inside: `{pname}.versions`
formattedOutputs = outputs // {
packages =
let
formattedOutputs =
outputs
// {
packages = let
allPackages = outputs.packages or {};
latestPackages =
lib.mapAttrs'
(pname: releases:
let
(pname: releases: let
latest =
releases."${utils.latestVersion (b.attrNames releases)}";
in
(lib.nameValuePair
in (lib.nameValuePair
"${pname}"
(latest // {
(latest
// {
versions = releases;
})))
allPackages;
in
latestPackages;
};
in
formattedOutputs;
riseAndShine = throw ''
Use makeOutputs instead of riseAndShine.
'';
makeOutputsForDreamLock =
{
makeOutputsForDreamLock = {
dreamLock,
builder ? null,
fetcher ? null,
@ -431,32 +400,30 @@ let
builderArgs ? {},
translator ? null,
translatorArgs ? {},
}@args:
let
} @ args: let
# parse dreamLock
dreamLockLoaded = utils.readDreamLock {inherit (args) dreamLock;};
dreamLock = dreamLockLoaded.lock;
dreamLockInterface = dreamLockLoaded.interface;
builder' =
if builder == null then
findBuilder dreamLock
else
builder;
if builder == null
then findBuilder dreamLock
else builder;
fetcher' =
if fetcher == null then
findFetcher dreamLock
else
fetcher;
if fetcher == null
then findFetcher dreamLock
else fetcher;
fetchedSources = (fetchSources {
fetchedSources =
(fetchSources {
inherit dreamLock sourceOverrides;
fetcher = fetcher';
}).fetchedSources;
})
.fetchedSources;
builderOutputs = callBuilder {
inherit
dreamLock
fetchedSources
@ -478,23 +445,18 @@ let
};
allOutputs = builderOutputs;
in
allOutputs;
translateProjects =
{
discoveredProjects ? dlib.discoverers.discoverProjects
translateProjects = {
discoveredProjects ?
dlib.discoverers.discoverProjects
{inherit settings tree;},
source ?
throw "Pass either `source` or `tree` to translateProjects",
source ? throw "Pass either `source` or `tree` to translateProjects",
tree ? dlib.prepareSourceTree {inherit source;},
pname,
settings ? [],
}@args:
let
} @ args: let
# This influences error messages only
flakeMode = ! builtins ? currentSystem;
@ -512,33 +474,39 @@ let
translator = project.translator;
};
isResolved = project:
let
isResolved = project: let
dreamLockExists =
l.pathExists "${config.projectRoot}/${project.dreamLockPath}";
dreamLockValid =
project.dreamLock._generic.invalidationHash or ""
project.dreamLock._generic.invalidationHash
or ""
== project.invalidationHash;
in
dreamLockExists && dreamLockValid;
getProjectKey = project:
"${project.name}_|_${project.subsystem}_|_${project.relPath}";
getProjectKey = project: "${project.name}_|_${project.subsystem}_|_${project.relPath}";
# list of projects extended with some information requried for processing
projectsList =
l.map
(project: (let self = project // rec {
dreamLock = (utils.readDreamLock {
(project: (let
self =
project
// rec {
dreamLock =
(utils.readDreamLock {
dreamLock = "${config.projectRoot}/${project.dreamLockPath}";
}).lock;
})
.lock;
impure = isImpure project translator;
invalidationHash = getInvalidationHash project;
key = getProjectKey project;
resolved = isResolved self;
translator = project.translator or (l.head project.translators);
}; in self))
};
in
self))
discoveredProjects;
# attrset of projects by key
@ -573,28 +541,33 @@ let
# list of pure projects extended with 'dreamLock' attribute
projectsResolvedOnTheFly =
l.forEach projectsPureUnresolved
(proj:
let
(proj: let
translator = getTranslator proj.subsystem proj.translator;
dreamLock' = translator.translate {
inherit source tree;
project = proj;
};
dreamLock = dreamLock' // {
_generic = dreamLock'._generic // {
dreamLock =
dreamLock'
// {
_generic =
dreamLock'._generic
// {
invalidationHash = proj.invalidationHash;
};
};
in
proj // {
proj
// {
inherit dreamLock;
});
resolvedProjects = projectsResolved ++ projectsResolvedOnTheFly;
in
if projectsImpureUnresolved != [] then
if flakeMode then
if projectsImpureUnresolved != []
then
if flakeMode
then
l.trace ''
${"\n"}
Run `nix run .#resolveImpure` once to resolve impure projects.
@ -609,57 +582,50 @@ let
${l.concatStringsSep "\n " projectsImpureUnresolvedInfo}
''
resolvedProjects
else if projectsPureUnresolved != [] then
resolvedProjects
else
resolvedProjects;
else if projectsPureUnresolved != []
then resolvedProjects
else resolvedProjects;
# transform a list of resolved projects to buildable outputs
realizeProjects =
{
realizeProjects = {
translatedProjects ? translateProjects {inherit pname settings source;},
# alternative way of calling (for debugging)
pname ? null,
source ? null,
packageOverrides ? {},
settings ? [],
}:
let
}: let
dreamLocks = l.forEach translatedProjects (proj: proj.dreamLock);
defaultSourceOverride = dreamLock:
if source == null then
{}
else
let
if source == null
then {}
else let
defaultPackage = dreamLock._generic.defaultPackage;
defaultPackageVersion =
dreamLock._generic.packages."${defaultPackage}";
in
{
"${defaultPackage}"."${defaultPackageVersion}" =
"${source}/${dreamLock._generic.location}";
in {
"${defaultPackage}"."${defaultPackageVersion}" = "${source}/${dreamLock._generic.location}";
};
# extends each package with a `.resolve` attribute
outputsForProject = proj:
let
outputsForProject = proj: let
outputs = makeOutputsForDreamLock rec {
inherit packageOverrides;
dreamLock = proj.dreamLock;
sourceOverrides = oldSources:
(defaultSourceOverride proj.dreamLock);
sourceOverrides = oldSources: (defaultSourceOverride proj.dreamLock);
};
in
outputs
// {
packages =
l.mapAttrs
(pname: pkg: pkg.overrideAttrs (old: {
passthru = old.passthru or {} // {
(pname: pkg:
pkg.overrideAttrs (old: {
passthru =
old.passthru
or {}
// {
resolve = utils.makeTranslateScript {
inherit source;
invalidationHash = proj.invalidationHash;
@ -677,20 +643,18 @@ let
mergedOutputs =
l.foldl'
(all: outputs: all // {
(all: outputs:
all
// {
packages = all.packages or {} // outputs.packages;
})
{}
projectOutputs;
in
mergedOutputs;
# produce outputs for a dream-lock or a source
makeOutputs =
{
makeOutputs = {
source, # source tree or dream-lock
builder ? null,
fetcher ? null,
@ -700,18 +664,15 @@ let
builderArgs ? {},
translator ? null,
translatorArgs ? {},
}@args:
let
} @ args: let
dreamLock' =
# in case of a dream-lock.json file or dream-lock attributes
if ( lib.isAttrs args.source && args.source ? _generic && args.source ? _subsytem )
|| lib.hasSuffix "dream-lock.json" source then
args.source
if
(lib.isAttrs args.source && args.source ? _generic && args.source ? _subsytem)
|| lib.hasSuffix "dream-lock.json" source
then args.source
# input is a source tree -> generate the dream-lock
else
makeDreamLockForSource { inherit source translator translatorArgs; };
else makeDreamLockForSource {inherit source translator translatorArgs;};
# parse dreamLock
dreamLockLoaded = utils.readDreamLock {dreamLock = dreamLock';};
@ -726,8 +687,10 @@ let
(args // {source = dreamLock.lock;}))
dreamLockInterface.subDreamLocks;
builderOutputs = makeOutputsForDreamLock
((b.removeAttrs args ["source"]) // {
builderOutputs =
makeOutputsForDreamLock
((b.removeAttrs args ["source"])
// {
inherit dreamLock;
});
@ -736,18 +699,16 @@ let
//
# merge with sub package outputs
b.foldl'
(old: new: old // {
(old: new:
old
// {
packages = new.packages or {} // old.packages;
})
builderOutputs
(b.attrValues builderOutputsSub);
in
allOutputs;
in
{
in {
inherit
apps
builders
@ -764,7 +725,8 @@ in
utils
;
inherit (dlib)
inherit
(dlib)
discoverers
;
}

42
src/discoverers/default.nix
View File

@ -2,9 +2,7 @@
config,
dlib,
lib,
}:
let
}: let
l = lib // builtins;
subsystems = dlib.dirNames ./.;
@ -14,8 +12,7 @@ let
(v: v ? discover)
discoverers;
discoverProjects =
{
discoverProjects = {
source ? throw "Pass either `source` or `tree` to discoverProjects",
tree ? dlib.prepareSourceTree {inherit source;},
settings ? [],
@ -28,8 +25,10 @@ let
rootProjectName = l.head discoveredProjects;
projectsExtended = l.forEach discoveredProjects
(proj: proj
projectsExtended =
l.forEach discoveredProjects
(proj:
proj
// {
translator = l.head proj.translators;
dreamLockPath = getDreamLockPath proj rootProjectName;
@ -37,23 +36,21 @@ let
in
applyProjectSettings projectsExtended settings;
getDreamLockPath = project: rootProject:
let
getDreamLockPath = project: rootProject: let
root =
if config.projectRoot == null then
"."
else
config.projectRoot;
if config.projectRoot == null
then "."
else config.projectRoot;
in
dlib.sanitizeRelativePath
"${config.packagesDir}/${rootProject.name}/${project.relPath}/dream-lock.json";
applyProjectSettings = projects: settingsList:
let
applyProjectSettings = projects: settingsList: let
settingsListForProject = project:
l.filter
(settings:
if ! settings ? filter then true
if ! settings ? filter
then true
else settings.filter project)
settingsList;
@ -69,16 +66,13 @@ let
settingsApplied =
l.forEach projects
(proj: applyAllSettings proj);
in settingsApplied;
discoverers = l.genAttrs subsystems (subsystem:
(import (./. + "/${subsystem}") { inherit dlib lib subsystem; })
);
in
settingsApplied;
{
discoverers = l.genAttrs subsystems (
subsystem: (import (./. + "/${subsystem}") {inherit dlib lib subsystem;})
);
in {
inherit
applyProjectSettings
discoverProjects

92
src/discoverers/nodejs/default.nix
View File

@ -1,18 +1,11 @@
{
dlib,
lib,
subsystem,
}:
let
}: let
l = lib // builtins;
discover =
{
tree,
}:
let
discover = {tree}: let
projects = discoverInternal {
inherit tree;
};
@ -22,8 +15,7 @@ let
# One translator call can process a whole workspace containing all
# sub-packages of that workspace.
# Therefore we can filter out projects which are children of a workspace.
filterProjects = projects:
let
filterProjects = projects: let
workspaceRoots =
l.filter
(proj: proj.subsystemInfo.workspaces or [] != [])
@ -37,15 +29,12 @@ let
childrenRemoved =
l.filter
(proj:
(! l.elem proj.relPath allWorkspaceChildren))
(proj: (! l.elem proj.relPath allWorkspaceChildren))
projects;
in
childrenRemoved;
getTranslatorNames = path:
let
getTranslatorNames = path: let
nodes = l.readDir path;
packageJson = l.fromJSON (l.readFile "${path}/package.json");
translators =
@ -53,7 +42,6 @@ let
# package-lock translator with `packageLock = null`
if ! packageJson ? dependencies && ! packageJson ? devDependencies
then ["package-lock"]
else
l.optionals (nodes ? "package-lock.json") ["package-lock"]
++ l.optionals (nodes ? "yarn.lock") ["yarn-lock"]
@ -67,8 +55,8 @@ let
# returns all relative paths to workspaces defined by a glob
getWorkspacePaths = glob: tree:
if l.hasSuffix "*" glob then
let
if l.hasSuffix "*" glob
then let
prefix = l.removeSuffix "*" glob;
path = "${tree.fullPath}/${prefix}";
@ -85,8 +73,7 @@ let
(wsPath:
if l.pathExists "${path}/${wsPath}/package.json"
then true
else
let
else let
notExistingPath =
dlib.sanitizeRelativePath "${prefix}/${wsPath}";
in
@ -96,9 +83,7 @@ let
dirNames;
in
l.map (dname: "${prefix}/${dname}") existingWsPaths
else
if l.pathExists "${tree.fullPath}/${glob}/package.json"
else if l.pathExists "${tree.fullPath}/${glob}/package.json"
then [glob]
else
l.trace
@ -106,8 +91,7 @@ let
[];
# collect project info for workspaces defined by current package.json
getWorkspaces = tree: parentInfo:
let
getWorkspaces = tree: parentInfo: let
packageJson = tree.files."package.json".jsonContent;
workspacesRaw = packageJson.workspaces or [];
@ -118,24 +102,19 @@ let
(l.mapAttrsToList
(category: workspaces: workspaces)
workspacesRaw)
else if l.isList workspacesRaw
then workspacesRaw
else throw "Error parsing workspaces in ${tree.files."package.json".relPath}";
in
l.flatten
(l.forEach workspacesFlattened
(glob:
let
(glob: let
workspacePaths = getWorkspacePaths glob tree;
in
l.forEach workspacePaths
(wPath: makeWorkspaceProjectInfo tree wPath parentInfo)));
makeWorkspaceProjectInfo = tree: wsRelPath: parentInfo:
{
makeWorkspaceProjectInfo = tree: wsRelPath: parentInfo: {
inherit subsystem;
name =
(getPackageJson "${tree.fullPath}/${wsRelPath}").name
@ -152,41 +131,38 @@ let
};
};
discoverInternal =
{
discoverInternal = {
tree,
# Internal parameter preventing workspace projects from being discovered
# twice.
alreadyDiscovered ? {},
}:
let
}: let
foundSubProjects = alreadyDiscovered:
l.flatten
((l.mapAttrsToList
(dname: dir: discoverInternal {
(l.mapAttrsToList
(dname: dir:
discoverInternal {
inherit alreadyDiscovered;
tree = dir;
})
(tree.directories or {})));
(tree.directories or {}));
in
# skip if not a nodajs project
if alreadyDiscovered ? "${tree.relPath}"
|| ! tree ? files."package.json" then
if
alreadyDiscovered
? "${tree.relPath}"
|| ! tree ? files."package.json"
then
# this will be cleaned by `flatten` for sub-directories
foundSubProjects alreadyDiscovered
else
let
else let
# project info of current directory
currentProjectInfo =
{
currentProjectInfo = {
inherit subsystem;
inherit (tree) relPath;
name = tree.files."package.json".jsonContent.name or tree.relPath;
translators = getTranslatorNames tree.fullPath;
subsystemInfo =
l.optionalAttrs (workspaces != []) {
subsystemInfo = l.optionalAttrs (workspaces != []) {
workspaces =
l.map
(w: l.removePrefix tree.relPath w.relPath)
@ -196,37 +172,29 @@ let
workspaces = getWorkspaces tree currentProjectInfo;
# list of all projects infos found by the current iteration
foundProjects =
# current directories project info
[currentProjectInfo]
# workspaces defined by the current directory
++
workspaces;
++ workspaces;
# index of already found projects
# This is needed, because sub-projects also contain a `package.json`,
# and would otherwise be discovered again as an independent project.
alreadyDiscovered' =
alreadyDiscovered
//
(l.genAttrs
// (l.genAttrs
(l.map (p: p.relPath) foundProjects)
(relPath: null));
in
# l.trace tree.directories
# the current directory
foundProjects
# sub-directories
# Thanks to `alreadyDiscovered`, workspace projects won't be discovered
# a second time.
++
(foundSubProjects alreadyDiscovered');
in
{
++ (foundSubProjects alreadyDiscovered');
in {
inherit discover;
}

93
src/fetchers/combined-fetcher.nix
View File

@ -9,60 +9,53 @@
nix,
stdenv,
writeScript,
# dream2nix
defaultFetcher,
utils,
...
}:
{
}: {
# sources attrset from dream lock
sources,
sourcesAggregatedHash,
...
}@args:
let
} @ args: let
b = builtins;
# resolve to individual fetcher calls
defaultFetched = (defaultFetcher args).fetchedSources;
# extract the arguments from the individual fetcher calls
FODArgsAll =
let
FODArgsAll = let
FODArgsAll' =
lib.mapAttrs
(name: versions:
(
name: versions:
lib.mapAttrs
(version: fetched:
# handle FOD sources
if lib.all
if
lib.all
(attr: fetched ? "${attr}")
[ "outputHash" "outputHashAlgo" "outputHashMode" ] then
["outputHash" "outputHashAlgo" "outputHashMode"]
then
(fetched.overrideAttrs (args: {
passthru.originalArgs = args;
})).originalArgs // {
outPath =
let
}))
.originalArgs
// {
outPath = let
sanitizedName = utils.sanitizeDerivationName name;
in
"${sanitizedName}/${version}/${fetched.name}";
in "${sanitizedName}/${version}/${fetched.name}";
}
# handle path sources
else if lib.isString fetched then
"ignore"
else if lib.isString fetched
then "ignore"
# handle store path sources
else if lib.isStorePath fetched then
"ignore"
else if lib.isStorePath fetched
then "ignore"
# handle unknown sources
else if fetched == "unknown" then
"ignore"
else if fetched == "unknown"
then "ignore"
# error out on unknown source types
else
throw ''
@ -91,17 +84,16 @@ let
# convert arbitrary types to string, like nix does with derivation arguments
toString' = x:
if lib.isBool x then
if x then
"1"
else
""
else if lib.isList x then
''"${lib.concatStringsSep " " (lib.forEach x (y: toString' y))}"''
else if x == null then
""
else
b.toJSON x;
if lib.isBool x
then
if x
then "1"
else ""
else if lib.isList x
then ''"${lib.concatStringsSep " " (lib.forEach x (y: toString' y))}"''
else if x == null
then ""
else b.toJSON x;
# set up nix build env for signle item
setupEnvForItem = fetcherArgs: ''
@ -109,8 +101,10 @@ let
# export arguments for builder
${lib.concatStringsSep "\n" (lib.mapAttrsToList (argName: argVal: ''
export ${argName}=${
lib.replaceStrings [ "$" ''\n'' ] [ ''\$'' "\n" ] (toString' argVal)}
'') fetcherArgs)}
lib.replaceStrings ["$" ''\n''] [''\$'' "\n"] (toString' argVal)
}
'')
fetcherArgs)}
# run builder
bash ${fetcherArgs.builder}
@ -159,8 +153,7 @@ let
echo "FOD_HASH=$(${nix}/bin/nix hash path $out)"
'';
FODAllSources =
let
FODAllSources = let
nativeBuildInputs' =
lib.unique
(lib.foldl (a: b: a ++ b) []
@ -171,17 +164,16 @@ let
stdenv.mkDerivation rec {
name = "sources-combined";
inherit builder;
nativeBuildInputs = nativeBuildInputs' ++ [
nativeBuildInputs =
nativeBuildInputs'
++ [
coreutils
];
outputHashAlgo = "sha256";
outputHashMode = "recursive";
outputHash = sourcesAggregatedHash;
};
in
{
in {
FOD = FODAllSources;
fetchedSources =
@ -189,10 +181,9 @@ in
(name: versions:
lib.mapAttrs
(version: source:
if FODArgsAll ? "${name}"."${version}" then
"${FODAllSources}/${FODArgsAll."${name}"."${version}".outPath}"
else
defaultFetched."${name}"."${version}")
if FODArgsAll ? "${name}"."${version}"
then "${FODAllSources}/${FODArgsAll."${name}"."${version}".outPath}"
else defaultFetched."${name}"."${version}")
versions)
sources;
}

21
src/fetchers/crates-io/default.nix
View File

@ -2,29 +2,28 @@
lib,
fetchurl,
runCommand,
utils,
...
}:
{
}: {
inputs = ["pname" "version"];
versionField = "version";
outputs = { pname, version, ... }@inp:
let
outputs = {
pname,
version,
...
} @ inp: let
b = builtins;
# See https://github.com/rust-lang/crates.io-index/blob/master/config.json#L2
url = "https://crates.io/api/v1/crates/${pname}/${version}/download";
in
{
calcHash = algo: utils.hashFile algo (b.fetchurl {
in {
calcHash = algo:
utils.hashFile algo (b.fetchurl {
inherit url;
});
fetched = hash:
let
fetched = hash: let
fetched = fetchurl {
inherit url;
sha256 = hash;

35
src/fetchers/default-fetcher.nix
View File

@ -1,41 +1,38 @@
{
lib,
# dream2nix attributes
fetchSource,
fetchers,
...
}:
{
}: {
# sources attrset from dream lock
defaultPackage,
defaultPackageVersion,
sourceOverrides,
sources,
...
}:
let
}: let
b = builtins;
fetchedSources =
lib.mapAttrs
(name: versions:
lib.mapAttrs
(version: source:
if source.type == "unknown" then
"unknown"
else if source.type == "path" then
if lib.isStorePath source.path then
source.path
if source.type == "unknown"
then "unknown"
else if source.type == "path"
then
if lib.isStorePath source.path
then source.path
# assume path relative to main package source
else
"${overriddenSources."${defaultPackage}"."${defaultPackageVersion}"}/${source.path}"
else if fetchers.fetchers ? "${source.type}" then
else "${overriddenSources."${defaultPackage}"."${defaultPackageVersion}"}/${source.path}"
else if fetchers.fetchers ? "${source.type}"
then
fetchSource {
source = source // {
source =
source
// {
pname = name;
inherit version;
};
@ -48,9 +45,7 @@ let
lib.recursiveUpdate
fetchedSources
(sourceOverrides fetchedSources);
in
{
in {
# attrset: pname -> path of downloaded source
fetchedSources = overriddenSources;
}

149
src/fetchers/default.nix
View File

@ -1,20 +1,15 @@
{
lib,
# dream2nix
callPackageDream,
utils,
...
}:
let
}: let
b = builtins;
callFetcher = file: args: callPackageDream file args;
in
rec {
fetchers = lib.genAttrs (utils.dirNames ./.) (name:
in rec {
fetchers = lib.genAttrs (utils.dirNames ./.) (
name:
callFetcher (./. + "/${name}") {}
);
@ -22,13 +17,11 @@ rec {
combinedFetcher = callPackageDream ./combined-fetcher.nix {inherit defaultFetcher;};
constructSource =
{
constructSource = {
type,
reComputeHash ? false,
...
}@args:
let
} @ args: let
fetcher = fetchers."${type}";
argsKeep = b.removeAttrs args ["reComputeHash"];
fetcherOutputs =
@ -42,40 +35,44 @@ rec {
});
# update source spec to different version
updateSource =
{
updateSource = {
source,
newVersion,
...
}:
constructSource (source // {
constructSource (source
// {
reComputeHash = true;
} // {
}
// {
"${fetchers."${source.type}".versionField}" = newVersion;
});
# fetch a source defined via a dream lock source spec
fetchSource = { source, extract ? false }:
let
fetchSource = {
source,
extract ? false,
}: let
fetcher = fetchers."${source.type}";
fetcherArgs = b.removeAttrs source ["dir" "hash" "type"];
fetcherOutputs = fetcher.outputs fetcherArgs;
maybeArchive = fetcherOutputs.fetched (source.hash or null);
in
if source ? dir then
"${maybeArchive}/${source.dir}"
else
maybeArchive;
if source ? dir
then "${maybeArchive}/${source.dir}"
else maybeArchive;
# fetch a source defined by a shortcut
fetchShortcut = { shortcut, extract ? false, }:
fetchShortcut = {
shortcut,
extract ? false,
}:
fetchSource {
source = translateShortcut {inherit shortcut;};
inherit extract;
};
parseShortcut = shortcut:
let
parseShortcut = shortcut: let
# in: "git+https://foo.com/bar?kwarg1=lol&kwarg2=hello"
# out: [ "git+" "git" "https" "//" "foo.com/bar" "?kwarg1=lol&kwarg2=hello" "kwarg1=lol&kwarg2=hello" ]
split =
@ -93,63 +90,54 @@ rec {
};
kwargs_ =
if parsed.allArgs == null then
{}
if parsed.allArgs == null
then {}
else
lib.listToAttrs
(map
(kwarg:
let
(kwarg: let
split = lib.splitString "=" kwarg;
in
lib.nameValuePair
(b.elemAt split 0)
(b.elemAt split 1))
(lib.splitString "&" parsed.allArgs));
in
if split == null then
throw "Unable to parse shortcut: ${shortcut}"
else
parsed;
if split == null
then throw "Unable to parse shortcut: ${shortcut}"
else parsed;
renderUrlArgs = kwargs:
let
renderUrlArgs = kwargs: let
asStr =
(lib.concatStringsSep
lib.concatStringsSep
"&"
(lib.mapAttrsToList
(name: val: "${name}=${val}")
kwargs));
kwargs);
in
if asStr == "" then
""
else
"?" + asStr;
if asStr == ""
then ""
else "?" + asStr;
# translate shortcut to dream lock source spec
translateShortcut = { shortcut, computeHash ? true, }:
let
translateShortcut = {
shortcut,
computeHash ? true,
}: let
parsed = parseShortcut shortcut;
checkArgs = fetcherName: args:
let
checkArgs = fetcherName: args: let
fetcher = fetchers."${fetcherName}";
unknownArgNames = lib.filter (argName: ! lib.elem argName fetcher.inputs) (lib.attrNames args);
missingArgNames = lib.filter (inputName: ! args ? "${inputName}") fetcher.inputs;
in
if lib.length unknownArgNames > 0 then
throw "Received unknown arguments for fetcher '${fetcherName}': ${b.toString unknownArgNames}"
else if lib.length missingArgNames > 0 then
throw "Missing arguments for fetcher '${fetcherName}': ${b.toString missingArgNames}"
else
args;
if lib.length unknownArgNames > 0
then throw "Received unknown arguments for fetcher '${fetcherName}': ${b.toString unknownArgNames}"
else if lib.length missingArgNames > 0
then throw "Missing arguments for fetcher '${fetcherName}': ${b.toString missingArgNames}"
else args;
translateHttpUrl =
let
translateHttpUrl = let
fetcher = fetchers.http;
urlArgsFinal = renderUrlArgs parsed.kwargs;
@ -159,7 +147,6 @@ rec {
fetcherOutputs = fetchers.http.outputs {
inherit url;
};
in
constructSource
{
@ -173,9 +160,7 @@ rec {
hash = fetcherOutputs.calcHash "sha256";
});
translateProtoShortcut =
let
translateProtoShortcut = let
kwargsUrl = b.removeAttrs parsed.kwargs fetcher.inputs;
urlArgs = renderUrlArgs kwargsUrl;
@ -189,10 +174,10 @@ rec {
args = parsed.kwargs // {inherit url;};
fetcherOutputs = fetcher.outputs (checkArgs fetcherName args);
in
constructSource
(parsed.kwargs // {
(parsed.kwargs
// {
type = fetcherName;
inherit url;
}
@ -203,8 +188,7 @@ rec {
hash = fetcherOutputs.calcHash "sha256";
}));
translateRegularShortcut =
let
translateRegularShortcut = let
fetcherName = parsed.proto2;
path = lib.removeSuffix "/" parsed.path;
@ -214,29 +198,30 @@ rec {
fetcher = fetchers."${fetcherName}";
args =
if fetcher ? parseParams then
fetcher.parseParams params
else if b.length params != b.length fetcher.inputs then
if fetcher ? parseParams
then fetcher.parseParams params
else if b.length params != b.length fetcher.inputs
then
throw ''
Wrong number of arguments provided in shortcut for fetcher '${fetcherName}'
Should be ${fetcherName}:${lib.concatStringsSep "/" fetcher.inputs}
''
else
lib.listToAttrs
(lib.forEach
(lib.range 0 ((lib.length fetcher.inputs) - 1))
(idx:
(
idx:
lib.nameValuePair
(lib.elemAt fetcher.inputs idx)
(lib.elemAt params idx)
));
fetcherOutputs = fetcher.outputs (args // parsed.kwargs);
in
constructSource (args // parsed.kwargs // {
constructSource (args
// parsed.kwargs
// {
type = fetcherName;
}
// (lib.optionalAttrs (parsed.dir != null) {
@ -245,14 +230,12 @@ rec {
// (lib.optionalAttrs computeHash {
hash = fetcherOutputs.calcHash "sha256";
}));
in
if parsed.proto1 != null then
translateProtoShortcut
else if lib.hasPrefix "http://" shortcut
|| lib.hasPrefix "https://" shortcut then
translateHttpUrl
else
translateRegularShortcut;
if parsed.proto1 != null
then translateProtoShortcut
else if
lib.hasPrefix "http://" shortcut
|| lib.hasPrefix "https://" shortcut
then translateHttpUrl
else translateRegularShortcut;
}

53
src/fetchers/git/default.nix
View File

@ -1,15 +1,11 @@
{
fetchgit,
lib,
utils,
...
}:
let
}: let
b = builtins;
in
{
in {
inputs = [
"url"
"rev"
@ -17,26 +13,26 @@ in
versionField = "rev";
outputs = { url, rev, ... }@inp:
if b.match "refs/(heads|tags)/.*" rev == null && builtins.match "[a-f0-9]*" rev == null then
throw ''rev must either be a sha1 revision or "refs/heads/branch-name" or "refs/tags/tag-name"''
else
let
outputs = {
url,
rev,
...
} @ inp:
if b.match "refs/(heads|tags)/.*" rev == null && builtins.match "[a-f0-9]*" rev == null
then throw ''rev must either be a sha1 revision or "refs/heads/branch-name" or "refs/tags/tag-name"''
else let
b = builtins;
refAndRev =
if b.match "refs/(heads|tags)/.*" inp.rev != null then
{ ref = inp.rev; }
else
{ rev = inp.rev; };
in
{
calcHash = algo: utils.hashPath algo
if b.match "refs/(heads|tags)/.*" inp.rev != null
then {ref = inp.rev;}
else {rev = inp.rev;};
in {
calcHash = algo:
utils.hashPath algo
(b.fetchGit
(refAndRev // {
(refAndRev
// {
inherit url;
allRefs = true;
submodules = true;
@ -45,19 +41,22 @@ in
# git can either be verified via revision or hash.
# In case revision is used for verification, `hash` will be null.
fetched = hash:
if hash == null then
if ! refAndRev ? rev then
throw "Cannot fetch git repo without integrity. Specify at least 'rev' or 'sha256'"
if hash == null
then
if ! refAndRev ? rev
then throw "Cannot fetch git repo without integrity. Specify at least 'rev' or 'sha256'"
else
b.fetchGit
(refAndRev // {
(refAndRev
// {
inherit url;
allRefs = true;
submodules = true;
})
else
fetchgit
(refAndRev // {
(refAndRev
// {
inherit url;
fetchSubmodules = true;
sha256 = hash;

21
src/fetchers/github/default.nix
View File

@ -3,12 +3,9 @@
lib,
nix,
runCommand,
utils,
...
}:
{
}: {
inputs = [
"owner"
"repo"
@ -19,13 +16,16 @@
defaultUpdater = "githubNewestReleaseTag";
outputs = { owner, repo, rev, ... }@inp:
let
outputs = {
owner,
repo,
rev,
...
} @ inp: let
b = builtins;
in
{
calcHash = algo: utils.hashPath algo (b.fetchTarball {
in {
calcHash = algo:
utils.hashPath algo (b.fetchTarball {
url = "https://github.com/${owner}/${repo}/tarball/${rev}";
});
@ -33,6 +33,5 @@
fetchFromGitHub {
inherit owner repo rev hash;
};
};
}

21
src/fetchers/gitlab/default.nix
View File

@ -1,11 +1,8 @@
{
fetchFromGitLab,
utils,
...
}:
{
}: {
inputs = [
"owner"
"repo"
@ -14,13 +11,16 @@
versionField = "rev";
outputs = { owner, repo, rev, ... }@inp:
let
outputs = {
owner,
repo,
rev,
...
} @ inp: let
b = builtins;
in
{
calcHash = algo: utils.hashPath algo (b.fetchTarball {
in {
calcHash = algo:
utils.hashPath algo (b.fetchTarball {
url = "https://gitlab.com/${owner}/${repo}/-/archive/${rev}/${repo}-${rev}.tar.gz";
});
@ -28,6 +28,5 @@
fetchFromGitLab {
inherit owner repo rev hash;
};
};
}

29
src/fetchers/http/default.nix
View File

@ -1,30 +1,25 @@
{
lib,
fetchurl,
utils,
...
}:
{
}: {
inputs = [
"url"
];
outputs = { url, ... }@inp:
let
outputs = {url, ...} @ inp: let
b = builtins;
in
{
calcHash = algo: utils.hashFile algo (b.fetchurl {
in {
calcHash = algo:
utils.hashFile algo (b.fetchurl {
inherit url;
});
fetched = hash:
let
fetched = hash: let
drv =
if hash != null && lib.stringLength hash == 40 then
if hash != null && lib.stringLength hash == 40
then
fetchurl {
inherit url;
sha1 = hash;
@ -34,18 +29,14 @@
inherit url hash;
};
drvSanitized =
drv.overrideAttrs (old: {
drvSanitized = drv.overrideAttrs (old: {
name = lib.strings.sanitizeDerivationName old.name;
});
extracted =
utils.extractSource {
extracted = utils.extractSource {
source = drvSanitized;
};
in
extracted;
};
}

39
src/fetchers/npm/default.nix
View File

@ -2,16 +2,11 @@
fetchurl,
lib,
python3,
utils,
...
}:
let
}: let
b = builtins;
in
rec {
in rec {
inputs = ["pname" "version"];
versionField = "version";
@ -21,16 +16,19 @@ rec {
# becuase some node packages contain submodules like `@hhhtj/draw.io`
# the amount of arguments can vary and a custom parser is needed
parseParams = params:
if b.length params == b.length inputs then
if b.length params == b.length inputs
then
lib.listToAttrs
(lib.forEach
(lib.range 0 ((lib.length inputs) - 1))
(idx:
(
idx:
lib.nameValuePair
(lib.elemAt inputs idx)
(lib.elemAt params idx)
))
else if b.length params == (b.length inputs) + 1 then
else if b.length params == (b.length inputs) + 1
then
parseParams [
"${b.elemAt params 0}/${b.elemAt params 1}"
(b.elemAt params 2)
@ -41,35 +39,34 @@ rec {
Should be npm:${lib.concatStringsSep "/" inputs}
'';
# defaultUpdater = "";
outputs = { pname, version, }@inp:
let
outputs = {
pname,
version,
} @ inp: let
b = builtins;
submodule = lib.last (lib.splitString "/" pname);
url = "https://registry.npmjs.org/${pname}/-/${submodule}-${version}.tgz";
in
{
calcHash = algo: utils.hashPath algo (
in {
calcHash = algo:
utils.hashPath algo (
b.fetchurl {inherit url;}
);
fetched = hash:
let
fetched = hash: let
source =
(fetchurl {
inherit url;
sha256 = hash;
}).overrideAttrs (old: {
})
.overrideAttrs (old: {
outputHashMode = "recursive";
});
in
utils.extractSource {
inherit source;
};
};
}

15
src/fetchers/path/default.nix
View File

@ -1,22 +1,13 @@
{
utils,
...
}:
{
{utils, ...}: {
inputs = [
"path"
];
outputs = { path, ... }@inp:
let
outputs = {path, ...} @ inp: let
b = builtins;
in
{
in {
calcHash = algo: utils.hashPath "${path}";
fetched = hash: "${path}";
};
}

25
src/fetchers/pypi-sdist/default.nix
View File

@ -1,40 +1,39 @@
{
fetchurl,
python3,
utils,
...
}:
{
}: {
inputs = ["pname" "version"];
versionField = "version";
defaultUpdater = "pypiNewestReleaseVersion";
outputs = { pname, version, extension ? "tar.gz", }@inp:
let
outputs = {
pname,
version,
extension ? "tar.gz",
} @ inp: let
b = builtins;
firstChar = builtins.substring 0 1 pname;
url =
"https://files.pythonhosted.org/packages/source/"
+ "${firstChar}/${pname}/${pname}-${version}.${extension}";
in
{
calcHash = algo: utils.hashPath algo (
in {
calcHash = algo:
utils.hashPath algo (
b.fetchurl {inherit url;}
);
fetched = hash:
let
fetched = hash: let
source =
(fetchurl {
inherit url;
sha256 = hash;
}).overrideAttrs (old: {
})
.overrideAttrs (old: {
outputHashMode = "recursive";
});
in

125
src/lib.nix
View File

@ -1,7 +1,6 @@
# like ./default.nix but system intependent
# (allows to generate outputs for several systems)
# follows flake output schema
{
dlib,
nixpkgsSrc,
@ -9,11 +8,7 @@
overridesDirs,
externalSources,
externalPaths,
}@args:
let
} @ args: let
b = builtins;
l = lib // builtins;
@ -22,65 +17,57 @@ let
import ./default.nix
{inherit config externalPaths externalSources pkgs;};
# TODO: design output schema for cross compiled packages
makePkgsKey = pkgs:
let
makePkgsKey = pkgs: let
build = pkgs.buildPlatform.system;
host = pkgs.hostPlatform.system;
in
if build == host then build
if build == host
then build
else throw "cross compiling currently not supported";
makeNixpkgs = pkgsList: systems:
# fail if neither pkgs nor systems are defined
if pkgsList == null && systems == [] then
throw "Either `systems` or `pkgs` must be defined"
if pkgsList == null && systems == []
then throw "Either `systems` or `pkgs` must be defined"
# fail if pkgs and systems are both defined
else if pkgsList != null && systems != [] then
throw "Define either `systems` or `pkgs`, not both"
else if pkgsList != null && systems != []
then throw "Define either `systems` or `pkgs`, not both"
# only pkgs is specified
else if pkgsList != null then
if b.isList pkgsList then
else if pkgsList != null
then
if b.isList pkgsList
then
lib.listToAttrs
(pkgs: lib.nameValuePair (makePkgsKey pkgs) pkgs)
pkgsList
else
{ "${makePkgsKey pkgsList}" = pkgsList; }
else {"${makePkgsKey pkgsList}" = pkgsList;}
# only systems is specified
else
lib.genAttrs systems
(system: import nixpkgsSrc {inherit system;});
flakifyBuilderOutputs = system: outputs:
(lib.optionalAttrs (outputs ? "defaultPackage") {
defaultPackage."${system}" = outputs.defaultPackage;
})
//
(lib.optionalAttrs (outputs ? "packages") {
// (lib.optionalAttrs (outputs ? "packages") {
packages."${system}" = outputs.packages;
})
//
(lib.optionalAttrs (outputs ? "devShell") {
// (lib.optionalAttrs (outputs ? "devShell") {
devShell."${system}" = outputs.devShell;
});
init =
{
init = {
pkgs ? null,
systems ? [],
config ? {},
}@argsInit:
let
} @ argsInit: let
config' = (import ./utils/config.nix).loadConfig argsInit.config or {};
config = config' // {
config =
config'
// {
overridesDirs = args.overridesDirs ++ config'.overridesDirs;
};
@ -89,12 +76,11 @@ let
forAllSystems = f: lib.mapAttrs f allPkgs;
dream2nixFor = forAllSystems (dream2nixForSystem config);
in
{
in {
riseAndShine = throw "Use makeFlakeOutputs instead of riseAndShine.";
makeFlakeOutputs = mArgs: makeFlakeOutputsFunc
makeFlakeOutputs = mArgs:
makeFlakeOutputsFunc
(
{inherit config pkgs systems;}
// mArgs
@ -116,8 +102,7 @@ let
dream2nixFor."${system}".builders);
};
makeFlakeOutputsFunc =
{
makeFlakeOutputsFunc = {
builder ? null,
pname ? null,
pkgs ? null,
@ -126,9 +111,7 @@ let
translator ? null,
translatorArgs ? {},
...
}@args:
let
} @ args: let
config = args.config or ((import ./utils/config.nix).loadConfig {});
argsForward = b.removeAttrs args ["config" "pname" "pkgs" "systems"];
@ -144,10 +127,16 @@ let
translatorName = args.translator or null;
};
translatorFoundFor = forAllSystems (system: pkgs:
translatorFoundFor = forAllSystems (
system: pkgs:
with translatorFound;
dream2nixFor."${system}".translators.translators
."${subsystem}"."${type}"."${name}"
dream2nixFor
."${system}"
.translators
.translators
."${subsystem}"
."${type}"
."${name}"
);
invalidationHash = dlib.calcInvalidationHash {
@ -163,21 +152,18 @@ let
detectedName = translatorFound.projectName;
pname =
if args.pname or null != null then
args.pname
else if detectedName != null then
detectedName
else
specifyPnameError;
if args.pname or null != null
then args.pname
else if detectedName != null
then detectedName
else specifyPnameError;
allBuilderOutputs =
lib.mapAttrs
(system: pkgs:
let
(system: pkgs: let
dream2nix = dream2nixFor."${system}";
dreamLockJsonPath = with config;
"${projectRoot}/${packagesDir}/${pname}/dream-lock.json";
dreamLockJsonPath = with config; "${projectRoot}/${packagesDir}/${pname}/dream-lock.json";
dreamLock = dream2nix.utils.readDreamLock {
dreamLock = dreamLockJsonPath;
@ -188,13 +174,14 @@ let
&& dreamLock.lock._generic.invalidationHash or "" == invalidationHash;
result = translator: args:
dream2nix.makeOutputs (argsForward // {
dream2nix.makeOutputs (argsForward
// {
# TODO: this triggers the translator finding routine a second time
translator = translatorFound.name;
});
in
if dreamLockExistsAndValid then
if dreamLockExistsAndValid
then
# we need to override the source here as it is coming from
# a flake input
let
@ -209,8 +196,8 @@ let
args.source;
};
}
else if b.elem translatorFound.type [ "pure" "ifd" ] then
else if b.elem translatorFound.type ["pure" "ifd"]
then
# warn the user about potentially slow on-the-fly evaluation
b.trace ''
${"\n"}
@ -219,8 +206,8 @@ let
To speed up future evalutations run once:
nix run .#resolve
''
result translatorFound {}
result
translatorFound {}
else
# print error because impure translation is required first.
# continue the evaluation anyways, as otherwise we won't have
@ -232,7 +219,6 @@ let
Please execute nix run .#resolve to resolve all impurities.
''
{})
allPkgs;
flakifiedOutputsList =
@ -245,21 +231,18 @@ let
(allOutputs: output: lib.recursiveUpdate allOutputs output)
{}
flakifiedOutputsList;
in
lib.recursiveUpdate
flakeOutputs
{
apps = forAllSystems (system: pkgs: {
resolve.type = "app";
resolve.program =
let
utils = (dream2nixFor."${system}".utils);
resolve.program = let
utils = dream2nixFor."${system}".utils;
# TODO: Too many calls to findOneTranslator.
# -> make findOneTranslator system independent
translatorFound =
dream2nixFor."${system}".translators.findOneTranslator {
translatorFound = dream2nixFor."${system}".translators.findOneTranslator {
inherit source;
translatorName = args.translator or null;
};
@ -272,9 +255,7 @@ let
});
});
};
in
{
in {
inherit dlib init;
riseAndShine = throw "Use makeFlakeOutputs instead of riseAndShine.";
makeFlakeOutpus = makeFlakeOutputsFunc;

80
src/lib/default.nix
View File

@ -2,12 +2,9 @@
lib,
config ? (import ../utils/config.nix).loadConfig {},
...
}:
let
}: let
l = lib // builtins;
# exported attributes
dlib = {
inherit
@ -29,7 +26,8 @@ let
traceJ
;
inherit (parseUtils)
inherit
(parseUtils)
identifyGitUrl
parseGitUrl
;
@ -41,7 +39,6 @@ let
parseUtils = import ./parsing.nix {inherit lib;};
# INTERNAL
# Calls any function with an attrset arugment, even if that function
@ -49,21 +46,18 @@ let
# recursively applied as parameters.
# For this to work, the function parameters defined by the called function
# must always be ordered alphabetically.
callWithAttrArgs = func: args:
let
callWithAttrArgs = func: args: let
applyParamsRec = func: params:
if l.length params == 1 then
func (l.head params)
if l.length params == 1
then func (l.head params)
else
applyParamsRec
(func (l.head params))
(l.tail params);
in
if lib.functionArgs func == {} then
applyParamsRec func (l.attrValues args)
else
func args;
if lib.functionArgs func == {}
then applyParamsRec func (l.attrValues args)
else func args;
# prepare source tree for executing discovery phase
# produces this structure:
@ -90,8 +84,7 @@ let
# };
# };
# }
prepareSourceTreeInternal = sourceRoot: relPath: name: depth:
let
prepareSourceTreeInternal = sourceRoot: relPath: name: depth: let
relPath' = relPath;
fullPath' = "${sourceRoot}/${relPath}";
current = l.readDir fullPath';
@ -103,10 +96,9 @@ let
l.filterAttrs (n: v: v == "directory") current;
makeNewPath = prefix: name:
if prefix == "" then
name
else
"${prefix}/${name}";
if prefix == ""
then name
else "${prefix}/${name}";
directories =
l.mapAttrs
@ -130,8 +122,7 @@ let
})
fileNames;
getNodeFromPath = path:
let
getNodeFromPath = path: let
cleanPath = l.removePrefix "/" path;
pathSplit = l.splitString "/" cleanPath;
dirSplit = l.init pathSplit;
@ -147,22 +138,19 @@ let
dirSplit);
dir =
if (l.length dirSplit == 0) || dirAttrPath == [ "" ] then
self
else if ! l.hasAttrByPath dirAttrPath directories then
error
else
l.getAttrFromPath dirAttrPath directories;
if (l.length dirSplit == 0) || dirAttrPath == [""]
then self
else if ! l.hasAttrByPath dirAttrPath directories
then error
else l.getAttrFromPath dirAttrPath directories;
in
if path == "" then
self
else if dir ? directories."${leaf}" then
dir.directories."${leaf}"
else if dir ? files."${leaf}" then
dir.files."${leaf}"
else
error;
if path == ""
then self
else if dir ? directories."${leaf}"
then dir.directories."${leaf}"
else if dir ? files."${leaf}"
then dir.files."${leaf}"
else error;
self =
{
@ -174,19 +162,16 @@ let
// (l.optionalAttrs (depth > 0) {
inherit directories;
});
in
self;
# determines if version v1 is greater than version v2
versionGreater = v1: v2: l.compareVersions v1 v2 == 1;
# EXPORTED
# calculate an invalidation hash for given source translation inputs
calcInvalidationHash =
{
calcInvalidationHash = {
source,
translator,
translatorArgs,
@ -199,8 +184,7 @@ let
'';
# call a function using arguments defined by the env var FUNC_ARGS
callViaEnv = func:
let
callViaEnv = func: let
funcArgs = l.fromJSON (l.readFile (l.getEnv "FUNC_ARGS"));
in
callWithAttrArgs func funcArgs;
@ -225,11 +209,9 @@ let
listFiles = path: l.attrNames (l.filterAttrs (n: v: v == "regular") (builtins.readDir path));
nameVersionPair = name: version:
{ inherit name version; };
nameVersionPair = name: version: {inherit name version;};
prepareSourceTree =
{
prepareSourceTree = {
source,
depth ? 10,
}:
@ -248,7 +230,5 @@ let
l.removePrefix "/" (l.toString (l.toPath "/${path}"));
traceJ = toTrace: eval: l.trace (l.toJSON toTrace) eval;
in
dlib

36
src/lib/parsing.nix
View File

@ -1,50 +1,38 @@
{
lib,
...
}:
let
{lib, ...}: let
b = builtins;
identifyGitUrl = url:
lib.hasPrefix "git+" url
|| b.match ''^github:.*/.*#.*'' url != null;
parseGitUrl = url:
let
parseGitUrl = url: let
githubMatch = b.match ''^github:(.*)/(.*)#(.*)$'' url;
in
if githubMatch != null then
let
if githubMatch != null
then let
owner = b.elemAt githubMatch 0;
repo = b.elemAt githubMatch 1;
rev = b.elemAt githubMatch 2;
in
{
in {
url = "https://github.com/${owner}/${repo}";
inherit rev;
}
else
let
else let
splitUrlRev = lib.splitString "#" url;
rev = lib.last splitUrlRev;
urlOnly = lib.head splitUrlRev;
in
if lib.hasPrefix "git+ssh://" urlOnly then
{
if lib.hasPrefix "git+ssh://" urlOnly
then {
inherit rev;
url = "https://${(lib.last (lib.splitString "@" url))}";
}
else if lib.hasPrefix "git+https://" urlOnly then
{
else if lib.hasPrefix "git+https://" urlOnly
then {
inherit rev;
url = lib.removePrefix "git+" urlOnly;
}
else
throw "Cannot parse git url: ${url}";
in
{
else throw "Cannot parse git url: ${url}";
in {
inherit identifyGitUrl parseGitUrl;
}

118
src/lib/translators.nix
View File

@ -1,8 +1,7 @@
{
dlib,
lib,
}:
let
}: let
l = lib // builtins;
# INTERNAL
@ -15,8 +14,7 @@ let
mkTranslatorsSet = function:
l.genAttrs
(dlib.dirNames ../translators)
(subsystem:
let
(subsystem: let
availableTypes =
l.filter
(type: l.pathExists (../translators + "/${subsystem}/${type}"))
@ -26,9 +24,9 @@ let
l.genAttrs
availableTypes
(transType: function subsystem transType);
in
translatorsForTypes // {
translatorsForTypes
// {
all =
l.foldl'
(a: b: a // b)
@ -37,45 +35,40 @@ let
});
# flat list of all translators sorted by priority (pure translators first)
translatorsList =
let
translatorsList = let
list = l.collect (v: v ? subsystem) translators;
prio = translator:
if translator.type == "pure" then
0
else if translator.type == "ifd" then
1
else if translator.type == "impure" then
2
else
3;
if translator.type == "pure"
then 0
else if translator.type == "ifd"
then 1
else if translator.type == "impure"
then 2
else 3;
in
l.sort
(a: b: (prio a) < (prio b))
list;
callTranslator = subsystem: type: name: file: args:
let
callTranslator = subsystem: type: name: file: args: let
translatorModule = import file {
inherit dlib lib;
};
in
translatorModule // {
translatorModule
// {
inherit name subsystem type;
projectName =
if translatorModule ? projectName then
translatorModule.projectName
else
{ ... }: null;
if translatorModule ? projectName
then translatorModule.projectName
else {...}: null;
};
# EXPORTED
# attrset of: subsystem -> translator-type -> translator
translators = mkTranslatorsSet (subsystem: type:
let
translators = mkTranslatorsSet (
subsystem: type: let
translatorNames =
dlib.dirNames (../translators + "/${subsystem}/${type}");
@ -107,13 +100,11 @@ let
# returns the list of translators including their special args
# and adds a flag `compatible` to each translator indicating
# if the translator is compatible to all given paths
translatorsForInput =
{
source,
}:
translatorsForInput = {source}:
l.forEach translatorsList
(t: rec {
inherit (t)
inherit
(t)
extraArgs
name
subsystem
@ -125,99 +116,84 @@ let
# also includes subdirectories of the given paths up to a certain depth
# to check for translator compatibility
translatorsForInputRecursive =
{
translatorsForInputRecursive = {
source,
depth ? 2,
}:
let
listDirsRec = dir: depth:
let
}: let
listDirsRec = dir: depth: let
subDirs =
l.map
(subdir: "${dir}/${subdir}")
(dlib.listDirs dir);
in
if depth == 0 then
subDirs
if depth == 0
then subDirs
else
subDirs
++
(l.flatten
++ (l.flatten
(map
(subDir: listDirsRec subDir (depth - 1))
subDirs));
dirsToCheck =
[source]
++
(l.flatten
++ (l.flatten
(map
(inputDir: listDirsRec inputDir depth)
[source]));
in
l.genAttrs
dirsToCheck
(dir:
(
dir:
translatorsForInput {
source = dir;
}
);
# pupulates a translators special args with defaults
getextraArgsDefaults = extraArgsDef:
l.mapAttrs
(name: def:
if def.type == "flag" then
false
else
def.default or null
(
name: def:
if def.type == "flag"
then false
else def.default or null
)
extraArgsDef;
# return one compatible translator or throw error
findOneTranslator =
{
findOneTranslator = {
source,
translatorName ? null,
}@args:
let
} @ args: let
translatorsForSource = translatorsForInput {
inherit source;
};
nameFilter =
if translatorName != null then
(translator: translator.name == translatorName)
else
(translator: true);
if translatorName != null
then (translator: translator.name == translatorName)
else (translator: true);
compatibleTranslators =
let
compatibleTranslators = let
result =
l.filter
(t: t.compatible)
translatorsForSource;
in
if result == [] then
throw "Could not find a compatible translator for input"
else
result;
if result == []
then throw "Could not find a compatible translator for input"
else result;
translator =
l.findFirst
nameFilter
(throw ''Specified translator ${translatorName} not found or incompatible'')
compatibleTranslators;
in
translator;
in
{
in {
inherit
findOneTranslator
getextraArgsDefaults

93
src/libV2.nix
View File

@ -1,7 +1,6 @@
# like ./default.nix but system intependent
# (allows to generate outputs for several systems)
# follows flake output schema
{
dlib,
nixpkgsSrc,
@ -9,11 +8,7 @@
overridesDirs,
externalSources,
externalPaths,
}@args:
let
} @ args: let
b = builtins;
l = lib // builtins;
@ -22,57 +17,51 @@ let
import ./default.nix
{inherit config externalPaths externalSources pkgs;};
# TODO: design output schema for cross compiled packages
makePkgsKey = pkgs:
let
makePkgsKey = pkgs: let
build = pkgs.buildPlatform.system;
host = pkgs.hostPlatform.system;
in
if build == host then build
if build == host
then build
else throw "cross compiling currently not supported";
makeNixpkgs = pkgsList: systems:
# fail if neither pkgs nor systems are defined
if pkgsList == null && systems == [] then
throw "Either `systems` or `pkgs` must be defined"
if pkgsList == null && systems == []
then throw "Either `systems` or `pkgs` must be defined"
# fail if pkgs and systems are both defined
else if pkgsList != null && systems != [] then
throw "Define either `systems` or `pkgs`, not both"
else if pkgsList != null && systems != []
then throw "Define either `systems` or `pkgs`, not both"
# only pkgs is specified
else if pkgsList != null then
if b.isList pkgsList then
else if pkgsList != null
then
if b.isList pkgsList
then
lib.listToAttrs
(pkgs: lib.nameValuePair (makePkgsKey pkgs) pkgs)
pkgsList
else
{ "${makePkgsKey pkgsList}" = pkgsList; }
else {"${makePkgsKey pkgsList}" = pkgsList;}
# only systems is specified
else
lib.genAttrs systems
(system: import nixpkgsSrc {inherit system;});
flakifyBuilderOutputs = system: outputs:
l.mapAttrs
(ouputType: outputValue: {"${system}" = outputValue;})
outputs;
init =
{
init = {
pkgs ? null,
systems ? [],
config ? {},
}@argsInit:
let
} @ argsInit: let
config' = (import ./utils/config.nix).loadConfig argsInit.config or {};
config = config' // {
config =
config'
// {
overridesDirs = args.overridesDirs ++ config'.overridesDirs;
};
@ -81,12 +70,11 @@ let
forAllSystems = f: lib.mapAttrs f allPkgs;
dream2nixFor = forAllSystems (dream2nixForSystem config);
in
{
in {
riseAndShine = throw "Use makeFlakeOutputs instead of riseAndShine.";
makeFlakeOutputs = mArgs: makeFlakeOutputsFunc
makeFlakeOutputs = mArgs:
makeFlakeOutputsFunc
(
{inherit config pkgs systems;}
// mArgs
@ -101,11 +89,9 @@ let
forAllSystems
(system: pkgs:
dream2nixFor."${system}".apps.flakeApps.dream2nix);
};
makeFlakeOutputsFunc =
{
makeFlakeOutputsFunc = {
pname ? throw "Please pass `pname` to makeFlakeOutputs",
pkgs ? null,
packageOverrides ? {},
@ -115,9 +101,7 @@ let
translator ? null,
translatorArgs ? {},
...
}@args:
let
} @ args: let
config = args.config or ((import ./utils/config.nix).loadConfig {});
allPkgs = makeNixpkgs pkgs systems;
forAllSystems = f: b.mapAttrs f allPkgs;
@ -138,15 +122,20 @@ let
allBuilderOutputs =
lib.mapAttrs
(system: pkgs:
let
(system: pkgs: let
dream2nix = dream2nixFor."${system}";
impureDiscoveredProjects =
l.filter
(proj:
dream2nix.translators.translatorsV2."${proj.subsystem}".all
."${proj.translator}".type == "impure")
dream2nix
.translators
.translatorsV2
."${proj.subsystem}"
.all
."${proj.translator}"
.type
== "impure")
discoveredProjects;
impureResolveScriptsList =
@ -178,8 +167,7 @@ let
inherit pname settings source;
};
realizedProjects =
dream2nix.realizeProjects {
realizedProjects = dream2nix.realizeProjects {
inherit packageOverrides translatedProjects source;
};
@ -207,23 +195,20 @@ let
flakifiedOutputsList;
flakeOutputs =
{ projectsJson = l.toJSON (discoveredProjects); }
{projectsJson = l.toJSON discoveredProjects;}
// flakeOutputsBuilders;
in
lib.recursiveUpdate
flakeOutputs
{
apps = forAllSystems (system: pkgs: {
resolve.type = "app";
resolve.program =
let
utils = (dream2nixFor."${system}".utils);
resolve.program = let
utils = dream2nixFor."${system}".utils;
# TODO: Too many calls to findOneTranslator.
# -> make findOneTranslator system independent
translatorFound =
dream2nixFor."${system}".translators.findOneTranslator {
translatorFound = dream2nixFor."${system}".translators.findOneTranslator {
inherit source;
translatorName = args.translator or null;
};
@ -236,9 +221,7 @@ let
});
});
};
in
{
in {
inherit dlib init;
riseAndShine = throw "Use makeFlakeOutputs instead of riseAndShine.";
makeFlakeOutpus = makeFlakeOutputsFunc;

28
src/templates/builders/default.nix
View File

@ -2,47 +2,36 @@
lib,
pkgs,
stdenv,
# dream2nix inputs
builders,
externals,
utils,
...
}:
{
}: {
# Funcs
# AttrSet -> Bool) -> AttrSet -> [x]
getCyclicDependencies, # name: version: -> [ {name=; version=; } ]
getDependencies, # name: version: -> [ {name=; version=; } ]
getSource, # name: version: -> store-path
buildPackageWithOtherBuilder, # { builder, name, version }: -> drv
# Attributes
subsystemAttrs, # attrset
defaultPackageName, # string
defaultPackageVersion, # string
# attrset of pname -> versions,
# where versions is a list of version strings
packageVersions,
# function which applies overrides to a package
# It must be applied by the builder to each individual derivation
# Example:
# produceDerivation name (mkDerivation {...})
produceDerivation,
# Custom Options: (parametrize builder behavior)
# These can be passed by the user via `builderArgs`.
# All options must provide default
standalonePackageNames ? [],
...
}@args:
let
} @ args: let
b = builtins;
# the main package
@ -58,11 +47,8 @@ let
packageVersions;
# Generates a derivation for a specific package name + version
makeOnePackage = name: version:
let
pkg =
stdenv.mkDerivation rec {
makeOnePackage = name: version: let
pkg = stdenv.mkDerivation rec {
pname = utils.sanitizeDerivationName name;
inherit version;
@ -74,14 +60,10 @@ let
(getDependencies name version);
# Implement build phases
};
in
# apply packageOverrides to current derivation
(utils.applyOverridesToPackage packageOverrides pkg name);
in
{
in {
inherit defaultPackage packages;
}

24
src/templates/translators/impure.nix
View File

@ -1,10 +1,7 @@
{
dlib,
lib,
}:
{
}: {
# A derivation which outputs a single executable at `$out`.
# The executable will be called by dream2nix for translation
# The input format is specified in /specifications/translator-call-example.json.
@ -15,11 +12,9 @@
# The program is expected to create a file at the location specified
# by the input parameter `outFile`.
# The output file must contain the dream lock data encoded as json.
translateBin =
{
translateBin = {
# dream2nix utils
utils,
# nixpkgs dependenies
bash,
jq,
@ -47,23 +42,15 @@
# containing the dream lock similar to /specifications/dream-lock-example.json
'';
# This function should return the projects name.
# The computational complexity of this should be kept as lightweight
# as possible, as this migth be executed on a large amount of inputs at once.
projectName =
{
source,
}:
projectName = {source}:
null;
# This allows the framework to detect if the translator is compatible with the given input
# to automatically select the right translator.
compatible =
{
source,
}:
compatible = {source}:
# TODO: insert regex here that matches valid input file names
# examples:
# - ''.*requirements.*\.txt''
@ -75,14 +62,12 @@
]
source;
# If the translator requires additional arguments, specify them here.
# There are only two types of arguments:
# - string argument (type = "argument")
# - boolean flag (type = "flag")
# String arguments contain a default value and examples. Flags do not.
extraArgs = {
# Example: boolean option
# Flags always default to 'false' if not specified by the user
noDev = {
@ -100,6 +85,5 @@
];
type = "argument";
};
};
}

85
src/translators/default.nix
View File

@ -6,50 +6,47 @@
nix,
pkgs,
python3,
callPackageDream,
externals,
dream2nixWithExternals,
utils,
...
}:
let
}: let
b = builtins;
l = lib // builtins;
# transforms V1 translators to V2 translators
ensureTranslatorV2 = translator:
let
ensureTranslatorV2 = translator: let
version = translator.version or 1;
cleanedArgs = args: l.removeAttrs args ["project" "tree"];
upgradedTranslator =
translator
// (lib.optionalAttrs (translator ? translate) {
translate = args:
let
translate = args: let
dreamLock =
translator.translate
((cleanedArgs args) // {
((cleanedArgs args)
// {
source = "${args.source}/${args.project.relPath}";
name = args.project.name;
});
in
dreamLock // {
_generic = dreamLock._generic // {
dreamLock
// {
_generic =
dreamLock._generic
// {
location = args.project.relPath;
};
};
});
finalTranslator =
if version == 2 then
translator
else
upgradedTranslator;
if version == 2
then translator
else upgradedTranslator;
in
finalTranslator
// (lib.optionalAttrs (finalTranslator ? translate) {
@ -58,22 +55,23 @@ let
(with translator; [subsystem type name]);
# ensure `tree` is passed
translate = args:
finalTranslator.translate (args // {
finalTranslator.translate (args
// {
tree =
args.tree or (dlib.prepareSourceTree {inherit (args) source;});
});
});
# transforms V2 translators to V1 translators
ensureTranslatorV1 = translator:
let
ensureTranslatorV1 = translator: let
version = translator.version or 1;
downgradeTranslator =
translator
// (lib.optionalAttrs (translator ? translate) {
translate = args:
translator.translate (args // {
translator.translate (args
// {
inherit (args) source;
tree = dlib.prepareSourceTree {inherit (args) source;};
project = {
@ -85,38 +83,32 @@ let
});
finalTranslator =
if version == 1 then
translator
else
downgradeTranslator;
if version == 1
then translator
else downgradeTranslator;
in
finalTranslator;
makeTranslatorV2 = translatorModule:
ensureTranslatorV2 (makeTranslator translatorModule);
makeTranslatorV1 = translatorModule:
ensureTranslatorV1 (makeTranslator translatorModule);
makeTranslator =
translatorModule:
let
makeTranslator = translatorModule: let
translator =
translatorModule
# for pure translators
# - import the `translate` function
# - generate `translateBin`
// (lib.optionalAttrs (translatorModule ? translate) {
translate =
let
translate = let
translateOriginal = callPackageDream translatorModule.translate {
translatorName = translatorModule.name;
};
in
args: translateOriginal
args:
translateOriginal
(
(dlib.translators.getextraArgsDefaults
(translatorModule.extraArgs or {}))
@ -126,28 +118,26 @@ let
wrapPureTranslator
(with translatorModule; [subsystem type name]);
})
# for impure translators:
# - import the `translateBin` function
// (lib.optionalAttrs (translatorModule ? translateBin) {
translateBin = callPackageDream translatorModule.translateBin
translateBin =
callPackageDream translatorModule.translateBin
{
translatorName = translatorModule.name;
};
});
in
translator;
translators = dlib.translators.mapTranslators makeTranslatorV1;
translatorsV2 = dlib.translators.mapTranslators makeTranslatorV2;
# adds a translateBin to a pure translator
wrapPureTranslator2 = translatorAttrPath:
let
bin = utils.writePureShellScript
wrapPureTranslator2 = translatorAttrPath: let
bin =
utils.writePureShellScript
[
coreutils
jq
@ -190,9 +180,9 @@ let
});
# adds a translateBin to a pure translator
wrapPureTranslator = translatorAttrPath:
let
bin = utils.writePureShellScript
wrapPureTranslator = translatorAttrPath: let
bin =
utils.writePureShellScript
[
coreutils
jq
@ -233,17 +223,14 @@ let
bin.overrideAttrs (old: {
name = "translator-${lib.concatStringsSep "-" translatorAttrPath}";
});
in
{
in {
inherit
translators
translatorsV2
;
inherit (dlib.translators)
inherit
(dlib.translators)
findOneTranslator
translatorsForInput
translatorsForInputRecursive

33
src/translators/go/impure/gomod2nix/default.nix
View File

@ -1,20 +1,15 @@
{
dlib,
lib,
}:
let
}: let
l = lib // builtins;
in
{
in {
# the input format is specified in /specifications/translator-call-example.json
# this script receives a json file including the input paths and specialArgs
translateBin =
{
translateBin = {
# dream2nix utils
utils,
dream2nixWithExternals,
bash,
coreutils,
jq,
@ -47,29 +42,19 @@ in
nix eval --show-trace --impure --raw --expr "import ${./translate.nix} ${dream2nixWithExternals} ./." > $outputFile
'';
projectName =
{
source,
}:
let
projectName = {source}: let
goModFile = "${source}/go.mod";
firstLine = (l.elemAt (l.splitString "\n" (l.readFile goModFile)) 0);
firstLine = l.elemAt (l.splitString "\n" (l.readFile goModFile)) 0;
in
if l.pathExists goModFile then
l.last (l.splitString "/" (l.elemAt (l.splitString " " firstLine) 1))
else
null;
if l.pathExists goModFile
then l.last (l.splitString "/" (l.elemAt (l.splitString " " firstLine) 1))
else null;
# This allows the framework to detect if the translator is compatible with the given input
# to automatically select the right translator.
compatible =
{
source,
}:
compatible = {source}:
dlib.containsMatchingFile [''go\.sum'' ''go\.mod''] source;
# If the translator requires additional arguments, specify them here.
# There are only two types of arguments:
# - string argument (type = "argument")

21
src/translators/go/impure/gomod2nix/translate.nix
View File

@ -1,6 +1,4 @@
dream2nixWithExternals:
cwd:
let
dream2nixWithExternals: cwd: let
dream2nix = import dream2nixWithExternals {};
b = builtins;
parsed = b.fromTOML (builtins.readFile "${cwd}/gomod2nix.toml");
@ -16,17 +14,13 @@ let
getDepByNameVer,
dependenciesByOriginalID,
...
}:
rec {
}: rec {
translatorName = "gomod2nix";
inputData = parsed;
defaultPackage =
let
firstLine = (b.elemAt (lib.splitString "\n" (b.readFile "${cwd}/go.mod")) 0);
defaultPackage = let
firstLine = b.elemAt (lib.splitString "\n" (b.readFile "${cwd}/go.mod")) 0;
in
lib.last (lib.splitString "/" (b.elemAt (lib.splitString " " firstLine) 1));
@ -55,21 +49,18 @@ let
getVersion = dependencyObject:
lib.removePrefix "v" dependencyObject.sumVersion;
getDependencies = dependencyObject:
[];
getDependencies = dependencyObject: [];
getSourceType = dependencyObject: "git";
sourceConstructors = {
git = dependencyObject:
{
git = dependencyObject: {
type = "git";
hash = dependencyObject.fetch.sha256;
url = dependencyObject.fetch.url;
rev = dependencyObject.fetch.rev;
};
};
});
in
dream2nix.utils.dreamLock.toJSON translated

20
src/translators/nodejs/impure/package-json/default.nix
View File

@ -1,18 +1,13 @@
{
dlib,
lib,
}:
{
}: {
# the input format is specified in /specifications/translator-call-example.json
# this script receives a json file including the input paths and specialArgs
translateBin =
{
translateBin = {
# dream2nix utils
translators,
utils,
# nixpkgs dependenies
bash,
coreutils,
@ -21,7 +16,6 @@
writeScriptBin,
...
}:
utils.writePureShellScript
[
bash
@ -57,22 +51,18 @@
${translators.translators.nodejs.pure.package-lock.translateBin} $TMPDIR/newJsonInput
'';
# inherit projectName function from package-lock translator
projectName = dlib.translators.translators.nodejs.pure.package-lock.projectName;
# This allows the framework to detect if the translator is compatible with the given input
# to automatically select the right translator.
compatible =
{
source,
}:
compatible = {source}:
dlib.containsMatchingFile [''.*package.json''] source;
# inherit options from package-lock translator
extraArgs =
dlib.translators.translators.nodejs.pure.package-lock.extraArgs // {
dlib.translators.translators.nodejs.pure.package-lock.extraArgs
// {
npmArgs = {
description = "Additional arguments for npm";
type = "argument";

150
src/translators/nodejs/pure/package-lock/default.nix
View File

@ -1,9 +1,7 @@
{
dlib,
lib,
}:
let
}: let
b = builtins;
l = lib // builtins;
nodejsUtils = import ../../utils.nix {inherit lib;};
@ -11,24 +9,19 @@ let
getPackageLock = tree: project:
nodejsUtils.getWorkspaceLockFile tree project "package-lock.json";
translate =
{
translate = {
translatorName,
utils,
...
}:
{
}: {
project,
source,
tree,
# translator args
noDev,
nodejs,
...
}@args:
let
} @ args: let
b = builtins;
dev = ! noDev;
@ -44,10 +37,9 @@ let
(tree.getNodeFromPath "package.json").jsonContent;
packageLockDeps =
if packageLock == null then
{}
else
packageLock.dependencies or {};
if packageLock == null
then {}
else packageLock.dependencies or {};
rootDependencies = packageLockDeps;
@ -63,40 +55,40 @@ let
# ! (dependencyObject ? integrity) &&
dlib.identifyGitUrl dependencyObject.version;
getVersion = dependencyObject:
let
getVersion = dependencyObject: let
# example: "version": "npm:@tailwindcss/postcss7-compat@2.2.4",
npmMatch = b.match ''^npm:.*@(.*)$'' dependencyObject.version;
in
if npmMatch != null then
b.elemAt npmMatch 0
else if identifyGitSource dependencyObject then
"0.0.0-rc.${b.substring 0 8 (dlib.parseGitUrl dependencyObject.version).rev}"
else if lib.hasPrefix "file:" dependencyObject.version then
let
if npmMatch != null
then b.elemAt npmMatch 0
else if identifyGitSource dependencyObject
then "0.0.0-rc.${b.substring 0 8 (dlib.parseGitUrl dependencyObject.version).rev}"
else if lib.hasPrefix "file:" dependencyObject.version
then let
path = getPath dependencyObject;
in
(b.fromJSON
(
b.fromJSON
(b.readFile "${source}/${path}/package.json")
).version
else if lib.hasPrefix "https://" dependencyObject.version then
"unknown"
else
dependencyObject.version;
)
.version
else if lib.hasPrefix "https://" dependencyObject.version
then "unknown"
else dependencyObject.version;
getPath = dependencyObject:
lib.removePrefix "file:" dependencyObject.version;
pinVersions = dependencies: parentScopeDeps:
lib.mapAttrs
(pname: pdata:
let
(
pname: pdata: let
selfScopeDeps = parentScopeDeps // dependencies;
requires = pdata.requires or {};
dependencies = pdata.dependencies or {};
in
pdata // {
pdata
// {
depsExact =
lib.forEach
(lib.attrNames requires)
@ -112,23 +104,17 @@ let
pinnedRootDeps =
pinVersions rootDependencies rootDependencies;
createMissingSource = name: version:
{
createMissingSource = name: version: {
type = "http";
url = "https://registry.npmjs.org/${name}/-/${name}-${version}.tgz";
};
in
utils.simpleTranslate
({
getDepByNameVer,
dependenciesByOriginalID,
...
}:
rec {
}: rec {
inherit translatorName;
location = relPath;
@ -136,10 +122,11 @@ let
inputData = pinnedRootDeps;
defaultPackage =
if name != "{automatic}" then
name
if name != "{automatic}"
then name
else
packageJson.name or (throw (
packageJson.name
or (throw (
"Could not identify package name. "
+ "Please specify extra argument 'name'"
));
@ -150,8 +137,10 @@ let
mainPackageDependencies =
lib.mapAttrsToList
(pname: pdata:
{ name = pname; version = getVersion pdata; })
(pname: pdata: {
name = pname;
version = getVersion pdata;
})
(lib.filterAttrs
(pname: pdata: ! (pdata.dev or false) || dev)
parsedDependencies);
@ -161,21 +150,22 @@ let
subsystemAttrs = {nodejsVersion = args.nodejs;};
# functions
serializePackages = inputData:
let
serializePackages = inputData: let
serialize = inputData:
lib.mapAttrsToList # returns list of lists
(pname: pdata:
[ (pdata // {
[
(pdata
// {
inherit pname;
depsExact =
lib.filter
(req:
(! (pdata.dependencies."${req.name}".bundled or false)))
(req: (! (pdata.dependencies."${req.name}".bundled or false)))
pdata.depsExact or {};
}) ]
++
(lib.optionals (pdata ? dependencies)
})
]
++ (lib.optionals (pdata ? dependencies)
(lib.flatten
(serialize
(lib.filterAttrs
@ -193,40 +183,37 @@ let
inherit getVersion;
getSourceType = dependencyObject:
if identifyGitSource dependencyObject then
"git"
else if lib.hasPrefix "file:" dependencyObject.version then
"path"
else
"http";
if identifyGitSource dependencyObject
then "git"
else if lib.hasPrefix "file:" dependencyObject.version
then "path"
else "http";
sourceConstructors = {
git = dependencyObject:
dlib.parseGitUrl dependencyObject.version;
http = dependencyObject:
if lib.hasPrefix "https://" dependencyObject.version then
rec {
if lib.hasPrefix "https://" dependencyObject.version
then rec {
version = getVersion dependencyObject;
url = dependencyObject.version;
hash = dependencyObject.integrity;
}
else if dependencyObject.resolved == false then
else if dependencyObject.resolved == false
then
(createMissingSource
(getName dependencyObject)
(getVersion dependencyObject))
// {
hash = dependencyObject.integrity;
}
else
rec {
else rec {
url = dependencyObject.resolved;
hash = dependencyObject.integrity;
};
path = dependencyObject:
rec {
path = dependencyObject: rec {
path = getPath dependencyObject;
};
};
@ -234,33 +221,20 @@ let
getDependencies = dependencyObject:
dependencyObject.depsExact;
});
in
rec {
in rec {
version = 2;
inherit translate;
projectName =
{
source,
}:
let
projectName = {source}: let
packageJson = "${source}/package.json";
parsed = b.fromJSON (b.readFile packageJson);
in
if b.pathExists packageJson && parsed ? name then
parsed.name
else
null;
if b.pathExists packageJson && parsed ? name
then parsed.name
else null;
compatible =
{
source,
}:
compatible = {source}:
dlib.containsMatchingFile
[
''.*package-lock\.json''
@ -269,7 +243,6 @@ rec {
source;
extraArgs = {
name = {
description = "The name of the main package";
examples = [
@ -296,6 +269,5 @@ rec {
];
type = "argument";
};
};
}

175
src/translators/nodejs/pure/yarn-lock/default.nix
View File

@ -1,9 +1,7 @@
{
dlib,
lib,
}:
let
}: let
l = lib // builtins;
nodejsUtils = import ../../utils.nix {inherit lib;};
parser = import ./parser.nix {inherit lib;};
@ -11,24 +9,19 @@ let
getYarnLock = tree: proj:
tree.getNodeFromPath "${proj.relPath}/yarn.lock";
translate =
{
translate = {
translatorName,
utils,
...
}:
{
}: {
project,
source,
tree,
# extraArgs
nodejs,
noDev,
...
}@args:
let
} @ args: let
b = builtins;
dev = ! noDev;
name = project.name;
@ -38,10 +31,11 @@ let
yarnLock = parser.parse (tree.getNodeFromPath "yarn.lock").content;
defaultPackage =
if name != "{automatic}" then
name
if name != "{automatic}"
then name
else
packageJson.name or (throw (
packageJson.name
or (throw (
"Could not identify package name. "
+ "Please specify extra argument 'name'"
));
@ -52,21 +46,14 @@ let
packageJsonDeps = nodejsUtils.getPackageJsonDeps packageJson noDev;
workspacesPackageJson = nodejsUtils.getWorkspacePackageJson tree workspaces;
in
utils.simpleTranslate2
({
objectsByKey,
...
}: let
({objectsByKey, ...}: let
makeWorkspaceExtraObject = workspace: let
json = workspacesPackageJson."${workspace}";
name = json.name or workspace;
version = json.version or "unknown";
in
{
in {
inherit name version;
dependencies =
@ -76,12 +63,14 @@ let
depObject = objectsByKey.yarnName."${yarnName}";
in
if exportedWorkspacePackages ? "${depName}"
then
{
then {
name = depName;
version = exportedWorkspacePackages."${depName}";
}
else {name = depName; version = depObject.version;})
else {
name = depName;
version = depObject.version;
})
(nodejsUtils.getPackageJsonDeps json noDev);
sourceSpec = {
@ -108,12 +97,10 @@ let
in
if
lib.hasInfix "@github:" dObj.yarnName
|| (dObj ? resolved
|| (dObj
? resolved
&& lib.hasInfix "codeload.github.com/" dObj.resolved)
|| lib.hasInfix "@git+" dObj.yarnName
# example:
# "jest-image-snapshot@https://github.com/machard/jest-image-snapshot#machard-patch-1":
# version "4.2.0"
@ -129,16 +116,12 @@ let
)
"git"
else "git"
else if
lib.hasInfix "@link:" dObj.yarnName
|| lib.hasInfix "@file:" dObj.yarnName
then "path"
else "http";
in rec {
inherit defaultPackage extraObjects translatorName;
exportedPackages =
@ -156,38 +139,35 @@ let
extractors = {
name = rawObj: finalObj:
if lib.hasInfix "@git+" rawObj.yarnName then
lib.head (lib.splitString "@git+" rawObj.yarnName)
if lib.hasInfix "@git+" rawObj.yarnName
then lib.head (lib.splitString "@git+" rawObj.yarnName)
# Example:
# @matrix-org/olm@https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.3.tgz
else if lib.hasInfix "@https://" rawObj.yarnName then
lib.head (lib.splitString "@https://" rawObj.yarnName)
else
let
else if lib.hasInfix "@https://" rawObj.yarnName
then lib.head (lib.splitString "@https://" rawObj.yarnName)
else let
split = lib.splitString "@" rawObj.yarnName;
version = lib.last split;
in
if lib.hasPrefix "@" rawObj.yarnName then
lib.removeSuffix "@${version}" rawObj.yarnName
else
lib.head split;
if lib.hasPrefix "@" rawObj.yarnName
then lib.removeSuffix "@${version}" rawObj.yarnName
else lib.head split;
version = rawObj: finalObj:
if l.hasInfix "@git+" rawObj.yarnName
then
let
then let
split = l.splitString "@git+" rawObj.yarnName;
gitUrl = l.last split;
in
# l.strings.sanitizeDerivationName
"${rawObj.version}@git+${gitUrl}"
else rawObj.version;
dependencies = rawObj: finalObj: let
dependencies = let
deps =
rawObj.dependencies or {}
rawObj.dependencies
or {}
// rawObj.optionalDependencies or {};
in
lib.mapAttrsToList
@ -196,26 +176,27 @@ let
in
lib.forEach
dependencies
(dependency:
(
dependency:
builtins.head (
lib.mapAttrsToList
(name: versionSpec: let
(
name: versionSpec: let
yarnName = "${name}@${versionSpec}";
depObject = objectsByKey.yarnName."${yarnName}";
version = depObject.version;
in
if ! objectsByKey.yarnName ? ${yarnName} then
if ! objectsByKey.yarnName ? ${yarnName}
then
# handle missing lock file entry
let
versionMatch =
b.match ''.*\^([[:digit:]|\.]+)'' versionSpec;
in
{
in {
inherit name;
version = b.elemAt versionMatch 0;
}
else
{ inherit name version; }
else {inherit name version;}
)
dependency
)
@ -225,32 +206,31 @@ let
type = getSourceType rawObj finalObj;
in
{inherit type;}
//
(if type == "git"
// (
if type == "git"
then
if utils.identifyGitUrl rawObj.resolved then
(utils.parseGitUrl rawObj.resolved) // {
if utils.identifyGitUrl rawObj.resolved
then
(utils.parseGitUrl rawObj.resolved)
// {
version = rawObj.version;
}
else
let
else let
githubUrlInfos = lib.splitString "/" rawObj.resolved;
owner = lib.elemAt githubUrlInfos 3;
repo = lib.elemAt githubUrlInfos 4;
in
if b.length githubUrlInfos == 7 then
let
if b.length githubUrlInfos == 7
then let
rev = lib.elemAt githubUrlInfos 6;
in
{
in {
url = "https://github.com/${owner}/${repo}";
inherit rev;
}
else if b.length githubUrlInfos == 5 then
let
else if b.length githubUrlInfos == 5
then let
urlAndRev = lib.splitString "#" rawObj.resolved;
in
{
in {
url = lib.head urlAndRev;
rev = lib.last urlAndRev;
}
@ -259,39 +239,35 @@ let
"Unable to parse git dependency for: "
+ "${finalObj.name}#${finalObj.version}"
)
else if type == "path"
then
if lib.hasInfix "@link:" rawObj.yarnName then
{
if lib.hasInfix "@link:" rawObj.yarnName
then {
path =
lib.last (lib.splitString "@link:" rawObj.yarnName);
}
else if lib.hasInfix "@file:" rawObj.yarnName then
{
else if lib.hasInfix "@file:" rawObj.yarnName
then {
path =
lib.last (lib.splitString "@file:" rawObj.yarnName);
}
else
throw "unknown path format ${b.toJSON rawObj}"
else throw "unknown path format ${b.toJSON rawObj}"
else # type == "http"
{
type = "http";
hash =
if rawObj ? integrity then
rawObj.integrity
else
let
if rawObj ? integrity
then rawObj.integrity
else let
hash =
lib.last (lib.splitString "#" rawObj.resolved);
in
if lib.stringLength hash == 40 then
hash
else
throw "Missing integrity for ${rawObj.yarnName}";
if lib.stringLength hash == 40
then hash
else throw "Missing integrity for ${rawObj.yarnName}";
url = lib.head (lib.splitString "#" rawObj.resolved);
});
}
);
};
extraDependencies =
@ -299,16 +275,17 @@ let
(name: semVer: let
depYarnKey = "${name}@${semVer}";
dependencyAttrs =
if ! yarnLock ? "${depYarnKey}" then
throw "Cannot find entry for top level dependency: '${depYarnKey}'"
else
yarnLock."${depYarnKey}";
in
{
if ! yarnLock ? "${depYarnKey}"
then throw "Cannot find entry for top level dependency: '${depYarnKey}'"
else yarnLock."${depYarnKey}";
in {
name = defaultPackage;
version = packageJson.version or "unknown";
dependencies = [
{inherit name; version = dependencyAttrs.version;}
{
inherit name;
version = dependencyAttrs.version;
}
];
})
packageJsonDeps;
@ -317,11 +294,8 @@ let
lib.mapAttrsToList
(yarnName: depAttrs: depAttrs // {inherit yarnName;})
yarnLock;
});
in {
version = 2;
inherit translate;
@ -329,23 +303,17 @@ in {
# inherit projectName function from package-lock translator
projectName = dlib.translators.translators.nodejs.pure.package-lock.projectName;
# This allows the framework to detect if the translator is compatible with the given input
# to automatically select the right translator.
compatible =
{
source,
}:
compatible = {source}:
dlib.containsMatchingFile [''.*yarn\.lock'' ''.*package.json''] source;
# If the translator requires additional arguments, specify them here.
# There are only two types of arguments:
# - string argument (type = "argument")
# - boolean flag (type = "flag")
# String arguments contain a default value and examples. Flags do not.
extraArgs = {
name = {
description = "The name of the main package";
examples = [
@ -370,6 +338,5 @@ in {
];
type = "argument";
};
};
}

98
src/translators/nodejs/pure/yarn-lock/parser.nix
View File

@ -1,26 +1,18 @@
{
lib ? (import <nixpkgs> {}).lib,
...
}:
let
{lib ? (import <nixpkgs> {}).lib, ...}: let
l = lib // builtins;
parse = text:
let
parse = text: let
lines = l.splitString "\n" text;
findStartLineNum = num:
let
findStartLineNum = num: let
line = l.elemAt lines num;
in
if ! l.hasPrefix "#" line
if
! l.hasPrefix "#" line
&& ! l.hasPrefix " " line
&& ! l.hasPrefix "_" line then
num
else
findStartLineNum (num + 1);
&& ! l.hasPrefix "_" line
then num
else findStartLineNum (num + 1);
contentLines =
l.sublist
@ -28,8 +20,7 @@ let
((l.length lines) - 1)
lines;
matchLine = line:
let
matchLine = line: let
# yarn v2
m1 = l.match ''( *)(.*): (.*)'' line;
m2 = l.match ''( *)(.*):$'' line;
@ -38,14 +29,14 @@ let
m3 = l.match ''( *)(.*) "(.*)"'' line;
m4 = l.match ''( *)(.*) (.*)'' line;
in
if m1 != null then
{
if m1 != null
then {
indent = (l.stringLength (l.elemAt m1 0)) / 2;
key = l.elemAt m1 1;
value = l.elemAt m1 2;
}
else if m2 != null then
{
else if m2 != null
then {
indent = (l.stringLength (l.elemAt m2 0)) / 2;
# transform yarn 1 to yarn 2 tyle
key =
@ -53,63 +44,56 @@ let
(l.replaceStrings [''", ''] ['', ''] (l.elemAt m2 1));
value = null;
}
else if m3 != null then
{
else if m3 != null
then {
indent = (l.stringLength (l.elemAt m3 0)) / 2;
key = l.elemAt m3 1;
value = l.elemAt m3 2;
}
else if m4 != null then
{
else if m4 != null
then {
indent = (l.stringLength (l.elemAt m4 0)) / 2;
key = l.elemAt m4 1;
value = l.elemAt m4 2;
}
else
null;
else null;
closingParenthesis = num:
if num == 1 then "}" else "}" + (closingParenthesis (num - 1));
if num == 1
then "}"
else "}" + (closingParenthesis (num - 1));
jsonLines = lines:
let
jsonLines = lines: let
filtered = l.filter (line: l.match ''[[:space:]]*'' line == null) lines;
matched = l.map (line: matchLine line) filtered;
in
l.imap0
(i: line:
let
(i: line: let
mNext = l.elemAt matched (i + 1);
m = l.elemAt matched i;
keyParenthesis =
let
keyParenthesis = let
beginOK = l.hasPrefix ''"'' m.key;
endOK = l.hasSuffix ''"'' m.key;
begin = l.optionalString (! beginOK) ''"'';
end = l.optionalString (! endOK) ''"'';
in
''${begin}${m.key}${end}'';
in ''${begin}${m.key}${end}'';
valParenthesis =
if l.hasPrefix ''"'' m.value then
m.value
else
''"${m.value}"'';
if l.hasPrefix ''"'' m.value
then m.value
else ''"${m.value}"'';
in
if l.length filtered == i + 1 then
let
if l.length filtered == i + 1
then let
end = closingParenthesis m.indent;
in
''${keyParenthesis}: ${valParenthesis}${end}}''
else if m.value == null then
''${keyParenthesis}: {''
in ''${keyParenthesis}: ${valParenthesis}${end}}''
else if m.value == null
then ''${keyParenthesis}: {''
# if indent of next line is smaller, close the object
else if mNext.indent < m.indent then
let
else if mNext.indent < m.indent
then let
end = closingParenthesis (m.indent - mNext.indent);
in
''${keyParenthesis}: ${valParenthesis}${end},''
else
''${keyParenthesis}: ${valParenthesis},'')
in ''${keyParenthesis}: ${valParenthesis}${end},''
else ''${keyParenthesis}: ${valParenthesis},'')
filtered;
json = "{${l.concatStringsSep "\n" (jsonLines contentLines)}";
@ -123,17 +107,13 @@ let
l.listToAttrs
(l.flatten
(l.mapAttrsToList
(n: v:
let
(n: v: let
keys = l.splitString ", " n;
in
l.map (k: l.nameValuePair k v) keys)
dataRaw));
in
data;
in
{
in {
inherit parse;
}

28
src/translators/nodejs/utils.nix
View File

@ -1,35 +1,27 @@
{
lib,
}: let
{lib}: let
l = lib // builtins;
in rec {
getPackageJsonDeps = packageJson: noDev:
packageJson.dependencies or {}
packageJson.dependencies
or {}
// (lib.optionalAttrs (! noDev) (packageJson.devDependencies or {}));
getWorkspaceLockFile = tree: project: fname: let
# returns the parsed package-lock.json for a given project
dirRelPath =
if project ? subsystemInfo.workspaceParent then
"${project.subsystemInfo.workspaceParent}"
else
"${project.relPath}";
if project ? subsystemInfo.workspaceParent
then "${project.subsystemInfo.workspaceParent}"
else "${project.relPath}";
packageJson =
(tree.getNodeFromPath "${dirRelPath}/package.json").jsonContent;
hasNoDependencies =
! packageJson ? dependencies && ! packageJson ? devDependencies;
in
if hasNoDependencies then
null
else
tree.getNodeFromPath "${dirRelPath}/${fname}";
if hasNoDependencies
then null
else tree.getNodeFromPath "${dirRelPath}/${fname}";
getWorkspacePackageJson = tree: workspaces:
l.genAttrs
@ -44,6 +36,4 @@ in rec {
json.name
json.version)
(getWorkspacePackageJson tree workspaces);
}

25
src/translators/python/impure/pip/default.nix
View File

@ -1,22 +1,15 @@
{
dlib,
lib,
}:
let
}: let
b = builtins;
in
{
in {
# the input format is specified in /specifications/translator-call-example.json
# this script receives a json file including the input paths and extraArgs
translateBin =
{
translateBin = {
# dream2nix
externalSources,
utils,
bash,
coreutils,
jq,
@ -24,8 +17,7 @@ in
python3,
writeScriptBin,
...
}:
let
}: let
machNixExtractor = "${externalSources.mach-nix}/lib/default.nix";
setuptools_shim = ''
@ -36,7 +28,6 @@ in
exec(compile(code, __file__, 'exec'))
'';
in
utils.writePureShellScript
[
bash
@ -106,11 +97,7 @@ in
rm -rf $tmp $tmpBuild
'';
compatible =
{
source,
}:
compatible = {source}:
dlib.containsMatchingFile
[
''.*requirements.*\.txt''
@ -119,7 +106,6 @@ in
# define special args and provide defaults
extraArgs = {
# the python attribute
pythonAttr = {
default = "python3";
@ -136,6 +122,5 @@ in
description = "build application instead of package";
type = "flag";
};
};
}

129
src/translators/rust/pure/cargo-lock/default.nix
View File

@ -1,36 +1,28 @@
{
dlib,
lib,
}:
let
}: let
l = lib // builtins;
in
{
translate =
{
in {
translate = {
externals,
translatorName,
utils,
...
}:
{
}: {
source,
packageName,
...
}@args:
let
} @ args: let
inputDir = source;
recurseFiles = path:
l.flatten (
l.mapAttrsToList
(n: v:
if v == "directory" then
recurseFiles "${path}/${n}"
else
"${path}/${n}")
if v == "directory"
then recurseFiles "${path}/${n}"
else "${path}/${n}")
(l.readDir path)
);
@ -53,8 +45,7 @@ in
packageName =
if args.packageName == "{automatic}"
then
let
then let
# Small function to check if a given package path has a package
# that has binaries
hasBinaries = toml:
@ -64,8 +55,8 @@ in
# Try to find a package with a binary
pkg = l.findFirst hasBinaries (l.elemAt cargoPackages 0) cargoPackages;
in pkg.value.package.name
in
pkg.value.package.name
else args.packageName;
# Find the Cargo.toml matching the package name
@ -81,25 +72,28 @@ in
parsedDeps = parsedLock.package;
# This parses a "package-name version" entry in the "dependencies"
# field of a dependency in Cargo.lock
makeDepNameVersion = entry:
let
makeDepNameVersion = entry: let
parsed = l.splitString " " entry;
name = l.head parsed;
maybeVersion = if l.length parsed > 1 then l.last parsed else null;
in
{
maybeVersion =
if l.length parsed > 1
then l.last parsed
else null;
in {
inherit name;
version =
# If there is no version, search through the lockfile to
# find the dependency's version
if maybeVersion != null
then maybeVersion
else (
else
(
l.findFirst
(dep: dep.name == name)
(throw "no dependency found with name ${name} in Cargo.lock")
parsedDeps
).version;
)
.version;
};
package = rec {
@ -111,41 +105,41 @@ in
};
# Parses a git source, taken straight from nixpkgs.
parseGitSource = src:
let
parseGitSource = src: let
parts = builtins.match ''git\+([^?]+)(\?(rev|tag|branch)=(.*))?#(.*)'' src;
type = builtins.elemAt parts 2; # rev, tag or branch
value = builtins.elemAt parts 3;
in
if parts == null then null
else {
if parts == null
then null
else
{
url = builtins.elemAt parts 0;
sha = builtins.elemAt parts 4;
} // lib.optionalAttrs (type != null) { inherit type value; };
}
// lib.optionalAttrs (type != null) {inherit type value;};
# Extracts a source type from a dependency.
getSourceTypeFrom = dependencyObject:
let checkType = type: l.hasPrefix "${type}+" dependencyObject.source; in
getSourceTypeFrom = dependencyObject: let
checkType = type: l.hasPrefix "${type}+" dependencyObject.source;
in
if !(l.hasAttr "source" dependencyObject)
then "path"
else if checkType "git" then
"git"
else if checkType "registry" then
else if checkType "git"
then "git"
else if checkType "registry"
then
if dependencyObject.source == "registry+https://github.com/rust-lang/crates.io-index"
then "crates-io"
else throw "registries other than crates.io are not supported yet"
else
throw "unknown or unsupported source type: ${dependencyObject.source}";
else throw "unknown or unsupported source type: ${dependencyObject.source}";
in
utils.simpleTranslate
({
getDepByNameVer,
dependenciesByOriginalID,
...
}:
rec {
}: rec {
# VALUES
inherit translatorName;
@ -165,11 +159,9 @@ in
toml.value.package.name
toml.value.package.version)
cargoPackages))
//
{ "${defaultPackage}" = package.version; };
// {"${defaultPackage}" = package.version;};
mainPackageDependencies =
let
mainPackageDependencies = let
mainPackage =
l.findFirst
(dep: dep.name == package.name)
@ -187,7 +179,8 @@ in
subsystemAttrs = rec {
gitSources = let
gitDeps = l.filter (dep: (getSourceTypeFrom dep) == "git") parsedDeps;
in l.unique (l.map (dep: parseGitSource dep.source) gitDeps);
in
l.unique (l.map (dep: parseGitSource dep.source) gitDeps);
};
# FUNCTIONS
@ -212,61 +205,45 @@ in
# Given a dependency object and a source type, construct the
# source definition containing url, hash, etc.
sourceConstructors = {
path = dependencyObject:
let
toml =
(l.findFirst
path = dependencyObject: let
toml = (
l.findFirst
(toml: toml.value.package.name == dependencyObject.name)
(throw "could not find crate ${dependencyObject.name}")
cargoPackages
);
relDir = lib.removePrefix "${inputDir}/" (l.dirOf toml.path);
in
{
in {
path = relDir;
rootName = package.name;
rootVersion = package.version;
};
git = dependencyObject:
let
git = dependencyObject: let
parsed = parseGitSource dependencyObject.source;
in
{
in {
url = parsed.url;
rev = parsed.sha;
};
crates-io = dependencyObject:
{
crates-io = dependencyObject: {
hash = dependencyObject.checksum;
};
};
});
projectName =
{
source,
}:
let
projectName = {source}: let
cargoToml = "${source}/Cargo.toml";
in
if l.pathExists cargoToml then
(l.fromTOML (l.readFile cargoToml)).package.name or null
else
null;
if l.pathExists cargoToml
then (l.fromTOML (l.readFile cargoToml)).package.name or null
else null;
# This allows the framework to detect if the translator is compatible with the given input
# to automatically select the right translator.
compatible =
{
source,
}:
compatible = {source}:
dlib.containsMatchingFile [''.*Cargo\.lock''] source;
# If the translator requires additional arguments, specify them here.
# When users run the CLI, they will be asked to specify these arguments.
# There are only two types of arguments:

22
src/updaters/default.nix
View File

@ -5,24 +5,17 @@
lib,
python3,
writeText,
# dream2nix inputs
callPackageDream,
fetchers,
utils,
...
}:
let
}: let
lockUtils = utils.dreamLock;
updaters = callPackageDream ./updaters.nix {};
getUpdaterName =
{
dreamLock,
}:
let
getUpdaterName = {dreamLock}: let
lock = (utils.readDreamLock {inherit dreamLock;}).lock;
source = lockUtils.getMainPackageSource lock;
in
@ -30,20 +23,15 @@ let
or fetchers.fetchers."${source.type}".defaultUpdater
or null;
makeUpdateScript =
{
makeUpdateScript = {
dreamLock,
updater ? getUpdaterName {inherit dreamLock;},
}:
let
}: let
lock = (utils.readDreamLock {inherit dreamLock;}).lock;
source = lockUtils.getMainPackageSource lock;
updater' = updaters."${updater}";
in
updater' source;
in
{
in {
inherit getUpdaterName makeUpdateScript updaters;
}

23
src/updaters/updaters.nix
View File

@ -5,14 +5,11 @@
lib,
python3,
writeText,
# dream2nix inputs
utils,
...
}:
{
githubNewestReleaseTag =
{
}: {
githubNewestReleaseTag = {
owner,
repo,
...
@ -21,20 +18,12 @@
curl -s "https://api.github.com/repos/${owner}/${repo}/releases?per_page=1" | jq -r '.[0].tag_name'
'';
pypiNewestReleaseVersion =
{
pname,
...
}:
pypiNewestReleaseVersion = {pname, ...}:
utils.writePureShellScript [curl jq] ''
curl -s https://pypi.org/pypi/${pname}/json | jq -r '.info.version'
'';
npmNewestReleaseVersion =
{
pname,
...
}:
npmNewestReleaseVersion = {pname, ...}:
# api docs: https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#get
utils.writePureShellScript [curl jq] ''
curl -s https://registry.npmjs.com/${pname} | jq -r '."dist-tags".latest'
@ -50,8 +39,7 @@
url,
regex,
...
}:
let
}: let
reFile = writeText "regex" regex;
in
utils.writePureShellScript [curl gnugrep python3] ''
@ -59,5 +47,4 @@
| python3 -c \
'import re, sys; print(re.search(open("${reFile}").read(), sys.stdin.read()).group("ver"), end="")'
'';
}

23
src/utils/config.nix
View File

@ -1,5 +1,4 @@
let
b = builtins;
# loads attrs either from s:
@ -7,18 +6,16 @@ let
# - json string
# - attrset (no changes)
loadAttrs = input:
if b.isPath input then
b.fromJSON (b.readFile input)
else if b.isString input then
b.fromJSON input
else if b.isAttrs input then
input
else
throw "input for loadAttrs must be json file or string or attrs";
if b.isPath input
then b.fromJSON (b.readFile input)
else if b.isString input
then b.fromJSON input
else if b.isAttrs input
then input
else throw "input for loadAttrs must be json file or string or attrs";
# load dream2nix config extending with defaults
loadConfig = configInput:
let
loadConfig = configInput: let
config = loadAttrs configInput;
defaults = {
overridesDirs = [];
@ -28,8 +25,6 @@ let
};
in
defaults // config;
in
{
in {
inherit loadConfig;
}

47
src/utils/default.nix
View File

@ -13,16 +13,13 @@
stdenv,
writeScript,
writeScriptBin,
# dream2nix inputs
apps,
callPackageDream,
externalSources,
translators,
...
}:
let
}: let
b = builtins;
l = lib // builtins;
@ -38,19 +35,20 @@ let
# copied from poetry2nix
ireplace = idx: value: list: (
lib.genList
(i: if i == idx then value else (b.elemAt list i))
(i:
if i == idx
then value
else (b.elemAt list i))
(b.length list)
);
};
in
overrideUtils
// translatorUtils
// translatorUtils2
// rec {
inherit (dlib)
inherit
(dlib)
dirNames
callViaEnv
identifyGitUrl
@ -74,8 +72,7 @@ overrideUtils
toTOML = import ./toTOML.nix {inherit lib;};
# hash the contents of a path via `nix hash path`
hashPath = algo: path:
let
hashPath = algo: path: let
hashPath = runCommand "hash-${algo}" {} ''
${nix}/bin/nix --option experimental-features nix-command hash path ${path} | tr --delete '\n' > $out
'';
@ -83,8 +80,7 @@ overrideUtils
b.readFile hashPath;
# hash a file via `nix hash file`
hashFile = algo: path:
let
hashFile = algo: path: let
hashFile = runCommand "hash-${algo}" {} ''
${nix}/bin/nix --option experimental-features nix-command hash file ${path} | tr --delete '\n' > $out
'';
@ -92,7 +88,8 @@ overrideUtils
b.readFile hashFile;
# builder to create a shell script that has it's own PATH
writePureShellScript = availablePrograms: script: writeScript "script.sh" ''
writePureShellScript = availablePrograms: script:
writeScript "script.sh" ''
#!${bash}/bin/bash
set -Eeuo pipefail
@ -128,8 +125,7 @@ overrideUtils
${coreutils}/bin/rm -rf $TMPDIR
'';
extractSource =
{
extractSource = {
source,
dir ? "",
}:
@ -141,15 +137,14 @@ overrideUtils
dontInstall = true;
dontFixup = true;
unpackCmd =
if lib.hasSuffix ".tgz" source.name then
''
if lib.hasSuffix ".tgz" source.name
then ''
tar --delay-directory-restore -xf $src
# set executable flag only on directories
chmod -R +X .
''
else
null;
else null;
# sometimes tarballs do not end with .tar.??
preUnpack = ''
unpackFallback(){
@ -168,17 +163,16 @@ overrideUtils
satisfiesSemver = poetry2nixSemver.satisfiesSemver;
makeTranslateScript =
{
makeTranslateScript = {
invalidationHash,
source,
project,
}@args:
let
} @ args: let
translator =
translators.translatorsV2."${project.subsystem}".all."${project.translator}";
argsJsonFile = pkgs.writeText "translator-args.json"
argsJsonFile =
pkgs.writeText "translator-args.json"
(l.toJSON
(args
// {
@ -220,8 +214,7 @@ overrideUtils
'';
# a script that produces and dumps the dream-lock json for a given source
makePackageLockScript =
{
makePackageLockScript = {
packagesDir,
source,
translator,

120
src/utils/dream-lock.nix
View File

@ -1,57 +1,43 @@
{
lib,
# dream2nix
utils,
...
}:
let
}: let
b = builtins;
subDreamLockNames = dreamLockFile:
let
subDreamLockNames = dreamLockFile: let
dir = b.dirOf dreamLockFile;
directories = utils.listDirs dir;
dreamLockDirs =
lib.filter
(d: b.pathExists ("${dir}/${d}/dream-lock.json"))
(d: b.pathExists "${dir}/${d}/dream-lock.json")
directories;
in
dreamLockDirs;
readDreamLock =
{
dreamLock,
}@args:
let
readDreamLock = {dreamLock} @ args: let
isFile =
b.isPath dreamLock
|| b.isString dreamLock
|| lib.isDerivation dreamLock;
lockMaybeCompressed =
if isFile then
b.fromJSON (b.readFile dreamLock)
else
dreamLock;
if isFile
then b.fromJSON (b.readFile dreamLock)
else dreamLock;
lock =
if lockMaybeCompressed.decompressed or false then
lockMaybeCompressed
else
decompressDreamLock lockMaybeCompressed;
if lockMaybeCompressed.decompressed or false
then lockMaybeCompressed
else decompressDreamLock lockMaybeCompressed;
subDreamLocks =
if ! isFile then
{}
else
let
if ! isFile
then {}
else let
dir = b.dirOf dreamLock;
in
lib.genAttrs
@ -79,7 +65,8 @@ let
cyclicDependencies = lock.cyclicDependencies;
getSourceSpec = pname: version:
sources."${pname}"."${version}" or (
sources."${pname}"."${version}"
or (
throw "The source spec for ${pname}#${version} is not defined in lockfile."
);
@ -91,21 +78,18 @@ let
getCyclicDependencies = pname: version:
cyclicDependencies."${pname}"."${version}" or [];
getRoot = pname: version:
let spec = getSourceSpec pname version; in
if spec.type == "path" then
{
getRoot = pname: version: let
spec = getSourceSpec pname version;
in
if spec.type == "path"
then {
pname = spec.rootName;
version = spec.rootVersion;
}
else
{ inherit pname version; };
in
{
else {inherit pname version;};
in {
inherit lock;
interface = {
inherit
defaultPackageName
defaultPackageVersion
@ -122,7 +106,8 @@ let
};
getMainPackageSource = dreamLock:
dreamLock.sources
dreamLock
.sources
."${dreamLock._generic.defaultPackage}"
."${dreamLock._generic.packages."${dreamLock._generic.defaultPackage}"}"
// rec {
@ -131,9 +116,11 @@ let
};
getSource = fetchedSources: pname: version:
if fetchedSources ? "${pname}"."${version}"
&& fetchedSources."${pname}"."${version}" != "unknown" then
fetchedSources."${pname}"."${version}"
if
fetchedSources
? "${pname}"."${version}"
&& fetchedSources."${pname}"."${version}" != "unknown"
then fetchedSources."${pname}"."${version}"
else
throw ''
The source for ${pname}#${version} is not defined.
@ -150,23 +137,27 @@ let
'';
# generate standalone dreamLock for a depenndency of an existing dreamLock
getSubDreamLock = dreamLock: name: version:
let
getSubDreamLock = dreamLock: name: version: let
lock = (readDreamLock {inherit dreamLock;}).lock;
in
lock // {
_generic = lock._generic // {
lock
// {
_generic =
lock._generic
// {
defaultPackage = name;
packages = lock._generic.packages // {
packages =
lock._generic.packages
// {
"${name}" = version;
};
};
};
injectDependencies = dreamLock: inject:
if inject == {} then dreamLock else
let
if inject == {}
then dreamLock
else let
lock = (readDreamLock {inherit dreamLock;}).lock;
oldDependencyGraph = lock.dependencies;
@ -181,7 +172,6 @@ let
oldDependencyGraph
inject
];
in
lib.recursiveUpdate lock {
dependencies = newDependencyGraph;
@ -209,8 +199,7 @@ let
versions)
decompGraph;
decompressDreamLock = comp:
let
decompressDreamLock = comp: let
dependencyGraphDecomp =
decompressDependencyGraph (comp.dependencies or {});
@ -229,16 +218,15 @@ let
lib.recursiveUpdate
emptyDependencyGraph
dependencyGraphDecomp;
in
comp // {
comp
// {
decompressed = true;
cyclicDependencies = cyclicDependencies;
dependencies = dependencyGraph;
};
compressDreamLock = uncomp:
let
compressDreamLock = uncomp: let
dependencyGraphComp =
compressDependencyGraph
uncomp.dependencies;
@ -257,26 +245,22 @@ let
versions)
dependencyGraphComp);
in
(b.removeAttrs uncomp [ "decompressed" ]) // {
(b.removeAttrs uncomp ["decompressed"])
// {
inherit cyclicDependencies;
dependencies = dependencyGraph;
};
toJSON = dreamLock:
let
toJSON = dreamLock: let
lock =
if dreamLock.decompressed or false then
compressDreamLock dreamLock
else
dreamLock;
if dreamLock.decompressed or false
then compressDreamLock dreamLock
else dreamLock;
json = b.toJSON lock;
in
json;
in
{
in {
inherit
compressDreamLock
decompressDreamLock

1
src/utils/external-dir.nix
View File

@ -4,7 +4,6 @@
externalSources,
externalPaths,
}:
pkgs.runCommand "dream2nix-external-dir" {}
(lib.concatStringsSep "\n"
(lib.mapAttrsToList

94
src/utils/override.nix
View File

@ -1,16 +1,12 @@
{
lib,
# dream2nix
utils,
...
}:
let
}: let
b = builtins;
loadOverridesDirs = overridesDirs: pkgs:
let
loadOverridesDirs = overridesDirs: pkgs: let
loadOverrides = dir:
lib.genAttrs (utils.dirNames dir) (name:
import (dir + "/${name}") {
@ -54,36 +50,31 @@ let
```
'';
getOverrideFunctionArgs = function:
let
getOverrideFunctionArgs = function: let
funcArgs = lib.functionArgs function;
in
if funcArgs != {} then
b.attrNames funcArgs
if funcArgs != {}
then b.attrNames funcArgs
else
(
function (old: {passthru.funcArgs = lib.attrNames old;})
).funcArgs;
)
.funcArgs;
applyOverridesToPackage =
{
applyOverridesToPackage = {
conditionalOverrides,
pkg,
pname,
outputs,
}:
let
}: let
# if condition is unset, it will be assumed true
evalCondition = condOverride: pkg:
if condOverride ? _condition then
condOverride._condition pkg
else
true;
if condOverride ? _condition
then condOverride._condition pkg
else true;
# filter the overrides by the package name and conditions
overridesToApply =
let
overridesToApply = let
# TODO: figure out if regex names will be useful
regexOverrides = {};
# lib.filterAttrs
@ -101,52 +92,50 @@ let
overridesListForPackage =
lib.mapAttrsToList
(_name: data:
(
_name: data:
data // {inherit _name;}
)
overridesForPackage;
in
(lib.filter
in (lib.filter
(condOverride: evalCondition condOverride pkg)
overridesListForPackage);
# apply single attribute override
applySingleAttributeOverride = oldVal: functionOrValue:
if b.isFunction functionOrValue then
if lib.functionArgs functionOrValue == {} then
functionOrValue oldVal
if b.isFunction functionOrValue
then
if lib.functionArgs functionOrValue == {}
then functionOrValue oldVal
else
functionOrValue {
old = oldVal;
inherit outputs;
}
else
functionOrValue;
else functionOrValue;
# helper to apply one conditional override
# the condition is not evaluated anymore here
applyOneOverride = pkg: condOverride:
let
applyOneOverride = pkg: condOverride: let
base_derivation =
if condOverride ? _replace then
if lib.isFunction condOverride._replace then
condOverride._replace pkg
else if lib.isDerivation condOverride._replace then
condOverride._replace
if condOverride ? _replace
then
if lib.isFunction condOverride._replace
then condOverride._replace pkg
else if lib.isDerivation condOverride._replace
then condOverride._replace
else
throw
("override attr ${pname}.${condOverride._name}._replace"
+ " must either be a derivation or a function")
else
pkg;
else pkg;
overrideFuncs =
lib.mapAttrsToList
(funcName: func: {inherit funcName func;})
(lib.filterAttrs (n: v: lib.hasPrefix "override" n) condOverride);
singleArgOverrideFuncs =
let
singleArgOverrideFuncs = let
availableFunctions =
lib.mapAttrs
(funcName: func: getOverrideFunctionArgs func)
@ -154,26 +143,23 @@ let
(funcName: func: lib.hasPrefix "override" funcName)
base_derivation);
getOverrideFuncNameForAttrName = attrName:
let
getOverrideFuncNameForAttrName = attrName: let
applicableFuncs =
lib.attrNames
(lib.filterAttrs
(funcName: args: b.elem attrName args)
availableFunctions);
in
if b.length applicableFuncs == 0 then
"overrideAttrs"
else if b.length applicableFuncs > 1 then
throwErrorUnclearAttributeOverride pname condOverride._name attrName
else
b.elemAt applicableFuncs 0;
if b.length applicableFuncs == 0
then "overrideAttrs"
else if b.length applicableFuncs > 1
then throwErrorUnclearAttributeOverride pname condOverride._name attrName
else b.elemAt applicableFuncs 0;
attributeOverrides =
lib.filterAttrs
(n: v: ! lib.hasPrefix "override" n && ! lib.hasPrefix "_" n)
condOverride;
in
lib.mapAttrsToList
(attrName: funcOrValue: {
@ -181,13 +167,11 @@ let
func = oldAttrs: {"${attrName}" = funcOrValue;};
})
attributeOverrides;
in
b.foldl'
(pkg: overrideFunc:
pkg."${overrideFunc.funcName}"
(old:
let
(old: let
updateAttrsFuncs = overrideFunc.func old;
in
lib.mapAttrs
@ -202,8 +186,6 @@ let
(pkg: condOverride: applyOneOverride pkg condOverride)
pkg
overridesToApply);
in
{
in {
inherit applyOverridesToPackage loadOverridesDirs;
}

21
src/utils/toTOML.nix
View File

@ -1,5 +1,4 @@
{ lib }:
let
{lib}: let
inherit
(lib)
length
@ -31,8 +30,7 @@ let
ty = tomlTy v;
in
if ty == "set"
then
let
then let
vals =
mapAttrsToList
(k': v': "${quoteKey k'} = ${outputValInner v'}")
@ -49,8 +47,7 @@ let
else if ty == "string"
then quoteString v
else if ty == "list" || ty == "list_of_attrs"
then
let
then let
vals = map quoteString v;
valsStr = concatStringsSep ", " vals;
in "[ ${valsStr} ]"
@ -62,8 +59,7 @@ let
ty = tomlTy v;
in
if ty == "set"
then
let
then let
vals =
mapAttrsToList
(k': v': "${quoteKey k'} = ${outputValInner v'}")
@ -88,8 +84,7 @@ let
)
v
else if ty == "list"
then
let
then let
vals = map quoteString v;
valsStr = concatStringsSep ", " vals;
in ["${quoteKey k} = [ ${valsStr} ]"]
@ -115,8 +110,7 @@ let
then
if length x == 0
then "list"
else
let
else let
ty = typeOf (elemAt x 0);
in
#assert (all (v: typeOf v == ty) x);
@ -127,7 +121,8 @@ let
toTOML = attrs:
assert (typeOf attrs == "set"); let
byTy = lib.foldl
byTy =
lib.foldl
(
acc: x: let
ty = tomlTy x.v;

136
src/utils/translator.nix
View File

@ -1,28 +1,28 @@
{
lib,
# dream2nix
fetchers,
dlib,
...
}:
let
}: let
b = builtins;
overrideWarning = fields: args:
lib.filterAttrs (name: _:
lib.filterAttrs (
name: _:
if lib.any (field: name == field) fields
then lib.warn ''
then
lib.warn ''
you are trying to pass a "${name}" key from your source
constructor, this will be overrided with a value passed
by dream2nix.
'' false
''
false
else true
) args;
)
args;
simpleTranslate = func:
let
simpleTranslate = func: let
final =
func
{
@ -33,8 +33,10 @@ let
getDepByNameVer = name: version:
final.allDependencies."${name}"."${version}" or null;
dependenciesByOriginalID = b.foldl'
(result: pkgData: lib.recursiveUpdate result {
dependenciesByOriginalID =
b.foldl'
(result: pkgData:
lib.recursiveUpdate result {
"${final.getOriginalID pkgData}" = pkgData;
})
{}
@ -44,8 +46,7 @@ let
dreamLockData = magic final;
magic =
{
magic = {
# values
defaultPackage,
inputData,
@ -55,23 +56,21 @@ let
subsystemName,
subsystemAttrs,
translatorName,
# functions
serializePackages,
getName,
getVersion,
getSourceType,
sourceConstructors,
createMissingSource ?
(name: version: throw "Cannot find source for ${name}:${version}"),
createMissingSource ? (name: version: throw "Cannot find source for ${name}:${version}"),
getDependencies ? null,
getOriginalID ? null,
mainPackageSource ? {type = "unknown";},
}:
let
allDependencies = b.foldl'
(result: pkgData: lib.recursiveUpdate result {
}: let
allDependencies =
b.foldl'
(result: pkgData:
lib.recursiveUpdate result {
"${getName pkgData}" = {
"${getVersion pkgData}" = pkgData;
};
@ -79,15 +78,15 @@ let
{}
serializedPackagesList;
sources = b.foldl'
(result: pkgData:
let
sources =
b.foldl'
(result: pkgData: let
pkgName = getName pkgData;
pkgVersion = getVersion pkgData;
in lib.recursiveUpdate result {
in
lib.recursiveUpdate result {
"${pkgName}" = {
"${pkgVersion}" =
let
"${pkgVersion}" = let
type = getSourceType pkgData;
constructedArgs = sourceConstructors."${type}" pkgData;
@ -95,13 +94,12 @@ let
constructedArgsKeep =
overrideWarning ["pname" "version"] constructedArgs;
constructedSource =
fetchers.constructSource (constructedArgsKeep // {
constructedSource = fetchers.constructSource (constructedArgsKeep
// {
inherit type;
pname = pkgName;
version = pkgVersion;
});
in
b.removeAttrs constructedSource ["pname" "version"];
};
@ -109,24 +107,26 @@ let
{}
serializedPackagesList;
dependencyGraph =
let
dependencyGraph = let
depGraph =
(lib.mapAttrs
lib.mapAttrs
(name: versions:
lib.mapAttrs
(version: pkgData: getDependencies pkgData)
versions)
allDependencies);
allDependencies;
in
depGraph // {
"${defaultPackage}" = depGraph."${defaultPackage}" or {} // {
depGraph
// {
"${defaultPackage}" =
depGraph."${defaultPackage}"
or {}
// {
"${packages."${defaultPackage}"}" = mainPackageDependencies;
};
};
allDependencyKeys =
let
allDependencyKeys = let
depsWithDuplicates =
lib.flatten
(lib.flatten
@ -140,18 +140,18 @@ let
lib.flatten
(lib.forEach allDependencyKeys
(dep:
if sources ? "${dep.name}"."${dep.version}" then
[]
else
dep));
if sources ? "${dep.name}"."${dep.version}"
then []
else dep));
generatedSources =
if missingDependencies == [] then
{}
if missingDependencies == []
then {}
else
lib.listToAttrs
(b.map
(dep: lib.nameValuePair
(dep:
lib.nameValuePair
"${dep.name}"
{
"${dep.version}" =
@ -165,9 +165,7 @@ let
cyclicDependencies =
# TODO: inefficient! Implement some kind of early cutoff
let
findCycles = node: prevNodes: cycles:
let
findCycles = node: prevNodes: cycles: let
children = dependencyGraph."${node.name}"."${node.version}";
cyclicChildren =
@ -182,17 +180,19 @@ let
cycles' =
cycles
++
(b.map (child: { from = node; to = child; }) cyclicChildren);
++ (b.map (child: {
from = node;
to = child;
})
cyclicChildren);
# use set for efficient lookups
prevNodes' =
prevNodes
// {"${node.name}#${node.version}" = null;};
in
if nonCyclicChildren == [] then
cycles'
if nonCyclicChildren == []
then cycles'
else
lib.flatten
(b.map
@ -206,8 +206,7 @@ let
[];
in
b.foldl'
(cycles: cycle:
(
(cycles: cycle: (
let
existing =
cycles."${cycle.from.name}"."${cycle.from.version}"
@ -216,28 +215,27 @@ let
reverse =
cycles."${cycle.to.name}"."${cycle.to.version}"
or [];
in
# if edge or reverse edge already in cycles, do nothing
if b.elem cycle.from reverse
|| b.elem cycle.to existing then
cycles
if
b.elem cycle.from reverse
|| b.elem cycle.to existing
then cycles
else
lib.recursiveUpdate
cycles
{
"${cycle.from.name}"."${cycle.from.version}" =
existing ++ [cycle.to];
}))
}
))
{}
cyclesList;
in
{
decompressed = true;
_generic =
{
_generic = {
inherit
defaultPackage
location
@ -254,17 +252,11 @@ let
sources = allSources;
}
//
(lib.optionalAttrs
// (lib.optionalAttrs
(getDependencies != null)
{dependencies = dependencyGraph;});
in
dreamLockData;
in
{
in {
inherit simpleTranslate;
}

78
src/utils/translator2.nix
View File

@ -2,13 +2,10 @@
dlib,
lib,
...
}:
let
}: let
l = lib // builtins;
simpleTranslate2 = func:
let
simpleTranslate2 = func: let
final =
func
{
@ -41,7 +38,8 @@ let
l.map
(finalObj:
if l.any (field: ! finalObj ? "${field}") expectedFields
then throw
then
throw
''
Translator ${final.translatorName} failed.
The following object does not contain all required fields:
@ -65,11 +63,9 @@ let
finalObjects')
final.keys;
dreamLockData = magic final;
magic =
{
magic = {
defaultPackage,
exportedPackages,
extractors,
@ -81,10 +77,9 @@ let
subsystemName,
subsystemAttrs ? {},
translatorName,
}:
let
allDependencies = l.foldl'
}: let
allDependencies =
l.foldl'
(result: finalObj:
lib.recursiveUpdate
result
@ -104,25 +99,27 @@ let
versions)
allDependencies;
dependencyGraph =
let
dependencyGraph = let
depGraph =
(lib.mapAttrs
lib.mapAttrs
(name: versions:
lib.mapAttrs
(version: finalObj: finalObj.dependencies)
versions)
allDependencies);
allDependencies;
in
# add extraDependencies to dependency graph
l.foldl'
(all: new:
all // {
all
// {
"${new.name}" =
all."${new.name}" or {}
all."${new.name}"
or {}
// {
"${new.version}" =
all."${new.name}"."${new.version}" or []
all."${new.name}"."${new.version}"
or []
++ new.dependencies;
};
})
@ -142,9 +139,7 @@ let
exportedPackages;
};
findCycles = node: prevNodes: cycles:
let
findCycles = node: prevNodes: cycles: let
children =
depGraphWithFakeRoot."${node.name}"."${node.version}";
@ -160,17 +155,19 @@ let
cycles' =
cycles
++
(l.map (child: { from = node; to = child; }) cyclicChildren);
++ (l.map (child: {
from = node;
to = child;
})
cyclicChildren);
# use set for efficient lookups
prevNodes' =
prevNodes
// {"${node.name}#${node.version}" = null;};
in
if nonCyclicChildren == [] then
cycles'
if nonCyclicChildren == []
then cycles'
else
lib.flatten
(l.map
@ -186,8 +183,7 @@ let
[];
in
l.foldl'
(cycles: cycle:
(
(cycles: cycle: (
let
existing =
cycles."${cycle.from.name}"."${cycle.from.version}"
@ -196,28 +192,27 @@ let
reverse =
cycles."${cycle.to.name}"."${cycle.to.version}"
or [];
in
# if edge or reverse edge already in cycles, do nothing
if l.elem cycle.from reverse
|| l.elem cycle.to existing then
cycles
if
l.elem cycle.from reverse
|| l.elem cycle.to existing
then cycles
else
lib.recursiveUpdate
cycles
{
"${cycle.from.name}"."${cycle.from.version}" =
existing ++ [cycle.to];
}))
}
))
{}
cyclesList;
in
{
decompressed = true;
_generic =
{
_generic = {
inherit
defaultPackage
location
@ -233,13 +228,8 @@ let
inherit cyclicDependencies sources;
}
// {dependencies = dependencyGraph;};
in
dreamLockData;
in
{
in {
inherit simpleTranslate2;
}

12
tests/impure/default.nix
View File

@ -2,14 +2,11 @@
async,
coreutils,
lib,
# dream2nix
callPackageDream,
utils,
...
}:
let
}: let
l = lib // builtins;
allTestFiles =
@ -20,10 +17,11 @@ let
allTests =
l.map
(file: callPackageDream ("${./.}/${file}") {})
(file: callPackageDream "${./.}/${file}" {})
allTestFiles;
executeAll = utils.writePureShellScript
executeAll =
utils.writePureShellScript
[
async
coreutils
@ -39,7 +37,5 @@ let
async -s=$S wait
rm $S
'';
in
executeAll

6
tests/impure/test_combined_fetching.nix
View File

@ -1,17 +1,13 @@
{
lib,
# dream2nix
apps,
utils,
...
}:
let
}: let
l = lib // builtins;
cli = apps.cli.program;
in
utils.writePureShellScript
[]

6
tests/impure/test_go.nix
View File

@ -1,17 +1,13 @@
{
lib,
# dream2nix
apps,
utils,
...
}:
let
}: let
l = lib // builtins;
cli = apps.cli.program;
in
utils.writePureShellScript
[]

6
tests/impure/test_package-json.nix
View File

@ -1,17 +1,13 @@
{
lib,
# dream2nix
apps,
utils,
...
}:
let
}: let
l = lib // builtins;
cli = apps.cli.program;
in
utils.writePureShellScript
[]

6
tests/impure/test_package-lock.nix
View File

@ -1,17 +1,13 @@
{
lib,
# dream2nix
apps,
utils,
...
}:
let
}: let
l = lib // builtins;
cli = apps.cli.program;
in
utils.writePureShellScript
[]

6
tests/impure/test_rust.nix
View File

@ -1,17 +1,13 @@
{
lib,
# dream2nix
apps,
utils,
...
}:
let
}: let
l = lib // builtins;
cli = apps.cli.program;
in
utils.writePureShellScript
[]

6
tests/impure/test_yarn-lock-prettier.nix
View File

@ -1,17 +1,13 @@
{
lib,
# dream2nix
apps,
utils,
...
}:
let
}: let
l = lib // builtins;
cli = apps.cli.program;
in
utils.writePureShellScript
[]

8
tests/pure/default.nix
View File

@ -2,9 +2,7 @@
lib ? pkgs.lib,
pkgs ? import <nixpkgs> {},
dream2nix ? import ./src {inherit pkgs;},
}:
let
}: let
l = pkgs.lib // builtins;
buildProjectsTests = import ./projects.nix {
@ -14,8 +12,6 @@ let
otherTests = import ./other {
inherit lib pkgs dream2nix;
};
in
buildProjectsTests
//
otherTests
// otherTests

15
tests/pure/other/default.nix
View File

@ -2,20 +2,17 @@
lib ? pkgs.lib,
pkgs ? import <nixpkgs> {},
dream2nix ? import ./src {inherit pkgs;},
}:
let
}: let
l = pkgs.lib // builtins;
fetchAggrgatedGithub =
dream2nix.utils.toDrv
(dream2nix.fetchSources {
dreamLock = ./prettier-github-aggregated.json;
}).fetchedSources.prettier."2.4.1";
in
{
})
.fetchedSources
.prettier
."2.4.1";
in {
inherit fetchAggrgatedGithub;
}

17
tests/pure/projects.nix
View File

@ -2,18 +2,14 @@
lib ? pkgs.lib,
pkgs ? import <nixpkgs> {},
dream2nix ? import ./src {inherit pkgs;},
}:
let
}: let
lib = pkgs.lib // builtins;
makeTest =
{
makeTest = {
name,
source,
cmds,
}:
let
}: let
outputs = dream2nix.makeOutputs {
inherit source;
};
@ -28,13 +24,11 @@ let
url = "https://github.com/prettier/prettier/tarball/2.4.1";
sha256 = "19b37qakhlsnr2n5bgv83aih5npgzbad1d2p2rs3zbq5syqbxdyi";
};
cmds = outputs:
let
cmds = outputs: let
prettier = outputs.defaultPackage.overrideAttrs (old: {
dontBuild = true;
});
in
[
in [
"${prettier}/bin/prettier --version | grep -q 2.4.1 && mkdir $out"
];
};
@ -44,6 +38,5 @@ let
lib.mapAttrs
(name: args: makeTest (args // {inherit name;}))
projects;
in
allTests

6
tests/unit/default.nix
View File

@ -1,17 +1,13 @@
{
self,
lib,
nix,
python3,
utils,
dream2nixWithExternals,
...
}:
let
}: let
l = lib // builtins;
in
utils.writePureShellScript
[