nix-community/dream2nix
1
1
Fork 0
mirror of https://github.com/nix-community/dream2nix.git synced 2024-10-26 15:23:14 +03:00
Code Issues Projects Releases Wiki Activity

treewide: format with alejandra 1.0.0

Browse Source 
closes #84
This commit is contained in:
DavHau 2022-03-07 17:12:07 +07:00
parent 7641236307
commit 076a1c9aff
69 changed files with 5774 additions and 6468 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
ci.nix
View File

@ -1,22 +1,17 @@
let let
b = builtins; b = builtins;
flakeCompatSrc = b.fetchurl "https://raw.githubusercontent.com/edolstra/flake-compat/12c64ca55c1014cdc1b16ed5a804aa8576601ff2/default.nix"; flakeCompatSrc = b.fetchurl "https://raw.githubusercontent.com/edolstra/flake-compat/12c64ca55c1014cdc1b16ed5a804aa8576601ff2/default.nix";
flake = (import flakeCompatSrc { src = ./.; }).defaultNix; flake = (import flakeCompatSrc {src = ./.;}).defaultNix;
pkgs = import flake.inputs.nixpkgs {}; pkgs = import flake.inputs.nixpkgs {};
recurseIntoAll = b.mapAttrs (name: val: pkgs.recurseIntoAttrs val); recurseIntoAll = b.mapAttrs (name: val: pkgs.recurseIntoAttrs val);
in in
# { # {
# inherit flake; # inherit flake;
# } # }
# // (recurseIntoAll {
# // (recurseIntoAll { # checks = flake.checks.x86_64-linux;
# })
# checks = flake.checks.x86_64-linux; # hercules ci's nix version cannot fetch submodules and crashes
{
# }) inherit (pkgs) hello;
}
# hercules ci's nix version cannot fetch submodules and crashes
{
inherit (pkgs) hello;
}

411
flake.nix
View File

@ -14,22 +14,36 @@
flake-utils-pre-commit.url = "github:numtide/flake-utils"; flake-utils-pre-commit.url = "github:numtide/flake-utils";
pre-commit-hooks.inputs.flake-utils.follows = "flake-utils-pre-commit"; pre-commit-hooks.inputs.flake-utils.follows = "flake-utils-pre-commit";
### framework dependencies ### framework dependencies
# required for builder go/gomod2nix # required for builder go/gomod2nix
gomod2nix = { url = "github:tweag/gomod2nix"; flake = false; }; gomod2nix = {
url = "github:tweag/gomod2nix";
flake = false;
};
# required for translator pip # required for translator pip
mach-nix = { url = "mach-nix"; flake = false; }; mach-nix = {
url = "mach-nix";
flake = false;
};
# required for builder nodejs/node2nix # required for builder nodejs/node2nix
node2nix = { url = "github:svanderburg/node2nix"; flake = false; }; node2nix = {
url = "github:svanderburg/node2nix";
flake = false;
};
# required for utils.satisfiesSemver # required for utils.satisfiesSemver
poetry2nix = { url = "github:nix-community/poetry2nix/1.21.0"; flake = false; }; poetry2nix = {
url = "github:nix-community/poetry2nix/1.21.0";
flake = false;
};
# required for builder rust/crane # required for builder rust/crane
crane = { url = "github:ipetkov/crane"; flake = false; }; crane = {
url = "github:ipetkov/crane";
flake = false;
};
}; };
outputs = { outputs = {
@ -43,84 +57,85 @@
pre-commit-hooks, pre-commit-hooks,
crane, crane,
... ...
}@inp: } @ inp: let
let b = builtins;
l = lib // builtins;
b = builtins; lib = nixpkgs.lib;
l = lib // builtins;
lib = nixpkgs.lib; # dream2nix lib (system independent utils)
dlib = import ./src/lib {inherit lib;};
# dream2nix lib (system independent utils) supportedSystems = ["x86_64-linux" "x86_64-darwin" "aarch64-darwin"];
dlib = import ./src/lib { inherit lib; };
supportedSystems = [ "x86_64-linux" "x86_64-darwin" "aarch64-darwin" ]; forSystems = systems: f:
lib.genAttrs systems
(system: f system nixpkgs.legacyPackages.${system});
forSystems = systems: f: lib.genAttrs systems forAllSystems = forSystems supportedSystems;
(system: f system nixpkgs.legacyPackages.${system});
forAllSystems = forSystems supportedSystems; # To use dream2nix in non-flake + non-IFD enabled repos, the source code of dream2nix
# must be installed into these repos (using nix run dream2nix#install).
# The problem is, all of dream2nix' dependecies need to be installed as well.
# Therefore 'externalPaths' contains all relevant files of external projects
# which dream2nix depends on. Exactly these files will be installed.
externalPaths = {
mach-nix = [
"lib/extractor/default.nix"
"lib/extractor/distutils.patch"
"lib/extractor/setuptools.patch"
"LICENSE"
];
node2nix = [
"nix/node-env.nix"
"LICENSE"
];
poetry2nix = [
"semver.nix"
"LICENSE"
];
crane = [
"lib/buildDepsOnly.nix"
"lib/buildPackage.nix"
"lib/cargoBuild.nix"
"lib/cleanCargoToml.nix"
"lib/findCargoFiles.nix"
"lib/mkCargoDerivation.nix"
"lib/mkDummySrc.nix"
"lib/writeTOML.nix"
"pkgs/configureCargoCommonVarsHook.sh"
"pkgs/configureCargoVendoredDepsHook.sh"
"pkgs/installFromCargoBuildLogHook.sh"
"pkgs/inheritCargoArtifactsHook.sh"
"pkgs/installCargoArtifactsHook.sh"
"pkgs/remapSourcePathPrefixHook.sh"
"LICENSE"
];
};
# To use dream2nix in non-flake + non-IFD enabled repos, the source code of dream2nix # create a directory containing the files listed in externalPaths
# must be installed into these repos (using nix run dream2nix#install). makeExternalDir = import ./src/utils/external-dir.nix;
# The problem is, all of dream2nix' dependecies need to be installed as well.
# Therefore 'externalPaths' contains all relevant files of external projects
# which dream2nix depends on. Exactly these files will be installed.
externalPaths = {
mach-nix = [
"lib/extractor/default.nix"
"lib/extractor/distutils.patch"
"lib/extractor/setuptools.patch"
"LICENSE"
];
node2nix = [
"nix/node-env.nix"
"LICENSE"
];
poetry2nix = [
"semver.nix"
"LICENSE"
];
crane = [
"lib/buildDepsOnly.nix"
"lib/buildPackage.nix"
"lib/cargoBuild.nix"
"lib/cleanCargoToml.nix"
"lib/findCargoFiles.nix"
"lib/mkCargoDerivation.nix"
"lib/mkDummySrc.nix"
"lib/writeTOML.nix"
"pkgs/configureCargoCommonVarsHook.sh"
"pkgs/configureCargoVendoredDepsHook.sh"
"pkgs/installFromCargoBuildLogHook.sh"
"pkgs/inheritCargoArtifactsHook.sh"
"pkgs/installCargoArtifactsHook.sh"
"pkgs/remapSourcePathPrefixHook.sh"
"LICENSE"
];
};
# create a directory containing the files listed in externalPaths externalDirFor = forAllSystems (system: pkgs:
makeExternalDir = import ./src/utils/external-dir.nix; makeExternalDir {
externalDirFor = forAllSystems (system: pkgs: makeExternalDir {
inherit externalPaths externalSources pkgs; inherit externalPaths externalSources pkgs;
}); });
# An interface to access files of external projects. # An interface to access files of external projects.
# This implementation accesses the flake inputs directly, # This implementation accesses the flake inputs directly,
# but if dream2nix is used without flakes, it defaults # but if dream2nix is used without flakes, it defaults
# to another implementation of that function which # to another implementation of that function which
# uses the installed external paths instead (see default.nix) # uses the installed external paths instead (see default.nix)
externalSources = externalSources =
lib.genAttrs lib.genAttrs
(lib.attrNames externalPaths) (lib.attrNames externalPaths)
(inputName: inp."${inputName}"); (inputName: inp."${inputName}");
overridesDirs = [ "${./overrides}" ]; overridesDirs = ["${./overrides}"];
# system specific dream2nix api # system specific dream2nix api
dream2nixFor = forAllSystems (system: pkgs: import ./src rec { dream2nixFor = forAllSystems (system: pkgs:
import ./src rec {
externalDir = externalDirFor."${system}"; externalDir = externalDirFor."${system}";
inherit dlib externalPaths externalSources lib pkgs; inherit dlib externalPaths externalSources lib pkgs;
config = { config = {
@ -128,7 +143,8 @@
}; };
}); });
pre-commit-check = forAllSystems (system: pkgs: pre-commit-check = forAllSystems (
system: pkgs:
pre-commit-hooks.lib.${system}.run { pre-commit-hooks.lib.${system}.run {
src = ./.; src = ./.;
hooks = { hooks = {
@ -144,139 +160,144 @@
}; };
}; };
} }
); );
in {
# System independent dream2nix api.
# Similar to drem2nixFor but will require 'system(s)' or 'pkgs' as an argument.
# Produces flake-like output schema.
lib =
(import ./src/lib.nix {
inherit dlib externalPaths externalSources overridesDirs lib;
nixpkgsSrc = "${nixpkgs}";
})
# system specific dream2nix library
// (forAllSystems (system: pkgs: dream2nixFor."${system}"));
in # with project discovery enabled
{ lib2 = import ./src/libV2.nix {
# System independent dream2nix api. inherit dlib externalPaths externalSources overridesDirs lib;
# Similar to drem2nixFor but will require 'system(s)' or 'pkgs' as an argument. nixpkgsSrc = "${nixpkgs}";
# Produces flake-like output schema. };
lib = (import ./src/lib.nix {
inherit dlib externalPaths externalSources overridesDirs lib;
nixpkgsSrc = "${nixpkgs}";
})
# system specific dream2nix library
// (forAllSystems (system: pkgs: dream2nixFor."${system}"));
# with project discovery enabled # the dream2nix cli to be used with 'nix run dream2nix'
lib2 = (import ./src/libV2.nix { defaultApp =
inherit dlib externalPaths externalSources overridesDirs lib; forAllSystems (system: pkgs: self.apps."${system}".dream2nix);
nixpkgsSrc = "${nixpkgs}";
});
# the dream2nix cli to be used with 'nix run dream2nix' # all apps including cli, install, etc.
defaultApp = apps = forAllSystems (
forAllSystems (system: pkgs: self.apps."${system}".dream2nix); system: pkgs:
dream2nixFor."${system}".apps.flakeApps
// {
tests-impure.type = "app";
tests-impure.program =
b.toString
(dream2nixFor."${system}".callPackageDream ./tests/impure {});
# all apps including cli, install, etc. tests-unit.type = "app";
apps = forAllSystems (system: pkgs: tests-unit.program =
dream2nixFor."${system}".apps.flakeApps // { b.toString
tests-impure.type = "app"; (dream2nixFor."${system}".callPackageDream ./tests/unit {
tests-impure.program = b.toString inherit self;
(dream2nixFor."${system}".callPackageDream ./tests/impure {}); });
tests-unit.type = "app"; tests-all.type = "app";
tests-unit.program = b.toString tests-all.program =
(dream2nixFor."${system}".callPackageDream ./tests/unit { l.toString
inherit self; (dream2nixFor.${system}.utils.writePureShellScript
}); [
alejandra.defaultPackage.${system}
pkgs.coreutils
pkgs.gitMinimal
pkgs.nix
]
''
echo "running unit tests"
${self.apps.${system}.tests-unit.program}
tests-all.type = "app"; echo "running impure CLI tests"
tests-all.program = l.toString ${self.apps.${system}.tests-impure.program}
(dream2nixFor.${system}.utils.writePureShellScript
[
alejandra.defaultPackage.${system}
pkgs.coreutils
pkgs.gitMinimal
pkgs.nix
]
''
echo "running unit tests"
${self.apps.${system}.tests-unit.program}
echo "running impure CLI tests" echo "running nix flake check"
${self.apps.${system}.tests-impure.program} cd $WORKDIR
nix flake check
echo "running nix flake check"
cd $WORKDIR
nix flake check
'');
# passes through extra flags to treefmt
format.type = "app";
format.program = l.toString
(pkgs.writeScript "format" ''
export PATH="${alejandra.defaultPackage.${system}}/bin"
${pkgs.treefmt}/bin/treefmt "$@"
''); '');
}
);
# a dev shell for working on dream2nix # passes through extra flags to treefmt
# use via 'nix develop . -c $SHELL' format.type = "app";
devShell = forAllSystems (system: pkgs: pkgs.mkShell { format.program =
l.toString
(pkgs.writeScript "format" ''
export PATH="${alejandra.defaultPackage.${system}}/bin"
${pkgs.treefmt}/bin/treefmt "$@"
'');
}
);
buildInputs = # a dev shell for working on dream2nix
(with pkgs; [ # use via 'nix develop . -c $SHELL'
nix devShell = forAllSystems (system: pkgs:
treefmt pkgs.mkShell {
]) buildInputs =
++ [ (with pkgs; [
alejandra.defaultPackage."${system}" nix
] treefmt
# using linux is highly recommended as cntr is amazing for debugging builds ])
++ lib.optionals pkgs.stdenv.isLinux [ pkgs.cntr ]; ++ [
alejandra.defaultPackage."${system}"
]
# using linux is highly recommended as cntr is amazing for debugging builds
++ lib.optionals pkgs.stdenv.isLinux [pkgs.cntr];
shellHook = shellHook =
# TODO: enable this once code base is formatted
# self.checks.${system}.pre-commit-check.shellHook
''
export NIX_PATH=nixpkgs=${nixpkgs}
export d2nExternalDir=${externalDirFor."${system}"}
export dream2nixWithExternals=${dream2nixFor."${system}".dream2nixWithExternals}
if [ -e ./overrides ]; then
export d2nOverridesDir=$(realpath ./overrides)
else
export d2nOverridesDir=${./overrides}
echo -e "\nManually execute 'export d2nOverridesDir={path to your dream2nix overrides dir}'"
fi
if [ -e ../dream2nix ]; then
export dream2nixWithExternals=$(realpath ./src)
else
export dream2nixWithExternals=${./src}
echo -e "\nManually execute 'export dream2nixWithExternals={path to your dream2nix checkout}'"
fi
'';
});
checks = l.recursiveUpdate
(forAllSystems (system: pkgs:
(import ./tests/pure {
inherit lib pkgs;
dream2nix = dream2nixFor."${system}";
})))
{}
# TODO: enable this once code base is formatted # TODO: enable this once code base is formatted
# (forAllSystems (system: pkgs:{ # self.checks.${system}.pre-commit-check.shellHook
# pre-commit-check = ''
# pre-commit-hooks.lib.${system}.run { export NIX_PATH=nixpkgs=${nixpkgs}
# src = ./.; export d2nExternalDir=${externalDirFor."${system}"}
# hooks = { export dream2nixWithExternals=${dream2nixFor."${system}".dream2nixWithExternals}
# treefmt = {
# enable = true; if [ -e ./overrides ]; then
# name = "treefmt"; export d2nOverridesDir=$(realpath ./overrides)
# pass_filenames = false; else
# entry = l.toString (pkgs.writeScript "treefmt" '' export d2nOverridesDir=${./overrides}
# #!${pkgs.bash}/bin/bash echo -e "\nManually execute 'export d2nOverridesDir={path to your dream2nix overrides dir}'"
# export PATH="$PATH:${alejandra.defaultPackage.${system}}/bin" fi
# ${pkgs.treefmt}/bin/treefmt --fail-on-change
# ''); if [ -e ../dream2nix ]; then
# }; export dream2nixWithExternals=$(realpath ./src)
# }; else
# }; export dream2nixWithExternals=${./src}
# })) echo -e "\nManually execute 'export dream2nixWithExternals={path to your dream2nix checkout}'"
; fi
}; '';
});
checks =
l.recursiveUpdate
(forAllSystems (system: pkgs: (import ./tests/pure {
inherit lib pkgs;
dream2nix = dream2nixFor."${system}";
})))
{}
# TODO: enable this once code base is formatted
# (forAllSystems (system: pkgs:{
# pre-commit-check =
# pre-commit-hooks.lib.${system}.run {
# src = ./.;
# hooks = {
# treefmt = {
# enable = true;
# name = "treefmt";
# pass_filenames = false;
# entry = l.toString (pkgs.writeScript "treefmt" ''
# #!${pkgs.bash}/bin/bash
# export PATH="$PATH:${alejandra.defaultPackage.${system}}/bin"
# ${pkgs.treefmt}/bin/treefmt --fail-on-change
# '');
# };
# };
# };
# }))
;
};
} }

1105
overrides/nodejs/default.nix
View File

File diff suppressed because it is too large Load Diff

95
src/apps/cli/default.nix
View File

@ -6,68 +6,59 @@
nix, nix,
translators, translators,
utils, utils,
# from nixpkgs # from nixpkgs
gitMinimal, gitMinimal,
lib, lib,
python3, python3,
... ...
}: }: let
let
b = builtins; b = builtins;
cliPython = python3.withPackages (ps: [ ps.networkx ps.cleo ps.jsonschema ]); cliPython = python3.withPackages (ps: [ps.networkx ps.cleo ps.jsonschema]);
in {
in
{
program = program =
utils.writePureShellScript utils.writePureShellScript
[ [
gitMinimal gitMinimal
nix nix
] ]
''
# escape the temp dir created by writePureShellScript
cd - > /dev/null
# run the cli
dream2nixConfig=${configFile} \
dream2nixSrc=${dream2nixWithExternals} \
fetcherNames="${b.toString (lib.attrNames fetchers.fetchers)}" \
${cliPython}/bin/python ${./.}/cli.py "$@"
'';
templateDefaultNix =
{
dream2nixLocationRelative,
dreamLock,
sourcePathRelative,
}:
let
defaultPackage = dreamLock._generic.defaultPackage;
defaultPackageVersion = dreamLock._generic.packages."${defaultPackage}";
in
'' ''
{ # escape the temp dir created by writePureShellScript
dream2nix ? import ( cd - > /dev/null
let
dream2nixWithExternals = (builtins.getEnv "dream2nixWithExternals");
in
if dream2nixWithExternals != "" then dream2nixWithExternals else
throw '''
This default.nix is for debugging purposes and can only be evaluated within the dream2nix devShell env.
''') {},
}:
dream2nix.makeOutputs { # run the cli
source = ./dream-lock.json; dream2nixConfig=${configFile} \
${lib.optionalString (dreamLock.sources."${defaultPackage}"."${defaultPackageVersion}".type == "unknown") '' dream2nixSrc=${dream2nixWithExternals} \
sourceOverrides = oldSources: { fetcherNames="${b.toString (lib.attrNames fetchers.fetchers)}" \
"${defaultPackage}"."${defaultPackageVersion}" = ./${sourcePathRelative}; ${cliPython}/bin/python ${./.}/cli.py "$@"
}; '';
''}
} templateDefaultNix = {
dream2nixLocationRelative,
dreamLock,
sourcePathRelative,
}: let
defaultPackage = dreamLock._generic.defaultPackage;
defaultPackageVersion = dreamLock._generic.packages."${defaultPackage}";
in ''
{
dream2nix ? import (
let
dream2nixWithExternals = (builtins.getEnv "dream2nixWithExternals");
in
if dream2nixWithExternals != "" then dream2nixWithExternals else
throw '''
This default.nix is for debugging purposes and can only be evaluated within the dream2nix devShell env.
''') {},
}:
dream2nix.makeOutputs {
source = ./dream-lock.json;
${lib.optionalString (dreamLock.sources."${defaultPackage}"."${defaultPackageVersion}".type == "unknown") ''
sourceOverrides = oldSources: {
"${defaultPackage}"."${defaultPackageVersion}" = ./${sourcePathRelative};
};
''}
}
''; '';
} }

95
src/apps/cli2/default.nix
View File

@ -6,68 +6,59 @@
nix, nix,
translators, translators,
utils, utils,
# from nixpkgs # from nixpkgs
gitMinimal, gitMinimal,
lib, lib,
python3, python3,
... ...
}: }: let
let
b = builtins; b = builtins;
cliPython = python3.withPackages (ps: [ ps.networkx ps.cleo ps.jsonschema ]); cliPython = python3.withPackages (ps: [ps.networkx ps.cleo ps.jsonschema]);
in {
in
{
program = program =
utils.writePureShellScript utils.writePureShellScript
[ [
gitMinimal gitMinimal
nix nix
] ]
''
# escape the temp dir created by writePureShellScript
cd - > /dev/null
# run the cli
dream2nixConfig=${configFile} \
dream2nixSrc=${dream2nixWithExternals} \
fetcherNames="${b.toString (lib.attrNames fetchers.fetchers)}" \
${cliPython}/bin/python ${./.}/cli.py "$@"
'';
templateDefaultNix =
{
dream2nixLocationRelative,
dreamLock,
sourcePathRelative,
}:
let
defaultPackage = dreamLock._generic.defaultPackage;
defaultPackageVersion = dreamLock._generic.packages."${defaultPackage}";
in
'' ''
{ # escape the temp dir created by writePureShellScript
dream2nix ? import ( cd - > /dev/null
let
dream2nixWithExternals = (builtins.getEnv "dream2nixWithExternals");
in
if dream2nixWithExternals != "" then dream2nixWithExternals else
throw '''
This default.nix is for debugging purposes and can only be evaluated within the dream2nix devShell env.
''') {},
}:
dream2nix.makeOutputs { # run the cli
source = ./dream-lock.json; dream2nixConfig=${configFile} \
${lib.optionalString (dreamLock.sources."${defaultPackage}"."${defaultPackageVersion}".type == "unknown") '' dream2nixSrc=${dream2nixWithExternals} \
sourceOverrides = oldSources: { fetcherNames="${b.toString (lib.attrNames fetchers.fetchers)}" \
"${defaultPackage}"."${defaultPackageVersion}" = ./${sourcePathRelative}; ${cliPython}/bin/python ${./.}/cli.py "$@"
}; '';
''}
} templateDefaultNix = {
dream2nixLocationRelative,
dreamLock,
sourcePathRelative,
}: let
defaultPackage = dreamLock._generic.defaultPackage;
defaultPackageVersion = dreamLock._generic.packages."${defaultPackage}";
in ''
{
dream2nix ? import (
let
dream2nixWithExternals = (builtins.getEnv "dream2nixWithExternals");
in
if dream2nixWithExternals != "" then dream2nixWithExternals else
throw '''
This default.nix is for debugging purposes and can only be evaluated within the dream2nix devShell env.
''') {},
}:
dream2nix.makeOutputs {
source = ./dream-lock.json;
${lib.optionalString (dreamLock.sources."${defaultPackage}"."${defaultPackageVersion}".type == "unknown") ''
sourceOverrides = oldSources: {
"${defaultPackage}"."${defaultPackageVersion}" = ./${sourcePathRelative};
};
''}
}
''; '';
} }

9
src/apps/contribute/default.nix
View File

@ -3,12 +3,9 @@
python3, python3,
writeScript, writeScript,
... ...
}: }: let
cliPython = python3.withPackages (ps: [ps.cleo]);
let in {
cliPython = python3.withPackages (ps: [ ps.cleo ]);
in
{
program = writeScript "contribute" '' program = writeScript "contribute" ''
dream2nixSrc=${../../.} \ dream2nixSrc=${../../.} \
${cliPython}/bin/python ${./contribute.py} contribute "$@" ${cliPython}/bin/python ${./contribute.py} contribute "$@"

23
src/apps/default.nix
View File

@ -1,30 +1,25 @@
{ {
lib, lib,
pkgs, pkgs,
# dream2nix # dream2nix
callPackageDream, callPackageDream,
translators, translators,
... ...
}: }: let
let
b = builtins; b = builtins;
in in rec {
rec {
apps = { apps = {
inherit cli cli2 contribute install; inherit cli cli2 contribute install;
dream2nix = cli; dream2nix = cli;
}; };
flakeApps = flakeApps = lib.mapAttrs (
lib.mapAttrs (appName: app: appName: app: {
{ type = "app";
type = "app"; program = b.toString app.program;
program = b.toString app.program; }
} )
) apps; apps;
# the dream2nix cli # the dream2nix cli
cli = callPackageDream (import ./cli) {}; cli = callPackageDream (import ./cli) {};

10
src/apps/install/default.nix
View File

@ -1,14 +1,12 @@
{ {
runCommand, runCommand,
writeScript, writeScript,
# dream2nix inputs # dream2nix inputs
dream2nixWithExternals, dream2nixWithExternals,
... ...
}: }: {
program =
{ writeScript "install"
program = writeScript "install"
'' ''
target="$1" target="$1"
if [[ "$target" == "" ]]; then if [[ "$target" == "" ]]; then
@ -28,4 +26,4 @@
echo "Installed dream2nix successfully to '$target'." echo "Installed dream2nix successfully to '$target'."
echo "Please check/modify settings in '$target/config.json'" echo "Please check/modify settings in '$target/config.json'"
''; '';
} }

13
src/builders/default.nix
View File

@ -2,24 +2,21 @@
builders, builders,
callPackageDream, callPackageDream,
... ...
}: }: {
{ python = rec {
python = rec {
default = simpleBuilder; default = simpleBuilder;
simpleBuilder = callPackageDream ./python/simple-builder {}; simpleBuilder = callPackageDream ./python/simple-builder {};
}; };
nodejs = rec { nodejs = rec {
default = granular; default = granular;
node2nix = callPackageDream ./nodejs/node2nix {}; node2nix = callPackageDream ./nodejs/node2nix {};
granular = callPackageDream ./nodejs/granular { inherit builders; }; granular = callPackageDream ./nodejs/granular {inherit builders;};
}; };
rust = rec { rust = rec {
default = buildRustPackage; default = buildRustPackage;

22
src/builders/go/gomod2nix/default.nix
View File

@ -3,19 +3,17 @@
pkgs, pkgs,
externals, externals,
... ...
}: }: {
{
fetchedSources, fetchedSources,
dreamLock, dreamLock,
}: }: let
let gomod2nixTOML =
gomod2nixTOML = fetchedSources.mapAttrs fetchedSources.mapAttrs
dependencyObject.goName; dependencyObject.goName;
in in
externals.gomod2nixBuilder rec { externals.gomod2nixBuilder rec {
pname = dreamLock.generic.mainPackage; pname = dreamLock.generic.mainPackage;
version = dreamLock.sources."${pname}".version; version = dreamLock.sources."${pname}".version;
src = fetchedSources."${pname}"; src = fetchedSources."${pname}";
modules = ./gomod2nix.toml; modules = ./gomod2nix.toml;
} }

620
src/builders/nodejs/granular/default.nix
View File

@ -7,50 +7,38 @@
runCommand, runCommand,
stdenv, stdenv,
writeText, writeText,
# dream2nix inputs # dream2nix inputs
builders, builders,
externals, externals,
utils, utils,
... ...
}: }: {
{
# Funcs # Funcs
# AttrSet -> Bool) -> AttrSet -> [x] # AttrSet -> Bool) -> AttrSet -> [x]
getCyclicDependencies, # name: version: -> [ {name=; version=; } ] getCyclicDependencies, # name: version: -> [ {name=; version=; } ]
getDependencies, # name: version: -> [ {name=; version=; } ] getDependencies, # name: version: -> [ {name=; version=; } ]
getSource, # name: version: -> store-path getSource, # name: version: -> store-path
buildPackageWithOtherBuilder, # { builder, name, version }: -> drv buildPackageWithOtherBuilder, # { builder, name, version }: -> drv
# Attributes # Attributes
subsystemAttrs, # attrset subsystemAttrs, # attrset
defaultPackageName, # string defaultPackageName, # string
defaultPackageVersion, # string defaultPackageVersion, # string
packages, # list packages, # list
# attrset of pname -> versions, # attrset of pname -> versions,
# where versions is a list of version strings # where versions is a list of version strings
packageVersions, packageVersions,
# function which applies overrides to a package # function which applies overrides to a package
# It must be applied by the builder to each individual derivation # It must be applied by the builder to each individual derivation
# Example: # Example:
# produceDerivation name (mkDerivation {...}) # produceDerivation name (mkDerivation {...})
produceDerivation, produceDerivation,
# Custom Options: (parametrize builder behavior) # Custom Options: (parametrize builder behavior)
# These can be passed by the user via `builderArgs`. # These can be passed by the user via `builderArgs`.
# All options must provide default # All options must provide default
standalonePackageNames ? [], standalonePackageNames ? [],
nodejs ? null, nodejs ? null,
... ...
}@args: } @ args: let
let
b = builtins; b = builtins;
nodejsVersion = subsystemAttrs.nodejsVersion; nodejsVersion = subsystemAttrs.nodejsVersion;
@ -59,8 +47,8 @@ let
(args.packages."${name}" or null) == version; (args.packages."${name}" or null) == version;
nodejs = nodejs =
if args ? nodejs then if args ? nodejs
args.nodejs then args.nodejs
else else
pkgs."nodejs-${builtins.toString nodejsVersion}_x" pkgs."nodejs-${builtins.toString nodejsVersion}_x"
or (throw "Could not find nodejs version '${nodejsVersion}' in pkgs"); or (throw "Could not find nodejs version '${nodejsVersion}' in pkgs");
@ -74,12 +62,12 @@ let
allPackages = allPackages =
lib.mapAttrs lib.mapAttrs
(name: versions: (name: versions:
lib.genAttrs lib.genAttrs
versions versions
(version: (version:
makePackage name version)) makePackage name version))
packageVersions; packageVersions;
outputs = { outputs = {
inherit defaultPackage; inherit defaultPackage;
@ -87,13 +75,13 @@ let
# select only the packages listed in dreamLock as main packages # select only the packages listed in dreamLock as main packages
packages = packages =
b.foldl' b.foldl'
(ps: p: ps // p) (ps: p: ps // p)
{} {}
(lib.mapAttrsToList (lib.mapAttrsToList
(name: version:{ (name: version: {
"${name}"."${version}" = allPackages."${name}"."${version}"; "${name}"."${version}" = allPackages."${name}"."${version}";
}) })
args.packages); args.packages);
}; };
# This is only executed for electron based packages. # This is only executed for electron based packages.
@ -159,302 +147,294 @@ let
''; '';
# Generates a derivation for a specific package name + version # Generates a derivation for a specific package name + version
makePackage = name: version: makePackage = name: version: let
let deps = getDependencies name version;
deps = getDependencies name version; nodeDeps =
lib.forEach
deps
(dep: allPackages."${dep.name}"."${dep.version}");
nodeDeps = dependenciesJson =
lib.forEach b.toJSON
deps (lib.listToAttrs
(dep: allPackages."${dep.name}"."${dep.version}" ); (b.map
(dep: lib.nameValuePair dep.name dep.version)
deps));
dependenciesJson = b.toJSON electronDep =
(lib.listToAttrs if ! isMainPackage name version
(b.map then null
(dep: lib.nameValuePair dep.name dep.version) else
deps)); lib.findFirst
(dep: dep.name == "electron")
null
deps;
electronDep = electronVersionMajor =
if ! isMainPackage name version then lib.versions.major electronDep.version;
null
electronHeaders =
if electronDep == null
then null
else pkgs."electron_${electronVersionMajor}".headers;
pkg = produceDerivation name (stdenv.mkDerivation rec {
inherit
dependenciesJson
electronHeaders
nodeDeps
nodeSources
version
;
packageName = name;
pname = utils.sanitizeDerivationName name;
installMethod = "symlink";
electronAppDir = ".";
# only run build on the main package
runBuild = isMainPackage name version;
src = getSource name version;
nativeBuildInputs = [makeWrapper];
buildInputs = [jq nodejs python3];
# prevents running into ulimits
passAsFile = ["dependenciesJson" "nodeDeps"];
preConfigurePhases = ["d2nLoadFuncsPhase" "d2nPatchPhase"];
# can be overridden to define alternative install command
# (defaults to 'npm run postinstall')
buildScript = null;
# python script to modify some metadata to support installation
# (see comments below on d2nPatchPhase)
fixPackage = "${./fix-package.py}";
# script to install (symlink or copy) dependencies.
installDeps = "${./install-deps.py}";
# costs performance and doesn't seem beneficial in most scenarios
dontStrip = true;
# declare some useful shell functions
d2nLoadFuncsPhase = ''
# function to resolve symlinks to copies
symlinksToCopies() {
local dir="$1"
echo "transforming symlinks to copies..."
for f in $(find -L "$dir" -xtype l); do
if [ -f $f ]; then
continue
fi
echo "copying $f"
chmod +wx $(dirname "$f")
mv "$f" "$f.bak"
mkdir "$f"
if [ -n "$(ls -A "$f.bak/")" ]; then
cp -r "$f.bak"/* "$f/"
chmod -R +w $f
fi
rm "$f.bak"
done
}
'';
# TODO: upstream fix to nixpkgs
# example which requires this:
# https://registry.npmjs.org/react-window-infinite-loader/-/react-window-infinite-loader-1.0.7.tgz
unpackCmd =
if lib.hasSuffix ".tgz" src
then "tar --delay-directory-restore -xf $src"
else null;
unpackPhase = ''
runHook preUnpack
nodeModules=$out/lib/node_modules
export sourceRoot="$nodeModules/$packageName"
# sometimes tarballs do not end with .tar.??
unpackFallback(){
local fn="$1"
tar xf "$fn"
}
unpackCmdHooks+=(unpackFallback)
unpackFile $src
# Make the base dir in which the target dependency resides in first
mkdir -p "$(dirname "$sourceRoot")"
# install source
if [ -f "$src" ]
then
# Figure out what directory has been unpacked
export packageDir="$(find . -maxdepth 1 -type d | tail -1)"
# Restore write permissions
find "$packageDir" -type d -exec chmod u+x {} \;
chmod -R u+w "$packageDir"
# Move the extracted tarball into the output folder
mv "$packageDir" "$sourceRoot"
elif [ -d "$src" ]
then
export strippedName="$(stripHash $src)"
# Restore write permissions
chmod -R u+w "$strippedName"
# Move the extracted directory into the output folder
mv "$strippedName" "$sourceRoot"
fi
runHook postUnpack
'';
# The python script wich is executed in this phase:
# - ensures that the package is compatible to the current system
# - ensures the main version in package.json matches the expected
# - pins dependency versions in package.json
# (some npm commands might otherwise trigger networking)
# - creates symlinks for executables declared in package.json
# Apart from that:
# - Any usage of 'link:' in package.json is replaced with 'file:'
# - If package-lock.json exists, it is deleted, as it might conflict
# with the parent package-lock.json.
d2nPatchPhase = ''
# delete package-lock.json as it can lead to conflicts
rm -f package-lock.json
# repair 'link:' -> 'file:'
mv $nodeModules/$packageName/package.json $nodeModules/$packageName/package.json.old
cat $nodeModules/$packageName/package.json.old | sed 's!link:!file\:!g' > $nodeModules/$packageName/package.json
rm $nodeModules/$packageName/package.json.old
# run python script (see comment above):
cp package.json package.json.bak
python $fixPackage \
|| \
# exit code 3 -> the package is incompatible to the current platform
# -> Let the build succeed, but don't create lib/node_packages
if [ "$?" == "3" ]; then
rm -r $out/*
echo "Not compatible with system $system" > $out/error
exit 0
else else
lib.findFirst exit 1
(dep: dep.name == "electron") fi
null
deps;
electronVersionMajor = # configure typescript
lib.versions.major electronDep.version; if [ -f ./tsconfig.json ] \
&& node -e 'require("typescript")' &>/dev/null; then
node ${./tsconfig-to-json.js}
${pkgs.jq}/bin/jq ".compilerOptions.preserveSymlinks = true" tsconfig.json \
| ${pkgs.moreutils}/bin/sponge tsconfig.json
fi
'';
electronHeaders = # - installs dependencies into the node_modules directory
if electronDep == null then # - adds executables of direct node module dependencies to PATH
null # - adds the current node module to NODE_PATH
# - sets HOME=$TMPDIR, as this is required by some npm scripts
# TODO: don't install dev dependencies. Load into NODE_PATH instead
configurePhase = ''
runHook preConfigure
# symlink sub dependencies as well as this imitates npm better
python $installDeps
# add bin path entries collected by python script
if [ -e $TMP/ADD_BIN_PATH ]; then
export PATH="$PATH:$(cat $TMP/ADD_BIN_PATH)"
fi
# add dependencies to NODE_PATH
export NODE_PATH="$NODE_PATH:$nodeModules/$packageName/node_modules"
export HOME=$TMPDIR
runHook postConfigure
'';
# Runs the install command which defaults to 'npm run postinstall'.
# Allows using custom install command by overriding 'buildScript'.
buildPhase = ''
runHook preBuild
# execute electron-rebuild
if [ -n "$electronHeaders" ]; then
${electron-rebuild}
fi
# execute install command
if [ -n "$buildScript" ]; then
if [ -f "$buildScript" ]; then
$buildScript
else
eval "$buildScript"
fi
# by default, only for top level packages, `npm run build` is executed
elif [ -n "$runBuild" ] && [ "$(jq '.scripts.build' ./package.json)" != "null" ]; then
npm run build
else else
pkgs."electron_${electronVersionMajor}".headers; if [ "$(jq '.scripts.install' ./package.json)" != "null" ]; then
npm --production --offline --nodedir=$nodeSources run install
fi
if [ "$(jq '.scripts.postinstall' ./package.json)" != "null" ]; then
npm --production --offline --nodedir=$nodeSources run postinstall
fi
fi
runHook postBuild
'';
pkg = # Symlinks executables and manual pages to correct directories
produceDerivation name (stdenv.mkDerivation rec { installPhase = ''
runHook preInstall
inherit echo "Symlinking exectuables to /bin"
dependenciesJson if [ -d "$nodeModules/.bin" ]
electronHeaders then
nodeDeps chmod +x $nodeModules/.bin/*
nodeSources ln -s $nodeModules/.bin $out/bin
version fi
;
packageName = name; echo "Symlinking manual pages"
if [ -d "$nodeModules/$packageName/man" ]
then
mkdir -p $out/share
for dir in "$nodeModules/$packageName/man/"*
do
mkdir -p $out/share/man/$(basename "$dir")
for page in "$dir"/*
do
ln -s $page $out/share/man/$(basename "$dir")
done
done
fi
pname = utils.sanitizeDerivationName name; # wrap electron app
# execute electron-rebuild
installMethod = "symlink"; if [ -n "$electronHeaders" ]; then
${electron-wrap}
electronAppDir = "."; fi
# only run build on the main package
runBuild = isMainPackage name version;
src = getSource name version;
nativeBuildInputs = [ makeWrapper ];
buildInputs = [ jq nodejs python3 ];
# prevents running into ulimits
passAsFile = [ "dependenciesJson" "nodeDeps" ];
preConfigurePhases = [ "d2nLoadFuncsPhase" "d2nPatchPhase" ];
# can be overridden to define alternative install command
# (defaults to 'npm run postinstall')
buildScript = null;
# python script to modify some metadata to support installation
# (see comments below on d2nPatchPhase)
fixPackage = "${./fix-package.py}";
# script to install (symlink or copy) dependencies.
installDeps = "${./install-deps.py}";
# costs performance and doesn't seem beneficial in most scenarios
dontStrip = true;
# declare some useful shell functions
d2nLoadFuncsPhase = ''
# function to resolve symlinks to copies
symlinksToCopies() {
local dir="$1"
echo "transforming symlinks to copies..."
for f in $(find -L "$dir" -xtype l); do
if [ -f $f ]; then
continue
fi
echo "copying $f"
chmod +wx $(dirname "$f")
mv "$f" "$f.bak"
mkdir "$f"
if [ -n "$(ls -A "$f.bak/")" ]; then
cp -r "$f.bak"/* "$f/"
chmod -R +w $f
fi
rm "$f.bak"
done
}
'';
# TODO: upstream fix to nixpkgs
# example which requires this:
# https://registry.npmjs.org/react-window-infinite-loader/-/react-window-infinite-loader-1.0.7.tgz
unpackCmd =
if lib.hasSuffix ".tgz" src then
"tar --delay-directory-restore -xf $src"
else
null;
unpackPhase = ''
runHook preUnpack
nodeModules=$out/lib/node_modules
export sourceRoot="$nodeModules/$packageName"
# sometimes tarballs do not end with .tar.??
unpackFallback(){
local fn="$1"
tar xf "$fn"
}
unpackCmdHooks+=(unpackFallback)
unpackFile $src
# Make the base dir in which the target dependency resides in first
mkdir -p "$(dirname "$sourceRoot")"
# install source
if [ -f "$src" ]
then
# Figure out what directory has been unpacked
export packageDir="$(find . -maxdepth 1 -type d | tail -1)"
# Restore write permissions
find "$packageDir" -type d -exec chmod u+x {} \;
chmod -R u+w "$packageDir"
# Move the extracted tarball into the output folder
mv "$packageDir" "$sourceRoot"
elif [ -d "$src" ]
then
export strippedName="$(stripHash $src)"
# Restore write permissions
chmod -R u+w "$strippedName"
# Move the extracted directory into the output folder
mv "$strippedName" "$sourceRoot"
fi
runHook postUnpack
'';
# The python script wich is executed in this phase:
# - ensures that the package is compatible to the current system
# - ensures the main version in package.json matches the expected
# - pins dependency versions in package.json
# (some npm commands might otherwise trigger networking)
# - creates symlinks for executables declared in package.json
# Apart from that:
# - Any usage of 'link:' in package.json is replaced with 'file:'
# - If package-lock.json exists, it is deleted, as it might conflict
# with the parent package-lock.json.
d2nPatchPhase = ''
# delete package-lock.json as it can lead to conflicts
rm -f package-lock.json
# repair 'link:' -> 'file:'
mv $nodeModules/$packageName/package.json $nodeModules/$packageName/package.json.old
cat $nodeModules/$packageName/package.json.old | sed 's!link:!file\:!g' > $nodeModules/$packageName/package.json
rm $nodeModules/$packageName/package.json.old
# run python script (see comment above):
cp package.json package.json.bak
python $fixPackage \
|| \
# exit code 3 -> the package is incompatible to the current platform
# -> Let the build succeed, but don't create lib/node_packages
if [ "$?" == "3" ]; then
rm -r $out/*
echo "Not compatible with system $system" > $out/error
exit 0
else
exit 1
fi
# configure typescript
if [ -f ./tsconfig.json ] \
&& node -e 'require("typescript")' &>/dev/null; then
node ${./tsconfig-to-json.js}
${pkgs.jq}/bin/jq ".compilerOptions.preserveSymlinks = true" tsconfig.json \
| ${pkgs.moreutils}/bin/sponge tsconfig.json
fi
'';
# - installs dependencies into the node_modules directory
# - adds executables of direct node module dependencies to PATH
# - adds the current node module to NODE_PATH
# - sets HOME=$TMPDIR, as this is required by some npm scripts
# TODO: don't install dev dependencies. Load into NODE_PATH instead
configurePhase = ''
runHook preConfigure
# symlink sub dependencies as well as this imitates npm better
python $installDeps
# add bin path entries collected by python script
if [ -e $TMP/ADD_BIN_PATH ]; then
export PATH="$PATH:$(cat $TMP/ADD_BIN_PATH)"
fi
# add dependencies to NODE_PATH
export NODE_PATH="$NODE_PATH:$nodeModules/$packageName/node_modules"
export HOME=$TMPDIR
runHook postConfigure
'';
# Runs the install command which defaults to 'npm run postinstall'.
# Allows using custom install command by overriding 'buildScript'.
buildPhase = ''
runHook preBuild
# execute electron-rebuild
if [ -n "$electronHeaders" ]; then
${electron-rebuild}
fi
# execute install command
if [ -n "$buildScript" ]; then
if [ -f "$buildScript" ]; then
$buildScript
else
eval "$buildScript"
fi
# by default, only for top level packages, `npm run build` is executed
elif [ -n "$runBuild" ] && [ "$(jq '.scripts.build' ./package.json)" != "null" ]; then
npm run build
else
if [ "$(jq '.scripts.install' ./package.json)" != "null" ]; then
npm --production --offline --nodedir=$nodeSources run install
fi
if [ "$(jq '.scripts.postinstall' ./package.json)" != "null" ]; then
npm --production --offline --nodedir=$nodeSources run postinstall
fi
fi
runHook postBuild
'';
# Symlinks executables and manual pages to correct directories
installPhase = ''
runHook preInstall
echo "Symlinking exectuables to /bin"
if [ -d "$nodeModules/.bin" ]
then
chmod +x $nodeModules/.bin/*
ln -s $nodeModules/.bin $out/bin
fi
echo "Symlinking manual pages"
if [ -d "$nodeModules/$packageName/man" ]
then
mkdir -p $out/share
for dir in "$nodeModules/$packageName/man/"*
do
mkdir -p $out/share/man/$(basename "$dir")
for page in "$dir"/*
do
ln -s $page $out/share/man/$(basename "$dir")
done
done
fi
# wrap electron app
# execute electron-rebuild
if [ -n "$electronHeaders" ]; then
${electron-wrap}
fi
runHook postInstall
'';
});
in
pkg;
runHook postInstall
'';
});
in
pkg;
in in
outputs outputs

87
src/builders/nodejs/node2nix/default.nix
View File

@ -1,17 +1,13 @@
# builder imported from node2nix # builder imported from node2nix
{ {
lib, lib,
pkgs, pkgs,
# dream2nix inputs # dream2nix inputs
externals, externals,
node2nix ? externals.node2nix, node2nix ? externals.node2nix,
utils, utils,
... ...
}: }: {
{
subsystemAttrs, subsystemAttrs,
defaultPackageName, defaultPackageName,
defaultPackageVersion, defaultPackageVersion,
@ -19,12 +15,10 @@
getDependencies, getDependencies,
getSource, getSource,
packageVersions, packageVersions,
# overrides # overrides
packageOverrides ? {}, packageOverrides ? {},
... ...
}@args: } @ args: let
let
b = builtins; b = builtins;
getAllDependencies = name: version: getAllDependencies = name: version:
@ -43,56 +37,49 @@ let
node2nixEnv = node2nix nodejs; node2nixEnv = node2nix nodejs;
makeSource = packageName: version: prevDeps: makeSource = packageName: version: prevDeps: let
let depsFiltered =
depsFiltered = lib.filter
(lib.filter (dep:
(dep: ! b.elem dep prevDeps)
! b.elem dep prevDeps) (getAllDependencies packageName version);
(getAllDependencies packageName version)); parentDeps =
parentDeps = prevDeps ++ depsFiltered;
prevDeps ++ depsFiltered; in rec {
in inherit packageName version;
rec { name = utils.sanitizeDerivationName packageName;
inherit packageName version; src = getSource packageName version;
name = utils.sanitizeDerivationName packageName; dependencies =
src = getSource packageName version; lib.forEach
dependencies = depsFiltered
lib.forEach (dep: makeSource dep.name dep.version parentDeps);
depsFiltered };
(dep: makeSource dep.name dep.version parentDeps);
};
node2nixDependencies = node2nixDependencies =
lib.forEach lib.forEach
mainPackageDependencies mainPackageDependencies
(dep: makeSource dep.name dep.version mainPackageDependencies); (dep: makeSource dep.name dep.version mainPackageDependencies);
# (dep: allSources."${dep.name}"."${dep.version}"); # (dep: allSources."${dep.name}"."${dep.version}");
callNode2Nix = funcName: args: callNode2Nix = funcName: args:
node2nixEnv."${funcName}" (rec { node2nixEnv."${funcName}" (rec {
name = utils.sanitizeDerivationName packageName; name = utils.sanitizeDerivationName packageName;
packageName = defaultPackageName; packageName = defaultPackageName;
version = defaultPackageVersion; version = defaultPackageVersion;
dependencies = node2nixDependencies; dependencies = node2nixDependencies;
production = true; production = true;
bypassCache = true; bypassCache = true;
reconstructLock = true; reconstructLock = true;
src = getSource defaultPackageName defaultPackageVersion; src = getSource defaultPackageName defaultPackageVersion;
} }
// args); // args);
in rec {
in
rec {
packages."${defaultPackageName}"."${defaultPackageVersion}" = defaultPackage; packages."${defaultPackageName}"."${defaultPackageVersion}" = defaultPackage;
defaultPackage = defaultPackage = let
let pkg = callNode2Nix "buildNodePackage" {};
pkg = callNode2Nix "buildNodePackage" {}; in
in utils.applyOverridesToPackage packageOverrides pkg defaultPackageName;
utils.applyOverridesToPackage packageOverrides pkg defaultPackageName;
devShell = callNode2Nix "buildNodeShell" {}; devShell = callNode2Nix "buildNodeShell" {};
} }

34
src/builders/python/simple-builder/default.nix
View File

@ -1,41 +1,34 @@
# A very simple single derivation python builder # A very simple single derivation python builder
{ {
lib, lib,
pkgs, pkgs,
... ...
}: }: {
{
fetchedSources, fetchedSources,
dreamLock, dreamLock,
}: }: let
let
python = pkgs."${dreamLock._subsystem.pythonAttr}"; python = pkgs."${dreamLock._subsystem.pythonAttr}";
buildFunc = buildFunc =
if dreamLock._subsystem.application then if dreamLock._subsystem.application
python.pkgs.buildPythonApplication then python.pkgs.buildPythonApplication
else else python.pkgs.buildPythonPackage;
python.pkgs.buildPythonPackage;
defaultPackage = dreamLock._generic.defaultPackage; defaultPackage = dreamLock._generic.defaultPackage;
packageName = packageName =
if defaultPackage == null then if defaultPackage == null
if dreamLock._subsystem.application then then
"application" if dreamLock._subsystem.application
else then "application"
"environment" else "environment"
else else defaultPackage;
defaultPackage;
defaultPackage = buildFunc { defaultPackage = buildFunc {
name = packageName; name = packageName;
format = ""; format = "";
buildInputs = pkgs.pythonManylinuxPackages.manylinux1; buildInputs = pkgs.pythonManylinuxPackages.manylinux1;
nativeBuildInputs = [ pkgs.autoPatchelfHook python.pkgs.wheelUnpackHook ]; nativeBuildInputs = [pkgs.autoPatchelfHook python.pkgs.wheelUnpackHook];
unpackPhase = '' unpackPhase = ''
mkdir dist mkdir dist
for file in ${builtins.toString (lib.attrValues fetchedSources)}; do for file in ${builtins.toString (lib.attrValues fetchedSources)}; do
@ -57,8 +50,7 @@ let
--ignore-installed --ignore-installed
runHook postInstall runHook postInstall
''; '';
}; };
in { in {
inherit defaultPackage; inherit defaultPackage;
} }

31
src/builders/rust/build-rust-package/default.nix
View File

@ -1,11 +1,8 @@
{ {
lib, lib,
pkgs, pkgs,
... ...
}: }: {
{
subsystemAttrs, subsystemAttrs,
defaultPackageName, defaultPackageName,
defaultPackageVersion, defaultPackageVersion,
@ -15,23 +12,19 @@
getSourceSpec, getSourceSpec,
packages, packages,
produceDerivation, produceDerivation,
... ...
}@args: } @ args: let
let
l = lib // builtins; l = lib // builtins;
utils = import ../utils.nix args; utils = import ../utils.nix args;
vendoring = import ../vendor.nix (args // { inherit lib pkgs utils; }); vendoring = import ../vendor.nix (args // {inherit lib pkgs utils;});
buildPackage = pname: version: buildPackage = pname: version: let
let src = utils.getRootSource pname version;
src = utils.getRootSource pname version; vendorDir = vendoring.vendorDependencies pname version;
vendorDir = vendoring.vendorDependencies pname version;
cargoBuildFlags = "--package ${pname}"; cargoBuildFlags = "--package ${pname}";
in in
produceDerivation pname (pkgs.rustPlatform.buildRustPackage { produceDerivation pname (pkgs.rustPlatform.buildRustPackage {
inherit pname version src; inherit pname version src;
@ -51,13 +44,11 @@ let
${vendoring.writeGitVendorEntries "vendored-sources"} ${vendoring.writeGitVendorEntries "vendored-sources"}
''; '';
}); });
in in rec {
rec {
packages = packages =
l.mapAttrs l.mapAttrs
(name: version: (name: version: {"${version}" = buildPackage name version;})
{ "${version}" = buildPackage name version; }) args.packages;
args.packages;
defaultPackage = packages."${defaultPackageName}"."${defaultPackageVersion}"; defaultPackage = packages."${defaultPackageName}"."${defaultPackageVersion}";
} }

57
src/builders/rust/crane/default.nix
View File

@ -1,12 +1,9 @@
{ {
lib, lib,
pkgs, pkgs,
externals, externals,
... ...
}: }: {
{
subsystemAttrs, subsystemAttrs,
defaultPackageName, defaultPackageName,
defaultPackageVersion, defaultPackageVersion,
@ -16,51 +13,47 @@
getSourceSpec, getSourceSpec,
packages, packages,
produceDerivation, produceDerivation,
... ...
}@args: } @ args: let
let
l = lib // builtins; l = lib // builtins;
utils = import ../utils.nix args; utils = import ../utils.nix args;
vendoring = import ../vendor.nix (args // { inherit lib pkgs utils; }); vendoring = import ../vendor.nix (args // {inherit lib pkgs utils;});
crane = externals.crane; crane = externals.crane;
buildPackage = pname: version: buildPackage = pname: version: let
let src = utils.getRootSource pname version;
src = utils.getRootSource pname version; cargoVendorDir = vendoring.vendorDependencies pname version;
cargoVendorDir = vendoring.vendorDependencies pname version; postUnpack = ''
postUnpack = '' export CARGO_HOME=$(pwd)/.cargo_home
export CARGO_HOME=$(pwd)/.cargo_home '';
''; preConfigure = ''
preConfigure = '' ${vendoring.writeGitVendorEntries "nix-sources"}
${vendoring.writeGitVendorEntries "nix-sources"} '';
''; # The deps-only derivation will use this as a prefix to the `pname`
# The deps-only derivation will use this as a prefix to the `pname` depsNameSuffix = "-deps";
depsNameSuffix = "-deps";
common = {inherit pname version src cargoVendorDir preConfigure postUnpack;}; common = {inherit pname version src cargoVendorDir preConfigure postUnpack;};
depsArgs = common // { pnameSuffix = depsNameSuffix; }; depsArgs = common // {pnameSuffix = depsNameSuffix;};
deps = produceDerivation "${pname}${depsNameSuffix}" (crane.buildDepsOnly depsArgs); deps = produceDerivation "${pname}${depsNameSuffix}" (crane.buildDepsOnly depsArgs);
buildArgs = common // { buildArgs =
common
// {
cargoArtifacts = deps; cargoArtifacts = deps;
# Make sure cargo only builds & tests the package we want # Make sure cargo only builds & tests the package we want
cargoBuildCommand = "cargo build --release --package ${pname}"; cargoBuildCommand = "cargo build --release --package ${pname}";
cargoTestCommand = "cargo test --release --package ${pname}"; cargoTestCommand = "cargo test --release --package ${pname}";
}; };
in in
produceDerivation pname (crane.buildPackage buildArgs); produceDerivation pname (crane.buildPackage buildArgs);
in in rec {
rec {
packages = packages =
l.mapAttrs l.mapAttrs
(name: version: (name: version: {"${version}" = buildPackage name version;})
{ "${version}" = buildPackage name version; }) args.packages;
args.packages;
defaultPackage = packages."${defaultPackageName}"."${defaultPackageVersion}"; defaultPackage = packages."${defaultPackageName}"."${defaultPackageVersion}";
} }

11
src/builders/rust/utils.nix
View File

@ -2,11 +2,10 @@
getSourceSpec, getSourceSpec,
getSource, getSource,
getRoot, getRoot,
... ...
}: }: rec {
rec { getRootSource = pname: version: let
getRootSource = pname: version: root = getRoot pname version;
let root = getRoot pname version; in in
getSource root.pname root.version; getSource root.pname root.version;
} }

134
src/builders/rust/vendor.nix
View File

@ -1,17 +1,14 @@
{ {
lib, lib,
pkgs, pkgs,
getRoot, getRoot,
getSource, getSource,
getSourceSpec, getSourceSpec,
getDependencies, getDependencies,
getCyclicDependencies, getCyclicDependencies,
subsystemAttrs, subsystemAttrs,
... ...
}: }: let
let
l = lib // builtins; l = lib // builtins;
isCyclic = cyclic: dep: isCyclic = cyclic: dep:
@ -44,78 +41,72 @@ let
in rec { in rec {
# Generates a shell script that writes git vendor entries to .cargo/config. # Generates a shell script that writes git vendor entries to .cargo/config.
# `replaceWith` is the name of the vendored source(s) to use. # `replaceWith` is the name of the vendored source(s) to use.
writeGitVendorEntries = replaceWith: writeGitVendorEntries = replaceWith: let
let makeEntry = source: ''
makeEntry = source: [source."${source.url}${l.optionalString (source ? type) "?${source.type}=${source.value}"}"]
'' replace-with = "${replaceWith}"
[source."${source.url}${l.optionalString (source ? type) "?${source.type}=${source.value}"}"] git = "${source.url}"
replace-with = "${replaceWith}" ${l.optionalString (source ? type) "${source.type} = \"${source.value}\""}
git = "${source.url}"
${l.optionalString (source ? type) "${source.type} = \"${source.value}\""}
'';
entries = l.map makeEntry subsystemAttrs.gitSources;
in ''
mkdir -p $CARGO_HOME && touch $CARGO_HOME/config.toml
cat >> $CARGO_HOME/config.toml <<EOF
${l.concatStringsSep "\n" entries}
EOF
''; '';
entries = l.map makeEntry subsystemAttrs.gitSources;
in ''
mkdir -p $CARGO_HOME && touch $CARGO_HOME/config.toml
cat >> $CARGO_HOME/config.toml <<EOF
${l.concatStringsSep "\n" entries}
EOF
'';
# Vendor a package's dependencies like how `cargo vendor` would do, # Vendor a package's dependencies like how `cargo vendor` would do,
# so we can use it with `cargo`. # so we can use it with `cargo`.
vendorPackageDependencies = pname: version: vendorPackageDependencies = pname: version: let
let deps = getAllTransitiveDependencies pname version;
deps = getAllTransitiveDependencies pname version;
makeSource = dep: makeSource = dep: let
let path = getSource dep.name dep.version;
path = getSource dep.name dep.version; spec = getSourceSpec dep.name dep.version;
spec = getSourceSpec dep.name dep.version; in {
in { inherit path spec dep;
inherit path spec dep; name = "${dep.name}-${dep.version}";
name = "${dep.name}-${dep.version}"; };
}; sources = l.map makeSource deps;
sources = l.map makeSource deps;
findCrateSource = source: findCrateSource = source: let
let inherit (pkgs) cargo jq;
inherit (pkgs) cargo jq; pkg = source.dep;
pkg = source.dep; in ''
in '' # If the target package is in a workspace, or if it's the top-level
# If the target package is in a workspace, or if it's the top-level # crate, we should find the crate path using `cargo metadata`.
# crate, we should find the crate path using `cargo metadata`. crateCargoTOML=$(${cargo}/bin/cargo metadata --format-version 1 --no-deps --manifest-path $tree/Cargo.toml | \
crateCargoTOML=$(${cargo}/bin/cargo metadata --format-version 1 --no-deps --manifest-path $tree/Cargo.toml | \ ${jq}/bin/jq -r '.packages[] | select(.name == "${pkg.name}") | .manifest_path')
${jq}/bin/jq -r '.packages[] | select(.name == "${pkg.name}") | .manifest_path') # If the repository is not a workspace the package might be in a subdirectory.
# If the repository is not a workspace the package might be in a subdirectory. if [[ -z $crateCargoTOML ]]; then
if [[ -z $crateCargoTOML ]]; then for manifest in $(find $tree -name "Cargo.toml"); do
for manifest in $(find $tree -name "Cargo.toml"); do echo Looking at $manifest
echo Looking at $manifest crateCargoTOML=$(${cargo}/bin/cargo metadata --format-version 1 --no-deps --manifest-path "$manifest" | ${jq}/bin/jq -r '.packages[] | select(.name == "${pkg.name}") | .manifest_path' || :)
crateCargoTOML=$(${cargo}/bin/cargo metadata --format-version 1 --no-deps --manifest-path "$manifest" | ${jq}/bin/jq -r '.packages[] | select(.name == "${pkg.name}") | .manifest_path' || :) if [[ ! -z $crateCargoTOML ]]; then
if [[ ! -z $crateCargoTOML ]]; then break
break
fi
done
if [[ -z $crateCargoTOML ]]; then
>&2 echo "Cannot find path for crate '${pkg.name}-${pkg.version}' in the tree in: $tree"
exit 1
fi
fi fi
echo Found crate ${pkg.name} at $crateCargoTOML done
tree="$(dirname $crateCargoTOML)" if [[ -z $crateCargoTOML ]]; then
''; >&2 echo "Cannot find path for crate '${pkg.name}-${pkg.version}' in the tree in: $tree"
makeScript = source: exit 1
let fi
isGit = source.spec.type == "git"; fi
isPath = source.spec.type == "path"; echo Found crate ${pkg.name} at $crateCargoTOML
in tree="$(dirname $crateCargoTOML)"
l.optionalString (!isPath) '' '';
tree="${source.path}" makeScript = source: let
${l.optionalString isGit (findCrateSource source)} isGit = source.spec.type == "git";
cp -prvd "$tree" $out/${source.name} isPath = source.spec.type == "path";
chmod u+w $out/${source.name}
${l.optionalString isGit "printf '{\"files\":{},\"package\":null}' > \"$out/${source.name}/.cargo-checksum.json\""}
'';
in in
l.optionalString (!isPath) ''
tree="${source.path}"
${l.optionalString isGit (findCrateSource source)}
cp -prvd "$tree" $out/${source.name}
chmod u+w $out/${source.name}
${l.optionalString isGit "printf '{\"files\":{},\"package\":null}' > \"$out/${source.name}/.cargo-checksum.json\""}
'';
in
pkgs.runCommand "vendor-${pname}-${version}" {} '' pkgs.runCommand "vendor-${pname}-${version}" {} ''
mkdir -p $out mkdir -p $out
@ -123,11 +114,12 @@ in rec {
l.concatMapStringsSep "\n" l.concatMapStringsSep "\n"
makeScript makeScript
sources sources
} }
''; '';
# Vendors a package's roots dependencies. # Vendors a package's roots dependencies.
vendorDependencies = pname: version: vendorDependencies = pname: version: let
let root = getRoot pname version; in root = getRoot pname version;
in
vendorPackageDependencies root.pname root.version; vendorPackageDependencies root.pname root.version;
} }

1200
src/default.nix
View File

File diff suppressed because it is too large Load Diff

118
src/discoverers/default.nix
View File

@ -2,86 +2,80 @@
config, config,
dlib, dlib,
lib, lib,
}: }: let
let
l = lib // builtins; l = lib // builtins;
subsystems = dlib.dirNames ./.; subsystems = dlib.dirNames ./.;
allDiscoverers = allDiscoverers =
l.collect l.collect
(v: v ? discover) (v: v ? discover)
discoverers; discoverers;
discoverProjects = discoverProjects = {
{ source ? throw "Pass either `source` or `tree` to discoverProjects",
source ? throw "Pass either `source` or `tree` to discoverProjects", tree ? dlib.prepareSourceTree {inherit source;},
tree ? dlib.prepareSourceTree { inherit source; }, settings ? [],
settings ? [], }: let
}: let discoveredProjects =
discoveredProjects = l.flatten
l.flatten (l.map
(l.map (discoverer: discoverer.discover {inherit tree;})
(discoverer: discoverer.discover { inherit tree; }) allDiscoverers);
allDiscoverers);
rootProjectName = l.head discoveredProjects; rootProjectName = l.head discoveredProjects;
projectsExtended = l.forEach discoveredProjects projectsExtended =
(proj: proj l.forEach discoveredProjects
// { (proj:
translator = l.head proj.translators; proj
dreamLockPath = getDreamLockPath proj rootProjectName; // {
}); translator = l.head proj.translators;
in dreamLockPath = getDreamLockPath proj rootProjectName;
applyProjectSettings projectsExtended settings; });
in
applyProjectSettings projectsExtended settings;
getDreamLockPath = project: rootProject: getDreamLockPath = project: rootProject: let
let root =
root = if config.projectRoot == null
if config.projectRoot == null then then "."
"." else config.projectRoot;
else in
config.projectRoot; dlib.sanitizeRelativePath
in "${config.packagesDir}/${rootProject.name}/${project.relPath}/dream-lock.json";
dlib.sanitizeRelativePath
"${config.packagesDir}/${rootProject.name}/${project.relPath}/dream-lock.json";
applyProjectSettings = projects: settingsList: applyProjectSettings = projects: settingsList: let
let settingsListForProject = project:
settingsListForProject = project: l.filter
l.filter (settings:
(settings: if ! settings ? filter
if ! settings ? filter then true then true
else settings.filter project) else settings.filter project)
settingsList; settingsList;
applySettings = project: settings: applySettings = project: settings:
l.recursiveUpdate project settings; l.recursiveUpdate project settings;
applyAllSettings = project: applyAllSettings = project:
l.foldl' l.foldl'
(proj: settings: applySettings proj settings) (proj: settings: applySettings proj settings)
project project
(settingsListForProject project); (settingsListForProject project);
settingsApplied = settingsApplied =
l.forEach projects l.forEach projects
(proj: applyAllSettings proj); (proj: applyAllSettings proj);
in
settingsApplied;
in settingsApplied; discoverers = l.genAttrs subsystems (
subsystem: (import (./. + "/${subsystem}") {inherit dlib lib subsystem;})
discoverers = l.genAttrs subsystems (subsystem:
(import (./. + "/${subsystem}") { inherit dlib lib subsystem; })
); );
in in {
{
inherit inherit
applyProjectSettings applyProjectSettings
discoverProjects discoverProjects
discoverers discoverers
; ;
} }

362
src/discoverers/nodejs/default.nix
View File

@ -1,65 +1,53 @@
{ {
dlib, dlib,
lib, lib,
subsystem, subsystem,
}: }: let
let
l = lib // builtins; l = lib // builtins;
discover = discover = {tree}: let
{ projects = discoverInternal {
tree, inherit tree;
}: };
let in
projects = discoverInternal { filterProjects projects;
inherit tree;
};
in
filterProjects projects;
# One translator call can process a whole workspace containing all # One translator call can process a whole workspace containing all
# sub-packages of that workspace. # sub-packages of that workspace.
# Therefore we can filter out projects which are children of a workspace. # Therefore we can filter out projects which are children of a workspace.
filterProjects = projects: filterProjects = projects: let
let workspaceRoots =
workspaceRoots = l.filter
l.filter (proj: proj.subsystemInfo.workspaces or [] != [])
(proj: proj.subsystemInfo.workspaces or [] != []) projects;
projects;
allWorkspaceChildren = allWorkspaceChildren =
l.flatten l.flatten
(l.map (l.map
(root: root.subsystemInfo.workspaces) (root: root.subsystemInfo.workspaces)
workspaceRoots); workspaceRoots);
childrenRemoved = childrenRemoved =
l.filter l.filter
(proj: (proj: (! l.elem proj.relPath allWorkspaceChildren))
(! l.elem proj.relPath allWorkspaceChildren)) projects;
projects; in
childrenRemoved;
in getTranslatorNames = path: let
childrenRemoved; nodes = l.readDir path;
packageJson = l.fromJSON (l.readFile "${path}/package.json");
getTranslatorNames = path: translators =
let # if the package has no dependencies we use the
nodes = l.readDir path; # package-lock translator with `packageLock = null`
packageJson = l.fromJSON (l.readFile "${path}/package.json"); if ! packageJson ? dependencies && ! packageJson ? devDependencies
translators = then ["package-lock"]
# if the package has no dependencies we use the else
# package-lock translator with `packageLock = null` l.optionals (nodes ? "package-lock.json") ["package-lock"]
if ! packageJson ? dependencies && ! packageJson ? devDependencies ++ l.optionals (nodes ? "yarn.lock") ["yarn-lock"]
then [ "package-lock" ] ++ ["package-json"];
in
else translators;
l.optionals (nodes ? "package-lock.json") [ "package-lock" ]
++ l.optionals (nodes ? "yarn.lock") [ "yarn-lock" ]
++ [ "package-json" ];
in
translators;
# returns the parsed package.json of a given directory # returns the parsed package.json of a given directory
getPackageJson = dirPath: getPackageJson = dirPath:
@ -67,166 +55,146 @@ let
# returns all relative paths to workspaces defined by a glob # returns all relative paths to workspaces defined by a glob
getWorkspacePaths = glob: tree: getWorkspacePaths = glob: tree:
if l.hasSuffix "*" glob then if l.hasSuffix "*" glob
let then let
prefix = l.removeSuffix "*" glob; prefix = l.removeSuffix "*" glob;
path = "${tree.fullPath}/${prefix}"; path = "${tree.fullPath}/${prefix}";
dirNames = dirNames =
if l.pathExists path if l.pathExists path
then dlib.listDirs path then dlib.listDirs path
else else
l.trace l.trace
"WARNING: Detected workspace ${glob} does not exist."
[];
existingWsPaths =
l.filter
(wsPath:
if l.pathExists "${path}/${wsPath}/package.json"
then true
else
let
notExistingPath =
dlib.sanitizeRelativePath "${prefix}/${wsPath}";
in
l.trace
"WARNING: Detected workspace ${notExistingPath} does not exist."
false)
dirNames;
in
l.map (dname: "${prefix}/${dname}") existingWsPaths
else
if l.pathExists "${tree.fullPath}/${glob}/package.json"
then [ glob ]
else
l.trace
"WARNING: Detected workspace ${glob} does not exist." "WARNING: Detected workspace ${glob} does not exist."
[]; [];
existingWsPaths =
l.filter
(wsPath:
if l.pathExists "${path}/${wsPath}/package.json"
then true
else let
notExistingPath =
dlib.sanitizeRelativePath "${prefix}/${wsPath}";
in
l.trace
"WARNING: Detected workspace ${notExistingPath} does not exist."
false)
dirNames;
in
l.map (dname: "${prefix}/${dname}") existingWsPaths
else if l.pathExists "${tree.fullPath}/${glob}/package.json"
then [glob]
else
l.trace
"WARNING: Detected workspace ${glob} does not exist."
[];
# collect project info for workspaces defined by current package.json # collect project info for workspaces defined by current package.json
getWorkspaces = tree: parentInfo: getWorkspaces = tree: parentInfo: let
let packageJson = tree.files."package.json".jsonContent;
packageJson = tree.files."package.json".jsonContent; workspacesRaw = packageJson.workspaces or [];
workspacesRaw = packageJson.workspaces or [];
workspacesFlattened = workspacesFlattened =
if l.isAttrs workspacesRaw if l.isAttrs workspacesRaw
then then
l.flatten
(l.mapAttrsToList
(category: workspaces: workspaces)
workspacesRaw)
else if l.isList workspacesRaw
then workspacesRaw
else throw "Error parsing workspaces in ${tree.files."package.json".relPath}";
in
l.flatten
(l.forEach workspacesFlattened
(glob:
let
workspacePaths = getWorkspacePaths glob tree;
in
l.forEach workspacePaths
(wPath: makeWorkspaceProjectInfo tree wPath parentInfo)));
makeWorkspaceProjectInfo = tree: wsRelPath: parentInfo:
{
inherit subsystem;
name =
(getPackageJson "${tree.fullPath}/${wsRelPath}").name
or "${parentInfo.name}/${wsRelPath}";
relPath = dlib.sanitizeRelativePath "${tree.relPath}/${wsRelPath}";
translators =
l.unique
(
(lib.filter (trans: l.elem trans ["package-lock" "yarn-lock"]) parentInfo.translators)
++ (getTranslatorNames "${tree.fullPath}/${wsRelPath}")
);
subsystemInfo = {
workspaceParent = tree.relPath;
};
};
discoverInternal =
{
tree,
# Internal parameter preventing workspace projects from being discovered
# twice.
alreadyDiscovered ? {},
}:
let
foundSubProjects = alreadyDiscovered:
l.flatten l.flatten
((l.mapAttrsToList (l.mapAttrsToList
(dname: dir: discoverInternal { (category: workspaces: workspaces)
inherit alreadyDiscovered; workspacesRaw)
tree = dir; else if l.isList workspacesRaw
}) then workspacesRaw
(tree.directories or {}))); else throw "Error parsing workspaces in ${tree.files."package.json".relPath}";
in in
l.flatten
(l.forEach workspacesFlattened
(glob: let
workspacePaths = getWorkspacePaths glob tree;
in
l.forEach workspacePaths
(wPath: makeWorkspaceProjectInfo tree wPath parentInfo)));
makeWorkspaceProjectInfo = tree: wsRelPath: parentInfo: {
inherit subsystem;
name =
(getPackageJson "${tree.fullPath}/${wsRelPath}").name
or "${parentInfo.name}/${wsRelPath}";
relPath = dlib.sanitizeRelativePath "${tree.relPath}/${wsRelPath}";
translators =
l.unique
(
(lib.filter (trans: l.elem trans ["package-lock" "yarn-lock"]) parentInfo.translators)
++ (getTranslatorNames "${tree.fullPath}/${wsRelPath}")
);
subsystemInfo = {
workspaceParent = tree.relPath;
};
};
discoverInternal = {
tree,
# Internal parameter preventing workspace projects from being discovered
# twice.
alreadyDiscovered ? {},
}: let
foundSubProjects = alreadyDiscovered:
l.flatten
(l.mapAttrsToList
(dname: dir:
discoverInternal {
inherit alreadyDiscovered;
tree = dir;
})
(tree.directories or {}));
in
# skip if not a nodajs project # skip if not a nodajs project
if alreadyDiscovered ? "${tree.relPath}" if
|| ! tree ? files."package.json" then alreadyDiscovered
? "${tree.relPath}"
|| ! tree ? files."package.json"
then
# this will be cleaned by `flatten` for sub-directories # this will be cleaned by `flatten` for sub-directories
foundSubProjects alreadyDiscovered foundSubProjects alreadyDiscovered
else else let
let # project info of current directory
currentProjectInfo = {
inherit subsystem;
inherit (tree) relPath;
name = tree.files."package.json".jsonContent.name or tree.relPath;
translators = getTranslatorNames tree.fullPath;
subsystemInfo = l.optionalAttrs (workspaces != []) {
workspaces =
l.map
(w: l.removePrefix tree.relPath w.relPath)
workspaces;
};
};
# project info of current directory workspaces = getWorkspaces tree currentProjectInfo;
currentProjectInfo =
{
inherit subsystem;
inherit (tree) relPath;
name = tree.files."package.json".jsonContent.name or tree.relPath;
translators = getTranslatorNames tree.fullPath;
subsystemInfo =
l.optionalAttrs (workspaces != []) {
workspaces =
l.map
(w: l.removePrefix tree.relPath w.relPath)
workspaces;
};
};
workspaces = getWorkspaces tree currentProjectInfo; # list of all projects infos found by the current iteration
foundProjects =
# current directories project info
[currentProjectInfo]
# workspaces defined by the current directory
++ workspaces;
# index of already found projects
# list of all projects infos found by the current iteration # This is needed, because sub-projects also contain a `package.json`,
foundProjects = # and would otherwise be discovered again as an independent project.
# current directories project info alreadyDiscovered' =
[ currentProjectInfo ] alreadyDiscovered
// (l.genAttrs
# workspaces defined by the current directory (l.map (p: p.relPath) foundProjects)
++ (relPath: null));
workspaces; in
# l.trace tree.directories
# index of already found projects # the current directory
# This is needed, because sub-projects also contain a `package.json`, foundProjects
# and would otherwise be discovered again as an independent project. # sub-directories
alreadyDiscovered' = # Thanks to `alreadyDiscovered`, workspace projects won't be discovered
alreadyDiscovered # a second time.
// ++ (foundSubProjects alreadyDiscovered');
(l.genAttrs in {
(l.map (p: p.relPath) foundProjects)
(relPath: null));
in
# l.trace tree.directories
# the current directory
foundProjects
# sub-directories
# Thanks to `alreadyDiscovered`, workspace projects won't be discovered
# a second time.
++
(foundSubProjects alreadyDiscovered');
in
{
inherit discover; inherit discover;
} }

207
src/fetchers/combined-fetcher.nix
View File

@ -9,108 +9,102 @@
nix, nix,
stdenv, stdenv,
writeScript, writeScript,
# dream2nix # dream2nix
defaultFetcher, defaultFetcher,
utils, utils,
... ...
}: }: {
{
# sources attrset from dream lock # sources attrset from dream lock
sources, sources,
sourcesAggregatedHash, sourcesAggregatedHash,
... ...
}@args: } @ args: let
let
b = builtins; b = builtins;
# resolve to individual fetcher calls # resolve to individual fetcher calls
defaultFetched = (defaultFetcher args).fetchedSources; defaultFetched = (defaultFetcher args).fetchedSources;
# extract the arguments from the individual fetcher calls # extract the arguments from the individual fetcher calls
FODArgsAll = FODArgsAll = let
let FODArgsAll' =
FODArgsAll' = lib.mapAttrs
lib.mapAttrs (
(name: versions: name: versions:
lib.mapAttrs lib.mapAttrs
(version: fetched: (version: fetched:
# handle FOD sources # handle FOD sources
if lib.all if
(attr: fetched ? "${attr}") lib.all
[ "outputHash" "outputHashAlgo" "outputHashMode" ] then (attr: fetched ? "${attr}")
["outputHash" "outputHashAlgo" "outputHashMode"]
then
(fetched.overrideAttrs (args: {
passthru.originalArgs = args;
}))
.originalArgs
// {
outPath = let
sanitizedName = utils.sanitizeDerivationName name;
in "${sanitizedName}/${version}/${fetched.name}";
}
# handle path sources
else if lib.isString fetched
then "ignore"
# handle store path sources
else if lib.isStorePath fetched
then "ignore"
# handle unknown sources
else if fetched == "unknown"
then "ignore"
# error out on unknown source types
else
throw ''
Error while generating FOD fetcher for combined sources.
Cannot classify source of ${name}#${version}.
'')
versions
)
defaultFetched;
in
lib.filterAttrs
(name: versions: versions != {})
(lib.mapAttrs
(name: versions:
lib.filterAttrs
(version: fetcherArgs: fetcherArgs != "ignore")
versions)
FODArgsAll');
(fetched.overrideAttrs (args: { FODArgsAllList =
passthru.originalArgs = args; lib.flatten
})).originalArgs // { (lib.mapAttrsToList
outPath = (name: versions:
let b.attrValues versions)
sanitizedName = utils.sanitizeDerivationName name; FODArgsAll);
in
"${sanitizedName}/${version}/${fetched.name}";
}
# handle path sources
else if lib.isString fetched then
"ignore"
# handle store path sources
else if lib.isStorePath fetched then
"ignore"
# handle unknown sources
else if fetched == "unknown" then
"ignore"
# error out on unknown source types
else
throw ''
Error while generating FOD fetcher for combined sources.
Cannot classify source of ${name}#${version}.
'')
versions
)
defaultFetched;
in
lib.filterAttrs
(name: versions: versions != {})
(lib.mapAttrs
(name: versions:
lib.filterAttrs
(version: fetcherArgs: fetcherArgs != "ignore")
versions)
FODArgsAll');
FODArgsAllList =
lib.flatten
(lib.mapAttrsToList
(name: versions:
b.attrValues versions)
FODArgsAll);
# convert arbitrary types to string, like nix does with derivation arguments # convert arbitrary types to string, like nix does with derivation arguments
toString' = x: toString' = x:
if lib.isBool x then if lib.isBool x
if x then then
"1" if x
else then "1"
"" else ""
else if lib.isList x then else if lib.isList x
''"${lib.concatStringsSep " " (lib.forEach x (y: toString' y))}"'' then ''"${lib.concatStringsSep " " (lib.forEach x (y: toString' y))}"''
else if x == null then else if x == null
"" then ""
else else b.toJSON x;
b.toJSON x;
# set up nix build env for signle item # set up nix build env for signle item
setupEnvForItem = fetcherArgs: '' setupEnvForItem = fetcherArgs: ''
# export arguments for builder # export arguments for builder
${lib.concatStringsSep "\n" (lib.mapAttrsToList (argName: argVal: '' ${lib.concatStringsSep "\n" (lib.mapAttrsToList (argName: argVal: ''
export ${argName}=${ export ${argName}=${
lib.replaceStrings [ "$" ''\n'' ] [ ''\$'' "\n" ] (toString' argVal)} lib.replaceStrings ["$" ''\n''] [''\$'' "\n"] (toString' argVal)
'') fetcherArgs)} }
'')
fetcherArgs)}
# run builder # run builder
bash ${fetcherArgs.builder} bash ${fetcherArgs.builder}
@ -159,40 +153,37 @@ let
echo "FOD_HASH=$(${nix}/bin/nix hash path $out)" echo "FOD_HASH=$(${nix}/bin/nix hash path $out)"
''; '';
FODAllSources = FODAllSources = let
let nativeBuildInputs' =
nativeBuildInputs' = lib.unique
lib.unique (lib.foldl (a: b: a ++ b) []
(lib.foldl (a: b: a ++ b) [] (b.map
(b.map (fetcherArgs: (fetcherArgs.nativeBuildInputs or []))
(fetcherArgs: (fetcherArgs.nativeBuildInputs or [])) FODArgsAllList));
FODArgsAllList)); in
in stdenv.mkDerivation rec {
stdenv.mkDerivation rec { name = "sources-combined";
name = "sources-combined"; inherit builder;
inherit builder; nativeBuildInputs =
nativeBuildInputs = nativeBuildInputs' ++ [ nativeBuildInputs'
++ [
coreutils coreutils
]; ];
outputHashAlgo = "sha256"; outputHashAlgo = "sha256";
outputHashMode = "recursive"; outputHashMode = "recursive";
outputHash = sourcesAggregatedHash; outputHash = sourcesAggregatedHash;
}; };
in {
in
{
FOD = FODAllSources; FOD = FODAllSources;
fetchedSources = fetchedSources =
lib.mapAttrs lib.mapAttrs
(name: versions: (name: versions:
lib.mapAttrs lib.mapAttrs
(version: source: (version: source:
if FODArgsAll ? "${name}"."${version}" then if FODArgsAll ? "${name}"."${version}"
"${FODAllSources}/${FODArgsAll."${name}"."${version}".outPath}" then "${FODAllSources}/${FODArgsAll."${name}"."${version}".outPath}"
else else defaultFetched."${name}"."${version}")
defaultFetched."${name}"."${version}") versions)
versions) sources;
sources;
} }

55
src/fetchers/crates-io/default.nix
View File

@ -2,40 +2,39 @@
lib, lib,
fetchurl, fetchurl,
runCommand, runCommand,
utils, utils,
... ...
}: }: {
{ inputs = ["pname" "version"];
inputs = [ "pname" "version" ];
versionField = "version"; versionField = "version";
outputs = { pname, version, ... }@inp: outputs = {
let pname,
b = builtins; version,
# See https://github.com/rust-lang/crates.io-index/blob/master/config.json#L2 ...
url = "https://crates.io/api/v1/crates/${pname}/${version}/download"; } @ inp: let
in b = builtins;
{ # See https://github.com/rust-lang/crates.io-index/blob/master/config.json#L2
calcHash = algo: utils.hashFile algo (b.fetchurl { url = "https://crates.io/api/v1/crates/${pname}/${version}/download";
in {
calcHash = algo:
utils.hashFile algo (b.fetchurl {
inherit url; inherit url;
}); });
fetched = hash: fetched = hash: let
let fetched = fetchurl {
fetched = fetchurl { inherit url;
inherit url; sha256 = hash;
sha256 = hash; name = "download-${pname}-${version}";
name = "download-${pname}-${version}"; };
}; in
in runCommand "unpack-${pname}-${version}" {}
runCommand "unpack-${pname}-${version}" {} ''
'' mkdir -p $out
mkdir -p $out tar --strip-components 1 -xzf ${fetched} -C $out
tar --strip-components 1 -xzf ${fetched} -C $out echo '{"package":"${hash}","files":{}}' > $out/.cargo-checksum.json
echo '{"package":"${hash}","files":{}}' > $out/.cargo-checksum.json '';
''; };
};
} }

63
src/fetchers/default-fetcher.nix
View File

@ -1,56 +1,51 @@
{ {
lib, lib,
# dream2nix attributes # dream2nix attributes
fetchSource, fetchSource,
fetchers, fetchers,
... ...
}: }: {
{
# sources attrset from dream lock # sources attrset from dream lock
defaultPackage, defaultPackage,
defaultPackageVersion, defaultPackageVersion,
sourceOverrides, sourceOverrides,
sources, sources,
... ...
}: }: let
let
b = builtins; b = builtins;
fetchedSources = fetchedSources =
lib.mapAttrs lib.mapAttrs
(name: versions: (name: versions:
lib.mapAttrs lib.mapAttrs
(version: source: (version: source:
if source.type == "unknown" then if source.type == "unknown"
"unknown" then "unknown"
else if source.type == "path" then else if source.type == "path"
if lib.isStorePath source.path then then
source.path if lib.isStorePath source.path
# assume path relative to main package source then source.path
else # assume path relative to main package source
"${overriddenSources."${defaultPackage}"."${defaultPackageVersion}"}/${source.path}" else "${overriddenSources."${defaultPackage}"."${defaultPackageVersion}"}/${source.path}"
else if fetchers.fetchers ? "${source.type}" then else if fetchers.fetchers ? "${source.type}"
fetchSource { then
source = source // { fetchSource {
pname = name; source =
inherit version; source
}; // {
} pname = name;
else throw "unsupported source type '${source.type}'") inherit version;
versions) };
sources; }
else throw "unsupported source type '${source.type}'")
versions)
sources;
overriddenSources = overriddenSources =
lib.recursiveUpdate lib.recursiveUpdate
fetchedSources fetchedSources
(sourceOverrides fetchedSources); (sourceOverrides fetchedSources);
in {
in
{
# attrset: pname -> path of downloaded source # attrset: pname -> path of downloaded source
fetchedSources = overriddenSources; fetchedSources = overriddenSources;
} }

437
src/fetchers/default.nix
View File

@ -1,258 +1,241 @@
{ {
lib, lib,
# dream2nix # dream2nix
callPackageDream, callPackageDream,
utils, utils,
... ...
}: }: let
let
b = builtins; b = builtins;
callFetcher = file: args: callPackageDream file args; callFetcher = file: args: callPackageDream file args;
in in rec {
fetchers = lib.genAttrs (utils.dirNames ./.) (
rec { name:
callFetcher (./. + "/${name}") {}
fetchers = lib.genAttrs (utils.dirNames ./.) (name:
callFetcher (./. + "/${name}") {}
); );
defaultFetcher = callPackageDream ./default-fetcher.nix { inherit fetchers fetchSource; }; defaultFetcher = callPackageDream ./default-fetcher.nix {inherit fetchers fetchSource;};
combinedFetcher = callPackageDream ./combined-fetcher.nix { inherit defaultFetcher; }; combinedFetcher = callPackageDream ./combined-fetcher.nix {inherit defaultFetcher;};
constructSource = constructSource = {
{ type,
type, reComputeHash ? false,
reComputeHash ? false, ...
... } @ args: let
}@args: fetcher = fetchers."${type}";
let argsKeep = b.removeAttrs args ["reComputeHash"];
fetcher = fetchers."${type}"; fetcherOutputs =
argsKeep = b.removeAttrs args [ "reComputeHash" ]; fetcher.outputs
fetcherOutputs = (b.removeAttrs argsKeep ["dir" "hash" "type"]);
fetcher.outputs in
(b.removeAttrs argsKeep [ "dir" "hash" "type" ]); argsKeep
in # if the hash was not provided, calculate hash on the fly (impure)
argsKeep // (lib.optionalAttrs reComputeHash {
# if the hash was not provided, calculate hash on the fly (impure) hash = fetcherOutputs.calcHash "sha256";
// (lib.optionalAttrs reComputeHash {
hash = fetcherOutputs.calcHash "sha256";
});
# update source spec to different version
updateSource =
{
source,
newVersion,
...
}:
constructSource (source // {
reComputeHash = true;
} // {
"${fetchers."${source.type}".versionField}" = newVersion;
}); });
# update source spec to different version
updateSource = {
source,
newVersion,
...
}:
constructSource (source
// {
reComputeHash = true;
}
// {
"${fetchers."${source.type}".versionField}" = newVersion;
});
# fetch a source defined via a dream lock source spec # fetch a source defined via a dream lock source spec
fetchSource = { source, extract ? false }: fetchSource = {
let source,
fetcher = fetchers."${source.type}"; extract ? false,
fetcherArgs = b.removeAttrs source [ "dir" "hash" "type" ]; }: let
fetcherOutputs = fetcher.outputs fetcherArgs; fetcher = fetchers."${source.type}";
maybeArchive = fetcherOutputs.fetched (source.hash or null); fetcherArgs = b.removeAttrs source ["dir" "hash" "type"];
in fetcherOutputs = fetcher.outputs fetcherArgs;
if source ? dir then maybeArchive = fetcherOutputs.fetched (source.hash or null);
"${maybeArchive}/${source.dir}" in
else if source ? dir
maybeArchive; then "${maybeArchive}/${source.dir}"
else maybeArchive;
# fetch a source defined by a shortcut # fetch a source defined by a shortcut
fetchShortcut = { shortcut, extract ? false, }: fetchShortcut = {
shortcut,
extract ? false,
}:
fetchSource { fetchSource {
source = translateShortcut { inherit shortcut; }; source = translateShortcut {inherit shortcut;};
inherit extract; inherit extract;
}; };
parseShortcut = shortcut: parseShortcut = shortcut: let
let # in: "git+https://foo.com/bar?kwarg1=lol&kwarg2=hello"
# in: "git+https://foo.com/bar?kwarg1=lol&kwarg2=hello" # out: [ "git+" "git" "https" "//" "foo.com/bar" "?kwarg1=lol&kwarg2=hello" "kwarg1=lol&kwarg2=hello" ]
# out: [ "git+" "git" "https" "//" "foo.com/bar" "?kwarg1=lol&kwarg2=hello" "kwarg1=lol&kwarg2=hello" ] split =
split = b.match
b.match ''(([[:alnum:]]+)\+)?([[:alnum:]-]+):(//)?([^\?]*)(\?(.*))?''
''(([[:alnum:]]+)\+)?([[:alnum:]-]+):(//)?([^\?]*)(\?(.*))?'' shortcut;
shortcut;
parsed = { parsed = {
proto1 = b.elemAt split 1; proto1 = b.elemAt split 1;
proto2 = b.elemAt split 2; proto2 = b.elemAt split 2;
path = b.elemAt split 4; path = b.elemAt split 4;
allArgs = b.elemAt split 6; allArgs = b.elemAt split 6;
kwargs = b.removeAttrs kwargs_ [ "dir" ]; kwargs = b.removeAttrs kwargs_ ["dir"];
dir = kwargs_.dir or null; dir = kwargs_.dir or null;
}; };
kwargs_ = kwargs_ =
if parsed.allArgs == null then if parsed.allArgs == null
{} then {}
else
lib.listToAttrs
(map
(kwarg:
let
split = lib.splitString "=" kwarg;
in
lib.nameValuePair
(b.elemAt split 0)
(b.elemAt split 1))
(lib.splitString "&" parsed.allArgs));
in
if split == null then
throw "Unable to parse shortcut: ${shortcut}"
else else
parsed; lib.listToAttrs
(map
renderUrlArgs = kwargs: (kwarg: let
let split = lib.splitString "=" kwarg;
asStr = in
(lib.concatStringsSep lib.nameValuePair
"&" (b.elemAt split 0)
(lib.mapAttrsToList (b.elemAt split 1))
(name: val: "${name}=${val}") (lib.splitString "&" parsed.allArgs));
kwargs)); in
if split == null
in then throw "Unable to parse shortcut: ${shortcut}"
if asStr == "" then else parsed;
""
else
"?" + asStr;
renderUrlArgs = kwargs: let
asStr =
lib.concatStringsSep
"&"
(lib.mapAttrsToList
(name: val: "${name}=${val}")
kwargs);
in
if asStr == ""
then ""
else "?" + asStr;
# translate shortcut to dream lock source spec # translate shortcut to dream lock source spec
translateShortcut = { shortcut, computeHash ? true, }: translateShortcut = {
let shortcut,
computeHash ? true,
parsed = parseShortcut shortcut; }: let
parsed = parseShortcut shortcut;
checkArgs = fetcherName: args:
let
fetcher = fetchers."${fetcherName}";
unknownArgNames = lib.filter (argName: ! lib.elem argName fetcher.inputs) (lib.attrNames args);
missingArgNames = lib.filter (inputName: ! args ? "${inputName}") fetcher.inputs;
in
if lib.length unknownArgNames > 0 then
throw "Received unknown arguments for fetcher '${fetcherName}': ${b.toString unknownArgNames}"
else if lib.length missingArgNames > 0 then
throw "Missing arguments for fetcher '${fetcherName}': ${b.toString missingArgNames}"
else
args;
translateHttpUrl =
let
fetcher = fetchers.http;
urlArgsFinal = renderUrlArgs parsed.kwargs;
url = with parsed; "${proto2}://${path}${urlArgsFinal}";
fetcherOutputs = fetchers.http.outputs {
inherit url;
};
in
constructSource
{
inherit url;
type = "http";
}
// (lib.optionalAttrs (parsed.dir != null) {
dir = parsed.dir;
})
// (lib.optionalAttrs computeHash {
hash = fetcherOutputs.calcHash "sha256";
});
translateProtoShortcut =
let
kwargsUrl = b.removeAttrs parsed.kwargs fetcher.inputs;
urlArgs = renderUrlArgs kwargsUrl;
url = with parsed; "${proto2}://${path}${urlArgs}";
fetcherName = parsed.proto1;
fetcher = fetchers."${fetcherName}";
args = parsed.kwargs // { inherit url; };
fetcherOutputs = fetcher.outputs (checkArgs fetcherName args);
in
constructSource
(parsed.kwargs // {
type = fetcherName;
inherit url;
}
// (lib.optionalAttrs (parsed.dir != null) {
dir = parsed.dir;
})
// (lib.optionalAttrs computeHash {
hash = fetcherOutputs.calcHash "sha256";
}));
translateRegularShortcut =
let
fetcherName = parsed.proto2;
path = lib.removeSuffix "/" parsed.path;
params = lib.splitString "/" path;
fetcher = fetchers."${fetcherName}";
args =
if fetcher ? parseParams then
fetcher.parseParams params
else if b.length params != b.length fetcher.inputs then
throw ''
Wrong number of arguments provided in shortcut for fetcher '${fetcherName}'
Should be ${fetcherName}:${lib.concatStringsSep "/" fetcher.inputs}
''
else
lib.listToAttrs
(lib.forEach
(lib.range 0 ((lib.length fetcher.inputs) - 1))
(idx:
lib.nameValuePair
(lib.elemAt fetcher.inputs idx)
(lib.elemAt params idx)
));
fetcherOutputs = fetcher.outputs (args // parsed.kwargs);
in
constructSource (args // parsed.kwargs // {
type = fetcherName;
}
// (lib.optionalAttrs (parsed.dir != null) {
dir = parsed.dir;
})
// (lib.optionalAttrs computeHash {
hash = fetcherOutputs.calcHash "sha256";
}));
checkArgs = fetcherName: args: let
fetcher = fetchers."${fetcherName}";
unknownArgNames = lib.filter (argName: ! lib.elem argName fetcher.inputs) (lib.attrNames args);
missingArgNames = lib.filter (inputName: ! args ? "${inputName}") fetcher.inputs;
in in
if parsed.proto1 != null then if lib.length unknownArgNames > 0
translateProtoShortcut then throw "Received unknown arguments for fetcher '${fetcherName}': ${b.toString unknownArgNames}"
else if lib.hasPrefix "http://" shortcut else if lib.length missingArgNames > 0
|| lib.hasPrefix "https://" shortcut then then throw "Missing arguments for fetcher '${fetcherName}': ${b.toString missingArgNames}"
translateHttpUrl else args;
else
translateRegularShortcut;
translateHttpUrl = let
fetcher = fetchers.http;
urlArgsFinal = renderUrlArgs parsed.kwargs;
url = with parsed; "${proto2}://${path}${urlArgsFinal}";
fetcherOutputs = fetchers.http.outputs {
inherit url;
};
in
constructSource
{
inherit url;
type = "http";
}
// (lib.optionalAttrs (parsed.dir != null) {
dir = parsed.dir;
})
// (lib.optionalAttrs computeHash {
hash = fetcherOutputs.calcHash "sha256";
});
translateProtoShortcut = let
kwargsUrl = b.removeAttrs parsed.kwargs fetcher.inputs;
urlArgs = renderUrlArgs kwargsUrl;
url = with parsed; "${proto2}://${path}${urlArgs}";
fetcherName = parsed.proto1;
fetcher = fetchers."${fetcherName}";
args = parsed.kwargs // {inherit url;};
fetcherOutputs = fetcher.outputs (checkArgs fetcherName args);
in
constructSource
(parsed.kwargs
// {
type = fetcherName;
inherit url;
}
// (lib.optionalAttrs (parsed.dir != null) {
dir = parsed.dir;
})
// (lib.optionalAttrs computeHash {
hash = fetcherOutputs.calcHash "sha256";
}));
translateRegularShortcut = let
fetcherName = parsed.proto2;
path = lib.removeSuffix "/" parsed.path;
params = lib.splitString "/" path;
fetcher = fetchers."${fetcherName}";
args =
if fetcher ? parseParams
then fetcher.parseParams params
else if b.length params != b.length fetcher.inputs
then
throw ''
Wrong number of arguments provided in shortcut for fetcher '${fetcherName}'
Should be ${fetcherName}:${lib.concatStringsSep "/" fetcher.inputs}
''
else
lib.listToAttrs
(lib.forEach
(lib.range 0 ((lib.length fetcher.inputs) - 1))
(
idx:
lib.nameValuePair
(lib.elemAt fetcher.inputs idx)
(lib.elemAt params idx)
));
fetcherOutputs = fetcher.outputs (args // parsed.kwargs);
in
constructSource (args
// parsed.kwargs
// {
type = fetcherName;
}
// (lib.optionalAttrs (parsed.dir != null) {
dir = parsed.dir;
})
// (lib.optionalAttrs computeHash {
hash = fetcherOutputs.calcHash "sha256";
}));
in
if parsed.proto1 != null
then translateProtoShortcut
else if
lib.hasPrefix "http://" shortcut
|| lib.hasPrefix "https://" shortcut
then translateHttpUrl
else translateRegularShortcut;
} }

61
src/fetchers/git/default.nix
View File

@ -1,15 +1,11 @@
{ {
fetchgit, fetchgit,
lib, lib,
utils, utils,
... ...
}: }: let
let
b = builtins; b = builtins;
in in {
{
inputs = [ inputs = [
"url" "url"
"rev" "rev"
@ -17,47 +13,50 @@ in
versionField = "rev"; versionField = "rev";
outputs = { url, rev, ... }@inp: outputs = {
if b.match "refs/(heads|tags)/.*" rev == null && builtins.match "[a-f0-9]*" rev == null then url,
throw ''rev must either be a sha1 revision or "refs/heads/branch-name" or "refs/tags/tag-name"'' rev,
else ...
let } @ inp:
if b.match "refs/(heads|tags)/.*" rev == null && builtins.match "[a-f0-9]*" rev == null
then throw ''rev must either be a sha1 revision or "refs/heads/branch-name" or "refs/tags/tag-name"''
else let
b = builtins; b = builtins;
refAndRev = refAndRev =
if b.match "refs/(heads|tags)/.*" inp.rev != null then if b.match "refs/(heads|tags)/.*" inp.rev != null
{ ref = inp.rev; } then {ref = inp.rev;}
else else {rev = inp.rev;};
{ rev = inp.rev; }; in {
calcHash = algo:
in utils.hashPath algo
{
calcHash = algo: utils.hashPath algo
(b.fetchGit (b.fetchGit
(refAndRev // { (refAndRev
inherit url; // {
allRefs = true; inherit url;
submodules = true; allRefs = true;
})); submodules = true;
}));
# git can either be verified via revision or hash. # git can either be verified via revision or hash.
# In case revision is used for verification, `hash` will be null. # In case revision is used for verification, `hash` will be null.
fetched = hash: fetched = hash:
if hash == null then if hash == null
if ! refAndRev ? rev then then
throw "Cannot fetch git repo without integrity. Specify at least 'rev' or 'sha256'" if ! refAndRev ? rev
then throw "Cannot fetch git repo without integrity. Specify at least 'rev' or 'sha256'"
else else
b.fetchGit b.fetchGit
(refAndRev // { (refAndRev
// {
inherit url; inherit url;
allRefs = true; allRefs = true;
submodules = true; submodules = true;
}) })
else else
fetchgit fetchgit
(refAndRev // { (refAndRev
// {
inherit url; inherit url;
fetchSubmodules = true; fetchSubmodules = true;
sha256 = hash; sha256 = hash;

33
src/fetchers/github/default.nix
View File

@ -3,12 +3,9 @@
lib, lib,
nix, nix,
runCommand, runCommand,
utils, utils,
... ...
}: }: {
{
inputs = [ inputs = [
"owner" "owner"
"repo" "repo"
@ -19,20 +16,22 @@
defaultUpdater = "githubNewestReleaseTag"; defaultUpdater = "githubNewestReleaseTag";
outputs = { owner, repo, rev, ... }@inp: outputs = {
let owner,
b = builtins; repo,
in rev,
{ ...
} @ inp: let
calcHash = algo: utils.hashPath algo (b.fetchTarball { b = builtins;
in {
calcHash = algo:
utils.hashPath algo (b.fetchTarball {
url = "https://github.com/${owner}/${repo}/tarball/${rev}"; url = "https://github.com/${owner}/${repo}/tarball/${rev}";
}); });
fetched = hash: fetched = hash:
fetchFromGitHub { fetchFromGitHub {
inherit owner repo rev hash; inherit owner repo rev hash;
}; };
};
};
} }

33
src/fetchers/gitlab/default.nix
View File

@ -1,11 +1,8 @@
{ {
fetchFromGitLab, fetchFromGitLab,
utils, utils,
... ...
}: }: {
{
inputs = [ inputs = [
"owner" "owner"
"repo" "repo"
@ -14,20 +11,22 @@
versionField = "rev"; versionField = "rev";
outputs = { owner, repo, rev, ... }@inp: outputs = {
let owner,
b = builtins; repo,
in rev,
{ ...
} @ inp: let
calcHash = algo: utils.hashPath algo (b.fetchTarball { b = builtins;
in {
calcHash = algo:
utils.hashPath algo (b.fetchTarball {
url = "https://gitlab.com/${owner}/${repo}/-/archive/${rev}/${repo}-${rev}.tar.gz"; url = "https://gitlab.com/${owner}/${repo}/-/archive/${rev}/${repo}-${rev}.tar.gz";
}); });
fetched = hash: fetched = hash:
fetchFromGitLab { fetchFromGitLab {
inherit owner repo rev hash; inherit owner repo rev hash;
}; };
};
};
} }

63
src/fetchers/http/default.nix
View File

@ -1,51 +1,42 @@
{ {
lib, lib,
fetchurl, fetchurl,
utils, utils,
... ...
}: }: {
{
inputs = [ inputs = [
"url" "url"
]; ];
outputs = { url, ... }@inp: outputs = {url, ...} @ inp: let
let b = builtins;
b = builtins; in {
in calcHash = algo:
{ utils.hashFile algo (b.fetchurl {
calcHash = algo: utils.hashFile algo (b.fetchurl {
inherit url; inherit url;
}); });
fetched = hash: fetched = hash: let
let drv =
drv = if hash != null && lib.stringLength hash == 40
if hash != null && lib.stringLength hash == 40 then then
fetchurl { fetchurl {
inherit url; inherit url;
sha1 = hash; sha1 = hash;
} }
else else
fetchurl { fetchurl {
inherit url hash; inherit url hash;
}; };
drvSanitized = drvSanitized = drv.overrideAttrs (old: {
drv.overrideAttrs (old: { name = lib.strings.sanitizeDerivationName old.name;
name = lib.strings.sanitizeDerivationName old.name; });
});
extracted = extracted = utils.extractSource {
utils.extractSource { source = drvSanitized;
source = drvSanitized; };
}; in
extracted;
in };
extracted;
};
} }

99
src/fetchers/npm/default.nix
View File

@ -2,17 +2,12 @@
fetchurl, fetchurl,
lib, lib,
python3, python3,
utils, utils,
... ...
}: }: let
let
b = builtins; b = builtins;
in in rec {
inputs = ["pname" "version"];
rec {
inputs = [ "pname" "version" ];
versionField = "version"; versionField = "version";
@ -21,55 +16,57 @@ rec {
# becuase some node packages contain submodules like `@hhhtj/draw.io` # becuase some node packages contain submodules like `@hhhtj/draw.io`
# the amount of arguments can vary and a custom parser is needed # the amount of arguments can vary and a custom parser is needed
parseParams = params: parseParams = params:
if b.length params == b.length inputs then if b.length params == b.length inputs
then
lib.listToAttrs lib.listToAttrs
(lib.forEach (lib.forEach
(lib.range 0 ((lib.length inputs) - 1)) (lib.range 0 ((lib.length inputs) - 1))
(idx: (
idx:
lib.nameValuePair lib.nameValuePair
(lib.elemAt inputs idx) (lib.elemAt inputs idx)
(lib.elemAt params idx) (lib.elemAt params idx)
)) ))
else if b.length params == (b.length inputs) + 1 then else if b.length params == (b.length inputs) + 1
parseParams [ then
"${b.elemAt params 0}/${b.elemAt params 1}" parseParams [
(b.elemAt params 2) "${b.elemAt params 0}/${b.elemAt params 1}"
] (b.elemAt params 2)
else ]
throw '' else
Wrong number of arguments provided in shortcut for fetcher 'npm' throw ''
Should be npm:${lib.concatStringsSep "/" inputs} Wrong number of arguments provided in shortcut for fetcher 'npm'
''; Should be npm:${lib.concatStringsSep "/" inputs}
'';
# defaultUpdater = ""; # defaultUpdater = "";
outputs = { pname, version, }@inp: outputs = {
let pname,
b = builtins; version,
} @ inp: let
b = builtins;
submodule = lib.last (lib.splitString "/" pname); submodule = lib.last (lib.splitString "/" pname);
url = "https://registry.npmjs.org/${pname}/-/${submodule}-${version}.tgz"; url = "https://registry.npmjs.org/${pname}/-/${submodule}-${version}.tgz";
in in {
{ calcHash = algo:
utils.hashPath algo (
calcHash = algo: utils.hashPath algo ( b.fetchurl {inherit url;}
b.fetchurl { inherit url; }
); );
fetched = hash: fetched = hash: let
let source =
source = (fetchurl {
(fetchurl { inherit url;
inherit url; sha256 = hash;
sha256 = hash; })
}).overrideAttrs (old: { .overrideAttrs (old: {
outputHashMode = "recursive"; outputHashMode = "recursive";
}); });
in in
utils.extractSource { utils.extractSource {
inherit source; inherit source;
}; };
};
};
} }

23
src/fetchers/path/default.nix
View File

@ -1,22 +1,13 @@
{ {utils, ...}: {
utils,
...
}:
{
inputs = [ inputs = [
"path" "path"
]; ];
outputs = { path, ... }@inp: outputs = {path, ...} @ inp: let
let b = builtins;
b = builtins; in {
in calcHash = algo: utils.hashPath "${path}";
{
calcHash = algo: utils.hashPath "${path}"; fetched = hash: "${path}";
};
fetched = hash: "${path}";
};
} }

61
src/fetchers/pypi-sdist/default.nix
View File

@ -1,45 +1,44 @@
{ {
fetchurl, fetchurl,
python3, python3,
utils, utils,
... ...
}: }: {
{ inputs = ["pname" "version"];
inputs = [ "pname" "version" ];
versionField = "version"; versionField = "version";
defaultUpdater = "pypiNewestReleaseVersion"; defaultUpdater = "pypiNewestReleaseVersion";
outputs = { pname, version, extension ? "tar.gz", }@inp: outputs = {
let pname,
b = builtins; version,
extension ? "tar.gz",
} @ inp: let
b = builtins;
firstChar = builtins.substring 0 1 pname; firstChar = builtins.substring 0 1 pname;
url = url =
"https://files.pythonhosted.org/packages/source/" "https://files.pythonhosted.org/packages/source/"
+ "${firstChar}/${pname}/${pname}-${version}.${extension}"; + "${firstChar}/${pname}/${pname}-${version}.${extension}";
in in {
{ calcHash = algo:
utils.hashPath algo (
calcHash = algo: utils.hashPath algo ( b.fetchurl {inherit url;}
b.fetchurl { inherit url; }
); );
fetched = hash: fetched = hash: let
let source =
source = (fetchurl {
(fetchurl { inherit url;
inherit url; sha256 = hash;
sha256 = hash; })
}).overrideAttrs (old: { .overrideAttrs (old: {
outputHashMode = "recursive"; outputHashMode = "recursive";
}); });
in in
utils.extractSource { utils.extractSource {
inherit source; inherit source;
}; };
}; };
} }

425
src/lib.nix
View File

@ -1,7 +1,6 @@
# like ./default.nix but system intependent # like ./default.nix but system intependent
# (allows to generate outputs for several systems) # (allows to generate outputs for several systems)
# follows flake output schema # follows flake output schema
{ {
dlib, dlib,
nixpkgsSrc, nixpkgsSrc,
@ -9,272 +8,254 @@
overridesDirs, overridesDirs,
externalSources, externalSources,
externalPaths, externalPaths,
} @ args: let
}@args:
let
b = builtins; b = builtins;
l = lib // builtins; l = lib // builtins;
dream2nixForSystem = config: system: pkgs: dream2nixForSystem = config: system: pkgs:
import ./default.nix import ./default.nix
{ inherit config externalPaths externalSources pkgs; }; {inherit config externalPaths externalSources pkgs;};
# TODO: design output schema for cross compiled packages # TODO: design output schema for cross compiled packages
makePkgsKey = pkgs: makePkgsKey = pkgs: let
let build = pkgs.buildPlatform.system;
build = pkgs.buildPlatform.system; host = pkgs.hostPlatform.system;
host = pkgs.hostPlatform.system; in
in if build == host
if build == host then build then build
else throw "cross compiling currently not supported"; else throw "cross compiling currently not supported";
makeNixpkgs = pkgsList: systems: makeNixpkgs = pkgsList: systems:
# fail if neither pkgs nor systems are defined
# fail if neither pkgs nor systems are defined if pkgsList == null && systems == []
if pkgsList == null && systems == [] then then throw "Either `systems` or `pkgs` must be defined"
throw "Either `systems` or `pkgs` must be defined"
# fail if pkgs and systems are both defined # fail if pkgs and systems are both defined
else if pkgsList != null && systems != [] then else if pkgsList != null && systems != []
throw "Define either `systems` or `pkgs`, not both" then throw "Define either `systems` or `pkgs`, not both"
# only pkgs is specified # only pkgs is specified
else if pkgsList != null then else if pkgsList != null
if b.isList pkgsList then then
if b.isList pkgsList
then
lib.listToAttrs lib.listToAttrs
(pkgs: lib.nameValuePair (makePkgsKey pkgs) pkgs) (pkgs: lib.nameValuePair (makePkgsKey pkgs) pkgs)
pkgsList pkgsList
else else {"${makePkgsKey pkgsList}" = pkgsList;}
{ "${makePkgsKey pkgsList}" = pkgsList; }
# only systems is specified # only systems is specified
else else
lib.genAttrs systems lib.genAttrs systems
(system: import nixpkgsSrc { inherit system; }); (system: import nixpkgsSrc {inherit system;});
flakifyBuilderOutputs = system: outputs:
(lib.optionalAttrs (outputs ? "defaultPackage") {
defaultPackage."${system}" = outputs.defaultPackage;
})
// (lib.optionalAttrs (outputs ? "packages") {
packages."${system}" = outputs.packages;
})
// (lib.optionalAttrs (outputs ? "devShell") {
devShell."${system}" = outputs.devShell;
});
flakifyBuilderOutputs = system: outputs: init = {
(lib.optionalAttrs (outputs ? "defaultPackage") { pkgs ? null,
defaultPackage."${system}" = outputs.defaultPackage; systems ? [],
}) config ? {},
// } @ argsInit: let
(lib.optionalAttrs (outputs ? "packages") { config' = (import ./utils/config.nix).loadConfig argsInit.config or {};
packages."${system}" = outputs.packages;
})
//
(lib.optionalAttrs (outputs ? "devShell") {
devShell."${system}" = outputs.devShell;
});
init = config =
{ config'
pkgs ? null, // {
systems ? [],
config ? {},
}@argsInit:
let
config' = (import ./utils/config.nix).loadConfig argsInit.config or {};
config = config' // {
overridesDirs = args.overridesDirs ++ config'.overridesDirs; overridesDirs = args.overridesDirs ++ config'.overridesDirs;
}; };
allPkgs = makeNixpkgs pkgs systems; allPkgs = makeNixpkgs pkgs systems;
forAllSystems = f: lib.mapAttrs f allPkgs; forAllSystems = f: lib.mapAttrs f allPkgs;
dream2nixFor = forAllSystems (dream2nixForSystem config); dream2nixFor = forAllSystems (dream2nixForSystem config);
in in {
{ riseAndShine = throw "Use makeFlakeOutputs instead of riseAndShine.";
riseAndShine = throw "Use makeFlakeOutputs instead of riseAndShine."; makeFlakeOutputs = mArgs:
makeFlakeOutputsFunc
makeFlakeOutputs = mArgs: makeFlakeOutputsFunc (
( {inherit config pkgs systems;}
{ inherit config pkgs systems; } // mArgs
// mArgs
);
apps =
forAllSystems
(system: pkgs:
dream2nixFor."${system}".apps.flakeApps);
defaultApp =
forAllSystems
(system: pkgs:
dream2nixFor."${system}".apps.flakeApps.dream2nix);
builders =
forAllSystems
(system: pkgs:
dream2nixFor."${system}".builders);
};
makeFlakeOutputsFunc =
{
builder ? null,
pname ? null,
pkgs ? null,
source,
systems ? [],
translator ? null,
translatorArgs ? {},
...
}@args:
let
config = args.config or ((import ./utils/config.nix).loadConfig {});
argsForward = b.removeAttrs args [ "config" "pname" "pkgs" "systems" ];
allPkgs = makeNixpkgs pkgs systems;
forAllSystems = f: b.mapAttrs f allPkgs;
dream2nixFor = forAllSystems (dream2nixForSystem config);
translatorFound = dlib.translators.findOneTranslator {
inherit source;
translatorName = args.translator or null;
};
translatorFoundFor = forAllSystems (system: pkgs:
with translatorFound;
dream2nixFor."${system}".translators.translators
."${subsystem}"."${type}"."${name}"
); );
invalidationHash = dlib.calcInvalidationHash { apps =
inherit source translatorArgs; forAllSystems
translator = translatorFound.name; (system: pkgs:
}; dream2nixFor."${system}".apps.flakeApps);
specifyPnameError = throw '' defaultApp =
Translator `${translatorFound.name}` could not automatically determine `pname`. forAllSystems
Please specify `pname` when calling `makeFlakeOutputs` (system: pkgs:
''; dream2nixFor."${system}".apps.flakeApps.dream2nix);
detectedName = translatorFound.projectName; builders =
forAllSystems
(system: pkgs:
dream2nixFor."${system}".builders);
};
pname = makeFlakeOutputsFunc = {
if args.pname or null != null then builder ? null,
args.pname pname ? null,
else if detectedName != null then pkgs ? null,
detectedName source,
else systems ? [],
specifyPnameError; translator ? null,
translatorArgs ? {},
...
} @ args: let
config = args.config or ((import ./utils/config.nix).loadConfig {});
allBuilderOutputs = argsForward = b.removeAttrs args ["config" "pname" "pkgs" "systems"];
lib.mapAttrs
(system: pkgs:
let
dream2nix = dream2nixFor."${system}";
dreamLockJsonPath = with config; allPkgs = makeNixpkgs pkgs systems;
"${projectRoot}/${packagesDir}/${pname}/dream-lock.json";
dreamLock = dream2nix.utils.readDreamLock { forAllSystems = f: b.mapAttrs f allPkgs;
dreamLock = dreamLockJsonPath;
};
dreamLockExistsAndValid = dream2nixFor = forAllSystems (dream2nixForSystem config);
b.pathExists dreamLockJsonPath
&& dreamLock.lock._generic.invalidationHash or "" == invalidationHash;
result = translator: args: translatorFound = dlib.translators.findOneTranslator {
dream2nix.makeOutputs (argsForward // { inherit source;
# TODO: this triggers the translator finding routine a second time translatorName = args.translator or null;
translator = translatorFound.name; };
});
in
if dreamLockExistsAndValid then translatorFoundFor = forAllSystems (
# we need to override the source here as it is coming from system: pkgs:
# a flake input with translatorFound;
let dream2nixFor
defaultPackage = dreamLock.lock._generic.defaultPackage; ."${system}"
defaultPackageVersion = .translators
dreamLock.lock._generic.packages."${defaultPackage}"; .translators
in ."${subsystem}"
result translatorFound { ."${type}"
source = dreamLockJsonPath; ."${name}"
sourceOverrides = oldSources: { );
"${defaultPackage}"."${defaultPackageVersion}" =
args.source;
};
}
else if b.elem translatorFound.type [ "pure" "ifd" ] then invalidationHash = dlib.calcInvalidationHash {
# warn the user about potentially slow on-the-fly evaluation inherit source translatorArgs;
b.trace '' translator = translatorFound.name;
${"\n"} };
The dream-lock.json for input '${pname}' doesn't exist or is outdated.
...Falling back to on-the-fly evaluation (possibly slow).
To speed up future evalutations run once:
nix run .#resolve
''
result translatorFound {}
else specifyPnameError = throw ''
# print error because impure translation is required first. Translator `${translatorFound.name}` could not automatically determine `pname`.
# continue the evaluation anyways, as otherwise we won't have Please specify `pname` when calling `makeFlakeOutputs`
# the `resolve` app '';
b.trace ''
${"\n"}
ERROR:
Some information is missing to build this project reproducibly.
Please execute nix run .#resolve to resolve all impurities.
''
{})
allPkgs; detectedName = translatorFound.projectName;
flakifiedOutputsList = pname =
lib.mapAttrsToList if args.pname or null != null
(system: outputs: flakifyBuilderOutputs system outputs) then args.pname
allBuilderOutputs; else if detectedName != null
then detectedName
else specifyPnameError;
flakeOutputs = allBuilderOutputs =
b.foldl' lib.mapAttrs
(allOutputs: output: lib.recursiveUpdate allOutputs output) (system: pkgs: let
{} dream2nix = dream2nixFor."${system}";
flakifiedOutputsList;
in dreamLockJsonPath = with config; "${projectRoot}/${packagesDir}/${pname}/dream-lock.json";
lib.recursiveUpdate
flakeOutputs
{
apps = forAllSystems (system: pkgs: {
resolve.type = "app";
resolve.program =
let
utils = (dream2nixFor."${system}".utils);
# TODO: Too many calls to findOneTranslator. dreamLock = dream2nix.utils.readDreamLock {
# -> make findOneTranslator system independent dreamLock = dreamLockJsonPath;
translatorFound =
dream2nixFor."${system}".translators.findOneTranslator {
inherit source;
translatorName = args.translator or null;
};
in
b.toString
(utils.makePackageLockScript {
inherit source translatorArgs;
packagesDir = config.packagesDir;
translator = translatorFound.name;
});
});
}; };
in dreamLockExistsAndValid =
{ b.pathExists dreamLockJsonPath
&& dreamLock.lock._generic.invalidationHash or "" == invalidationHash;
result = translator: args:
dream2nix.makeOutputs (argsForward
// {
# TODO: this triggers the translator finding routine a second time
translator = translatorFound.name;
});
in
if dreamLockExistsAndValid
then
# we need to override the source here as it is coming from
# a flake input
let
defaultPackage = dreamLock.lock._generic.defaultPackage;
defaultPackageVersion =
dreamLock.lock._generic.packages."${defaultPackage}";
in
result translatorFound {
source = dreamLockJsonPath;
sourceOverrides = oldSources: {
"${defaultPackage}"."${defaultPackageVersion}" =
args.source;
};
}
else if b.elem translatorFound.type ["pure" "ifd"]
then
# warn the user about potentially slow on-the-fly evaluation
b.trace ''
${"\n"}
The dream-lock.json for input '${pname}' doesn't exist or is outdated.
...Falling back to on-the-fly evaluation (possibly slow).
To speed up future evalutations run once:
nix run .#resolve
''
result
translatorFound {}
else
# print error because impure translation is required first.
# continue the evaluation anyways, as otherwise we won't have
# the `resolve` app
b.trace ''
${"\n"}
ERROR:
Some information is missing to build this project reproducibly.
Please execute nix run .#resolve to resolve all impurities.
''
{})
allPkgs;
flakifiedOutputsList =
lib.mapAttrsToList
(system: outputs: flakifyBuilderOutputs system outputs)
allBuilderOutputs;
flakeOutputs =
b.foldl'
(allOutputs: output: lib.recursiveUpdate allOutputs output)
{}
flakifiedOutputsList;
in
lib.recursiveUpdate
flakeOutputs
{
apps = forAllSystems (system: pkgs: {
resolve.type = "app";
resolve.program = let
utils = dream2nixFor."${system}".utils;
# TODO: Too many calls to findOneTranslator.
# -> make findOneTranslator system independent
translatorFound = dream2nixFor."${system}".translators.findOneTranslator {
inherit source;
translatorName = args.translator or null;
};
in
b.toString
(utils.makePackageLockScript {
inherit source translatorArgs;
packagesDir = config.packagesDir;
translator = translatorFound.name;
});
});
};
in {
inherit dlib init; inherit dlib init;
riseAndShine = throw "Use makeFlakeOutputs instead of riseAndShine."; riseAndShine = throw "Use makeFlakeOutputs instead of riseAndShine.";
makeFlakeOutpus = makeFlakeOutputsFunc; makeFlakeOutpus = makeFlakeOutputsFunc;

240
src/lib/default.nix
View File

@ -2,12 +2,9 @@
lib, lib,
config ? (import ../utils/config.nix).loadConfig {}, config ? (import ../utils/config.nix).loadConfig {},
... ...
}: }: let
let
l = lib // builtins; l = lib // builtins;
# exported attributes # exported attributes
dlib = { dlib = {
inherit inherit
@ -27,20 +24,20 @@ let
sanitizeDerivationName sanitizeDerivationName
sanitizeRelativePath sanitizeRelativePath
traceJ traceJ
; ;
inherit (parseUtils) inherit
(parseUtils)
identifyGitUrl identifyGitUrl
parseGitUrl parseGitUrl
; ;
}; };
# other libs # other libs
translators = import ./translators.nix { inherit dlib lib; }; translators = import ./translators.nix {inherit dlib lib;};
discoverers = import ../discoverers { inherit config dlib lib; }; discoverers = import ../discoverers {inherit config dlib lib;};
parseUtils = import ./parsing.nix { inherit lib; };
parseUtils = import ./parsing.nix {inherit lib;};
# INTERNAL # INTERNAL
@ -49,21 +46,18 @@ let
# recursively applied as parameters. # recursively applied as parameters.
# For this to work, the function parameters defined by the called function # For this to work, the function parameters defined by the called function
# must always be ordered alphabetically. # must always be ordered alphabetically.
callWithAttrArgs = func: args: callWithAttrArgs = func: args: let
let applyParamsRec = func: params:
applyParamsRec = func: params: if l.length params == 1
if l.length params == 1 then then func (l.head params)
func (l.head params)
else
applyParamsRec
(func (l.head params))
(l.tail params);
in
if lib.functionArgs func == {} then
applyParamsRec func (l.attrValues args)
else else
func args; applyParamsRec
(func (l.head params))
(l.tail params);
in
if lib.functionArgs func == {}
then applyParamsRec func (l.attrValues args)
else func args;
# prepare source tree for executing discovery phase # prepare source tree for executing discovery phase
# produces this structure: # produces this structure:
@ -90,107 +84,98 @@ let
# }; # };
# }; # };
# } # }
prepareSourceTreeInternal = sourceRoot: relPath: name: depth: prepareSourceTreeInternal = sourceRoot: relPath: name: depth: let
let relPath' = relPath;
relPath' = relPath; fullPath' = "${sourceRoot}/${relPath}";
fullPath' = "${sourceRoot}/${relPath}"; current = l.readDir fullPath';
current = l.readDir fullPath';
fileNames = fileNames =
l.filterAttrs (n: v: v == "regular") current; l.filterAttrs (n: v: v == "regular") current;
directoryNames = directoryNames =
l.filterAttrs (n: v: v == "directory") current; l.filterAttrs (n: v: v == "directory") current;
makeNewPath = prefix: name: makeNewPath = prefix: name:
if prefix == "" then if prefix == ""
name then name
else else "${prefix}/${name}";
"${prefix}/${name}";
directories = directories =
l.mapAttrs l.mapAttrs
(dname: _: (dname: _:
prepareSourceTreeInternal prepareSourceTreeInternal
sourceRoot sourceRoot
(makeNewPath relPath dname) (makeNewPath relPath dname)
dname dname
(depth - 1)) (depth - 1))
directoryNames; directoryNames;
files = files =
l.mapAttrs l.mapAttrs
(fname: _: rec { (fname: _: rec {
name = fname; name = fname;
fullPath = "${fullPath'}/${fname}"; fullPath = "${fullPath'}/${fname}";
relPath = makeNewPath relPath' fname; relPath = makeNewPath relPath' fname;
content = readTextFile fullPath; content = readTextFile fullPath;
jsonContent = l.fromJSON content; jsonContent = l.fromJSON content;
tomlContent = l.fromTOML content; tomlContent = l.fromTOML content;
}) })
fileNames; fileNames;
getNodeFromPath = path: getNodeFromPath = path: let
let cleanPath = l.removePrefix "/" path;
cleanPath = l.removePrefix "/" path; pathSplit = l.splitString "/" cleanPath;
pathSplit = l.splitString "/" cleanPath; dirSplit = l.init pathSplit;
dirSplit = l.init pathSplit; leaf = l.last pathSplit;
leaf = l.last pathSplit; error = throw ''
error = throw '' Failed while trying to navigate to ${path} from ${fullPath'}
Failed while trying to navigate to ${path} from ${fullPath'} '';
'';
dirAttrPath = dirAttrPath =
l.init l.init
(l.concatMap (l.concatMap
(x: [x] ++ ["directories"]) (x: [x] ++ ["directories"])
dirSplit); dirSplit);
dir =
if (l.length dirSplit == 0) || dirAttrPath == [ "" ] then
self
else if ! l.hasAttrByPath dirAttrPath directories then
error
else
l.getAttrFromPath dirAttrPath directories;
in
if path == "" then
self
else if dir ? directories."${leaf}" then
dir.directories."${leaf}"
else if dir ? files."${leaf}" then
dir.files."${leaf}"
else
error;
self =
{
inherit files getNodeFromPath name relPath;
fullPath = fullPath';
}
# stop recursion if depth is reached
// (l.optionalAttrs (depth > 0) {
inherit directories;
});
dir =
if (l.length dirSplit == 0) || dirAttrPath == [""]
then self
else if ! l.hasAttrByPath dirAttrPath directories
then error
else l.getAttrFromPath dirAttrPath directories;
in in
self; if path == ""
then self
else if dir ? directories."${leaf}"
then dir.directories."${leaf}"
else if dir ? files."${leaf}"
then dir.files."${leaf}"
else error;
self =
{
inherit files getNodeFromPath name relPath;
fullPath = fullPath';
}
# stop recursion if depth is reached
// (l.optionalAttrs (depth > 0) {
inherit directories;
});
in
self;
# determines if version v1 is greater than version v2 # determines if version v1 is greater than version v2
versionGreater = v1: v2: l.compareVersions v1 v2 == 1; versionGreater = v1: v2: l.compareVersions v1 v2 == 1;
# EXPORTED # EXPORTED
# calculate an invalidation hash for given source translation inputs # calculate an invalidation hash for given source translation inputs
calcInvalidationHash = calcInvalidationHash = {
{ source,
source, translator,
translator, translatorArgs,
translatorArgs, }:
}:
l.hashString "sha256" '' l.hashString "sha256" ''
${source} ${source}
${translator} ${translator}
@ -199,19 +184,18 @@ let
''; '';
# call a function using arguments defined by the env var FUNC_ARGS # call a function using arguments defined by the env var FUNC_ARGS
callViaEnv = func: callViaEnv = func: let
let funcArgs = l.fromJSON (l.readFile (l.getEnv "FUNC_ARGS"));
funcArgs = l.fromJSON (l.readFile (l.getEnv "FUNC_ARGS")); in
in callWithAttrArgs func funcArgs;
callWithAttrArgs func funcArgs;
# Returns true if every given pattern is satisfied by at least one file name # Returns true if every given pattern is satisfied by at least one file name
# inside the given directory. # inside the given directory.
# Sub-directories are not recursed. # Sub-directories are not recursed.
containsMatchingFile = patterns: dir: containsMatchingFile = patterns: dir:
l.all l.all
(pattern: l.any (file: l.match pattern file != null) (listFiles dir)) (pattern: l.any (file: l.match pattern file != null) (listFiles dir))
patterns; patterns;
# directory names of a given directory # directory names of a given directory
dirNames = dir: l.attrNames (l.filterAttrs (name: type: type == "directory") (builtins.readDir dir)); dirNames = dir: l.attrNames (l.filterAttrs (name: type: type == "directory") (builtins.readDir dir));
@ -219,36 +203,32 @@ let
# picks the latest version from a list of version strings # picks the latest version from a list of version strings
latestVersion = versions: latestVersion = versions:
l.head l.head
(lib.sort versionGreater versions); (lib.sort versionGreater versions);
listDirs = path: l.attrNames (l.filterAttrs (n: v: v == "directory") (builtins.readDir path)); listDirs = path: l.attrNames (l.filterAttrs (n: v: v == "directory") (builtins.readDir path));
listFiles = path: l.attrNames (l.filterAttrs (n: v: v == "regular") (builtins.readDir path)); listFiles = path: l.attrNames (l.filterAttrs (n: v: v == "regular") (builtins.readDir path));
nameVersionPair = name: version: nameVersionPair = name: version: {inherit name version;};
{ inherit name version; };
prepareSourceTree = prepareSourceTree = {
{ source,
source, depth ? 10,
depth ? 10, }:
}:
prepareSourceTreeInternal source "" "" depth; prepareSourceTreeInternal source "" "" depth;
readTextFile = file: l.replaceStrings [ "\r\n" ] [ "\n" ] (l.readFile file); readTextFile = file: l.replaceStrings ["\r\n"] ["\n"] (l.readFile file);
# like nixpkgs recursiveUpdateUntil, but with the depth as a stop condition # like nixpkgs recursiveUpdateUntil, but with the depth as a stop condition
recursiveUpdateUntilDepth = depth: lhs: rhs: recursiveUpdateUntilDepth = depth: lhs: rhs:
lib.recursiveUpdateUntil (path: _: _: (l.length path) > depth) lhs rhs; lib.recursiveUpdateUntil (path: _: _: (l.length path) > depth) lhs rhs;
sanitizeDerivationName = name: sanitizeDerivationName = name:
lib.replaceStrings [ "@" "/" ] [ "__at__" "__slash__" ] name; lib.replaceStrings ["@" "/"] ["__at__" "__slash__"] name;
sanitizeRelativePath = path: sanitizeRelativePath = path:
l.removePrefix "/" (l.toString (l.toPath "/${path}")); l.removePrefix "/" (l.toString (l.toPath "/${path}"));
traceJ = toTrace: eval: l.trace (l.toJSON toTrace) eval; traceJ = toTrace: eval: l.trace (l.toJSON toTrace) eval;
in in
dlib
dlib

70
src/lib/parsing.nix
View File

@ -1,50 +1,38 @@
{ {lib, ...}: let
lib,
...
}:
let
b = builtins; b = builtins;
identifyGitUrl = url: identifyGitUrl = url:
lib.hasPrefix "git+" url lib.hasPrefix "git+" url
|| b.match ''^github:.*/.*#.*'' url != null; || b.match ''^github:.*/.*#.*'' url != null;
parseGitUrl = url: parseGitUrl = url: let
let githubMatch = b.match ''^github:(.*)/(.*)#(.*)$'' url;
githubMatch = b.match ''^github:(.*)/(.*)#(.*)$'' url; in
if githubMatch != null
then let
owner = b.elemAt githubMatch 0;
repo = b.elemAt githubMatch 1;
rev = b.elemAt githubMatch 2;
in {
url = "https://github.com/${owner}/${repo}";
inherit rev;
}
else let
splitUrlRev = lib.splitString "#" url;
rev = lib.last splitUrlRev;
urlOnly = lib.head splitUrlRev;
in in
if githubMatch != null then if lib.hasPrefix "git+ssh://" urlOnly
let then {
owner = b.elemAt githubMatch 0; inherit rev;
repo = b.elemAt githubMatch 1; url = "https://${(lib.last (lib.splitString "@" url))}";
rev = b.elemAt githubMatch 2; }
in else if lib.hasPrefix "git+https://" urlOnly
{ then {
url = "https://github.com/${owner}/${repo}"; inherit rev;
inherit rev; url = lib.removePrefix "git+" urlOnly;
} }
else else throw "Cannot parse git url: ${url}";
let in {
splitUrlRev = lib.splitString "#" url;
rev = lib.last splitUrlRev;
urlOnly = lib.head splitUrlRev;
in
if lib.hasPrefix "git+ssh://" urlOnly then
{
inherit rev;
url = "https://${(lib.last (lib.splitString "@" url))}";
}
else if lib.hasPrefix "git+https://" urlOnly then
{
inherit rev;
url = lib.removePrefix "git+" urlOnly;
}
else
throw "Cannot parse git url: ${url}";
in
{
inherit identifyGitUrl parseGitUrl; inherit identifyGitUrl parseGitUrl;
} }

310
src/lib/translators.nix
View File

@ -1,223 +1,199 @@
{ {
dlib, dlib,
lib, lib,
}: }: let
let
l = lib // builtins; l = lib // builtins;
# INTERNAL # INTERNAL
subsystems = dlib.dirNames ../translators; subsystems = dlib.dirNames ../translators;
translatorTypes = [ "impure" "ifd" "pure" ]; translatorTypes = ["impure" "ifd" "pure"];
# attrset of: subsystem -> translator-type -> (function subsystem translator-type) # attrset of: subsystem -> translator-type -> (function subsystem translator-type)
mkTranslatorsSet = function: mkTranslatorsSet = function:
l.genAttrs l.genAttrs
(dlib.dirNames ../translators) (dlib.dirNames ../translators)
(subsystem: (subsystem: let
let availableTypes =
availableTypes = l.filter
l.filter (type: l.pathExists (../translators + "/${subsystem}/${type}"))
(type: l.pathExists (../translators + "/${subsystem}/${type}")) translatorTypes;
translatorTypes;
translatorsForTypes = translatorsForTypes =
l.genAttrs l.genAttrs
availableTypes availableTypes
(transType: function subsystem transType); (transType: function subsystem transType);
in
in translatorsForTypes
translatorsForTypes // { // {
all = all =
l.foldl' l.foldl'
(a: b: a // b) (a: b: a // b)
{} {}
(l.attrValues translatorsForTypes); (l.attrValues translatorsForTypes);
}); });
# flat list of all translators sorted by priority (pure translators first) # flat list of all translators sorted by priority (pure translators first)
translatorsList = translatorsList = let
let list = l.collect (v: v ? subsystem) translators;
list = l.collect (v: v ? subsystem) translators; prio = translator:
prio = translator: if translator.type == "pure"
if translator.type == "pure" then then 0
0 else if translator.type == "ifd"
else if translator.type == "ifd" then then 1
1 else if translator.type == "impure"
else if translator.type == "impure" then then 2
2 else 3;
else in
3; l.sort
in (a: b: (prio a) < (prio b))
l.sort list;
(a: b: (prio a) < (prio b))
list;
callTranslator = subsystem: type: name: file: args:
let
translatorModule = import file {
inherit dlib lib;
};
in
translatorModule // {
inherit name subsystem type;
projectName =
if translatorModule ? projectName then
translatorModule.projectName
else
{ ... }: null;
};
callTranslator = subsystem: type: name: file: args: let
translatorModule = import file {
inherit dlib lib;
};
in
translatorModule
// {
inherit name subsystem type;
projectName =
if translatorModule ? projectName
then translatorModule.projectName
else {...}: null;
};
# EXPORTED # EXPORTED
# attrset of: subsystem -> translator-type -> translator # attrset of: subsystem -> translator-type -> translator
translators = mkTranslatorsSet (subsystem: type: translators = mkTranslatorsSet (
let subsystem: type: let
translatorNames = translatorNames =
dlib.dirNames (../translators + "/${subsystem}/${type}"); dlib.dirNames (../translators + "/${subsystem}/${type}");
translatorsLoaded = translatorsLoaded =
l.genAttrs l.genAttrs
translatorNames translatorNames
(translatorName: (translatorName:
callTranslator callTranslator
subsystem subsystem
type type
translatorName translatorName
(../translators + "/${subsystem}/${type}/${translatorName}") (../translators + "/${subsystem}/${type}/${translatorName}")
{}); {});
in in
translatorsLoaded translatorsLoaded
); );
mapTranslators = f: mapTranslators = f:
l.mapAttrs l.mapAttrs
(subsystem: types: (subsystem: types:
l.mapAttrs
(type: names:
l.mapAttrs l.mapAttrs
(type: names: (name: translator: f translator)
l.mapAttrs names)
(name: translator: f translator) types)
names) translators;
types)
translators;
# returns the list of translators including their special args # returns the list of translators including their special args
# and adds a flag `compatible` to each translator indicating # and adds a flag `compatible` to each translator indicating
# if the translator is compatible to all given paths # if the translator is compatible to all given paths
translatorsForInput = translatorsForInput = {source}:
{
source,
}:
l.forEach translatorsList l.forEach translatorsList
(t: rec { (t: rec {
inherit (t) inherit
extraArgs (t)
name extraArgs
subsystem name
type subsystem
type
; ;
compatible = t.compatible { inherit source; }; compatible = t.compatible {inherit source;};
projectName = t.projectName { inherit source; }; projectName = t.projectName {inherit source;};
}); });
# also includes subdirectories of the given paths up to a certain depth # also includes subdirectories of the given paths up to a certain depth
# to check for translator compatibility # to check for translator compatibility
translatorsForInputRecursive = translatorsForInputRecursive = {
{ source,
source, depth ? 2,
depth ? 2, }: let
}: listDirsRec = dir: depth: let
let subDirs =
listDirsRec = dir: depth: l.map
let (subdir: "${dir}/${subdir}")
subDirs = (dlib.listDirs dir);
l.map
(subdir: "${dir}/${subdir}")
(dlib.listDirs dir);
in
if depth == 0 then
subDirs
else
subDirs
++
(l.flatten
(map
(subDir: listDirsRec subDir (depth -1))
subDirs));
dirsToCheck =
[ source ]
++
(l.flatten
(map
(inputDir: listDirsRec inputDir depth)
[ source ]));
in in
l.genAttrs if depth == 0
dirsToCheck then subDirs
(dir: else
translatorsForInput { subDirs
source = dir; ++ (l.flatten
} (map
); (subDir: listDirsRec subDir (depth - 1))
subDirs));
dirsToCheck =
[source]
++ (l.flatten
(map
(inputDir: listDirsRec inputDir depth)
[source]));
in
l.genAttrs
dirsToCheck
(
dir:
translatorsForInput {
source = dir;
}
);
# pupulates a translators special args with defaults # pupulates a translators special args with defaults
getextraArgsDefaults = extraArgsDef: getextraArgsDefaults = extraArgsDef:
l.mapAttrs l.mapAttrs
(name: def: (
if def.type == "flag" then name: def:
false if def.type == "flag"
else then false
def.default or null else def.default or null
) )
extraArgsDef; extraArgsDef;
# return one compatible translator or throw error # return one compatible translator or throw error
findOneTranslator = findOneTranslator = {
{ source,
source, translatorName ? null,
translatorName ? null, } @ args: let
}@args: translatorsForSource = translatorsForInput {
let inherit source;
translatorsForSource = translatorsForInput { };
inherit source;
};
nameFilter = nameFilter =
if translatorName != null then if translatorName != null
(translator: translator.name == translatorName) then (translator: translator.name == translatorName)
else else (translator: true);
(translator: true);
compatibleTranslators =
let
result =
l.filter
(t: t.compatible)
translatorsForSource;
in
if result == [] then
throw "Could not find a compatible translator for input"
else
result;
translator =
l.findFirst
nameFilter
(throw ''Specified translator ${translatorName} not found or incompatible'')
compatibleTranslators;
compatibleTranslators = let
result =
l.filter
(t: t.compatible)
translatorsForSource;
in in
translator; if result == []
then throw "Could not find a compatible translator for input"
else result;
in translator =
{ l.findFirst
nameFilter
(throw ''Specified translator ${translatorName} not found or incompatible'')
compatibleTranslators;
in
translator;
in {
inherit inherit
findOneTranslator findOneTranslator
getextraArgsDefaults getextraArgsDefaults
@ -225,5 +201,5 @@ in
translators translators
translatorsForInput translatorsForInput
translatorsForInputRecursive translatorsForInputRecursive
; ;
} }

377
src/libV2.nix
View File

@ -1,7 +1,6 @@
# like ./default.nix but system intependent # like ./default.nix but system intependent
# (allows to generate outputs for several systems) # (allows to generate outputs for several systems)
# follows flake output schema # follows flake output schema
{ {
dlib, dlib,
nixpkgsSrc, nixpkgsSrc,
@ -9,236 +8,220 @@
overridesDirs, overridesDirs,
externalSources, externalSources,
externalPaths, externalPaths,
} @ args: let
}@args:
let
b = builtins; b = builtins;
l = lib // builtins; l = lib // builtins;
dream2nixForSystem = config: system: pkgs: dream2nixForSystem = config: system: pkgs:
import ./default.nix import ./default.nix
{ inherit config externalPaths externalSources pkgs; }; {inherit config externalPaths externalSources pkgs;};
# TODO: design output schema for cross compiled packages # TODO: design output schema for cross compiled packages
makePkgsKey = pkgs: makePkgsKey = pkgs: let
let build = pkgs.buildPlatform.system;
build = pkgs.buildPlatform.system; host = pkgs.hostPlatform.system;
host = pkgs.hostPlatform.system; in
in if build == host
if build == host then build then build
else throw "cross compiling currently not supported"; else throw "cross compiling currently not supported";
makeNixpkgs = pkgsList: systems: makeNixpkgs = pkgsList: systems:
# fail if neither pkgs nor systems are defined
# fail if neither pkgs nor systems are defined if pkgsList == null && systems == []
if pkgsList == null && systems == [] then then throw "Either `systems` or `pkgs` must be defined"
throw "Either `systems` or `pkgs` must be defined"
# fail if pkgs and systems are both defined # fail if pkgs and systems are both defined
else if pkgsList != null && systems != [] then else if pkgsList != null && systems != []
throw "Define either `systems` or `pkgs`, not both" then throw "Define either `systems` or `pkgs`, not both"
# only pkgs is specified # only pkgs is specified
else if pkgsList != null then else if pkgsList != null
if b.isList pkgsList then then
if b.isList pkgsList
then
lib.listToAttrs lib.listToAttrs
(pkgs: lib.nameValuePair (makePkgsKey pkgs) pkgs) (pkgs: lib.nameValuePair (makePkgsKey pkgs) pkgs)
pkgsList pkgsList
else else {"${makePkgsKey pkgsList}" = pkgsList;}
{ "${makePkgsKey pkgsList}" = pkgsList; }
# only systems is specified # only systems is specified
else else
lib.genAttrs systems lib.genAttrs systems
(system: import nixpkgsSrc { inherit system; }); (system: import nixpkgsSrc {inherit system;});
flakifyBuilderOutputs = system: outputs:
l.mapAttrs
(ouputType: outputValue: {"${system}" = outputValue;})
outputs;
flakifyBuilderOutputs = system: outputs: init = {
l.mapAttrs pkgs ? null,
(ouputType: outputValue: { "${system}" = outputValue; }) systems ? [],
outputs; config ? {},
} @ argsInit: let
config' = (import ./utils/config.nix).loadConfig argsInit.config or {};
init = config =
{ config'
pkgs ? null, // {
systems ? [],
config ? {},
}@argsInit:
let
config' = (import ./utils/config.nix).loadConfig argsInit.config or {};
config = config' // {
overridesDirs = args.overridesDirs ++ config'.overridesDirs; overridesDirs = args.overridesDirs ++ config'.overridesDirs;
}; };
allPkgs = makeNixpkgs pkgs systems; allPkgs = makeNixpkgs pkgs systems;
forAllSystems = f: lib.mapAttrs f allPkgs; forAllSystems = f: lib.mapAttrs f allPkgs;
dream2nixFor = forAllSystems (dream2nixForSystem config); dream2nixFor = forAllSystems (dream2nixForSystem config);
in in {
{ riseAndShine = throw "Use makeFlakeOutputs instead of riseAndShine.";
riseAndShine = throw "Use makeFlakeOutputs instead of riseAndShine."; makeFlakeOutputs = mArgs:
makeFlakeOutputsFunc
(
{inherit config pkgs systems;}
// mArgs
);
makeFlakeOutputs = mArgs: makeFlakeOutputsFunc apps =
( forAllSystems
{ inherit config pkgs systems; } (system: pkgs:
// mArgs dream2nixFor."${system}".apps.flakeApps);
);
apps = defaultApp =
forAllSystems forAllSystems
(system: pkgs: (system: pkgs:
dream2nixFor."${system}".apps.flakeApps); dream2nixFor."${system}".apps.flakeApps.dream2nix);
};
defaultApp = makeFlakeOutputsFunc = {
forAllSystems pname ? throw "Please pass `pname` to makeFlakeOutputs",
(system: pkgs: pkgs ? null,
dream2nixFor."${system}".apps.flakeApps.dream2nix); packageOverrides ? {},
settings ? [],
source,
systems ? [],
translator ? null,
translatorArgs ? {},
...
} @ args: let
config = args.config or ((import ./utils/config.nix).loadConfig {});
allPkgs = makeNixpkgs pkgs systems;
forAllSystems = f: b.mapAttrs f allPkgs;
dream2nixFor = forAllSystems (dream2nixForSystem config);
getInvalidationHash = project:
dlib.calcInvalidationHash {
inherit source;
# TODO: add translatorArgs
translatorArgs = {};
translator = project.translator;
}; };
makeFlakeOutputsFunc = discoveredProjects = dlib.discoverers.discoverProjects {
inherit settings;
tree = dlib.prepareSourceTree {inherit source;};
};
allBuilderOutputs =
lib.mapAttrs
(system: pkgs: let
dream2nix = dream2nixFor."${system}";
impureDiscoveredProjects =
l.filter
(proj:
dream2nix
.translators
.translatorsV2
."${proj.subsystem}"
.all
."${proj.translator}"
.type
== "impure")
discoveredProjects;
impureResolveScriptsList =
l.listToAttrs
(l.forEach impureDiscoveredProjects
(project:
l.nameValuePair
"Name: ${project.name}; Subsystem: ${project.subsystem}; relPath: ${project.relPath}"
(dream2nix.utils.makeTranslateScript {
inherit project source;
invalidationHash = getInvalidationHash project;
})));
resolveImpureScript =
dream2nix.utils.writePureShellScript
[]
''
cd $WORKDIR
${l.concatStringsSep "\n"
(l.mapAttrsToList
(title: script: ''
echo "Resolving:: ${title}"
${script}/bin/resolve
'')
impureResolveScriptsList)}
'';
translatedProjects = dream2nix.translateProjects {
inherit pname settings source;
};
realizedProjects = dream2nix.realizeProjects {
inherit packageOverrides translatedProjects source;
};
allOutputs =
realizedProjects
// {
apps.resolveImpure = {
type = "app";
program = l.toString resolveImpureScript;
};
};
in
allOutputs)
allPkgs;
flakifiedOutputsList =
lib.mapAttrsToList
(system: outputs: flakifyBuilderOutputs system outputs)
allBuilderOutputs;
flakeOutputsBuilders =
b.foldl'
(allOutputs: output: lib.recursiveUpdate allOutputs output)
{}
flakifiedOutputsList;
flakeOutputs =
{projectsJson = l.toJSON discoveredProjects;}
// flakeOutputsBuilders;
in
lib.recursiveUpdate
flakeOutputs
{ {
pname ? throw "Please pass `pname` to makeFlakeOutputs", apps = forAllSystems (system: pkgs: {
pkgs ? null, resolve.type = "app";
packageOverrides ? {}, resolve.program = let
settings ? [], utils = dream2nixFor."${system}".utils;
source,
systems ? [],
translator ? null,
translatorArgs ? {},
...
}@args:
let
config = args.config or ((import ./utils/config.nix).loadConfig {}); # TODO: Too many calls to findOneTranslator.
allPkgs = makeNixpkgs pkgs systems; # -> make findOneTranslator system independent
forAllSystems = f: b.mapAttrs f allPkgs; translatorFound = dream2nixFor."${system}".translators.findOneTranslator {
dream2nixFor = forAllSystems (dream2nixForSystem config); inherit source;
translatorName = args.translator or null;
getInvalidationHash = project: };
dlib.calcInvalidationHash { in
inherit source; b.toString
# TODO: add translatorArgs (utils.makePackageLockScript {
translatorArgs = {}; inherit source translatorArgs;
translator = project.translator; packagesDir = config.packagesDir;
}; translator = translatorFound.name;
discoveredProjects = dlib.discoverers.discoverProjects {
inherit settings;
tree = dlib.prepareSourceTree { inherit source; };
};
allBuilderOutputs =
lib.mapAttrs
(system: pkgs:
let
dream2nix = dream2nixFor."${system}";
impureDiscoveredProjects =
l.filter
(proj:
dream2nix.translators.translatorsV2."${proj.subsystem}".all
."${proj.translator}".type == "impure")
discoveredProjects;
impureResolveScriptsList =
l.listToAttrs
(l.forEach impureDiscoveredProjects
(project:
l.nameValuePair
"Name: ${project.name}; Subsystem: ${project.subsystem}; relPath: ${project.relPath}"
(dream2nix.utils.makeTranslateScript {
inherit project source;
invalidationHash = getInvalidationHash project;
})));
resolveImpureScript =
dream2nix.utils.writePureShellScript
[]
''
cd $WORKDIR
${l.concatStringsSep "\n"
(l.mapAttrsToList
(title: script: ''
echo "Resolving:: ${title}"
${script}/bin/resolve
'')
impureResolveScriptsList)}
'';
translatedProjects = dream2nix.translateProjects {
inherit pname settings source;
};
realizedProjects =
dream2nix.realizeProjects {
inherit packageOverrides translatedProjects source;
};
allOutputs =
realizedProjects
// {
apps.resolveImpure = {
type = "app";
program = l.toString resolveImpureScript;
};
};
in
allOutputs)
allPkgs;
flakifiedOutputsList =
lib.mapAttrsToList
(system: outputs: flakifyBuilderOutputs system outputs)
allBuilderOutputs;
flakeOutputsBuilders =
b.foldl'
(allOutputs: output: lib.recursiveUpdate allOutputs output)
{}
flakifiedOutputsList;
flakeOutputs =
{ projectsJson = l.toJSON (discoveredProjects); }
// flakeOutputsBuilders;
in
lib.recursiveUpdate
flakeOutputs
{
apps = forAllSystems (system: pkgs: {
resolve.type = "app";
resolve.program =
let
utils = (dream2nixFor."${system}".utils);
# TODO: Too many calls to findOneTranslator.
# -> make findOneTranslator system independent
translatorFound =
dream2nixFor."${system}".translators.findOneTranslator {
inherit source;
translatorName = args.translator or null;
};
in
b.toString
(utils.makePackageLockScript {
inherit source translatorArgs;
packagesDir = config.packagesDir;
translator = translatorFound.name;
});
}); });
}; });
};
in in {
{
inherit dlib init; inherit dlib init;
riseAndShine = throw "Use makeFlakeOutputs instead of riseAndShine."; riseAndShine = throw "Use makeFlakeOutputs instead of riseAndShine.";
makeFlakeOutpus = makeFlakeOutputsFunc; makeFlakeOutpus = makeFlakeOutputsFunc;

74
src/templates/builders/default.nix
View File

@ -2,47 +2,36 @@
lib, lib,
pkgs, pkgs,
stdenv, stdenv,
# dream2nix inputs # dream2nix inputs
builders, builders,
externals, externals,
utils, utils,
... ...
}: }: {
{
# Funcs # Funcs
# AttrSet -> Bool) -> AttrSet -> [x] # AttrSet -> Bool) -> AttrSet -> [x]
getCyclicDependencies, # name: version: -> [ {name=; version=; } ] getCyclicDependencies, # name: version: -> [ {name=; version=; } ]
getDependencies, # name: version: -> [ {name=; version=; } ] getDependencies, # name: version: -> [ {name=; version=; } ]
getSource, # name: version: -> store-path getSource, # name: version: -> store-path
buildPackageWithOtherBuilder, # { builder, name, version }: -> drv buildPackageWithOtherBuilder, # { builder, name, version }: -> drv
# Attributes # Attributes
subsystemAttrs, # attrset subsystemAttrs, # attrset
defaultPackageName, # string defaultPackageName, # string
defaultPackageVersion, # string defaultPackageVersion, # string
# attrset of pname -> versions, # attrset of pname -> versions,
# where versions is a list of version strings # where versions is a list of version strings
packageVersions, packageVersions,
# function which applies overrides to a package # function which applies overrides to a package
# It must be applied by the builder to each individual derivation # It must be applied by the builder to each individual derivation
# Example: # Example:
# produceDerivation name (mkDerivation {...}) # produceDerivation name (mkDerivation {...})
produceDerivation, produceDerivation,
# Custom Options: (parametrize builder behavior) # Custom Options: (parametrize builder behavior)
# These can be passed by the user via `builderArgs`. # These can be passed by the user via `builderArgs`.
# All options must provide default # All options must provide default
standalonePackageNames ? [], standalonePackageNames ? [],
... ...
}@args: } @ args: let
let
b = builtins; b = builtins;
# the main package # the main package
@ -51,37 +40,30 @@ let
# manage pakcages in attrset to prevent duplicated evaluation # manage pakcages in attrset to prevent duplicated evaluation
packages = packages =
lib.mapAttrs lib.mapAttrs
(name: versions: (name: versions:
lib.genAttrs lib.genAttrs
versions versions
(version: makeOnePackage name version)) (version: makeOnePackage name version))
packageVersions; packageVersions;
# Generates a derivation for a specific package name + version # Generates a derivation for a specific package name + version
makeOnePackage = name: version: makeOnePackage = name: version: let
let pkg = stdenv.mkDerivation rec {
pkg = pname = utils.sanitizeDerivationName name;
stdenv.mkDerivation rec { inherit version;
pname = utils.sanitizeDerivationName name; src = getSource name version;
inherit version;
src = getSource name version; buildInputs =
map
(dep: packages."${dep.name}"."${dep.version}")
(getDependencies name version);
buildInputs = # Implement build phases
map };
(dep: packages."${dep.name}"."${dep.version}") in
(getDependencies name version); # apply packageOverrides to current derivation
(utils.applyOverridesToPackage packageOverrides pkg name);
# Implement build phases in {
};
in
# apply packageOverrides to current derivation
(utils.applyOverridesToPackage packageOverrides pkg name);
in
{
inherit defaultPackage packages; inherit defaultPackage packages;
} }

92
src/templates/translators/impure.nix
View File

@ -1,10 +1,7 @@
{ {
dlib, dlib,
lib, lib,
}: }: {
{
# A derivation which outputs a single executable at `$out`. # A derivation which outputs a single executable at `$out`.
# The executable will be called by dream2nix for translation # The executable will be called by dream2nix for translation
# The input format is specified in /specifications/translator-call-example.json. # The input format is specified in /specifications/translator-call-example.json.
@ -15,66 +12,55 @@
# The program is expected to create a file at the location specified # The program is expected to create a file at the location specified
# by the input parameter `outFile`. # by the input parameter `outFile`.
# The output file must contain the dream lock data encoded as json. # The output file must contain the dream lock data encoded as json.
translateBin = translateBin = {
{ # dream2nix utils
# dream2nix utils utils,
utils, # nixpkgs dependenies
bash,
# nixpkgs dependenies jq,
bash, writeScriptBin,
jq, ...
writeScriptBin, }:
...
}:
utils.writePureShellScript utils.writePureShellScript
[ [
bash bash
coreutils coreutils
jq jq
nix nix
] ]
'' ''
# accroding to the spec, the translator reads the input from a json file # accroding to the spec, the translator reads the input from a json file
jsonInput=$1 jsonInput=$1
# read the json input # read the json input
outputFile=$(${jq}/bin/jq '.outputFile' -c -r $jsonInput) outputFile=$(${jq}/bin/jq '.outputFile' -c -r $jsonInput)
source=$(${jq}/bin/jq '.source' -c -r $jsonInput) source=$(${jq}/bin/jq '.source' -c -r $jsonInput)
inputFiles=$(${jq}/bin/jq '.inputFiles | .[]' -c -r $jsonInput) inputFiles=$(${jq}/bin/jq '.inputFiles | .[]' -c -r $jsonInput)
# TODO:
# read input files/dirs and produce a json file at $outputFile
# containing the dream lock similar to /specifications/dream-lock-example.json
'';
# TODO:
# read input files/dirs and produce a json file at $outputFile
# containing the dream lock similar to /specifications/dream-lock-example.json
'';
# This function should return the projects name. # This function should return the projects name.
# The computational complexity of this should be kept as lightweight # The computational complexity of this should be kept as lightweight
# as possible, as this migth be executed on a large amount of inputs at once. # as possible, as this migth be executed on a large amount of inputs at once.
projectName = projectName = {source}:
{
source,
}:
null; null;
# This allows the framework to detect if the translator is compatible with the given input # This allows the framework to detect if the translator is compatible with the given input
# to automatically select the right translator. # to automatically select the right translator.
compatible = compatible = {source}:
{ # TODO: insert regex here that matches valid input file names
source, # examples:
}: # - ''.*requirements.*\.txt''
# TODO: insert regex here that matches valid input file names # - ''.*package-lock\.json''
# examples:
# - ''.*requirements.*\.txt''
# - ''.*package-lock\.json''
dlib.containsMatchingFile dlib.containsMatchingFile
[ [
''TODO: regex1'' ''TODO: regex1''
''TODO: regex2'' ''TODO: regex2''
] ]
source; source;
# If the translator requires additional arguments, specify them here. # If the translator requires additional arguments, specify them here.
# There are only two types of arguments: # There are only two types of arguments:
@ -82,7 +68,6 @@
# - boolean flag (type = "flag") # - boolean flag (type = "flag")
# String arguments contain a default value and examples. Flags do not. # String arguments contain a default value and examples. Flags do not.
extraArgs = { extraArgs = {
# Example: boolean option # Example: boolean option
# Flags always default to 'false' if not specified by the user # Flags always default to 'false' if not specified by the user
noDev = { noDev = {
@ -100,6 +85,5 @@
]; ];
type = "argument"; type = "argument";
}; };
}; };
} }

369
src/translators/default.nix
View File

@ -6,92 +6,88 @@
nix, nix,
pkgs, pkgs,
python3, python3,
callPackageDream, callPackageDream,
externals, externals,
dream2nixWithExternals, dream2nixWithExternals,
utils, utils,
... ...
}: }: let
let
b = builtins; b = builtins;
l = lib // builtins; l = lib // builtins;
# transforms V1 translators to V2 translators # transforms V1 translators to V2 translators
ensureTranslatorV2 = translator: ensureTranslatorV2 = translator: let
let version = translator.version or 1;
version = translator.version or 1; cleanedArgs = args: l.removeAttrs args ["project" "tree"];
cleanedArgs = args: l.removeAttrs args [ "project" "tree" ];
upgradedTranslator = upgradedTranslator =
translator translator
// (lib.optionalAttrs (translator ? translate) { // (lib.optionalAttrs (translator ? translate) {
translate = args: translate = args: let
let dreamLock =
dreamLock = translator.translate
translator.translate ((cleanedArgs args)
((cleanedArgs args) // { // {
source = "${args.source}/${args.project.relPath}"; source = "${args.source}/${args.project.relPath}";
name = args.project.name; name = args.project.name;
}); });
in in
dreamLock // { dreamLock
_generic = dreamLock._generic // { // {
location = args.project.relPath; _generic =
}; dreamLock._generic
}; // {
}); location = args.project.relPath;
};
finalTranslator = };
if version == 2 then
translator
else
upgradedTranslator;
in
finalTranslator
// (lib.optionalAttrs (finalTranslator ? translate) {
translateBin =
wrapPureTranslator2
(with translator; [ subsystem type name ]);
# ensure `tree` is passed
translate = args:
finalTranslator.translate (args // {
tree =
args.tree or (dlib.prepareSourceTree { inherit (args) source; });
});
}); });
# transforms V2 translators to V1 translators finalTranslator =
ensureTranslatorV1 = translator: if version == 2
let then translator
version = translator.version or 1; else upgradedTranslator;
in
finalTranslator
// (lib.optionalAttrs (finalTranslator ? translate) {
translateBin =
wrapPureTranslator2
(with translator; [subsystem type name]);
# ensure `tree` is passed
translate = args:
finalTranslator.translate (args
// {
tree =
args.tree or (dlib.prepareSourceTree {inherit (args) source;});
});
});
downgradeTranslator = # transforms V2 translators to V1 translators
translator ensureTranslatorV1 = translator: let
// (lib.optionalAttrs (translator ? translate) { version = translator.version or 1;
translate = args:
translator.translate (args // { downgradeTranslator =
translator
// (lib.optionalAttrs (translator ? translate) {
translate = args:
translator.translate (args
// {
inherit (args) source; inherit (args) source;
tree = dlib.prepareSourceTree { inherit (args) source; }; tree = dlib.prepareSourceTree {inherit (args) source;};
project = { project = {
name = translator.projectName { inherit (args) source; }; name = translator.projectName {inherit (args) source;};
relPath = ""; relPath = "";
subsystem = translator.subsystem; subsystem = translator.subsystem;
}; };
}); });
}); });
finalTranslator =
if version == 1 then
translator
else
downgradeTranslator;
in
finalTranslator;
finalTranslator =
if version == 1
then translator
else downgradeTranslator;
in
finalTranslator;
makeTranslatorV2 = translatorModule: makeTranslatorV2 = translatorModule:
ensureTranslatorV2 (makeTranslator translatorModule); ensureTranslatorV2 (makeTranslator translatorModule);
@ -99,153 +95,144 @@ let
makeTranslatorV1 = translatorModule: makeTranslatorV1 = translatorModule:
ensureTranslatorV1 (makeTranslator translatorModule); ensureTranslatorV1 (makeTranslator translatorModule);
makeTranslator = translatorModule: let
makeTranslator = translator =
translatorModule: translatorModule
let # for pure translators
translator = # - import the `translate` function
translatorModule # - generate `translateBin`
// (lib.optionalAttrs (translatorModule ? translate) {
# for pure translators translate = let
# - import the `translate` function translateOriginal = callPackageDream translatorModule.translate {
# - generate `translateBin` translatorName = translatorModule.name;
// (lib.optionalAttrs (translatorModule ? translate) { };
translate = in
let args:
translateOriginal = callPackageDream translatorModule.translate { translateOriginal
translatorName = translatorModule.name; (
}; (dlib.translators.getextraArgsDefaults
in (translatorModule.extraArgs or {}))
args: translateOriginal // args
( );
(dlib.translators.getextraArgsDefaults translateBin =
(translatorModule.extraArgs or {})) wrapPureTranslator
// args (with translatorModule; [subsystem type name]);
); })
translateBin = # for impure translators:
wrapPureTranslator # - import the `translateBin` function
(with translatorModule; [ subsystem type name ]); // (lib.optionalAttrs (translatorModule ? translateBin) {
}) translateBin =
callPackageDream translatorModule.translateBin
# for impure translators: {
# - import the `translateBin` function translatorName = translatorModule.name;
// (lib.optionalAttrs (translatorModule ? translateBin) { };
translateBin = callPackageDream translatorModule.translateBin });
{ in
translatorName = translatorModule.name; translator;
};
});
in
translator;
translators = dlib.translators.mapTranslators makeTranslatorV1; translators = dlib.translators.mapTranslators makeTranslatorV1;
translatorsV2 = dlib.translators.mapTranslators makeTranslatorV2; translatorsV2 = dlib.translators.mapTranslators makeTranslatorV2;
# adds a translateBin to a pure translator # adds a translateBin to a pure translator
wrapPureTranslator2 = translatorAttrPath: wrapPureTranslator2 = translatorAttrPath: let
let bin =
bin = utils.writePureShellScript utils.writePureShellScript
[ [
coreutils coreutils
jq jq
nix nix
python3 python3
] ]
'' ''
jsonInputFile=$(realpath $1) jsonInputFile=$(realpath $1)
outputFile=$WORKDIR/$(jq '.outputFile' -c -r $jsonInputFile) outputFile=$WORKDIR/$(jq '.outputFile' -c -r $jsonInputFile)
cd $WORKDIR cd $WORKDIR
mkdir -p $(dirname $outputFile) mkdir -p $(dirname $outputFile)
nix eval \ nix eval \
--option experimental-features "nix-command flakes"\ --option experimental-features "nix-command flakes"\
--show-trace --impure --raw --expr " --show-trace --impure --raw --expr "
let let
dream2nix = import ${dream2nixWithExternals} {}; dream2nix = import ${dream2nixWithExternals} {};
translatorArgs = translatorArgs =
(builtins.fromJSON (builtins.fromJSON
(builtins.unsafeDiscardStringContext (builtins.readFile '''$1'''))); (builtins.unsafeDiscardStringContext (builtins.readFile '''$1''')));
dreamLock = dreamLock =
dream2nix.translators.translatorsV2.${ dream2nix.translators.translatorsV2.${
lib.concatStringsSep "." translatorAttrPath lib.concatStringsSep "." translatorAttrPath
}.translate }.translate
translatorArgs; translatorArgs;
in in
dream2nix.utils.dreamLock.toJSON dream2nix.utils.dreamLock.toJSON
# don't use nix to detect cycles, this will be more efficient in python # don't use nix to detect cycles, this will be more efficient in python
(dreamLock // { (dreamLock // {
_generic = builtins.removeAttrs dreamLock._generic [ \"cyclicDependencies\" ]; _generic = builtins.removeAttrs dreamLock._generic [ \"cyclicDependencies\" ];
}) })
" | python3 ${../apps/cli2/format-dream-lock.py} > $outputFile " | python3 ${../apps/cli2/format-dream-lock.py} > $outputFile
''; '';
in in
bin.overrideAttrs (old: { bin.overrideAttrs (old: {
name = "translator-${lib.concatStringsSep "-" translatorAttrPath}"; name = "translator-${lib.concatStringsSep "-" translatorAttrPath}";
}); });
# adds a translateBin to a pure translator # adds a translateBin to a pure translator
wrapPureTranslator = translatorAttrPath: wrapPureTranslator = translatorAttrPath: let
let bin =
bin = utils.writePureShellScript utils.writePureShellScript
[ [
coreutils coreutils
jq jq
nix nix
python3 python3
] ]
'' ''
jsonInputFile=$(realpath $1) jsonInputFile=$(realpath $1)
outputFile=$(jq '.outputFile' -c -r $jsonInputFile) outputFile=$(jq '.outputFile' -c -r $jsonInputFile)
cd $WORKDIR cd $WORKDIR
mkdir -p $(dirname $outputFile) mkdir -p $(dirname $outputFile)
nix eval \ nix eval \
--option experimental-features "nix-command flakes"\ --option experimental-features "nix-command flakes"\
--show-trace --impure --raw --expr " --show-trace --impure --raw --expr "
let let
dream2nix = import ${dream2nixWithExternals} {}; dream2nix = import ${dream2nixWithExternals} {};
translatorArgs = translatorArgs =
(builtins.fromJSON (builtins.fromJSON
(builtins.unsafeDiscardStringContext (builtins.readFile '''$1'''))); (builtins.unsafeDiscardStringContext (builtins.readFile '''$1''')));
dreamLock = dreamLock =
dream2nix.translators.translators.${ dream2nix.translators.translators.${
lib.concatStringsSep "." translatorAttrPath lib.concatStringsSep "." translatorAttrPath
}.translate }.translate
translatorArgs; translatorArgs;
in in
dream2nix.utils.dreamLock.toJSON dream2nix.utils.dreamLock.toJSON
# don't use nix to detect cycles, this will be more efficient in python # don't use nix to detect cycles, this will be more efficient in python
(dreamLock // { (dreamLock // {
_generic = builtins.removeAttrs dreamLock._generic [ \"cyclicDependencies\" ]; _generic = builtins.removeAttrs dreamLock._generic [ \"cyclicDependencies\" ];
}) })
" | python3 ${../apps/cli2/format-dream-lock.py} > $outputFile " | python3 ${../apps/cli2/format-dream-lock.py} > $outputFile
''; '';
in in
bin.overrideAttrs (old: { bin.overrideAttrs (old: {
name = "translator-${lib.concatStringsSep "-" translatorAttrPath}"; name = "translator-${lib.concatStringsSep "-" translatorAttrPath}";
}); });
in {
in
{
inherit inherit
translators translators
translatorsV2 translatorsV2
; ;
inherit (dlib.translators) inherit
(dlib.translators)
findOneTranslator findOneTranslator
translatorsForInput translatorsForInput
translatorsForInputRecursive translatorsForInputRecursive
; ;
} }

99
src/translators/go/impure/gomod2nix/default.nix
View File

@ -1,74 +1,59 @@
{ {
dlib, dlib,
lib, lib,
}: }: let
let
l = lib // builtins; l = lib // builtins;
in in {
{
# the input format is specified in /specifications/translator-call-example.json # the input format is specified in /specifications/translator-call-example.json
# this script receives a json file including the input paths and specialArgs # this script receives a json file including the input paths and specialArgs
translateBin = translateBin = {
{ # dream2nix utils
# dream2nix utils utils,
utils, dream2nixWithExternals,
dream2nixWithExternals, bash,
coreutils,
bash, jq,
coreutils, nix,
jq, writeScriptBin,
nix, ...
writeScriptBin, }:
...
}:
utils.writePureShellScript utils.writePureShellScript
[ [
bash bash
coreutils coreutils
jq jq
nix nix
] ]
'' ''
# accroding to the spec, the translator reads the input from a json file # accroding to the spec, the translator reads the input from a json file
jsonInput=$1 jsonInput=$1
# read the json input # read the json input
outputFile=$(${jq}/bin/jq '.outputFile' -c -r $jsonInput) outputFile=$(${jq}/bin/jq '.outputFile' -c -r $jsonInput)
source=$(${jq}/bin/jq '.source' -c -r $jsonInput) source=$(${jq}/bin/jq '.source' -c -r $jsonInput)
tmpBuild=$(mktemp -d) tmpBuild=$(mktemp -d)
cd $tmpBuild cd $tmpBuild
cp -r $source/* . cp -r $source/* .
chmod -R +w . chmod -R +w .
# This should be in sync with gomod2nix version in flake.lock # This should be in sync with gomod2nix version in flake.lock
nix run github:tweag/gomod2nix/67f22dd738d092c6ba88e420350ada0ed4992ae8 nix run github:tweag/gomod2nix/67f22dd738d092c6ba88e420350ada0ed4992ae8
nix eval --show-trace --impure --raw --expr "import ${./translate.nix} ${dream2nixWithExternals} ./." > $outputFile nix eval --show-trace --impure --raw --expr "import ${./translate.nix} ${dream2nixWithExternals} ./." > $outputFile
''; '';
projectName =
{
source,
}:
let
goModFile = "${source}/go.mod";
firstLine = (l.elemAt (l.splitString "\n" (l.readFile goModFile)) 0);
in
if l.pathExists goModFile then
l.last (l.splitString "/" (l.elemAt (l.splitString " " firstLine) 1))
else
null;
projectName = {source}: let
goModFile = "${source}/go.mod";
firstLine = l.elemAt (l.splitString "\n" (l.readFile goModFile)) 0;
in
if l.pathExists goModFile
then l.last (l.splitString "/" (l.elemAt (l.splitString " " firstLine) 1))
else null;
# This allows the framework to detect if the translator is compatible with the given input # This allows the framework to detect if the translator is compatible with the given input
# to automatically select the right translator. # to automatically select the right translator.
compatible = compatible = {source}:
{ dlib.containsMatchingFile [''go\.sum'' ''go\.mod''] source;
source,
}:
dlib.containsMatchingFile [ ''go\.sum'' ''go\.mod'' ] source;
# If the translator requires additional arguments, specify them here. # If the translator requires additional arguments, specify them here.
# There are only two types of arguments: # There are only two types of arguments:

95
src/translators/go/impure/gomod2nix/translate.nix
View File

@ -1,75 +1,66 @@
dream2nixWithExternals: dream2nixWithExternals: cwd: let
cwd: dream2nix = import dream2nixWithExternals {};
let
dream2nix = import dream2nixWithExternals { };
b = builtins; b = builtins;
parsed = b.fromTOML (builtins.readFile "${cwd}/gomod2nix.toml"); parsed = b.fromTOML (builtins.readFile "${cwd}/gomod2nix.toml");
pkgs = import <nixpkgs> { }; pkgs = import <nixpkgs> {};
lib = pkgs.lib; lib = pkgs.lib;
serializePackages = inputData: serializePackages = inputData:
lib.mapAttrsToList lib.mapAttrsToList
(goName: depAttrs: depAttrs // { inherit goName; }) (goName: depAttrs: depAttrs // {inherit goName;})
parsed; parsed;
translated = translated =
dream2nix.utils.simpleTranslate dream2nix.utils.simpleTranslate
({ ({
getDepByNameVer, getDepByNameVer,
dependenciesByOriginalID, dependenciesByOriginalID,
... ...
}: }: rec {
translatorName = "gomod2nix";
rec { inputData = parsed;
translatorName = "gomod2nix"; defaultPackage = let
firstLine = b.elemAt (lib.splitString "\n" (b.readFile "${cwd}/go.mod")) 0;
in
lib.last (lib.splitString "/" (b.elemAt (lib.splitString " " firstLine) 1));
inputData = parsed; packages."${defaultPackage}" = "unknown";
defaultPackage = subsystemName = "go";
let
firstLine = (b.elemAt (lib.splitString "\n" (b.readFile "${cwd}/go.mod")) 0);
in
lib.last (lib.splitString "/" (b.elemAt (lib.splitString " " firstLine) 1));
packages."${defaultPackage}" = "unknown"; subsystemAttrs = {};
subsystemName = "go"; inherit serializePackages;
subsystemAttrs = { }; mainPackageDependencies =
lib.forEach
(serializePackages parsed)
(dep: {
name = getName dep;
version = getVersion dep;
});
inherit serializePackages; getOriginalID = dependencyObject:
null;
mainPackageDependencies = getName = dependencyObject:
lib.forEach dependencyObject.goName;
(serializePackages parsed)
(dep: {
name = getName dep;
version = getVersion dep;
});
getOriginalID = dependencyObject: getVersion = dependencyObject:
null; lib.removePrefix "v" dependencyObject.sumVersion;
getName = dependencyObject: getDependencies = dependencyObject: [];
dependencyObject.goName;
getVersion = dependencyObject: getSourceType = dependencyObject: "git";
lib.removePrefix "v" dependencyObject.sumVersion;
getDependencies = dependencyObject: sourceConstructors = {
[]; git = dependencyObject: {
type = "git";
getSourceType = dependencyObject: "git"; hash = dependencyObject.fetch.sha256;
url = dependencyObject.fetch.url;
sourceConstructors = { rev = dependencyObject.fetch.rev;
git = dependencyObject:
{
type = "git";
hash = dependencyObject.fetch.sha256;
url = dependencyObject.fetch.url;
rev = dependencyObject.fetch.rev;
};
}; };
};
}); });
in in
dream2nix.utils.dreamLock.toJSON translated dream2nix.utils.dreamLock.toJSON translated

100
src/translators/nodejs/impure/package-json/default.nix
View File

@ -1,78 +1,68 @@
{ {
dlib, dlib,
lib, lib,
}: }: {
{
# the input format is specified in /specifications/translator-call-example.json # the input format is specified in /specifications/translator-call-example.json
# this script receives a json file including the input paths and specialArgs # this script receives a json file including the input paths and specialArgs
translateBin = translateBin = {
{ # dream2nix utils
# dream2nix utils translators,
translators, utils,
utils, # nixpkgs dependenies
bash,
# nixpkgs dependenies coreutils,
bash, jq,
coreutils, nodePackages,
jq, writeScriptBin,
nodePackages, ...
writeScriptBin, }:
...
}:
utils.writePureShellScript utils.writePureShellScript
[ [
bash bash
coreutils coreutils
jq jq
nodePackages.npm nodePackages.npm
] ]
'' ''
# accroding to the spec, the translator reads the input from a json file # accroding to the spec, the translator reads the input from a json file
jsonInput=$1 jsonInput=$1
# read the json input # read the json input
outputFile=$(jq '.outputFile' -c -r $jsonInput) outputFile=$(jq '.outputFile' -c -r $jsonInput)
source=$(jq '.source' -c -r $jsonInput) source=$(jq '.source' -c -r $jsonInput)
npmArgs=$(jq '.npmArgs' -c -r $jsonInput) npmArgs=$(jq '.npmArgs' -c -r $jsonInput)
cp -r $source/* ./ cp -r $source/* ./
chmod -R +w ./ chmod -R +w ./
rm -rf package-lock.json rm -rf package-lock.json
if [ "$(jq '.noDev' -c -r $jsonInput)" == "true" ]; then if [ "$(jq '.noDev' -c -r $jsonInput)" == "true" ]; then
echo "excluding dev dependencies" echo "excluding dev dependencies"
jq '.devDependencies = {}' ./package.json > package.json.mod jq '.devDependencies = {}' ./package.json > package.json.mod
mv package.json.mod package.json mv package.json.mod package.json
npm install --package-lock-only --production $npmArgs npm install --package-lock-only --production $npmArgs
else else
npm install --package-lock-only $npmArgs npm install --package-lock-only $npmArgs
fi fi
jq ".source = \"$(pwd)\"" -c -r $jsonInput > $TMPDIR/newJsonInput jq ".source = \"$(pwd)\"" -c -r $jsonInput > $TMPDIR/newJsonInput
cd $WORKDIR
${translators.translators.nodejs.pure.package-lock.translateBin} $TMPDIR/newJsonInput
'';
cd $WORKDIR
${translators.translators.nodejs.pure.package-lock.translateBin} $TMPDIR/newJsonInput
'';
# inherit projectName function from package-lock translator # inherit projectName function from package-lock translator
projectName = dlib.translators.translators.nodejs.pure.package-lock.projectName; projectName = dlib.translators.translators.nodejs.pure.package-lock.projectName;
# This allows the framework to detect if the translator is compatible with the given input # This allows the framework to detect if the translator is compatible with the given input
# to automatically select the right translator. # to automatically select the right translator.
compatible = compatible = {source}:
{ dlib.containsMatchingFile [''.*package.json''] source;
source,
}:
dlib.containsMatchingFile [ ''.*package.json'' ] source;
# inherit options from package-lock translator # inherit options from package-lock translator
extraArgs = extraArgs =
dlib.translators.translators.nodejs.pure.package-lock.extraArgs // { dlib.translators.translators.nodejs.pure.package-lock.extraArgs
// {
npmArgs = { npmArgs = {
description = "Additional arguments for npm"; description = "Additional arguments for npm";
type = "argument"; type = "argument";

426
src/translators/nodejs/pure/package-lock/default.nix
View File

@ -1,275 +1,248 @@
{ {
dlib, dlib,
lib, lib,
}: }: let
let
b = builtins; b = builtins;
l = lib // builtins; l = lib // builtins;
nodejsUtils = import ../../utils.nix { inherit lib; }; nodejsUtils = import ../../utils.nix {inherit lib;};
getPackageLock = tree: project: getPackageLock = tree: project:
nodejsUtils.getWorkspaceLockFile tree project "package-lock.json"; nodejsUtils.getWorkspaceLockFile tree project "package-lock.json";
translate = translate = {
{ translatorName,
translatorName, utils,
utils, ...
... }: {
}: project,
{ source,
project, tree,
source, # translator args
tree, noDev,
nodejs,
...
} @ args: let
b = builtins;
# translator args dev = ! noDev;
noDev, name = project.name;
nodejs, tree = args.tree.getNodeFromPath project.relPath;
... relPath = project.relPath;
}@args: source = "${args.source}/${relPath}";
let workspaces = project.subsystemInfo.workspaces or [];
b = builtins; packageLock = (getPackageLock args.tree project).jsonContent or null;
dev = ! noDev; packageJson =
name = project.name; (tree.getNodeFromPath "package.json").jsonContent;
tree = args.tree.getNodeFromPath project.relPath;
relPath = project.relPath;
source = "${args.source}/${relPath}";
workspaces = project.subsystemInfo.workspaces or [];
packageLock = (getPackageLock args.tree project).jsonContent or null; packageLockDeps =
if packageLock == null
then {}
else packageLock.dependencies or {};
packageJson = rootDependencies = packageLockDeps;
(tree.getNodeFromPath "package.json").jsonContent;
packageLockDeps = packageJsonDeps = nodejsUtils.getPackageJsonDeps packageJson noDev;
if packageLock == null then
{}
else
packageLock.dependencies or {};
rootDependencies = packageLockDeps; parsedDependencies =
l.filterAttrs
(name: dep: packageJsonDeps ? "${name}")
packageLockDeps;
packageJsonDeps = nodejsUtils.getPackageJsonDeps packageJson noDev; identifyGitSource = dependencyObject:
# TODO: when integrity is there, and git url is github then use tarball instead
parsedDependencies = # ! (dependencyObject ? integrity) &&
l.filterAttrs dlib.identifyGitUrl dependencyObject.version;
(name: dep: packageJsonDeps ? "${name}")
packageLockDeps;
identifyGitSource = dependencyObject:
# TODO: when integrity is there, and git url is github then use tarball instead
# ! (dependencyObject ? integrity) &&
dlib.identifyGitUrl dependencyObject.version;
getVersion = dependencyObject:
let
# example: "version": "npm:@tailwindcss/postcss7-compat@2.2.4",
npmMatch = b.match ''^npm:.*@(.*)$'' dependencyObject.version;
in
if npmMatch != null then
b.elemAt npmMatch 0
else if identifyGitSource dependencyObject then
"0.0.0-rc.${b.substring 0 8 (dlib.parseGitUrl dependencyObject.version).rev}"
else if lib.hasPrefix "file:" dependencyObject.version then
let
path = getPath dependencyObject;
in
(b.fromJSON
(b.readFile "${source}/${path}/package.json")
).version
else if lib.hasPrefix "https://" dependencyObject.version then
"unknown"
else
dependencyObject.version;
getPath = dependencyObject:
lib.removePrefix "file:" dependencyObject.version;
pinVersions = dependencies: parentScopeDeps:
lib.mapAttrs
(pname: pdata:
let
selfScopeDeps = parentScopeDeps // dependencies;
requires = pdata.requires or {};
dependencies = pdata.dependencies or {};
in
pdata // {
depsExact =
lib.forEach
(lib.attrNames requires)
(reqName: {
name = reqName;
version = getVersion selfScopeDeps."${reqName}";
});
dependencies = pinVersions dependencies selfScopeDeps;
}
)
dependencies;
pinnedRootDeps =
pinVersions rootDependencies rootDependencies;
createMissingSource = name: version:
{
type = "http";
url = "https://registry.npmjs.org/${name}/-/${name}-${version}.tgz";
};
getVersion = dependencyObject: let
# example: "version": "npm:@tailwindcss/postcss7-compat@2.2.4",
npmMatch = b.match ''^npm:.*@(.*)$'' dependencyObject.version;
in in
if npmMatch != null
then b.elemAt npmMatch 0
else if identifyGitSource dependencyObject
then "0.0.0-rc.${b.substring 0 8 (dlib.parseGitUrl dependencyObject.version).rev}"
else if lib.hasPrefix "file:" dependencyObject.version
then let
path = getPath dependencyObject;
in
(
b.fromJSON
(b.readFile "${source}/${path}/package.json")
)
.version
else if lib.hasPrefix "https://" dependencyObject.version
then "unknown"
else dependencyObject.version;
utils.simpleTranslate getPath = dependencyObject:
({ lib.removePrefix "file:" dependencyObject.version;
getDepByNameVer,
dependenciesByOriginalID,
...
}:
rec { pinVersions = dependencies: parentScopeDeps:
lib.mapAttrs
(
pname: pdata: let
selfScopeDeps = parentScopeDeps // dependencies;
requires = pdata.requires or {};
dependencies = pdata.dependencies or {};
in
pdata
// {
depsExact =
lib.forEach
(lib.attrNames requires)
(reqName: {
name = reqName;
version = getVersion selfScopeDeps."${reqName}";
});
dependencies = pinVersions dependencies selfScopeDeps;
}
)
dependencies;
inherit translatorName; pinnedRootDeps =
location = relPath; pinVersions rootDependencies rootDependencies;
# values createMissingSource = name: version: {
inputData = pinnedRootDeps; type = "http";
url = "https://registry.npmjs.org/${name}/-/${name}-${version}.tgz";
};
in
utils.simpleTranslate
({
getDepByNameVer,
dependenciesByOriginalID,
...
}: rec {
inherit translatorName;
location = relPath;
defaultPackage = # values
if name != "{automatic}" then inputData = pinnedRootDeps;
name
else
packageJson.name or (throw (
"Could not identify package name. "
+ "Please specify extra argument 'name'"
));
packages = defaultPackage =
{ "${defaultPackage}" = packageJson.version or "unknown"; } if name != "{automatic}"
// (nodejsUtils.getWorkspacePackages tree workspaces); then name
else
packageJson.name
or (throw (
"Could not identify package name. "
+ "Please specify extra argument 'name'"
));
mainPackageDependencies = packages =
lib.mapAttrsToList {"${defaultPackage}" = packageJson.version or "unknown";}
(pname: pdata: // (nodejsUtils.getWorkspacePackages tree workspaces);
{ name = pname; version = getVersion pdata; })
(lib.filterAttrs
(pname: pdata: ! (pdata.dev or false) || dev)
parsedDependencies);
subsystemName = "nodejs"; mainPackageDependencies =
lib.mapAttrsToList
(pname: pdata: {
name = pname;
version = getVersion pdata;
})
(lib.filterAttrs
(pname: pdata: ! (pdata.dev or false) || dev)
parsedDependencies);
subsystemAttrs = { nodejsVersion = args.nodejs; }; subsystemName = "nodejs";
# functions subsystemAttrs = {nodejsVersion = args.nodejs;};
serializePackages = inputData:
let
serialize = inputData:
lib.mapAttrsToList # returns list of lists
(pname: pdata:
[ (pdata // {
inherit pname;
depsExact =
lib.filter
(req:
(! (pdata.dependencies."${req.name}".bundled or false)))
pdata.depsExact or {};
}) ]
++
(lib.optionals (pdata ? dependencies)
(lib.flatten
(serialize
(lib.filterAttrs
(pname: data: ! data.bundled or false)
pdata.dependencies)))))
inputData;
in
lib.filter
(pdata:
dev || ! (pdata.dev or false))
(lib.flatten (serialize inputData));
getName = dependencyObject: dependencyObject.pname; # functions
serializePackages = inputData: let
serialize = inputData:
lib.mapAttrsToList # returns list of lists
(pname: pdata:
[
(pdata
// {
inherit pname;
depsExact =
lib.filter
(req: (! (pdata.dependencies."${req.name}".bundled or false)))
pdata.depsExact or {};
})
]
++ (lib.optionals (pdata ? dependencies)
(lib.flatten
(serialize
(lib.filterAttrs
(pname: data: ! data.bundled or false)
pdata.dependencies)))))
inputData;
in
lib.filter
(pdata:
dev || ! (pdata.dev or false))
(lib.flatten (serialize inputData));
inherit getVersion; getName = dependencyObject: dependencyObject.pname;
getSourceType = dependencyObject: inherit getVersion;
if identifyGitSource dependencyObject then
"git"
else if lib.hasPrefix "file:" dependencyObject.version then
"path"
else
"http";
sourceConstructors = { getSourceType = dependencyObject:
if identifyGitSource dependencyObject
then "git"
else if lib.hasPrefix "file:" dependencyObject.version
then "path"
else "http";
git = dependencyObject: sourceConstructors = {
dlib.parseGitUrl dependencyObject.version; git = dependencyObject:
dlib.parseGitUrl dependencyObject.version;
http = dependencyObject: http = dependencyObject:
if lib.hasPrefix "https://" dependencyObject.version then if lib.hasPrefix "https://" dependencyObject.version
rec { then rec {
version = getVersion dependencyObject; version = getVersion dependencyObject;
url = dependencyObject.version; url = dependencyObject.version;
hash = dependencyObject.integrity; hash = dependencyObject.integrity;
} }
else if dependencyObject.resolved == false then else if dependencyObject.resolved == false
(createMissingSource then
(getName dependencyObject) (createMissingSource
(getVersion dependencyObject)) (getName dependencyObject)
// { (getVersion dependencyObject))
hash = dependencyObject.integrity; // {
} hash = dependencyObject.integrity;
else }
rec { else rec {
url = dependencyObject.resolved; url = dependencyObject.resolved;
hash = dependencyObject.integrity; hash = dependencyObject.integrity;
}; };
path = dependencyObject: path = dependencyObject: rec {
rec { path = getPath dependencyObject;
path = getPath dependencyObject;
};
}; };
};
getDependencies = dependencyObject: getDependencies = dependencyObject:
dependencyObject.depsExact; dependencyObject.depsExact;
}); });
in in rec {
rec {
version = 2; version = 2;
inherit translate; inherit translate;
projectName = {source}: let
packageJson = "${source}/package.json";
parsed = b.fromJSON (b.readFile packageJson);
in
if b.pathExists packageJson && parsed ? name
then parsed.name
else null;
projectName = compatible = {source}:
{
source,
}:
let
packageJson = "${source}/package.json";
parsed = b.fromJSON (b.readFile packageJson);
in
if b.pathExists packageJson && parsed ? name then
parsed.name
else
null;
compatible =
{
source,
}:
dlib.containsMatchingFile dlib.containsMatchingFile
[ [
''.*package-lock\.json'' ''.*package-lock\.json''
''.*package.json'' ''.*package.json''
] ]
source; source;
extraArgs = { extraArgs = {
name = { name = {
description = "The name of the main package"; description = "The name of the main package";
examples = [ examples = [
@ -296,6 +269,5 @@ rec {
]; ];
type = "argument"; type = "argument";
}; };
}; };
} }

593
src/translators/nodejs/pure/yarn-lock/default.nix
View File

@ -1,327 +1,301 @@
{ {
dlib, dlib,
lib, lib,
}: }: let
let
l = lib // builtins; l = lib // builtins;
nodejsUtils = import ../../utils.nix { inherit lib; }; nodejsUtils = import ../../utils.nix {inherit lib;};
parser = import ./parser.nix { inherit lib; }; parser = import ./parser.nix {inherit lib;};
getYarnLock = tree: proj: getYarnLock = tree: proj:
tree.getNodeFromPath "${proj.relPath}/yarn.lock"; tree.getNodeFromPath "${proj.relPath}/yarn.lock";
translate = translate = {
{ translatorName,
translatorName, utils,
utils, ...
... }: {
}: project,
{ source,
project, tree,
source, # extraArgs
tree, nodejs,
noDev,
...
} @ args: let
b = builtins;
dev = ! noDev;
name = project.name;
relPath = project.relPath;
tree = args.tree.getNodeFromPath project.relPath;
workspaces = project.subsystemInfo.workspaces or [];
yarnLock = parser.parse (tree.getNodeFromPath "yarn.lock").content;
# extraArgs defaultPackage =
nodejs, if name != "{automatic}"
noDev, then name
... else
}@args: packageJson.name
or (throw (
"Could not identify package name. "
+ "Please specify extra argument 'name'"
));
let packageJson =
b = builtins; (tree.getNodeFromPath "package.json").jsonContent;
dev = ! noDev;
name = project.name;
relPath = project.relPath;
tree = args.tree.getNodeFromPath project.relPath;
workspaces = project.subsystemInfo.workspaces or [];
yarnLock = parser.parse (tree.getNodeFromPath "yarn.lock").content;
defaultPackage = packageJsonDeps = nodejsUtils.getPackageJsonDeps packageJson noDev;
if name != "{automatic}" then
name
else
packageJson.name or (throw (
"Could not identify package name. "
+ "Please specify extra argument 'name'"
));
packageJson =
(tree.getNodeFromPath "package.json").jsonContent;
packageJsonDeps = nodejsUtils.getPackageJsonDeps packageJson noDev;
workspacesPackageJson = nodejsUtils.getWorkspacePackageJson tree workspaces;
in
workspacesPackageJson = nodejsUtils.getWorkspacePackageJson tree workspaces;
in
utils.simpleTranslate2 utils.simpleTranslate2
({ ({objectsByKey, ...}: let
objectsByKey, makeWorkspaceExtraObject = workspace: let
... json = workspacesPackageJson."${workspace}";
}: let name = json.name or workspace;
version = json.version or "unknown";
in {
inherit name version;
makeWorkspaceExtraObject = workspace: let dependencies =
json = workspacesPackageJson."${workspace}";
name = json.name or workspace;
version = json.version or "unknown";
in
{
inherit name version;
dependencies =
l.mapAttrsToList
(depName: semVer: let
yarnName = "${depName}@${semVer}";
depObject = objectsByKey.yarnName."${yarnName}";
in
if exportedWorkspacePackages ? "${depName}"
then
{
name = depName;
version = exportedWorkspacePackages."${depName}";
}
else {name = depName; version = depObject.version;})
(nodejsUtils.getPackageJsonDeps json noDev);
sourceSpec = {
type = "path";
path = workspace;
rootName = defaultPackage;
rootVersion = packageJson.version or "unknown";
};
};
extraObjects = l.map makeWorkspaceExtraObject workspaces;
exportedWorkspacePackages =
l.listToAttrs
(l.map
(wsObject:
l.nameValuePair
wsObject.name
wsObject.version)
extraObjects);
getSourceType = rawObj: finalObj: let
dObj = rawObj;
in
if
lib.hasInfix "@github:" dObj.yarnName
|| (dObj ? resolved
&& lib.hasInfix "codeload.github.com/" dObj.resolved)
|| lib.hasInfix "@git+" dObj.yarnName
# example:
# "jest-image-snapshot@https://github.com/machard/jest-image-snapshot#machard-patch-1":
# version "4.2.0"
# resolved "https://github.com/machard/jest-image-snapshot#d087e8683859dba2964b5866a4d1eb02ba64e7b9"
|| (lib.hasInfix "@https://github.com" dObj.yarnName
&& lib.hasPrefix "https://github.com" dObj.resolved)
then
if dObj ? integrity
then
b.trace (
"Warning: Using git despite integrity exists for"
+ "${finalObj.name}"
)
"git"
else "git"
else if
lib.hasInfix "@link:" dObj.yarnName
|| lib.hasInfix "@file:" dObj.yarnName
then "path"
else "http";
in rec {
inherit defaultPackage extraObjects translatorName;
exportedPackages =
{ "${defaultPackage}" = packageJson.version or "unknown"; }
// exportedWorkspacePackages;
subsystemName = "nodejs";
subsystemAttrs = { nodejsVersion = args.nodejs; };
keys = {
yarnName = rawObj: finalObj:
rawObj.yarnName;
};
extractors = {
name = rawObj: finalObj:
if lib.hasInfix "@git+" rawObj.yarnName then
lib.head (lib.splitString "@git+" rawObj.yarnName)
# Example:
# @matrix-org/olm@https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.3.tgz
else if lib.hasInfix "@https://" rawObj.yarnName then
lib.head (lib.splitString "@https://" rawObj.yarnName)
else
let
split = lib.splitString "@" rawObj.yarnName;
version = lib.last split;
in
if lib.hasPrefix "@" rawObj.yarnName then
lib.removeSuffix "@${version}" rawObj.yarnName
else
lib.head split;
version = rawObj: finalObj:
if l.hasInfix "@git+" rawObj.yarnName
then
let
split = l.splitString "@git+" rawObj.yarnName;
gitUrl = l.last split;
in
# l.strings.sanitizeDerivationName
"${rawObj.version}@git+${gitUrl}"
else rawObj.version;
dependencies = rawObj: finalObj: let
dependencies = let
deps =
rawObj.dependencies or {}
// rawObj.optionalDependencies or {};
in
lib.mapAttrsToList
(name: version: { "${name}" = version; })
deps;
in
lib.forEach
dependencies
(dependency:
builtins.head (
lib.mapAttrsToList
(name: versionSpec: let
yarnName = "${name}@${versionSpec}";
depObject = objectsByKey.yarnName."${yarnName}";
version = depObject.version;
in
if ! objectsByKey.yarnName ? ${yarnName} then
# handle missing lock file entry
let
versionMatch =
b.match ''.*\^([[:digit:]|\.]+)'' versionSpec;
in
{
inherit name;
version = b.elemAt versionMatch 0;
}
else
{ inherit name version; }
)
dependency
)
);
sourceSpec = rawObj: finalObj: let
type = getSourceType rawObj finalObj;
in
{ inherit type; }
//
(if type == "git"
then
if utils.identifyGitUrl rawObj.resolved then
(utils.parseGitUrl rawObj.resolved) // {
version = rawObj.version;
}
else
let
githubUrlInfos = lib.splitString "/" rawObj.resolved;
owner = lib.elemAt githubUrlInfos 3;
repo = lib.elemAt githubUrlInfos 4;
in
if b.length githubUrlInfos == 7 then
let
rev = lib.elemAt githubUrlInfos 6;
in
{
url = "https://github.com/${owner}/${repo}";
inherit rev;
}
else if b.length githubUrlInfos == 5 then
let
urlAndRev = lib.splitString "#" rawObj.resolved;
in
{
url = lib.head urlAndRev;
rev = lib.last urlAndRev;
}
else
throw (
"Unable to parse git dependency for: "
+ "${finalObj.name}#${finalObj.version}"
)
else if type == "path"
then
if lib.hasInfix "@link:" rawObj.yarnName then
{
path =
lib.last (lib.splitString "@link:" rawObj.yarnName);
}
else if lib.hasInfix "@file:" rawObj.yarnName then
{
path =
lib.last (lib.splitString "@file:" rawObj.yarnName);
}
else
throw "unknown path format ${b.toJSON rawObj}"
else # type == "http"
{
type = "http";
hash =
if rawObj ? integrity then
rawObj.integrity
else
let
hash =
lib.last (lib.splitString "#" rawObj.resolved);
in
if lib.stringLength hash == 40 then
hash
else
throw "Missing integrity for ${rawObj.yarnName}";
url = lib.head (lib.splitString "#" rawObj.resolved);
});
};
extraDependencies =
l.mapAttrsToList l.mapAttrsToList
(name: semVer: let (depName: semVer: let
depYarnKey = "${name}@${semVer}"; yarnName = "${depName}@${semVer}";
dependencyAttrs = depObject = objectsByKey.yarnName."${yarnName}";
if ! yarnLock ? "${depYarnKey}" then in
throw "Cannot find entry for top level dependency: '${depYarnKey}'" if exportedWorkspacePackages ? "${depName}"
then {
name = depName;
version = exportedWorkspacePackages."${depName}";
}
else {
name = depName;
version = depObject.version;
})
(nodejsUtils.getPackageJsonDeps json noDev);
sourceSpec = {
type = "path";
path = workspace;
rootName = defaultPackage;
rootVersion = packageJson.version or "unknown";
};
};
extraObjects = l.map makeWorkspaceExtraObject workspaces;
exportedWorkspacePackages =
l.listToAttrs
(l.map
(wsObject:
l.nameValuePair
wsObject.name
wsObject.version)
extraObjects);
getSourceType = rawObj: finalObj: let
dObj = rawObj;
in
if
lib.hasInfix "@github:" dObj.yarnName
|| (dObj
? resolved
&& lib.hasInfix "codeload.github.com/" dObj.resolved)
|| lib.hasInfix "@git+" dObj.yarnName
# example:
# "jest-image-snapshot@https://github.com/machard/jest-image-snapshot#machard-patch-1":
# version "4.2.0"
# resolved "https://github.com/machard/jest-image-snapshot#d087e8683859dba2964b5866a4d1eb02ba64e7b9"
|| (lib.hasInfix "@https://github.com" dObj.yarnName
&& lib.hasPrefix "https://github.com" dObj.resolved)
then
if dObj ? integrity
then
b.trace (
"Warning: Using git despite integrity exists for"
+ "${finalObj.name}"
)
"git"
else "git"
else if
lib.hasInfix "@link:" dObj.yarnName
|| lib.hasInfix "@file:" dObj.yarnName
then "path"
else "http";
in rec {
inherit defaultPackage extraObjects translatorName;
exportedPackages =
{"${defaultPackage}" = packageJson.version or "unknown";}
// exportedWorkspacePackages;
subsystemName = "nodejs";
subsystemAttrs = {nodejsVersion = args.nodejs;};
keys = {
yarnName = rawObj: finalObj:
rawObj.yarnName;
};
extractors = {
name = rawObj: finalObj:
if lib.hasInfix "@git+" rawObj.yarnName
then lib.head (lib.splitString "@git+" rawObj.yarnName)
# Example:
# @matrix-org/olm@https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.3.tgz
else if lib.hasInfix "@https://" rawObj.yarnName
then lib.head (lib.splitString "@https://" rawObj.yarnName)
else let
split = lib.splitString "@" rawObj.yarnName;
version = lib.last split;
in
if lib.hasPrefix "@" rawObj.yarnName
then lib.removeSuffix "@${version}" rawObj.yarnName
else lib.head split;
version = rawObj: finalObj:
if l.hasInfix "@git+" rawObj.yarnName
then let
split = l.splitString "@git+" rawObj.yarnName;
gitUrl = l.last split;
in
# l.strings.sanitizeDerivationName
"${rawObj.version}@git+${gitUrl}"
else rawObj.version;
dependencies = rawObj: finalObj: let
dependencies = let
deps =
rawObj.dependencies
or {}
// rawObj.optionalDependencies or {};
in
lib.mapAttrsToList
(name: version: {"${name}" = version;})
deps;
in
lib.forEach
dependencies
(
dependency:
builtins.head (
lib.mapAttrsToList
(
name: versionSpec: let
yarnName = "${name}@${versionSpec}";
depObject = objectsByKey.yarnName."${yarnName}";
version = depObject.version;
in
if ! objectsByKey.yarnName ? ${yarnName}
then
# handle missing lock file entry
let
versionMatch =
b.match ''.*\^([[:digit:]|\.]+)'' versionSpec;
in {
inherit name;
version = b.elemAt versionMatch 0;
}
else {inherit name version;}
)
dependency
)
);
sourceSpec = rawObj: finalObj: let
type = getSourceType rawObj finalObj;
in
{inherit type;}
// (
if type == "git"
then
if utils.identifyGitUrl rawObj.resolved
then
(utils.parseGitUrl rawObj.resolved)
// {
version = rawObj.version;
}
else let
githubUrlInfos = lib.splitString "/" rawObj.resolved;
owner = lib.elemAt githubUrlInfos 3;
repo = lib.elemAt githubUrlInfos 4;
in
if b.length githubUrlInfos == 7
then let
rev = lib.elemAt githubUrlInfos 6;
in {
url = "https://github.com/${owner}/${repo}";
inherit rev;
}
else if b.length githubUrlInfos == 5
then let
urlAndRev = lib.splitString "#" rawObj.resolved;
in {
url = lib.head urlAndRev;
rev = lib.last urlAndRev;
}
else else
yarnLock."${depYarnKey}"; throw (
in "Unable to parse git dependency for: "
+ "${finalObj.name}#${finalObj.version}"
)
else if type == "path"
then
if lib.hasInfix "@link:" rawObj.yarnName
then {
path =
lib.last (lib.splitString "@link:" rawObj.yarnName);
}
else if lib.hasInfix "@file:" rawObj.yarnName
then {
path =
lib.last (lib.splitString "@file:" rawObj.yarnName);
}
else throw "unknown path format ${b.toJSON rawObj}"
else # type == "http"
{ {
name = defaultPackage; type = "http";
version = packageJson.version or "unknown"; hash =
dependencies = [ if rawObj ? integrity
{inherit name; version = dependencyAttrs.version;} then rawObj.integrity
]; else let
}) hash =
packageJsonDeps; lib.last (lib.splitString "#" rawObj.resolved);
in
if lib.stringLength hash == 40
then hash
else throw "Missing integrity for ${rawObj.yarnName}";
url = lib.head (lib.splitString "#" rawObj.resolved);
}
);
};
serializedRawObjects = extraDependencies =
lib.mapAttrsToList l.mapAttrsToList
(yarnName: depAttrs: depAttrs // { inherit yarnName; }) (name: semVer: let
yarnLock; depYarnKey = "${name}@${semVer}";
dependencyAttrs =
}); if ! yarnLock ? "${depYarnKey}"
then throw "Cannot find entry for top level dependency: '${depYarnKey}'"
else yarnLock."${depYarnKey}";
in {
name = defaultPackage;
version = packageJson.version or "unknown";
dependencies = [
{
inherit name;
version = dependencyAttrs.version;
}
];
})
packageJsonDeps;
serializedRawObjects =
lib.mapAttrsToList
(yarnName: depAttrs: depAttrs // {inherit yarnName;})
yarnLock;
});
in { in {
version = 2; version = 2;
inherit translate; inherit translate;
@ -329,15 +303,10 @@ in {
# inherit projectName function from package-lock translator # inherit projectName function from package-lock translator
projectName = dlib.translators.translators.nodejs.pure.package-lock.projectName; projectName = dlib.translators.translators.nodejs.pure.package-lock.projectName;
# This allows the framework to detect if the translator is compatible with the given input # This allows the framework to detect if the translator is compatible with the given input
# to automatically select the right translator. # to automatically select the right translator.
compatible = compatible = {source}:
{ dlib.containsMatchingFile [''.*yarn\.lock'' ''.*package.json''] source;
source,
}:
dlib.containsMatchingFile [ ''.*yarn\.lock'' ''.*package.json'' ] source;
# If the translator requires additional arguments, specify them here. # If the translator requires additional arguments, specify them here.
# There are only two types of arguments: # There are only two types of arguments:
@ -345,7 +314,6 @@ in {
# - boolean flag (type = "flag") # - boolean flag (type = "flag")
# String arguments contain a default value and examples. Flags do not. # String arguments contain a default value and examples. Flags do not.
extraArgs = { extraArgs = {
name = { name = {
description = "The name of the main package"; description = "The name of the main package";
examples = [ examples = [
@ -370,6 +338,5 @@ in {
]; ];
type = "argument"; type = "argument";
}; };
}; };
} }

234
src/translators/nodejs/pure/yarn-lock/parser.nix
View File

@ -1,139 +1,119 @@
{lib ? (import <nixpkgs> {}).lib, ...}: let
{
lib ? (import <nixpkgs> {}).lib,
...
}:
let
l = lib // builtins; l = lib // builtins;
parse = text: parse = text: let
let lines = l.splitString "\n" text;
lines = l.splitString "\n" text;
findStartLineNum = num: findStartLineNum = num: let
let line = l.elemAt lines num;
line = l.elemAt lines num; in
in if
if ! l.hasPrefix "#" line ! l.hasPrefix "#" line
&& ! l.hasPrefix " " line && ! l.hasPrefix " " line
&& ! l.hasPrefix "_" line then && ! l.hasPrefix "_" line
num then num
else else findStartLineNum (num + 1);
findStartLineNum (num + 1);
contentLines = contentLines =
l.sublist l.sublist
(findStartLineNum 0) (findStartLineNum 0)
((l.length lines) - 1) ((l.length lines) - 1)
lines; lines;
matchLine = line: matchLine = line: let
let # yarn v2
# yarn v2 m1 = l.match ''( *)(.*): (.*)'' line;
m1 = l.match ''( *)(.*): (.*)'' line; m2 = l.match ''( *)(.*):$'' line;
m2 = l.match ''( *)(.*):$'' line;
# yarn v1 # yarn v1
m3 = l.match ''( *)(.*) "(.*)"'' line; m3 = l.match ''( *)(.*) "(.*)"'' line;
m4 = l.match ''( *)(.*) (.*)'' line; m4 = l.match ''( *)(.*) (.*)'' line;
in in
if m1 != null then if m1 != null
{ then {
indent = (l.stringLength (l.elemAt m1 0)) / 2; indent = (l.stringLength (l.elemAt m1 0)) / 2;
key = l.elemAt m1 1; key = l.elemAt m1 1;
value = l.elemAt m1 2; value = l.elemAt m1 2;
} }
else if m2 != null then else if m2 != null
{ then {
indent = (l.stringLength (l.elemAt m2 0)) / 2; indent = (l.stringLength (l.elemAt m2 0)) / 2;
# transform yarn 1 to yarn 2 tyle # transform yarn 1 to yarn 2 tyle
key = key =
l.replaceStrings [ '', "'' ] [ '', '' ] l.replaceStrings ['', "''] ['', '']
(l.replaceStrings [ ''", '' ] [ '', '' ] (l.elemAt m2 1)); (l.replaceStrings [''", ''] ['', ''] (l.elemAt m2 1));
value = null; value = null;
} }
else if m3 != null then else if m3 != null
{ then {
indent = (l.stringLength (l.elemAt m3 0)) / 2; indent = (l.stringLength (l.elemAt m3 0)) / 2;
key = l.elemAt m3 1; key = l.elemAt m3 1;
value = l.elemAt m3 2; value = l.elemAt m3 2;
} }
else if m4 != null then else if m4 != null
{ then {
indent = (l.stringLength (l.elemAt m4 0)) / 2; indent = (l.stringLength (l.elemAt m4 0)) / 2;
key = l.elemAt m4 1; key = l.elemAt m4 1;
value = l.elemAt m4 2; value = l.elemAt m4 2;
} }
else else null;
null;
closingParenthesis = num: closingParenthesis = num:
if num == 1 then "}" else "}" + (closingParenthesis (num - 1)); if num == 1
then "}"
jsonLines = lines: else "}" + (closingParenthesis (num - 1));
let
filtered = l.filter (line: l.match ''[[:space:]]*'' line == null) lines;
matched = l.map (line: matchLine line) filtered;
in
l.imap0
(i: line:
let
mNext = l.elemAt matched (i + 1);
m = l.elemAt matched i;
keyParenthesis =
let
beginOK = l.hasPrefix ''"'' m.key;
endOK = l.hasSuffix ''"'' m.key;
begin = l.optionalString (! beginOK) ''"'';
end = l.optionalString (! endOK) ''"'';
in
''${begin}${m.key}${end}'';
valParenthesis =
if l.hasPrefix ''"'' m.value then
m.value
else
''"${m.value}"'';
in
if l.length filtered == i + 1 then
let
end = closingParenthesis m.indent;
in
''${keyParenthesis}: ${valParenthesis}${end}}''
else if m.value == null then
''${keyParenthesis}: {''
# if indent of next line is smaller, close the object
else if mNext.indent < m.indent then
let
end = closingParenthesis (m.indent - mNext.indent);
in
''${keyParenthesis}: ${valParenthesis}${end},''
else
''${keyParenthesis}: ${valParenthesis},'')
filtered;
json = "{${l.concatStringsSep "\n" (jsonLines contentLines)}";
dataRaw = l.fromJSON json;
# transform key collections like:
# "@babel/code-frame@^7.0.0, @babel/code-frame@^7.10.4"
# ... to individual entries
data =
l.listToAttrs
(l.flatten
(l.mapAttrsToList
(n: v:
let
keys = l.splitString ", " n;
in
l.map (k: l.nameValuePair k v) keys)
dataRaw));
jsonLines = lines: let
filtered = l.filter (line: l.match ''[[:space:]]*'' line == null) lines;
matched = l.map (line: matchLine line) filtered;
in
l.imap0
(i: line: let
mNext = l.elemAt matched (i + 1);
m = l.elemAt matched i;
keyParenthesis = let
beginOK = l.hasPrefix ''"'' m.key;
endOK = l.hasSuffix ''"'' m.key;
begin = l.optionalString (! beginOK) ''"'';
end = l.optionalString (! endOK) ''"'';
in ''${begin}${m.key}${end}'';
valParenthesis =
if l.hasPrefix ''"'' m.value
then m.value
else ''"${m.value}"'';
in in
data; if l.length filtered == i + 1
then let
end = closingParenthesis m.indent;
in ''${keyParenthesis}: ${valParenthesis}${end}}''
else if m.value == null
then ''${keyParenthesis}: {''
# if indent of next line is smaller, close the object
else if mNext.indent < m.indent
then let
end = closingParenthesis (m.indent - mNext.indent);
in ''${keyParenthesis}: ${valParenthesis}${end},''
else ''${keyParenthesis}: ${valParenthesis},'')
filtered;
in json = "{${l.concatStringsSep "\n" (jsonLines contentLines)}";
{
dataRaw = l.fromJSON json;
# transform key collections like:
# "@babel/code-frame@^7.0.0, @babel/code-frame@^7.10.4"
# ... to individual entries
data =
l.listToAttrs
(l.flatten
(l.mapAttrsToList
(n: v: let
keys = l.splitString ", " n;
in
l.map (k: l.nameValuePair k v) keys)
dataRaw));
in
data;
in {
inherit parse; inherit parse;
} }

46
src/translators/nodejs/utils.nix
View File

@ -1,49 +1,39 @@
{ {lib}: let
lib, l = lib // builtins;
}: let
l = lib // builtins;
in rec { in rec {
getPackageJsonDeps = packageJson: noDev: getPackageJsonDeps = packageJson: noDev:
packageJson.dependencies or {} packageJson.dependencies
or {}
// (lib.optionalAttrs (! noDev) (packageJson.devDependencies or {})); // (lib.optionalAttrs (! noDev) (packageJson.devDependencies or {}));
getWorkspaceLockFile = tree: project: fname: let getWorkspaceLockFile = tree: project: fname: let
# returns the parsed package-lock.json for a given project # returns the parsed package-lock.json for a given project
dirRelPath = dirRelPath =
if project ? subsystemInfo.workspaceParent then if project ? subsystemInfo.workspaceParent
"${project.subsystemInfo.workspaceParent}" then "${project.subsystemInfo.workspaceParent}"
else else "${project.relPath}";
"${project.relPath}";
packageJson = packageJson =
(tree.getNodeFromPath "${dirRelPath}/package.json").jsonContent; (tree.getNodeFromPath "${dirRelPath}/package.json").jsonContent;
hasNoDependencies = hasNoDependencies =
! packageJson ? dependencies && ! packageJson ? devDependencies; ! packageJson ? dependencies && ! packageJson ? devDependencies;
in in
if hasNoDependencies then if hasNoDependencies
null then null
else else tree.getNodeFromPath "${dirRelPath}/${fname}";
tree.getNodeFromPath "${dirRelPath}/${fname}";
getWorkspacePackageJson = tree: workspaces: getWorkspacePackageJson = tree: workspaces:
l.genAttrs l.genAttrs
workspaces workspaces
(wsRelPath: (wsRelPath:
(tree.getNodeFromPath "${wsRelPath}/package.json").jsonContent); (tree.getNodeFromPath "${wsRelPath}/package.json").jsonContent);
getWorkspacePackages = tree: workspaces: getWorkspacePackages = tree: workspaces:
lib.mapAttrs' lib.mapAttrs'
(wsRelPath: json: (wsRelPath: json:
l.nameValuePair l.nameValuePair
json.name json.name
json.version) json.version)
(getWorkspacePackageJson tree workspaces); (getWorkspacePackageJson tree workspaces);
} }

189
src/translators/python/impure/pip/default.nix
View File

@ -1,125 +1,111 @@
{ {
dlib, dlib,
lib, lib,
}: }: let
let
b = builtins; b = builtins;
in in {
{
# the input format is specified in /specifications/translator-call-example.json # the input format is specified in /specifications/translator-call-example.json
# this script receives a json file including the input paths and extraArgs # this script receives a json file including the input paths and extraArgs
translateBin = translateBin = {
{ # dream2nix
# dream2nix externalSources,
externalSources, utils,
utils, bash,
coreutils,
bash, jq,
coreutils, nix,
jq, python3,
nix, writeScriptBin,
python3, ...
writeScriptBin, }: let
... machNixExtractor = "${externalSources.mach-nix}/lib/default.nix";
}:
let
machNixExtractor = "${externalSources.mach-nix}/lib/default.nix";
setuptools_shim = ''
import sys, setuptools, tokenize, os; sys.argv[0] = 'setup.py'; __file__='setup.py';
f=getattr(tokenize, 'open', open)(__file__);
code=f.read().replace('\r\n', '\n');
f.close();
exec(compile(code, __file__, 'exec'))
'';
in
setuptools_shim = ''
import sys, setuptools, tokenize, os; sys.argv[0] = 'setup.py'; __file__='setup.py';
f=getattr(tokenize, 'open', open)(__file__);
code=f.read().replace('\r\n', '\n');
f.close();
exec(compile(code, __file__, 'exec'))
'';
in
utils.writePureShellScript utils.writePureShellScript
[ [
bash bash
coreutils coreutils
jq jq
nix nix
] ]
'' ''
# accroding to the spec, the translator reads the input from a json file # accroding to the spec, the translator reads the input from a json file
jsonInput=$1 jsonInput=$1
# read the json input # read the json input
outputFile=$(${jq}/bin/jq '.outputFile' -c -r $jsonInput) outputFile=$(${jq}/bin/jq '.outputFile' -c -r $jsonInput)
source=$(${jq}/bin/jq '.source' -c -r $jsonInput) source=$(${jq}/bin/jq '.source' -c -r $jsonInput)
pythonAttr=$(${jq}/bin/jq '.pythonAttr' -c -r $jsonInput) pythonAttr=$(${jq}/bin/jq '.pythonAttr' -c -r $jsonInput)
application=$(${jq}/bin/jq '.application' -c -r $jsonInput) application=$(${jq}/bin/jq '.application' -c -r $jsonInput)
# build python and pip executables # build python and pip executables
tmpBuild=$(mktemp -d) tmpBuild=$(mktemp -d)
nix build --show-trace --impure --expr \ nix build --show-trace --impure --expr \
" "
(import ${machNixExtractor} {}).mkPy (import ${machNixExtractor} {}).mkPy
(import <nixpkgs> {}).$pythonAttr (import <nixpkgs> {}).$pythonAttr
" \ " \
-o $tmpBuild/python -o $tmpBuild/python
nix build --impure --expr "(import <nixpkgs> {}).$pythonAttr.pkgs.pip" -o $tmpBuild/pip nix build --impure --expr "(import <nixpkgs> {}).$pythonAttr.pkgs.pip" -o $tmpBuild/pip
python=$tmpBuild/python/bin/python python=$tmpBuild/python/bin/python
pip=$tmpBuild/pip/bin/pip pip=$tmpBuild/pip/bin/pip
# prepare temporary directory # prepare temporary directory
tmp=$(mktemp -d) tmp=$(mktemp -d)
# extract python requirements from setup.py # extract python requirements from setup.py
cp -r $source $tmpBuild/src cp -r $source $tmpBuild/src
chmod -R +w $tmpBuild/src chmod -R +w $tmpBuild/src
cd $tmpBuild/src cd $tmpBuild/src
chmod +x setup.py || true chmod +x setup.py || true
echo "extracting dependencies" echo "extracting dependencies"
out_file=$tmpBuild/python.json \ out_file=$tmpBuild/python.json \
dump_setup_attrs=y \ dump_setup_attrs=y \
PYTHONIOENCODING=utf8 \ PYTHONIOENCODING=utf8 \
LANG=C.utf8 \ LANG=C.utf8 \
$python -c "${setuptools_shim}" install &> $tmpBuild/python.log || true $python -c "${setuptools_shim}" install &> $tmpBuild/python.log || true
# extract requirements from json result # extract requirements from json result
$python -c " $python -c "
import json import json
result = json.load(open('$tmpBuild/python.json')) result = json.load(open('$tmpBuild/python.json'))
for key in ('install_requires', 'setup_requires'): for key in ('install_requires', 'setup_requires'):
if key in result: if key in result:
print('\n'.join(result[key])) print('\n'.join(result[key]))
" > $tmpBuild/computed_requirements " > $tmpBuild/computed_requirements
# download files according to requirements # download files according to requirements
$tmpBuild/pip/bin/pip download \ $tmpBuild/pip/bin/pip download \
--no-cache \ --no-cache \
--dest $tmp \ --dest $tmp \
--progress-bar off \ --progress-bar off \
-r $tmpBuild/computed_requirements -r $tmpBuild/computed_requirements
# -r ''${inputFiles/$'\n'/$' -r '} # -r ''${inputFiles/$'\n'/$' -r '}
# generate the dream lock from the downloaded list of files # generate the dream lock from the downloaded list of files
NAME=$(${jq}/bin/jq '.name' -c -r $tmpBuild/python.json) \ NAME=$(${jq}/bin/jq '.name' -c -r $tmpBuild/python.json) \
VERSION=$(${jq}/bin/jq '.version' -c -r $tmpBuild/python.json) \ VERSION=$(${jq}/bin/jq '.version' -c -r $tmpBuild/python.json) \
$tmpBuild/python/bin/python ${./generate-dream-lock.py} $tmp $jsonInput $tmpBuild/python/bin/python ${./generate-dream-lock.py} $tmp $jsonInput
rm -rf $tmp $tmpBuild rm -rf $tmp $tmpBuild
''; '';
compatible = {source}:
compatible =
{
source,
}:
dlib.containsMatchingFile dlib.containsMatchingFile
[ [
''.*requirements.*\.txt'' ''.*requirements.*\.txt''
] ]
source; source;
# define special args and provide defaults # define special args and provide defaults
extraArgs = { extraArgs = {
# the python attribute # the python attribute
pythonAttr = { pythonAttr = {
default = "python3"; default = "python3";
@ -136,6 +122,5 @@ in
description = "build application instead of package"; description = "build application instead of package";
type = "flag"; type = "flag";
}; };
}; };
} }

453
src/translators/rust/pure/cargo-lock/default.nix
View File

@ -1,271 +1,248 @@
{ {
dlib, dlib,
lib, lib,
}: }: let
let
l = lib // builtins; l = lib // builtins;
in in {
translate = {
externals,
translatorName,
utils,
...
}: {
source,
packageName,
...
} @ args: let
inputDir = source;
{ recurseFiles = path:
translate = l.flatten (
{ l.mapAttrsToList
externals, (n: v:
translatorName, if v == "directory"
utils, then recurseFiles "${path}/${n}"
... else "${path}/${n}")
}: (l.readDir path)
{ );
source,
packageName,
...
}@args:
let
inputDir = source;
recurseFiles = path: # Find all Cargo.toml files and parse them
l.flatten ( allFiles = l.flatten (l.map recurseFiles [inputDir]);
l.mapAttrsToList cargoTomlPaths = l.filter (path: l.baseNameOf path == "Cargo.toml") allFiles;
(n: v: cargoTomls =
if v == "directory" then l.map
recurseFiles "${path}/${n}" (path: {
else inherit path;
"${path}/${n}") value = l.fromTOML (l.readFile path);
(l.readDir path) })
); cargoTomlPaths;
# Find all Cargo.toml files and parse them # Filter cargo-tomls to for files that actually contain packages
allFiles = l.flatten (l.map recurseFiles [ inputDir ]); cargoPackages =
cargoTomlPaths = l.filter (path: l.baseNameOf path == "Cargo.toml") allFiles; l.filter
cargoTomls = (toml: l.hasAttrByPath ["package" "name"] toml.value)
l.map cargoTomls;
(path: {
inherit path;
value = l.fromTOML (l.readFile path);
})
cargoTomlPaths;
# Filter cargo-tomls to for files that actually contain packages packageName =
cargoPackages = if args.packageName == "{automatic}"
l.filter then let
(toml: l.hasAttrByPath [ "package" "name" ] toml.value) # Small function to check if a given package path has a package
cargoTomls; # that has binaries
hasBinaries = toml:
l.hasAttr "bin" toml.value
|| l.pathExists "${l.dirOf toml.path}/src/main.rs"
|| l.pathExists "${l.dirOf toml.path}/src/bin";
packageName = # Try to find a package with a binary
if args.packageName == "{automatic}" pkg = l.findFirst hasBinaries (l.elemAt cargoPackages 0) cargoPackages;
then in
let pkg.value.package.name
# Small function to check if a given package path has a package else args.packageName;
# that has binaries
hasBinaries = toml:
l.hasAttr "bin" toml.value
|| l.pathExists "${l.dirOf toml.path}/src/main.rs"
|| l.pathExists "${l.dirOf toml.path}/src/bin";
# Try to find a package with a binary # Find the Cargo.toml matching the package name
pkg = l.findFirst hasBinaries (l.elemAt cargoPackages 0) cargoPackages; checkForPackageName = cargoToml: (cargoToml.value.package.name or null) == packageName;
packageToml =
l.findFirst
checkForPackageName
(throw "no Cargo.toml found with the package name passed: ${packageName}")
cargoTomls;
in pkg.value.package.name # Parse Cargo.lock and extract dependencies
else args.packageName; parsedLock = l.fromTOML (l.readFile "${inputDir}/Cargo.lock");
parsedDeps = parsedLock.package;
# This parses a "package-name version" entry in the "dependencies"
# field of a dependency in Cargo.lock
makeDepNameVersion = entry: let
parsed = l.splitString " " entry;
name = l.head parsed;
maybeVersion =
if l.length parsed > 1
then l.last parsed
else null;
in {
inherit name;
version =
# If there is no version, search through the lockfile to
# find the dependency's version
if maybeVersion != null
then maybeVersion
else
(
l.findFirst
(dep: dep.name == name)
(throw "no dependency found with name ${name} in Cargo.lock")
parsedDeps
)
.version;
};
# Find the Cargo.toml matching the package name package = rec {
checkForPackageName = cargoToml: (cargoToml.value.package.name or null) == packageName; toml = packageToml.value;
packageToml = tomlPath = packageToml.path;
l.findFirst
checkForPackageName
(throw "no Cargo.toml found with the package name passed: ${packageName}")
cargoTomls;
# Parse Cargo.lock and extract dependencies name = toml.package.name;
parsedLock = l.fromTOML (l.readFile "${inputDir}/Cargo.lock"); version = toml.package.version or (l.warn "no version found in Cargo.toml for ${name}, defaulting to unknown" "unknown");
parsedDeps = parsedLock.package; };
# This parses a "package-name version" entry in the "dependencies"
# field of a dependency in Cargo.lock # Parses a git source, taken straight from nixpkgs.
makeDepNameVersion = entry: parseGitSource = src: let
let parts = builtins.match ''git\+([^?]+)(\?(rev|tag|branch)=(.*))?#(.*)'' src;
parsed = l.splitString " " entry; type = builtins.elemAt parts 2; # rev, tag or branch
name = l.head parsed; value = builtins.elemAt parts 3;
maybeVersion = if l.length parsed > 1 then l.last parsed else null; in
in if parts == null
then null
else
{ {
inherit name; url = builtins.elemAt parts 0;
version = sha = builtins.elemAt parts 4;
# If there is no version, search through the lockfile to }
# find the dependency's version // lib.optionalAttrs (type != null) {inherit type value;};
if maybeVersion != null
then maybeVersion
else (
l.findFirst
(dep: dep.name == name)
(throw "no dependency found with name ${name} in Cargo.lock")
parsedDeps
).version;
};
package = rec { # Extracts a source type from a dependency.
toml = packageToml.value; getSourceTypeFrom = dependencyObject: let
tomlPath = packageToml.path; checkType = type: l.hasPrefix "${type}+" dependencyObject.source;
in
if !(l.hasAttr "source" dependencyObject)
then "path"
else if checkType "git"
then "git"
else if checkType "registry"
then
if dependencyObject.source == "registry+https://github.com/rust-lang/crates.io-index"
then "crates-io"
else throw "registries other than crates.io are not supported yet"
else throw "unknown or unsupported source type: ${dependencyObject.source}";
in
utils.simpleTranslate
({
getDepByNameVer,
dependenciesByOriginalID,
...
}: rec {
# VALUES
name = toml.package.name; inherit translatorName;
version = toml.package.version or (l.warn "no version found in Cargo.toml for ${name}, defaulting to unknown" "unknown");
# The raw input data as an attribute set.
# This will then be processed by `serializePackages` (see below) and
# transformed into a flat list.
inputData = parsedDeps;
defaultPackage = package.name;
packages =
(l.listToAttrs
(l.map
(toml:
l.nameValuePair
toml.value.package.name
toml.value.package.version)
cargoPackages))
// {"${defaultPackage}" = package.version;};
mainPackageDependencies = let
mainPackage =
l.findFirst
(dep: dep.name == package.name)
(throw "could not find main package in Cargo.lock")
parsedDeps;
in
l.map makeDepNameVersion (mainPackage.dependencies or []);
# the name of the subsystem
subsystemName = "rust";
# Extract subsystem specific attributes.
# The structure of this should be defined in:
# ./src/specifications/{subsystem}
subsystemAttrs = rec {
gitSources = let
gitDeps = l.filter (dep: (getSourceTypeFrom dep) == "git") parsedDeps;
in
l.unique (l.map (dep: parseGitSource dep.source) gitDeps);
}; };
# Parses a git source, taken straight from nixpkgs. # FUNCTIONS
parseGitSource = src:
let
parts = builtins.match ''git\+([^?]+)(\?(rev|tag|branch)=(.*))?#(.*)'' src;
type = builtins.elemAt parts 2; # rev, tag or branch
value = builtins.elemAt parts 3;
in
if parts == null then null
else {
url = builtins.elemAt parts 0;
sha = builtins.elemAt parts 4;
} // lib.optionalAttrs (type != null) { inherit type value; };
# Extracts a source type from a dependency. # return a list of package objects of arbitrary structure
getSourceTypeFrom = dependencyObject: serializePackages = inputData: inputData;
let checkType = type: l.hasPrefix "${type}+" dependencyObject.source; in
if !(l.hasAttr "source" dependencyObject)
then "path"
else if checkType "git" then
"git"
else if checkType "registry" then
if dependencyObject.source == "registry+https://github.com/rust-lang/crates.io-index"
then "crates-io"
else throw "registries other than crates.io are not supported yet"
else
throw "unknown or unsupported source type: ${dependencyObject.source}";
in
utils.simpleTranslate # return the name for a package object
({ getName = dependencyObject: dependencyObject.name;
getDepByNameVer,
dependenciesByOriginalID,
...
}:
rec { # return the version for a package object
# VALUES getVersion = dependencyObject: dependencyObject.version;
inherit translatorName; # get dependencies of a dependency object
getDependencies = dependencyObject:
l.map makeDepNameVersion (dependencyObject.dependencies or []);
# The raw input data as an attribute set. # return the source type of a package object
# This will then be processed by `serializePackages` (see below) and getSourceType = getSourceTypeFrom;
# transformed into a flat list.
inputData = parsedDeps;
defaultPackage = package.name; # An attrset of constructor functions.
# Given a dependency object and a source type, construct the
# source definition containing url, hash, etc.
sourceConstructors = {
path = dependencyObject: let
toml = (
l.findFirst
(toml: toml.value.package.name == dependencyObject.name)
(throw "could not find crate ${dependencyObject.name}")
cargoPackages
);
relDir = lib.removePrefix "${inputDir}/" (l.dirOf toml.path);
in {
path = relDir;
rootName = package.name;
rootVersion = package.version;
};
packages = git = dependencyObject: let
(l.listToAttrs parsed = parseGitSource dependencyObject.source;
(l.map in {
(toml: url = parsed.url;
l.nameValuePair rev = parsed.sha;
toml.value.package.name };
toml.value.package.version)
cargoPackages))
//
{ "${defaultPackage}" = package.version; };
mainPackageDependencies = crates-io = dependencyObject: {
let hash = dependencyObject.checksum;
mainPackage = };
l.findFirst };
(dep: dep.name == package.name) });
(throw "could not find main package in Cargo.lock")
parsedDeps;
in
l.map makeDepNameVersion (mainPackage.dependencies or [ ]);
# the name of the subsystem
subsystemName = "rust";
# Extract subsystem specific attributes.
# The structure of this should be defined in:
# ./src/specifications/{subsystem}
subsystemAttrs = rec {
gitSources = let
gitDeps = l.filter (dep: (getSourceTypeFrom dep) == "git") parsedDeps;
in l.unique (l.map (dep: parseGitSource dep.source) gitDeps);
};
# FUNCTIONS
# return a list of package objects of arbitrary structure
serializePackages = inputData: inputData;
# return the name for a package object
getName = dependencyObject: dependencyObject.name;
# return the version for a package object
getVersion = dependencyObject: dependencyObject.version;
# get dependencies of a dependency object
getDependencies = dependencyObject:
l.map makeDepNameVersion (dependencyObject.dependencies or [ ]);
# return the source type of a package object
getSourceType = getSourceTypeFrom;
# An attrset of constructor functions.
# Given a dependency object and a source type, construct the
# source definition containing url, hash, etc.
sourceConstructors = {
path = dependencyObject:
let
toml =
(l.findFirst
(toml: toml.value.package.name == dependencyObject.name)
(throw "could not find crate ${dependencyObject.name}")
cargoPackages
);
relDir = lib.removePrefix "${inputDir}/" (l.dirOf toml.path);
in
{
path = relDir;
rootName = package.name;
rootVersion = package.version;
};
git = dependencyObject:
let
parsed = parseGitSource dependencyObject.source;
in
{
url = parsed.url;
rev = parsed.sha;
};
crates-io = dependencyObject:
{
hash = dependencyObject.checksum;
};
};
});
projectName =
{
source,
}:
let
cargoToml = "${source}/Cargo.toml";
in
if l.pathExists cargoToml then
(l.fromTOML (l.readFile cargoToml)).package.name or null
else
null;
projectName = {source}: let
cargoToml = "${source}/Cargo.toml";
in
if l.pathExists cargoToml
then (l.fromTOML (l.readFile cargoToml)).package.name or null
else null;
# This allows the framework to detect if the translator is compatible with the given input # This allows the framework to detect if the translator is compatible with the given input
# to automatically select the right translator. # to automatically select the right translator.
compatible = compatible = {source}:
{ dlib.containsMatchingFile [''.*Cargo\.lock''] source;
source,
}:
dlib.containsMatchingFile [ ''.*Cargo\.lock'' ] source;
# If the translator requires additional arguments, specify them here. # If the translator requires additional arguments, specify them here.
# When users run the CLI, they will be asked to specify these arguments. # When users run the CLI, they will be asked to specify these arguments.

48
src/updaters/default.nix
View File

@ -5,45 +5,33 @@
lib, lib,
python3, python3,
writeText, writeText,
# dream2nix inputs # dream2nix inputs
callPackageDream, callPackageDream,
fetchers, fetchers,
utils, utils,
... ...
}: }: let
let
lockUtils = utils.dreamLock; lockUtils = utils.dreamLock;
updaters = callPackageDream ./updaters.nix {}; updaters = callPackageDream ./updaters.nix {};
getUpdaterName = getUpdaterName = {dreamLock}: let
{ lock = (utils.readDreamLock {inherit dreamLock;}).lock;
dreamLock, source = lockUtils.getMainPackageSource lock;
}: in
let lock.updater
lock = (utils.readDreamLock { inherit dreamLock; }).lock; or fetchers.fetchers."${source.type}".defaultUpdater
source = lockUtils.getMainPackageSource lock; or null;
in
lock.updater
or fetchers.fetchers."${source.type}".defaultUpdater
or null;
makeUpdateScript = makeUpdateScript = {
{ dreamLock,
dreamLock, updater ? getUpdaterName {inherit dreamLock;},
updater ? getUpdaterName { inherit dreamLock; }, }: let
}: lock = (utils.readDreamLock {inherit dreamLock;}).lock;
let source = lockUtils.getMainPackageSource lock;
lock = (utils.readDreamLock { inherit dreamLock; }).lock; updater' = updaters."${updater}";
source = lockUtils.getMainPackageSource lock; in
updater' = updaters."${updater}"; updater' source;
in in {
updater' source;
in
{
inherit getUpdaterName makeUpdateScript updaters; inherit getUpdaterName makeUpdateScript updaters;
} }

45
src/updaters/updaters.nix
View File

@ -5,42 +5,31 @@
lib, lib,
python3, python3,
writeText, writeText,
# dream2nix inputs # dream2nix inputs
utils, utils,
... ...
}: }: {
{ githubNewestReleaseTag = {
githubNewestReleaseTag = owner,
{ repo,
owner, ...
repo, }:
... utils.writePureShellScript [curl jq] ''
}:
utils.writePureShellScript [ curl jq ] ''
curl -s "https://api.github.com/repos/${owner}/${repo}/releases?per_page=1" | jq -r '.[0].tag_name' curl -s "https://api.github.com/repos/${owner}/${repo}/releases?per_page=1" | jq -r '.[0].tag_name'
''; '';
pypiNewestReleaseVersion = pypiNewestReleaseVersion = {pname, ...}:
{ utils.writePureShellScript [curl jq] ''
pname,
...
}:
utils.writePureShellScript [ curl jq ] ''
curl -s https://pypi.org/pypi/${pname}/json | jq -r '.info.version' curl -s https://pypi.org/pypi/${pname}/json | jq -r '.info.version'
''; '';
npmNewestReleaseVersion = npmNewestReleaseVersion = {pname, ...}:
{ # api docs: https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#get
pname, utils.writePureShellScript [curl jq] ''
...
}:
# api docs: https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#get
utils.writePureShellScript [ curl jq ] ''
curl -s https://registry.npmjs.com/${pname} | jq -r '."dist-tags".latest' curl -s https://registry.npmjs.com/${pname} | jq -r '."dist-tags".latest'
''; '';
urlRegexPython = urlRegexPython =
# Don't forget to use double quoted strings # Don't forget to use double quoted strings
# or double escape ('\\' instead of '\'). # or double escape ('\\' instead of '\').
# Expects named group 'rev' to be defined. # Expects named group 'rev' to be defined.
@ -50,14 +39,12 @@
url, url,
regex, regex,
... ...
}: }: let
let
reFile = writeText "regex" regex; reFile = writeText "regex" regex;
in in
utils.writePureShellScript [ curl gnugrep python3 ] '' utils.writePureShellScript [curl gnugrep python3] ''
curl -s ${url} \ curl -s ${url} \
| python3 -c \ | python3 -c \
'import re, sys; print(re.search(open("${reFile}").read(), sys.stdin.read()).group("ver"), end="")' 'import re, sys; print(re.search(open("${reFile}").read(), sys.stdin.read()).group("ver"), end="")'
''; '';
} }

41
src/utils/config.nix
View File

@ -1,5 +1,4 @@
let let
b = builtins; b = builtins;
# loads attrs either from s: # loads attrs either from s:
@ -7,29 +6,25 @@ let
# - json string # - json string
# - attrset (no changes) # - attrset (no changes)
loadAttrs = input: loadAttrs = input:
if b.isPath input then if b.isPath input
b.fromJSON (b.readFile input) then b.fromJSON (b.readFile input)
else if b.isString input then else if b.isString input
b.fromJSON input then b.fromJSON input
else if b.isAttrs input then else if b.isAttrs input
input then input
else else throw "input for loadAttrs must be json file or string or attrs";
throw "input for loadAttrs must be json file or string or attrs";
# load dream2nix config extending with defaults # load dream2nix config extending with defaults
loadConfig = configInput: loadConfig = configInput: let
let config = loadAttrs configInput;
config = loadAttrs configInput; defaults = {
defaults = { overridesDirs = [];
overridesDirs = []; packagesDir = "./packages";
packagesDir = "./packages"; projectRoot = null;
projectRoot = null; repoName = null;
repoName = null; };
}; in
in defaults // config;
defaults // config; in {
in
{
inherit loadConfig; inherit loadConfig;
} }

217
src/utils/default.nix
View File

@ -13,16 +13,13 @@
stdenv, stdenv,
writeScript, writeScript,
writeScriptBin, writeScriptBin,
# dream2nix inputs # dream2nix inputs
apps, apps,
callPackageDream, callPackageDream,
externalSources, externalSources,
translators, translators,
... ...
}: }: let
let
b = builtins; b = builtins;
l = lib // builtins; l = lib // builtins;
@ -38,153 +35,150 @@ let
# copied from poetry2nix # copied from poetry2nix
ireplace = idx: value: list: ( ireplace = idx: value: list: (
lib.genList lib.genList
(i: if i == idx then value else (b.elemAt list i)) (i:
(b.length list) if i == idx
then value
else (b.elemAt list i))
(b.length list)
); );
}; };
in in
overrideUtils
// translatorUtils
// translatorUtils2
// rec {
inherit
(dlib)
dirNames
callViaEnv
identifyGitUrl
latestVersion
listDirs
listFiles
nameVersionPair
parseGitUrl
readTextFile
recursiveUpdateUntilDepth
sanitizeDerivationName
traceJ
;
overrideUtils dreamLock = dreamLockUtils;
// translatorUtils
// translatorUtils2
// rec {
inherit (dlib) inherit (dreamLockUtils) readDreamLock;
dirNames
callViaEnv
identifyGitUrl
latestVersion
listDirs
listFiles
nameVersionPair
parseGitUrl
readTextFile
recursiveUpdateUntilDepth
sanitizeDerivationName
traceJ
;
dreamLock = dreamLockUtils; toDrv = path: runCommand "some-drv" {} "cp -r ${path} $out";
inherit (dreamLockUtils) readDreamLock; toTOML = import ./toTOML.nix {inherit lib;};
toDrv = path: runCommand "some-drv" {} "cp -r ${path} $out"; # hash the contents of a path via `nix hash path`
hashPath = algo: path: let
toTOML = import ./toTOML.nix { inherit lib; };
# hash the contents of a path via `nix hash path`
hashPath = algo: path:
let
hashPath = runCommand "hash-${algo}" {} '' hashPath = runCommand "hash-${algo}" {} ''
${nix}/bin/nix --option experimental-features nix-command hash path ${path} | tr --delete '\n' > $out ${nix}/bin/nix --option experimental-features nix-command hash path ${path} | tr --delete '\n' > $out
''; '';
in in
b.readFile hashPath; b.readFile hashPath;
# hash a file via `nix hash file` # hash a file via `nix hash file`
hashFile = algo: path: hashFile = algo: path: let
let
hashFile = runCommand "hash-${algo}" {} '' hashFile = runCommand "hash-${algo}" {} ''
${nix}/bin/nix --option experimental-features nix-command hash file ${path} | tr --delete '\n' > $out ${nix}/bin/nix --option experimental-features nix-command hash file ${path} | tr --delete '\n' > $out
''; '';
in in
b.readFile hashFile; b.readFile hashFile;
# builder to create a shell script that has it's own PATH # builder to create a shell script that has it's own PATH
writePureShellScript = availablePrograms: script: writeScript "script.sh" '' writePureShellScript = availablePrograms: script:
#!${bash}/bin/bash writeScript "script.sh" ''
set -Eeuo pipefail #!${bash}/bin/bash
set -Eeuo pipefail
export PATH="${lib.makeBinPath availablePrograms}" export PATH="${lib.makeBinPath availablePrograms}"
export NIX_PATH=nixpkgs=${pkgs.path} export NIX_PATH=nixpkgs=${pkgs.path}
export WORKDIR="$PWD" export WORKDIR="$PWD"
TMPDIR=$(${coreutils}/bin/mktemp -d) TMPDIR=$(${coreutils}/bin/mktemp -d)
cd $TMPDIR cd $TMPDIR
${script} ${script}
cd cd
${coreutils}/bin/rm -rf $TMPDIR ${coreutils}/bin/rm -rf $TMPDIR
''; '';
# builder to create a shell script that has it's own PATH # builder to create a shell script that has it's own PATH
writePureShellScriptBin = binName: availablePrograms: script: writePureShellScriptBin = binName: availablePrograms: script:
writeScriptBin binName '' writeScriptBin binName ''
#!${bash}/bin/bash #!${bash}/bin/bash
set -Eeuo pipefail set -Eeuo pipefail
export PATH="${lib.makeBinPath availablePrograms}" export PATH="${lib.makeBinPath availablePrograms}"
export NIX_PATH=nixpkgs=${pkgs.path} export NIX_PATH=nixpkgs=${pkgs.path}
export WORKDIR="$PWD" export WORKDIR="$PWD"
TMPDIR=$(${coreutils}/bin/mktemp -d) TMPDIR=$(${coreutils}/bin/mktemp -d)
cd $TMPDIR cd $TMPDIR
${script} ${script}
cd cd
${coreutils}/bin/rm -rf $TMPDIR ${coreutils}/bin/rm -rf $TMPDIR
''; '';
extractSource = extractSource = {
{
source, source,
dir ? "", dir ? "",
}: }:
stdenv.mkDerivation { stdenv.mkDerivation {
name = "${(source.name or "")}-extracted"; name = "${(source.name or "")}-extracted";
src = source; src = source;
inherit dir; inherit dir;
phases = [ "unpackPhase" ]; phases = ["unpackPhase"];
dontInstall = true; dontInstall = true;
dontFixup = true; dontFixup = true;
unpackCmd = unpackCmd =
if lib.hasSuffix ".tgz" source.name then if lib.hasSuffix ".tgz" source.name
'' then ''
tar --delay-directory-restore -xf $src tar --delay-directory-restore -xf $src
# set executable flag only on directories # set executable flag only on directories
chmod -R +X . chmod -R +X .
'' ''
else else null;
null; # sometimes tarballs do not end with .tar.??
# sometimes tarballs do not end with .tar.?? preUnpack = ''
preUnpack = '' unpackFallback(){
unpackFallback(){ local fn="$1"
local fn="$1" tar xf "$fn"
tar xf "$fn" }
}
unpackCmdHooks+=(unpackFallback) unpackCmdHooks+=(unpackFallback)
''; '';
postUnpack = '' postUnpack = ''
echo postUnpack echo postUnpack
mv "$sourceRoot/$dir" $out mv "$sourceRoot/$dir" $out
exit exit
''; '';
}; };
satisfiesSemver = poetry2nixSemver.satisfiesSemver; satisfiesSemver = poetry2nixSemver.satisfiesSemver;
makeTranslateScript = makeTranslateScript = {
{
invalidationHash, invalidationHash,
source, source,
project, project,
}@args: } @ args: let
let
translator = translator =
translators.translatorsV2."${project.subsystem}".all."${project.translator}"; translators.translatorsV2."${project.subsystem}".all."${project.translator}";
argsJsonFile = pkgs.writeText "translator-args.json" argsJsonFile =
pkgs.writeText "translator-args.json"
(l.toJSON (l.toJSON
(args (args
// { // {
project = l.removeAttrs args.project ["dreamLock"]; project = l.removeAttrs args.project ["dreamLock"];
outputFile = project.dreamLockPath; outputFile = project.dreamLockPath;
})); }));
in in
writePureShellScriptBin "resolve" writePureShellScriptBin "resolve"
[ [
@ -219,15 +213,14 @@ overrideUtils
fi fi
''; '';
# a script that produces and dumps the dream-lock json for a given source # a script that produces and dumps the dream-lock json for a given source
makePackageLockScript = makePackageLockScript = {
{
packagesDir, packagesDir,
source, source,
translator, translator,
translatorArgs, translatorArgs,
}: }:
writePureShellScript writePureShellScript
[] []
'' ''
cd $WORKDIR cd $WORKDIR
@ -236,12 +229,12 @@ overrideUtils
--no-default-nix \ --no-default-nix \
--translator ${translator} \ --translator ${translator} \
--invalidation-hash ${dlib.calcInvalidationHash { --invalidation-hash ${dlib.calcInvalidationHash {
inherit source translator translatorArgs; inherit source translator translatorArgs;
}} \ }} \
--packages-root $WORKDIR/${packagesDir} \ --packages-root $WORKDIR/${packagesDir} \
${lib.concatStringsSep " \\\n" ${lib.concatStringsSep " \\\n"
(lib.mapAttrsToList (lib.mapAttrsToList
(key: val: "--arg ${key}=${b.toString val}") (key: val: "--arg ${key}=${b.toString val}")
translatorArgs)} translatorArgs)}
''; '';
} }

474
src/utils/dream-lock.nix
View File

@ -1,139 +1,126 @@
{ {
lib, lib,
# dream2nix # dream2nix
utils, utils,
... ...
}: }: let
let
b = builtins; b = builtins;
subDreamLockNames = dreamLockFile: subDreamLockNames = dreamLockFile: let
let dir = b.dirOf dreamLockFile;
dir = b.dirOf dreamLockFile;
directories = utils.listDirs dir; directories = utils.listDirs dir;
dreamLockDirs = dreamLockDirs =
lib.filter lib.filter
(d: b.pathExists ("${dir}/${d}/dream-lock.json")) (d: b.pathExists "${dir}/${d}/dream-lock.json")
directories; directories;
in
dreamLockDirs;
readDreamLock = {dreamLock} @ args: let
isFile =
b.isPath dreamLock
|| b.isString dreamLock
|| lib.isDerivation dreamLock;
lockMaybeCompressed =
if isFile
then b.fromJSON (b.readFile dreamLock)
else dreamLock;
lock =
if lockMaybeCompressed.decompressed or false
then lockMaybeCompressed
else decompressDreamLock lockMaybeCompressed;
subDreamLocks =
if ! isFile
then {}
else let
dir = b.dirOf dreamLock;
in
lib.genAttrs
(subDreamLockNames dreamLock)
(d:
readDreamLock
{dreamLock = "${dir}/${d}/dream-lock.json";});
packages = lock._generic.packages;
defaultPackageName = lock._generic.defaultPackage;
defaultPackageVersion = packages."${defaultPackageName}";
subsystemAttrs = lock._subsystem;
sources = lock.sources;
dependencyGraph = lock.dependencies;
packageVersions =
lib.mapAttrs
(name: versions: lib.attrNames versions)
dependencyGraph;
cyclicDependencies = lock.cyclicDependencies;
getSourceSpec = pname: version:
sources."${pname}"."${version}"
or (
throw "The source spec for ${pname}#${version} is not defined in lockfile."
);
getDependencies = pname: version:
b.filter
(dep: ! b.elem dep cyclicDependencies."${pname}"."${version}" or [])
dependencyGraph."${pname}"."${version}" or [];
getCyclicDependencies = pname: version:
cyclicDependencies."${pname}"."${version}" or [];
getRoot = pname: version: let
spec = getSourceSpec pname version;
in in
dreamLockDirs; if spec.type == "path"
then {
pname = spec.rootName;
readDreamLock = version = spec.rootVersion;
{ }
dreamLock, else {inherit pname version;};
}@args: in {
let inherit lock;
interface = {
isFile = inherit
b.isPath dreamLock defaultPackageName
|| b.isString dreamLock defaultPackageVersion
|| lib.isDerivation dreamLock; subsystemAttrs
getCyclicDependencies
lockMaybeCompressed = getDependencies
if isFile then getSourceSpec
b.fromJSON (b.readFile dreamLock) getRoot
else packages
dreamLock; packageVersions
subDreamLocks
lock = ;
if lockMaybeCompressed.decompressed or false then };
lockMaybeCompressed };
else
decompressDreamLock lockMaybeCompressed;
subDreamLocks =
if ! isFile then
{}
else
let
dir = b.dirOf dreamLock;
in
lib.genAttrs
(subDreamLockNames dreamLock)
(d:
readDreamLock
{ dreamLock = "${dir}/${d}/dream-lock.json"; });
packages = lock._generic.packages;
defaultPackageName = lock._generic.defaultPackage;
defaultPackageVersion = packages."${defaultPackageName}";
subsystemAttrs = lock._subsystem;
sources = lock.sources;
dependencyGraph = lock.dependencies;
packageVersions =
lib.mapAttrs
(name: versions: lib.attrNames versions)
dependencyGraph;
cyclicDependencies = lock.cyclicDependencies;
getSourceSpec = pname: version:
sources."${pname}"."${version}" or (
throw "The source spec for ${pname}#${version} is not defined in lockfile."
);
getDependencies = pname: version:
b.filter
(dep: ! b.elem dep cyclicDependencies."${pname}"."${version}" or [])
dependencyGraph."${pname}"."${version}" or [];
getCyclicDependencies = pname: version:
cyclicDependencies."${pname}"."${version}" or [];
getRoot = pname: version:
let spec = getSourceSpec pname version; in
if spec.type == "path" then
{
pname = spec.rootName;
version = spec.rootVersion;
}
else
{ inherit pname version; };
in
{
inherit lock;
interface = {
inherit
defaultPackageName
defaultPackageVersion
subsystemAttrs
getCyclicDependencies
getDependencies
getSourceSpec
getRoot
packages
packageVersions
subDreamLocks
;
};
};
getMainPackageSource = dreamLock: getMainPackageSource = dreamLock:
dreamLock.sources dreamLock
."${dreamLock._generic.defaultPackage}" .sources
."${dreamLock._generic.packages."${dreamLock._generic.defaultPackage}"}" ."${dreamLock._generic.defaultPackage}"
."${dreamLock._generic.packages."${dreamLock._generic.defaultPackage}"}"
// rec { // rec {
pname = dreamLock._generic.defaultPackage; pname = dreamLock._generic.defaultPackage;
version = dreamLock._generic.packages."${pname}" ; version = dreamLock._generic.packages."${pname}";
}; };
getSource = fetchedSources: pname: version: getSource = fetchedSources: pname: version:
if fetchedSources ? "${pname}"."${version}" if
&& fetchedSources."${pname}"."${version}" != "unknown" then fetchedSources
fetchedSources."${pname}"."${version}" ? "${pname}"."${version}"
&& fetchedSources."${pname}"."${version}" != "unknown"
then fetchedSources."${pname}"."${version}"
else else
throw '' throw ''
The source for ${pname}#${version} is not defined. The source for ${pname}#${version} is not defined.
@ -149,143 +136,140 @@ let
``` ```
''; '';
# generate standalone dreamLock for a depenndency of an existing dreamLock # generate standalone dreamLock for a depenndency of an existing dreamLock
getSubDreamLock = dreamLock: name: version: getSubDreamLock = dreamLock: name: version: let
let lock = (readDreamLock {inherit dreamLock;}).lock;
lock = (readDreamLock { inherit dreamLock; }).lock; in
lock
in // {
lock // { _generic =
_generic = lock._generic // { lock._generic
defaultPackage = name; // {
packages = lock._generic.packages // { defaultPackage = name;
packages =
lock._generic.packages
// {
"${name}" = version; "${name}" = version;
}; };
};
}; };
};
injectDependencies = dreamLock: inject: injectDependencies = dreamLock: inject:
if inject == {} then dreamLock else if inject == {}
let then dreamLock
lock = (readDreamLock { inherit dreamLock; }).lock; else let
lock = (readDreamLock {inherit dreamLock;}).lock;
oldDependencyGraph = lock.dependencies; oldDependencyGraph = lock.dependencies;
newDependencyGraph = newDependencyGraph =
lib.zipAttrsWith
(name: versions:
lib.zipAttrsWith lib.zipAttrsWith
(name: versions: (version: deps: lib.unique (lib.flatten deps))
lib.zipAttrsWith versions)
(version: deps: lib.unique (lib.flatten deps)) [
versions) oldDependencyGraph
[ inject
oldDependencyGraph ];
inject in
]; lib.recursiveUpdate lock {
dependencies = newDependencyGraph;
in
lib.recursiveUpdate lock {
dependencies = newDependencyGraph;
};
decompressDependencyGraph = compGraph:
lib.mapAttrs
(name: versions:
lib.mapAttrs
(version: deps:
map
(dep: {
name = b.elemAt dep 0;
version = b.elemAt dep 1;
})
deps)
versions)
compGraph;
compressDependencyGraph = decompGraph:
lib.mapAttrs
(name: versions:
lib.mapAttrs
(version: deps: map ( dep: [ dep.name dep.version ]) deps)
versions)
decompGraph;
decompressDreamLock = comp:
let
dependencyGraphDecomp =
decompressDependencyGraph (comp.dependencies or {});
cyclicDependencies =
decompressDependencyGraph (comp.cyclicDependencies or {});
emptyDependencyGraph =
lib.mapAttrs
(name: versions:
lib.mapAttrs
(version: source: [])
versions)
comp.sources;
dependencyGraph =
lib.recursiveUpdate
emptyDependencyGraph
dependencyGraphDecomp;
in
comp // {
decompressed = true;
cyclicDependencies = cyclicDependencies;
dependencies = dependencyGraph;
};
compressDreamLock = uncomp:
let
dependencyGraphComp =
compressDependencyGraph
uncomp.dependencies;
cyclicDependencies =
compressDependencyGraph
uncomp.cyclicDependencies;
dependencyGraph =
lib.filterAttrs
(name: versions: versions != {})
(lib.mapAttrs
(name: versions:
lib.filterAttrs
(version: deps: deps != [])
versions)
dependencyGraphComp);
in
(b.removeAttrs uncomp [ "decompressed" ]) // {
inherit cyclicDependencies;
dependencies = dependencyGraph;
}; };
toJSON = dreamLock: decompressDependencyGraph = compGraph:
let lib.mapAttrs
lock = (name: versions:
if dreamLock.decompressed or false then lib.mapAttrs
compressDreamLock dreamLock (version: deps:
else map
dreamLock; (dep: {
name = b.elemAt dep 0;
version = b.elemAt dep 1;
})
deps)
versions)
compGraph;
json = b.toJSON lock; compressDependencyGraph = decompGraph:
lib.mapAttrs
(name: versions:
lib.mapAttrs
(version: deps: map (dep: [dep.name dep.version]) deps)
versions)
decompGraph;
in decompressDreamLock = comp: let
json; dependencyGraphDecomp =
decompressDependencyGraph (comp.dependencies or {});
in cyclicDependencies =
{ decompressDependencyGraph (comp.cyclicDependencies or {});
inherit
compressDreamLock emptyDependencyGraph =
decompressDreamLock lib.mapAttrs
decompressDependencyGraph (name: versions:
getMainPackageSource lib.mapAttrs
getSource (version: source: [])
getSubDreamLock versions)
readDreamLock comp.sources;
injectDependencies
toJSON dependencyGraph =
lib.recursiveUpdate
emptyDependencyGraph
dependencyGraphDecomp;
in
comp
// {
decompressed = true;
cyclicDependencies = cyclicDependencies;
dependencies = dependencyGraph;
};
compressDreamLock = uncomp: let
dependencyGraphComp =
compressDependencyGraph
uncomp.dependencies;
cyclicDependencies =
compressDependencyGraph
uncomp.cyclicDependencies;
dependencyGraph =
lib.filterAttrs
(name: versions: versions != {})
(lib.mapAttrs
(name: versions:
lib.filterAttrs
(version: deps: deps != [])
versions)
dependencyGraphComp);
in
(b.removeAttrs uncomp ["decompressed"])
// {
inherit cyclicDependencies;
dependencies = dependencyGraph;
};
toJSON = dreamLock: let
lock =
if dreamLock.decompressed or false
then compressDreamLock dreamLock
else dreamLock;
json = b.toJSON lock;
in
json;
in {
inherit
compressDreamLock
decompressDreamLock
decompressDependencyGraph
getMainPackageSource
getSource
getSubDreamLock
readDreamLock
injectDependencies
toJSON
; ;
} }

23
src/utils/external-dir.nix
View File

@ -4,16 +4,15 @@
externalSources, externalSources,
externalPaths, externalPaths,
}: }:
pkgs.runCommand "dream2nix-external-dir" {} pkgs.runCommand "dream2nix-external-dir" {}
(lib.concatStringsSep "\n" (lib.concatStringsSep "\n"
(lib.mapAttrsToList (lib.mapAttrsToList
(inputName: paths: (inputName: paths:
lib.concatStringsSep "\n" lib.concatStringsSep "\n"
(lib.forEach (lib.forEach
paths paths
(path: '' (path: ''
mkdir -p $out/${inputName}/$(dirname ${path}) mkdir -p $out/${inputName}/$(dirname ${path})
cp ${externalSources."${inputName}"}/${path} $out/${inputName}/${path} cp ${externalSources."${inputName}"}/${path} $out/${inputName}/${path}
''))) '')))
externalPaths)) externalPaths))

304
src/utils/override.nix
View File

@ -1,29 +1,25 @@
{ {
lib, lib,
# dream2nix # dream2nix
utils, utils,
... ...
}: }: let
let
b = builtins; b = builtins;
loadOverridesDirs = overridesDirs: pkgs: loadOverridesDirs = overridesDirs: pkgs: let
let loadOverrides = dir:
loadOverrides = dir: lib.genAttrs (utils.dirNames dir) (name:
lib.genAttrs (utils.dirNames dir) (name: import (dir + "/${name}") {
import (dir + "/${name}") { inherit lib pkgs;
inherit lib pkgs; satisfiesSemver = constraint: pkg:
satisfiesSemver = constraint: pkg: utils.satisfiesSemver pkg.version constraint;
utils.satisfiesSemver pkg.version constraint; });
}); in
in b.foldl'
b.foldl' (loaded: nextDir:
(loaded: nextDir: utils.recursiveUpdateUntilDepth 3 loaded (loadOverrides nextDir))
utils.recursiveUpdateUntilDepth 3 loaded (loadOverrides nextDir)) {}
{} overridesDirs;
overridesDirs;
throwErrorUnclearAttributeOverride = pname: overrideName: attrName: throwErrorUnclearAttributeOverride = pname: overrideName: attrName:
throw '' throw ''
@ -54,156 +50,142 @@ let
``` ```
''; '';
getOverrideFunctionArgs = function: getOverrideFunctionArgs = function: let
let funcArgs = lib.functionArgs function;
funcArgs = lib.functionArgs function; in
in if funcArgs != {}
if funcArgs != {} then then b.attrNames funcArgs
b.attrNames funcArgs else
else (
function (old: {passthru.funcArgs = lib.attrNames old;})
)
.funcArgs;
applyOverridesToPackage = {
conditionalOverrides,
pkg,
pname,
outputs,
}: let
# if condition is unset, it will be assumed true
evalCondition = condOverride: pkg:
if condOverride ? _condition
then condOverride._condition pkg
else true;
# filter the overrides by the package name and conditions
overridesToApply = let
# TODO: figure out if regex names will be useful
regexOverrides = {};
# lib.filterAttrs
# (name: data:
# lib.hasPrefix "^" name
# &&
# b.match name pname != null)
# conditionalOverrides;
overridesForPackage =
b.foldl'
(overrides: new: overrides // new)
conditionalOverrides."${pname}" or {}
(lib.attrValues regexOverrides);
overridesListForPackage =
lib.mapAttrsToList
( (
function (old: {passthru.funcArgs = lib.attrNames old;}) _name: data:
).funcArgs; data // {inherit _name;}
)
overridesForPackage;
in (lib.filter
(condOverride: evalCondition condOverride pkg)
overridesListForPackage);
applyOverridesToPackage = # apply single attribute override
{ applySingleAttributeOverride = oldVal: functionOrValue:
conditionalOverrides, if b.isFunction functionOrValue
pkg, then
pname, if lib.functionArgs functionOrValue == {}
outputs, then functionOrValue oldVal
}: else
let functionOrValue {
old = oldVal;
inherit outputs;
}
else functionOrValue;
# if condition is unset, it will be assumed true # helper to apply one conditional override
evalCondition = condOverride: pkg: # the condition is not evaluated anymore here
if condOverride ? _condition then applyOneOverride = pkg: condOverride: let
condOverride._condition pkg base_derivation =
if condOverride ? _replace
then
if lib.isFunction condOverride._replace
then condOverride._replace pkg
else if lib.isDerivation condOverride._replace
then condOverride._replace
else else
true; throw
("override attr ${pname}.${condOverride._name}._replace"
+ " must either be a derivation or a function")
else pkg;
# filter the overrides by the package name and conditions overrideFuncs =
overridesToApply = lib.mapAttrsToList
let (funcName: func: {inherit funcName func;})
# TODO: figure out if regex names will be useful (lib.filterAttrs (n: v: lib.hasPrefix "override" n) condOverride);
regexOverrides = {};
# lib.filterAttrs
# (name: data:
# lib.hasPrefix "^" name
# &&
# b.match name pname != null)
# conditionalOverrides;
overridesForPackage = singleArgOverrideFuncs = let
b.foldl' availableFunctions =
(overrides: new: overrides // new) lib.mapAttrs
conditionalOverrides."${pname}" or {} (funcName: func: getOverrideFunctionArgs func)
(lib.attrValues regexOverrides); (lib.filterAttrs
(funcName: func: lib.hasPrefix "override" funcName)
base_derivation);
overridesListForPackage = getOverrideFuncNameForAttrName = attrName: let
lib.mapAttrsToList applicableFuncs =
(_name: data: lib.attrNames
data // { inherit _name; } (lib.filterAttrs
) (funcName: args: b.elem attrName args)
overridesForPackage; availableFunctions);
in
(lib.filter
(condOverride: evalCondition condOverride pkg)
overridesListForPackage);
# apply single attribute override
applySingleAttributeOverride = oldVal: functionOrValue:
if b.isFunction functionOrValue then
if lib.functionArgs functionOrValue == {} then
functionOrValue oldVal
else
functionOrValue {
old = oldVal;
inherit outputs;
}
else
functionOrValue;
# helper to apply one conditional override
# the condition is not evaluated anymore here
applyOneOverride = pkg: condOverride:
let
base_derivation =
if condOverride ? _replace then
if lib.isFunction condOverride._replace then
condOverride._replace pkg
else if lib.isDerivation condOverride._replace then
condOverride._replace
else
throw
("override attr ${pname}.${condOverride._name}._replace"
+ " must either be a derivation or a function")
else
pkg;
overrideFuncs =
lib.mapAttrsToList
(funcName: func: { inherit funcName func; })
(lib.filterAttrs (n: v: lib.hasPrefix "override" n) condOverride);
singleArgOverrideFuncs =
let
availableFunctions =
lib.mapAttrs
(funcName: func: getOverrideFunctionArgs func)
(lib.filterAttrs
(funcName: func: lib.hasPrefix "override" funcName)
base_derivation);
getOverrideFuncNameForAttrName = attrName:
let
applicableFuncs =
lib.attrNames
(lib.filterAttrs
(funcName: args: b.elem attrName args)
availableFunctions);
in
if b.length applicableFuncs == 0 then
"overrideAttrs"
else if b.length applicableFuncs > 1 then
throwErrorUnclearAttributeOverride pname condOverride._name attrName
else
b.elemAt applicableFuncs 0;
attributeOverrides =
lib.filterAttrs
(n: v: ! lib.hasPrefix "override" n && ! lib.hasPrefix "_" n)
condOverride;
in
lib.mapAttrsToList
(attrName: funcOrValue: {
funcName = getOverrideFuncNameForAttrName attrName;
func = oldAttrs: { "${attrName}" = funcOrValue; };
})
attributeOverrides;
in
b.foldl'
(pkg: overrideFunc:
pkg."${overrideFunc.funcName}"
(old:
let
updateAttrsFuncs = overrideFunc.func old;
in
lib.mapAttrs
(attrName: functionOrValue:
applySingleAttributeOverride old."${attrName}" functionOrValue)
updateAttrsFuncs))
base_derivation
(overrideFuncs ++ singleArgOverrideFuncs);
in in
# apply the overrides to the given pkg if b.length applicableFuncs == 0
(lib.foldl then "overrideAttrs"
(pkg: condOverride: applyOneOverride pkg condOverride) else if b.length applicableFuncs > 1
pkg then throwErrorUnclearAttributeOverride pname condOverride._name attrName
overridesToApply); else b.elemAt applicableFuncs 0;
in attributeOverrides =
{ lib.filterAttrs
(n: v: ! lib.hasPrefix "override" n && ! lib.hasPrefix "_" n)
condOverride;
in
lib.mapAttrsToList
(attrName: funcOrValue: {
funcName = getOverrideFuncNameForAttrName attrName;
func = oldAttrs: {"${attrName}" = funcOrValue;};
})
attributeOverrides;
in
b.foldl'
(pkg: overrideFunc:
pkg."${overrideFunc.funcName}"
(old: let
updateAttrsFuncs = overrideFunc.func old;
in
lib.mapAttrs
(attrName: functionOrValue:
applySingleAttributeOverride old."${attrName}" functionOrValue)
updateAttrsFuncs))
base_derivation
(overrideFuncs ++ singleArgOverrideFuncs);
in
# apply the overrides to the given pkg
(lib.foldl
(pkg: condOverride: applyOneOverride pkg condOverride)
pkg
overridesToApply);
in {
inherit applyOverridesToPackage loadOverridesDirs; inherit applyOverridesToPackage loadOverridesDirs;
} }

83
src/utils/toTOML.nix
View File

@ -1,5 +1,4 @@
{ lib }: {lib}: let
let
inherit inherit
(lib) (lib)
length length
@ -31,14 +30,13 @@ let
ty = tomlTy v; ty = tomlTy v;
in in
if ty == "set" if ty == "set"
then then let
let vals =
vals = mapAttrsToList
mapAttrsToList (k': v': "${quoteKey k'} = ${outputValInner v'}")
(k': v': "${quoteKey k'} = ${outputValInner v'}") v;
v; valsStr = concatStringsSep ", " vals;
valsStr = concatStringsSep ", " vals; in "{ ${valsStr} }"
in "{ ${valsStr} }"
else outputVal v; else outputVal v;
outputVal = v: let outputVal = v: let
@ -49,11 +47,10 @@ let
else if ty == "string" else if ty == "string"
then quoteString v then quoteString v
else if ty == "list" || ty == "list_of_attrs" else if ty == "list" || ty == "list_of_attrs"
then then let
let vals = map quoteString v;
vals = map quoteString v; valsStr = concatStringsSep ", " vals;
valsStr = concatStringsSep ", " vals; in "[ ${valsStr} ]"
in "[ ${valsStr} ]"
else if ty == "set" else if ty == "set"
then abort "unsupported set for not-inner value" then abort "unsupported set for not-inner value"
else abort "Not implemented: type ${ty}"; else abort "Not implemented: type ${ty}";
@ -62,14 +59,13 @@ let
ty = tomlTy v; ty = tomlTy v;
in in
if ty == "set" if ty == "set"
then then let
let vals =
vals = mapAttrsToList
mapAttrsToList (k': v': "${quoteKey k'} = ${outputValInner v'}")
(k': v': "${quoteKey k'} = ${outputValInner v'}") v;
v; valsStr = concatStringsSep ", " vals;
valsStr = concatStringsSep ", " vals; in ["${quoteKey k} = { ${valsStr} }"]
in ["${quoteKey k} = { ${valsStr} }"]
else outputKeyVal k v; else outputKeyVal k v;
# Returns a list of strings; one string per line # Returns a list of strings; one string per line
@ -88,11 +84,10 @@ let
) )
v v
else if ty == "list" else if ty == "list"
then then let
let vals = map quoteString v;
vals = map quoteString v; valsStr = concatStringsSep ", " vals;
valsStr = concatStringsSep ", " vals; in ["${quoteKey k} = [ ${valsStr} ]"]
in ["${quoteKey k} = [ ${valsStr} ]"]
else if ty == "set" else if ty == "set"
then ["[${k}]"] ++ (concatLists (mapAttrsToList outputKeyValInner v)) then ["[${k}]"] ++ (concatLists (mapAttrsToList outputKeyValInner v))
else abort "Not implemented: type ${ty} for key ${k}"; else abort "Not implemented: type ${ty} for key ${k}";
@ -115,26 +110,26 @@ let
then then
if length x == 0 if length x == 0
then "list" then "list"
else else let
let ty = typeOf (elemAt x 0);
ty = typeOf (elemAt x 0); in
in #assert (all (v: typeOf v == ty) x);
#assert (all (v: typeOf v == ty) x); if ty == "set"
if ty == "set" then "list_of_attrs"
then "list_of_attrs" else "list"
else "list"
else abort "Not implemented: toml type for ${typeOf x}"; else abort "Not implemented: toml type for ${typeOf x}";
toTOML = attrs: toTOML = attrs:
assert (typeOf attrs == "set"); let assert (typeOf attrs == "set"); let
byTy = lib.foldl byTy =
( lib.foldl
acc: x: let (
ty = tomlTy x.v; acc: x: let
in ty = tomlTy x.v;
acc // { "${ty}" = (acc.${ty} or []) ++ [x]; } in
) acc // {"${ty}" = (acc.${ty} or []) ++ [x];}
{} (mapAttrsToList (k: v: { inherit k v; }) attrs); )
{} (mapAttrsToList (k: v: {inherit k v;}) attrs);
in in
concatMapStringsSep "\n" concatMapStringsSep "\n"
(kv: concatStringsSep "\n" (outputKeyVal kv.k kv.v)) (kv: concatStringsSep "\n" (outputKeyVal kv.k kv.v))

476
src/utils/translator.nix
View File

@ -1,270 +1,262 @@
{ {
lib, lib,
# dream2nix # dream2nix
fetchers, fetchers,
dlib, dlib,
... ...
}: }: let
let
b = builtins; b = builtins;
overrideWarning = fields: args: overrideWarning = fields: args:
lib.filterAttrs (name: _: lib.filterAttrs (
if lib.any (field: name == field) fields name: _:
then lib.warn '' if lib.any (field: name == field) fields
you are trying to pass a "${name}" key from your source then
constructor, this will be overrided with a value passed lib.warn ''
by dream2nix. you are trying to pass a "${name}" key from your source
'' false constructor, this will be overrided with a value passed
else true by dream2nix.
) args; ''
false
else true
)
args;
simpleTranslate = func: simpleTranslate = func: let
let final =
final = func
func {
{ inherit getDepByNameVer;
inherit getDepByNameVer; inherit dependenciesByOriginalID;
inherit dependenciesByOriginalID; };
};
getDepByNameVer = name: version: getDepByNameVer = name: version:
final.allDependencies."${name}"."${version}" or null; final.allDependencies."${name}"."${version}" or null;
dependenciesByOriginalID = b.foldl' dependenciesByOriginalID =
(result: pkgData: lib.recursiveUpdate result { b.foldl'
(result: pkgData:
lib.recursiveUpdate result {
"${final.getOriginalID pkgData}" = pkgData; "${final.getOriginalID pkgData}" = pkgData;
}) })
{}
serializedPackagesList;
serializedPackagesList = final.serializePackages final.inputData;
dreamLockData = magic final;
magic = {
# values
defaultPackage,
inputData,
location ? "",
mainPackageDependencies,
packages,
subsystemName,
subsystemAttrs,
translatorName,
# functions
serializePackages,
getName,
getVersion,
getSourceType,
sourceConstructors,
createMissingSource ? (name: version: throw "Cannot find source for ${name}:${version}"),
getDependencies ? null,
getOriginalID ? null,
mainPackageSource ? {type = "unknown";},
}: let
allDependencies =
b.foldl'
(result: pkgData:
lib.recursiveUpdate result {
"${getName pkgData}" = {
"${getVersion pkgData}" = pkgData;
};
})
{} {}
serializedPackagesList; serializedPackagesList;
serializedPackagesList = final.serializePackages final.inputData; sources =
b.foldl'
dreamLockData = magic final; (result: pkgData: let
pkgName = getName pkgData;
magic = pkgVersion = getVersion pkgData;
{
# values
defaultPackage,
inputData,
location ? "",
mainPackageDependencies,
packages,
subsystemName,
subsystemAttrs,
translatorName,
# functions
serializePackages,
getName,
getVersion,
getSourceType,
sourceConstructors,
createMissingSource ?
(name: version: throw "Cannot find source for ${name}:${version}"),
getDependencies ? null,
getOriginalID ? null,
mainPackageSource ? { type = "unknown"; },
}:
let
allDependencies = b.foldl'
(result: pkgData: lib.recursiveUpdate result {
"${getName pkgData}" = {
"${getVersion pkgData}" = pkgData;
};
})
{}
serializedPackagesList;
sources = b.foldl'
(result: pkgData:
let
pkgName = getName pkgData;
pkgVersion = getVersion pkgData;
in lib.recursiveUpdate result {
"${pkgName}" = {
"${pkgVersion}" =
let
type = getSourceType pkgData;
constructedArgs = sourceConstructors."${type}" pkgData;
constructedArgsKeep =
overrideWarning [ "pname" "version" ] constructedArgs;
constructedSource =
fetchers.constructSource (constructedArgsKeep // {
inherit type;
pname = pkgName;
version = pkgVersion;
});
in
b.removeAttrs constructedSource [ "pname" "version" ];
};
})
{}
serializedPackagesList;
dependencyGraph =
let
depGraph =
(lib.mapAttrs
(name: versions:
lib.mapAttrs
(version: pkgData: getDependencies pkgData)
versions)
allDependencies);
in
depGraph // {
"${defaultPackage}" = depGraph."${defaultPackage}" or {} // {
"${packages."${defaultPackage}"}" = mainPackageDependencies;
};
};
allDependencyKeys =
let
depsWithDuplicates =
lib.flatten
(lib.flatten
(lib.mapAttrsToList
(name: versions: lib.attrValues versions)
dependencyGraph));
in
lib.unique depsWithDuplicates;
missingDependencies =
lib.flatten
(lib.forEach allDependencyKeys
(dep:
if sources ? "${dep.name}"."${dep.version}" then
[]
else
dep));
generatedSources =
if missingDependencies == [] then
{}
else
lib.listToAttrs
(b.map
(dep: lib.nameValuePair
"${dep.name}"
{
"${dep.version}" =
createMissingSource dep.name dep.version;
})
missingDependencies);
allSources =
lib.recursiveUpdate sources generatedSources;
cyclicDependencies =
# TODO: inefficient! Implement some kind of early cutoff
let
findCycles = node: prevNodes: cycles:
let
children = dependencyGraph."${node.name}"."${node.version}";
cyclicChildren =
lib.filter
(child: prevNodes ? "${child.name}#${child.version}")
children;
nonCyclicChildren =
lib.filter
(child: ! prevNodes ? "${child.name}#${child.version}")
children;
cycles' =
cycles
++
(b.map (child: { from = node; to = child; }) cyclicChildren);
# use set for efficient lookups
prevNodes' =
prevNodes
// { "${node.name}#${node.version}" = null; };
in
if nonCyclicChildren == [] then
cycles'
else
lib.flatten
(b.map
(child: findCycles child prevNodes' cycles')
nonCyclicChildren);
cyclesList =
findCycles
(dlib.nameVersionPair defaultPackage packages."${defaultPackage}")
{}
[];
in
b.foldl'
(cycles: cycle:
(
let
existing =
cycles."${cycle.from.name}"."${cycle.from.version}"
or [];
reverse =
cycles."${cycle.to.name}"."${cycle.to.version}"
or [];
in
# if edge or reverse edge already in cycles, do nothing
if b.elem cycle.from reverse
|| b.elem cycle.to existing then
cycles
else
lib.recursiveUpdate
cycles
{
"${cycle.from.name}"."${cycle.from.version}" =
existing ++ [ cycle.to ];
}))
{}
cyclesList;
in in
{ lib.recursiveUpdate result {
decompressed = true; "${pkgName}" = {
"${pkgVersion}" = let
type = getSourceType pkgData;
_generic = constructedArgs = sourceConstructors."${type}" pkgData;
constructedArgsKeep =
overrideWarning ["pname" "version"] constructedArgs;
constructedSource = fetchers.constructSource (constructedArgsKeep
// {
inherit type;
pname = pkgName;
version = pkgVersion;
});
in
b.removeAttrs constructedSource ["pname" "version"];
};
})
{}
serializedPackagesList;
dependencyGraph = let
depGraph =
lib.mapAttrs
(name: versions:
lib.mapAttrs
(version: pkgData: getDependencies pkgData)
versions)
allDependencies;
in
depGraph
// {
"${defaultPackage}" =
depGraph."${defaultPackage}"
or {}
// {
"${packages."${defaultPackage}"}" = mainPackageDependencies;
};
};
allDependencyKeys = let
depsWithDuplicates =
lib.flatten
(lib.flatten
(lib.mapAttrsToList
(name: versions: lib.attrValues versions)
dependencyGraph));
in
lib.unique depsWithDuplicates;
missingDependencies =
lib.flatten
(lib.forEach allDependencyKeys
(dep:
if sources ? "${dep.name}"."${dep.version}"
then []
else dep));
generatedSources =
if missingDependencies == []
then {}
else
lib.listToAttrs
(b.map
(dep:
lib.nameValuePair
"${dep.name}"
{ {
inherit "${dep.version}" =
defaultPackage createMissingSource dep.name dep.version;
location })
packages missingDependencies);
;
subsystem = subsystemName;
sourcesAggregatedHash = null;
};
# build system specific attributes allSources =
_subsystem = subsystemAttrs; lib.recursiveUpdate sources generatedSources;
inherit cyclicDependencies; cyclicDependencies =
# TODO: inefficient! Implement some kind of early cutoff
let
findCycles = node: prevNodes: cycles: let
children = dependencyGraph."${node.name}"."${node.version}";
sources = allSources; cyclicChildren =
} lib.filter
// (child: prevNodes ? "${child.name}#${child.version}")
(lib.optionalAttrs children;
(getDependencies != null)
{ dependencies = dependencyGraph; });
nonCyclicChildren =
lib.filter
(child: ! prevNodes ? "${child.name}#${child.version}")
children;
cycles' =
cycles
++ (b.map (child: {
from = node;
to = child;
})
cyclicChildren);
# use set for efficient lookups
prevNodes' =
prevNodes
// {"${node.name}#${node.version}" = null;};
in
if nonCyclicChildren == []
then cycles'
else
lib.flatten
(b.map
(child: findCycles child prevNodes' cycles')
nonCyclicChildren);
cyclesList =
findCycles
(dlib.nameVersionPair defaultPackage packages."${defaultPackage}")
{}
[];
in
b.foldl'
(cycles: cycle: (
let
existing =
cycles."${cycle.from.name}"."${cycle.from.version}"
or [];
reverse =
cycles."${cycle.to.name}"."${cycle.to.version}"
or [];
in
# if edge or reverse edge already in cycles, do nothing
if
b.elem cycle.from reverse
|| b.elem cycle.to existing
then cycles
else
lib.recursiveUpdate
cycles
{
"${cycle.from.name}"."${cycle.from.version}" =
existing ++ [cycle.to];
}
))
{}
cyclesList;
in in
{
decompressed = true;
dreamLockData; _generic = {
inherit
defaultPackage
location
packages
;
subsystem = subsystemName;
sourcesAggregatedHash = null;
};
in # build system specific attributes
{ _subsystem = subsystemAttrs;
inherit simpleTranslate;
}
inherit cyclicDependencies;
sources = allSources;
}
// (lib.optionalAttrs
(getDependencies != null)
{dependencies = dependencyGraph;});
in
dreamLockData;
in {
inherit simpleTranslate;
}

444
src/utils/translator2.nix
View File

@ -2,244 +2,234 @@
dlib, dlib,
lib, lib,
... ...
}: }: let
let
l = lib // builtins; l = lib // builtins;
simpleTranslate2 = func: simpleTranslate2 = func: let
let final =
final = func
func {
inherit objectsByKey;
};
rawObjects = final.serializedRawObjects;
expectedFields = [
"name"
"version"
"sourceSpec"
"dependencies"
];
finalObjects' =
l.map
(rawObj: let
finalObj =
{inherit rawObj;}
// l.mapAttrs
(key: extractFunc: extractFunc rawObj finalObj)
final.extractors;
in
finalObj)
rawObjects;
# checks validity of all objects by iterating over them
finalObjects =
l.map
(finalObj:
if l.any (field: ! finalObj ? "${field}") expectedFields
then
throw
''
Translator ${final.translatorName} failed.
The following object does not contain all required fields:
Object:
${l.toJSON finalObj}
Missing fields:
${l.subtractLists expectedFields (l.attrNames finalObj)}
''
# TODO: validate sourceSpec as well
else finalObj)
(finalObjects' ++ (final.extraObjects or []));
objectsByKey =
l.mapAttrs
(key: keyFunc:
l.foldl'
(merged: finalObj:
merged
// {"${keyFunc finalObj.rawObj finalObj}" = finalObj;})
{}
finalObjects')
final.keys;
dreamLockData = magic final;
magic = {
defaultPackage,
exportedPackages,
extractors,
extraDependencies ? {},
extraObjects ? [],
keys ? {},
location ? "",
serializedRawObjects,
subsystemName,
subsystemAttrs ? {},
translatorName,
}: let
allDependencies =
l.foldl'
(result: finalObj:
lib.recursiveUpdate
result
{ {
inherit objectsByKey; "${finalObj.name}" = {
}; "${finalObj.version}" = finalObj;
};
})
{}
finalObjects;
rawObjects = final.serializedRawObjects; sources =
expectedFields = [
"name"
"version"
"sourceSpec"
"dependencies"
];
finalObjects' =
l.map
(rawObj: let
finalObj =
{ inherit rawObj; }
// l.mapAttrs
(key: extractFunc: extractFunc rawObj finalObj)
final.extractors;
in
finalObj)
rawObjects;
# checks validity of all objects by iterating over them
finalObjects =
l.map
(finalObj:
if l.any (field: ! finalObj ? "${field}") expectedFields
then throw
''
Translator ${final.translatorName} failed.
The following object does not contain all required fields:
Object:
${l.toJSON finalObj}
Missing fields:
${l.subtractLists expectedFields (l.attrNames finalObj)}
''
# TODO: validate sourceSpec as well
else finalObj)
(finalObjects' ++ (final.extraObjects or []));
objectsByKey =
l.mapAttrs l.mapAttrs
(key: keyFunc: (name: versions:
l.foldl' l.mapAttrs
(merged: finalObj: (version: finalObj: finalObj.sourceSpec)
merged versions)
// {"${keyFunc finalObj.rawObj finalObj}" = finalObj;}) allDependencies;
{}
finalObjects')
final.keys;
dependencyGraph = let
dreamLockData = magic final; depGraph =
lib.mapAttrs
magic = (name: versions:
{ lib.mapAttrs
defaultPackage, (version: finalObj: finalObj.dependencies)
exportedPackages, versions)
extractors, allDependencies;
extraDependencies ? {}, in
extraObjects ? [], # add extraDependencies to dependency graph
keys ? {}, l.foldl'
location ? "", (all: new:
serializedRawObjects, all
subsystemName, // {
subsystemAttrs ? {}, "${new.name}" =
translatorName, all."${new.name}"
}: or {}
let // {
"${new.version}" =
allDependencies = l.foldl' all."${new.name}"."${new.version}"
(result: finalObj: or []
lib.recursiveUpdate ++ new.dependencies;
result
{
"${finalObj.name}" = {
"${finalObj.version}" = finalObj;
};
})
{}
finalObjects;
sources =
l.mapAttrs
(name: versions:
l.mapAttrs
(version: finalObj: finalObj.sourceSpec)
versions)
allDependencies;
dependencyGraph =
let
depGraph =
(lib.mapAttrs
(name: versions:
lib.mapAttrs
(version: finalObj: finalObj.dependencies)
versions)
allDependencies);
in
# add extraDependencies to dependency graph
l.foldl'
(all: new:
all // {
"${new.name}" =
all."${new.name}" or {}
// {
"${new.version}" =
all."${new.name}"."${new.version}" or []
++ new.dependencies;
};
})
depGraph
extraDependencies;
cyclicDependencies =
# TODO: inefficient! Implement some kind of early cutoff
let
depGraphWithFakeRoot =
l.recursiveUpdate
dependencyGraph
{
__fake-entry.__fake-version =
l.mapAttrsToList
dlib.nameVersionPair
exportedPackages;
};
findCycles = node: prevNodes: cycles:
let
children =
depGraphWithFakeRoot."${node.name}"."${node.version}";
cyclicChildren =
lib.filter
(child: prevNodes ? "${child.name}#${child.version}")
children;
nonCyclicChildren =
lib.filter
(child: ! prevNodes ? "${child.name}#${child.version}")
children;
cycles' =
cycles
++
(l.map (child: { from = node; to = child; }) cyclicChildren);
# use set for efficient lookups
prevNodes' =
prevNodes
// { "${node.name}#${node.version}" = null; };
in
if nonCyclicChildren == [] then
cycles'
else
lib.flatten
(l.map
(child: findCycles child prevNodes' cycles')
nonCyclicChildren);
cyclesList =
findCycles
(dlib.nameVersionPair
"__fake-entry"
"__fake-version")
{}
[];
in
l.foldl'
(cycles: cycle:
(
let
existing =
cycles."${cycle.from.name}"."${cycle.from.version}"
or [];
reverse =
cycles."${cycle.to.name}"."${cycle.to.version}"
or [];
in
# if edge or reverse edge already in cycles, do nothing
if l.elem cycle.from reverse
|| l.elem cycle.to existing then
cycles
else
lib.recursiveUpdate
cycles
{
"${cycle.from.name}"."${cycle.from.version}" =
existing ++ [ cycle.to ];
}))
{}
cyclesList;
in
{
decompressed = true;
_generic =
{
inherit
defaultPackage
location
;
packages = exportedPackages;
subsystem = subsystemName;
sourcesAggregatedHash = null;
}; };
})
depGraph
extraDependencies;
# build system specific attributes cyclicDependencies =
_subsystem = subsystemAttrs; # TODO: inefficient! Implement some kind of early cutoff
let
depGraphWithFakeRoot =
l.recursiveUpdate
dependencyGraph
{
__fake-entry.__fake-version =
l.mapAttrsToList
dlib.nameVersionPair
exportedPackages;
};
inherit cyclicDependencies sources; findCycles = node: prevNodes: cycles: let
} children =
// { dependencies = dependencyGraph; }; depGraphWithFakeRoot."${node.name}"."${node.version}";
cyclicChildren =
lib.filter
(child: prevNodes ? "${child.name}#${child.version}")
children;
nonCyclicChildren =
lib.filter
(child: ! prevNodes ? "${child.name}#${child.version}")
children;
cycles' =
cycles
++ (l.map (child: {
from = node;
to = child;
})
cyclicChildren);
# use set for efficient lookups
prevNodes' =
prevNodes
// {"${node.name}#${node.version}" = null;};
in
if nonCyclicChildren == []
then cycles'
else
lib.flatten
(l.map
(child: findCycles child prevNodes' cycles')
nonCyclicChildren);
cyclesList =
findCycles
(dlib.nameVersionPair
"__fake-entry"
"__fake-version")
{}
[];
in
l.foldl'
(cycles: cycle: (
let
existing =
cycles."${cycle.from.name}"."${cycle.from.version}"
or [];
reverse =
cycles."${cycle.to.name}"."${cycle.to.version}"
or [];
in
# if edge or reverse edge already in cycles, do nothing
if
l.elem cycle.from reverse
|| l.elem cycle.to existing
then cycles
else
lib.recursiveUpdate
cycles
{
"${cycle.from.name}"."${cycle.from.version}" =
existing ++ [cycle.to];
}
))
{}
cyclesList;
in in
{
decompressed = true;
dreamLockData; _generic = {
inherit
defaultPackage
location
;
packages = exportedPackages;
subsystem = subsystemName;
sourcesAggregatedHash = null;
};
in # build system specific attributes
{ _subsystem = subsystemAttrs;
inherit simpleTranslate2;
}
inherit cyclicDependencies sources;
}
// {dependencies = dependencyGraph;};
in
dreamLockData;
in {
inherit simpleTranslate2;
}

20
tests/impure/default.nix
View File

@ -2,28 +2,26 @@
async, async,
coreutils, coreutils,
lib, lib,
# dream2nix # dream2nix
callPackageDream, callPackageDream,
utils, utils,
... ...
}: }: let
let
l = lib // builtins; l = lib // builtins;
allTestFiles = allTestFiles =
l.attrNames l.attrNames
(l.filterAttrs (l.filterAttrs
(name: type: type == "regular" && l.hasPrefix "test_" name) (name: type: type == "regular" && l.hasPrefix "test_" name)
(l.readDir ./.)); (l.readDir ./.));
allTests = allTests =
l.map l.map
(file: callPackageDream ("${./.}/${file}") {}) (file: callPackageDream "${./.}/${file}" {})
allTestFiles; allTestFiles;
executeAll = utils.writePureShellScript executeAll =
utils.writePureShellScript
[ [
async async
coreutils coreutils
@ -39,7 +37,5 @@ let
async -s=$S wait async -s=$S wait
rm $S rm $S
''; '';
in in
executeAll executeAll

32
tests/impure/test_combined_fetching.nix
View File

@ -1,28 +1,24 @@
{ {
lib, lib,
# dream2nix # dream2nix
apps, apps,
utils, utils,
... ...
}: }: let
let
l = lib // builtins; l = lib // builtins;
cli = apps.cli.program; cli = apps.cli.program;
in in
utils.writePureShellScript utils.writePureShellScript
[] []
'' ''
${cli} add github:prettier/prettier/2.4.1 \ ${cli} add github:prettier/prettier/2.4.1 \
--no-default-nix \ --no-default-nix \
--translator yarn-lock \ --translator yarn-lock \
--attribute-name prettier \ --attribute-name prettier \
--arg name="{automatic}" \ --arg name="{automatic}" \
--arg noDev=false \ --arg noDev=false \
--arg nodejs=14 \ --arg nodejs=14 \
--arg peer=false \ --arg peer=false \
--aggregate --aggregate
'' ''

22
tests/impure/test_go.nix
View File

@ -1,23 +1,19 @@
{ {
lib, lib,
# dream2nix # dream2nix
apps, apps,
utils, utils,
... ...
}: }: let
let
l = lib // builtins; l = lib // builtins;
cli = apps.cli.program; cli = apps.cli.program;
in in
utils.writePureShellScript utils.writePureShellScript
[] []
'' ''
${cli} add github:tweag/gomod2nix/67f22dd738d092c6ba88e420350ada0ed4992ae8 \ ${cli} add github:tweag/gomod2nix/67f22dd738d092c6ba88e420350ada0ed4992ae8 \
--no-default-nix \ --no-default-nix \
--translator gomod2nix \ --translator gomod2nix \
--attribute-name gomod2nix --attribute-name gomod2nix
'' ''

32
tests/impure/test_package-json.nix
View File

@ -1,29 +1,25 @@
{ {
lib, lib,
# dream2nix # dream2nix
apps, apps,
utils, utils,
... ...
}: }: let
let
l = lib // builtins; l = lib // builtins;
cli = apps.cli.program; cli = apps.cli.program;
in in
utils.writePureShellScript utils.writePureShellScript
[] []
'' ''
${cli} add npm:eslint/8.4.0 \ ${cli} add npm:eslint/8.4.0 \
--no-default-nix \ --no-default-nix \
--translator package-json \ --translator package-json \
--attribute-name eslint \ --attribute-name eslint \
--arg name="{automatic}" \ --arg name="{automatic}" \
--arg noDev=true \ --arg noDev=true \
--arg nodejs=14 \ --arg nodejs=14 \
--arg npmArgs= --arg npmArgs=
${cli} update eslint --to-version 8.4.1 ${cli} update eslint --to-version 8.4.1
'' ''

28
tests/impure/test_package-lock.nix
View File

@ -1,26 +1,22 @@
{ {
lib, lib,
# dream2nix # dream2nix
apps, apps,
utils, utils,
... ...
}: }: let
let
l = lib // builtins; l = lib // builtins;
cli = apps.cli.program; cli = apps.cli.program;
in in
utils.writePureShellScript utils.writePureShellScript
[] []
'' ''
${cli} add github:mattermost/mattermost-webapp/v6.1.0 \ ${cli} add github:mattermost/mattermost-webapp/v6.1.0 \
--no-default-nix \ --no-default-nix \
--translator package-lock \ --translator package-lock \
--attribute-name mattermost-webapp \ --attribute-name mattermost-webapp \
--arg name="{automatic}" \ --arg name="{automatic}" \
--arg noDev=false \ --arg noDev=false \
--arg nodejs=14 --arg nodejs=14
'' ''

24
tests/impure/test_rust.nix
View File

@ -1,24 +1,20 @@
{ {
lib, lib,
# dream2nix # dream2nix
apps, apps,
utils, utils,
... ...
}: }: let
let
l = lib // builtins; l = lib // builtins;
cli = apps.cli.program; cli = apps.cli.program;
in in
utils.writePureShellScript utils.writePureShellScript
[] []
'' ''
${cli} add github:BurntSushi/ripgrep/13.0.0 \ ${cli} add github:BurntSushi/ripgrep/13.0.0 \
--no-default-nix \ --no-default-nix \
--translator cargo-lock \ --translator cargo-lock \
--arg packageName="ripgrep" \ --arg packageName="ripgrep" \
--attribute-name ripgrep --attribute-name ripgrep
'' ''

28
tests/impure/test_yarn-lock-prettier.nix
View File

@ -1,26 +1,22 @@
{ {
lib, lib,
# dream2nix # dream2nix
apps, apps,
utils, utils,
... ...
}: }: let
let
l = lib // builtins; l = lib // builtins;
cli = apps.cli.program; cli = apps.cli.program;
in in
utils.writePureShellScript utils.writePureShellScript
[] []
'' ''
${cli} add github:prettier/prettier/2.4.1 \ ${cli} add github:prettier/prettier/2.4.1 \
--no-default-nix \ --no-default-nix \
--translator yarn-lock \ --translator yarn-lock \
--attribute-name prettier \ --attribute-name prettier \
--arg name="{automatic}" \ --arg name="{automatic}" \
--arg noDev=false \ --arg noDev=false \
--arg nodejs=14 --arg nodejs=14
'' ''

12
tests/pure/default.nix
View File

@ -1,10 +1,8 @@
{ {
lib ? pkgs.lib, lib ? pkgs.lib,
pkgs ? import <nixpkgs> {}, pkgs ? import <nixpkgs> {},
dream2nix ? import ./src { inherit pkgs; }, dream2nix ? import ./src {inherit pkgs;},
}: }: let
let
l = pkgs.lib // builtins; l = pkgs.lib // builtins;
buildProjectsTests = import ./projects.nix { buildProjectsTests = import ./projects.nix {
@ -14,8 +12,6 @@ let
otherTests = import ./other { otherTests = import ./other {
inherit lib pkgs dream2nix; inherit lib pkgs dream2nix;
}; };
in in
buildProjectsTests buildProjectsTests
// // otherTests
otherTests

21
tests/pure/other/default.nix
View File

@ -1,21 +1,18 @@
{ {
lib ? pkgs.lib, lib ? pkgs.lib,
pkgs ? import <nixpkgs> {}, pkgs ? import <nixpkgs> {},
dream2nix ? import ./src { inherit pkgs; }, dream2nix ? import ./src {inherit pkgs;},
}: }: let
let
l = pkgs.lib // builtins; l = pkgs.lib // builtins;
fetchAggrgatedGithub = fetchAggrgatedGithub =
dream2nix.utils.toDrv dream2nix.utils.toDrv
(dream2nix.fetchSources { (dream2nix.fetchSources {
dreamLock = ./prettier-github-aggregated.json; dreamLock = ./prettier-github-aggregated.json;
}).fetchedSources.prettier."2.4.1"; })
.fetchedSources
in .prettier
{ ."2.4.1";
in {
inherit fetchAggrgatedGithub; inherit fetchAggrgatedGithub;
} }

55
tests/pure/projects.nix
View File

@ -1,26 +1,22 @@
{ {
lib ? pkgs.lib, lib ? pkgs.lib,
pkgs ? import <nixpkgs> {}, pkgs ? import <nixpkgs> {},
dream2nix ? import ./src { inherit pkgs; }, dream2nix ? import ./src {inherit pkgs;},
}: }: let
let
lib = pkgs.lib // builtins; lib = pkgs.lib // builtins;
makeTest = makeTest = {
{ name,
name, source,
source, cmds,
cmds, }: let
}: outputs = dream2nix.makeOutputs {
let inherit source;
outputs = dream2nix.makeOutputs { };
inherit source; commandsToRun = cmds outputs;
}; in
commandsToRun = cmds outputs; pkgs.runCommand "test-${name}" {}
in (lib.concatStringsSep "\n" commandsToRun);
pkgs.runCommand "test-${name}" {}
(lib.concatStringsSep "\n" commandsToRun);
projects = { projects = {
prettier = { prettier = {
@ -28,22 +24,19 @@ let
url = "https://github.com/prettier/prettier/tarball/2.4.1"; url = "https://github.com/prettier/prettier/tarball/2.4.1";
sha256 = "19b37qakhlsnr2n5bgv83aih5npgzbad1d2p2rs3zbq5syqbxdyi"; sha256 = "19b37qakhlsnr2n5bgv83aih5npgzbad1d2p2rs3zbq5syqbxdyi";
}; };
cmds = outputs: cmds = outputs: let
let prettier = outputs.defaultPackage.overrideAttrs (old: {
prettier = outputs.defaultPackage.overrideAttrs (old: { dontBuild = true;
dontBuild = true; });
}); in [
in "${prettier}/bin/prettier --version | grep -q 2.4.1 && mkdir $out"
[ ];
"${prettier}/bin/prettier --version | grep -q 2.4.1 && mkdir $out"
];
}; };
}; };
allTests = allTests =
lib.mapAttrs lib.mapAttrs
(name: args: makeTest (args // { inherit name; })) (name: args: makeTest (args // {inherit name;}))
projects; projects;
in in
allTests allTests

22
tests/unit/default.nix
View File

@ -1,23 +1,19 @@
{ {
self, self,
lib, lib,
nix, nix,
python3, python3,
utils, utils,
dream2nixWithExternals, dream2nixWithExternals,
... ...
}: }: let
let l = lib // builtins;
l = lib // builtins;
in in
utils.writePureShellScript utils.writePureShellScript
[ [
nix nix
] ]
'' ''
export dream2nixSrc=${dream2nixWithExternals} export dream2nixSrc=${dream2nixWithExternals}
${python3.pkgs.pytest}/bin/pytest ${self}/tests/unit "$@" ${python3.pkgs.pytest}/bin/pytest ${self}/tests/unit "$@"
'' ''