Merge remote-tracking branch 'origin/main' into indexers

This commit is contained in:
DavHau 2022-10-15 00:21:18 +02:00
commit 26e4adffcc
92 changed files with 1005 additions and 2348 deletions

View File

@ -1,9 +1,11 @@
{
{config, ...}: let
inherit (config.pkgs) hello;
in {
builders.dummy = {...}: {
name = "dummy";
subsystem = "hello";
type = "pure";
build = {hello, ...}: {...}: {
build = {...}: {
packages.${hello.pname}.${hello.version} =
hello;
};

View File

@ -1,9 +1,9 @@
{dlib, ...}: {
{config, ...}: {
discoverers.dummy = rec {
name = "dummy";
subsystem = "hello";
discover = {tree}: [
(dlib.construct.discoveredProject {
(config.dlib.construct.discoveredProject {
inherit subsystem name;
inherit (tree) relPath;
translators = ["dummy"];

View File

@ -1,9 +1,11 @@
{
{config, ...}: let
inherit (config.pkgs) hello;
in {
translators.dummy = {...}: {
type = "pure";
name = "dummy";
subsystem = "hello";
translate = {hello, ...}: {...}: {
translate = {...}: {
result = {
_generic = {
subsystem = "hello";

View File

@ -17,7 +17,7 @@
(builtins.toFile "cargo-toml-new.nix" ''
{
translators.cargo-toml-new = {
imports = ["${inp.dream2nix}/src/subsystems/rust/translators/cargo-toml"];
imports = [(attrs: import "${inp.dream2nix}/src/subsystems/rust/translators/cargo-toml" attrs.framework)];
name = "cargo-toml-new";
subsystem = "rust";
};
@ -26,7 +26,7 @@
(builtins.toFile "brp-new.nix" ''
{
builders.brp-new = {
imports = ["${inp.dream2nix}/src/subsystems/rust/builders/build-rust-package"];
imports = [(attrs: import "${inp.dream2nix}/src/subsystems/rust/builders/build-rust-package" attrs.framework)];
name = "brp-new";
subsystem = "rust";
};
@ -35,16 +35,16 @@
(builtins.toFile "cargo-new.nix" ''
{
discoverers.cargo-new = {
imports = ["${inp.dream2nix}/src/subsystems/rust/discoverers/cargo"];
imports = [(attrs: import "${inp.dream2nix}/src/subsystems/rust/discoverers/cargo" attrs.framework)];
name = "cargo-new";
subsystem = "rust";
};
}
'')
(builtins.toFile "crates-io-new.nix" ''
{lib, ...}: {
fetchers.crates-io = lib.mkForce {
imports = ["${inp.dream2nix}/src/fetchers/crates-io"];
{config, ...}: {
fetchers.crates-io = config.lib.mkForce {
imports = [(attrs: import "${inp.dream2nix}/src/fetchers/crates-io" attrs.framework)];
};
}
'')

View File

@ -65,7 +65,7 @@ in
if [ ! -e "$bin" ]; then
echo "building executable for translator $translator"
${callNixWithD2N} build -o "$bin" "
dream2nix.framework.translatorInstances.$translator.translateBin
dream2nix.framework.translators.$translator.finalTranslateBin
"
fi
done

View File

@ -51,7 +51,7 @@ utils.writePureShellScriptBin
translateBin="$TRANSLATOR_DIR/$translator"
else
translateBin=$(${callNixWithD2N} build --print-out-paths --no-link "
dream2nix.framework.translatorInstances.$translator.translateBin
dream2nix.framework.translators.$translator.finalTranslateBin
")
fi

View File

@ -51,7 +51,17 @@ in let
};
framework = import ./modules/framework.nix {
inherit lib dlib callPackageDream pkgs utils;
inherit
apps
lib
dlib
pkgs
utils
externals
externalSources
dream2nixWithExternals
;
dream2nixConfigFile = configFile;
dream2nixConfig = config;
};
@ -71,13 +81,11 @@ in let
inherit dlib;
inherit externals;
inherit externalSources;
inherit fetchers;
inherit framework;
inherit indexers;
inherit dream2nixWithExternals;
inherit utils;
inherit nix;
inherit subsystems;
dream2nixInterface = {
inherit
makeOutputsForDreamLock
@ -98,17 +106,12 @@ in let
# apps for CLI and installation
apps = callPackageDream ./apps {};
# fetcher implementations
fetchers = callPackageDream ./fetchers {};
# indexer implementations
indexers = callPackageDream ./indexers {};
# updater modules to find newest package versions
updaters = callPackageDream ./updaters {};
subsystems = callPackageDream ./subsystems {};
externals = {
devshell = {
makeShell = import "${externalSources.devshell}/modules" pkgs;
@ -434,7 +437,7 @@ in let
settings ? [],
} @ args: let
getTranslator = translatorName:
framework.translatorInstances.${translatorName};
framework.translators.${translatorName};
isImpure = project: translatorName:
(getTranslator translatorName).type == "impure";
@ -499,7 +502,7 @@ in let
l.forEach projectsPureUnresolved
(proj: let
translator = getTranslator proj.translator;
dreamLock'' = translator.translate {
dreamLock'' = translator.finalTranslate {
inherit source tree discoveredProjects;
project = proj;
};
@ -650,7 +653,7 @@ in let
impureDiscoveredProjects =
l.filter
(proj:
framework.translatorInstances."${proj.translator}".type
framework.translators."${proj.translator}".type
== "impure")
discoveredProjects;
@ -716,7 +719,6 @@ in {
apps
callPackageDream
dream2nixWithExternals
fetchers
framework
indexers
fetchSources
@ -727,6 +729,5 @@ in {
utils
makeOutputsForDreamLock
makeOutputs
subsystems
;
}

View File

@ -1,228 +0,0 @@
# this fetcher takes an attrset of sources and combines all contained FODs
# to one large FOD. Non-FOD sources like derivations and store paths are
# not touched
{
bash,
async,
coreutils,
lib,
nix,
stdenv,
writeScript,
# dream2nix
defaultFetcher,
dlib,
utils,
...
}: {
# sources attrset from dream lock
sources,
sourcesAggregatedHash,
sourceOverrides,
...
} @ args: let
b = builtins;
l = lib // builtins;
# resolve to individual fetcher calls
defaultFetched = (defaultFetcher args).fetchedSources;
isFOD = drv:
lib.all
(attr: drv ? "${attr}")
["outputHash" "outputHashAlgo" "outputHashMode"];
drvArgs = drv:
(drv.overrideAttrs (args: {
passthru.originalArgs = args;
}))
.originalArgs;
# extract the arguments from the individual fetcher calls
FODArgsAll = let
FODArgsAll' =
lib.mapAttrs
(
name: versions:
lib.mapAttrs
(version: fetched:
# handle FOD sources
if isFOD fetched
then
(drvArgs fetched)
// {
isOriginal = false;
outPath = let
sanitizedName = l.strings.sanitizeDerivationName name;
in "${sanitizedName}/${version}/${fetched.name}";
}
# handle already extracted sources
else if fetched ? original && isFOD fetched.original
then
(drvArgs fetched.original)
// {
isOriginal = true;
outPath = let
sanitizedName = l.strings.sanitizeDerivationName name;
in "${sanitizedName}/${version}/${fetched.original.name}";
}
# handle path sources
else if lib.isString fetched
then "ignore"
# handle store path sources
else if lib.isStorePath fetched
then "ignore"
# handle unknown sources
else if fetched == "unknown"
then "ignore"
# error out on unknown source types
else
throw ''
Error while generating FOD fetcher for combined sources.
Cannot classify source of ${name}#${version}.
'')
versions
)
defaultFetched;
in
lib.filterAttrs
(name: versions: versions != {})
(lib.mapAttrs
(name: versions:
lib.filterAttrs
(version: fetcherArgs: fetcherArgs != "ignore")
versions)
FODArgsAll');
FODArgsAllList =
lib.flatten
(lib.mapAttrsToList
(name: versions:
b.attrValues versions)
FODArgsAll);
# convert arbitrary types to string, like nix does with derivation arguments
toString' = x:
if lib.isBool x
then
if x
then "1"
else ""
else if lib.isList x
then ''"${lib.concatStringsSep " " (lib.forEach x (y: toString' y))}"''
else if x == null
then ""
else b.toJSON x;
# set up nix build env for signle item
itemScript = fetcherArgs: ''
# export arguments for builder
${lib.concatStringsSep "\n" (lib.mapAttrsToList (argName: argVal: ''
export ${argName}=${
lib.replaceStrings ["$" ''\n''] [''\$'' "\n"] (toString' argVal)
}
'')
fetcherArgs)}
# run builder
bash ${fetcherArgs.builder}
'';
mkScriptForItem = fetcherArgs: ''
# configure $out
OUT_ORIG=$out
export out=$OUT_ORIG/${fetcherArgs.outPath}
mkdir -p $(dirname $out)
# set up TMP and TMPDIR
workdir=$(mktemp -d)
TMP=$workdir/TMP
TMPDIR=$TMP
mkdir -p $TMP
# do the work
pushd $workdir
${itemScript fetcherArgs}
popd
rm -r $workdir
'';
# builder which wraps several other FOD builders
# and executes these after each other inside a single build
builder = writeScript "multi-source-fetcher" ''
#!${bash}/bin/bash
export PATH=${coreutils}/bin:${bash}/bin
mkdir $out
S="/$TMP/async_socket"
async=${async}/bin/async
$async -s="$S" server --start -j40
# remove if resolved: https://github.com/ctbur/async/issues/6
sleep 1
${lib.concatStringsSep "\n"
(b.map
(fetcherArgs: ''
$async -s="$S" cmd -- bash -c '${mkScriptForItem fetcherArgs}'
'')
FODArgsAllList)}
$async -s="$S" wait
${lib.concatStringsSep "\n"
(b.map
(fetcherArgs: ''
if [ ! -e "$out/${fetcherArgs.outPath}" ]; then
echo "builder for ${fetcherArgs.name} terminated without creating out path: ${fetcherArgs.outPath}"
exit 1
fi
'')
FODArgsAllList)}
echo "FOD_HASH=$(${nix}/bin/nix --extra-experimental-features "nix-command flakes" hash path $out)"
'';
FODAllSources = let
nativeBuildInputs' =
lib.unique
(lib.foldl (a: b: a ++ b) []
(b.map
(fetcherArgs: (fetcherArgs.nativeBuildInputs or []))
FODArgsAllList));
in
stdenv.mkDerivation rec {
name = "sources-combined";
inherit builder;
nativeBuildInputs =
nativeBuildInputs'
++ [
coreutils
];
outputHashAlgo = "sha256";
outputHashMode = "recursive";
outputHash = sourcesAggregatedHash;
};
in {
FOD = FODAllSources;
fetchedSources =
lib.mapAttrs
(name: versions:
lib.mapAttrs
(version: source:
if FODArgsAll ? "${name}"."${version}".outPath
then
if FODArgsAll ? "${name}"."${version}".isOriginal
then
utils.extractSource {
source = "${FODAllSources}/${FODArgsAll."${name}"."${version}".outPath}";
name = l.strings.sanitizeDerivationName name;
}
else "${FODAllSources}/${FODArgsAll."${name}"."${version}".outPath}"
else defaultFetched."${name}"."${version}")
versions)
defaultFetched;
}

View File

@ -1,18 +1,17 @@
{lib, ...}: {
{
pkgs,
utils,
...
}: {
inputs = ["pname" "version"];
versionField = "version";
outputs = {
fetchurl,
runCommandLocal,
utils,
...
}: {
pname,
version,
...
} @ inp: let
}: let
b = builtins;
# See https://github.com/rust-lang/crates.io-index/blob/master/config.json#L2
url = "https://crates.io/api/v1/crates/${pname}/${version}/download";
@ -23,13 +22,13 @@
});
fetched = hash: let
fetched = fetchurl {
fetched = pkgs.fetchurl {
inherit url;
sha256 = hash;
name = "download-${pname}-${version}";
};
in
runCommandLocal "unpack-${pname}-${version}" {}
pkgs.runCommandLocal "unpack-${pname}-${version}" {}
''
mkdir -p $out
tar --strip-components 1 -xzf ${fetched} -C $out

View File

@ -1,70 +0,0 @@
{
lib,
# dream2nix attributes
dlib,
fetchSource,
fetchers,
...
}: {
# sources attrset from dream lock
defaultPackage,
defaultPackageVersion,
sourceOverrides,
sources,
sourceRoot ? null,
...
}: let
l = lib // builtins;
fetchedSources =
l.mapAttrs
(name: versions:
l.mapAttrs
(version: source:
if source.type == "unknown"
then "unknown"
else if source.type == "path"
then let
path =
if l.isStorePath (l.concatStringsSep "/" (l.take 4 (l.splitString "/" source.path)))
then source.path
else if name == source.rootName && version == source.rootVersion
then throw "source for ${name}@${version} is referencing itself"
else if source.rootName != null && source.rootVersion != null
then "${overriddenSources."${source.rootName}"."${source.rootVersion}"}/${source.path}"
else if sourceRoot != null
then "${sourceRoot}/${source.path}"
else throw "${name}-${version}: cannot determine path source";
in
l.path {
inherit path;
name = l.strings.sanitizeDerivationName "${name}-${version}-source";
}
else if fetchers.fetchers ? "${source.type}"
then
fetchSource {
source =
source
// {
pname = source.pname or name;
version = source.version or version;
};
}
else throw "unsupported source type '${source.type}'")
versions)
sources;
overriddenSources =
l.zipAttrsWith
(name: versionsSets:
l.zipAttrsWith
(version: sources: l.last sources)
versionsSets)
[
fetchedSources
(sourceOverrides fetchedSources)
];
in {
# attrset: pname -> path of downloaded source
fetchedSources = overriddenSources;
}

View File

@ -1,240 +0,0 @@
{
lib,
# dream2nix
callPackageDream,
dlib,
...
}: let
b = builtins;
callFetcher = module:
module // {outputs = callPackageDream module.outputs {};};
in rec {
fetchers = dlib.fetchers.mapFetchers callFetcher;
defaultFetcher = callPackageDream ./default-fetcher.nix {inherit fetchSource;};
combinedFetcher = callPackageDream ./combined-fetcher.nix {inherit defaultFetcher;};
constructSource = {
type,
reComputeHash ? false,
...
} @ args: let
fetcher = fetchers."${type}";
argsKeep = b.removeAttrs args ["reComputeHash"];
fetcherOutputs =
fetcher.outputs
(b.removeAttrs argsKeep ["dir" "hash" "type"]);
in
argsKeep
# if the hash was not provided, calculate hash on the fly (impure)
// (lib.optionalAttrs reComputeHash {
hash = fetcherOutputs.calcHash "sha256";
});
# update source spec to different version
updateSource = {
source,
newVersion,
...
}:
constructSource (source
// {
reComputeHash = true;
}
// {
"${fetchers."${source.type}".versionField}" = newVersion;
});
# fetch a source defined via a dream lock source spec
fetchSource = {
source,
extract ? false,
}: let
fetcher = fetchers."${source.type}";
fetcherArgs = b.removeAttrs source ["dir" "hash" "type"];
fetcherOutputs = fetcher.outputs fetcherArgs;
maybeArchive = fetcherOutputs.fetched (source.hash or null);
in
if source ? dir
then "${maybeArchive}/${source.dir}"
else maybeArchive;
# fetch a source defined by a shortcut
fetchShortcut = {
shortcut,
extract ? false,
}:
fetchSource {
source = translateShortcut {inherit shortcut;};
inherit extract;
};
parseShortcut = shortcut: let
# in: "git+https://foo.com/bar?kwarg1=lol&kwarg2=hello"
# out: [ "git+" "git" "https" "//" "foo.com/bar" "?kwarg1=lol&kwarg2=hello" "kwarg1=lol&kwarg2=hello" ]
split =
b.match
''(([[:alnum:]]+)\+)?([[:alnum:]-]+):(//)?([^\?]*)(\?(.*))?''
shortcut;
parsed = {
proto1 = b.elemAt split 1;
proto2 = b.elemAt split 2;
path = b.elemAt split 4;
allArgs = b.elemAt split 6;
kwargs = b.removeAttrs kwargs_ ["dir"];
dir = kwargs_.dir or null;
};
kwargs_ =
if parsed.allArgs == null
then {}
else
lib.listToAttrs
(map
(kwarg: let
split = lib.splitString "=" kwarg;
in
lib.nameValuePair
(b.elemAt split 0)
(b.elemAt split 1))
(lib.splitString "&" parsed.allArgs));
in
if split == null
then throw "Unable to parse shortcut: ${shortcut}"
else parsed;
renderUrlArgs = kwargs: let
asStr =
lib.concatStringsSep
"&"
(lib.mapAttrsToList
(name: val: "${name}=${val}")
kwargs);
in
if asStr == ""
then ""
else "?" + asStr;
# translate shortcut to dream lock source spec
translateShortcut = {
shortcut ? throw "pass shortcut",
parsed ? parseShortcut shortcut,
computeHash ? (! parsed.kwargs ? hash),
}: let
parsed = parseShortcut shortcut;
checkArgs = fetcherName: args: let
fetcher = fetchers."${fetcherName}";
unknownArgNames = lib.filter (argName: ! lib.elem argName fetcher.inputs) (lib.attrNames args);
missingArgNames = lib.filter (inputName: ! args ? "${inputName}") fetcher.inputs;
in
if lib.length unknownArgNames > 0
then throw "Received unknown arguments for fetcher '${fetcherName}': ${b.toString unknownArgNames}"
else if lib.length missingArgNames > 0
then throw "Missing arguments for fetcher '${fetcherName}': ${b.toString missingArgNames}"
else args;
translateHttpUrl = let
fetcher = fetchers.http;
urlArgsFinal = renderUrlArgs parsed.kwargs;
url = with parsed; "${proto2}://${path}${urlArgsFinal}";
fetcherOutputs = fetchers.http.outputs {
inherit url;
};
in
constructSource
{
inherit url;
type = "http";
}
// (lib.optionalAttrs (parsed.dir != null) {
dir = parsed.dir;
})
// (lib.optionalAttrs computeHash {
hash = fetcherOutputs.calcHash "sha256";
});
translateProtoShortcut = let
kwargsUrl = b.removeAttrs parsed.kwargs fetcher.inputs;
urlArgs = renderUrlArgs kwargsUrl;
url = with parsed; "${proto2}://${path}${urlArgs}";
fetcherName = parsed.proto1;
fetcher = fetchers."${fetcherName}";
args = parsed.kwargs // {inherit url;};
fetcherOutputs = fetcher.outputs (checkArgs fetcherName args);
in
constructSource
(parsed.kwargs
// {
type = fetcherName;
inherit url;
}
// (lib.optionalAttrs (parsed.dir != null) {
dir = parsed.dir;
})
// (lib.optionalAttrs computeHash {
hash = fetcherOutputs.calcHash "sha256";
}));
translateRegularShortcut = let
fetcherName = parsed.proto2;
path = lib.removeSuffix "/" parsed.path;
params = lib.splitString "/" path;
fetcher = fetchers."${fetcherName}";
args =
if fetcher ? parseParams
then fetcher.parseParams params
else if b.length params != b.length fetcher.inputs
then
throw ''
Wrong number of arguments provided in shortcut for fetcher '${fetcherName}'
Should be ${fetcherName}:${lib.concatStringsSep "/" fetcher.inputs}
''
else
lib.listToAttrs
(lib.forEach
(lib.range 0 ((lib.length fetcher.inputs) - 1))
(
idx:
lib.nameValuePair
(lib.elemAt fetcher.inputs idx)
(lib.elemAt params idx)
));
fetcherOutputs = fetcher.outputs (args // parsed.kwargs);
in
constructSource (args
// parsed.kwargs
// {
type = fetcherName;
}
// (lib.optionalAttrs (parsed.dir != null) {
dir = parsed.dir;
})
// (lib.optionalAttrs computeHash {
hash = fetcherOutputs.calcHash "sha256";
}));
in
if parsed.proto1 != null
then translateProtoShortcut
else if
lib.hasPrefix "http://" shortcut
|| lib.hasPrefix "https://" shortcut
then translateHttpUrl
else translateRegularShortcut;
}

View File

@ -1,4 +1,9 @@
{lib, ...}: let
{
pkgs,
lib,
utils,
...
}: let
b = builtins;
# check if a string is a git ref
@ -14,10 +19,6 @@ in {
versionField = "rev";
outputs = {
fetchgit,
utils,
...
}: {
url,
rev,
submodules ? true,
@ -78,7 +79,7 @@ in {
inherit submodules;
})
else
fetchgit
pkgs.fetchgit
(refAndRev
// {
inherit url;

View File

@ -1,4 +1,9 @@
{lib, ...}: {
{
pkgs,
lib,
utils,
...
}: {
inputs = [
"owner"
"repo"
@ -10,15 +15,11 @@
defaultUpdater = "githubNewestReleaseTag";
outputs = {
fetchFromGitHub,
utils,
...
}: {
owner,
repo,
rev,
...
} @ inp: let
}: let
b = builtins;
in {
calcHash = algo:
@ -27,7 +28,7 @@
});
fetched = hash:
fetchFromGitHub {
pkgs.fetchFromGitHub {
inherit owner repo rev hash;
};
};

View File

@ -1,4 +1,8 @@
{...}: {
{
pkgs,
utils,
...
}: {
inputs = [
"owner"
"repo"
@ -8,15 +12,11 @@
versionField = "rev";
outputs = {
fetchFromGitLab,
utils,
...
}: {
owner,
repo,
rev,
...
} @ inp: let
}: let
b = builtins;
in {
calcHash = algo:
@ -25,7 +25,7 @@
});
fetched = hash:
fetchFromGitLab {
pkgs.fetchFromGitLab {
inherit owner repo rev hash;
};
};

View File

@ -1,13 +1,14 @@
{lib, ...}: {
{
pkgs,
lib,
utils,
...
}: {
inputs = [
"url"
];
outputs = {
fetchurl,
utils,
...
}: {url, ...} @ inp: let
outputs = {url, ...}: let
b = builtins;
in {
calcHash = algo:
@ -19,12 +20,12 @@
drv =
if hash != null && lib.stringLength hash == 40
then
fetchurl {
pkgs.fetchurl {
inherit url;
sha1 = hash;
}
else
fetchurl {
pkgs.fetchurl {
inherit url hash;
};

View File

@ -1,4 +1,9 @@
{lib, ...}: let
{
pkgs,
lib,
utils,
...
}: let
b = builtins;
in rec {
inputs = ["pname" "version"];
@ -36,14 +41,9 @@ in rec {
# defaultUpdater = "";
outputs = {
python3,
utils,
fetchurl,
...
}: {
pname,
version,
} @ inp: let
}: let
b = builtins;
submodule = lib.last (lib.splitString "/" pname);
@ -56,7 +56,7 @@ in rec {
fetched = hash: let
source =
(fetchurl {
(pkgs.fetchurl {
inherit url;
sha256 = hash;
})

View File

@ -1,11 +1,9 @@
{...}: {
{utils, ...}: {
inputs = [
"path"
];
outputs = {utils, ...}: {path, ...} @ inp: let
b = builtins;
in {
outputs = {path, ...}: {
calcHash = algo: utils.hashPath "${path}";
fetched = hash: "${path}";

View File

@ -1,4 +1,8 @@
{...}: {
{
pkgs,
utils,
...
}: {
inputs = ["pname" "version"];
versionField = "version";
@ -6,15 +10,10 @@
defaultUpdater = "pypiNewestReleaseVersion";
outputs = {
fetchurl,
python3,
utils,
...
}: {
pname,
version,
extension ? "tar.gz",
} @ inp: let
}: let
b = builtins;
firstChar = builtins.substring 0 1 pname;
@ -29,7 +28,7 @@
fetched = hash: let
source =
(fetchurl {
(pkgs.fetchurl {
inherit url;
sha256 = hash;
})

View File

@ -90,12 +90,14 @@
dlib = import ./lib {inherit lib config;};
framework = import ./modules/framework.nix {
inherit lib dlib;
inherit lib dlib externalSources;
dream2nixConfig = config;
callPackageDream =
throw "callPackageDream is not available before nixpkgs is imported";
dream2nixConfigFile = l.toFile "dream2nix-config.json" (l.toJSON config);
apps = throw "apps is not available before nixpkgs is imported";
pkgs = throw "pkgs is not available before nixpkgs is imported";
utils = throw "utils is not available before nixpkgs is imported";
externals = throw "externals is not available before nixpkgs is imported";
dream2nixWithExternals = throw "not available before nixpkgs is imported";
};
initD2N = initDream2nix config;

View File

@ -1,32 +0,0 @@
{
dlib,
lib,
config,
}: let
l = lib // builtins;
defaults = {
rust = "build-rust-package";
nodejs = "granular";
python = "simple-builder";
php = "granular";
racket = "simple";
};
# TODO
validator = module: true;
modules = dlib.modules.makeSubsystemModules {
inherit validator defaults;
modulesCategory = "builders";
};
builders =
dlib.modules.mapSubsystemModules
(b: b // {build = dlib.warnIfIfd b b.build;})
modules.modules;
mapBuilders = f: dlib.modules.mapSubsystemModules f builders;
in {
inherit builders mapBuilders;
callBuilder = modules.callModule;
}

View File

@ -8,14 +8,11 @@
# exported attributes
dlib = {
inherit
builders
calcInvalidationHash
callViaEnv
construct
containsMatchingFile
dirNames
discoverers
fetchers
indexers
latestVersion
listDirs
@ -25,7 +22,6 @@
readTextFile
recursiveUpdateUntilDepth
simpleTranslate2
translators
sanitizePath
sanitizeRelativePath
subsystems
@ -46,11 +42,7 @@
subsystems = dirNames ../subsystems;
# other libs
builders = import ./builders.nix {inherit dlib lib config;};
construct = import ./construct.nix {inherit lib;};
discoverers = import ./discoverers.nix {inherit config dlib lib;};
fetchers = import ./fetchers.nix {inherit dlib lib;};
translators = import ./translators.nix {inherit dlib lib;};
indexers = import ./indexers.nix {inherit dlib lib;};
modules = import ./modules.nix {inherit config dlib lib;};

View File

@ -1,147 +0,0 @@
{
dlib,
lib,
config,
}: let
l = lib // builtins;
allDiscoverers =
(dlib.modules.collectSubsystemModules modules.modules)
++ [defaultDiscoverer];
allTranslators =
dlib.modules.collectSubsystemModules dlib.translators.translators;
translatorsWithDiscoverFunc =
l.filter (translator: translator.discoverProject or null != null) allTranslators;
defaultDiscoverer.discover = {tree}: let
translatorsCurrentDir =
l.filter
(t: t.discoverProject tree)
translatorsWithDiscoverFunc;
projectsCurrentDir =
l.map
(t: {
name = "main";
relPath = tree.relPath;
translators = [t.name];
subsystem = t.subsystem;
})
translatorsCurrentDir;
# If there are multiple projects detected for the same subsystem,
# merge them to a single one with translators = [...]
projectsCurrentDirMerged =
l.attrValues
(l.foldl
(all: curr:
all
// {
"${curr.subsystem}" =
all.${curr.subsystem}
or curr
// {
translators =
l.unique (all.${curr.subsystem}.translators or [] ++ curr.translators);
};
})
{}
projectsCurrentDir);
subdirProjects =
l.flatten
(l.mapAttrsToList
(dirName: tree:
defaultDiscoverer.discover {
inherit tree;
})
tree.directories or {});
in (
if translatorsCurrentDir == []
then subdirProjects
else projectsCurrentDirMerged ++ subdirProjects
);
discoverProjects = {
projects,
source ? throw "Pass either `source` or `tree` to discoverProjects",
tree ? dlib.prepareSourceTree {inherit source;},
settings ? [],
}: let
discoveredProjects =
l.flatten
(l.map
(discoverer: discoverer.discover {inherit tree;})
allDiscoverers);
discoveredProjectsSorted = let
sorted =
l.sort
(p1: p2: l.hasPrefix p1.relPath or "" p2.relPath or "")
discoveredProjects;
in
sorted;
allProjects = discoveredProjectsSorted ++ (l.attrValues projects);
rootProject = l.head allProjects;
projectsExtended =
l.forEach allProjects
(proj:
proj
// {
relPath = proj.relPath or "";
translator = proj.translator or (l.head proj.translators);
dreamLockPath = getDreamLockPath proj rootProject;
});
in
applyProjectSettings projectsExtended settings;
getDreamLockPath = project: rootProject:
dlib.sanitizeRelativePath
"${config.packagesDir}/${rootProject.name}/${project.relPath or ""}/dream-lock.json";
applyProjectSettings = projects: settingsList: let
settingsListForProject = project:
l.filter
(settings:
if ! settings ? filter
then true
else settings.filter project)
settingsList;
applySettings = project: settings:
l.recursiveUpdate project settings;
applyAllSettings = project:
l.foldl'
(proj: settings: applySettings proj settings)
project
(settingsListForProject project);
settingsApplied =
l.forEach projects
(proj: applyAllSettings proj);
in
settingsApplied;
# TODO
validator = module: true;
modules = dlib.modules.makeSubsystemModules {
modulesCategory = "discoverers";
inherit validator;
};
in {
inherit
applyProjectSettings
discoverProjects
;
discoverers = modules.modules;
callDiscoverer = modules.callModule;
mapDiscoverers = modules.mapModules;
}

View File

@ -1,40 +0,0 @@
{
dlib,
lib,
}: let
l = lib // builtins;
# TODO:
validator = module: true;
callFetcher = file: extraArgs:
dlib.modules.importModule {inherit file extraArgs validator;};
# get information for builtin fetchers
fetchersDir = ../fetchers;
fetcherNames = dlib.dirNames fetchersDir;
# get the builtin fetchers
builtinFetchers =
l.map
(name: {
file = "${fetchersDir}/${name}";
extraArgs = {inherit name;};
})
fetcherNames;
# get extra fetchers
extraFetchers = dlib.modules.extra.fetchers or [];
# import fetchers
importedFetchers =
l.map
(
module:
(callFetcher module.file module.extraArgs)
// {inherit (module.extraArgs) name;}
)
(builtinFetchers ++ extraFetchers);
# create the attrset
fetchers = l.listToAttrs (l.map (f: l.nameValuePair f.name f) importedFetchers);
mapFetchers = f: l.mapAttrs (_: fetcher: f fetcher) fetchers;
in {inherit fetchers callFetcher mapFetchers;}

View File

@ -1,197 +0,0 @@
{
config,
dlib,
lib,
}: let
l = lib // builtins;
configFuncMsg = ''
consider passing a path to a file instead of a function:
functions can't be encoded to JSON, and as such most features of
dream2nix won't work because of this since they require passing around
the config as JSON.
'';
# imports a module.
#
# - 'file' can be a function or a path to a function.
# - 'dlib', 'lib', 'config' and attributes in
# 'extraArgs' are passed to the function.
importModule = {
file,
validator ? _: true,
extraArgs ? {},
}: let
_module =
if l.isFunction file
then file
else import file;
module =
if l.isFunction _module
then _module ({inherit config dlib lib;} // extraArgs)
else throw "module file (${file}) must return a function that takes an attrset";
in
l.seq (validator module) module;
# collects extra modules from a list
# ex: [{translators = [module1];} {translators = [module2];}] -> {translators = [module1 module2];}
collectExtraModules =
l.foldl'
(acc: el:
acc
// (
l.mapAttrs
(category: modules: modules ++ (acc.${category} or []))
el
))
{};
# processes one extra (config.extra)
# returns extra modules like {fetchers = [...]; translators = [...];}
processOneExtra = _extra: let
isAttrs = val: l.isAttrs val && (! val ? drvPath);
# imports a modules declaration
importDecl = decl:
if l.isFunction decl
then l.throw configFuncMsg
else if isAttrs decl
then decl
else import decl {inherit config dlib lib;};
# extra attrset itself
# config.extra is imported here if it's a path
extra = importDecl _extra;
# throw if declaring a module as a function
throwIfModuleNotPath = module:
l.throwIf ((isAttrs _extra) && (l.isFunction module)) configFuncMsg module;
# collect subsystem modules (translators, discoverers, builders)
_extraSubsystemModules =
l.mapAttrsToList
(subsystem: categories:
l.mapAttrs
(category: modules:
l.mapAttrsToList
(name: module: {
file = throwIfModuleNotPath module;
extraArgs = {inherit subsystem name;};
})
(importDecl modules))
categories)
(importDecl (extra.subsystems or {}));
extraSubsystemModules =
collectExtraModules (l.flatten _extraSubsystemModules);
# collect fetcher modules
extraFetcherModules =
l.mapAttrsToList
(name: fetcher: {
file = throwIfModuleNotPath fetcher;
extraArgs = {inherit name;};
})
(importDecl (extra.fetchers or {}));
in
extraSubsystemModules
// {
fetchers = extraFetcherModules;
};
# collect all extras
extra =
collectExtraModules
(l.map processOneExtra (l.flatten [(config.extra or [])]));
mapSubsystemModules = f: modules:
l.mapAttrs
(
subsystem: names:
l.mapAttrs
(name: module: f module)
names
)
modules;
# collect subsystem modules into a list
# ex: {rust.cargo-lock = cargo-lock; go.gomod2nix = gomod2nix;} -> [cargo-lock gomod2nix]
collectSubsystemModules = modules: let
allModules = l.flatten (l.map l.attrValues (l.attrValues modules));
hasModule = module: modules:
l.any
(
omodule:
(omodule.name == module.name)
&& (omodule.subsystem == module.subsystem)
)
modules;
in
l.foldl'
(acc: el:
if hasModule el acc
then acc
else acc ++ [el])
[]
allModules;
# create subsystem modules
# returns ex: {rust = {moduleName = module;}; go = {moduleName = module;};}
makeSubsystemModules = {
modulesCategory,
validator,
extraModules ? extra.${modulesCategory} or [],
defaults ? {},
}: let
callModule = file: extraArgs:
importModule {inherit file validator extraArgs;};
# extract builtin modules from subsystems directory
modulesBuiltin = l.flatten (
l.map
(
subsystem: let
dir = "${toString ../subsystems}/${subsystem}/${modulesCategory}";
moduleNames =
if l.pathExists dir
then dlib.dirNames dir
else [];
makeModule = name: {
file = "${dir}/${name}";
extraArgs = {inherit subsystem name;};
};
in
l.map makeModule moduleNames
)
dlib.subsystems
);
# import the modules
importedModules = l.filter (t: t.disabled or false == false) (
l.map
(
module:
(callModule module.file module.extraArgs)
// {inherit (module.extraArgs) subsystem name;}
)
(modulesBuiltin ++ extraModules)
);
# create the modules attrset
_modules =
l.foldl'
(acc: el: l.recursiveUpdate acc {"${el.subsystem}"."${el.name}" = el;})
{}
importedModules;
# add default module attribute to a subsystem if declared in `defaults`
modules =
l.mapAttrs
(
subsystem: modules:
if l.hasAttr subsystem defaults
then modules // {default = modules.${defaults.${subsystem}};}
else modules
)
_modules;
mapModules = f: mapSubsystemModules f modules;
in {inherit callModule mapModules modules;};
in {
inherit
importModule
makeSubsystemModules
collectSubsystemModules
mapSubsystemModules
extra
;
}

View File

@ -1,27 +0,0 @@
{
dlib,
lib,
}: let
l = lib // builtins;
# TODO
validator = module: true;
modules = dlib.modules.makeSubsystemModules {
inherit validator;
modulesCategory = "translators";
};
translators =
dlib.modules.mapSubsystemModules
(t:
t
// (lib.optionalAttrs (t.translate or null != null) {
translate = l.trace t dlib.warnIfIfd t t.translate;
}))
modules.modules;
mapTranslators = f: dlib.modules.mapSubsystemModules f translators;
in {
inherit translators mapTranslators;
callTranslator = modules.callModule;
}

View File

@ -1,9 +1,5 @@
{
config,
callPackageDream,
lib,
...
}: let
{config, ...}: let
lib = config.lib;
defaults = {
rust = "build-rust-package";
nodejs = "granular-nodejs";
@ -13,20 +9,13 @@
debian = "simple-debian";
racket = "simple-racket";
};
loader = b: b // {build = callPackageDream b.build {};};
funcs = config.functions.subsystem-loading;
funcs = import ../subsystem-loading.nix config;
collectedModules = funcs.collect "builders";
in {
config = {
# The user can add more translators by extending this attribute
builders = funcs.import_ collectedModules;
/*
translators wrapped with extra logic to add extra attributes,
like .translateBin for pure translators
*/
builderInstances = funcs.instantiate config.builders loader;
buildersBySubsystem =
lib.mapAttrs
(
@ -36,6 +25,6 @@ in {
default = builders.${defaults.${subsystem}};
}
)
(funcs.structureBySubsystem config.builderInstances);
(funcs.structureBySubsystem config.builders);
};
}

View File

@ -1,23 +1,17 @@
{
lib,
specialArgs,
...
}: let
{config, ...}: let
lib = config.lib;
t = lib.types;
in {
options = {
builders = lib.mkOption {
type = t.attrsOf (t.submoduleWith {
modules = [./builder/default.nix];
inherit specialArgs;
modules = [../interfaces.builder];
specialArgs = {framework = config;};
});
description = ''
builder module definitions
'';
};
builderInstances = lib.mkOption {
type = t.attrsOf t.anything;
};
buildersBySubsystem = lib.mkOption {
type = t.attrsOf (t.attrsOf t.anything);
};

View File

@ -1,9 +1,5 @@
{
config,
lib,
...
}: let
l = lib // builtins;
{config, ...}: let
l = config.lib // builtins;
translatorsWithDiscoverFunc =
l.filter

View File

@ -1,11 +1,13 @@
{config, ...}: let
funcs = config.functions.subsystem-loading;
funcs = import ../subsystem-loading.nix config;
collectedModules = funcs.collect "discoverers";
in {
config = {
# The user can add more discoverers by extending this attribute
discoverers = funcs.import_ collectedModules;
discoverersBySubsystem = funcs.structureBySubsystem config.discoverers;
discoverersBySubsystem = funcs.structureBySubsystem (
# remove the "default" discoverer we create, as it's not subsystem specific.
builtins.removeAttrs config.discoverers ["default"]
);
};
}

View File

@ -1,15 +1,12 @@
{
lib,
specialArgs,
...
}: let
{config, ...}: let
lib = config.lib;
t = lib.types;
in {
options = {
discoverers = lib.mkOption {
type = t.attrsOf (t.submoduleWith {
modules = [../interfaces.discoverer];
inherit specialArgs;
specialArgs = {framework = config;};
});
description = ''
discoverer module definitions

View File

@ -1,10 +1,6 @@
{
callPackageDream,
dlib,
lib,
...
}: let
l = lib // builtins;
{config, ...}: let
l = config.lib;
fetchersDir = ../../fetchers;
fetcherNames = l.attrNames (
l.filterAttrs
@ -14,12 +10,7 @@
fetcherModules =
l.genAttrs
fetcherNames
(
name:
import "${fetchersDir}/${name}" {
inherit dlib lib;
}
);
(name: import "${fetchersDir}/${name}" config);
in {
config = {
fetchers = fetcherModules;

View File

@ -1,9 +1,5 @@
{
lib,
specialArgs,
...
}: let
l = lib // builtins;
{config, ...}: let
l = config.lib // builtins;
t = l.types;
in {
options = {
@ -11,7 +7,7 @@ in {
type = t.attrsOf (
t.submoduleWith {
modules = [../interfaces.fetcher];
inherit specialArgs;
specialArgs = {framework = config;};
}
);
};

View File

@ -1,24 +1,11 @@
{
dream2nixConfig,
callPackageDream,
pkgs,
dlib,
lib,
utils,
}: let
dream2nixConfig,
...
} @ args: let
topLevel = import ./top-level.nix args;
evaledModules = lib.evalModules {
modules = [./top-level.nix] ++ (dream2nixConfig.modules or []);
# TODO: remove specialArgs once all functionality is moved to /src/modules
specialArgs = {
inherit
dream2nixConfig
callPackageDream
pkgs
dlib
utils
;
};
modules = [topLevel] ++ (dream2nixConfig.modules or []);
};
framework = evaledModules.config;

View File

@ -1,11 +1,5 @@
{
lib,
dlib,
utils,
config,
pkgs,
...
}: let
{config, ...}: let
inherit (config) lib pkgs utils;
defaultFetcher = config.functions.defaultFetcher;
inherit
(pkgs)

View File

@ -1,10 +1,10 @@
{lib, ...}: let
l = lib // builtins;
{config, ...}: let
l = config.lib // builtins;
t = l.types;
in {
options = {
functions.combinedFetcher = l.mkOption {
type = t.functionTo t.attrs;
type = t.uniq (t.functionTo t.attrs);
};
};
}

View File

@ -1,9 +1,6 @@
{
lib,
dlib,
config,
...
}: let
{config, ...}: let
inherit (config) lib;
fetchers = config.fetchers;
fetchSource = config.functions.fetchers.fetchSource;
func = {

View File

@ -1,10 +1,10 @@
{lib, ...}: let
l = lib // builtins;
{config, ...}: let
l = config.lib // builtins;
t = l.types;
in {
options = {
functions.defaultFetcher = l.mkOption {
type = t.functionTo t.attrs;
type = t.uniq (t.functionTo t.attrs);
};
};
}

View File

@ -1,11 +1,6 @@
{
config,
dream2nixConfig,
dlib,
lib,
...
}: let
l = lib // builtins;
{config, ...}: let
l = config.lib // builtins;
dlib = config.dlib;
discoverProjects = {
projects,
@ -47,7 +42,7 @@
getDreamLockPath = project: rootProject:
dlib.sanitizeRelativePath
"${dream2nixConfig.packagesDir}/${rootProject.name}/${project.relPath or ""}/dream-lock.json";
"${config.dream2nixConfig.packagesDir}/${rootProject.name}/${project.relPath or ""}/dream-lock.json";
applyProjectSettings = projects: settingsList: let
settingsListForProject = project:

View File

@ -1,16 +1,16 @@
{lib, ...}: let
l = lib // builtins;
{config, ...}: let
l = config.lib // builtins;
t = l.types;
in {
options.functions.discoverers = {
discoverProjects = l.mkOption {
type = t.functionTo (t.listOf t.attrs);
type = t.uniq (t.functionTo (t.listOf t.attrs));
};
applyProjectSettings = l.mkOption {
type = t.functionTo (t.functionTo (t.listOf t.attrs));
type = t.uniq (t.functionTo (t.functionTo (t.listOf t.attrs)));
};
getDreamLockPath = l.mkOption {
type = t.functionTo (t.functionTo t.path);
type = t.uniq (t.functionTo (t.functionTo t.path));
};
};
}

View File

@ -1,10 +1,6 @@
{
config,
lib,
...
}: let
{config, ...}: let
inherit (config) fetchers lib;
b = builtins;
fetchers = config.fetchers;
renderUrlArgs = kwargs: let
asStr =
@ -28,7 +24,7 @@ in {
fetcher = fetchers."${type}";
argsKeep = b.removeAttrs args ["reComputeHash"];
fetcherOutputs =
fetcher.outputsInstanced
fetcher.outputs
(b.removeAttrs argsKeep ["dir" "hash" "type"]);
in
argsKeep
@ -58,7 +54,7 @@ in {
}: let
fetcher = fetchers."${source.type}";
fetcherArgs = b.removeAttrs source ["dir" "hash" "type"];
fetcherOutputs = fetcher.outputsInstanced fetcherArgs;
fetcherOutputs = fetcher.outputs fetcherArgs;
maybeArchive = fetcherOutputs.fetched (source.hash or null);
in
if source ? dir
@ -136,7 +132,7 @@ in {
url = with parsed; "${proto2}://${path}${urlArgsFinal}";
fetcherOutputs = fetcher.outputsInstanced {
fetcherOutputs = fetcher.outputs {
inherit url;
};
in
@ -165,7 +161,7 @@ in {
args = parsed.kwargs // {inherit url;};
fetcherOutputs = fetcher.outputsInstanced (checkArgs fetcherName args);
fetcherOutputs = fetcher.outputs (checkArgs fetcherName args);
in
constructSource
(parsed.kwargs
@ -190,7 +186,7 @@ in {
fetcher = fetchers."${fetcherName}";
args =
if fetcher ? parseParams
if fetcher.parseParams != null
then fetcher.parseParams params
else if b.length params != b.length fetcher.inputs
then
@ -209,7 +205,7 @@ in {
(lib.elemAt params idx)
));
fetcherOutputs = fetcher.outputsInstanced (args // parsed.kwargs);
fetcherOutputs = fetcher.outputs (args // parsed.kwargs);
in
constructSource (args
// parsed.kwargs

View File

@ -1,35 +1,35 @@
{lib, ...}: let
l = lib // builtins;
{config, ...}: let
l = config.lib // builtins;
t = l.types;
in {
options = {
functions.fetchers = {
constructSource = l.mkOption {
type = t.functionTo t.attrs;
type = t.uniq (t.functionTo t.attrs);
};
updateSource = l.mkOption {
type = t.functionTo t.attrs;
type = t.uniq (t.functionTo t.attrs);
description = ''
update source spec to different version
'';
};
fetchSource = l.mkOption {
type = t.functionTo t.path;
type = t.uniq (t.functionTo t.path);
description = ''
fetch a source defined via a dream lock source spec
'';
};
fetchShortcut = l.mkOption {
type = t.functionTo t.path;
type = t.uniq (t.functionTo t.path);
description = ''
fetch a source defined by a shortcut
'';
};
parseShortcut = l.mkOption {
type = t.functionTo t.attrs;
type = t.uniq (t.functionTo t.attrs);
};
translateShortcut = l.mkOption {
type = t.functionTo t.attrs;
type = t.uniq (t.functionTo t.attrs);
description = ''
translate shortcut to dream lock source spec
'';

View File

@ -1,35 +0,0 @@
{config, ...}: let
lib = config.lib;
t = lib.types;
in {
options.functions.subsystem-loading = {
collect = lib.mkOption {
type = t.functionTo t.anything;
description = ''
Discover modules in /src/subsystems/{subsystem}/{module-type}/{module-name}
'';
};
import_ = lib.mkOption {
type = t.functionTo t.anything;
description = ''
Imports discovered module files.
Adds name and subsystem attributes to each module derived from the path.
'';
};
instantiate = lib.mkOption {
type = t.functionTo t.anything;
description = ''
To keep module implementations simpler, additional generic logic is added
by a loader.
The loader is subsytem specific and needs to be passed as an argument.
'';
};
structureBySubsystem = lib.mkOption {
type = t.functionTo t.anything;
description = ''
re-structures the instantiated instances into a deeper attrset like:
{subsytem}.{module-name} = ...
'';
};
};
}

View File

@ -11,10 +11,67 @@
else def.default or null
)
extraArgsDef;
# adds a translateBin to a pure translator
wrapPureTranslator = {
subsystem,
name,
}: let
inherit
(config)
utils
pkgs
dream2nixWithExternals
dream2nixConfigFile
;
bin =
utils.writePureShellScript
(with pkgs; [
coreutils
jq
nix
python3
])
''
jsonInputFile=$(realpath $1)
outputFile=$(realpath -m $(jq '.outputFile' -c -r $jsonInputFile))
pushd $TMPDIR
nix eval \
--option experimental-features "nix-command flakes"\
--show-trace --impure --raw --expr "
let
dream2nix = import ${dream2nixWithExternals} {
config = ${dream2nixConfigFile};
};
translatorArgs =
(builtins.fromJSON
(builtins.unsafeDiscardStringContext (builtins.readFile '''$1''')));
dreamLock' =
dream2nix.framework.translatorsBySubsystem.${subsystem}.${name}.finalTranslate
translatorArgs;
# simpleTranslate2 puts dream-lock in result
dreamLock = dreamLock'.result or dreamLock';
in
dream2nix.utils.dreamLock.toJSON
# don't use nix to detect cycles, this will be more efficient in python
(dreamLock // {
_generic = builtins.removeAttrs dreamLock._generic [ \"cyclicDependencies\" ];
})
" | python3 ${../../apps/cli/format-dream-lock.py} > out
tmpOut=$(realpath out)
popd
mkdir -p $(dirname $outputFile)
cp $tmpOut $outputFile
'';
in
bin.overrideAttrs (old: {
name = "translator-${subsystem}-pure-${name}";
});
in {
functions.translators = {
inherit
makeTranslatorDefaultArgs
wrapPureTranslator
;
};
}

View File

@ -3,6 +3,11 @@
t = lib.types;
in {
options.functions.translators = {
makeTranslatorDefaultArgs = lib.mkOption {type = t.functionTo t.anything;};
makeTranslatorDefaultArgs = lib.mkOption {
type = t.uniq (t.functionTo t.attrs);
};
wrapPureTranslator = lib.mkOption {
type = t.uniq (t.functionTo t.package);
};
};
}

View File

@ -1,8 +1,5 @@
{
config,
lib,
...
}: let
{framework, ...}: let
lib = framework.lib;
t = lib.types;
in {
options = {
@ -20,8 +17,8 @@ in {
description = "Subsystem of the builder.";
};
build = lib.mkOption {
type = t.functionTo (t.functionTo t.attrs);
default = _: _: {};
type = t.uniq (t.functionTo t.attrs);
default = _: {};
};
type = lib.mkOption {
type = t.enum [

View File

@ -1,8 +1,5 @@
{
config,
lib,
...
}: let
{framework, ...}: let
lib = framework.lib;
t = lib.types;
in {
options = {
@ -20,7 +17,7 @@ in {
description = "Subsystem of the discoverer.";
};
discover = lib.mkOption {
type = t.functionTo (t.listOf t.attrs);
type = t.uniq (t.functionTo (t.listOf t.attrs));
default = _: {};
};
};

View File

@ -1,6 +1,5 @@
{
imports = [
./interface.nix
./implementation.nix
];
}

View File

@ -1,9 +0,0 @@
{
config,
callPackageDream,
...
}: {
config = {
outputsInstanced = callPackageDream config.outputs {};
};
}

View File

@ -1,5 +1,5 @@
{lib, ...}: let
l = lib // builtins;
{framework, ...}: let
l = framework.lib // builtins;
t = l.types;
outputsOptions = {
@ -41,22 +41,15 @@ in {
'';
};
parseParams = l.mkOption {
type = t.functionTo t.attrs;
type = t.nullOr (t.uniq (t.functionTo t.attrs));
default = null;
};
outputs = l.mkOption {
type = t.functionTo (t.functionTo (
type = t.uniq (t.functionTo (
t.submoduleWith {
modules = [outputsOptions];
}
));
};
outputsInstanced = l.mkOption {
type = t.functionTo (
t.submoduleWith {
modules = [outputsOptions];
}
);
readOnly = true;
};
};
}

View File

@ -1,6 +1,6 @@
{
imports = [
./implementation.nix
./interface.nix
./implementation.nix
];
}

View File

@ -0,0 +1,31 @@
{
config,
framework,
...
}: {
config = {
finalTranslate = args:
if config.translate != null
then
config.translate
(
(framework.functions.translators.makeTranslatorDefaultArgs
(config.extraArgs or {}))
// args
// (args.project.subsystemInfo or {})
// {
tree =
args.tree or (framework.dlib.prepareSourceTree {inherit (args) source;});
}
)
else null;
finalTranslateBin =
if config.translate != null
then
framework.functions.translators.wrapPureTranslator
{inherit (config) subsystem name;}
else if config.translateBin != null
then config.translateBin
else null;
};
}

View File

@ -1,8 +1,5 @@
{
config,
lib,
...
}: let
{framework, ...}: let
lib = framework.lib;
t = lib.types;
in {
options = {
@ -11,7 +8,7 @@ in {
default = false;
};
discoverProject = lib.mkOption {
type = t.nullOr (t.functionTo (t.anything));
type = t.nullOr (t.uniq (t.functionTo (t.anything)));
default = null;
};
extraArgs = lib.mkOption {
@ -46,13 +43,27 @@ in {
type = t.str;
};
translate = lib.mkOption {
type = t.nullOr (t.functionTo (t.functionTo (t.attrs)));
type = t.nullOr (t.uniq (t.functionTo t.attrs));
default = null;
};
finalTranslate = lib.mkOption {
type = t.nullOr (t.uniq (t.functionTo t.attrs));
readOnly = true;
description = ''
the final translate that you should use.
'';
};
translateBin = lib.mkOption {
type = t.nullOr (t.functionTo t.package);
type = t.nullOr t.package;
default = null;
};
finalTranslateBin = lib.mkOption {
type = t.nullOr t.package;
readOnly = true;
description = ''
the final translateBin that you should use.
'';
};
type = lib.mkOption {
type = t.enum [
"ifd"

View File

@ -5,16 +5,12 @@ translators and builders.
This recurses through the following directory structure to discover and load
modules:
/src/subsystems/{subsystem}/{module-type}/{module-name}
This is not included in `config.functions` because it causes infinite recursion.
*/
{
config,
dlib,
callPackageDream,
...
}: let
lib = config.lib;
t = lib.types;
subsystemsDir = lib.toString ../../subsystems;
config: let
inherit (config) dlib lib;
subsystemsDir = lib.toString ../subsystems;
subsystems = dlib.dirNames subsystemsDir;
collect = moduleDirName:
@ -41,9 +37,12 @@ modules:
import_ = collectedModules:
lib.mapAttrs
(name: description:
(import description.path {inherit dlib lib;})
// {inherit (description) name subsystem;})
(
name: description: let
attrs = {inherit (description) name subsystem;};
in
(import description.path (config // attrs)) // attrs
)
(
lib.foldl'
(
@ -60,13 +59,6 @@ modules:
collectedModules
);
instantiate = importedModules: loader:
lib.mapAttrs
(name: module:
(loader module)
// {inherit (module) name subsystem;})
importedModules;
structureBySubsystem = instances:
lib.foldl
lib.recursiveUpdate
@ -75,15 +67,9 @@ modules:
(tName: t: {"${t.subsystem}"."${tName}" = t;})
instances);
in {
/*
Expose the functions via via the modules system.
*/
config.functions.subsystem-loading = {
inherit
collect
import_
instantiate
structureBySubsystem
;
};
inherit
collect
import_
structureBySubsystem
;
}

View File

@ -1,8 +1,15 @@
{
config,
apps,
dlib,
externals,
externalSources,
lib,
...
}: let
pkgs,
utils,
dream2nixConfig,
dream2nixConfigFile,
dream2nixWithExternals,
} @ args: let
t = lib.types;
in {
imports = [
@ -11,7 +18,6 @@ in {
./functions.default-fetcher
./functions.combined-fetcher
./functions.translators
./functions.subsystem-loading
./builders
./discoverers
./discoverers.default-discoverer
@ -19,11 +25,40 @@ in {
./translators
];
options = {
apps = lib.mkOption {
type = t.raw;
};
lib = lib.mkOption {
type = t.raw;
};
dlib = lib.mkOption {
type = t.raw;
};
externals = lib.mkOption {
type = t.raw;
};
externalSources = lib.mkOption {
type = t.raw;
};
pkgs = lib.mkOption {
type = t.raw;
};
utils = lib.mkOption {
type = t.raw;
};
dream2nixConfig = lib.mkOption {
type = t.raw;
};
dream2nixWithExternals = lib.mkOption {
type = t.path;
};
dream2nixConfigFile = lib.mkOption {
type = t.path;
};
};
config = {
lib = lib // builtins;
};
config =
args
// {
lib = args.lib // builtins;
};
}

View File

@ -1,25 +1,11 @@
{
config,
callPackageDream,
...
}: let
lib = config.lib;
t = lib.types;
# TODO: the makeTranslator logic should be moved somewhere to /src/modules
loader = (callPackageDream ../../subsystems/translators.nix {}).makeTranslator;
funcs = config.functions.subsystem-loading;
{config, ...}: let
funcs = import ../subsystem-loading.nix config;
collectedModules = funcs.collect "translators";
in {
config = {
# The user can add more translators by extending this attribute
translators = funcs.import_ collectedModules;
/*
translators wrapped with extra logic to add extra attributes,
like .translateBin for pure translators
*/
translatorInstances = funcs.instantiate config.translators loader;
translatorsBySubsystem = funcs.structureBySubsystem config.translatorInstances;
translatorsBySubsystem = funcs.structureBySubsystem config.translators;
};
}

View File

@ -1,24 +1,17 @@
{
config,
lib,
specialArgs,
...
}: let
{config, ...}: let
lib = config.lib;
t = lib.types;
in {
options = {
translators = lib.mkOption {
type = t.attrsOf (t.submoduleWith {
modules = [./interface-translator.nix];
inherit specialArgs;
modules = [../interfaces.translator];
specialArgs = {framework = config;};
});
description = ''
Translator module definitions
'';
};
translatorInstances = lib.mkOption {
type = t.attrsOf t.anything;
};
translatorsBySubsystem = lib.mkOption {
type = t.attrsOf (t.attrsOf t.anything);
};

View File

@ -1,18 +0,0 @@
{
dlib,
callPackageDream,
...
}: let
makeBuilder = builderModule:
builderModule
// {
build = callPackageDream builderModule.build {};
};
builders = dlib.builders.mapBuilders makeBuilder;
in {
inherit
makeBuilder
builders
;
}

View File

@ -1,14 +1,11 @@
{...}: {
{
lib,
pkgs,
...
}: {
type = "pure";
build = {
lib,
pkgs,
stdenv,
# dream2nix inputs
externals,
...
}: {
### FUNCTIONS
# AttrSet -> Bool) -> AttrSet -> [x]
getCyclicDependencies, # name: version: -> [ {name=; version=; } ]
@ -54,7 +51,7 @@
(src: src.original or src)
allDependencySources';
package = produceDerivation defaultPackageName (stdenv.mkDerivation {
package = produceDerivation defaultPackageName (pkgs.stdenv.mkDerivation {
name = defaultPackageName;
src = ":";
dontUnpack = true;

View File

@ -1,10 +1,8 @@
{
dlib,
lib,
pkgs,
utils,
...
}: let
l = lib // builtins;
in {
}: {
type = "impure";
# A derivation which outputs a single executable at `$out`.
@ -18,28 +16,16 @@ in {
# by the input parameter `outFile`.
# The output file must contain the dream lock data encoded as json.
# See /src/specifications/dream-lock-example.json
translateBin = {
# dream2nix utils
utils,
# nixpkgs dependencies
bash,
coreutils,
jq,
writeScriptBin,
nix,
callPackage,
python3,
...
}:
translateBin =
utils.writePureShellScript
[
(with pkgs; [
bash
coreutils
jq
nix
(callPackage ./aptdream {})
python3
]
])
''
# accroding to the spec, the translator reads the input from a json file
jsonInput=$1

View File

@ -1,36 +0,0 @@
{
lib,
dlib,
callPackageDream,
...
}: let
l = lib // builtins;
inherit (dlib.modules) collectSubsystemModules;
builders = callPackageDream ./builders.nix {};
translators = callPackageDream ./translators.nix {};
discoverers = callPackageDream ./discoverers.nix {};
# maps modules
# ex: {rust = <translators attrset>;} -> {rust.translators = <translators attrset>;}
mapModules = modules: name:
l.mapAttrs'
(
subsystem: modules:
l.nameValuePair subsystem {${name} = modules;}
)
modules;
modules = [
(mapModules builders.builders "builders")
(mapModules translators.translators "translators")
(mapModules discoverers.discoverers "discoverers")
];
allModules = l.foldl' l.recursiveUpdate {} modules;
in
allModules
// {
allTranslators = collectSubsystemModules translators.translators;
allBuilders = collectSubsystemModules builders.builders;
allDiscoverers = collectSubsystemModules discoverers.discoverers;
}

View File

@ -1 +0,0 @@
{dlib, ...}: dlib.discoverers

View File

@ -1,17 +1,15 @@
{lib, ...}: let
{
pkgs,
utils,
externals,
lib,
...
}: let
l = lib // builtins;
in {
type = "pure";
build = {
lib,
pkgs,
stdenv,
# dream2nix inputs
externals,
utils,
...
}: {
### FUNCTIONS
# AttrSet -> Bool) -> AttrSet -> [x]
getCyclicDependencies, # name: version: -> [ {name=; version=; } ]
@ -37,8 +35,6 @@ in {
produceDerivation,
...
} @ args: let
b = builtins;
compiler =
pkgs
.haskell

View File

@ -1,6 +1,7 @@
{
dlib,
lib,
name,
...
}: let
l = lib // builtins;
@ -37,7 +38,7 @@ in {
l.pathExists "${tree.fullPath}/dist-newstyle/cache/plan.json";
# translate from a given source and a project specification to a dream-lock.
translate = {translatorName, ...}: {
translate = {
/*
A list of projects returned by `discoverProjects`
Example:
@ -189,7 +190,7 @@ in {
in
dlib.simpleTranslate2.translate
({objectsByKey, ...}: rec {
inherit translatorName;
translatorName = name;
# relative path of the project within the source tree.
location = project.relPath;

View File

@ -1,10 +1,9 @@
{
dlib,
lib,
pkgs,
utils,
translators,
...
}: let
l = lib // builtins;
in {
}: {
type = "impure";
# A derivation which outputs a single executable at `$out`.
@ -18,27 +17,9 @@ in {
# by the input parameter `outFile`.
# The output file must contain the dream lock data encoded as json.
# See /src/specifications/dream-lock-example.json
translateBin = {
# dream2nix utils
subsystems,
utils,
# nixpkgs dependenies
bash,
coreutils,
curl,
gnutar,
gzip,
haskellPackages,
jq,
moreutils,
nix,
python3,
util-linux,
writeScriptBin,
...
}:
translateBin =
utils.writePureShellScript
[
(with pkgs; [
bash
coreutils
curl
@ -51,7 +32,7 @@ in {
nix
python3
util-linux
]
])
''
# accroding to the spec, the translator reads the input from a json file
jsonInput=$1
@ -106,7 +87,7 @@ in {
popd
# execute cabal-plan translator
${subsystems.haskell.translators.cabal-plan.translateBin} $TMPDIR/newJsonInput
${translators.cabal-plan.finalTranslateBin} $TMPDIR/newJsonInput
# finalize dream-lock. Add source and export default package
# set correct package version under `packages`
@ -125,22 +106,5 @@ in {
# - boolean flag (type = "flag")
# String arguments contain a default value and examples. Flags do not.
extraArgs = {
# Example: boolean option
# Flags always default to 'false' if not specified by the user
noDev = {
description = "Exclude dev dependencies";
type = "flag";
};
# Example: string option
theAnswer = {
default = "42";
description = "The Answer to the Ultimate Question of Life";
examples = [
"0"
"1234"
];
type = "argument";
};
};
}

View File

@ -1,6 +1,9 @@
{
dlib,
lib,
pkgs,
utils,
name,
...
}: let
l = lib // builtins;
@ -71,12 +74,7 @@ in {
(l.attrNames tree.files);
# translate from a given source and a project specification to a dream-lock.
translate = {
translatorName,
pkgs,
utils,
...
}: let
translate = let
stackLockUtils = import ./utils.nix {inherit dlib lib pkgs;};
all-cabal-hashes = let
all-cabal-hashes' = pkgs.runCommandLocal "all-cabal-hashes" {} ''
@ -190,8 +188,7 @@ in {
};
getDependencyNames = finalObj: objectsByName: let
cabal = with finalObj;
cabalData.${name}.${version};
cabal = cabalData.${finalObj.name}.${finalObj.version};
targetBuildDepends =
cabal.library.condTreeData.build-info.targetBuildDepends or [];
@ -231,7 +228,7 @@ in {
in
dlib.simpleTranslate2.translate
({objectsByKey, ...}: rec {
inherit translatorName;
translatorName = name;
# relative path of the project within the source tree.
location = project.relPath;
@ -309,7 +306,7 @@ in {
{
type = "http";
url = haskellUtils.getHackageUrl finalObj;
hash = with finalObj; "sha256:${all-cabal-hashes.${name}.${version}.SHA256}";
hash = "sha256:${all-cabal-hashes.${finalObj.name}.${finalObj.version}.SHA256}";
};
};

View File

@ -1,511 +1,515 @@
{...}: {
{
pkgs,
lib,
...
}: {
type = "pure";
build = {
jq,
lib,
makeWrapper,
mkShell,
pkgs,
python3,
runCommandLocal,
stdenv,
writeText,
...
}: {
# Funcs
# AttrSet -> Bool) -> AttrSet -> [x]
getCyclicDependencies, # name: version: -> [ {name=; version=; } ]
getDependencies, # name: version: -> [ {name=; version=; } ]
getSource, # name: version: -> store-path
# Attributes
subsystemAttrs, # attrset
defaultPackageName, # string
defaultPackageVersion, # string
packages, # list
# attrset of pname -> versions,
# where versions is a list of version strings
packageVersions,
# function which applies overrides to a package
# It must be applied by the builder to each individual derivation
# Example:
# produceDerivation name (mkDerivation {...})
produceDerivation,
nodejs ? null,
...
} @ args: let
b = builtins;
l = lib // builtins;
build = let
inherit
(pkgs)
jq
makeWrapper
mkShell
python3
runCommandLocal
stdenv
;
in
{
# Funcs
# AttrSet -> Bool) -> AttrSet -> [x]
getCyclicDependencies, # name: version: -> [ {name=; version=; } ]
getDependencies, # name: version: -> [ {name=; version=; } ]
getSource, # name: version: -> store-path
# Attributes
subsystemAttrs, # attrset
defaultPackageName, # string
defaultPackageVersion, # string
packages, # list
# attrset of pname -> versions,
# where versions is a list of version strings
packageVersions,
# function which applies overrides to a package
# It must be applied by the builder to each individual derivation
# Example:
# produceDerivation name (mkDerivation {...})
produceDerivation,
nodejs ? null,
...
} @ args: let
b = builtins;
l = lib // builtins;
nodejsVersion = subsystemAttrs.nodejsVersion;
nodejsVersion = subsystemAttrs.nodejsVersion;
isMainPackage = name: version:
(args.packages."${name}" or null) == version;
isMainPackage = name: version:
(args.packages."${name}" or null) == version;
nodejs =
if args ? nodejs
then b.toString args.nodejs
else
pkgs."nodejs-${nodejsVersion}_x"
or (throw "Could not find nodejs version '${nodejsVersion}' in pkgs");
nodejs =
if args ? nodejs
then b.toString args.nodejs
else
pkgs."nodejs-${nodejsVersion}_x"
or (throw "Could not find nodejs version '${nodejsVersion}' in pkgs");
nodeSources = runCommandLocal "node-sources" {} ''
tar --no-same-owner --no-same-permissions -xf ${nodejs.src}
mv node-* $out
'';
allPackages =
lib.mapAttrs
(name: versions:
lib.genAttrs
versions
(version:
makePackage name version))
packageVersions;
outputs = rec {
# select only the packages listed in dreamLock as main packages
packages =
b.foldl'
(ps: p: ps // p)
{}
(lib.mapAttrsToList
(name: version: {
"${name}"."${version}" = allPackages."${name}"."${version}";
})
args.packages);
devShells =
{default = devShells.${defaultPackageName};}
// (
l.mapAttrs
(name: version: allPackages.${name}.${version}.devShell)
args.packages
);
};
# This is only executed for electron based packages.
# Electron ships its own version of node, requiring a rebuild of native
# extensions.
# Theoretically this requires headers for the exact electron version in use,
# but we use the headers from nixpkgs' electron instead which might have a
# different minor version.
# Alternatively the headers can be specified via `electronHeaders`.
# Also a custom electron version can be specified via `electronPackage`
electron-rebuild = ''
# prepare node headers for electron
if [ -n "$electronPackage" ]; then
export electronDist="$electronPackage/lib/electron"
else
export electronDist="$nodeModules/$packageName/node_modules/electron/dist"
fi
local ver
ver="v$(cat $electronDist/version | tr -d '\n')"
mkdir $TMP/$ver
cp $electronHeaders $TMP/$ver/node-$ver-headers.tar.gz
# calc checksums
cd $TMP/$ver
sha256sum ./* > SHASUMS256.txt
cd -
# serve headers via http
python -m http.server 45034 --directory $TMP &
# copy electron distribution
cp -r $electronDist $TMP/electron
chmod -R +w $TMP/electron
# configure electron toolchain
${pkgs.jq}/bin/jq ".build.electronDist = \"$TMP/electron\"" package.json \
| ${pkgs.moreutils}/bin/sponge package.json
${pkgs.jq}/bin/jq ".build.linux.target = \"dir\"" package.json \
| ${pkgs.moreutils}/bin/sponge package.json
${pkgs.jq}/bin/jq ".build.npmRebuild = false" package.json \
| ${pkgs.moreutils}/bin/sponge package.json
# execute electron-rebuild if available
export headers=http://localhost:45034/
if command -v electron-rebuild &> /dev/null; then
pushd $electronAppDir
electron-rebuild -d $headers
popd
fi
'';
# Only executed for electron based packages.
# Creates an executable script under /bin starting the electron app
electron-wrap =
if pkgs.stdenv.isLinux
then ''
mkdir -p $out/bin
makeWrapper \
$electronDist/electron \
$out/bin/$(basename "$packageName") \
--add-flags "$(realpath $electronAppDir)"
''
else ''
mkdir -p $out/bin
makeWrapper \
$electronDist/Electron.app/Contents/MacOS/Electron \
$out/bin/$(basename "$packageName") \
--add-flags "$(realpath $electronAppDir)"
nodeSources = runCommandLocal "node-sources" {} ''
tar --no-same-owner --no-same-permissions -xf ${nodejs.src}
mv node-* $out
'';
# Generates a derivation for a specific package name + version
makePackage = name: version: let
pname = lib.replaceStrings ["@" "/"] ["__at__" "__slash__"] name;
allPackages =
lib.mapAttrs
(name: versions:
lib.genAttrs
versions
(version:
makePackage name version))
packageVersions;
deps = getDependencies name version;
outputs = rec {
# select only the packages listed in dreamLock as main packages
packages =
b.foldl'
(ps: p: ps // p)
{}
(lib.mapAttrsToList
(name: version: {
"${name}"."${version}" = allPackages."${name}"."${version}";
})
args.packages);
nodeDeps =
lib.forEach
deps
(dep: allPackages."${dep.name}"."${dep.version}");
devShells =
{default = devShells.${defaultPackageName};}
// (
l.mapAttrs
(name: version: allPackages.${name}.${version}.devShell)
args.packages
);
};
passthruDeps =
l.listToAttrs
(l.forEach deps
(dep:
l.nameValuePair
dep.name
allPackages."${dep.name}"."${dep.version}"));
dependenciesJson =
b.toJSON
(lib.listToAttrs
(b.map
(dep: lib.nameValuePair dep.name dep.version)
deps));
electronDep =
if ! isMainPackage name version
then null
# This is only executed for electron based packages.
# Electron ships its own version of node, requiring a rebuild of native
# extensions.
# Theoretically this requires headers for the exact electron version in use,
# but we use the headers from nixpkgs' electron instead which might have a
# different minor version.
# Alternatively the headers can be specified via `electronHeaders`.
# Also a custom electron version can be specified via `electronPackage`
electron-rebuild = ''
# prepare node headers for electron
if [ -n "$electronPackage" ]; then
export electronDist="$electronPackage/lib/electron"
else
lib.findFirst
(dep: dep.name == "electron")
null
deps;
export electronDist="$nodeModules/$packageName/node_modules/electron/dist"
fi
local ver
ver="v$(cat $electronDist/version | tr -d '\n')"
mkdir $TMP/$ver
cp $electronHeaders $TMP/$ver/node-$ver-headers.tar.gz
electronVersionMajor =
lib.versions.major electronDep.version;
# calc checksums
cd $TMP/$ver
sha256sum ./* > SHASUMS256.txt
cd -
electronHeaders =
if
(electronDep == null)
# hashes seem unavailable for electron < 4
|| ((l.toInt electronVersionMajor) <= 2)
then null
else pkgs."electron_${electronVersionMajor}".headers;
# serve headers via http
python -m http.server 45034 --directory $TMP &
pkg = produceDerivation name (stdenv.mkDerivation rec {
inherit
dependenciesJson
electronHeaders
nodeDeps
nodeSources
version
;
# copy electron distribution
cp -r $electronDist $TMP/electron
chmod -R +w $TMP/electron
packageName = name;
# configure electron toolchain
${pkgs.jq}/bin/jq ".build.electronDist = \"$TMP/electron\"" package.json \
| ${pkgs.moreutils}/bin/sponge package.json
inherit pname;
${pkgs.jq}/bin/jq ".build.linux.target = \"dir\"" package.json \
| ${pkgs.moreutils}/bin/sponge package.json
meta = let
meta = subsystemAttrs.meta;
in
meta
// {
license = l.map (name: l.licenses.${name}) meta.license;
${pkgs.jq}/bin/jq ".build.npmRebuild = false" package.json \
| ${pkgs.moreutils}/bin/sponge package.json
# execute electron-rebuild if available
export headers=http://localhost:45034/
if command -v electron-rebuild &> /dev/null; then
pushd $electronAppDir
electron-rebuild -d $headers
popd
fi
'';
# Only executed for electron based packages.
# Creates an executable script under /bin starting the electron app
electron-wrap =
if pkgs.stdenv.isLinux
then ''
mkdir -p $out/bin
makeWrapper \
$electronDist/electron \
$out/bin/$(basename "$packageName") \
--add-flags "$(realpath $electronAppDir)"
''
else ''
mkdir -p $out/bin
makeWrapper \
$electronDist/Electron.app/Contents/MacOS/Electron \
$out/bin/$(basename "$packageName") \
--add-flags "$(realpath $electronAppDir)"
'';
# Generates a derivation for a specific package name + version
makePackage = name: version: let
pname = lib.replaceStrings ["@" "/"] ["__at__" "__slash__"] name;
deps = getDependencies name version;
nodeDeps =
lib.forEach
deps
(dep: allPackages."${dep.name}"."${dep.version}");
passthruDeps =
l.listToAttrs
(l.forEach deps
(dep:
l.nameValuePair
dep.name
allPackages."${dep.name}"."${dep.version}"));
dependenciesJson =
b.toJSON
(lib.listToAttrs
(b.map
(dep: lib.nameValuePair dep.name dep.version)
deps));
electronDep =
if ! isMainPackage name version
then null
else
lib.findFirst
(dep: dep.name == "electron")
null
deps;
electronVersionMajor =
lib.versions.major electronDep.version;
electronHeaders =
if
(electronDep == null)
# hashes seem unavailable for electron < 4
|| ((l.toInt electronVersionMajor) <= 2)
then null
else pkgs."electron_${electronVersionMajor}".headers;
pkg = produceDerivation name (stdenv.mkDerivation rec {
inherit
dependenciesJson
electronHeaders
nodeDeps
nodeSources
version
;
packageName = name;
inherit pname;
meta = let
meta = subsystemAttrs.meta;
in
meta
// {
license = l.map (name: l.licenses.${name}) meta.license;
};
passthru.dependencies = passthruDeps;
passthru.devShell = import ./devShell.nix {
inherit
mkShell
nodejs
packageName
pkg
;
};
passthru.dependencies = passthruDeps;
/*
For top-level packages install dependencies as full copies, as this
reduces errors with build tooling that doesn't cope well with
symlinking.
*/
installMethod =
if isMainPackage name version
then "copy"
else "symlink";
passthru.devShell = import ./devShell.nix {
inherit
mkShell
nodejs
packageName
pkg
;
};
electronAppDir = ".";
/*
For top-level packages install dependencies as full copies, as this
reduces errors with build tooling that doesn't cope well with
symlinking.
*/
installMethod =
if isMainPackage name version
then "copy"
else "symlink";
# only run build on the main package
runBuild = isMainPackage name version;
electronAppDir = ".";
src = getSource name version;
# only run build on the main package
runBuild = isMainPackage name version;
nativeBuildInputs = [makeWrapper];
src = getSource name version;
buildInputs = [jq nodejs python3];
nativeBuildInputs = [makeWrapper];
# prevents running into ulimits
passAsFile = ["dependenciesJson" "nodeDeps"];
buildInputs = [jq nodejs python3];
preConfigurePhases = ["d2nLoadFuncsPhase" "d2nPatchPhase"];
# prevents running into ulimits
passAsFile = ["dependenciesJson" "nodeDeps"];
# can be overridden to define alternative install command
# (defaults to 'npm run postinstall')
buildScript = null;
preConfigurePhases = ["d2nLoadFuncsPhase" "d2nPatchPhase"];
# python script to modify some metadata to support installation
# (see comments below on d2nPatchPhase)
fixPackage = "${./fix-package.py}";
# can be overridden to define alternative install command
# (defaults to 'npm run postinstall')
buildScript = null;
# script to install (symlink or copy) dependencies.
installDeps = "${./install-deps.py}";
# python script to modify some metadata to support installation
# (see comments below on d2nPatchPhase)
fixPackage = "${./fix-package.py}";
# python script to link bin entries from package.json
linkBins = "${./link-bins.py}";
# script to install (symlink or copy) dependencies.
installDeps = "${./install-deps.py}";
# costs performance and doesn't seem beneficial in most scenarios
dontStrip = true;
# python script to link bin entries from package.json
linkBins = "${./link-bins.py}";
# declare some useful shell functions
d2nLoadFuncsPhase = ''
# function to resolve symlinks to copies
symlinksToCopies() {
local dir="$1"
# costs performance and doesn't seem beneficial in most scenarios
dontStrip = true;
# declare some useful shell functions
d2nLoadFuncsPhase = ''
# function to resolve symlinks to copies
symlinksToCopies() {
local dir="$1"
echo "transforming symlinks to copies..."
for f in $(find -L "$dir" -xtype l); do
if [ -f $f ]; then
continue
fi
echo "copying $f"
chmod +wx $(dirname "$f")
mv "$f" "$f.bak"
mkdir "$f"
if [ -n "$(ls -A "$f.bak/")" ]; then
cp -r "$f.bak"/* "$f/"
chmod -R +w $f
fi
rm "$f.bak"
done
}
'';
# TODO: upstream fix to nixpkgs
# example which requires this:
# https://registry.npmjs.org/react-window-infinite-loader/-/react-window-infinite-loader-1.0.7.tgz
unpackCmd =
if lib.hasSuffix ".tgz" src
then "tar --delay-directory-restore -xf $src"
else null;
unpackPhase = ''
runHook preUnpack
nodeModules=$out/lib/node_modules
export sourceRoot="$nodeModules/$packageName"
# sometimes tarballs do not end with .tar.??
unpackFallback(){
local fn="$1"
tar xf "$fn"
}
unpackCmdHooks+=(unpackFallback)
unpackFile $src
# Make the base dir in which the target dependency resides in first
mkdir -p "$(dirname "$sourceRoot")"
# install source
if [ -f "$src" ]
then
# Figure out what directory has been unpacked
export packageDir="$(find . -maxdepth 1 -type d | tail -1)"
# Restore write permissions
find "$packageDir" -type d -exec chmod u+x {} \;
chmod -R u+w -- "$packageDir"
# Move the extracted tarball into the output folder
mv -- "$packageDir" "$sourceRoot"
elif [ -d "$src" ]
then
export strippedName="$(stripHash $src)"
# Restore write permissions
chmod -R u+w -- "$strippedName"
# Move the extracted directory into the output folder
mv -- "$strippedName" "$sourceRoot"
fi
runHook postUnpack
'';
# The python script wich is executed in this phase:
# - ensures that the package is compatible to the current system
# - ensures the main version in package.json matches the expected
# - pins dependency versions in package.json
# (some npm commands might otherwise trigger networking)
# - creates symlinks for executables declared in package.json
# Apart from that:
# - Any usage of 'link:' in package.json is replaced with 'file:'
# - If package-lock.json exists, it is deleted, as it might conflict
# with the parent package-lock.json.
d2nPatchPhase = ''
# delete package-lock.json as it can lead to conflicts
rm -f package-lock.json
# repair 'link:' -> 'file:'
mv $nodeModules/$packageName/package.json $nodeModules/$packageName/package.json.old
cat $nodeModules/$packageName/package.json.old | sed 's!link:!file\:!g' > $nodeModules/$packageName/package.json
rm $nodeModules/$packageName/package.json.old
# run python script (see comment above):
cp package.json package.json.bak
python $fixPackage \
|| \
# exit code 3 -> the package is incompatible to the current platform
# -> Let the build succeed, but don't create lib/node_packages
if [ "$?" == "3" ]; then
rm -r $out/*
echo "Not compatible with system $system" > $out/error
exit 0
else
exit 1
fi
# configure typescript
if [ -f ./tsconfig.json ] \
&& node -e 'require("typescript")' &>/dev/null; then
node ${./tsconfig-to-json.js}
${pkgs.jq}/bin/jq ".compilerOptions.preserveSymlinks = true" tsconfig.json \
| ${pkgs.moreutils}/bin/sponge tsconfig.json
fi
'';
# - installs dependencies into the node_modules directory
# - adds executables of direct node module dependencies to PATH
# - adds the current node module to NODE_PATH
# - sets HOME=$TMPDIR, as this is required by some npm scripts
# TODO: don't install dev dependencies. Load into NODE_PATH instead
configurePhase = ''
runHook preConfigure
# symlink sub dependencies as well as this imitates npm better
python $installDeps
echo "Symlinking transitive executables to $nodeModules/.bin"
for dep in ${l.toString nodeDeps}; do
binDir=$dep/lib/node_modules/.bin
if [ -e $binDir ]; then
for bin in $(ls $binDir/); do\
if [ ! -e $nodeModules/.bin ]; then
mkdir -p $nodeModules/.bin
echo "transforming symlinks to copies..."
for f in $(find -L "$dir" -xtype l); do
if [ -f $f ]; then
continue
fi
# symlink might have been already created by install-deps.py
# if installMethod=copy was selected
if [ ! -L $nodeModules/.bin/$bin ]; then
ln -s $binDir/$bin $nodeModules/.bin/$bin
else
echo "won't overwrite existing symlink $nodeModules/.bin/$bin. current target: $(readlink $nodeModules/.bin/$bin)"
echo "copying $f"
chmod +wx $(dirname "$f")
mv "$f" "$f.bak"
mkdir "$f"
if [ -n "$(ls -A "$f.bak/")" ]; then
cp -r "$f.bak"/* "$f/"
chmod -R +w $f
fi
rm "$f.bak"
done
}
'';
# TODO: upstream fix to nixpkgs
# example which requires this:
# https://registry.npmjs.org/react-window-infinite-loader/-/react-window-infinite-loader-1.0.7.tgz
unpackCmd =
if lib.hasSuffix ".tgz" src
then "tar --delay-directory-restore -xf $src"
else null;
unpackPhase = ''
runHook preUnpack
nodeModules=$out/lib/node_modules
export sourceRoot="$nodeModules/$packageName"
# sometimes tarballs do not end with .tar.??
unpackFallback(){
local fn="$1"
tar xf "$fn"
}
unpackCmdHooks+=(unpackFallback)
unpackFile $src
# Make the base dir in which the target dependency resides in first
mkdir -p "$(dirname "$sourceRoot")"
# install source
if [ -f "$src" ]
then
# Figure out what directory has been unpacked
export packageDir="$(find . -maxdepth 1 -type d | tail -1)"
# Restore write permissions
find "$packageDir" -type d -exec chmod u+x {} \;
chmod -R u+w -- "$packageDir"
# Move the extracted tarball into the output folder
mv -- "$packageDir" "$sourceRoot"
elif [ -d "$src" ]
then
export strippedName="$(stripHash $src)"
# Restore write permissions
chmod -R u+w -- "$strippedName"
# Move the extracted directory into the output folder
mv -- "$strippedName" "$sourceRoot"
fi
done
# add bin path entries collected by python script
export PATH="$PATH:$nodeModules/.bin"
runHook postUnpack
'';
# add dependencies to NODE_PATH
export NODE_PATH="$NODE_PATH:$nodeModules/$packageName/node_modules"
# The python script wich is executed in this phase:
# - ensures that the package is compatible to the current system
# - ensures the main version in package.json matches the expected
# - pins dependency versions in package.json
# (some npm commands might otherwise trigger networking)
# - creates symlinks for executables declared in package.json
# Apart from that:
# - Any usage of 'link:' in package.json is replaced with 'file:'
# - If package-lock.json exists, it is deleted, as it might conflict
# with the parent package-lock.json.
d2nPatchPhase = ''
# delete package-lock.json as it can lead to conflicts
rm -f package-lock.json
export HOME=$TMPDIR
# repair 'link:' -> 'file:'
mv $nodeModules/$packageName/package.json $nodeModules/$packageName/package.json.old
cat $nodeModules/$packageName/package.json.old | sed 's!link:!file\:!g' > $nodeModules/$packageName/package.json
rm $nodeModules/$packageName/package.json.old
runHook postConfigure
'';
# Runs the install command which defaults to 'npm run postinstall'.
# Allows using custom install command by overriding 'buildScript'.
buildPhase = ''
runHook preBuild
# execute electron-rebuild
if [ -n "$electronHeaders" ]; then
echo "executing electron-rebuild"
${electron-rebuild}
fi
# execute install command
if [ -n "$buildScript" ]; then
if [ -f "$buildScript" ]; then
$buildScript
# run python script (see comment above):
cp package.json package.json.bak
python $fixPackage \
|| \
# exit code 3 -> the package is incompatible to the current platform
# -> Let the build succeed, but don't create lib/node_packages
if [ "$?" == "3" ]; then
rm -r $out/*
echo "Not compatible with system $system" > $out/error
exit 0
else
eval "$buildScript"
exit 1
fi
# by default, only for top level packages, `npm run build` is executed
elif [ -n "$runBuild" ] && [ "$(jq '.scripts.build' ./package.json)" != "null" ]; then
npm run build
else
if [ "$(jq '.scripts.preinstall' ./package.json)" != "null" ]; then
npm --production --offline --nodedir=$nodeSources run preinstall
# configure typescript
if [ -f ./tsconfig.json ] \
&& node -e 'require("typescript")' &>/dev/null; then
node ${./tsconfig-to-json.js}
${pkgs.jq}/bin/jq ".compilerOptions.preserveSymlinks = true" tsconfig.json \
| ${pkgs.moreutils}/bin/sponge tsconfig.json
fi
if [ "$(jq '.scripts.install' ./package.json)" != "null" ]; then
npm --production --offline --nodedir=$nodeSources run install
fi
if [ "$(jq '.scripts.postinstall' ./package.json)" != "null" ]; then
npm --production --offline --nodedir=$nodeSources run postinstall
fi
fi
'';
runHook postBuild
'';
# - installs dependencies into the node_modules directory
# - adds executables of direct node module dependencies to PATH
# - adds the current node module to NODE_PATH
# - sets HOME=$TMPDIR, as this is required by some npm scripts
# TODO: don't install dev dependencies. Load into NODE_PATH instead
configurePhase = ''
runHook preConfigure
# Symlinks executables and manual pages to correct directories
installPhase = ''
runHook preInstall
# symlink sub dependencies as well as this imitates npm better
python $installDeps
echo "Symlinking bin entries from package.json"
python $linkBins
echo "Symlinking transitive executables to $nodeModules/.bin"
for dep in ${l.toString nodeDeps}; do
binDir=$dep/lib/node_modules/.bin
if [ -e $binDir ]; then
for bin in $(ls $binDir/); do\
if [ ! -e $nodeModules/.bin ]; then
mkdir -p $nodeModules/.bin
fi
echo "Symlinking manual pages"
if [ -d "$nodeModules/$packageName/man" ]
then
mkdir -p $out/share
for dir in "$nodeModules/$packageName/man/"*
do
mkdir -p $out/share/man/$(basename "$dir")
for page in "$dir"/*
do
ln -s $page $out/share/man/$(basename "$dir")
done
# symlink might have been already created by install-deps.py
# if installMethod=copy was selected
if [ ! -L $nodeModules/.bin/$bin ]; then
ln -s $binDir/$bin $nodeModules/.bin/$bin
else
echo "won't overwrite existing symlink $nodeModules/.bin/$bin. current target: $(readlink $nodeModules/.bin/$bin)"
fi
done
fi
done
fi
# wrap electron app
if [ -n "$electronHeaders" ]; then
echo "Wrapping electron app"
${electron-wrap}
fi
# add bin path entries collected by python script
export PATH="$PATH:$nodeModules/.bin"
runHook postInstall
'';
});
# add dependencies to NODE_PATH
export NODE_PATH="$NODE_PATH:$nodeModules/$packageName/node_modules"
export HOME=$TMPDIR
runHook postConfigure
'';
# Runs the install command which defaults to 'npm run postinstall'.
# Allows using custom install command by overriding 'buildScript'.
buildPhase = ''
runHook preBuild
# execute electron-rebuild
if [ -n "$electronHeaders" ]; then
echo "executing electron-rebuild"
${electron-rebuild}
fi
# execute install command
if [ -n "$buildScript" ]; then
if [ -f "$buildScript" ]; then
$buildScript
else
eval "$buildScript"
fi
# by default, only for top level packages, `npm run build` is executed
elif [ -n "$runBuild" ] && [ "$(jq '.scripts.build' ./package.json)" != "null" ]; then
npm run build
else
if [ "$(jq '.scripts.preinstall' ./package.json)" != "null" ]; then
npm --production --offline --nodedir=$nodeSources run preinstall
fi
if [ "$(jq '.scripts.install' ./package.json)" != "null" ]; then
npm --production --offline --nodedir=$nodeSources run install
fi
if [ "$(jq '.scripts.postinstall' ./package.json)" != "null" ]; then
npm --production --offline --nodedir=$nodeSources run postinstall
fi
fi
runHook postBuild
'';
# Symlinks executables and manual pages to correct directories
installPhase = ''
runHook preInstall
echo "Symlinking bin entries from package.json"
python $linkBins
echo "Symlinking manual pages"
if [ -d "$nodeModules/$packageName/man" ]
then
mkdir -p $out/share
for dir in "$nodeModules/$packageName/man/"*
do
mkdir -p $out/share/man/$(basename "$dir")
for page in "$dir"/*
do
ln -s $page $out/share/man/$(basename "$dir")
done
done
fi
# wrap electron app
if [ -n "$electronHeaders" ]; then
echo "Wrapping electron app"
${electron-wrap}
fi
runHook postInstall
'';
});
in
pkg;
in
pkg;
in
outputs;
outputs;
}

View File

@ -1,31 +1,17 @@
{
dlib,
lib,
apps,
utils,
pkgs,
translators,
...
}: {
type = "impure";
# the input format is specified in /specifications/translator-call-example.json
# this script receives a json file including the input paths and specialArgs
translateBin = {
# dream2nix utils
apps,
subsystems,
utils,
# nixpkgs dependencies
bash,
coreutils,
git,
jq,
moreutils,
nodePackages,
openssh,
python3,
writeScriptBin,
...
}:
translateBin =
utils.writePureShellScript
[
(with pkgs; [
bash
coreutils
git
@ -34,7 +20,7 @@
nodePackages.npm
openssh
python3
]
])
''
# accroding to the spec, the translator reads the input from a json file
jsonInput=$1
@ -64,7 +50,7 @@
popd
# call package-lock translator
${subsystems.nodejs.translators.package-lock.translateBin} $TMPDIR/newJsonInput
${translators.package-lock.finalTranslateBin} $TMPDIR/newJsonInput
# get resolved package version
export version=$(npm view $candidate version)
@ -78,7 +64,7 @@
# inherit options from package-lock translator
extraArgs =
dlib.translators.translators.nodejs.package-lock.extraArgs
translators.package-lock.extraArgs
// {
npmArgs = {
description = "Additional arguments for npm";

View File

@ -1,35 +1,23 @@
{
dlib,
lib,
utils,
pkgs,
translators,
...
}: {
type = "impure";
# the input format is specified in /specifications/translator-call-example.json
# this script receives a json file including the input paths and specialArgs
translateBin = {
# dream2nix utils
subsystems,
utils,
# nixpkgs dependencies
bash,
coreutils,
git,
jq,
nodePackages,
openssh,
writeScriptBin,
...
}:
translateBin =
utils.writePureShellScript
[
(with pkgs; [
bash
coreutils
git
jq
nodePackages.npm
openssh
]
])
''
# accroding to the spec, the translator reads the input from a json file
jsonInput=$1
@ -62,12 +50,12 @@
jq ".source = \"$newSource\"" -c -r $jsonInput > $TMPDIR/newJsonInput
popd
${subsystems.nodejs.translators.package-lock.translateBin} $TMPDIR/newJsonInput
${translators.package-lock.finalTranslateBin} $TMPDIR/newJsonInput
'';
# inherit options from package-lock translator
extraArgs =
dlib.translators.translators.nodejs.package-lock.extraArgs
translators.package-lock.extraArgs
// {
npmArgs = {
description = "Additional arguments for npm";

View File

@ -1,9 +1,10 @@
{
dlib,
lib,
utils,
name,
...
}: let
b = builtins;
l = lib // builtins;
nodejsUtils = import ../utils.nix {inherit dlib lib;};
@ -11,10 +12,6 @@
nodejsUtils.getWorkspaceLockFile tree project "package-lock.json";
translate = {
translatorName,
utils,
...
}: {
project,
source,
tree,
@ -146,7 +143,7 @@
dependenciesByOriginalID,
...
}: rec {
inherit translatorName;
translatorName = name;
location = relPath;
# values

View File

@ -1,6 +1,7 @@
{
dlib,
lib,
name,
...
}: let
l = lib // builtins;
@ -10,7 +11,7 @@
getYarnLock = tree: project:
nodejsUtils.getWorkspaceLockFile tree project "yarn.lock";
translate = {translatorName, ...}: {
translate = {
project,
source,
tree,
@ -130,7 +131,9 @@
then "path"
else "http";
in rec {
inherit defaultPackage translatorName;
inherit defaultPackage;
translatorName = name;
location = relPath;

View File

@ -1,15 +1,12 @@
{...}: {
{
lib,
pkgs,
externals,
...
}: {
type = "pure";
build = {
lib,
pkgs,
stdenvNoCC,
# dream2nix inputs
externals,
callPackageDream,
...
}: {
### FUNCTIONS
# AttrSet -> Bool) -> AttrSet -> [x]
getCyclicDependencies, # name: version: -> [ {name=; version=; } ]
@ -37,7 +34,7 @@
} @ args: let
l = lib // builtins;
inherit (callPackageDream ../../semver.nix {}) satisfies;
inherit (pkgs.callPackage ../../semver.nix {}) satisfies;
# php with required extensions
php =
@ -142,7 +139,7 @@
then "0.0.0"
else version;
pkg = stdenvNoCC.mkDerivation rec {
pkg = pkgs.stdenvNoCC.mkDerivation rec {
pname = l.strings.sanitizeDerivationName name;
inherit version;

View File

@ -1,15 +1,12 @@
{...}: {
{
lib,
pkgs,
externals,
...
}: {
type = "pure";
build = {
lib,
pkgs,
stdenvNoCC,
# dream2nix inputs
externals,
callPackageDream,
...
}: {
### FUNCTIONS
# AttrSet -> Bool) -> AttrSet -> [x]
getCyclicDependencies, # name: version: -> [ {name=; version=; } ]
@ -37,7 +34,7 @@
} @ args: let
l = lib // builtins;
inherit (callPackageDream ../../semver.nix {}) satisfies;
inherit (pkgs.callPackage ../../semver.nix {}) satisfies;
# php with required extensions
php =
@ -78,7 +75,7 @@
# Unpack dependency
cleanDependency = name: version:
stdenvNoCC.mkDerivation rec {
pkgs.stdenvNoCC.mkDerivation rec {
pname = "${l.strings.sanitizeDerivationName name}-source";
inherit version;
versionString =
@ -140,7 +137,7 @@
then "0.0.0"
else version;
pkg = stdenvNoCC.mkDerivation rec {
pkg = pkgs.stdenvNoCC.mkDerivation rec {
pname = l.strings.sanitizeDerivationName name;
inherit version;

View File

@ -1,6 +1,7 @@
{
dlib,
lib,
pkgs,
utils,
translators,
...
}: {
type = "impure";
@ -16,26 +17,15 @@
# by the input parameter `outFile`.
# The output file must contain the dream lock data encoded as json.
# See /src/specifications/dream-lock-example.json
translateBin = {
# dream2nix utils
subsystems,
utils,
# nixpkgs dependencies
bash,
coreutils,
moreutils,
jq,
phpPackages,
...
}:
translateBin =
utils.writePureShellScript
[
(with pkgs; [
bash
coreutils
moreutils
jq
phpPackages.composer
]
])
''
# accroding to the spec, the translator reads the input from a json file
jsonInput=$1
@ -76,9 +66,9 @@
jq ".source = \"$newSource\"" -c -r $jsonInput > $TMPDIR/newJsonInput
popd
${subsystems.php.translators.composer-lock.translateBin} $TMPDIR/newJsonInput
${translators.composer-lock.finalTranslateBin} $TMPDIR/newJsonInput
'';
# inherit options from composer-lock translator
extraArgs = dlib.translators.translators.php.composer-lock.extraArgs;
extraArgs = translators.composer-lock.extraArgs;
}

View File

@ -1,4 +1,5 @@
{
name,
dlib,
lib,
...
@ -24,10 +25,6 @@ in {
# translate from a given source and a project specification to a dream-lock.
translate = {
translatorName,
callPackageDream,
...
}: {
/*
A list of projects returned by `discoverProjects`
Example:
@ -95,7 +92,7 @@ in {
...
} @ args: let
inherit
(callPackageDream ../../semver.nix {})
(import ../../semver.nix {inherit lib;})
satisfies
multiSatisfies
;
@ -236,7 +233,7 @@ in {
in
dlib.simpleTranslate2.translate
({objectsByKey, ...}: rec {
inherit translatorName;
translatorName = name;
# relative path of the project within the source tree.
location = project.relPath;

View File

@ -1,33 +1,23 @@
{
dlib,
lib,
apps,
utils,
translators,
pkgs,
...
}: {
type = "impure";
# the input format is specified in /specifications/translator-call-example.json
# this script receives a json file including the input paths and specialArgs
translateBin = {
# dream2nix utils
apps,
subsystems,
utils,
# nixpkgs dependencies
coreutils,
curl,
jq,
git,
moreutils,
...
}:
translateBin =
utils.writePureShellScript
[
(with pkgs; [
coreutils
curl
jq
git
moreutils
]
])
''
# according to the spec, the translator reads the input from a json file
jsonInput=$1
@ -71,10 +61,10 @@
if [ -f $TMPDIR/source/composer.lock ]
then
echo 'Translating with composer-lock'
${subsystems.php.translators.composer-lock.translateBin} $TMPDIR/newJsonInput
${translators.composer-lock.finalTranslateBin} $TMPDIR/newJsonInput
else
echo 'Translating with composer-json'
${subsystems.php.translators.composer-json.translateBin} $TMPDIR/newJsonInput
${translators.composer-json.finalTranslateBin} $TMPDIR/newJsonInput
fi
# add main package source info to dream-lock.json
@ -91,5 +81,5 @@
'';
# inherit options from composer-json translator
extraArgs = dlib.translators.translators.php.composer-json.extraArgs;
extraArgs = translators.composer-json.extraArgs;
}

View File

@ -1,12 +1,12 @@
# A very simple single derivation python builder
{...}: {
{
pkgs,
lib,
...
}: {
type = "pure";
build = {
lib,
pkgs,
...
}: {
defaultPackageName,
defaultPackageVersion,
getSource,

View File

@ -1,36 +1,23 @@
{
dlib,
lib,
externalSources,
utils,
pkgs,
...
}: let
b = builtins;
in {
}: {
type = "impure";
# the input format is specified in /specifications/translator-call-example.json
# this script receives a json file including the input paths and extraArgs
translateBin = {
externalSources,
utils,
bash,
coreutils,
jq,
nix,
python3,
remarshal,
toml2json,
writeScriptBin,
...
}:
translateBin =
utils.writePureShellScript
[
(with pkgs; [
bash
coreutils
jq
nix
remarshal
toml2json
]
])
''
# accroding to the spec, the translator reads the input from a json file
jsonInput=$1

View File

@ -1,14 +1,12 @@
{...}: {
{
pkgs,
lib,
externals,
...
}: {
type = "pure";
build = {
lib,
pkgs,
stdenv,
# dream2nix inputs
externals,
...
}: {
### FUNCTIONS
# AttrSet -> Bool) -> AttrSet -> [x]
getCyclicDependencies, # name: version: -> [ {name=; version=; } ]

View File

@ -1,10 +1,8 @@
{
dlib,
lib,
utils,
pkgs,
...
}: let
l = lib // builtins;
in {
}: {
type = "impure";
# A derivation which outputs a single executable at `$out`.
@ -18,40 +16,27 @@ in {
# by the input parameter `outFile`.
# The output file must contain the dream lock data encoded as json.
# See /src/specifications/dream-lock-example.json
translateBin = {
# dream2nix utils
utils,
# nixpkgs dependencies
bash,
coreutils,
fetchurl,
jq,
nix,
racket,
runCommandLocal,
writeScriptBin,
...
}: let
translateBin = let
pruned-racket-catalog = let
src = fetchurl {
src = pkgs.fetchurl {
url = "https://github.com/nix-community/pruned-racket-catalog/tarball/9f11e5ea5765c8a732c5e3129ca2b71237ae2bac";
sha256 = "sha256-/n30lailqSndoqPGWcFquCpQWVQcciMiypXYLhNmFUo=";
};
in
runCommandLocal "pruned-racket-catalog" {} ''
pkgs.runCommandLocal "pruned-racket-catalog" {} ''
mkdir $out
cd $out
tar --strip-components 1 -xf ${src}
'';
in
utils.writePureShellScript
[
(with pkgs; [
bash
coreutils
jq
nix
racket
]
])
''
# according to the spec, the translator reads the input from a json file
jsonInput=$1

View File

@ -1,13 +1,13 @@
{...}: {
{
lib,
dlib,
pkgs,
externals,
...
} @ topArgs: {
type = "pure";
build = {
lib,
dlib,
pkgs,
callPackageDream,
...
} @ topArgs: {
subsystemAttrs,
defaultPackageName,
defaultPackageVersion,
@ -68,7 +68,10 @@
});
mkShellForPkg = pkg:
callPackageDream ../devshell.nix {drv = pkg;};
pkgs.callPackage ../devshell.nix {
inherit externals;
drv = pkg;
};
allPackages =
l.mapAttrs

View File

@ -1,12 +1,12 @@
{...}: {
{
lib,
pkgs,
externals,
...
} @ topArgs: {
type = "ifd";
build = {
lib,
pkgs,
externals,
callPackageDream,
...
} @ topArgs: {
subsystemAttrs,
defaultPackageName,
defaultPackageVersion,
@ -121,12 +121,16 @@
pname
(buildPackageWithToolchain defaultToolchain buildArgs);
# TODO: this does not carry over environment variables from the
# dependencies derivation. this could cause confusion for users.
mkShellForPkg = pkg: let
pkgDeps = pkg.passthru.dependencies;
depsShell = callPackageDream ../devshell.nix {drv = pkgDeps;};
mainShell = callPackageDream ../devshell.nix {drv = pkg;};
depsShell = pkgs.callPackage ../devshell.nix {
inherit externals;
drv = pkgDeps;
};
mainShell = pkgs.callPackage ../devshell.nix {
inherit externals;
drv = pkg;
};
shell = depsShell.combineWith mainShell;
in
shell;

View File

@ -1,6 +1,6 @@
{
dlib,
lib,
dlib,
...
}: let
l = lib // builtins;

View File

@ -1,4 +1,5 @@
{
name,
dlib,
lib,
...
@ -18,7 +19,7 @@ in {
})
];
translate = {translatorName, ...}: {
translate = {
project,
tree,
...
@ -180,8 +181,7 @@ in {
in
dlib.simpleTranslate2.translate
({...}: {
inherit translatorName;
translatorName = name;
# relative path of the project within the source tree.
location = project.relPath;

View File

@ -1,28 +1,20 @@
{
dlib,
lib,
pkgs,
utils,
translators,
...
}: {
type = "impure";
# the input format is specified in /specifications/translator-call-example.json
# this script receives a json file including the input paths and specialArgs
translateBin = {
# dream2nix utils
subsystems,
utils,
# nixpkgs dependencies
coreutils,
jq,
rustPlatform,
...
}:
translateBin =
utils.writePureShellScript
[
(with pkgs; [
coreutils
jq
rustPlatform.rust.cargo
]
])
''
# according to the spec, the translator reads the input from a json file
jsonInput=$1
@ -47,7 +39,7 @@
popd
if [ $cargoResult -eq 0 ]; then
${subsystems.rust.translators.cargo-lock.translateBin} $TMPDIR/newJsonInput
${translators.cargo-lock.finalTranslateBin} $TMPDIR/newJsonInput
else
echo "cargo failed to generate the lockfile"
exit 1
@ -56,7 +48,7 @@
# inherit options from cargo-lock translator
extraArgs =
dlib.translators.translators.rust.cargo-lock.extraArgs
translators.cargo-lock.extraArgs
// {
cargoArgs = {
description = "Additional arguments for Cargo";

View File

@ -1,29 +1,17 @@
{
dlib,
lib,
pkgs,
apps,
utils,
translators,
...
}: {
type = "impure";
# the input format is specified in /specifications/translator-call-example.json
# this script receives a json file including the input paths and specialArgs
translateBin = {
# dream2nix utils
apps,
subsystems,
utils,
# nixpkgs dependencies
coreutils,
curl,
gnutar,
gzip,
jq,
moreutils,
rustPlatform,
...
}:
translateBin =
utils.writePureShellScript
[
(with pkgs; [
coreutils
curl
gnutar
@ -31,7 +19,7 @@
jq
moreutils
rustPlatform.rust.cargo
]
])
''
# according to the spec, the translator reads the input from a json file
jsonInput=$1
@ -67,9 +55,9 @@
# we don't need to run cargo-toml translator if Cargo.lock exists
if [ -f "$TMPDIR/source/Cargo.lock" ]; then
${subsystems.rust.translators.cargo-lock.translateBin} $TMPDIR/newJsonInput
${translators.cargo-lock.finalTranslateBin} $TMPDIR/newJsonInput
else
${subsystems.rust.translators.cargo-toml.translateBin} $TMPDIR/newJsonInput
${translators.cargo-toml.finalTranslateBin} $TMPDIR/newJsonInput
fi
# add main package source info to dream-lock.json
@ -85,16 +73,5 @@
'';
# inherit options from cargo-lock translator
extraArgs =
dlib.translators.translators.rust.cargo-lock.extraArgs
// {
cargoArgs = {
description = "Additional arguments for Cargo";
type = "argument";
default = "";
examples = [
"--verbose"
];
};
};
extraArgs = translators.cargo-toml.extraArgs;
}

View File

@ -1,122 +0,0 @@
{
coreutils,
dlib,
jq,
lib,
nix,
pkgs,
python3,
callPackageDream,
externals,
dream2nixWithExternals,
utils,
config,
configFile,
framework,
...
}: let
b = builtins;
l = lib // builtins;
# adds a translateBin to a pure translator
wrapPureTranslator = {
subsystem,
name,
}: let
bin =
utils.writePureShellScript
[
coreutils
jq
nix
python3
]
''
jsonInputFile=$(realpath $1)
outputFile=$(realpath -m $(jq '.outputFile' -c -r $jsonInputFile))
pushd $TMPDIR
nix eval \
--option experimental-features "nix-command flakes"\
--show-trace --impure --raw --expr "
let
dream2nix = import ${dream2nixWithExternals} {
config = ${configFile};
};
translatorArgs =
(builtins.fromJSON
(builtins.unsafeDiscardStringContext (builtins.readFile '''$1''')));
dreamLock' =
dream2nix.subsystems.${subsystem}.translators.${name}.translate
translatorArgs;
# simpleTranslate2 puts dream-lock in result
dreamLock = dreamLock'.result or dreamLock';
in
dream2nix.utils.dreamLock.toJSON
# don't use nix to detect cycles, this will be more efficient in python
(dreamLock // {
_generic = builtins.removeAttrs dreamLock._generic [ \"cyclicDependencies\" ];
})
" | python3 ${../apps/cli/format-dream-lock.py} > out
tmpOut=$(realpath out)
popd
mkdir -p $(dirname $outputFile)
cp $tmpOut $outputFile
'';
in
bin.overrideAttrs (old: {
name = "translator-${subsystem}-pure-${name}";
});
makeTranslator = translatorModule: let
translator =
translatorModule
# for pure translators
# - import the `translate` function
# - generate `translateBin`
// (lib.optionalAttrs (translatorModule.translate or null != null) {
translate = let
translateOriginal = callPackageDream translatorModule.translate {
translatorName = translatorModule.name;
};
in
args:
translateOriginal
(
(framework.functions.translators.makeTranslatorDefaultArgs
(translatorModule.extraArgs or {}))
// args
// (args.project.subsystemInfo or {})
// {
tree =
args.tree or (dlib.prepareSourceTree {inherit (args) source;});
}
);
translateBin =
wrapPureTranslator
{inherit (translatorModule) subsystem name;};
})
# for impure translators:
# - import the `translateBin` function
// (lib.optionalAttrs (translatorModule.translateBin or null != null) {
translateBin =
callPackageDream translatorModule.translateBin
{
translatorName = translatorModule.name;
};
});
in
translator;
translators = framework.translatorsBySubsystem;
in {
inherit
translators
makeTranslator
;
}

View File

@ -19,7 +19,6 @@
callPackageDream,
dream2nixWithExternals,
externalSources,
subsystems,
config,
configFile,
framework,
@ -181,7 +180,7 @@ in
aggregate = project.aggregate or false;
translator =
framework.translatorInstances."${project.translator}";
framework.translators."${project.translator}";
argsJsonFile =
pkgs.writeText "translator-args.json"
@ -207,7 +206,7 @@ in
''
dreamLockPath="${project.dreamLockPath}"
${translator.translateBin} ${argsJsonFile}
${translator.finalTranslateBin} ${argsJsonFile}
# aggregate source hashes
if [ "${l.toJSON aggregate}" == "true" ]; then

View File

@ -1,8 +1,8 @@
{
lib,
# dream2nix
fetchers,
dlib,
framework,
...
}: let
b = builtins;
@ -89,12 +89,14 @@
constructedArgsKeep =
overrideWarning ["pname" "version"] constructedArgs;
constructedSource = fetchers.constructSource (constructedArgsKeep
constructedSource = framework.functions.fetchers.constructSource (
constructedArgsKeep
// {
inherit type;
pname = pkgName;
version = pkgVersion;
});
}
);
skip =
(type == "path")

View File

@ -5,7 +5,7 @@ import pytest
def get_projects_to_test():
tests = nix_ffi.eval(
'subsystems.allTranslators',
'framework.translators',
wrapper_code = '''
{result, ...}: let
lib = (import <nixpkgs> {}).lib;
@ -23,7 +23,7 @@ def get_projects_to_test():
})
(translator.generateUnitTestsForProjects or [])
)
result
(l.attrValues result)
)
''',
)
@ -65,7 +65,7 @@ def check_format_sourceSpec(sourceSpec):
@pytest.mark.parametrize("p", projects)
def test_packageName(p):
defaultPackage = nix_ffi.eval(
f"subsystems.{p['subsystem']}.translators.{p['translator']}.translate",
f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.finalTranslate",
params=dict(
project=p['project'],
source=p['source'],
@ -81,7 +81,7 @@ def test_packageName(p):
@pytest.mark.parametrize("p", projects)
def test_exportedPackages(p):
exportedPackages = nix_ffi.eval(
f"subsystems.{p['subsystem']}.translators.{p['translator']}.translate",
f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.finalTranslate",
params=dict(
project=p['project'],
source=p['source'],
@ -97,7 +97,7 @@ def test_exportedPackages(p):
@pytest.mark.parametrize("p", projects)
def test_extraObjects(p):
extraObjects = nix_ffi.eval(
f"subsystems.{p['subsystem']}.translators.{p['translator']}.translate",
f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.finalTranslate",
params=dict(
project=p['project'],
source=p['source'],
@ -121,7 +121,7 @@ def test_extraObjects(p):
@pytest.mark.parametrize("p", projects)
def test_location(p):
location = nix_ffi.eval(
f"subsystems.{p['subsystem']}.translators.{p['translator']}.translate",
f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.finalTranslate",
params=dict(
project=p['project'],
source=p['source'],
@ -136,7 +136,7 @@ def test_location(p):
@pytest.mark.parametrize("p", projects)
def test_serializedRawObjects(p):
serializedRawObjects = nix_ffi.eval(
f"subsystems.{p['subsystem']}.translators.{p['translator']}.translate",
f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.finalTranslate",
params=dict(
project=p['project'],
source=p['source'],
@ -159,7 +159,7 @@ def test_serializedRawObjects(p):
@pytest.mark.parametrize("p", projects)
def test_subsystemName(p):
subsystemName = nix_ffi.eval(
f"subsystems.{p['subsystem']}.translators.{p['translator']}.translate",
f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.finalTranslate",
params=dict(
project=p['project'],
source=p['source'],
@ -175,7 +175,7 @@ def test_subsystemName(p):
@pytest.mark.parametrize("p", projects)
def test_subsystemAttrs(p):
subsystemAttrs = nix_ffi.eval(
f"subsystems.{p['subsystem']}.translators.{p['translator']}.translate",
f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.finalTranslate",
params=dict(
project=p['project'],
source=p['source'],
@ -191,7 +191,7 @@ def test_subsystemAttrs(p):
@pytest.mark.parametrize("p", projects)
def test_translatorName(p):
translatorName = nix_ffi.eval(
f"subsystems.{p['subsystem']}.translators.{p['translator']}.translate",
f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.finalTranslate",
params=dict(
project=p['project'],
source=p['source'],
@ -207,7 +207,7 @@ def test_translatorName(p):
@pytest.mark.parametrize("p", projects)
def test_extractors(p):
finalObjects = nix_ffi.eval(
f"subsystems.{p['subsystem']}.translators.{p['translator']}.translate",
f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.finalTranslate",
params=dict(
project=p['project'],
source=p['source'],
@ -241,7 +241,7 @@ def test_extractors(p):
@pytest.mark.parametrize("p", projects)
def test_keys(p):
objectsByKey = nix_ffi.eval(
f"subsystems.{p['subsystem']}.translators.{p['translator']}.translate",
f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.finalTranslate",
params=dict(
project=p['project'],
source=p['source'],

View File

@ -68,7 +68,7 @@ import nix_ffi
])
def test_translateShortcut(shortcut, expected):
result = nix_ffi.callNixFunction(
'fetchers.translateShortcut',
'framework.functions.fetchers.translateShortcut',
shortcut=shortcut,
computeHash=False,
)