This commit is contained in:
Yusuf Bera Ertan 2022-10-06 19:11:06 +03:00
parent 12e45887b5
commit 65120af9f6
No known key found for this signature in database
GPG Key ID: 1D8F8FAF2294D6EA
52 changed files with 181 additions and 1446 deletions

View File

@ -51,7 +51,7 @@ in let
};
framework = import ./modules/framework.nix {
inherit lib dlib callPackageDream pkgs utils;
inherit apps lib dlib pkgs utils;
dream2nixConfig = config;
};
@ -77,7 +77,6 @@ in let
inherit dream2nixWithExternals;
inherit utils;
inherit nix;
inherit subsystems;
dream2nixInterface = {
inherit
makeOutputsForDreamLock
@ -107,8 +106,6 @@ in let
# updater modules to find newest package versions
updaters = callPackageDream ./updaters {};
subsystems = callPackageDream ./subsystems {};
externals = {
devshell = {
makeShell = import "${externalSources.devshell}/modules" pkgs;
@ -434,7 +431,7 @@ in let
settings ? [],
} @ args: let
getTranslator = translatorName:
framework.translatorInstances.${translatorName};
framework.translators.${translatorName};
isImpure = project: translatorName:
(getTranslator translatorName).type == "impure";
@ -499,7 +496,7 @@ in let
l.forEach projectsPureUnresolved
(proj: let
translator = getTranslator proj.translator;
dreamLock'' = translator.translate {
dreamLock'' = translator.translateInstanced {
inherit source tree discoveredProjects;
project = proj;
};
@ -650,7 +647,7 @@ in let
impureDiscoveredProjects =
l.filter
(proj:
framework.translatorInstances."${proj.translator}".type
framework.translators."${proj.translator}".type
== "impure")
discoveredProjects;
@ -727,6 +724,5 @@ in {
utils
makeOutputsForDreamLock
makeOutputs
subsystems
;
}

View File

@ -1,228 +0,0 @@
# this fetcher takes an attrset of sources and combines all contained FODs
# to one large FOD. Non-FOD sources like derivations and store paths are
# not touched
{
bash,
async,
coreutils,
lib,
nix,
stdenv,
writeScript,
# dream2nix
defaultFetcher,
dlib,
utils,
...
}: {
# sources attrset from dream lock
sources,
sourcesAggregatedHash,
sourceOverrides,
...
} @ args: let
b = builtins;
l = lib // builtins;
# resolve to individual fetcher calls
defaultFetched = (defaultFetcher args).fetchedSources;
isFOD = drv:
lib.all
(attr: drv ? "${attr}")
["outputHash" "outputHashAlgo" "outputHashMode"];
drvArgs = drv:
(drv.overrideAttrs (args: {
passthru.originalArgs = args;
}))
.originalArgs;
# extract the arguments from the individual fetcher calls
FODArgsAll = let
FODArgsAll' =
lib.mapAttrs
(
name: versions:
lib.mapAttrs
(version: fetched:
# handle FOD sources
if isFOD fetched
then
(drvArgs fetched)
// {
isOriginal = false;
outPath = let
sanitizedName = l.strings.sanitizeDerivationName name;
in "${sanitizedName}/${version}/${fetched.name}";
}
# handle already extracted sources
else if fetched ? original && isFOD fetched.original
then
(drvArgs fetched.original)
// {
isOriginal = true;
outPath = let
sanitizedName = l.strings.sanitizeDerivationName name;
in "${sanitizedName}/${version}/${fetched.original.name}";
}
# handle path sources
else if lib.isString fetched
then "ignore"
# handle store path sources
else if lib.isStorePath fetched
then "ignore"
# handle unknown sources
else if fetched == "unknown"
then "ignore"
# error out on unknown source types
else
throw ''
Error while generating FOD fetcher for combined sources.
Cannot classify source of ${name}#${version}.
'')
versions
)
defaultFetched;
in
lib.filterAttrs
(name: versions: versions != {})
(lib.mapAttrs
(name: versions:
lib.filterAttrs
(version: fetcherArgs: fetcherArgs != "ignore")
versions)
FODArgsAll');
FODArgsAllList =
lib.flatten
(lib.mapAttrsToList
(name: versions:
b.attrValues versions)
FODArgsAll);
# convert arbitrary types to string, like nix does with derivation arguments
toString' = x:
if lib.isBool x
then
if x
then "1"
else ""
else if lib.isList x
then ''"${lib.concatStringsSep " " (lib.forEach x (y: toString' y))}"''
else if x == null
then ""
else b.toJSON x;
# set up nix build env for signle item
itemScript = fetcherArgs: ''
# export arguments for builder
${lib.concatStringsSep "\n" (lib.mapAttrsToList (argName: argVal: ''
export ${argName}=${
lib.replaceStrings ["$" ''\n''] [''\$'' "\n"] (toString' argVal)
}
'')
fetcherArgs)}
# run builder
bash ${fetcherArgs.builder}
'';
mkScriptForItem = fetcherArgs: ''
# configure $out
OUT_ORIG=$out
export out=$OUT_ORIG/${fetcherArgs.outPath}
mkdir -p $(dirname $out)
# set up TMP and TMPDIR
workdir=$(mktemp -d)
TMP=$workdir/TMP
TMPDIR=$TMP
mkdir -p $TMP
# do the work
pushd $workdir
${itemScript fetcherArgs}
popd
rm -r $workdir
'';
# builder which wraps several other FOD builders
# and executes these after each other inside a single build
builder = writeScript "multi-source-fetcher" ''
#!${bash}/bin/bash
export PATH=${coreutils}/bin:${bash}/bin
mkdir $out
S="/$TMP/async_socket"
async=${async}/bin/async
$async -s="$S" server --start -j40
# remove if resolved: https://github.com/ctbur/async/issues/6
sleep 1
${lib.concatStringsSep "\n"
(b.map
(fetcherArgs: ''
$async -s="$S" cmd -- bash -c '${mkScriptForItem fetcherArgs}'
'')
FODArgsAllList)}
$async -s="$S" wait
${lib.concatStringsSep "\n"
(b.map
(fetcherArgs: ''
if [ ! -e "$out/${fetcherArgs.outPath}" ]; then
echo "builder for ${fetcherArgs.name} terminated without creating out path: ${fetcherArgs.outPath}"
exit 1
fi
'')
FODArgsAllList)}
echo "FOD_HASH=$(${nix}/bin/nix --extra-experimental-features "nix-command flakes" hash path $out)"
'';
FODAllSources = let
nativeBuildInputs' =
lib.unique
(lib.foldl (a: b: a ++ b) []
(b.map
(fetcherArgs: (fetcherArgs.nativeBuildInputs or []))
FODArgsAllList));
in
stdenv.mkDerivation rec {
name = "sources-combined";
inherit builder;
nativeBuildInputs =
nativeBuildInputs'
++ [
coreutils
];
outputHashAlgo = "sha256";
outputHashMode = "recursive";
outputHash = sourcesAggregatedHash;
};
in {
FOD = FODAllSources;
fetchedSources =
lib.mapAttrs
(name: versions:
lib.mapAttrs
(version: source:
if FODArgsAll ? "${name}"."${version}".outPath
then
if FODArgsAll ? "${name}"."${version}".isOriginal
then
utils.extractSource {
source = "${FODAllSources}/${FODArgsAll."${name}"."${version}".outPath}";
name = l.strings.sanitizeDerivationName name;
}
else "${FODAllSources}/${FODArgsAll."${name}"."${version}".outPath}"
else defaultFetched."${name}"."${version}")
versions)
defaultFetched;
}

View File

@ -1,70 +0,0 @@
{
lib,
# dream2nix attributes
dlib,
fetchSource,
fetchers,
...
}: {
# sources attrset from dream lock
defaultPackage,
defaultPackageVersion,
sourceOverrides,
sources,
sourceRoot ? null,
...
}: let
l = lib // builtins;
fetchedSources =
l.mapAttrs
(name: versions:
l.mapAttrs
(version: source:
if source.type == "unknown"
then "unknown"
else if source.type == "path"
then let
path =
if l.isStorePath (l.concatStringsSep "/" (l.take 4 (l.splitString "/" source.path)))
then source.path
else if name == source.rootName && version == source.rootVersion
then throw "source for ${name}@${version} is referencing itself"
else if source.rootName != null && source.rootVersion != null
then "${overriddenSources."${source.rootName}"."${source.rootVersion}"}/${source.path}"
else if sourceRoot != null
then "${sourceRoot}/${source.path}"
else throw "${name}-${version}: cannot determine path source";
in
l.path {
inherit path;
name = l.strings.sanitizeDerivationName "${name}-${version}-source";
}
else if fetchers.fetchers ? "${source.type}"
then
fetchSource {
source =
source
// {
pname = source.pname or name;
version = source.version or version;
};
}
else throw "unsupported source type '${source.type}'")
versions)
sources;
overriddenSources =
l.zipAttrsWith
(name: versionsSets:
l.zipAttrsWith
(version: sources: l.last sources)
versionsSets)
[
fetchedSources
(sourceOverrides fetchedSources)
];
in {
# attrset: pname -> path of downloaded source
fetchedSources = overriddenSources;
}

View File

@ -1,240 +0,0 @@
{
lib,
# dream2nix
callPackageDream,
dlib,
...
}: let
b = builtins;
callFetcher = module:
module // {outputs = callPackageDream module.outputs {};};
in rec {
fetchers = dlib.fetchers.mapFetchers callFetcher;
defaultFetcher = callPackageDream ./default-fetcher.nix {inherit fetchSource;};
combinedFetcher = callPackageDream ./combined-fetcher.nix {inherit defaultFetcher;};
constructSource = {
type,
reComputeHash ? false,
...
} @ args: let
fetcher = fetchers."${type}";
argsKeep = b.removeAttrs args ["reComputeHash"];
fetcherOutputs =
fetcher.outputs
(b.removeAttrs argsKeep ["dir" "hash" "type"]);
in
argsKeep
# if the hash was not provided, calculate hash on the fly (impure)
// (lib.optionalAttrs reComputeHash {
hash = fetcherOutputs.calcHash "sha256";
});
# update source spec to different version
updateSource = {
source,
newVersion,
...
}:
constructSource (source
// {
reComputeHash = true;
}
// {
"${fetchers."${source.type}".versionField}" = newVersion;
});
# fetch a source defined via a dream lock source spec
fetchSource = {
source,
extract ? false,
}: let
fetcher = fetchers."${source.type}";
fetcherArgs = b.removeAttrs source ["dir" "hash" "type"];
fetcherOutputs = fetcher.outputs fetcherArgs;
maybeArchive = fetcherOutputs.fetched (source.hash or null);
in
if source ? dir
then "${maybeArchive}/${source.dir}"
else maybeArchive;
# fetch a source defined by a shortcut
fetchShortcut = {
shortcut,
extract ? false,
}:
fetchSource {
source = translateShortcut {inherit shortcut;};
inherit extract;
};
parseShortcut = shortcut: let
# in: "git+https://foo.com/bar?kwarg1=lol&kwarg2=hello"
# out: [ "git+" "git" "https" "//" "foo.com/bar" "?kwarg1=lol&kwarg2=hello" "kwarg1=lol&kwarg2=hello" ]
split =
b.match
''(([[:alnum:]]+)\+)?([[:alnum:]-]+):(//)?([^\?]*)(\?(.*))?''
shortcut;
parsed = {
proto1 = b.elemAt split 1;
proto2 = b.elemAt split 2;
path = b.elemAt split 4;
allArgs = b.elemAt split 6;
kwargs = b.removeAttrs kwargs_ ["dir"];
dir = kwargs_.dir or null;
};
kwargs_ =
if parsed.allArgs == null
then {}
else
lib.listToAttrs
(map
(kwarg: let
split = lib.splitString "=" kwarg;
in
lib.nameValuePair
(b.elemAt split 0)
(b.elemAt split 1))
(lib.splitString "&" parsed.allArgs));
in
if split == null
then throw "Unable to parse shortcut: ${shortcut}"
else parsed;
renderUrlArgs = kwargs: let
asStr =
lib.concatStringsSep
"&"
(lib.mapAttrsToList
(name: val: "${name}=${val}")
kwargs);
in
if asStr == ""
then ""
else "?" + asStr;
# translate shortcut to dream lock source spec
translateShortcut = {
shortcut ? throw "pass shortcut",
parsed ? parseShortcut shortcut,
computeHash ? (! parsed.kwargs ? hash),
}: let
parsed = parseShortcut shortcut;
checkArgs = fetcherName: args: let
fetcher = fetchers."${fetcherName}";
unknownArgNames = lib.filter (argName: ! lib.elem argName fetcher.inputs) (lib.attrNames args);
missingArgNames = lib.filter (inputName: ! args ? "${inputName}") fetcher.inputs;
in
if lib.length unknownArgNames > 0
then throw "Received unknown arguments for fetcher '${fetcherName}': ${b.toString unknownArgNames}"
else if lib.length missingArgNames > 0
then throw "Missing arguments for fetcher '${fetcherName}': ${b.toString missingArgNames}"
else args;
translateHttpUrl = let
fetcher = fetchers.http;
urlArgsFinal = renderUrlArgs parsed.kwargs;
url = with parsed; "${proto2}://${path}${urlArgsFinal}";
fetcherOutputs = fetchers.http.outputs {
inherit url;
};
in
constructSource
{
inherit url;
type = "http";
}
// (lib.optionalAttrs (parsed.dir != null) {
dir = parsed.dir;
})
// (lib.optionalAttrs computeHash {
hash = fetcherOutputs.calcHash "sha256";
});
translateProtoShortcut = let
kwargsUrl = b.removeAttrs parsed.kwargs fetcher.inputs;
urlArgs = renderUrlArgs kwargsUrl;
url = with parsed; "${proto2}://${path}${urlArgs}";
fetcherName = parsed.proto1;
fetcher = fetchers."${fetcherName}";
args = parsed.kwargs // {inherit url;};
fetcherOutputs = fetcher.outputs (checkArgs fetcherName args);
in
constructSource
(parsed.kwargs
// {
type = fetcherName;
inherit url;
}
// (lib.optionalAttrs (parsed.dir != null) {
dir = parsed.dir;
})
// (lib.optionalAttrs computeHash {
hash = fetcherOutputs.calcHash "sha256";
}));
translateRegularShortcut = let
fetcherName = parsed.proto2;
path = lib.removeSuffix "/" parsed.path;
params = lib.splitString "/" path;
fetcher = fetchers."${fetcherName}";
args =
if fetcher ? parseParams
then fetcher.parseParams params
else if b.length params != b.length fetcher.inputs
then
throw ''
Wrong number of arguments provided in shortcut for fetcher '${fetcherName}'
Should be ${fetcherName}:${lib.concatStringsSep "/" fetcher.inputs}
''
else
lib.listToAttrs
(lib.forEach
(lib.range 0 ((lib.length fetcher.inputs) - 1))
(
idx:
lib.nameValuePair
(lib.elemAt fetcher.inputs idx)
(lib.elemAt params idx)
));
fetcherOutputs = fetcher.outputs (args // parsed.kwargs);
in
constructSource (args
// parsed.kwargs
// {
type = fetcherName;
}
// (lib.optionalAttrs (parsed.dir != null) {
dir = parsed.dir;
})
// (lib.optionalAttrs computeHash {
hash = fetcherOutputs.calcHash "sha256";
}));
in
if parsed.proto1 != null
then translateProtoShortcut
else if
lib.hasPrefix "http://" shortcut
|| lib.hasPrefix "https://" shortcut
then translateHttpUrl
else translateRegularShortcut;
}

View File

@ -92,8 +92,7 @@
framework = import ./modules/framework.nix {
inherit lib dlib;
dream2nixConfig = config;
callPackageDream =
throw "callPackageDream is not available before nixpkgs is imported";
apps = throw "apps is not available before nixpkgs is imported";
pkgs = throw "pkgs is not available before nixpkgs is imported";
utils = throw "utils is not available before nixpkgs is imported";
};

View File

@ -1,32 +0,0 @@
{
dlib,
lib,
config,
}: let
l = lib // builtins;
defaults = {
rust = "build-rust-package";
nodejs = "granular";
python = "simple-builder";
php = "granular";
racket = "simple";
};
# TODO
validator = module: true;
modules = dlib.modules.makeSubsystemModules {
inherit validator defaults;
modulesCategory = "builders";
};
builders =
dlib.modules.mapSubsystemModules
(b: b // {build = dlib.warnIfIfd b b.build;})
modules.modules;
mapBuilders = f: dlib.modules.mapSubsystemModules f builders;
in {
inherit builders mapBuilders;
callBuilder = modules.callModule;
}

View File

@ -8,14 +8,11 @@
# exported attributes
dlib = {
inherit
builders
calcInvalidationHash
callViaEnv
construct
containsMatchingFile
dirNames
discoverers
fetchers
indexers
latestVersion
listDirs
@ -25,7 +22,6 @@
readTextFile
recursiveUpdateUntilDepth
simpleTranslate2
translators
sanitizePath
sanitizeRelativePath
subsystems
@ -46,11 +42,7 @@
subsystems = dirNames ../subsystems;
# other libs
builders = import ./builders.nix {inherit dlib lib config;};
construct = import ./construct.nix {inherit lib;};
discoverers = import ./discoverers.nix {inherit config dlib lib;};
fetchers = import ./fetchers.nix {inherit dlib lib;};
translators = import ./translators.nix {inherit dlib lib;};
indexers = import ./indexers.nix {inherit dlib lib;};
modules = import ./modules.nix {inherit config dlib lib;};

View File

@ -1,147 +0,0 @@
{
dlib,
lib,
config,
}: let
l = lib // builtins;
allDiscoverers =
(dlib.modules.collectSubsystemModules modules.modules)
++ [defaultDiscoverer];
allTranslators =
dlib.modules.collectSubsystemModules dlib.translators.translators;
translatorsWithDiscoverFunc =
l.filter (translator: translator.discoverProject or null != null) allTranslators;
defaultDiscoverer.discover = {tree}: let
translatorsCurrentDir =
l.filter
(t: t.discoverProject tree)
translatorsWithDiscoverFunc;
projectsCurrentDir =
l.map
(t: {
name = "main";
relPath = tree.relPath;
translators = [t.name];
subsystem = t.subsystem;
})
translatorsCurrentDir;
# If there are multiple projects detected for the same subsystem,
# merge them to a single one with translators = [...]
projectsCurrentDirMerged =
l.attrValues
(l.foldl
(all: curr:
all
// {
"${curr.subsystem}" =
all.${curr.subsystem}
or curr
// {
translators =
l.unique (all.${curr.subsystem}.translators or [] ++ curr.translators);
};
})
{}
projectsCurrentDir);
subdirProjects =
l.flatten
(l.mapAttrsToList
(dirName: tree:
defaultDiscoverer.discover {
inherit tree;
})
tree.directories or {});
in (
if translatorsCurrentDir == []
then subdirProjects
else projectsCurrentDirMerged ++ subdirProjects
);
discoverProjects = {
projects,
source ? throw "Pass either `source` or `tree` to discoverProjects",
tree ? dlib.prepareSourceTree {inherit source;},
settings ? [],
}: let
discoveredProjects =
l.flatten
(l.map
(discoverer: discoverer.discover {inherit tree;})
allDiscoverers);
discoveredProjectsSorted = let
sorted =
l.sort
(p1: p2: l.hasPrefix p1.relPath or "" p2.relPath or "")
discoveredProjects;
in
sorted;
allProjects = discoveredProjectsSorted ++ (l.attrValues projects);
rootProject = l.head allProjects;
projectsExtended =
l.forEach allProjects
(proj:
proj
// {
relPath = proj.relPath or "";
translator = proj.translator or (l.head proj.translators);
dreamLockPath = getDreamLockPath proj rootProject;
});
in
applyProjectSettings projectsExtended settings;
getDreamLockPath = project: rootProject:
dlib.sanitizeRelativePath
"${config.packagesDir}/${rootProject.name}/${project.relPath or ""}/dream-lock.json";
applyProjectSettings = projects: settingsList: let
settingsListForProject = project:
l.filter
(settings:
if ! settings ? filter
then true
else settings.filter project)
settingsList;
applySettings = project: settings:
l.recursiveUpdate project settings;
applyAllSettings = project:
l.foldl'
(proj: settings: applySettings proj settings)
project
(settingsListForProject project);
settingsApplied =
l.forEach projects
(proj: applyAllSettings proj);
in
settingsApplied;
# TODO
validator = module: true;
modules = dlib.modules.makeSubsystemModules {
modulesCategory = "discoverers";
inherit validator;
};
in {
inherit
applyProjectSettings
discoverProjects
;
discoverers = modules.modules;
callDiscoverer = modules.callModule;
mapDiscoverers = modules.mapModules;
}

View File

@ -1,40 +0,0 @@
{
dlib,
lib,
}: let
l = lib // builtins;
# TODO:
validator = module: true;
callFetcher = file: extraArgs:
dlib.modules.importModule {inherit file extraArgs validator;};
# get information for builtin fetchers
fetchersDir = ../fetchers;
fetcherNames = dlib.dirNames fetchersDir;
# get the builtin fetchers
builtinFetchers =
l.map
(name: {
file = "${fetchersDir}/${name}";
extraArgs = {inherit name;};
})
fetcherNames;
# get extra fetchers
extraFetchers = dlib.modules.extra.fetchers or [];
# import fetchers
importedFetchers =
l.map
(
module:
(callFetcher module.file module.extraArgs)
// {inherit (module.extraArgs) name;}
)
(builtinFetchers ++ extraFetchers);
# create the attrset
fetchers = l.listToAttrs (l.map (f: l.nameValuePair f.name f) importedFetchers);
mapFetchers = f: l.mapAttrs (_: fetcher: f fetcher) fetchers;
in {inherit fetchers callFetcher mapFetchers;}

View File

@ -1,197 +0,0 @@
{
config,
dlib,
lib,
}: let
l = lib // builtins;
configFuncMsg = ''
consider passing a path to a file instead of a function:
functions can't be encoded to JSON, and as such most features of
dream2nix won't work because of this since they require passing around
the config as JSON.
'';
# imports a module.
#
# - 'file' can be a function or a path to a function.
# - 'dlib', 'lib', 'config' and attributes in
# 'extraArgs' are passed to the function.
importModule = {
file,
validator ? _: true,
extraArgs ? {},
}: let
_module =
if l.isFunction file
then file
else import file;
module =
if l.isFunction _module
then _module ({inherit config dlib lib;} // extraArgs)
else throw "module file (${file}) must return a function that takes an attrset";
in
l.seq (validator module) module;
# collects extra modules from a list
# ex: [{translators = [module1];} {translators = [module2];}] -> {translators = [module1 module2];}
collectExtraModules =
l.foldl'
(acc: el:
acc
// (
l.mapAttrs
(category: modules: modules ++ (acc.${category} or []))
el
))
{};
# processes one extra (config.extra)
# returns extra modules like {fetchers = [...]; translators = [...];}
processOneExtra = _extra: let
isAttrs = val: l.isAttrs val && (! val ? drvPath);
# imports a modules declaration
importDecl = decl:
if l.isFunction decl
then l.throw configFuncMsg
else if isAttrs decl
then decl
else import decl {inherit config dlib lib;};
# extra attrset itself
# config.extra is imported here if it's a path
extra = importDecl _extra;
# throw if declaring a module as a function
throwIfModuleNotPath = module:
l.throwIf ((isAttrs _extra) && (l.isFunction module)) configFuncMsg module;
# collect subsystem modules (translators, discoverers, builders)
_extraSubsystemModules =
l.mapAttrsToList
(subsystem: categories:
l.mapAttrs
(category: modules:
l.mapAttrsToList
(name: module: {
file = throwIfModuleNotPath module;
extraArgs = {inherit subsystem name;};
})
(importDecl modules))
categories)
(importDecl (extra.subsystems or {}));
extraSubsystemModules =
collectExtraModules (l.flatten _extraSubsystemModules);
# collect fetcher modules
extraFetcherModules =
l.mapAttrsToList
(name: fetcher: {
file = throwIfModuleNotPath fetcher;
extraArgs = {inherit name;};
})
(importDecl (extra.fetchers or {}));
in
extraSubsystemModules
// {
fetchers = extraFetcherModules;
};
# collect all extras
extra =
collectExtraModules
(l.map processOneExtra (l.flatten [(config.extra or [])]));
mapSubsystemModules = f: modules:
l.mapAttrs
(
subsystem: names:
l.mapAttrs
(name: module: f module)
names
)
modules;
# collect subsystem modules into a list
# ex: {rust.cargo-lock = cargo-lock; go.gomod2nix = gomod2nix;} -> [cargo-lock gomod2nix]
collectSubsystemModules = modules: let
allModules = l.flatten (l.map l.attrValues (l.attrValues modules));
hasModule = module: modules:
l.any
(
omodule:
(omodule.name == module.name)
&& (omodule.subsystem == module.subsystem)
)
modules;
in
l.foldl'
(acc: el:
if hasModule el acc
then acc
else acc ++ [el])
[]
allModules;
# create subsystem modules
# returns ex: {rust = {moduleName = module;}; go = {moduleName = module;};}
makeSubsystemModules = {
modulesCategory,
validator,
extraModules ? extra.${modulesCategory} or [],
defaults ? {},
}: let
callModule = file: extraArgs:
importModule {inherit file validator extraArgs;};
# extract builtin modules from subsystems directory
modulesBuiltin = l.flatten (
l.map
(
subsystem: let
dir = "${toString ../subsystems}/${subsystem}/${modulesCategory}";
moduleNames =
if l.pathExists dir
then dlib.dirNames dir
else [];
makeModule = name: {
file = "${dir}/${name}";
extraArgs = {inherit subsystem name;};
};
in
l.map makeModule moduleNames
)
dlib.subsystems
);
# import the modules
importedModules = l.filter (t: t.disabled or false == false) (
l.map
(
module:
(callModule module.file module.extraArgs)
// {inherit (module.extraArgs) subsystem name;}
)
(modulesBuiltin ++ extraModules)
);
# create the modules attrset
_modules =
l.foldl'
(acc: el: l.recursiveUpdate acc {"${el.subsystem}"."${el.name}" = el;})
{}
importedModules;
# add default module attribute to a subsystem if declared in `defaults`
modules =
l.mapAttrs
(
subsystem: modules:
if l.hasAttr subsystem defaults
then modules // {default = modules.${defaults.${subsystem}};}
else modules
)
_modules;
mapModules = f: mapSubsystemModules f modules;
in {inherit callModule mapModules modules;};
in {
inherit
importModule
makeSubsystemModules
collectSubsystemModules
mapSubsystemModules
extra
;
}

View File

@ -1,27 +0,0 @@
{
dlib,
lib,
}: let
l = lib // builtins;
# TODO
validator = module: true;
modules = dlib.modules.makeSubsystemModules {
inherit validator;
modulesCategory = "translators";
};
translators =
dlib.modules.mapSubsystemModules
(t:
t
// (lib.optionalAttrs (t.translate or null != null) {
translate = l.trace t dlib.warnIfIfd t t.translate;
}))
modules.modules;
mapTranslators = f: dlib.modules.mapSubsystemModules f translators;
in {
inherit translators mapTranslators;
callTranslator = modules.callModule;
}

View File

@ -1,9 +1,5 @@
{
config,
callPackageDream,
lib,
...
}: let
{config, ...}: let
lib = config.lib;
defaults = {
rust = "build-rust-package";
nodejs = "granular-nodejs";
@ -13,7 +9,6 @@
debian = "simple-debian";
racket = "simple-racket";
};
loader = b: b // {build = callPackageDream b.build {};};
funcs = config.functions.subsystem-loading;
collectedModules = funcs.collect "builders";
in {
@ -21,12 +16,6 @@ in {
# The user can add more translators by extending this attribute
builders = funcs.import_ collectedModules;
/*
translators wrapped with extra logic to add extra attributes,
like .translateBin for pure translators
*/
builderInstances = funcs.instantiate config.builders loader;
buildersBySubsystem =
lib.mapAttrs
(
@ -36,6 +25,6 @@ in {
default = builders.${defaults.${subsystem}};
}
)
(funcs.structureBySubsystem config.builderInstances);
(funcs.structureBySubsystem config.builders);
};
}

View File

@ -1,23 +1,17 @@
{
lib,
specialArgs,
...
}: let
{config, ...}: let
lib = config.lib;
t = lib.types;
in {
options = {
builders = lib.mkOption {
type = t.attrsOf (t.submoduleWith {
modules = [./builder/default.nix];
inherit specialArgs;
modules = [../interfaces.builder];
specialArgs = {framework = config;};
});
description = ''
builder module definitions
'';
};
builderInstances = lib.mkOption {
type = t.attrsOf t.anything;
};
buildersBySubsystem = lib.mkOption {
type = t.attrsOf (t.attrsOf t.anything);
};

View File

@ -1,9 +1,5 @@
{
config,
lib,
...
}: let
l = lib // builtins;
{config, ...}: let
l = config.lib // builtins;
translatorsWithDiscoverFunc =
l.filter

View File

@ -3,7 +3,6 @@
collectedModules = funcs.collect "discoverers";
in {
config = {
# The user can add more discoverers by extending this attribute
discoverers = funcs.import_ collectedModules;
discoverersBySubsystem = funcs.structureBySubsystem config.discoverers;

View File

@ -1,15 +1,12 @@
{
lib,
specialArgs,
...
}: let
{config, ...}: let
lib = config.lib;
t = lib.types;
in {
options = {
discoverers = lib.mkOption {
type = t.attrsOf (t.submoduleWith {
modules = [../interfaces.discoverer];
inherit specialArgs;
specialArgs = {framework = config;};
});
description = ''
discoverer module definitions

View File

@ -1,10 +1,6 @@
{
callPackageDream,
dlib,
lib,
...
}: let
l = lib // builtins;
{config, ...}: let
l = config.lib;
fetchersDir = ../../fetchers;
fetcherNames = l.attrNames (
l.filterAttrs
@ -14,12 +10,7 @@
fetcherModules =
l.genAttrs
fetcherNames
(
name:
import "${fetchersDir}/${name}" {
inherit dlib lib;
}
);
(name: import "${fetchersDir}/${name}" config);
in {
config = {
fetchers = fetcherModules;

View File

@ -1,9 +1,5 @@
{
lib,
specialArgs,
...
}: let
l = lib // builtins;
{config, ...}: let
l = config.lib // builtins;
t = l.types;
in {
options = {
@ -11,7 +7,7 @@ in {
type = t.attrsOf (
t.submoduleWith {
modules = [../interfaces.fetcher];
inherit specialArgs;
specialArgs = {framework = config;};
}
);
};

View File

@ -1,24 +1,16 @@
{
dream2nixConfig,
callPackageDream,
pkgs,
dlib,
lib,
utils,
apps,
}: let
topLevel = import ./top-level.nix {
inherit apps lib dlib utils pkgs dream2nixConfig;
};
evaledModules = lib.evalModules {
modules = [./top-level.nix] ++ (dream2nixConfig.modules or []);
# TODO: remove specialArgs once all functionality is moved to /src/modules
specialArgs = {
inherit
dream2nixConfig
callPackageDream
pkgs
dlib
utils
;
};
modules = [topLevel] ++ (dream2nixConfig.modules or []);
};
framework = evaledModules.config;

View File

@ -1,11 +1,5 @@
{
lib,
dlib,
utils,
config,
pkgs,
...
}: let
{config, ...}: let
inherit (config) lib pkgs utils;
defaultFetcher = config.functions.defaultFetcher;
inherit
(pkgs)

View File

@ -1,5 +1,5 @@
{lib, ...}: let
l = lib // builtins;
{config, ...}: let
l = config.lib // builtins;
t = l.types;
in {
options = {

View File

@ -1,9 +1,6 @@
{
lib,
dlib,
config,
...
}: let
{config, ...}: let
inherit (config) lib;
fetchers = config.fetchers;
fetchSource = config.functions.fetchers.fetchSource;
func = {

View File

@ -1,5 +1,5 @@
{lib, ...}: let
l = lib // builtins;
{config, ...}: let
l = config.lib // builtins;
t = l.types;
in {
options = {

View File

@ -1,11 +1,6 @@
{
config,
dream2nixConfig,
dlib,
lib,
...
}: let
l = lib // builtins;
{config, ...}: let
l = config.lib // builtins;
dlib = config.dlib;
discoverProjects = {
projects,
@ -47,7 +42,7 @@
getDreamLockPath = project: rootProject:
dlib.sanitizeRelativePath
"${dream2nixConfig.packagesDir}/${rootProject.name}/${project.relPath or ""}/dream-lock.json";
"${config.dream2nixConfig.packagesDir}/${rootProject.name}/${project.relPath or ""}/dream-lock.json";
applyProjectSettings = projects: settingsList: let
settingsListForProject = project:

View File

@ -1,5 +1,5 @@
{lib, ...}: let
l = lib // builtins;
{config, ...}: let
l = config.lib // builtins;
t = l.types;
in {
options.functions.discoverers = {

View File

@ -1,10 +1,6 @@
{
config,
lib,
...
}: let
{config, ...}: let
inherit (config) fetchers lib;
b = builtins;
fetchers = config.fetchers;
renderUrlArgs = kwargs: let
asStr =
@ -28,7 +24,7 @@ in {
fetcher = fetchers."${type}";
argsKeep = b.removeAttrs args ["reComputeHash"];
fetcherOutputs =
fetcher.outputsInstanced
fetcher.outputs
(b.removeAttrs argsKeep ["dir" "hash" "type"]);
in
argsKeep
@ -58,7 +54,7 @@ in {
}: let
fetcher = fetchers."${source.type}";
fetcherArgs = b.removeAttrs source ["dir" "hash" "type"];
fetcherOutputs = fetcher.outputsInstanced fetcherArgs;
fetcherOutputs = fetcher.outputs fetcherArgs;
maybeArchive = fetcherOutputs.fetched (source.hash or null);
in
if source ? dir
@ -136,7 +132,7 @@ in {
url = with parsed; "${proto2}://${path}${urlArgsFinal}";
fetcherOutputs = fetcher.outputsInstanced {
fetcherOutputs = fetcher.outputs {
inherit url;
};
in
@ -165,7 +161,7 @@ in {
args = parsed.kwargs // {inherit url;};
fetcherOutputs = fetcher.outputsInstanced (checkArgs fetcherName args);
fetcherOutputs = fetcher.outputs (checkArgs fetcherName args);
in
constructSource
(parsed.kwargs
@ -209,7 +205,7 @@ in {
(lib.elemAt params idx)
));
fetcherOutputs = fetcher.outputsInstanced (args // parsed.kwargs);
fetcherOutputs = fetcher.outputs (args // parsed.kwargs);
in
constructSource (args
// parsed.kwargs

View File

@ -1,5 +1,5 @@
{lib, ...}: let
l = lib // builtins;
{config, ...}: let
l = config.lib // builtins;
t = l.types;
in {
options = {

View File

@ -6,14 +6,8 @@ This recurses through the following directory structure to discover and load
modules:
/src/subsystems/{subsystem}/{module-type}/{module-name}
*/
{
config,
dlib,
callPackageDream,
...
}: let
lib = config.lib;
t = lib.types;
{config, ...}: let
inherit (config) dlib lib;
subsystemsDir = lib.toString ../../subsystems;
subsystems = dlib.dirNames subsystemsDir;
@ -42,7 +36,7 @@ modules:
import_ = collectedModules:
lib.mapAttrs
(name: description:
(import description.path {inherit dlib lib;})
(import description.path config)
// {inherit (description) name subsystem;})
(
lib.foldl'

View File

@ -1,8 +1,5 @@
{
config,
lib,
...
}: let
{framework, ...}: let
lib = framework.lib;
t = lib.types;
in {
options = {
@ -20,8 +17,8 @@ in {
description = "Subsystem of the builder.";
};
build = lib.mkOption {
type = t.functionTo (t.functionTo t.attrs);
default = _: _: {};
type = t.functionTo t.attrs;
default = _: {};
};
type = lib.mkOption {
type = t.enum [

View File

@ -1,8 +1,5 @@
{
config,
lib,
...
}: let
{framework, ...}: let
lib = framework.lib;
t = lib.types;
in {
options = {

View File

@ -1,6 +1,5 @@
{
imports = [
./interface.nix
./implementation.nix
];
}

View File

@ -1,9 +0,0 @@
{
config,
callPackageDream,
...
}: {
config = {
outputsInstanced = callPackageDream config.outputs {};
};
}

View File

@ -1,5 +1,5 @@
{lib, ...}: let
l = lib // builtins;
{framework, ...}: let
l = framework.lib // builtins;
t = l.types;
outputsOptions = {
@ -44,19 +44,11 @@ in {
type = t.functionTo t.attrs;
};
outputs = l.mkOption {
type = t.functionTo (t.functionTo (
t.submoduleWith {
modules = [outputsOptions];
}
));
};
outputsInstanced = l.mkOption {
type = t.functionTo (
t.submoduleWith {
modules = [outputsOptions];
}
);
readOnly = true;
};
};
}

View File

@ -0,0 +1,6 @@
{
imports = [
./interface.nix
./implementation.nix
];
}

View File

@ -0,0 +1,23 @@
{
config,
framework,
...
}: {
config = {
translateInstanced = args:
config.translate
(
(framework.functions.translators.makeTranslatorDefaultArgs
(config.extraArgs or {}))
// args
// (args.project.subsystemInfo or {})
// {
tree =
args.tree or (framework.dlib.prepareSourceTree {inherit (args) source;});
}
);
translateBin =
framework.functions.translators.wrapPureTranslator
{inherit (config) subsystem name;};
};
}

View File

@ -1,8 +1,5 @@
{
config,
lib,
...
}: let
{framework, ...}: let
lib = framework.lib;
t = lib.types;
in {
options = {
@ -46,11 +43,15 @@ in {
type = t.str;
};
translate = lib.mkOption {
type = t.nullOr (t.functionTo (t.functionTo (t.attrs)));
type = t.nullOr (t.functionTo t.attrs);
default = null;
};
translateInstanced = lib.mkOption {
type = t.nullOr (t.functionTo t.attrs);
readOnly = true;
};
translateBin = lib.mkOption {
type = t.nullOr (t.functionTo t.package);
type = t.nullOr t.package;
default = null;
};
type = lib.mkOption {

View File

@ -1,7 +1,10 @@
{
config,
apps,
dlib,
lib,
...
pkgs,
utils,
dream2nixConfig,
}: let
t = lib.types;
in {
@ -19,11 +22,27 @@ in {
./translators
];
options = {
apps = lib.mkOption {
type = t.raw;
};
lib = lib.mkOption {
type = t.raw;
};
dlib = lib.mkOption {
type = t.raw;
};
pkgs = lib.mkOption {
type = t.raw;
};
utils = lib.mkOption {
type = t.raw;
};
dream2nixConfig = lib.mkOption {
type = t.raw;
};
};
config = {
inherit apps dlib pkgs utils dream2nixConfig;
lib = lib // builtins;
};
}

View File

@ -1,12 +1,4 @@
{
config,
callPackageDream,
...
}: let
lib = config.lib;
t = lib.types;
# TODO: the makeTranslator logic should be moved somewhere to /src/modules
loader = (callPackageDream ../../subsystems/translators.nix {}).makeTranslator;
{config, ...}: let
funcs = config.functions.subsystem-loading;
collectedModules = funcs.collect "translators";
in {
@ -14,12 +6,6 @@ in {
# The user can add more translators by extending this attribute
translators = funcs.import_ collectedModules;
/*
translators wrapped with extra logic to add extra attributes,
like .translateBin for pure translators
*/
translatorInstances = funcs.instantiate config.translators loader;
translatorsBySubsystem = funcs.structureBySubsystem config.translatorInstances;
translatorsBySubsystem = funcs.structureBySubsystem config.translators;
};
}

View File

@ -1,24 +1,17 @@
{
config,
lib,
specialArgs,
...
}: let
{config, ...}: let
lib = config.lib;
t = lib.types;
in {
options = {
translators = lib.mkOption {
type = t.attrsOf (t.submoduleWith {
modules = [./interface-translator.nix];
inherit specialArgs;
modules = [../interfaces.translator];
specialArgs = {framework = config;};
});
description = ''
Translator module definitions
'';
};
translatorInstances = lib.mkOption {
type = t.attrsOf t.anything;
};
translatorsBySubsystem = lib.mkOption {
type = t.attrsOf (t.attrsOf t.anything);
};

View File

@ -1,18 +0,0 @@
{
dlib,
callPackageDream,
...
}: let
makeBuilder = builderModule:
builderModule
// {
build = callPackageDream builderModule.build {};
};
builders = dlib.builders.mapBuilders makeBuilder;
in {
inherit
makeBuilder
builders
;
}

View File

@ -1,14 +1,11 @@
{...}: {
{
lib,
pkgs,
...
}: {
type = "pure";
build = {
lib,
pkgs,
stdenv,
# dream2nix inputs
externals,
...
}: {
### FUNCTIONS
# AttrSet -> Bool) -> AttrSet -> [x]
getCyclicDependencies, # name: version: -> [ {name=; version=; } ]
@ -54,7 +51,7 @@
(src: src.original or src)
allDependencySources';
package = produceDerivation defaultPackageName (stdenv.mkDerivation {
package = produceDerivation defaultPackageName (pkgs.stdenv.mkDerivation {
name = defaultPackageName;
src = ":";
dontUnpack = true;

View File

@ -1,10 +1,8 @@
{
dlib,
lib,
pkgs,
utils,
...
}: let
l = lib // builtins;
in {
}: {
type = "impure";
# A derivation which outputs a single executable at `$out`.
@ -18,19 +16,17 @@ in {
# by the input parameter `outFile`.
# The output file must contain the dream lock data encoded as json.
# See /src/specifications/dream-lock-example.json
translateBin = {
# dream2nix utils
utils,
# nixpkgs dependencies
bash,
coreutils,
jq,
writeScriptBin,
nix,
callPackage,
python3,
...
}:
translateBin = let
inherit
(pkgs)
bash
coreutils
jq
nix
callPackage
python3
;
in
utils.writePureShellScript
[
bash

View File

@ -1,36 +0,0 @@
{
lib,
dlib,
callPackageDream,
...
}: let
l = lib // builtins;
inherit (dlib.modules) collectSubsystemModules;
builders = callPackageDream ./builders.nix {};
translators = callPackageDream ./translators.nix {};
discoverers = callPackageDream ./discoverers.nix {};
# maps modules
# ex: {rust = <translators attrset>;} -> {rust.translators = <translators attrset>;}
mapModules = modules: name:
l.mapAttrs'
(
subsystem: modules:
l.nameValuePair subsystem {${name} = modules;}
)
modules;
modules = [
(mapModules builders.builders "builders")
(mapModules translators.translators "translators")
(mapModules discoverers.discoverers "discoverers")
];
allModules = l.foldl' l.recursiveUpdate {} modules;
in
allModules
// {
allTranslators = collectSubsystemModules translators.translators;
allBuilders = collectSubsystemModules builders.builders;
allDiscoverers = collectSubsystemModules discoverers.discoverers;
}

View File

@ -1 +0,0 @@
{dlib, ...}: dlib.discoverers

View File

@ -1,6 +1,6 @@
{
dlib,
lib,
dlib,
...
}: let
l = lib // builtins;

View File

@ -18,7 +18,7 @@ in {
})
];
translate = {translatorName, ...}: {
translate = {
project,
tree,
...
@ -180,8 +180,6 @@ in {
in
dlib.simpleTranslate2.translate
({...}: {
inherit translatorName;
# relative path of the project within the source tree.
location = project.relPath;

View File

@ -1,23 +1,17 @@
{
dlib,
lib,
...
}: {
framework: {
type = "impure";
# the input format is specified in /specifications/translator-call-example.json
# this script receives a json file including the input paths and specialArgs
translateBin = {
# dream2nix utils
subsystems,
utils,
# nixpkgs dependencies
coreutils,
jq,
rustPlatform,
...
}:
utils.writePureShellScript
translateBin = let
inherit
(framework.pkgs)
coreutils
jq
rustPlatform
;
in
framework.utils.writePureShellScript
[
coreutils
jq
@ -47,7 +41,7 @@
popd
if [ $cargoResult -eq 0 ]; then
${subsystems.rust.translators.cargo-lock.translateBin} $TMPDIR/newJsonInput
${framework.translators.cargo-lock.translateBin} $TMPDIR/newJsonInput
else
echo "cargo failed to generate the lockfile"
exit 1
@ -56,7 +50,7 @@
# inherit options from cargo-lock translator
extraArgs =
dlib.translators.translators.rust.cargo-lock.extraArgs
framework.translators.cargo-lock.extraArgs
// {
cargoArgs = {
description = "Additional arguments for Cargo";

View File

@ -1,27 +1,26 @@
{
dlib,
lib,
pkgs,
apps,
utils,
translators,
...
}: {
type = "impure";
# the input format is specified in /specifications/translator-call-example.json
# this script receives a json file including the input paths and specialArgs
translateBin = {
# dream2nix utils
apps,
subsystems,
utils,
# nixpkgs dependencies
coreutils,
curl,
gnutar,
gzip,
jq,
moreutils,
rustPlatform,
...
}:
translateBin = let
inherit
(pkgs)
coreutils
curl
gnutar
gzip
jq
moreutils
rustPlatform
;
in
utils.writePureShellScript
[
coreutils
@ -67,9 +66,9 @@
# we don't need to run cargo-toml translator if Cargo.lock exists
if [ -f "$TMPDIR/source/Cargo.lock" ]; then
${subsystems.rust.translators.cargo-lock.translateBin} $TMPDIR/newJsonInput
${translators.cargo-lock.translateBin} $TMPDIR/newJsonInput
else
${subsystems.rust.translators.cargo-toml.translateBin} $TMPDIR/newJsonInput
${translators.cargo-toml.translateBin} $TMPDIR/newJsonInput
fi
# add main package source info to dream-lock.json
@ -85,16 +84,5 @@
'';
# inherit options from cargo-lock translator
extraArgs =
dlib.translators.translators.rust.cargo-lock.extraArgs
// {
cargoArgs = {
description = "Additional arguments for Cargo";
type = "argument";
default = "";
examples = [
"--verbose"
];
};
};
extraArgs = translators.cargo-toml.extraArgs;
}

View File

@ -1,122 +0,0 @@
{
coreutils,
dlib,
jq,
lib,
nix,
pkgs,
python3,
callPackageDream,
externals,
dream2nixWithExternals,
utils,
config,
configFile,
framework,
...
}: let
b = builtins;
l = lib // builtins;
# adds a translateBin to a pure translator
wrapPureTranslator = {
subsystem,
name,
}: let
bin =
utils.writePureShellScript
[
coreutils
jq
nix
python3
]
''
jsonInputFile=$(realpath $1)
outputFile=$(realpath -m $(jq '.outputFile' -c -r $jsonInputFile))
pushd $TMPDIR
nix eval \
--option experimental-features "nix-command flakes"\
--show-trace --impure --raw --expr "
let
dream2nix = import ${dream2nixWithExternals} {
config = ${configFile};
};
translatorArgs =
(builtins.fromJSON
(builtins.unsafeDiscardStringContext (builtins.readFile '''$1''')));
dreamLock' =
dream2nix.subsystems.${subsystem}.translators.${name}.translate
translatorArgs;
# simpleTranslate2 puts dream-lock in result
dreamLock = dreamLock'.result or dreamLock';
in
dream2nix.utils.dreamLock.toJSON
# don't use nix to detect cycles, this will be more efficient in python
(dreamLock // {
_generic = builtins.removeAttrs dreamLock._generic [ \"cyclicDependencies\" ];
})
" | python3 ${../apps/cli/format-dream-lock.py} > out
tmpOut=$(realpath out)
popd
mkdir -p $(dirname $outputFile)
cp $tmpOut $outputFile
'';
in
bin.overrideAttrs (old: {
name = "translator-${subsystem}-pure-${name}";
});
makeTranslator = translatorModule: let
translator =
translatorModule
# for pure translators
# - import the `translate` function
# - generate `translateBin`
// (lib.optionalAttrs (translatorModule.translate or null != null) {
translate = let
translateOriginal = callPackageDream translatorModule.translate {
translatorName = translatorModule.name;
};
in
args:
translateOriginal
(
(framework.functions.translators.makeTranslatorDefaultArgs
(translatorModule.extraArgs or {}))
// args
// (args.project.subsystemInfo or {})
// {
tree =
args.tree or (dlib.prepareSourceTree {inherit (args) source;});
}
);
translateBin =
wrapPureTranslator
{inherit (translatorModule) subsystem name;};
})
# for impure translators:
# - import the `translateBin` function
// (lib.optionalAttrs (translatorModule.translateBin or null != null) {
translateBin =
callPackageDream translatorModule.translateBin
{
translatorName = translatorModule.name;
};
});
in
translator;
translators = framework.translatorsBySubsystem;
in {
inherit
translators
makeTranslator
;
}

View File

@ -19,7 +19,6 @@
callPackageDream,
dream2nixWithExternals,
externalSources,
subsystems,
config,
configFile,
framework,