From 2772c26105737e3a90a503c3f569252cb2449df9 Mon Sep 17 00:00:00 2001 From: DavHau Date: Tue, 22 Feb 2022 15:04:16 +0700 Subject: [PATCH] integrate impure translation with flakes Now if dream2nix is used via flakes and impure translation is required, the user will be isntructed to `nix run ...` something to generate the missing dream-lock. --- src/default.nix | 27 +++++++---- src/lib.nix | 91 +++++++++++++++++++++++++------------ src/translators/default.nix | 60 +++++++++++++++++++++++- src/utils/default.nix | 24 +++++++++- 4 files changed, 162 insertions(+), 40 deletions(-) diff --git a/src/default.nix b/src/default.nix index 14ca2a98..f53f0dfa 100644 --- a/src/default.nix +++ b/src/default.nix @@ -186,6 +186,7 @@ let makeDreamLockForSource = { source, + translator ? null, translatorArgs ? {}, }@args: let @@ -201,17 +202,23 @@ let source = fetchers.fetchSource { source = sourceSpec; }; - translatorsForSource = translators.translatorsForInput { - inputFiles = []; - inputDirectories = [ source ]; - }; - t = let - trans = b.filter (t: t.compatible && b.elem t.type [ "pure" "ifd" ]) translatorsForSource; + translator = translators.findOneTranslator { + inherit source; + translatorName = args.translator or null; + }; + in - if trans != [] then lib.elemAt trans 0 else - throw "Could not find a suitable translator for input"; + if b.elem translator.type [ "pure" "ifd" ] then + translator + else + throw '' + All comaptible translators are impure and therefore require + pre-processing the input before evaluation. + Use the CLI to add this package: + nix run .# -- add ... + ''; dreamLock' = translators.translators."${t.subsystem}"."${t.type}"."${t.name}".translate (translatorArgs // { @@ -359,6 +366,7 @@ let sourceOverrides ? oldSources: {}, packageOverrides ? {}, builderArgs ? {}, + translator ? null, translatorArgs ? {}, }@args: @@ -369,8 +377,9 @@ let if ( lib.isAttrs args.source && args.source ? _generic && args.source ? _subsytem ) || lib.hasSuffix "dream-lock.json" source then args.source + # input is a source tree -> generate the dream-lock else - makeDreamLockForSource { inherit source translatorArgs; }; + makeDreamLockForSource { inherit source translator translatorArgs; }; # parse dreamLock dreamLockLoaded = utils.readDreamLock { dreamLock = dreamLock'; }; diff --git a/src/lib.nix b/src/lib.nix index 47f86178..b7783004 100644 --- a/src/lib.nix +++ b/src/lib.nix @@ -84,30 +84,17 @@ let allPkgs = makeNixpkgs pkgs systems; - forAllSystems = f: - lib.mapAttrs f allPkgs; + forAllSystems = f: lib.mapAttrs f allPkgs; dream2nixFor = forAllSystems (dream2nixForSystem config); in { - riseAndShine = riseAndShineArgs: - let - allBuilderOutputs = - lib.mapAttrs - (system: pkgs: - dream2nixFor."${system}".riseAndShine riseAndShineArgs) - allPkgs; - flakifiedOutputs = - lib.mapAttrsToList - (system: outputs: flakifyBuilderOutputs system outputs) - allBuilderOutputs; - - in - b.foldl' - (allOutputs: output: lib.recursiveUpdate allOutputs output) - {} - flakifiedOutputs; + riseAndShine = rArgs: riseAndShineFunc + ( + { inherit config pkgs systems; } + // rArgs + ); apps = forAllSystems @@ -121,15 +108,20 @@ let }; - riseAndShine = + riseAndShineFunc = { pkgs ? null, + source, systems ? [], + translator ? null, + translatorArgs ? {}, ... }@args: let - argsForward = b.removeAttrs args [ "pkgs" "systems" ]; + config = args.config or ((import ./utils/config.nix).loadConfig {}); + + argsForward = b.removeAttrs args [ "config" "pkgs" "systems" ]; allPkgs = makeNixpkgs pkgs systems; @@ -139,21 +131,64 @@ let allBuilderOutputs = lib.mapAttrs (system: pkgs: - dream2nixFor."${system}".riseAndShine argsForward) + let + dream2nix = dream2nixFor."${system}"; + + translatorFound = dream2nix.translators.findOneTranslator { + inherit source; + translatorName = args.translator or null; + }; + + result = translator: + dream2nix.riseAndShine (argsForward // { + # TODO: this triggers the translator finding routine a second time + translator = translatorFound.name; + }); + in + if b.elem translatorFound.type [ "pure" "ifd" ] then + result translatorFound + else + b.trace '' + Some information is missing to build this project reproducibly. + Please execute nix run .#resolve to resolve impurities. + '' + {}) + allPkgs; - flakifiedOutputs = + flakifiedOutputsList = lib.mapAttrsToList (system: outputs: flakifyBuilderOutputs system outputs) allBuilderOutputs; + flakeOutputs = + b.foldl' + (allOutputs: output: lib.recursiveUpdate allOutputs output) + {} + flakifiedOutputsList; + + forAllSystems = f: b.mapAttrs f allPkgs; + in - b.foldl' - (allOutputs: output: lib.recursiveUpdate allOutputs output) - {} - flakifiedOutputs; + lib.recursiveUpdate + flakeOutputs + { + apps = forAllSystems (system: pkgs: { + resolve.type = "app"; + resolve.program = + let + utils = (dream2nixFor."${system}".utils); + in + b.toString + (utils.makePackageLockScript { + inherit source translator translatorArgs; + packagesDir = config.packagesDir; + }); + }); + }; in { - inherit init riseAndShine; + inherit init; + riseAndShine = riseAndShineFunc; } diff --git a/src/translators/default.nix b/src/translators/default.nix index 8c385734..cc451179 100644 --- a/src/translators/default.nix +++ b/src/translators/default.nix @@ -92,8 +92,23 @@ let ) ); - # flat list of all translators - translatorsList = lib.collect (v: v ? translateBin) translators; + # flat list of all translators sorted by priority (pure translators first) + translatorsList = + let + list = lib.collect (v: v ? translateBin) translators; + prio = translator: + if translator.type == "pure" then + 0 + else if translator.type == "ifd" then + 1 + else if translator.type == "impure" then + 2 + else + 3; + in + b.sort + (a: b: (prio a) < (prio b)) + list; # returns the list of translators including their special args # and adds a flag `compatible` to each translator indicating @@ -170,9 +185,50 @@ let ) extraArgsDef; + + # return one compatible translator or throw error + findOneTranslator = + { + source, + translatorName ? null, + }@args: + let + translatorsForSource = translatorsForInput { + inputFiles = []; + inputDirectories = [ source ]; + }; + + nameFilter = + if translatorName != null then + (translator: translator.name == translatorName) + else + (translator: true); + + compatibleTranslators = + let + result = + b.filter + (t: t.compatible) + translatorsForSource; + in + if result == [] then + throw "Could not find a compatible translator for input" + else + result; + + translator = + lib.findFirst + nameFilter + (throw ''Specified translator ${translatorName} not found or incompatible'') + compatibleTranslators; + + in + translator; + in { inherit + findOneTranslator translators translatorsForInput translatorsForInputRecursive diff --git a/src/utils/default.nix b/src/utils/default.nix index 5740f046..f8019cbd 100644 --- a/src/utils/default.nix +++ b/src/utils/default.nix @@ -10,6 +10,7 @@ writeScript, # dream2nix inputs + apps, callPackageDream, externalSources, ... @@ -128,6 +129,7 @@ rec { export PATH="${lib.makeBinPath availablePrograms}" export NIX_PATH=nixpkgs=${pkgs.path} + export WORKDIR="$PWD" tmpdir=$(${coreutils}/bin/mktemp -d) cd $tmpdir @@ -192,8 +194,28 @@ rec { satisfiesSemver = poetry2nixSemver.satisfiesSemver; - # like nixpkgs recursiveUpdateUntil, but the depth of the + # like nixpkgs recursiveUpdateUntil, but with the depth as a stop condition recursiveUpdateUntilDepth = depth: lhs: rhs: lib.recursiveUpdateUntil (path: l: r: (b.length path) > depth) lhs rhs; + + # a script that produces and dumps the dream-lock json for a given source + makePackageLockScript = + { + packagesDir, + source, + translator, + translatorArgs, + }: + writePureShellScript + [] + '' + ${apps.cli.program} add ${source} \ + --translator ${translator} \ + --packages-root $WORKDIR/${packagesDir} \ + ${lib.concatStringsSep " \\\n" + (lib.mapAttrsToList + (key: val: "--arg ${key}=${val}") + translatorArgs)} + ''; }