From d8300cb90910eaadf7ff90971a1cc8ad920ac9fe Mon Sep 17 00:00:00 2001 From: DavHau Date: Tue, 22 Feb 2022 18:23:49 +0700 Subject: [PATCH] Always build from dream-lock.json if exists - add an invalidationHash to dream-lock.json - always evaluate dream-lock.json if exists - warn user if dream-lock.json doesn't exist --- src/apps/cli/commands/add.py | 4 +- src/default.nix | 6 +- src/lib.nix | 77 ++++++++++++++++++++--- src/specifications/dream-lock-schema.json | 1 + src/utils/config.nix | 3 +- src/utils/default.nix | 17 +++++ 6 files changed, 93 insertions(+), 15 deletions(-) diff --git a/src/apps/cli/commands/add.py b/src/apps/cli/commands/add.py index 716f47fa..4430817b 100644 --- a/src/apps/cli/commands/add.py +++ b/src/apps/cli/commands/add.py @@ -63,6 +63,7 @@ class AddCommand(Command): ), option("force", None, "override existing files", flag=True), option("no-default-nix", None, "create default.nix", flag=True), + option("invalidation-hash", None, "invalidation hash to attach", flag=False), ] def handle(self): @@ -383,6 +384,7 @@ class AddCommand(Command): ) + [ f"--arg {n}={v}" for n, v in specified_extra_args.items() ]) + lock['_generic']['invalidationHash'] = self.option('invalidation-hash') def calc_outputs(self, main_package_dir_name, packages_root): if self.option('target'): @@ -504,7 +506,7 @@ class AddCommand(Command): # assume defaults for unspecified extra args specified_extra_args.update( - {n: (True if v['type'] == 'flag' else v['default']) \ + {n: (False if v['type'] == 'flag' else v['default']) \ for n, v in translator['extraArgs'].items() \ if n not in specified_extra_args} ) diff --git a/src/default.nix b/src/default.nix index f53f0dfa..7a63bb8d 100644 --- a/src/default.nix +++ b/src/default.nix @@ -15,9 +15,9 @@ # default to empty dream2nix config config ? - # if called via CLI, load cnfig via env - if builtins ? getEnv && builtins.getEnv "d2nConfigFile" != "" then - builtins.toPath (builtins.getEnv "d2nConfigFile") + # if called via CLI, load config via env + if builtins ? getEnv && builtins.getEnv "dream2nixConfig" != "" then + builtins.toPath (builtins.getEnv "dream2nixConfig") # load from default directory else {}, diff --git a/src/lib.nix b/src/lib.nix index b7783004..a15966d2 100644 --- a/src/lib.nix +++ b/src/lib.nix @@ -110,6 +110,7 @@ let riseAndShineFunc = { + pname, pkgs ? null, source, systems ? [], @@ -121,12 +122,12 @@ let config = args.config or ((import ./utils/config.nix).loadConfig {}); - argsForward = b.removeAttrs args [ "config" "pkgs" "systems" ]; + argsForward = b.removeAttrs args [ "config" "pname" "pkgs" "systems" ]; allPkgs = makeNixpkgs pkgs systems; dream2nixFor = - lib.mapAttrs (dream2nixForSystem {}) allPkgs; + lib.mapAttrs (dream2nixForSystem config) allPkgs; allBuilderOutputs = lib.mapAttrs @@ -139,18 +140,65 @@ let translatorName = args.translator or null; }; - result = translator: - dream2nix.riseAndShine (argsForward // { + invalidationHash = dream2nix.utils.calcInvalidationHash { + inherit source translatorArgs; + translator = translatorFound.name; + }; + + dreamLockJsonPath = with config; + "${projectRoot}/${packagesDir}/${pname}/dream-lock.json"; + + dreamLock = dream2nix.utils.readDreamLock { + dreamLock = dreamLockJsonPath; + }; + + dreamLockExistsAndValid = + b.pathExists dreamLockJsonPath + && dreamLock.lock._generic.invalidationHash or "" == invalidationHash; + + result = translator: args: + dream2nix.riseAndShine (argsForward // args // { # TODO: this triggers the translator finding routine a second time translator = translatorFound.name; }); in - if b.elem translatorFound.type [ "pure" "ifd" ] then - result translatorFound - else + + if dreamLockExistsAndValid then + # we need to override the source here as it is coming from + # a flake input + let + defaultPackage = dreamLock.lock._generic.defaultPackage; + defaultPackageVersion = + dreamLock.lock._generic.packages."${defaultPackage}"; + in + result translatorFound { + source = dreamLockJsonPath; + sourceOverrides = oldSources: { + "${defaultPackage}"."${defaultPackageVersion}" = + args.source; + }; + } + + else if b.elem translatorFound.type [ "pure" "ifd" ] then + # warn the user about potentially slow on-the-fly evaluation b.trace '' - Some information is missing to build this project reproducibly. - Please execute nix run .#resolve to resolve impurities. + ${"\n"} + The dream-lock.json for input '${pname}' doesn't exist or is outdated. + ...Falling back to on-the-fly evaluation (possibly slow). + To speed up future evalutations run once: + nix run .#resolve + '' + result translatorFound {} + + else + # print error because impure translation is required first. + # continue the evaluation anyways, as otherwise we won't have + # the `resolve` app + b.trace '' + ${"\n"} + ERROR: + Some information is missing to build this project reproducibly. + Please execute nix run .#resolve to resolve all impurities. '' {}) @@ -178,11 +226,20 @@ let resolve.program = let utils = (dream2nixFor."${system}".utils); + + # TODO: Too many calls to findOneTranslator. + # -> make findOneTranslator system independent + translatorFound = + dream2nixFor."${system}".translators.findOneTranslator { + inherit source; + translatorName = args.translator or null; + }; in b.toString (utils.makePackageLockScript { - inherit source translator translatorArgs; + inherit source translatorArgs; packagesDir = config.packagesDir; + translator = translatorFound.name; }); }); }; diff --git a/src/specifications/dream-lock-schema.json b/src/specifications/dream-lock-schema.json index ad864899..362def64 100644 --- a/src/specifications/dream-lock-schema.json +++ b/src/specifications/dream-lock-schema.json @@ -154,6 +154,7 @@ "type": "object", "properties": { "defaultPackage": { "type": ["string", "null"] }, + "invalidationHash": { "type": ["string", "null"] }, "packages": { "type": "object" }, "sourcesAggregatedHash": { "type": ["string", "null"] }, "subsystem": { "type": "string" }, diff --git a/src/utils/config.nix b/src/utils/config.nix index eb5d50d0..97614864 100644 --- a/src/utils/config.nix +++ b/src/utils/config.nix @@ -23,7 +23,8 @@ let defaults = { overridesDirs = []; packagesDir = null; - repoName = "this repo"; + projectRoot = throw "projectRoot not specified in dream2nix config"; + repoName = null; }; in defaults // config; diff --git a/src/utils/default.nix b/src/utils/default.nix index 0879a0e6..b0005e06 100644 --- a/src/utils/default.nix +++ b/src/utils/default.nix @@ -198,6 +198,19 @@ rec { recursiveUpdateUntilDepth = depth: lhs: rhs: lib.recursiveUpdateUntil (path: l: r: (b.length path) > depth) lhs rhs; + # calculate an invalidation hash for given source translation inputs + calcInvalidationHash = + { + source, + translator, + translatorArgs, + }: + b.hashString "sha256" '' + ${source} + ${translator} + ${b.toString + (lib.mapAttrsToList (k: v: "${k}=${v}") translatorArgs)} + ''; # a script that produces and dumps the dream-lock json for a given source makePackageLockScript = @@ -210,10 +223,14 @@ rec { writePureShellScript [] '' + cd $WORKDIR ${apps.cli.program} add ${source} \ --force \ --no-default-nix \ --translator ${translator} \ + --invalidation-hash ${calcInvalidationHash { + inherit source translator translatorArgs; + }} \ --packages-root $WORKDIR/${packagesDir} \ ${lib.concatStringsSep " \\\n" (lib.mapAttrsToList