From 65120af9f691c0d500f95cae172cc532d3331eb7 Mon Sep 17 00:00:00 2001 From: Yusuf Bera Ertan Date: Thu, 6 Oct 2022 19:11:06 +0300 Subject: [PATCH 01/23] wip --- src/default.nix | 12 +- src/fetchers/combined-fetcher.nix | 228 ----------------- src/fetchers/default-fetcher.nix | 70 ----- src/fetchers/default.nix | 240 ------------------ src/lib.nix | 3 +- src/lib/builders.nix | 32 --- src/lib/default.nix | 8 - src/lib/discoverers.nix | 147 ----------- src/lib/fetchers.nix | 40 --- src/lib/modules.nix | 197 -------------- src/lib/translators.nix | 27 -- src/modules/builders/implementation.nix | 17 +- src/modules/builders/interface.nix | 14 +- .../implementation.nix | 8 +- src/modules/discoverers/implementation.nix | 1 - src/modules/discoverers/interface.nix | 9 +- src/modules/fetchers/implementation.nix | 17 +- src/modules/fetchers/interface.nix | 10 +- src/modules/framework.nix | 18 +- .../implementation.nix | 10 +- .../functions.combined-fetcher/interface.nix | 4 +- .../implementation.nix | 9 +- .../functions.default-fetcher/interface.nix | 4 +- .../functions.discoverers/implementation.nix | 13 +- .../functions.discoverers/interface.nix | 4 +- .../functions.fetchers/implementation.nix | 18 +- src/modules/functions.fetchers/interface.nix | 4 +- .../implementation.nix | 12 +- .../default.nix | 0 .../interface.nix | 11 +- .../interfaces.discoverer/interface.nix | 7 +- src/modules/interfaces.fetcher/default.nix | 1 - .../interfaces.fetcher/implementation.nix | 9 - src/modules/interfaces.fetcher/interface.nix | 12 +- src/modules/interfaces.translator/default.nix | 6 + .../interfaces.translator/implementation.nix | 23 ++ .../interface.nix} | 15 +- src/modules/top-level.nix | 23 +- src/modules/translators/implementation.nix | 18 +- src/modules/translators/interface.nix | 15 +- src/subsystems/builders.nix | 18 -- .../debian/builders/simple-debian/default.nix | 15 +- .../translators/debian-binary/default.nix | 32 +-- src/subsystems/default.nix | 36 --- src/subsystems/discoverers.nix | 1 - .../default.nix | 0 .../rust/discoverers/cargo/default.nix | 2 +- .../rust/translators/cargo-lock/default.nix | 4 +- .../rust/translators/cargo-toml/default.nix | 30 +-- .../rust/translators/crates-io/default.nix | 50 ++-- src/subsystems/translators.nix | 122 --------- src/utils/default.nix | 1 - 52 files changed, 181 insertions(+), 1446 deletions(-) delete mode 100644 src/fetchers/combined-fetcher.nix delete mode 100644 src/fetchers/default-fetcher.nix delete mode 100644 src/fetchers/default.nix delete mode 100644 src/lib/builders.nix delete mode 100644 src/lib/discoverers.nix delete mode 100644 src/lib/fetchers.nix delete mode 100644 src/lib/modules.nix delete mode 100644 src/lib/translators.nix rename src/modules/{builders/builder => interfaces.builder}/default.nix (100%) rename src/modules/{builders/builder => interfaces.builder}/interface.nix (83%) delete mode 100644 src/modules/interfaces.fetcher/implementation.nix create mode 100644 src/modules/interfaces.translator/default.nix create mode 100644 src/modules/interfaces.translator/implementation.nix rename src/modules/{translators/interface-translator.nix => interfaces.translator/interface.nix} (84%) delete mode 100644 src/subsystems/builders.nix delete mode 100644 src/subsystems/default.nix delete mode 100644 src/subsystems/discoverers.nix rename src/subsystems/racket/discoverers/{default => discoverer-racket}/default.nix (100%) delete mode 100644 src/subsystems/translators.nix diff --git a/src/default.nix b/src/default.nix index f41bdd94..045c7b53 100644 --- a/src/default.nix +++ b/src/default.nix @@ -51,7 +51,7 @@ in let }; framework = import ./modules/framework.nix { - inherit lib dlib callPackageDream pkgs utils; + inherit apps lib dlib pkgs utils; dream2nixConfig = config; }; @@ -77,7 +77,6 @@ in let inherit dream2nixWithExternals; inherit utils; inherit nix; - inherit subsystems; dream2nixInterface = { inherit makeOutputsForDreamLock @@ -107,8 +106,6 @@ in let # updater modules to find newest package versions updaters = callPackageDream ./updaters {}; - subsystems = callPackageDream ./subsystems {}; - externals = { devshell = { makeShell = import "${externalSources.devshell}/modules" pkgs; @@ -434,7 +431,7 @@ in let settings ? [], } @ args: let getTranslator = translatorName: - framework.translatorInstances.${translatorName}; + framework.translators.${translatorName}; isImpure = project: translatorName: (getTranslator translatorName).type == "impure"; @@ -499,7 +496,7 @@ in let l.forEach projectsPureUnresolved (proj: let translator = getTranslator proj.translator; - dreamLock'' = translator.translate { + dreamLock'' = translator.translateInstanced { inherit source tree discoveredProjects; project = proj; }; @@ -650,7 +647,7 @@ in let impureDiscoveredProjects = l.filter (proj: - framework.translatorInstances."${proj.translator}".type + framework.translators."${proj.translator}".type == "impure") discoveredProjects; @@ -727,6 +724,5 @@ in { utils makeOutputsForDreamLock makeOutputs - subsystems ; } diff --git a/src/fetchers/combined-fetcher.nix b/src/fetchers/combined-fetcher.nix deleted file mode 100644 index ef8e0f30..00000000 --- a/src/fetchers/combined-fetcher.nix +++ /dev/null @@ -1,228 +0,0 @@ -# this fetcher takes an attrset of sources and combines all contained FODs -# to one large FOD. Non-FOD sources like derivations and store paths are -# not touched -{ - bash, - async, - coreutils, - lib, - nix, - stdenv, - writeScript, - # dream2nix - defaultFetcher, - dlib, - utils, - ... -}: { - # sources attrset from dream lock - sources, - sourcesAggregatedHash, - sourceOverrides, - ... -} @ args: let - b = builtins; - l = lib // builtins; - - # resolve to individual fetcher calls - defaultFetched = (defaultFetcher args).fetchedSources; - - isFOD = drv: - lib.all - (attr: drv ? "${attr}") - ["outputHash" "outputHashAlgo" "outputHashMode"]; - - drvArgs = drv: - (drv.overrideAttrs (args: { - passthru.originalArgs = args; - })) - .originalArgs; - - # extract the arguments from the individual fetcher calls - FODArgsAll = let - FODArgsAll' = - lib.mapAttrs - ( - name: versions: - lib.mapAttrs - (version: fetched: - # handle FOD sources - if isFOD fetched - then - (drvArgs fetched) - // { - isOriginal = false; - outPath = let - sanitizedName = l.strings.sanitizeDerivationName name; - in "${sanitizedName}/${version}/${fetched.name}"; - } - # handle already extracted sources - else if fetched ? original && isFOD fetched.original - then - (drvArgs fetched.original) - // { - isOriginal = true; - outPath = let - sanitizedName = l.strings.sanitizeDerivationName name; - in "${sanitizedName}/${version}/${fetched.original.name}"; - } - # handle path sources - else if lib.isString fetched - then "ignore" - # handle store path sources - else if lib.isStorePath fetched - then "ignore" - # handle unknown sources - else if fetched == "unknown" - then "ignore" - # error out on unknown source types - else - throw '' - Error while generating FOD fetcher for combined sources. - Cannot classify source of ${name}#${version}. - '') - versions - ) - defaultFetched; - in - lib.filterAttrs - (name: versions: versions != {}) - (lib.mapAttrs - (name: versions: - lib.filterAttrs - (version: fetcherArgs: fetcherArgs != "ignore") - versions) - FODArgsAll'); - - FODArgsAllList = - lib.flatten - (lib.mapAttrsToList - (name: versions: - b.attrValues versions) - FODArgsAll); - - # convert arbitrary types to string, like nix does with derivation arguments - toString' = x: - if lib.isBool x - then - if x - then "1" - else "" - else if lib.isList x - then ''"${lib.concatStringsSep " " (lib.forEach x (y: toString' y))}"'' - else if x == null - then "" - else b.toJSON x; - - # set up nix build env for signle item - itemScript = fetcherArgs: '' - - # export arguments for builder - ${lib.concatStringsSep "\n" (lib.mapAttrsToList (argName: argVal: '' - export ${argName}=${ - lib.replaceStrings ["$" ''\n''] [''\$'' "\n"] (toString' argVal) - } - '') - fetcherArgs)} - - # run builder - bash ${fetcherArgs.builder} - ''; - - mkScriptForItem = fetcherArgs: '' - # configure $out - OUT_ORIG=$out - export out=$OUT_ORIG/${fetcherArgs.outPath} - mkdir -p $(dirname $out) - - # set up TMP and TMPDIR - workdir=$(mktemp -d) - TMP=$workdir/TMP - TMPDIR=$TMP - mkdir -p $TMP - - # do the work - pushd $workdir - ${itemScript fetcherArgs} - popd - rm -r $workdir - ''; - - # builder which wraps several other FOD builders - # and executes these after each other inside a single build - builder = writeScript "multi-source-fetcher" '' - #!${bash}/bin/bash - export PATH=${coreutils}/bin:${bash}/bin - - mkdir $out - - S="/$TMP/async_socket" - async=${async}/bin/async - $async -s="$S" server --start -j40 - - # remove if resolved: https://github.com/ctbur/async/issues/6 - sleep 1 - - ${lib.concatStringsSep "\n" - (b.map - (fetcherArgs: '' - $async -s="$S" cmd -- bash -c '${mkScriptForItem fetcherArgs}' - '') - FODArgsAllList)} - - $async -s="$S" wait - - ${lib.concatStringsSep "\n" - (b.map - (fetcherArgs: '' - if [ ! -e "$out/${fetcherArgs.outPath}" ]; then - echo "builder for ${fetcherArgs.name} terminated without creating out path: ${fetcherArgs.outPath}" - exit 1 - fi - '') - FODArgsAllList)} - - echo "FOD_HASH=$(${nix}/bin/nix --extra-experimental-features "nix-command flakes" hash path $out)" - ''; - - FODAllSources = let - nativeBuildInputs' = - lib.unique - (lib.foldl (a: b: a ++ b) [] - (b.map - (fetcherArgs: (fetcherArgs.nativeBuildInputs or [])) - FODArgsAllList)); - in - stdenv.mkDerivation rec { - name = "sources-combined"; - inherit builder; - nativeBuildInputs = - nativeBuildInputs' - ++ [ - coreutils - ]; - outputHashAlgo = "sha256"; - outputHashMode = "recursive"; - outputHash = sourcesAggregatedHash; - }; -in { - FOD = FODAllSources; - - fetchedSources = - lib.mapAttrs - (name: versions: - lib.mapAttrs - (version: source: - if FODArgsAll ? "${name}"."${version}".outPath - then - if FODArgsAll ? "${name}"."${version}".isOriginal - then - utils.extractSource { - source = "${FODAllSources}/${FODArgsAll."${name}"."${version}".outPath}"; - name = l.strings.sanitizeDerivationName name; - } - else "${FODAllSources}/${FODArgsAll."${name}"."${version}".outPath}" - else defaultFetched."${name}"."${version}") - versions) - defaultFetched; -} diff --git a/src/fetchers/default-fetcher.nix b/src/fetchers/default-fetcher.nix deleted file mode 100644 index 49df1f8e..00000000 --- a/src/fetchers/default-fetcher.nix +++ /dev/null @@ -1,70 +0,0 @@ -{ - lib, - # dream2nix attributes - dlib, - fetchSource, - fetchers, - ... -}: { - # sources attrset from dream lock - defaultPackage, - defaultPackageVersion, - sourceOverrides, - sources, - sourceRoot ? null, - ... -}: let - l = lib // builtins; - - fetchedSources = - l.mapAttrs - (name: versions: - l.mapAttrs - (version: source: - if source.type == "unknown" - then "unknown" - else if source.type == "path" - then let - path = - if l.isStorePath (l.concatStringsSep "/" (l.take 4 (l.splitString "/" source.path))) - then source.path - else if name == source.rootName && version == source.rootVersion - then throw "source for ${name}@${version} is referencing itself" - else if source.rootName != null && source.rootVersion != null - then "${overriddenSources."${source.rootName}"."${source.rootVersion}"}/${source.path}" - else if sourceRoot != null - then "${sourceRoot}/${source.path}" - else throw "${name}-${version}: cannot determine path source"; - in - l.path { - inherit path; - name = l.strings.sanitizeDerivationName "${name}-${version}-source"; - } - else if fetchers.fetchers ? "${source.type}" - then - fetchSource { - source = - source - // { - pname = source.pname or name; - version = source.version or version; - }; - } - else throw "unsupported source type '${source.type}'") - versions) - sources; - - overriddenSources = - l.zipAttrsWith - (name: versionsSets: - l.zipAttrsWith - (version: sources: l.last sources) - versionsSets) - [ - fetchedSources - (sourceOverrides fetchedSources) - ]; -in { - # attrset: pname -> path of downloaded source - fetchedSources = overriddenSources; -} diff --git a/src/fetchers/default.nix b/src/fetchers/default.nix deleted file mode 100644 index f1ade64c..00000000 --- a/src/fetchers/default.nix +++ /dev/null @@ -1,240 +0,0 @@ -{ - lib, - # dream2nix - callPackageDream, - dlib, - ... -}: let - b = builtins; - callFetcher = module: - module // {outputs = callPackageDream module.outputs {};}; -in rec { - fetchers = dlib.fetchers.mapFetchers callFetcher; - - defaultFetcher = callPackageDream ./default-fetcher.nix {inherit fetchSource;}; - - combinedFetcher = callPackageDream ./combined-fetcher.nix {inherit defaultFetcher;}; - - constructSource = { - type, - reComputeHash ? false, - ... - } @ args: let - fetcher = fetchers."${type}"; - argsKeep = b.removeAttrs args ["reComputeHash"]; - fetcherOutputs = - fetcher.outputs - (b.removeAttrs argsKeep ["dir" "hash" "type"]); - in - argsKeep - # if the hash was not provided, calculate hash on the fly (impure) - // (lib.optionalAttrs reComputeHash { - hash = fetcherOutputs.calcHash "sha256"; - }); - - # update source spec to different version - updateSource = { - source, - newVersion, - ... - }: - constructSource (source - // { - reComputeHash = true; - } - // { - "${fetchers."${source.type}".versionField}" = newVersion; - }); - - # fetch a source defined via a dream lock source spec - fetchSource = { - source, - extract ? false, - }: let - fetcher = fetchers."${source.type}"; - fetcherArgs = b.removeAttrs source ["dir" "hash" "type"]; - fetcherOutputs = fetcher.outputs fetcherArgs; - maybeArchive = fetcherOutputs.fetched (source.hash or null); - in - if source ? dir - then "${maybeArchive}/${source.dir}" - else maybeArchive; - - # fetch a source defined by a shortcut - fetchShortcut = { - shortcut, - extract ? false, - }: - fetchSource { - source = translateShortcut {inherit shortcut;}; - inherit extract; - }; - - parseShortcut = shortcut: let - # in: "git+https://foo.com/bar?kwarg1=lol&kwarg2=hello" - # out: [ "git+" "git" "https" "//" "foo.com/bar" "?kwarg1=lol&kwarg2=hello" "kwarg1=lol&kwarg2=hello" ] - split = - b.match - ''(([[:alnum:]]+)\+)?([[:alnum:]-]+):(//)?([^\?]*)(\?(.*))?'' - shortcut; - - parsed = { - proto1 = b.elemAt split 1; - proto2 = b.elemAt split 2; - path = b.elemAt split 4; - allArgs = b.elemAt split 6; - kwargs = b.removeAttrs kwargs_ ["dir"]; - dir = kwargs_.dir or null; - }; - - kwargs_ = - if parsed.allArgs == null - then {} - else - lib.listToAttrs - (map - (kwarg: let - split = lib.splitString "=" kwarg; - in - lib.nameValuePair - (b.elemAt split 0) - (b.elemAt split 1)) - (lib.splitString "&" parsed.allArgs)); - in - if split == null - then throw "Unable to parse shortcut: ${shortcut}" - else parsed; - - renderUrlArgs = kwargs: let - asStr = - lib.concatStringsSep - "&" - (lib.mapAttrsToList - (name: val: "${name}=${val}") - kwargs); - in - if asStr == "" - then "" - else "?" + asStr; - - # translate shortcut to dream lock source spec - translateShortcut = { - shortcut ? throw "pass shortcut", - parsed ? parseShortcut shortcut, - computeHash ? (! parsed.kwargs ? hash), - }: let - parsed = parseShortcut shortcut; - - checkArgs = fetcherName: args: let - fetcher = fetchers."${fetcherName}"; - unknownArgNames = lib.filter (argName: ! lib.elem argName fetcher.inputs) (lib.attrNames args); - missingArgNames = lib.filter (inputName: ! args ? "${inputName}") fetcher.inputs; - in - if lib.length unknownArgNames > 0 - then throw "Received unknown arguments for fetcher '${fetcherName}': ${b.toString unknownArgNames}" - else if lib.length missingArgNames > 0 - then throw "Missing arguments for fetcher '${fetcherName}': ${b.toString missingArgNames}" - else args; - - translateHttpUrl = let - fetcher = fetchers.http; - - urlArgsFinal = renderUrlArgs parsed.kwargs; - - url = with parsed; "${proto2}://${path}${urlArgsFinal}"; - - fetcherOutputs = fetchers.http.outputs { - inherit url; - }; - in - constructSource - { - inherit url; - type = "http"; - } - // (lib.optionalAttrs (parsed.dir != null) { - dir = parsed.dir; - }) - // (lib.optionalAttrs computeHash { - hash = fetcherOutputs.calcHash "sha256"; - }); - - translateProtoShortcut = let - kwargsUrl = b.removeAttrs parsed.kwargs fetcher.inputs; - - urlArgs = renderUrlArgs kwargsUrl; - - url = with parsed; "${proto2}://${path}${urlArgs}"; - - fetcherName = parsed.proto1; - - fetcher = fetchers."${fetcherName}"; - - args = parsed.kwargs // {inherit url;}; - - fetcherOutputs = fetcher.outputs (checkArgs fetcherName args); - in - constructSource - (parsed.kwargs - // { - type = fetcherName; - inherit url; - } - // (lib.optionalAttrs (parsed.dir != null) { - dir = parsed.dir; - }) - // (lib.optionalAttrs computeHash { - hash = fetcherOutputs.calcHash "sha256"; - })); - - translateRegularShortcut = let - fetcherName = parsed.proto2; - - path = lib.removeSuffix "/" parsed.path; - - params = lib.splitString "/" path; - - fetcher = fetchers."${fetcherName}"; - - args = - if fetcher ? parseParams - then fetcher.parseParams params - else if b.length params != b.length fetcher.inputs - then - throw '' - Wrong number of arguments provided in shortcut for fetcher '${fetcherName}' - Should be ${fetcherName}:${lib.concatStringsSep "/" fetcher.inputs} - '' - else - lib.listToAttrs - (lib.forEach - (lib.range 0 ((lib.length fetcher.inputs) - 1)) - ( - idx: - lib.nameValuePair - (lib.elemAt fetcher.inputs idx) - (lib.elemAt params idx) - )); - - fetcherOutputs = fetcher.outputs (args // parsed.kwargs); - in - constructSource (args - // parsed.kwargs - // { - type = fetcherName; - } - // (lib.optionalAttrs (parsed.dir != null) { - dir = parsed.dir; - }) - // (lib.optionalAttrs computeHash { - hash = fetcherOutputs.calcHash "sha256"; - })); - in - if parsed.proto1 != null - then translateProtoShortcut - else if - lib.hasPrefix "http://" shortcut - || lib.hasPrefix "https://" shortcut - then translateHttpUrl - else translateRegularShortcut; -} diff --git a/src/lib.nix b/src/lib.nix index 49435bf1..9b357f2c 100644 --- a/src/lib.nix +++ b/src/lib.nix @@ -92,8 +92,7 @@ framework = import ./modules/framework.nix { inherit lib dlib; dream2nixConfig = config; - callPackageDream = - throw "callPackageDream is not available before nixpkgs is imported"; + apps = throw "apps is not available before nixpkgs is imported"; pkgs = throw "pkgs is not available before nixpkgs is imported"; utils = throw "utils is not available before nixpkgs is imported"; }; diff --git a/src/lib/builders.nix b/src/lib/builders.nix deleted file mode 100644 index 625c3906..00000000 --- a/src/lib/builders.nix +++ /dev/null @@ -1,32 +0,0 @@ -{ - dlib, - lib, - config, -}: let - l = lib // builtins; - - defaults = { - rust = "build-rust-package"; - nodejs = "granular"; - python = "simple-builder"; - php = "granular"; - racket = "simple"; - }; - - # TODO - validator = module: true; - - modules = dlib.modules.makeSubsystemModules { - inherit validator defaults; - modulesCategory = "builders"; - }; - - builders = - dlib.modules.mapSubsystemModules - (b: b // {build = dlib.warnIfIfd b b.build;}) - modules.modules; - mapBuilders = f: dlib.modules.mapSubsystemModules f builders; -in { - inherit builders mapBuilders; - callBuilder = modules.callModule; -} diff --git a/src/lib/default.nix b/src/lib/default.nix index de5fb9b6..714fceca 100644 --- a/src/lib/default.nix +++ b/src/lib/default.nix @@ -8,14 +8,11 @@ # exported attributes dlib = { inherit - builders calcInvalidationHash callViaEnv construct containsMatchingFile dirNames - discoverers - fetchers indexers latestVersion listDirs @@ -25,7 +22,6 @@ readTextFile recursiveUpdateUntilDepth simpleTranslate2 - translators sanitizePath sanitizeRelativePath subsystems @@ -46,11 +42,7 @@ subsystems = dirNames ../subsystems; # other libs - builders = import ./builders.nix {inherit dlib lib config;}; construct = import ./construct.nix {inherit lib;}; - discoverers = import ./discoverers.nix {inherit config dlib lib;}; - fetchers = import ./fetchers.nix {inherit dlib lib;}; - translators = import ./translators.nix {inherit dlib lib;}; indexers = import ./indexers.nix {inherit dlib lib;}; modules = import ./modules.nix {inherit config dlib lib;}; diff --git a/src/lib/discoverers.nix b/src/lib/discoverers.nix deleted file mode 100644 index 579c2eaf..00000000 --- a/src/lib/discoverers.nix +++ /dev/null @@ -1,147 +0,0 @@ -{ - dlib, - lib, - config, -}: let - l = lib // builtins; - - allDiscoverers = - (dlib.modules.collectSubsystemModules modules.modules) - ++ [defaultDiscoverer]; - - allTranslators = - dlib.modules.collectSubsystemModules dlib.translators.translators; - - translatorsWithDiscoverFunc = - l.filter (translator: translator.discoverProject or null != null) allTranslators; - - defaultDiscoverer.discover = {tree}: let - translatorsCurrentDir = - l.filter - (t: t.discoverProject tree) - translatorsWithDiscoverFunc; - - projectsCurrentDir = - l.map - (t: { - name = "main"; - relPath = tree.relPath; - translators = [t.name]; - subsystem = t.subsystem; - }) - translatorsCurrentDir; - - # If there are multiple projects detected for the same subsystem, - # merge them to a single one with translators = [...] - projectsCurrentDirMerged = - l.attrValues - (l.foldl - (all: curr: - all - // { - "${curr.subsystem}" = - all.${curr.subsystem} - or curr - // { - translators = - l.unique (all.${curr.subsystem}.translators or [] ++ curr.translators); - }; - }) - {} - projectsCurrentDir); - - subdirProjects = - l.flatten - (l.mapAttrsToList - (dirName: tree: - defaultDiscoverer.discover { - inherit tree; - }) - tree.directories or {}); - in ( - if translatorsCurrentDir == [] - then subdirProjects - else projectsCurrentDirMerged ++ subdirProjects - ); - - discoverProjects = { - projects, - source ? throw "Pass either `source` or `tree` to discoverProjects", - tree ? dlib.prepareSourceTree {inherit source;}, - settings ? [], - }: let - discoveredProjects = - l.flatten - (l.map - (discoverer: discoverer.discover {inherit tree;}) - allDiscoverers); - - discoveredProjectsSorted = let - sorted = - l.sort - (p1: p2: l.hasPrefix p1.relPath or "" p2.relPath or "") - discoveredProjects; - in - sorted; - - allProjects = discoveredProjectsSorted ++ (l.attrValues projects); - - rootProject = l.head allProjects; - - projectsExtended = - l.forEach allProjects - (proj: - proj - // { - relPath = proj.relPath or ""; - translator = proj.translator or (l.head proj.translators); - dreamLockPath = getDreamLockPath proj rootProject; - }); - in - applyProjectSettings projectsExtended settings; - - getDreamLockPath = project: rootProject: - dlib.sanitizeRelativePath - "${config.packagesDir}/${rootProject.name}/${project.relPath or ""}/dream-lock.json"; - - applyProjectSettings = projects: settingsList: let - settingsListForProject = project: - l.filter - (settings: - if ! settings ? filter - then true - else settings.filter project) - settingsList; - - applySettings = project: settings: - l.recursiveUpdate project settings; - - applyAllSettings = project: - l.foldl' - (proj: settings: applySettings proj settings) - project - (settingsListForProject project); - - settingsApplied = - l.forEach projects - (proj: applyAllSettings proj); - in - settingsApplied; - - # TODO - validator = module: true; - - modules = dlib.modules.makeSubsystemModules { - modulesCategory = "discoverers"; - inherit validator; - }; -in { - inherit - applyProjectSettings - discoverProjects - ; - - discoverers = modules.modules; - callDiscoverer = modules.callModule; - mapDiscoverers = modules.mapModules; -} diff --git a/src/lib/fetchers.nix b/src/lib/fetchers.nix deleted file mode 100644 index 1eca8998..00000000 --- a/src/lib/fetchers.nix +++ /dev/null @@ -1,40 +0,0 @@ -{ - dlib, - lib, -}: let - l = lib // builtins; - - # TODO: - validator = module: true; - - callFetcher = file: extraArgs: - dlib.modules.importModule {inherit file extraArgs validator;}; - - # get information for builtin fetchers - fetchersDir = ../fetchers; - fetcherNames = dlib.dirNames fetchersDir; - - # get the builtin fetchers - builtinFetchers = - l.map - (name: { - file = "${fetchersDir}/${name}"; - extraArgs = {inherit name;}; - }) - fetcherNames; - # get extra fetchers - extraFetchers = dlib.modules.extra.fetchers or []; - - # import fetchers - importedFetchers = - l.map - ( - module: - (callFetcher module.file module.extraArgs) - // {inherit (module.extraArgs) name;} - ) - (builtinFetchers ++ extraFetchers); - # create the attrset - fetchers = l.listToAttrs (l.map (f: l.nameValuePair f.name f) importedFetchers); - mapFetchers = f: l.mapAttrs (_: fetcher: f fetcher) fetchers; -in {inherit fetchers callFetcher mapFetchers;} diff --git a/src/lib/modules.nix b/src/lib/modules.nix deleted file mode 100644 index afe29608..00000000 --- a/src/lib/modules.nix +++ /dev/null @@ -1,197 +0,0 @@ -{ - config, - dlib, - lib, -}: let - l = lib // builtins; - - configFuncMsg = '' - consider passing a path to a file instead of a function: - functions can't be encoded to JSON, and as such most features of - dream2nix won't work because of this since they require passing around - the config as JSON. - ''; - - # imports a module. - # - # - 'file' can be a function or a path to a function. - # - 'dlib', 'lib', 'config' and attributes in - # 'extraArgs' are passed to the function. - importModule = { - file, - validator ? _: true, - extraArgs ? {}, - }: let - _module = - if l.isFunction file - then file - else import file; - module = - if l.isFunction _module - then _module ({inherit config dlib lib;} // extraArgs) - else throw "module file (${file}) must return a function that takes an attrset"; - in - l.seq (validator module) module; - - # collects extra modules from a list - # ex: [{translators = [module1];} {translators = [module2];}] -> {translators = [module1 module2];} - collectExtraModules = - l.foldl' - (acc: el: - acc - // ( - l.mapAttrs - (category: modules: modules ++ (acc.${category} or [])) - el - )) - {}; - # processes one extra (config.extra) - # returns extra modules like {fetchers = [...]; translators = [...];} - processOneExtra = _extra: let - isAttrs = val: l.isAttrs val && (! val ? drvPath); - # imports a modules declaration - importDecl = decl: - if l.isFunction decl - then l.throw configFuncMsg - else if isAttrs decl - then decl - else import decl {inherit config dlib lib;}; - # extra attrset itself - # config.extra is imported here if it's a path - extra = importDecl _extra; - # throw if declaring a module as a function - throwIfModuleNotPath = module: - l.throwIf ((isAttrs _extra) && (l.isFunction module)) configFuncMsg module; - # collect subsystem modules (translators, discoverers, builders) - _extraSubsystemModules = - l.mapAttrsToList - (subsystem: categories: - l.mapAttrs - (category: modules: - l.mapAttrsToList - (name: module: { - file = throwIfModuleNotPath module; - extraArgs = {inherit subsystem name;}; - }) - (importDecl modules)) - categories) - (importDecl (extra.subsystems or {})); - extraSubsystemModules = - collectExtraModules (l.flatten _extraSubsystemModules); - # collect fetcher modules - extraFetcherModules = - l.mapAttrsToList - (name: fetcher: { - file = throwIfModuleNotPath fetcher; - extraArgs = {inherit name;}; - }) - (importDecl (extra.fetchers or {})); - in - extraSubsystemModules - // { - fetchers = extraFetcherModules; - }; - # collect all extras - extra = - collectExtraModules - (l.map processOneExtra (l.flatten [(config.extra or [])])); - - mapSubsystemModules = f: modules: - l.mapAttrs - ( - subsystem: names: - l.mapAttrs - (name: module: f module) - names - ) - modules; - - # collect subsystem modules into a list - # ex: {rust.cargo-lock = cargo-lock; go.gomod2nix = gomod2nix;} -> [cargo-lock gomod2nix] - collectSubsystemModules = modules: let - allModules = l.flatten (l.map l.attrValues (l.attrValues modules)); - hasModule = module: modules: - l.any - ( - omodule: - (omodule.name == module.name) - && (omodule.subsystem == module.subsystem) - ) - modules; - in - l.foldl' - (acc: el: - if hasModule el acc - then acc - else acc ++ [el]) - [] - allModules; - - # create subsystem modules - # returns ex: {rust = {moduleName = module;}; go = {moduleName = module;};} - makeSubsystemModules = { - modulesCategory, - validator, - extraModules ? extra.${modulesCategory} or [], - defaults ? {}, - }: let - callModule = file: extraArgs: - importModule {inherit file validator extraArgs;}; - - # extract builtin modules from subsystems directory - modulesBuiltin = l.flatten ( - l.map - ( - subsystem: let - dir = "${toString ../subsystems}/${subsystem}/${modulesCategory}"; - moduleNames = - if l.pathExists dir - then dlib.dirNames dir - else []; - makeModule = name: { - file = "${dir}/${name}"; - extraArgs = {inherit subsystem name;}; - }; - in - l.map makeModule moduleNames - ) - dlib.subsystems - ); - - # import the modules - importedModules = l.filter (t: t.disabled or false == false) ( - l.map - ( - module: - (callModule module.file module.extraArgs) - // {inherit (module.extraArgs) subsystem name;} - ) - (modulesBuiltin ++ extraModules) - ); - # create the modules attrset - _modules = - l.foldl' - (acc: el: l.recursiveUpdate acc {"${el.subsystem}"."${el.name}" = el;}) - {} - importedModules; - # add default module attribute to a subsystem if declared in `defaults` - modules = - l.mapAttrs - ( - subsystem: modules: - if l.hasAttr subsystem defaults - then modules // {default = modules.${defaults.${subsystem}};} - else modules - ) - _modules; - mapModules = f: mapSubsystemModules f modules; - in {inherit callModule mapModules modules;}; -in { - inherit - importModule - makeSubsystemModules - collectSubsystemModules - mapSubsystemModules - extra - ; -} diff --git a/src/lib/translators.nix b/src/lib/translators.nix deleted file mode 100644 index 40ffbdea..00000000 --- a/src/lib/translators.nix +++ /dev/null @@ -1,27 +0,0 @@ -{ - dlib, - lib, -}: let - l = lib // builtins; - - # TODO - validator = module: true; - - modules = dlib.modules.makeSubsystemModules { - inherit validator; - modulesCategory = "translators"; - }; - - translators = - dlib.modules.mapSubsystemModules - (t: - t - // (lib.optionalAttrs (t.translate or null != null) { - translate = l.trace t dlib.warnIfIfd t t.translate; - })) - modules.modules; - mapTranslators = f: dlib.modules.mapSubsystemModules f translators; -in { - inherit translators mapTranslators; - callTranslator = modules.callModule; -} diff --git a/src/modules/builders/implementation.nix b/src/modules/builders/implementation.nix index 8d7a8076..50b436ce 100644 --- a/src/modules/builders/implementation.nix +++ b/src/modules/builders/implementation.nix @@ -1,9 +1,5 @@ -{ - config, - callPackageDream, - lib, - ... -}: let +{config, ...}: let + lib = config.lib; defaults = { rust = "build-rust-package"; nodejs = "granular-nodejs"; @@ -13,7 +9,6 @@ debian = "simple-debian"; racket = "simple-racket"; }; - loader = b: b // {build = callPackageDream b.build {};}; funcs = config.functions.subsystem-loading; collectedModules = funcs.collect "builders"; in { @@ -21,12 +16,6 @@ in { # The user can add more translators by extending this attribute builders = funcs.import_ collectedModules; - /* - translators wrapped with extra logic to add extra attributes, - like .translateBin for pure translators - */ - builderInstances = funcs.instantiate config.builders loader; - buildersBySubsystem = lib.mapAttrs ( @@ -36,6 +25,6 @@ in { default = builders.${defaults.${subsystem}}; } ) - (funcs.structureBySubsystem config.builderInstances); + (funcs.structureBySubsystem config.builders); }; } diff --git a/src/modules/builders/interface.nix b/src/modules/builders/interface.nix index aa333324..ad5f7a83 100644 --- a/src/modules/builders/interface.nix +++ b/src/modules/builders/interface.nix @@ -1,23 +1,17 @@ -{ - lib, - specialArgs, - ... -}: let +{config, ...}: let + lib = config.lib; t = lib.types; in { options = { builders = lib.mkOption { type = t.attrsOf (t.submoduleWith { - modules = [./builder/default.nix]; - inherit specialArgs; + modules = [../interfaces.builder]; + specialArgs = {framework = config;}; }); description = '' builder module definitions ''; }; - builderInstances = lib.mkOption { - type = t.attrsOf t.anything; - }; buildersBySubsystem = lib.mkOption { type = t.attrsOf (t.attrsOf t.anything); }; diff --git a/src/modules/discoverers.default-discoverer/implementation.nix b/src/modules/discoverers.default-discoverer/implementation.nix index 6e352f48..e11a7ef1 100644 --- a/src/modules/discoverers.default-discoverer/implementation.nix +++ b/src/modules/discoverers.default-discoverer/implementation.nix @@ -1,9 +1,5 @@ -{ - config, - lib, - ... -}: let - l = lib // builtins; +{config, ...}: let + l = config.lib // builtins; translatorsWithDiscoverFunc = l.filter diff --git a/src/modules/discoverers/implementation.nix b/src/modules/discoverers/implementation.nix index 21426bf9..e9c68ac9 100644 --- a/src/modules/discoverers/implementation.nix +++ b/src/modules/discoverers/implementation.nix @@ -3,7 +3,6 @@ collectedModules = funcs.collect "discoverers"; in { config = { - # The user can add more discoverers by extending this attribute discoverers = funcs.import_ collectedModules; discoverersBySubsystem = funcs.structureBySubsystem config.discoverers; diff --git a/src/modules/discoverers/interface.nix b/src/modules/discoverers/interface.nix index c779c8ec..af8576ab 100644 --- a/src/modules/discoverers/interface.nix +++ b/src/modules/discoverers/interface.nix @@ -1,15 +1,12 @@ -{ - lib, - specialArgs, - ... -}: let +{config, ...}: let + lib = config.lib; t = lib.types; in { options = { discoverers = lib.mkOption { type = t.attrsOf (t.submoduleWith { modules = [../interfaces.discoverer]; - inherit specialArgs; + specialArgs = {framework = config;}; }); description = '' discoverer module definitions diff --git a/src/modules/fetchers/implementation.nix b/src/modules/fetchers/implementation.nix index ce7de6a5..5648efbc 100644 --- a/src/modules/fetchers/implementation.nix +++ b/src/modules/fetchers/implementation.nix @@ -1,10 +1,6 @@ -{ - callPackageDream, - dlib, - lib, - ... -}: let - l = lib // builtins; +{config, ...}: let + l = config.lib; + fetchersDir = ../../fetchers; fetcherNames = l.attrNames ( l.filterAttrs @@ -14,12 +10,7 @@ fetcherModules = l.genAttrs fetcherNames - ( - name: - import "${fetchersDir}/${name}" { - inherit dlib lib; - } - ); + (name: import "${fetchersDir}/${name}" config); in { config = { fetchers = fetcherModules; diff --git a/src/modules/fetchers/interface.nix b/src/modules/fetchers/interface.nix index f4e97987..88355cf2 100644 --- a/src/modules/fetchers/interface.nix +++ b/src/modules/fetchers/interface.nix @@ -1,9 +1,5 @@ -{ - lib, - specialArgs, - ... -}: let - l = lib // builtins; +{config, ...}: let + l = config.lib // builtins; t = l.types; in { options = { @@ -11,7 +7,7 @@ in { type = t.attrsOf ( t.submoduleWith { modules = [../interfaces.fetcher]; - inherit specialArgs; + specialArgs = {framework = config;}; } ); }; diff --git a/src/modules/framework.nix b/src/modules/framework.nix index ac036528..5e9c29d1 100644 --- a/src/modules/framework.nix +++ b/src/modules/framework.nix @@ -1,24 +1,16 @@ { dream2nixConfig, - callPackageDream, pkgs, dlib, lib, utils, + apps, }: let + topLevel = import ./top-level.nix { + inherit apps lib dlib utils pkgs dream2nixConfig; + }; evaledModules = lib.evalModules { - modules = [./top-level.nix] ++ (dream2nixConfig.modules or []); - - # TODO: remove specialArgs once all functionality is moved to /src/modules - specialArgs = { - inherit - dream2nixConfig - callPackageDream - pkgs - dlib - utils - ; - }; + modules = [topLevel] ++ (dream2nixConfig.modules or []); }; framework = evaledModules.config; diff --git a/src/modules/functions.combined-fetcher/implementation.nix b/src/modules/functions.combined-fetcher/implementation.nix index d5c43894..09e9fbef 100644 --- a/src/modules/functions.combined-fetcher/implementation.nix +++ b/src/modules/functions.combined-fetcher/implementation.nix @@ -1,11 +1,5 @@ -{ - lib, - dlib, - utils, - config, - pkgs, - ... -}: let +{config, ...}: let + inherit (config) lib pkgs utils; defaultFetcher = config.functions.defaultFetcher; inherit (pkgs) diff --git a/src/modules/functions.combined-fetcher/interface.nix b/src/modules/functions.combined-fetcher/interface.nix index 57ec9d32..4551042d 100644 --- a/src/modules/functions.combined-fetcher/interface.nix +++ b/src/modules/functions.combined-fetcher/interface.nix @@ -1,5 +1,5 @@ -{lib, ...}: let - l = lib // builtins; +{config, ...}: let + l = config.lib // builtins; t = l.types; in { options = { diff --git a/src/modules/functions.default-fetcher/implementation.nix b/src/modules/functions.default-fetcher/implementation.nix index d4e533e8..aa45c250 100644 --- a/src/modules/functions.default-fetcher/implementation.nix +++ b/src/modules/functions.default-fetcher/implementation.nix @@ -1,9 +1,6 @@ -{ - lib, - dlib, - config, - ... -}: let +{config, ...}: let + inherit (config) lib; + fetchers = config.fetchers; fetchSource = config.functions.fetchers.fetchSource; func = { diff --git a/src/modules/functions.default-fetcher/interface.nix b/src/modules/functions.default-fetcher/interface.nix index 77be25a8..e1dbb714 100644 --- a/src/modules/functions.default-fetcher/interface.nix +++ b/src/modules/functions.default-fetcher/interface.nix @@ -1,5 +1,5 @@ -{lib, ...}: let - l = lib // builtins; +{config, ...}: let + l = config.lib // builtins; t = l.types; in { options = { diff --git a/src/modules/functions.discoverers/implementation.nix b/src/modules/functions.discoverers/implementation.nix index 1eb0d11e..dea0d100 100644 --- a/src/modules/functions.discoverers/implementation.nix +++ b/src/modules/functions.discoverers/implementation.nix @@ -1,11 +1,6 @@ -{ - config, - dream2nixConfig, - dlib, - lib, - ... -}: let - l = lib // builtins; +{config, ...}: let + l = config.lib // builtins; + dlib = config.dlib; discoverProjects = { projects, @@ -47,7 +42,7 @@ getDreamLockPath = project: rootProject: dlib.sanitizeRelativePath - "${dream2nixConfig.packagesDir}/${rootProject.name}/${project.relPath or ""}/dream-lock.json"; + "${config.dream2nixConfig.packagesDir}/${rootProject.name}/${project.relPath or ""}/dream-lock.json"; applyProjectSettings = projects: settingsList: let settingsListForProject = project: diff --git a/src/modules/functions.discoverers/interface.nix b/src/modules/functions.discoverers/interface.nix index ca447326..d716f018 100644 --- a/src/modules/functions.discoverers/interface.nix +++ b/src/modules/functions.discoverers/interface.nix @@ -1,5 +1,5 @@ -{lib, ...}: let - l = lib // builtins; +{config, ...}: let + l = config.lib // builtins; t = l.types; in { options.functions.discoverers = { diff --git a/src/modules/functions.fetchers/implementation.nix b/src/modules/functions.fetchers/implementation.nix index 76e3c0f0..6c6750d4 100644 --- a/src/modules/functions.fetchers/implementation.nix +++ b/src/modules/functions.fetchers/implementation.nix @@ -1,10 +1,6 @@ -{ - config, - lib, - ... -}: let +{config, ...}: let + inherit (config) fetchers lib; b = builtins; - fetchers = config.fetchers; renderUrlArgs = kwargs: let asStr = @@ -28,7 +24,7 @@ in { fetcher = fetchers."${type}"; argsKeep = b.removeAttrs args ["reComputeHash"]; fetcherOutputs = - fetcher.outputsInstanced + fetcher.outputs (b.removeAttrs argsKeep ["dir" "hash" "type"]); in argsKeep @@ -58,7 +54,7 @@ in { }: let fetcher = fetchers."${source.type}"; fetcherArgs = b.removeAttrs source ["dir" "hash" "type"]; - fetcherOutputs = fetcher.outputsInstanced fetcherArgs; + fetcherOutputs = fetcher.outputs fetcherArgs; maybeArchive = fetcherOutputs.fetched (source.hash or null); in if source ? dir @@ -136,7 +132,7 @@ in { url = with parsed; "${proto2}://${path}${urlArgsFinal}"; - fetcherOutputs = fetcher.outputsInstanced { + fetcherOutputs = fetcher.outputs { inherit url; }; in @@ -165,7 +161,7 @@ in { args = parsed.kwargs // {inherit url;}; - fetcherOutputs = fetcher.outputsInstanced (checkArgs fetcherName args); + fetcherOutputs = fetcher.outputs (checkArgs fetcherName args); in constructSource (parsed.kwargs @@ -209,7 +205,7 @@ in { (lib.elemAt params idx) )); - fetcherOutputs = fetcher.outputsInstanced (args // parsed.kwargs); + fetcherOutputs = fetcher.outputs (args // parsed.kwargs); in constructSource (args // parsed.kwargs diff --git a/src/modules/functions.fetchers/interface.nix b/src/modules/functions.fetchers/interface.nix index 5a15428d..58be0fd3 100644 --- a/src/modules/functions.fetchers/interface.nix +++ b/src/modules/functions.fetchers/interface.nix @@ -1,5 +1,5 @@ -{lib, ...}: let - l = lib // builtins; +{config, ...}: let + l = config.lib // builtins; t = l.types; in { options = { diff --git a/src/modules/functions.subsystem-loading/implementation.nix b/src/modules/functions.subsystem-loading/implementation.nix index cdbbf01d..7ca647eb 100644 --- a/src/modules/functions.subsystem-loading/implementation.nix +++ b/src/modules/functions.subsystem-loading/implementation.nix @@ -6,14 +6,8 @@ This recurses through the following directory structure to discover and load modules: /src/subsystems/{subsystem}/{module-type}/{module-name} */ -{ - config, - dlib, - callPackageDream, - ... -}: let - lib = config.lib; - t = lib.types; +{config, ...}: let + inherit (config) dlib lib; subsystemsDir = lib.toString ../../subsystems; subsystems = dlib.dirNames subsystemsDir; @@ -42,7 +36,7 @@ modules: import_ = collectedModules: lib.mapAttrs (name: description: - (import description.path {inherit dlib lib;}) + (import description.path config) // {inherit (description) name subsystem;}) ( lib.foldl' diff --git a/src/modules/builders/builder/default.nix b/src/modules/interfaces.builder/default.nix similarity index 100% rename from src/modules/builders/builder/default.nix rename to src/modules/interfaces.builder/default.nix diff --git a/src/modules/builders/builder/interface.nix b/src/modules/interfaces.builder/interface.nix similarity index 83% rename from src/modules/builders/builder/interface.nix rename to src/modules/interfaces.builder/interface.nix index e77c0d34..9a925820 100644 --- a/src/modules/builders/builder/interface.nix +++ b/src/modules/interfaces.builder/interface.nix @@ -1,8 +1,5 @@ -{ - config, - lib, - ... -}: let +{framework, ...}: let + lib = framework.lib; t = lib.types; in { options = { @@ -20,8 +17,8 @@ in { description = "Subsystem of the builder."; }; build = lib.mkOption { - type = t.functionTo (t.functionTo t.attrs); - default = _: _: {}; + type = t.functionTo t.attrs; + default = _: {}; }; type = lib.mkOption { type = t.enum [ diff --git a/src/modules/interfaces.discoverer/interface.nix b/src/modules/interfaces.discoverer/interface.nix index 44961f7f..e84bcaab 100644 --- a/src/modules/interfaces.discoverer/interface.nix +++ b/src/modules/interfaces.discoverer/interface.nix @@ -1,8 +1,5 @@ -{ - config, - lib, - ... -}: let +{framework, ...}: let + lib = framework.lib; t = lib.types; in { options = { diff --git a/src/modules/interfaces.fetcher/default.nix b/src/modules/interfaces.fetcher/default.nix index 274989a4..bd4ca0b5 100644 --- a/src/modules/interfaces.fetcher/default.nix +++ b/src/modules/interfaces.fetcher/default.nix @@ -1,6 +1,5 @@ { imports = [ ./interface.nix - ./implementation.nix ]; } diff --git a/src/modules/interfaces.fetcher/implementation.nix b/src/modules/interfaces.fetcher/implementation.nix deleted file mode 100644 index 77a43011..00000000 --- a/src/modules/interfaces.fetcher/implementation.nix +++ /dev/null @@ -1,9 +0,0 @@ -{ - config, - callPackageDream, - ... -}: { - config = { - outputsInstanced = callPackageDream config.outputs {}; - }; -} diff --git a/src/modules/interfaces.fetcher/interface.nix b/src/modules/interfaces.fetcher/interface.nix index 63df356d..dba00e34 100644 --- a/src/modules/interfaces.fetcher/interface.nix +++ b/src/modules/interfaces.fetcher/interface.nix @@ -1,5 +1,5 @@ -{lib, ...}: let - l = lib // builtins; +{framework, ...}: let + l = framework.lib // builtins; t = l.types; outputsOptions = { @@ -44,19 +44,11 @@ in { type = t.functionTo t.attrs; }; outputs = l.mkOption { - type = t.functionTo (t.functionTo ( - t.submoduleWith { - modules = [outputsOptions]; - } - )); - }; - outputsInstanced = l.mkOption { type = t.functionTo ( t.submoduleWith { modules = [outputsOptions]; } ); - readOnly = true; }; }; } diff --git a/src/modules/interfaces.translator/default.nix b/src/modules/interfaces.translator/default.nix new file mode 100644 index 00000000..274989a4 --- /dev/null +++ b/src/modules/interfaces.translator/default.nix @@ -0,0 +1,6 @@ +{ + imports = [ + ./interface.nix + ./implementation.nix + ]; +} diff --git a/src/modules/interfaces.translator/implementation.nix b/src/modules/interfaces.translator/implementation.nix new file mode 100644 index 00000000..1ad225a7 --- /dev/null +++ b/src/modules/interfaces.translator/implementation.nix @@ -0,0 +1,23 @@ +{ + config, + framework, + ... +}: { + config = { + translateInstanced = args: + config.translate + ( + (framework.functions.translators.makeTranslatorDefaultArgs + (config.extraArgs or {})) + // args + // (args.project.subsystemInfo or {}) + // { + tree = + args.tree or (framework.dlib.prepareSourceTree {inherit (args) source;}); + } + ); + translateBin = + framework.functions.translators.wrapPureTranslator + {inherit (config) subsystem name;}; + }; +} diff --git a/src/modules/translators/interface-translator.nix b/src/modules/interfaces.translator/interface.nix similarity index 84% rename from src/modules/translators/interface-translator.nix rename to src/modules/interfaces.translator/interface.nix index c4645ee8..a38368a3 100644 --- a/src/modules/translators/interface-translator.nix +++ b/src/modules/interfaces.translator/interface.nix @@ -1,8 +1,5 @@ -{ - config, - lib, - ... -}: let +{framework, ...}: let + lib = framework.lib; t = lib.types; in { options = { @@ -46,11 +43,15 @@ in { type = t.str; }; translate = lib.mkOption { - type = t.nullOr (t.functionTo (t.functionTo (t.attrs))); + type = t.nullOr (t.functionTo t.attrs); default = null; }; + translateInstanced = lib.mkOption { + type = t.nullOr (t.functionTo t.attrs); + readOnly = true; + }; translateBin = lib.mkOption { - type = t.nullOr (t.functionTo t.package); + type = t.nullOr t.package; default = null; }; type = lib.mkOption { diff --git a/src/modules/top-level.nix b/src/modules/top-level.nix index 340d2681..82fa86b8 100644 --- a/src/modules/top-level.nix +++ b/src/modules/top-level.nix @@ -1,7 +1,10 @@ { - config, + apps, + dlib, lib, - ... + pkgs, + utils, + dream2nixConfig, }: let t = lib.types; in { @@ -19,11 +22,27 @@ in { ./translators ]; options = { + apps = lib.mkOption { + type = t.raw; + }; lib = lib.mkOption { type = t.raw; }; + dlib = lib.mkOption { + type = t.raw; + }; + pkgs = lib.mkOption { + type = t.raw; + }; + utils = lib.mkOption { + type = t.raw; + }; + dream2nixConfig = lib.mkOption { + type = t.raw; + }; }; config = { + inherit apps dlib pkgs utils dream2nixConfig; lib = lib // builtins; }; } diff --git a/src/modules/translators/implementation.nix b/src/modules/translators/implementation.nix index 9de266ee..49346d78 100644 --- a/src/modules/translators/implementation.nix +++ b/src/modules/translators/implementation.nix @@ -1,12 +1,4 @@ -{ - config, - callPackageDream, - ... -}: let - lib = config.lib; - t = lib.types; - # TODO: the makeTranslator logic should be moved somewhere to /src/modules - loader = (callPackageDream ../../subsystems/translators.nix {}).makeTranslator; +{config, ...}: let funcs = config.functions.subsystem-loading; collectedModules = funcs.collect "translators"; in { @@ -14,12 +6,6 @@ in { # The user can add more translators by extending this attribute translators = funcs.import_ collectedModules; - /* - translators wrapped with extra logic to add extra attributes, - like .translateBin for pure translators - */ - translatorInstances = funcs.instantiate config.translators loader; - - translatorsBySubsystem = funcs.structureBySubsystem config.translatorInstances; + translatorsBySubsystem = funcs.structureBySubsystem config.translators; }; } diff --git a/src/modules/translators/interface.nix b/src/modules/translators/interface.nix index d1358fbf..c74bf4cd 100644 --- a/src/modules/translators/interface.nix +++ b/src/modules/translators/interface.nix @@ -1,24 +1,17 @@ -{ - config, - lib, - specialArgs, - ... -}: let +{config, ...}: let + lib = config.lib; t = lib.types; in { options = { translators = lib.mkOption { type = t.attrsOf (t.submoduleWith { - modules = [./interface-translator.nix]; - inherit specialArgs; + modules = [../interfaces.translator]; + specialArgs = {framework = config;}; }); description = '' Translator module definitions ''; }; - translatorInstances = lib.mkOption { - type = t.attrsOf t.anything; - }; translatorsBySubsystem = lib.mkOption { type = t.attrsOf (t.attrsOf t.anything); }; diff --git a/src/subsystems/builders.nix b/src/subsystems/builders.nix deleted file mode 100644 index cfec2c38..00000000 --- a/src/subsystems/builders.nix +++ /dev/null @@ -1,18 +0,0 @@ -{ - dlib, - callPackageDream, - ... -}: let - makeBuilder = builderModule: - builderModule - // { - build = callPackageDream builderModule.build {}; - }; - - builders = dlib.builders.mapBuilders makeBuilder; -in { - inherit - makeBuilder - builders - ; -} diff --git a/src/subsystems/debian/builders/simple-debian/default.nix b/src/subsystems/debian/builders/simple-debian/default.nix index d4d44bd8..8bce84cb 100644 --- a/src/subsystems/debian/builders/simple-debian/default.nix +++ b/src/subsystems/debian/builders/simple-debian/default.nix @@ -1,14 +1,11 @@ -{...}: { +{ + lib, + pkgs, + ... +}: { type = "pure"; build = { - lib, - pkgs, - stdenv, - # dream2nix inputs - externals, - ... - }: { ### FUNCTIONS # AttrSet -> Bool) -> AttrSet -> [x] getCyclicDependencies, # name: version: -> [ {name=; version=; } ] @@ -54,7 +51,7 @@ (src: src.original or src) allDependencySources'; - package = produceDerivation defaultPackageName (stdenv.mkDerivation { + package = produceDerivation defaultPackageName (pkgs.stdenv.mkDerivation { name = defaultPackageName; src = ":"; dontUnpack = true; diff --git a/src/subsystems/debian/translators/debian-binary/default.nix b/src/subsystems/debian/translators/debian-binary/default.nix index 40002199..f86be0b0 100644 --- a/src/subsystems/debian/translators/debian-binary/default.nix +++ b/src/subsystems/debian/translators/debian-binary/default.nix @@ -1,10 +1,8 @@ { - dlib, - lib, + pkgs, + utils, ... -}: let - l = lib // builtins; -in { +}: { type = "impure"; # A derivation which outputs a single executable at `$out`. @@ -18,19 +16,17 @@ in { # by the input parameter `outFile`. # The output file must contain the dream lock data encoded as json. # See /src/specifications/dream-lock-example.json - translateBin = { - # dream2nix utils - utils, - # nixpkgs dependencies - bash, - coreutils, - jq, - writeScriptBin, - nix, - callPackage, - python3, - ... - }: + translateBin = let + inherit + (pkgs) + bash + coreutils + jq + nix + callPackage + python3 + ; + in utils.writePureShellScript [ bash diff --git a/src/subsystems/default.nix b/src/subsystems/default.nix deleted file mode 100644 index a8545d5d..00000000 --- a/src/subsystems/default.nix +++ /dev/null @@ -1,36 +0,0 @@ -{ - lib, - dlib, - callPackageDream, - ... -}: let - l = lib // builtins; - inherit (dlib.modules) collectSubsystemModules; - - builders = callPackageDream ./builders.nix {}; - translators = callPackageDream ./translators.nix {}; - discoverers = callPackageDream ./discoverers.nix {}; - - # maps modules - # ex: {rust = ;} -> {rust.translators = ;} - mapModules = modules: name: - l.mapAttrs' - ( - subsystem: modules: - l.nameValuePair subsystem {${name} = modules;} - ) - modules; - - modules = [ - (mapModules builders.builders "builders") - (mapModules translators.translators "translators") - (mapModules discoverers.discoverers "discoverers") - ]; - allModules = l.foldl' l.recursiveUpdate {} modules; -in - allModules - // { - allTranslators = collectSubsystemModules translators.translators; - allBuilders = collectSubsystemModules builders.builders; - allDiscoverers = collectSubsystemModules discoverers.discoverers; - } diff --git a/src/subsystems/discoverers.nix b/src/subsystems/discoverers.nix deleted file mode 100644 index 07ce7e57..00000000 --- a/src/subsystems/discoverers.nix +++ /dev/null @@ -1 +0,0 @@ -{dlib, ...}: dlib.discoverers diff --git a/src/subsystems/racket/discoverers/default/default.nix b/src/subsystems/racket/discoverers/discoverer-racket/default.nix similarity index 100% rename from src/subsystems/racket/discoverers/default/default.nix rename to src/subsystems/racket/discoverers/discoverer-racket/default.nix diff --git a/src/subsystems/rust/discoverers/cargo/default.nix b/src/subsystems/rust/discoverers/cargo/default.nix index 0c87430a..2f5eb35e 100644 --- a/src/subsystems/rust/discoverers/cargo/default.nix +++ b/src/subsystems/rust/discoverers/cargo/default.nix @@ -1,6 +1,6 @@ { - dlib, lib, + dlib, ... }: let l = lib // builtins; diff --git a/src/subsystems/rust/translators/cargo-lock/default.nix b/src/subsystems/rust/translators/cargo-lock/default.nix index e44a0657..d92bde9a 100644 --- a/src/subsystems/rust/translators/cargo-lock/default.nix +++ b/src/subsystems/rust/translators/cargo-lock/default.nix @@ -18,7 +18,7 @@ in { }) ]; - translate = {translatorName, ...}: { + translate = { project, tree, ... @@ -180,8 +180,6 @@ in { in dlib.simpleTranslate2.translate ({...}: { - inherit translatorName; - # relative path of the project within the source tree. location = project.relPath; diff --git a/src/subsystems/rust/translators/cargo-toml/default.nix b/src/subsystems/rust/translators/cargo-toml/default.nix index ea358f13..d22d0a88 100644 --- a/src/subsystems/rust/translators/cargo-toml/default.nix +++ b/src/subsystems/rust/translators/cargo-toml/default.nix @@ -1,23 +1,17 @@ -{ - dlib, - lib, - ... -}: { +framework: { type = "impure"; # the input format is specified in /specifications/translator-call-example.json # this script receives a json file including the input paths and specialArgs - translateBin = { - # dream2nix utils - subsystems, - utils, - # nixpkgs dependencies - coreutils, - jq, - rustPlatform, - ... - }: - utils.writePureShellScript + translateBin = let + inherit + (framework.pkgs) + coreutils + jq + rustPlatform + ; + in + framework.utils.writePureShellScript [ coreutils jq @@ -47,7 +41,7 @@ popd if [ $cargoResult -eq 0 ]; then - ${subsystems.rust.translators.cargo-lock.translateBin} $TMPDIR/newJsonInput + ${framework.translators.cargo-lock.translateBin} $TMPDIR/newJsonInput else echo "cargo failed to generate the lockfile" exit 1 @@ -56,7 +50,7 @@ # inherit options from cargo-lock translator extraArgs = - dlib.translators.translators.rust.cargo-lock.extraArgs + framework.translators.cargo-lock.extraArgs // { cargoArgs = { description = "Additional arguments for Cargo"; diff --git a/src/subsystems/rust/translators/crates-io/default.nix b/src/subsystems/rust/translators/crates-io/default.nix index 62ba2724..5ff30ca2 100644 --- a/src/subsystems/rust/translators/crates-io/default.nix +++ b/src/subsystems/rust/translators/crates-io/default.nix @@ -1,27 +1,26 @@ { - dlib, - lib, + pkgs, + apps, + utils, + translators, ... }: { type = "impure"; # the input format is specified in /specifications/translator-call-example.json # this script receives a json file including the input paths and specialArgs - translateBin = { - # dream2nix utils - apps, - subsystems, - utils, - # nixpkgs dependencies - coreutils, - curl, - gnutar, - gzip, - jq, - moreutils, - rustPlatform, - ... - }: + translateBin = let + inherit + (pkgs) + coreutils + curl + gnutar + gzip + jq + moreutils + rustPlatform + ; + in utils.writePureShellScript [ coreutils @@ -67,9 +66,9 @@ # we don't need to run cargo-toml translator if Cargo.lock exists if [ -f "$TMPDIR/source/Cargo.lock" ]; then - ${subsystems.rust.translators.cargo-lock.translateBin} $TMPDIR/newJsonInput + ${translators.cargo-lock.translateBin} $TMPDIR/newJsonInput else - ${subsystems.rust.translators.cargo-toml.translateBin} $TMPDIR/newJsonInput + ${translators.cargo-toml.translateBin} $TMPDIR/newJsonInput fi # add main package source info to dream-lock.json @@ -85,16 +84,5 @@ ''; # inherit options from cargo-lock translator - extraArgs = - dlib.translators.translators.rust.cargo-lock.extraArgs - // { - cargoArgs = { - description = "Additional arguments for Cargo"; - type = "argument"; - default = ""; - examples = [ - "--verbose" - ]; - }; - }; + extraArgs = translators.cargo-toml.extraArgs; } diff --git a/src/subsystems/translators.nix b/src/subsystems/translators.nix deleted file mode 100644 index 75d976c9..00000000 --- a/src/subsystems/translators.nix +++ /dev/null @@ -1,122 +0,0 @@ -{ - coreutils, - dlib, - jq, - lib, - nix, - pkgs, - python3, - callPackageDream, - externals, - dream2nixWithExternals, - utils, - config, - configFile, - framework, - ... -}: let - b = builtins; - - l = lib // builtins; - - # adds a translateBin to a pure translator - wrapPureTranslator = { - subsystem, - name, - }: let - bin = - utils.writePureShellScript - [ - coreutils - jq - nix - python3 - ] - '' - jsonInputFile=$(realpath $1) - outputFile=$(realpath -m $(jq '.outputFile' -c -r $jsonInputFile)) - - pushd $TMPDIR - - nix eval \ - --option experimental-features "nix-command flakes"\ - --show-trace --impure --raw --expr " - let - dream2nix = import ${dream2nixWithExternals} { - config = ${configFile}; - }; - - translatorArgs = - (builtins.fromJSON - (builtins.unsafeDiscardStringContext (builtins.readFile '''$1'''))); - - dreamLock' = - dream2nix.subsystems.${subsystem}.translators.${name}.translate - translatorArgs; - # simpleTranslate2 puts dream-lock in result - dreamLock = dreamLock'.result or dreamLock'; - in - dream2nix.utils.dreamLock.toJSON - # don't use nix to detect cycles, this will be more efficient in python - (dreamLock // { - _generic = builtins.removeAttrs dreamLock._generic [ \"cyclicDependencies\" ]; - }) - " | python3 ${../apps/cli/format-dream-lock.py} > out - - tmpOut=$(realpath out) - popd - mkdir -p $(dirname $outputFile) - cp $tmpOut $outputFile - ''; - in - bin.overrideAttrs (old: { - name = "translator-${subsystem}-pure-${name}"; - }); - - makeTranslator = translatorModule: let - translator = - translatorModule - # for pure translators - # - import the `translate` function - # - generate `translateBin` - // (lib.optionalAttrs (translatorModule.translate or null != null) { - translate = let - translateOriginal = callPackageDream translatorModule.translate { - translatorName = translatorModule.name; - }; - in - args: - translateOriginal - ( - (framework.functions.translators.makeTranslatorDefaultArgs - (translatorModule.extraArgs or {})) - // args - // (args.project.subsystemInfo or {}) - // { - tree = - args.tree or (dlib.prepareSourceTree {inherit (args) source;}); - } - ); - translateBin = - wrapPureTranslator - {inherit (translatorModule) subsystem name;}; - }) - # for impure translators: - # - import the `translateBin` function - // (lib.optionalAttrs (translatorModule.translateBin or null != null) { - translateBin = - callPackageDream translatorModule.translateBin - { - translatorName = translatorModule.name; - }; - }); - in - translator; - - translators = framework.translatorsBySubsystem; -in { - inherit - translators - makeTranslator - ; -} diff --git a/src/utils/default.nix b/src/utils/default.nix index 05a0d4be..f7fb3687 100644 --- a/src/utils/default.nix +++ b/src/utils/default.nix @@ -19,7 +19,6 @@ callPackageDream, dream2nixWithExternals, externalSources, - subsystems, config, configFile, framework, From fe38f28ba873d4d5c1da5cfcedb486f009329525 Mon Sep 17 00:00:00 2001 From: Yusuf Bera Ertan Date: Thu, 6 Oct 2022 20:42:04 +0300 Subject: [PATCH 02/23] wip --- src/default.nix | 2 +- src/fetchers/crates-io/default.nix | 17 +- src/fetchers/git/default.nix | 13 +- src/fetchers/github/default.nix | 15 +- src/fetchers/gitlab/default.nix | 14 +- src/fetchers/http/default.nix | 17 +- src/fetchers/npm/default.nix | 16 +- src/fetchers/path/default.nix | 6 +- src/fetchers/pypi-sdist/default.nix | 15 +- src/lib.nix | 3 +- src/modules/builders/implementation.nix | 2 +- src/modules/discoverers/implementation.nix | 2 +- src/modules/framework.nix | 8 +- .../functions.subsystem-loading/default.nix | 6 - .../functions.subsystem-loading/interface.nix | 35 - ...plementation.nix => subsystem-loading.nix} | 38 +- src/modules/top-level.nix | 11 +- src/modules/translators/implementation.nix | 2 +- .../builders/simple-haskell/default.nix | 18 +- .../translators/cabal-plan/default.nix | 5 +- .../haskell/translators/hackage/default.nix | 61 +- .../translators/stack-lock/default.nix | 12 +- .../builders/granular-nodejs/default.nix | 918 +++++++++--------- .../nodejs/translators/npm/default.nix | 40 +- .../translators/package-json/default.nix | 34 +- .../translators/package-lock/default.nix | 9 +- .../nodejs/translators/yarn-lock/default.nix | 7 +- .../php/builders/granular-php/default.nix | 19 +- .../php/builders/simple-php/default.nix | 21 +- .../php/translators/composer-json/default.nix | 31 +- .../php/translators/composer-lock/default.nix | 9 +- .../php/translators/packagist/default.nix | 35 +- .../python/builders/simple-python/default.nix | 10 +- .../python/translators/pip/default.nix | 33 +- .../racket/builders/simple-racket/default.nix | 14 +- .../translators/racket-impure/default.nix | 34 +- .../builders/build-rust-package/default.nix | 19 +- .../rust/builders/crane/default.nix | 26 +- .../rust/translators/cargo-lock/default.nix | 2 + .../rust/translators/cargo-toml/default.nix | 15 +- 40 files changed, 760 insertions(+), 834 deletions(-) delete mode 100644 src/modules/functions.subsystem-loading/default.nix delete mode 100644 src/modules/functions.subsystem-loading/interface.nix rename src/modules/{functions.subsystem-loading/implementation.nix => subsystem-loading.nix} (73%) diff --git a/src/default.nix b/src/default.nix index 045c7b53..83599c65 100644 --- a/src/default.nix +++ b/src/default.nix @@ -51,7 +51,7 @@ in let }; framework = import ./modules/framework.nix { - inherit apps lib dlib pkgs utils; + inherit apps lib dlib pkgs utils externals externalSources; dream2nixConfig = config; }; diff --git a/src/fetchers/crates-io/default.nix b/src/fetchers/crates-io/default.nix index 4f2e5a7e..960a03c2 100644 --- a/src/fetchers/crates-io/default.nix +++ b/src/fetchers/crates-io/default.nix @@ -1,18 +1,17 @@ -{lib, ...}: { +{ + pkgs, + utils, + ... +}: { inputs = ["pname" "version"]; versionField = "version"; outputs = { - fetchurl, - runCommandLocal, - utils, - ... - }: { pname, version, ... - } @ inp: let + }: let b = builtins; # See https://github.com/rust-lang/crates.io-index/blob/master/config.json#L2 url = "https://crates.io/api/v1/crates/${pname}/${version}/download"; @@ -23,13 +22,13 @@ }); fetched = hash: let - fetched = fetchurl { + fetched = pkgs.fetchurl { inherit url; sha256 = hash; name = "download-${pname}-${version}"; }; in - runCommandLocal "unpack-${pname}-${version}" {} + pkgs.runCommandLocal "unpack-${pname}-${version}" {} '' mkdir -p $out tar --strip-components 1 -xzf ${fetched} -C $out diff --git a/src/fetchers/git/default.nix b/src/fetchers/git/default.nix index ddfa4988..b0fc43b0 100644 --- a/src/fetchers/git/default.nix +++ b/src/fetchers/git/default.nix @@ -1,4 +1,9 @@ -{lib, ...}: let +{ + pkgs, + lib, + utils, + ... +}: let b = builtins; # check if a string is a git ref @@ -14,10 +19,6 @@ in { versionField = "rev"; outputs = { - fetchgit, - utils, - ... - }: { url, rev, submodules ? true, @@ -78,7 +79,7 @@ in { inherit submodules; }) else - fetchgit + pkgs.fetchgit (refAndRev // { inherit url; diff --git a/src/fetchers/github/default.nix b/src/fetchers/github/default.nix index 38350bfc..45478b98 100644 --- a/src/fetchers/github/default.nix +++ b/src/fetchers/github/default.nix @@ -1,4 +1,9 @@ -{lib, ...}: { +{ + pkgs, + lib, + utils, + ... +}: { inputs = [ "owner" "repo" @@ -10,15 +15,11 @@ defaultUpdater = "githubNewestReleaseTag"; outputs = { - fetchFromGitHub, - utils, - ... - }: { owner, repo, rev, ... - } @ inp: let + }: let b = builtins; in { calcHash = algo: @@ -27,7 +28,7 @@ }); fetched = hash: - fetchFromGitHub { + pkgs.fetchFromGitHub { inherit owner repo rev hash; }; }; diff --git a/src/fetchers/gitlab/default.nix b/src/fetchers/gitlab/default.nix index 53d552f5..47962f3d 100644 --- a/src/fetchers/gitlab/default.nix +++ b/src/fetchers/gitlab/default.nix @@ -1,4 +1,8 @@ -{...}: { +{ + pkgs, + utils, + ... +}: { inputs = [ "owner" "repo" @@ -8,15 +12,11 @@ versionField = "rev"; outputs = { - fetchFromGitLab, - utils, - ... - }: { owner, repo, rev, ... - } @ inp: let + }: let b = builtins; in { calcHash = algo: @@ -25,7 +25,7 @@ }); fetched = hash: - fetchFromGitLab { + pkgs.fetchFromGitLab { inherit owner repo rev hash; }; }; diff --git a/src/fetchers/http/default.nix b/src/fetchers/http/default.nix index f6e8a8b1..958ccf7e 100644 --- a/src/fetchers/http/default.nix +++ b/src/fetchers/http/default.nix @@ -1,13 +1,14 @@ -{lib, ...}: { +{ + pkgs, + lib, + utils, + ... +}: { inputs = [ "url" ]; - outputs = { - fetchurl, - utils, - ... - }: {url, ...} @ inp: let + outputs = {url, ...}: let b = builtins; in { calcHash = algo: @@ -19,12 +20,12 @@ drv = if hash != null && lib.stringLength hash == 40 then - fetchurl { + pkgs.fetchurl { inherit url; sha1 = hash; } else - fetchurl { + pkgs.fetchurl { inherit url hash; }; diff --git a/src/fetchers/npm/default.nix b/src/fetchers/npm/default.nix index 495add9b..f46cb7c7 100644 --- a/src/fetchers/npm/default.nix +++ b/src/fetchers/npm/default.nix @@ -1,4 +1,9 @@ -{lib, ...}: let +{ + pkgs, + lib, + utils, + ... +}: let b = builtins; in rec { inputs = ["pname" "version"]; @@ -36,14 +41,9 @@ in rec { # defaultUpdater = ""; outputs = { - python3, - utils, - fetchurl, - ... - }: { pname, version, - } @ inp: let + }: let b = builtins; submodule = lib.last (lib.splitString "/" pname); @@ -56,7 +56,7 @@ in rec { fetched = hash: let source = - (fetchurl { + (pkgs.fetchurl { inherit url; sha256 = hash; }) diff --git a/src/fetchers/path/default.nix b/src/fetchers/path/default.nix index bfef7104..598174af 100644 --- a/src/fetchers/path/default.nix +++ b/src/fetchers/path/default.nix @@ -1,11 +1,9 @@ -{...}: { +{utils, ...}: { inputs = [ "path" ]; - outputs = {utils, ...}: {path, ...} @ inp: let - b = builtins; - in { + outputs = {path, ...}: { calcHash = algo: utils.hashPath "${path}"; fetched = hash: "${path}"; diff --git a/src/fetchers/pypi-sdist/default.nix b/src/fetchers/pypi-sdist/default.nix index 1aee1c7d..a664a7ce 100644 --- a/src/fetchers/pypi-sdist/default.nix +++ b/src/fetchers/pypi-sdist/default.nix @@ -1,4 +1,8 @@ -{...}: { +{ + pkgs, + utils, + ... +}: { inputs = ["pname" "version"]; versionField = "version"; @@ -6,15 +10,10 @@ defaultUpdater = "pypiNewestReleaseVersion"; outputs = { - fetchurl, - python3, - utils, - ... - }: { pname, version, extension ? "tar.gz", - } @ inp: let + }: let b = builtins; firstChar = builtins.substring 0 1 pname; @@ -29,7 +28,7 @@ fetched = hash: let source = - (fetchurl { + (pkgs.fetchurl { inherit url; sha256 = hash; }) diff --git a/src/lib.nix b/src/lib.nix index 9b357f2c..2a995710 100644 --- a/src/lib.nix +++ b/src/lib.nix @@ -90,11 +90,12 @@ dlib = import ./lib {inherit lib config;}; framework = import ./modules/framework.nix { - inherit lib dlib; + inherit lib dlib externalSources; dream2nixConfig = config; apps = throw "apps is not available before nixpkgs is imported"; pkgs = throw "pkgs is not available before nixpkgs is imported"; utils = throw "utils is not available before nixpkgs is imported"; + externals = throw "externals is not available before nixpkgs is imported"; }; initD2N = initDream2nix config; diff --git a/src/modules/builders/implementation.nix b/src/modules/builders/implementation.nix index 50b436ce..50e2ef10 100644 --- a/src/modules/builders/implementation.nix +++ b/src/modules/builders/implementation.nix @@ -9,7 +9,7 @@ debian = "simple-debian"; racket = "simple-racket"; }; - funcs = config.functions.subsystem-loading; + funcs = import ../subsystem-loading.nix config; collectedModules = funcs.collect "builders"; in { config = { diff --git a/src/modules/discoverers/implementation.nix b/src/modules/discoverers/implementation.nix index e9c68ac9..13edcd9c 100644 --- a/src/modules/discoverers/implementation.nix +++ b/src/modules/discoverers/implementation.nix @@ -1,5 +1,5 @@ {config, ...}: let - funcs = config.functions.subsystem-loading; + funcs = import ../subsystem-loading.nix config; collectedModules = funcs.collect "discoverers"; in { config = { diff --git a/src/modules/framework.nix b/src/modules/framework.nix index 5e9c29d1..15cac543 100644 --- a/src/modules/framework.nix +++ b/src/modules/framework.nix @@ -2,13 +2,13 @@ dream2nixConfig, pkgs, dlib, + externals, + externalSources, lib, utils, apps, -}: let - topLevel = import ./top-level.nix { - inherit apps lib dlib utils pkgs dream2nixConfig; - }; +} @ args: let + topLevel = import ./top-level.nix args; evaledModules = lib.evalModules { modules = [topLevel] ++ (dream2nixConfig.modules or []); }; diff --git a/src/modules/functions.subsystem-loading/default.nix b/src/modules/functions.subsystem-loading/default.nix deleted file mode 100644 index e0a7730f..00000000 --- a/src/modules/functions.subsystem-loading/default.nix +++ /dev/null @@ -1,6 +0,0 @@ -{ - imports = [ - ./implementation.nix - ./interface.nix - ]; -} diff --git a/src/modules/functions.subsystem-loading/interface.nix b/src/modules/functions.subsystem-loading/interface.nix deleted file mode 100644 index c4887d70..00000000 --- a/src/modules/functions.subsystem-loading/interface.nix +++ /dev/null @@ -1,35 +0,0 @@ -{config, ...}: let - lib = config.lib; - t = lib.types; -in { - options.functions.subsystem-loading = { - collect = lib.mkOption { - type = t.functionTo t.anything; - description = '' - Discover modules in /src/subsystems/{subsystem}/{module-type}/{module-name} - ''; - }; - import_ = lib.mkOption { - type = t.functionTo t.anything; - description = '' - Imports discovered module files. - Adds name and subsystem attributes to each module derived from the path. - ''; - }; - instantiate = lib.mkOption { - type = t.functionTo t.anything; - description = '' - To keep module implementations simpler, additional generic logic is added - by a loader. - The loader is subsytem specific and needs to be passed as an argument. - ''; - }; - structureBySubsystem = lib.mkOption { - type = t.functionTo t.anything; - description = '' - re-structures the instantiated instances into a deeper attrset like: - {subsytem}.{module-name} = ... - ''; - }; - }; -} diff --git a/src/modules/functions.subsystem-loading/implementation.nix b/src/modules/subsystem-loading.nix similarity index 73% rename from src/modules/functions.subsystem-loading/implementation.nix rename to src/modules/subsystem-loading.nix index 7ca647eb..b8b02b76 100644 --- a/src/modules/functions.subsystem-loading/implementation.nix +++ b/src/modules/subsystem-loading.nix @@ -5,10 +5,12 @@ translators and builders. This recurses through the following directory structure to discover and load modules: /src/subsystems/{subsystem}/{module-type}/{module-name} + +This is not included in `config.functions` because it causes infinite recursion. */ -{config, ...}: let +config: let inherit (config) dlib lib; - subsystemsDir = lib.toString ../../subsystems; + subsystemsDir = lib.toString ../subsystems; subsystems = dlib.dirNames subsystemsDir; collect = moduleDirName: @@ -35,9 +37,12 @@ modules: import_ = collectedModules: lib.mapAttrs - (name: description: - (import description.path config) - // {inherit (description) name subsystem;}) + ( + name: description: let + attrs = {inherit (description) name subsystem;}; + in + (import description.path (config // attrs)) // attrs + ) ( lib.foldl' ( @@ -54,13 +59,6 @@ modules: collectedModules ); - instantiate = importedModules: loader: - lib.mapAttrs - (name: module: - (loader module) - // {inherit (module) name subsystem;}) - importedModules; - structureBySubsystem = instances: lib.foldl lib.recursiveUpdate @@ -69,15 +67,9 @@ modules: (tName: t: {"${t.subsystem}"."${tName}" = t;}) instances); in { - /* - Expose the functions via via the modules system. - */ - config.functions.subsystem-loading = { - inherit - collect - import_ - instantiate - structureBySubsystem - ; - }; + inherit + collect + import_ + structureBySubsystem + ; } diff --git a/src/modules/top-level.nix b/src/modules/top-level.nix index 82fa86b8..1c59d137 100644 --- a/src/modules/top-level.nix +++ b/src/modules/top-level.nix @@ -1,6 +1,8 @@ { apps, dlib, + externals, + externalSources, lib, pkgs, utils, @@ -14,7 +16,6 @@ in { ./functions.default-fetcher ./functions.combined-fetcher ./functions.translators - ./functions.subsystem-loading ./builders ./discoverers ./discoverers.default-discoverer @@ -31,6 +32,12 @@ in { dlib = lib.mkOption { type = t.raw; }; + externals = lib.mkOption { + type = t.raw; + }; + externalSources = lib.mkOption { + type = t.raw; + }; pkgs = lib.mkOption { type = t.raw; }; @@ -42,7 +49,7 @@ in { }; }; config = { - inherit apps dlib pkgs utils dream2nixConfig; + inherit apps dlib externals externalSources pkgs utils dream2nixConfig; lib = lib // builtins; }; } diff --git a/src/modules/translators/implementation.nix b/src/modules/translators/implementation.nix index 49346d78..003c8367 100644 --- a/src/modules/translators/implementation.nix +++ b/src/modules/translators/implementation.nix @@ -1,5 +1,5 @@ {config, ...}: let - funcs = config.functions.subsystem-loading; + funcs = import ../subsystem-loading.nix config; collectedModules = funcs.collect "translators"; in { config = { diff --git a/src/subsystems/haskell/builders/simple-haskell/default.nix b/src/subsystems/haskell/builders/simple-haskell/default.nix index bb6a8fbf..3eabce17 100644 --- a/src/subsystems/haskell/builders/simple-haskell/default.nix +++ b/src/subsystems/haskell/builders/simple-haskell/default.nix @@ -1,17 +1,15 @@ -{lib, ...}: let +{ + pkgs, + utils, + externals, + lib, + ... +}: let l = lib // builtins; in { type = "pure"; build = { - lib, - pkgs, - stdenv, - # dream2nix inputs - externals, - utils, - ... - }: { ### FUNCTIONS # AttrSet -> Bool) -> AttrSet -> [x] getCyclicDependencies, # name: version: -> [ {name=; version=; } ] @@ -37,8 +35,6 @@ in { produceDerivation, ... } @ args: let - b = builtins; - compiler = pkgs .haskell diff --git a/src/subsystems/haskell/translators/cabal-plan/default.nix b/src/subsystems/haskell/translators/cabal-plan/default.nix index 49730cd4..83989461 100644 --- a/src/subsystems/haskell/translators/cabal-plan/default.nix +++ b/src/subsystems/haskell/translators/cabal-plan/default.nix @@ -1,6 +1,7 @@ { dlib, lib, + name, ... }: let l = lib // builtins; @@ -37,7 +38,7 @@ in { l.pathExists "${tree.fullPath}/dist-newstyle/cache/plan.json"; # translate from a given source and a project specification to a dream-lock. - translate = {translatorName, ...}: { + translate = { /* A list of projects returned by `discoverProjects` Example: @@ -189,7 +190,7 @@ in { in dlib.simpleTranslate2.translate ({objectsByKey, ...}: rec { - inherit translatorName; + translatorName = name; # relative path of the project within the source tree. location = project.relPath; diff --git a/src/subsystems/haskell/translators/hackage/default.nix b/src/subsystems/haskell/translators/hackage/default.nix index f8f7cb87..e29f90d8 100644 --- a/src/subsystems/haskell/translators/hackage/default.nix +++ b/src/subsystems/haskell/translators/hackage/default.nix @@ -1,10 +1,9 @@ { - dlib, - lib, + pkgs, + utils, + translators, ... -}: let - l = lib // builtins; -in { +}: { type = "impure"; # A derivation which outputs a single executable at `$out`. @@ -18,24 +17,21 @@ in { # by the input parameter `outFile`. # The output file must contain the dream lock data encoded as json. # See /src/specifications/dream-lock-example.json - translateBin = { - # dream2nix utils - subsystems, - utils, - # nixpkgs dependenies - bash, - coreutils, - curl, - gnutar, - gzip, - haskellPackages, - jq, - moreutils, - nix, - python3, - writeScriptBin, - ... - }: + translateBin = let + inherit + (pkgs) + bash + coreutils + curl + gnutar + gzip + haskellPackages + jq + moreutils + nix + python3 + ; + in utils.writePureShellScript [ bash @@ -86,7 +82,7 @@ in { popd # execute cabal-plan translator - ${subsystems.haskell.translators.cabal-plan.translateBin} $TMPDIR/newJsonInput + ${translators.cabal-plan.translateBin} $TMPDIR/newJsonInput # finalize dream-lock. Add source and export default package # set correct package version under `packages` @@ -105,22 +101,5 @@ in { # - boolean flag (type = "flag") # String arguments contain a default value and examples. Flags do not. extraArgs = { - # Example: boolean option - # Flags always default to 'false' if not specified by the user - noDev = { - description = "Exclude dev dependencies"; - type = "flag"; - }; - - # Example: string option - theAnswer = { - default = "42"; - description = "The Answer to the Ultimate Question of Life"; - examples = [ - "0" - "1234" - ]; - type = "argument"; - }; }; } diff --git a/src/subsystems/haskell/translators/stack-lock/default.nix b/src/subsystems/haskell/translators/stack-lock/default.nix index 711fbb6d..c1411259 100644 --- a/src/subsystems/haskell/translators/stack-lock/default.nix +++ b/src/subsystems/haskell/translators/stack-lock/default.nix @@ -1,6 +1,9 @@ { dlib, lib, + pkgs, + utils, + name, ... }: let l = lib // builtins; @@ -71,12 +74,7 @@ in { (l.attrNames tree.files); # translate from a given source and a project specification to a dream-lock. - translate = { - translatorName, - pkgs, - utils, - ... - }: let + translate = let stackLockUtils = import ./utils.nix {inherit dlib lib pkgs;}; all-cabal-hashes = let all-cabal-hashes' = pkgs.runCommandLocal "all-cabal-hashes" {} '' @@ -231,7 +229,7 @@ in { in dlib.simpleTranslate2.translate ({objectsByKey, ...}: rec { - inherit translatorName; + translatorName = name; # relative path of the project within the source tree. location = project.relPath; diff --git a/src/subsystems/nodejs/builders/granular-nodejs/default.nix b/src/subsystems/nodejs/builders/granular-nodejs/default.nix index c9a8f4a5..ab4dcf0e 100644 --- a/src/subsystems/nodejs/builders/granular-nodejs/default.nix +++ b/src/subsystems/nodejs/builders/granular-nodejs/default.nix @@ -1,511 +1,515 @@ -{...}: { +{ + pkgs, + lib, + ... +}: { type = "pure"; - build = { - jq, - lib, - makeWrapper, - mkShell, - pkgs, - python3, - runCommandLocal, - stdenv, - writeText, - ... - }: { - # Funcs - # AttrSet -> Bool) -> AttrSet -> [x] - getCyclicDependencies, # name: version: -> [ {name=; version=; } ] - getDependencies, # name: version: -> [ {name=; version=; } ] - getSource, # name: version: -> store-path - # Attributes - subsystemAttrs, # attrset - defaultPackageName, # string - defaultPackageVersion, # string - packages, # list - # attrset of pname -> versions, - # where versions is a list of version strings - packageVersions, - # function which applies overrides to a package - # It must be applied by the builder to each individual derivation - # Example: - # produceDerivation name (mkDerivation {...}) - produceDerivation, - nodejs ? null, - ... - } @ args: let - b = builtins; - l = lib // builtins; + build = let + inherit + (pkgs) + jq + makeWrapper + mkShell + python3 + runCommandLocal + stdenv + ; + in + { + # Funcs + # AttrSet -> Bool) -> AttrSet -> [x] + getCyclicDependencies, # name: version: -> [ {name=; version=; } ] + getDependencies, # name: version: -> [ {name=; version=; } ] + getSource, # name: version: -> store-path + # Attributes + subsystemAttrs, # attrset + defaultPackageName, # string + defaultPackageVersion, # string + packages, # list + # attrset of pname -> versions, + # where versions is a list of version strings + packageVersions, + # function which applies overrides to a package + # It must be applied by the builder to each individual derivation + # Example: + # produceDerivation name (mkDerivation {...}) + produceDerivation, + nodejs ? null, + ... + } @ args: let + b = builtins; + l = lib // builtins; - nodejsVersion = subsystemAttrs.nodejsVersion; + nodejsVersion = subsystemAttrs.nodejsVersion; - isMainPackage = name: version: - (args.packages."${name}" or null) == version; + isMainPackage = name: version: + (args.packages."${name}" or null) == version; - nodejs = - if args ? nodejs - then b.toString args.nodejs - else - pkgs."nodejs-${nodejsVersion}_x" - or (throw "Could not find nodejs version '${nodejsVersion}' in pkgs"); + nodejs = + if args ? nodejs + then b.toString args.nodejs + else + pkgs."nodejs-${nodejsVersion}_x" + or (throw "Could not find nodejs version '${nodejsVersion}' in pkgs"); - nodeSources = runCommandLocal "node-sources" {} '' - tar --no-same-owner --no-same-permissions -xf ${nodejs.src} - mv node-* $out - ''; - - allPackages = - lib.mapAttrs - (name: versions: - lib.genAttrs - versions - (version: - makePackage name version)) - packageVersions; - - outputs = rec { - # select only the packages listed in dreamLock as main packages - packages = - b.foldl' - (ps: p: ps // p) - {} - (lib.mapAttrsToList - (name: version: { - "${name}"."${version}" = allPackages."${name}"."${version}"; - }) - args.packages); - - devShells = - {default = devShells.${defaultPackageName};} - // ( - l.mapAttrs - (name: version: allPackages.${name}.${version}.devShell) - args.packages - ); - }; - - # This is only executed for electron based packages. - # Electron ships its own version of node, requiring a rebuild of native - # extensions. - # Theoretically this requires headers for the exact electron version in use, - # but we use the headers from nixpkgs' electron instead which might have a - # different minor version. - # Alternatively the headers can be specified via `electronHeaders`. - # Also a custom electron version can be specified via `electronPackage` - electron-rebuild = '' - # prepare node headers for electron - if [ -n "$electronPackage" ]; then - export electronDist="$electronPackage/lib/electron" - else - export electronDist="$nodeModules/$packageName/node_modules/electron/dist" - fi - local ver - ver="v$(cat $electronDist/version | tr -d '\n')" - mkdir $TMP/$ver - cp $electronHeaders $TMP/$ver/node-$ver-headers.tar.gz - - # calc checksums - cd $TMP/$ver - sha256sum ./* > SHASUMS256.txt - cd - - - # serve headers via http - python -m http.server 45034 --directory $TMP & - - # copy electron distribution - cp -r $electronDist $TMP/electron - chmod -R +w $TMP/electron - - # configure electron toolchain - ${pkgs.jq}/bin/jq ".build.electronDist = \"$TMP/electron\"" package.json \ - | ${pkgs.moreutils}/bin/sponge package.json - - ${pkgs.jq}/bin/jq ".build.linux.target = \"dir\"" package.json \ - | ${pkgs.moreutils}/bin/sponge package.json - - ${pkgs.jq}/bin/jq ".build.npmRebuild = false" package.json \ - | ${pkgs.moreutils}/bin/sponge package.json - - # execute electron-rebuild if available - export headers=http://localhost:45034/ - if command -v electron-rebuild &> /dev/null; then - pushd $electronAppDir - - electron-rebuild -d $headers - popd - fi - ''; - - # Only executed for electron based packages. - # Creates an executable script under /bin starting the electron app - electron-wrap = - if pkgs.stdenv.isLinux - then '' - mkdir -p $out/bin - makeWrapper \ - $electronDist/electron \ - $out/bin/$(basename "$packageName") \ - --add-flags "$(realpath $electronAppDir)" - '' - else '' - mkdir -p $out/bin - makeWrapper \ - $electronDist/Electron.app/Contents/MacOS/Electron \ - $out/bin/$(basename "$packageName") \ - --add-flags "$(realpath $electronAppDir)" + nodeSources = runCommandLocal "node-sources" {} '' + tar --no-same-owner --no-same-permissions -xf ${nodejs.src} + mv node-* $out ''; - # Generates a derivation for a specific package name + version - makePackage = name: version: let - pname = lib.replaceStrings ["@" "/"] ["__at__" "__slash__"] name; + allPackages = + lib.mapAttrs + (name: versions: + lib.genAttrs + versions + (version: + makePackage name version)) + packageVersions; - deps = getDependencies name version; + outputs = rec { + # select only the packages listed in dreamLock as main packages + packages = + b.foldl' + (ps: p: ps // p) + {} + (lib.mapAttrsToList + (name: version: { + "${name}"."${version}" = allPackages."${name}"."${version}"; + }) + args.packages); - nodeDeps = - lib.forEach - deps - (dep: allPackages."${dep.name}"."${dep.version}"); + devShells = + {default = devShells.${defaultPackageName};} + // ( + l.mapAttrs + (name: version: allPackages.${name}.${version}.devShell) + args.packages + ); + }; - passthruDeps = - l.listToAttrs - (l.forEach deps - (dep: - l.nameValuePair - dep.name - allPackages."${dep.name}"."${dep.version}")); - - dependenciesJson = - b.toJSON - (lib.listToAttrs - (b.map - (dep: lib.nameValuePair dep.name dep.version) - deps)); - - electronDep = - if ! isMainPackage name version - then null + # This is only executed for electron based packages. + # Electron ships its own version of node, requiring a rebuild of native + # extensions. + # Theoretically this requires headers for the exact electron version in use, + # but we use the headers from nixpkgs' electron instead which might have a + # different minor version. + # Alternatively the headers can be specified via `electronHeaders`. + # Also a custom electron version can be specified via `electronPackage` + electron-rebuild = '' + # prepare node headers for electron + if [ -n "$electronPackage" ]; then + export electronDist="$electronPackage/lib/electron" else - lib.findFirst - (dep: dep.name == "electron") - null - deps; + export electronDist="$nodeModules/$packageName/node_modules/electron/dist" + fi + local ver + ver="v$(cat $electronDist/version | tr -d '\n')" + mkdir $TMP/$ver + cp $electronHeaders $TMP/$ver/node-$ver-headers.tar.gz - electronVersionMajor = - lib.versions.major electronDep.version; + # calc checksums + cd $TMP/$ver + sha256sum ./* > SHASUMS256.txt + cd - - electronHeaders = - if - (electronDep == null) - # hashes seem unavailable for electron < 4 - || ((l.toInt electronVersionMajor) <= 2) - then null - else pkgs."electron_${electronVersionMajor}".headers; + # serve headers via http + python -m http.server 45034 --directory $TMP & - pkg = produceDerivation name (stdenv.mkDerivation rec { - inherit - dependenciesJson - electronHeaders - nodeDeps - nodeSources - version - ; + # copy electron distribution + cp -r $electronDist $TMP/electron + chmod -R +w $TMP/electron - packageName = name; + # configure electron toolchain + ${pkgs.jq}/bin/jq ".build.electronDist = \"$TMP/electron\"" package.json \ + | ${pkgs.moreutils}/bin/sponge package.json - inherit pname; + ${pkgs.jq}/bin/jq ".build.linux.target = \"dir\"" package.json \ + | ${pkgs.moreutils}/bin/sponge package.json - meta = let - meta = subsystemAttrs.meta; - in - meta - // { - license = l.map (name: l.licenses.${name}) meta.license; + ${pkgs.jq}/bin/jq ".build.npmRebuild = false" package.json \ + | ${pkgs.moreutils}/bin/sponge package.json + + # execute electron-rebuild if available + export headers=http://localhost:45034/ + if command -v electron-rebuild &> /dev/null; then + pushd $electronAppDir + + electron-rebuild -d $headers + popd + fi + ''; + + # Only executed for electron based packages. + # Creates an executable script under /bin starting the electron app + electron-wrap = + if pkgs.stdenv.isLinux + then '' + mkdir -p $out/bin + makeWrapper \ + $electronDist/electron \ + $out/bin/$(basename "$packageName") \ + --add-flags "$(realpath $electronAppDir)" + '' + else '' + mkdir -p $out/bin + makeWrapper \ + $electronDist/Electron.app/Contents/MacOS/Electron \ + $out/bin/$(basename "$packageName") \ + --add-flags "$(realpath $electronAppDir)" + ''; + + # Generates a derivation for a specific package name + version + makePackage = name: version: let + pname = lib.replaceStrings ["@" "/"] ["__at__" "__slash__"] name; + + deps = getDependencies name version; + + nodeDeps = + lib.forEach + deps + (dep: allPackages."${dep.name}"."${dep.version}"); + + passthruDeps = + l.listToAttrs + (l.forEach deps + (dep: + l.nameValuePair + dep.name + allPackages."${dep.name}"."${dep.version}")); + + dependenciesJson = + b.toJSON + (lib.listToAttrs + (b.map + (dep: lib.nameValuePair dep.name dep.version) + deps)); + + electronDep = + if ! isMainPackage name version + then null + else + lib.findFirst + (dep: dep.name == "electron") + null + deps; + + electronVersionMajor = + lib.versions.major electronDep.version; + + electronHeaders = + if + (electronDep == null) + # hashes seem unavailable for electron < 4 + || ((l.toInt electronVersionMajor) <= 2) + then null + else pkgs."electron_${electronVersionMajor}".headers; + + pkg = produceDerivation name (stdenv.mkDerivation rec { + inherit + dependenciesJson + electronHeaders + nodeDeps + nodeSources + version + ; + + packageName = name; + + inherit pname; + + meta = let + meta = subsystemAttrs.meta; + in + meta + // { + license = l.map (name: l.licenses.${name}) meta.license; + }; + + passthru.dependencies = passthruDeps; + + passthru.devShell = import ./devShell.nix { + inherit + mkShell + nodejs + packageName + pkg + ; }; - passthru.dependencies = passthruDeps; + /* + For top-level packages install dependencies as full copies, as this + reduces errors with build tooling that doesn't cope well with + symlinking. + */ + installMethod = + if isMainPackage name version + then "copy" + else "symlink"; - passthru.devShell = import ./devShell.nix { - inherit - mkShell - nodejs - packageName - pkg - ; - }; + electronAppDir = "."; - /* - For top-level packages install dependencies as full copies, as this - reduces errors with build tooling that doesn't cope well with - symlinking. - */ - installMethod = - if isMainPackage name version - then "copy" - else "symlink"; + # only run build on the main package + runBuild = isMainPackage name version; - electronAppDir = "."; + src = getSource name version; - # only run build on the main package - runBuild = isMainPackage name version; + nativeBuildInputs = [makeWrapper]; - src = getSource name version; + buildInputs = [jq nodejs python3]; - nativeBuildInputs = [makeWrapper]; + # prevents running into ulimits + passAsFile = ["dependenciesJson" "nodeDeps"]; - buildInputs = [jq nodejs python3]; + preConfigurePhases = ["d2nLoadFuncsPhase" "d2nPatchPhase"]; - # prevents running into ulimits - passAsFile = ["dependenciesJson" "nodeDeps"]; + # can be overridden to define alternative install command + # (defaults to 'npm run postinstall') + buildScript = null; - preConfigurePhases = ["d2nLoadFuncsPhase" "d2nPatchPhase"]; + # python script to modify some metadata to support installation + # (see comments below on d2nPatchPhase) + fixPackage = "${./fix-package.py}"; - # can be overridden to define alternative install command - # (defaults to 'npm run postinstall') - buildScript = null; + # script to install (symlink or copy) dependencies. + installDeps = "${./install-deps.py}"; - # python script to modify some metadata to support installation - # (see comments below on d2nPatchPhase) - fixPackage = "${./fix-package.py}"; + # python script to link bin entries from package.json + linkBins = "${./link-bins.py}"; - # script to install (symlink or copy) dependencies. - installDeps = "${./install-deps.py}"; + # costs performance and doesn't seem beneficial in most scenarios + dontStrip = true; - # python script to link bin entries from package.json - linkBins = "${./link-bins.py}"; + # declare some useful shell functions + d2nLoadFuncsPhase = '' + # function to resolve symlinks to copies + symlinksToCopies() { + local dir="$1" - # costs performance and doesn't seem beneficial in most scenarios - dontStrip = true; - - # declare some useful shell functions - d2nLoadFuncsPhase = '' - # function to resolve symlinks to copies - symlinksToCopies() { - local dir="$1" - - echo "transforming symlinks to copies..." - for f in $(find -L "$dir" -xtype l); do - if [ -f $f ]; then - continue - fi - echo "copying $f" - chmod +wx $(dirname "$f") - mv "$f" "$f.bak" - mkdir "$f" - if [ -n "$(ls -A "$f.bak/")" ]; then - cp -r "$f.bak"/* "$f/" - chmod -R +w $f - fi - rm "$f.bak" - done - } - ''; - - # TODO: upstream fix to nixpkgs - # example which requires this: - # https://registry.npmjs.org/react-window-infinite-loader/-/react-window-infinite-loader-1.0.7.tgz - unpackCmd = - if lib.hasSuffix ".tgz" src - then "tar --delay-directory-restore -xf $src" - else null; - - unpackPhase = '' - runHook preUnpack - - nodeModules=$out/lib/node_modules - - export sourceRoot="$nodeModules/$packageName" - - # sometimes tarballs do not end with .tar.?? - unpackFallback(){ - local fn="$1" - tar xf "$fn" - } - - unpackCmdHooks+=(unpackFallback) - - unpackFile $src - - # Make the base dir in which the target dependency resides in first - mkdir -p "$(dirname "$sourceRoot")" - - # install source - if [ -f "$src" ] - then - # Figure out what directory has been unpacked - export packageDir="$(find . -maxdepth 1 -type d | tail -1)" - - # Restore write permissions - find "$packageDir" -type d -exec chmod u+x {} \; - chmod -R u+w -- "$packageDir" - - # Move the extracted tarball into the output folder - mv -- "$packageDir" "$sourceRoot" - elif [ -d "$src" ] - then - export strippedName="$(stripHash $src)" - - # Restore write permissions - chmod -R u+w -- "$strippedName" - - # Move the extracted directory into the output folder - mv -- "$strippedName" "$sourceRoot" - fi - - runHook postUnpack - ''; - - # The python script wich is executed in this phase: - # - ensures that the package is compatible to the current system - # - ensures the main version in package.json matches the expected - # - pins dependency versions in package.json - # (some npm commands might otherwise trigger networking) - # - creates symlinks for executables declared in package.json - # Apart from that: - # - Any usage of 'link:' in package.json is replaced with 'file:' - # - If package-lock.json exists, it is deleted, as it might conflict - # with the parent package-lock.json. - d2nPatchPhase = '' - # delete package-lock.json as it can lead to conflicts - rm -f package-lock.json - - # repair 'link:' -> 'file:' - mv $nodeModules/$packageName/package.json $nodeModules/$packageName/package.json.old - cat $nodeModules/$packageName/package.json.old | sed 's!link:!file\:!g' > $nodeModules/$packageName/package.json - rm $nodeModules/$packageName/package.json.old - - # run python script (see comment above): - cp package.json package.json.bak - python $fixPackage \ - || \ - # exit code 3 -> the package is incompatible to the current platform - # -> Let the build succeed, but don't create lib/node_packages - if [ "$?" == "3" ]; then - rm -r $out/* - echo "Not compatible with system $system" > $out/error - exit 0 - else - exit 1 - fi - - # configure typescript - if [ -f ./tsconfig.json ] \ - && node -e 'require("typescript")' &>/dev/null; then - node ${./tsconfig-to-json.js} - ${pkgs.jq}/bin/jq ".compilerOptions.preserveSymlinks = true" tsconfig.json \ - | ${pkgs.moreutils}/bin/sponge tsconfig.json - fi - ''; - - # - installs dependencies into the node_modules directory - # - adds executables of direct node module dependencies to PATH - # - adds the current node module to NODE_PATH - # - sets HOME=$TMPDIR, as this is required by some npm scripts - # TODO: don't install dev dependencies. Load into NODE_PATH instead - configurePhase = '' - runHook preConfigure - - # symlink sub dependencies as well as this imitates npm better - python $installDeps - - echo "Symlinking transitive executables to $nodeModules/.bin" - for dep in ${l.toString nodeDeps}; do - binDir=$dep/lib/node_modules/.bin - if [ -e $binDir ]; then - for bin in $(ls $binDir/); do\ - if [ ! -e $nodeModules/.bin ]; then - mkdir -p $nodeModules/.bin + echo "transforming symlinks to copies..." + for f in $(find -L "$dir" -xtype l); do + if [ -f $f ]; then + continue fi - - # symlink might have been already created by install-deps.py - # if installMethod=copy was selected - if [ ! -L $nodeModules/.bin/$bin ]; then - ln -s $binDir/$bin $nodeModules/.bin/$bin - else - echo "won't overwrite existing symlink $nodeModules/.bin/$bin. current target: $(readlink $nodeModules/.bin/$bin)" + echo "copying $f" + chmod +wx $(dirname "$f") + mv "$f" "$f.bak" + mkdir "$f" + if [ -n "$(ls -A "$f.bak/")" ]; then + cp -r "$f.bak"/* "$f/" + chmod -R +w $f fi + rm "$f.bak" done + } + ''; + + # TODO: upstream fix to nixpkgs + # example which requires this: + # https://registry.npmjs.org/react-window-infinite-loader/-/react-window-infinite-loader-1.0.7.tgz + unpackCmd = + if lib.hasSuffix ".tgz" src + then "tar --delay-directory-restore -xf $src" + else null; + + unpackPhase = '' + runHook preUnpack + + nodeModules=$out/lib/node_modules + + export sourceRoot="$nodeModules/$packageName" + + # sometimes tarballs do not end with .tar.?? + unpackFallback(){ + local fn="$1" + tar xf "$fn" + } + + unpackCmdHooks+=(unpackFallback) + + unpackFile $src + + # Make the base dir in which the target dependency resides in first + mkdir -p "$(dirname "$sourceRoot")" + + # install source + if [ -f "$src" ] + then + # Figure out what directory has been unpacked + export packageDir="$(find . -maxdepth 1 -type d | tail -1)" + + # Restore write permissions + find "$packageDir" -type d -exec chmod u+x {} \; + chmod -R u+w -- "$packageDir" + + # Move the extracted tarball into the output folder + mv -- "$packageDir" "$sourceRoot" + elif [ -d "$src" ] + then + export strippedName="$(stripHash $src)" + + # Restore write permissions + chmod -R u+w -- "$strippedName" + + # Move the extracted directory into the output folder + mv -- "$strippedName" "$sourceRoot" fi - done - # add bin path entries collected by python script - export PATH="$PATH:$nodeModules/.bin" + runHook postUnpack + ''; - # add dependencies to NODE_PATH - export NODE_PATH="$NODE_PATH:$nodeModules/$packageName/node_modules" + # The python script wich is executed in this phase: + # - ensures that the package is compatible to the current system + # - ensures the main version in package.json matches the expected + # - pins dependency versions in package.json + # (some npm commands might otherwise trigger networking) + # - creates symlinks for executables declared in package.json + # Apart from that: + # - Any usage of 'link:' in package.json is replaced with 'file:' + # - If package-lock.json exists, it is deleted, as it might conflict + # with the parent package-lock.json. + d2nPatchPhase = '' + # delete package-lock.json as it can lead to conflicts + rm -f package-lock.json - export HOME=$TMPDIR + # repair 'link:' -> 'file:' + mv $nodeModules/$packageName/package.json $nodeModules/$packageName/package.json.old + cat $nodeModules/$packageName/package.json.old | sed 's!link:!file\:!g' > $nodeModules/$packageName/package.json + rm $nodeModules/$packageName/package.json.old - runHook postConfigure - ''; - - # Runs the install command which defaults to 'npm run postinstall'. - # Allows using custom install command by overriding 'buildScript'. - buildPhase = '' - runHook preBuild - - # execute electron-rebuild - if [ -n "$electronHeaders" ]; then - echo "executing electron-rebuild" - ${electron-rebuild} - fi - - # execute install command - if [ -n "$buildScript" ]; then - if [ -f "$buildScript" ]; then - $buildScript + # run python script (see comment above): + cp package.json package.json.bak + python $fixPackage \ + || \ + # exit code 3 -> the package is incompatible to the current platform + # -> Let the build succeed, but don't create lib/node_packages + if [ "$?" == "3" ]; then + rm -r $out/* + echo "Not compatible with system $system" > $out/error + exit 0 else - eval "$buildScript" + exit 1 fi - # by default, only for top level packages, `npm run build` is executed - elif [ -n "$runBuild" ] && [ "$(jq '.scripts.build' ./package.json)" != "null" ]; then - npm run build - else - if [ "$(jq '.scripts.preinstall' ./package.json)" != "null" ]; then - npm --production --offline --nodedir=$nodeSources run preinstall + + # configure typescript + if [ -f ./tsconfig.json ] \ + && node -e 'require("typescript")' &>/dev/null; then + node ${./tsconfig-to-json.js} + ${pkgs.jq}/bin/jq ".compilerOptions.preserveSymlinks = true" tsconfig.json \ + | ${pkgs.moreutils}/bin/sponge tsconfig.json fi - if [ "$(jq '.scripts.install' ./package.json)" != "null" ]; then - npm --production --offline --nodedir=$nodeSources run install - fi - if [ "$(jq '.scripts.postinstall' ./package.json)" != "null" ]; then - npm --production --offline --nodedir=$nodeSources run postinstall - fi - fi + ''; - runHook postBuild - ''; + # - installs dependencies into the node_modules directory + # - adds executables of direct node module dependencies to PATH + # - adds the current node module to NODE_PATH + # - sets HOME=$TMPDIR, as this is required by some npm scripts + # TODO: don't install dev dependencies. Load into NODE_PATH instead + configurePhase = '' + runHook preConfigure - # Symlinks executables and manual pages to correct directories - installPhase = '' - runHook preInstall + # symlink sub dependencies as well as this imitates npm better + python $installDeps - echo "Symlinking bin entries from package.json" - python $linkBins + echo "Symlinking transitive executables to $nodeModules/.bin" + for dep in ${l.toString nodeDeps}; do + binDir=$dep/lib/node_modules/.bin + if [ -e $binDir ]; then + for bin in $(ls $binDir/); do\ + if [ ! -e $nodeModules/.bin ]; then + mkdir -p $nodeModules/.bin + fi - echo "Symlinking manual pages" - if [ -d "$nodeModules/$packageName/man" ] - then - mkdir -p $out/share - for dir in "$nodeModules/$packageName/man/"* - do - mkdir -p $out/share/man/$(basename "$dir") - for page in "$dir"/* - do - ln -s $page $out/share/man/$(basename "$dir") - done + # symlink might have been already created by install-deps.py + # if installMethod=copy was selected + if [ ! -L $nodeModules/.bin/$bin ]; then + ln -s $binDir/$bin $nodeModules/.bin/$bin + else + echo "won't overwrite existing symlink $nodeModules/.bin/$bin. current target: $(readlink $nodeModules/.bin/$bin)" + fi + done + fi done - fi - # wrap electron app - if [ -n "$electronHeaders" ]; then - echo "Wrapping electron app" - ${electron-wrap} - fi + # add bin path entries collected by python script + export PATH="$PATH:$nodeModules/.bin" - runHook postInstall - ''; - }); + # add dependencies to NODE_PATH + export NODE_PATH="$NODE_PATH:$nodeModules/$packageName/node_modules" + + export HOME=$TMPDIR + + runHook postConfigure + ''; + + # Runs the install command which defaults to 'npm run postinstall'. + # Allows using custom install command by overriding 'buildScript'. + buildPhase = '' + runHook preBuild + + # execute electron-rebuild + if [ -n "$electronHeaders" ]; then + echo "executing electron-rebuild" + ${electron-rebuild} + fi + + # execute install command + if [ -n "$buildScript" ]; then + if [ -f "$buildScript" ]; then + $buildScript + else + eval "$buildScript" + fi + # by default, only for top level packages, `npm run build` is executed + elif [ -n "$runBuild" ] && [ "$(jq '.scripts.build' ./package.json)" != "null" ]; then + npm run build + else + if [ "$(jq '.scripts.preinstall' ./package.json)" != "null" ]; then + npm --production --offline --nodedir=$nodeSources run preinstall + fi + if [ "$(jq '.scripts.install' ./package.json)" != "null" ]; then + npm --production --offline --nodedir=$nodeSources run install + fi + if [ "$(jq '.scripts.postinstall' ./package.json)" != "null" ]; then + npm --production --offline --nodedir=$nodeSources run postinstall + fi + fi + + runHook postBuild + ''; + + # Symlinks executables and manual pages to correct directories + installPhase = '' + runHook preInstall + + echo "Symlinking bin entries from package.json" + python $linkBins + + echo "Symlinking manual pages" + if [ -d "$nodeModules/$packageName/man" ] + then + mkdir -p $out/share + for dir in "$nodeModules/$packageName/man/"* + do + mkdir -p $out/share/man/$(basename "$dir") + for page in "$dir"/* + do + ln -s $page $out/share/man/$(basename "$dir") + done + done + fi + + # wrap electron app + if [ -n "$electronHeaders" ]; then + echo "Wrapping electron app" + ${electron-wrap} + fi + + runHook postInstall + ''; + }); + in + pkg; in - pkg; - in - outputs; + outputs; } diff --git a/src/subsystems/nodejs/translators/npm/default.nix b/src/subsystems/nodejs/translators/npm/default.nix index 157648c9..2d63a7af 100644 --- a/src/subsystems/nodejs/translators/npm/default.nix +++ b/src/subsystems/nodejs/translators/npm/default.nix @@ -1,29 +1,27 @@ { - dlib, - lib, + apps, + utils, + pkgs, + translators, ... }: { type = "impure"; # the input format is specified in /specifications/translator-call-example.json # this script receives a json file including the input paths and specialArgs - translateBin = { - # dream2nix utils - apps, - subsystems, - utils, - # nixpkgs dependencies - bash, - coreutils, - git, - jq, - moreutils, - nodePackages, - openssh, - python3, - writeScriptBin, - ... - }: + translateBin = let + inherit + (pkgs) + bash + coreutils + git + jq + moreutils + nodePackages + openssh + python3 + ; + in utils.writePureShellScript [ bash @@ -64,7 +62,7 @@ popd # call package-lock translator - ${subsystems.nodejs.translators.package-lock.translateBin} $TMPDIR/newJsonInput + ${translators.package-lock.translateBin} $TMPDIR/newJsonInput # get resolved package version export version=$(npm view $candidate version) @@ -78,7 +76,7 @@ # inherit options from package-lock translator extraArgs = - dlib.translators.translators.nodejs.package-lock.extraArgs + translators.package-lock.extraArgs // { npmArgs = { description = "Additional arguments for npm"; diff --git a/src/subsystems/nodejs/translators/package-json/default.nix b/src/subsystems/nodejs/translators/package-json/default.nix index ec8cfc3e..13ec3b79 100644 --- a/src/subsystems/nodejs/translators/package-json/default.nix +++ b/src/subsystems/nodejs/translators/package-json/default.nix @@ -1,26 +1,24 @@ { - dlib, - lib, + utils, + pkgs, + translators, ... }: { type = "impure"; # the input format is specified in /specifications/translator-call-example.json # this script receives a json file including the input paths and specialArgs - translateBin = { - # dream2nix utils - subsystems, - utils, - # nixpkgs dependencies - bash, - coreutils, - git, - jq, - nodePackages, - openssh, - writeScriptBin, - ... - }: + translateBin = let + inherit + (pkgs) + bash + coreutils + git + jq + nodePackages + openssh + ; + in utils.writePureShellScript [ bash @@ -62,12 +60,12 @@ jq ".source = \"$newSource\"" -c -r $jsonInput > $TMPDIR/newJsonInput popd - ${subsystems.nodejs.translators.package-lock.translateBin} $TMPDIR/newJsonInput + ${translators.package-lock.translateBin} $TMPDIR/newJsonInput ''; # inherit options from package-lock translator extraArgs = - dlib.translators.translators.nodejs.package-lock.extraArgs + translators.package-lock.extraArgs // { npmArgs = { description = "Additional arguments for npm"; diff --git a/src/subsystems/nodejs/translators/package-lock/default.nix b/src/subsystems/nodejs/translators/package-lock/default.nix index 940e2b25..7e8c8354 100644 --- a/src/subsystems/nodejs/translators/package-lock/default.nix +++ b/src/subsystems/nodejs/translators/package-lock/default.nix @@ -1,9 +1,10 @@ { dlib, lib, + utils, + name, ... }: let - b = builtins; l = lib // builtins; nodejsUtils = import ../utils.nix {inherit dlib lib;}; @@ -11,10 +12,6 @@ nodejsUtils.getWorkspaceLockFile tree project "package-lock.json"; translate = { - translatorName, - utils, - ... - }: { project, source, tree, @@ -146,7 +143,7 @@ dependenciesByOriginalID, ... }: rec { - inherit translatorName; + translatorName = name; location = relPath; # values diff --git a/src/subsystems/nodejs/translators/yarn-lock/default.nix b/src/subsystems/nodejs/translators/yarn-lock/default.nix index 9536ebbc..8fd0d66d 100644 --- a/src/subsystems/nodejs/translators/yarn-lock/default.nix +++ b/src/subsystems/nodejs/translators/yarn-lock/default.nix @@ -1,6 +1,7 @@ { dlib, lib, + name, ... }: let l = lib // builtins; @@ -10,7 +11,7 @@ getYarnLock = tree: project: nodejsUtils.getWorkspaceLockFile tree project "yarn.lock"; - translate = {translatorName, ...}: { + translate = { project, source, tree, @@ -130,7 +131,9 @@ then "path" else "http"; in rec { - inherit defaultPackage translatorName; + inherit defaultPackage; + + translatorName = name; location = relPath; diff --git a/src/subsystems/php/builders/granular-php/default.nix b/src/subsystems/php/builders/granular-php/default.nix index a301d638..f37619bd 100644 --- a/src/subsystems/php/builders/granular-php/default.nix +++ b/src/subsystems/php/builders/granular-php/default.nix @@ -1,15 +1,12 @@ -{...}: { +{ + lib, + pkgs, + externals, + ... +}: { type = "pure"; build = { - lib, - pkgs, - stdenvNoCC, - # dream2nix inputs - externals, - callPackageDream, - ... - }: { ### FUNCTIONS # AttrSet -> Bool) -> AttrSet -> [x] getCyclicDependencies, # name: version: -> [ {name=; version=; } ] @@ -37,7 +34,7 @@ } @ args: let l = lib // builtins; - inherit (callPackageDream ../../semver.nix {}) satisfies; + inherit (pkgs.callPackage ../../semver.nix {}) satisfies; # php with required extensions php = @@ -142,7 +139,7 @@ then "0.0.0" else version; - pkg = stdenvNoCC.mkDerivation rec { + pkg = pkgs.stdenvNoCC.mkDerivation rec { pname = l.strings.sanitizeDerivationName name; inherit version; diff --git a/src/subsystems/php/builders/simple-php/default.nix b/src/subsystems/php/builders/simple-php/default.nix index 0883a3bb..64923fdf 100644 --- a/src/subsystems/php/builders/simple-php/default.nix +++ b/src/subsystems/php/builders/simple-php/default.nix @@ -1,15 +1,12 @@ -{...}: { +{ + lib, + pkgs, + externals, + ... +}: { type = "pure"; build = { - lib, - pkgs, - stdenvNoCC, - # dream2nix inputs - externals, - callPackageDream, - ... - }: { ### FUNCTIONS # AttrSet -> Bool) -> AttrSet -> [x] getCyclicDependencies, # name: version: -> [ {name=; version=; } ] @@ -37,7 +34,7 @@ } @ args: let l = lib // builtins; - inherit (callPackageDream ../../semver.nix {}) satisfies; + inherit (pkgs.callPackage ../../semver.nix {}) satisfies; # php with required extensions php = @@ -78,7 +75,7 @@ # Unpack dependency cleanDependency = name: version: - stdenvNoCC.mkDerivation rec { + pkgs.stdenvNoCC.mkDerivation rec { pname = "${l.strings.sanitizeDerivationName name}-source"; inherit version; versionString = @@ -140,7 +137,7 @@ then "0.0.0" else version; - pkg = stdenvNoCC.mkDerivation rec { + pkg = pkgs.stdenvNoCC.mkDerivation rec { pname = l.strings.sanitizeDerivationName name; inherit version; diff --git a/src/subsystems/php/translators/composer-json/default.nix b/src/subsystems/php/translators/composer-json/default.nix index 27fd4b3f..8d7ae010 100644 --- a/src/subsystems/php/translators/composer-json/default.nix +++ b/src/subsystems/php/translators/composer-json/default.nix @@ -1,6 +1,7 @@ { - dlib, - lib, + pkgs, + utils, + translators, ... }: { type = "impure"; @@ -16,18 +17,16 @@ # by the input parameter `outFile`. # The output file must contain the dream lock data encoded as json. # See /src/specifications/dream-lock-example.json - translateBin = { - # dream2nix utils - subsystems, - utils, - # nixpkgs dependencies - bash, - coreutils, - moreutils, - jq, - phpPackages, - ... - }: + translateBin = let + inherit + (pkgs) + bash + coreutils + moreutils + jq + phpPackages + ; + in utils.writePureShellScript [ bash @@ -76,9 +75,9 @@ jq ".source = \"$newSource\"" -c -r $jsonInput > $TMPDIR/newJsonInput popd - ${subsystems.php.translators.composer-lock.translateBin} $TMPDIR/newJsonInput + ${translators.composer-lock.translateBin} $TMPDIR/newJsonInput ''; # inherit options from composer-lock translator - extraArgs = dlib.translators.translators.php.composer-lock.extraArgs; + extraArgs = translators.composer-lock.extraArgs; } diff --git a/src/subsystems/php/translators/composer-lock/default.nix b/src/subsystems/php/translators/composer-lock/default.nix index c6241d98..4537caae 100644 --- a/src/subsystems/php/translators/composer-lock/default.nix +++ b/src/subsystems/php/translators/composer-lock/default.nix @@ -1,4 +1,5 @@ { + name, dlib, lib, ... @@ -24,10 +25,6 @@ in { # translate from a given source and a project specification to a dream-lock. translate = { - translatorName, - callPackageDream, - ... - }: { /* A list of projects returned by `discoverProjects` Example: @@ -95,7 +92,7 @@ in { ... } @ args: let inherit - (callPackageDream ../../semver.nix {}) + (import ../../semver.nix {inherit lib;}) satisfies multiSatisfies ; @@ -236,7 +233,7 @@ in { in dlib.simpleTranslate2.translate ({objectsByKey, ...}: rec { - inherit translatorName; + translatorName = name; # relative path of the project within the source tree. location = project.relPath; diff --git a/src/subsystems/php/translators/packagist/default.nix b/src/subsystems/php/translators/packagist/default.nix index 3d4b9fa8..e82b6765 100644 --- a/src/subsystems/php/translators/packagist/default.nix +++ b/src/subsystems/php/translators/packagist/default.nix @@ -1,25 +1,24 @@ { - dlib, - lib, + apps, + utils, + translators, + pkgs, ... }: { type = "impure"; # the input format is specified in /specifications/translator-call-example.json # this script receives a json file including the input paths and specialArgs - translateBin = { - # dream2nix utils - apps, - subsystems, - utils, - # nixpkgs dependencies - coreutils, - curl, - jq, - git, - moreutils, - ... - }: + translateBin = let + inherit + (pkgs) + coreutils + curl + jq + git + moreutils + ; + in utils.writePureShellScript [ coreutils @@ -71,10 +70,10 @@ if [ -f $TMPDIR/source/composer.lock ] then echo 'Translating with composer-lock' - ${subsystems.php.translators.composer-lock.translateBin} $TMPDIR/newJsonInput + ${translators.composer-lock.translateBin} $TMPDIR/newJsonInput else echo 'Translating with composer-json' - ${subsystems.php.translators.composer-json.translateBin} $TMPDIR/newJsonInput + ${translators.composer-json.translateBin} $TMPDIR/newJsonInput fi # add main package source info to dream-lock.json @@ -91,5 +90,5 @@ ''; # inherit options from composer-json translator - extraArgs = dlib.translators.translators.php.composer-json.extraArgs; + extraArgs = translators.composer-json.extraArgs; } diff --git a/src/subsystems/python/builders/simple-python/default.nix b/src/subsystems/python/builders/simple-python/default.nix index fd76c6b3..b0651aec 100644 --- a/src/subsystems/python/builders/simple-python/default.nix +++ b/src/subsystems/python/builders/simple-python/default.nix @@ -1,12 +1,12 @@ # A very simple single derivation python builder -{...}: { +{ + pkgs, + lib, + ... +}: { type = "pure"; build = { - lib, - pkgs, - ... - }: { defaultPackageName, defaultPackageVersion, getSource, diff --git a/src/subsystems/python/translators/pip/default.nix b/src/subsystems/python/translators/pip/default.nix index efbec56a..3987ceaa 100644 --- a/src/subsystems/python/translators/pip/default.nix +++ b/src/subsystems/python/translators/pip/default.nix @@ -1,27 +1,24 @@ { - dlib, - lib, + externalSources, + utils, + pkgs, ... -}: let - b = builtins; -in { +}: { type = "impure"; # the input format is specified in /specifications/translator-call-example.json # this script receives a json file including the input paths and extraArgs - translateBin = { - externalSources, - utils, - bash, - coreutils, - jq, - nix, - python3, - remarshal, - toml2json, - writeScriptBin, - ... - }: + translateBin = let + inherit + (pkgs) + bash + coreutils + jq + nix + remarshal + toml2json + ; + in utils.writePureShellScript [ bash diff --git a/src/subsystems/racket/builders/simple-racket/default.nix b/src/subsystems/racket/builders/simple-racket/default.nix index 07ca79fa..0a32bf15 100644 --- a/src/subsystems/racket/builders/simple-racket/default.nix +++ b/src/subsystems/racket/builders/simple-racket/default.nix @@ -1,14 +1,12 @@ -{...}: { +{ + pkgs, + lib, + externals, + ... +}: { type = "pure"; build = { - lib, - pkgs, - stdenv, - # dream2nix inputs - externals, - ... - }: { ### FUNCTIONS # AttrSet -> Bool) -> AttrSet -> [x] getCyclicDependencies, # name: version: -> [ {name=; version=; } ] diff --git a/src/subsystems/racket/translators/racket-impure/default.nix b/src/subsystems/racket/translators/racket-impure/default.nix index 664f28cc..69433757 100644 --- a/src/subsystems/racket/translators/racket-impure/default.nix +++ b/src/subsystems/racket/translators/racket-impure/default.nix @@ -1,10 +1,8 @@ { - dlib, - lib, + utils, + pkgs, ... -}: let - l = lib // builtins; -in { +}: { type = "impure"; # A derivation which outputs a single executable at `$out`. @@ -18,20 +16,18 @@ in { # by the input parameter `outFile`. # The output file must contain the dream lock data encoded as json. # See /src/specifications/dream-lock-example.json - translateBin = { - # dream2nix utils - utils, - # nixpkgs dependencies - bash, - coreutils, - fetchurl, - jq, - nix, - racket, - runCommandLocal, - writeScriptBin, - ... - }: let + translateBin = let + inherit + (pkgs) + bash + coreutils + fetchurl + jq + nix + racket + runCommandLocal + ; + pruned-racket-catalog = let src = fetchurl { url = "https://github.com/nix-community/pruned-racket-catalog/tarball/9f11e5ea5765c8a732c5e3129ca2b71237ae2bac"; diff --git a/src/subsystems/rust/builders/build-rust-package/default.nix b/src/subsystems/rust/builders/build-rust-package/default.nix index d24313fe..2b45a767 100644 --- a/src/subsystems/rust/builders/build-rust-package/default.nix +++ b/src/subsystems/rust/builders/build-rust-package/default.nix @@ -1,13 +1,13 @@ -{...}: { +{ + lib, + dlib, + pkgs, + externals, + ... +} @ topArgs: { type = "pure"; build = { - lib, - dlib, - pkgs, - callPackageDream, - ... - } @ topArgs: { subsystemAttrs, defaultPackageName, defaultPackageVersion, @@ -68,7 +68,10 @@ }); mkShellForPkg = pkg: - callPackageDream ../devshell.nix {drv = pkg;}; + pkgs.callPackage ../devshell.nix { + inherit externals; + drv = pkg; + }; allPackages = l.mapAttrs diff --git a/src/subsystems/rust/builders/crane/default.nix b/src/subsystems/rust/builders/crane/default.nix index 37905f71..09951697 100644 --- a/src/subsystems/rust/builders/crane/default.nix +++ b/src/subsystems/rust/builders/crane/default.nix @@ -1,12 +1,12 @@ -{...}: { +{ + lib, + pkgs, + externals, + ... +} @ topArgs: { type = "ifd"; + build = { - lib, - pkgs, - externals, - callPackageDream, - ... - } @ topArgs: { subsystemAttrs, defaultPackageName, defaultPackageVersion, @@ -121,12 +121,16 @@ pname (buildPackageWithToolchain defaultToolchain buildArgs); - # TODO: this does not carry over environment variables from the - # dependencies derivation. this could cause confusion for users. mkShellForPkg = pkg: let pkgDeps = pkg.passthru.dependencies; - depsShell = callPackageDream ../devshell.nix {drv = pkgDeps;}; - mainShell = callPackageDream ../devshell.nix {drv = pkg;}; + depsShell = pkgs.callPackage ../devshell.nix { + inherit externals; + drv = pkgDeps; + }; + mainShell = pkgs.callPackage ../devshell.nix { + inherit externals; + drv = pkg; + }; shell = depsShell.combineWith mainShell; in shell; diff --git a/src/subsystems/rust/translators/cargo-lock/default.nix b/src/subsystems/rust/translators/cargo-lock/default.nix index d92bde9a..e9f04afe 100644 --- a/src/subsystems/rust/translators/cargo-lock/default.nix +++ b/src/subsystems/rust/translators/cargo-lock/default.nix @@ -1,4 +1,5 @@ { + name, dlib, lib, ... @@ -180,6 +181,7 @@ in { in dlib.simpleTranslate2.translate ({...}: { + translatorName = name; # relative path of the project within the source tree. location = project.relPath; diff --git a/src/subsystems/rust/translators/cargo-toml/default.nix b/src/subsystems/rust/translators/cargo-toml/default.nix index d22d0a88..3a21d76c 100644 --- a/src/subsystems/rust/translators/cargo-toml/default.nix +++ b/src/subsystems/rust/translators/cargo-toml/default.nix @@ -1,17 +1,22 @@ -framework: { +{ + pkgs, + utils, + translators, + ... +}: { type = "impure"; # the input format is specified in /specifications/translator-call-example.json # this script receives a json file including the input paths and specialArgs translateBin = let inherit - (framework.pkgs) + (pkgs) coreutils jq rustPlatform ; in - framework.utils.writePureShellScript + utils.writePureShellScript [ coreutils jq @@ -41,7 +46,7 @@ framework: { popd if [ $cargoResult -eq 0 ]; then - ${framework.translators.cargo-lock.translateBin} $TMPDIR/newJsonInput + ${translators.cargo-lock.translateBin} $TMPDIR/newJsonInput else echo "cargo failed to generate the lockfile" exit 1 @@ -50,7 +55,7 @@ framework: { # inherit options from cargo-lock translator extraArgs = - framework.translators.cargo-lock.extraArgs + translators.cargo-lock.extraArgs // { cargoArgs = { description = "Additional arguments for Cargo"; From 55db7931ce3b4ec1031945c6b3ce5b66f6681c77 Mon Sep 17 00:00:00 2001 From: Yusuf Bera Ertan Date: Thu, 6 Oct 2022 20:53:59 +0300 Subject: [PATCH 03/23] tests: fix unit tests, fix extended examples --- .../discoverers.nix | 4 ++-- examples/_d2n-extended/flake.nix | 4 ++-- src/default.nix | 4 ---- tests/unit/test_pure_translators.py | 24 +++++++++---------- tests/unit/test_source_shortcuts.py | 2 +- 5 files changed, 17 insertions(+), 21 deletions(-) diff --git a/examples/_d2n-extended-new-subsystem/discoverers.nix b/examples/_d2n-extended-new-subsystem/discoverers.nix index 35249634..f580512d 100644 --- a/examples/_d2n-extended-new-subsystem/discoverers.nix +++ b/examples/_d2n-extended-new-subsystem/discoverers.nix @@ -1,9 +1,9 @@ -{dlib, ...}: { +{config, ...}: { discoverers.dummy = rec { name = "dummy"; subsystem = "hello"; discover = {tree}: [ - (dlib.construct.discoveredProject { + (config.dlib.construct.discoveredProject { inherit subsystem name; inherit (tree) relPath; translators = ["dummy"]; diff --git a/examples/_d2n-extended/flake.nix b/examples/_d2n-extended/flake.nix index 73814f08..a32a625a 100644 --- a/examples/_d2n-extended/flake.nix +++ b/examples/_d2n-extended/flake.nix @@ -42,8 +42,8 @@ } '') (builtins.toFile "crates-io-new.nix" '' - {lib, ...}: { - fetchers.crates-io = lib.mkForce { + {config, ...}: { + fetchers.crates-io = config.lib.mkForce { imports = ["${inp.dream2nix}/src/fetchers/crates-io"]; }; } diff --git a/src/default.nix b/src/default.nix index 83599c65..a3c0e4e9 100644 --- a/src/default.nix +++ b/src/default.nix @@ -71,7 +71,6 @@ in let inherit dlib; inherit externals; inherit externalSources; - inherit fetchers; inherit framework; inherit indexers; inherit dream2nixWithExternals; @@ -97,9 +96,6 @@ in let # apps for CLI and installation apps = callPackageDream ./apps {}; - # fetcher implementations - fetchers = callPackageDream ./fetchers {}; - # indexer implementations indexers = callPackageDream ./indexers {}; diff --git a/tests/unit/test_pure_translators.py b/tests/unit/test_pure_translators.py index cdd5c202..e1be637d 100644 --- a/tests/unit/test_pure_translators.py +++ b/tests/unit/test_pure_translators.py @@ -5,7 +5,7 @@ import pytest def get_projects_to_test(): tests = nix_ffi.eval( - 'subsystems.allTranslators', + 'framework.translators', wrapper_code = ''' {result, ...}: let lib = (import {}).lib; @@ -23,7 +23,7 @@ def get_projects_to_test(): }) (translator.generateUnitTestsForProjects or []) ) - result + (l.attrValues result) ) ''', ) @@ -65,7 +65,7 @@ def check_format_sourceSpec(sourceSpec): @pytest.mark.parametrize("p", projects) def test_packageName(p): defaultPackage = nix_ffi.eval( - f"subsystems.{p['subsystem']}.translators.{p['translator']}.translate", + f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.translateInstanced", params=dict( project=p['project'], source=p['source'], @@ -81,7 +81,7 @@ def test_packageName(p): @pytest.mark.parametrize("p", projects) def test_exportedPackages(p): exportedPackages = nix_ffi.eval( - f"subsystems.{p['subsystem']}.translators.{p['translator']}.translate", + f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.translateınstanced", params=dict( project=p['project'], source=p['source'], @@ -97,7 +97,7 @@ def test_exportedPackages(p): @pytest.mark.parametrize("p", projects) def test_extraObjects(p): extraObjects = nix_ffi.eval( - f"subsystems.{p['subsystem']}.translators.{p['translator']}.translate", + f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.translateInstanced", params=dict( project=p['project'], source=p['source'], @@ -121,7 +121,7 @@ def test_extraObjects(p): @pytest.mark.parametrize("p", projects) def test_location(p): location = nix_ffi.eval( - f"subsystems.{p['subsystem']}.translators.{p['translator']}.translate", + f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.translateInstanced", params=dict( project=p['project'], source=p['source'], @@ -136,7 +136,7 @@ def test_location(p): @pytest.mark.parametrize("p", projects) def test_serializedRawObjects(p): serializedRawObjects = nix_ffi.eval( - f"subsystems.{p['subsystem']}.translators.{p['translator']}.translate", + f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.translateInstanced", params=dict( project=p['project'], source=p['source'], @@ -159,7 +159,7 @@ def test_serializedRawObjects(p): @pytest.mark.parametrize("p", projects) def test_subsystemName(p): subsystemName = nix_ffi.eval( - f"subsystems.{p['subsystem']}.translators.{p['translator']}.translate", + f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.translateInstanced", params=dict( project=p['project'], source=p['source'], @@ -175,7 +175,7 @@ def test_subsystemName(p): @pytest.mark.parametrize("p", projects) def test_subsystemAttrs(p): subsystemAttrs = nix_ffi.eval( - f"subsystems.{p['subsystem']}.translators.{p['translator']}.translate", + f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.translateInstanced", params=dict( project=p['project'], source=p['source'], @@ -191,7 +191,7 @@ def test_subsystemAttrs(p): @pytest.mark.parametrize("p", projects) def test_translatorName(p): translatorName = nix_ffi.eval( - f"subsystems.{p['subsystem']}.translators.{p['translator']}.translate", + f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.translateInstanced", params=dict( project=p['project'], source=p['source'], @@ -207,7 +207,7 @@ def test_translatorName(p): @pytest.mark.parametrize("p", projects) def test_extractors(p): finalObjects = nix_ffi.eval( - f"subsystems.{p['subsystem']}.translators.{p['translator']}.translate", + f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.translateInstanced", params=dict( project=p['project'], source=p['source'], @@ -241,7 +241,7 @@ def test_extractors(p): @pytest.mark.parametrize("p", projects) def test_keys(p): objectsByKey = nix_ffi.eval( - f"subsystems.{p['subsystem']}.translators.{p['translator']}.translate", + f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.translateInstanced", params=dict( project=p['project'], source=p['source'], diff --git a/tests/unit/test_source_shortcuts.py b/tests/unit/test_source_shortcuts.py index 48c5dc89..6c11b7b0 100644 --- a/tests/unit/test_source_shortcuts.py +++ b/tests/unit/test_source_shortcuts.py @@ -68,7 +68,7 @@ import nix_ffi ]) def test_translateShortcut(shortcut, expected): result = nix_ffi.callNixFunction( - 'fetchers.translateShortcut', + 'framework.functions.fetchers.translateShortcut', shortcut=shortcut, computeHash=False, ) From 9a0f3e1f8beaa91891fa2da0972ea18c74b70899 Mon Sep 17 00:00:00 2001 From: Yusuf Bera Ertan Date: Thu, 6 Oct 2022 20:56:25 +0300 Subject: [PATCH 04/23] fix: remove fetchers --- src/default.nix | 1 - 1 file changed, 1 deletion(-) diff --git a/src/default.nix b/src/default.nix index a3c0e4e9..a93e3d59 100644 --- a/src/default.nix +++ b/src/default.nix @@ -709,7 +709,6 @@ in { apps callPackageDream dream2nixWithExternals - fetchers framework indexers fetchSources From 6e3d49800634b6c5cbdce0706e04abe0c80f3d93 Mon Sep 17 00:00:00 2001 From: Yusuf Bera Ertan Date: Thu, 6 Oct 2022 20:57:54 +0300 Subject: [PATCH 05/23] fix: use framework in translator utils --- src/utils/translator.nix | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/utils/translator.nix b/src/utils/translator.nix index 22975192..480151c3 100644 --- a/src/utils/translator.nix +++ b/src/utils/translator.nix @@ -1,8 +1,8 @@ { lib, # dream2nix - fetchers, dlib, + framework, ... }: let b = builtins; @@ -89,12 +89,14 @@ constructedArgsKeep = overrideWarning ["pname" "version"] constructedArgs; - constructedSource = fetchers.constructSource (constructedArgsKeep + constructedSource = framework.functions.fetchers.constructSource ( + constructedArgsKeep // { inherit type; pname = pkgName; version = pkgVersion; - }); + } + ); skip = (type == "path") From df731a072247f0ad09819c239922e844745665e5 Mon Sep 17 00:00:00 2001 From: Yusuf Bera Ertan Date: Thu, 6 Oct 2022 21:00:19 +0300 Subject: [PATCH 06/23] fix: remove translatorInstances usages --- src/apps/translate-index/default.nix | 2 +- src/apps/translate/default.nix | 2 +- src/utils/default.nix | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/apps/translate-index/default.nix b/src/apps/translate-index/default.nix index 2aeb67e5..ca815730 100644 --- a/src/apps/translate-index/default.nix +++ b/src/apps/translate-index/default.nix @@ -54,7 +54,7 @@ in if [ ! -e "$bin" ]; then echo "building executable for translator $translator" ${callNixWithD2N} build -o "$bin" " - dream2nix.framework.translatorInstances.$translator.translateBin + dream2nix.framework.translators.$translator.translateBin " fi done diff --git a/src/apps/translate/default.nix b/src/apps/translate/default.nix index 76310504..2409b24a 100644 --- a/src/apps/translate/default.nix +++ b/src/apps/translate/default.nix @@ -51,7 +51,7 @@ utils.writePureShellScriptBin translateBin="$TRANSLATOR_DIR/$translator" else translateBin=$(${callNixWithD2N} build --print-out-paths --no-link " - dream2nix.framework.translatorInstances.$translator.translateBin + dream2nix.framework.translators.$translator.translateBin ") fi diff --git a/src/utils/default.nix b/src/utils/default.nix index f7fb3687..416131b5 100644 --- a/src/utils/default.nix +++ b/src/utils/default.nix @@ -180,7 +180,7 @@ in aggregate = project.aggregate or false; translator = - framework.translatorInstances."${project.translator}"; + framework.translators."${project.translator}"; argsJsonFile = pkgs.writeText "translator-args.json" From 08ae837a1aa97f2b7245dc38a057b208530e6a3c Mon Sep 17 00:00:00 2001 From: Yusuf Bera Ertan Date: Thu, 6 Oct 2022 21:26:15 +0300 Subject: [PATCH 07/23] fix: add wrapPureTranslator to functions.translators --- src/default.nix | 12 +++- src/lib.nix | 2 + src/modules/framework.nix | 9 +-- .../functions.translators/implementation.nix | 57 +++++++++++++++++++ src/modules/top-level.nix | 19 +++++-- 5 files changed, 86 insertions(+), 13 deletions(-) diff --git a/src/default.nix b/src/default.nix index a93e3d59..25268421 100644 --- a/src/default.nix +++ b/src/default.nix @@ -51,7 +51,17 @@ in let }; framework = import ./modules/framework.nix { - inherit apps lib dlib pkgs utils externals externalSources; + inherit + apps + lib + dlib + pkgs + utils + externals + externalSources + dream2nixWithExternals + ; + dream2nixConfigFile = configFile; dream2nixConfig = config; }; diff --git a/src/lib.nix b/src/lib.nix index 2a995710..98898c32 100644 --- a/src/lib.nix +++ b/src/lib.nix @@ -92,10 +92,12 @@ framework = import ./modules/framework.nix { inherit lib dlib externalSources; dream2nixConfig = config; + dream2nixConfigFile = l.toFile "dream2nix-config.json" (l.toJSON config); apps = throw "apps is not available before nixpkgs is imported"; pkgs = throw "pkgs is not available before nixpkgs is imported"; utils = throw "utils is not available before nixpkgs is imported"; externals = throw "externals is not available before nixpkgs is imported"; + dream2nixWithExternals = throw "not available before nixpkgs is imported"; }; initD2N = initDream2nix config; diff --git a/src/modules/framework.nix b/src/modules/framework.nix index 15cac543..5466863b 100644 --- a/src/modules/framework.nix +++ b/src/modules/framework.nix @@ -1,12 +1,7 @@ { - dream2nixConfig, - pkgs, - dlib, - externals, - externalSources, lib, - utils, - apps, + dream2nixConfig, + ... } @ args: let topLevel = import ./top-level.nix args; evaledModules = lib.evalModules { diff --git a/src/modules/functions.translators/implementation.nix b/src/modules/functions.translators/implementation.nix index 466cddf6..d4128f50 100644 --- a/src/modules/functions.translators/implementation.nix +++ b/src/modules/functions.translators/implementation.nix @@ -11,10 +11,67 @@ else def.default or null ) extraArgsDef; + + # adds a translateBin to a pure translator + wrapPureTranslator = { + subsystem, + name, + }: let + inherit + (config) + utils + pkgs + dream2nixWithExternals + configFile + ; + bin = + utils.writePureShellScript + (with pkgs; [ + coreutils + jq + nix + python3 + ]) + '' + jsonInputFile=$(realpath $1) + outputFile=$(realpath -m $(jq '.outputFile' -c -r $jsonInputFile)) + pushd $TMPDIR + nix eval \ + --option experimental-features "nix-command flakes"\ + --show-trace --impure --raw --expr " + let + dream2nix = import ${dream2nixWithExternals} { + config = ${configFile}; + }; + translatorArgs = + (builtins.fromJSON + (builtins.unsafeDiscardStringContext (builtins.readFile '''$1'''))); + dreamLock' = + dream2nix.framework.translatorsBySubsystem.${subsystem}.${name}.translate + translatorArgs; + # simpleTranslate2 puts dream-lock in result + dreamLock = dreamLock'.result or dreamLock'; + in + dream2nix.utils.dreamLock.toJSON + # don't use nix to detect cycles, this will be more efficient in python + (dreamLock // { + _generic = builtins.removeAttrs dreamLock._generic [ \"cyclicDependencies\" ]; + }) + " | python3 ${../apps/cli/format-dream-lock.py} > out + tmpOut=$(realpath out) + popd + mkdir -p $(dirname $outputFile) + cp $tmpOut $outputFile + ''; + in + bin.overrideAttrs (old: { + name = "translator-${subsystem}-pure-${name}"; + }); in { functions.translators = { inherit makeTranslatorDefaultArgs + wrapPureTranslator ; }; } diff --git a/src/modules/top-level.nix b/src/modules/top-level.nix index 1c59d137..9656334c 100644 --- a/src/modules/top-level.nix +++ b/src/modules/top-level.nix @@ -7,7 +7,9 @@ pkgs, utils, dream2nixConfig, -}: let + dream2nixConfigFile, + dream2nixWithExternals, +} @ args: let t = lib.types; in { imports = [ @@ -47,9 +49,16 @@ in { dream2nixConfig = lib.mkOption { type = t.raw; }; + dream2nixWithExternals = lib.mkOption { + type = t.path; + }; + dream2nixConfigFile = lib.mkOption { + type = t.path; + }; }; - config = { - inherit apps dlib externals externalSources pkgs utils dream2nixConfig; - lib = lib // builtins; - }; + config = + args + // { + lib = args.lib // builtins; + }; } From 65ffc551d0e59243da6612bf9b6f2c9889a67ade Mon Sep 17 00:00:00 2001 From: Yusuf Bera Ertan Date: Thu, 6 Oct 2022 21:33:56 +0300 Subject: [PATCH 08/23] fix: add wrapPureTranslator option --- src/modules/functions.translators/interface.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/src/modules/functions.translators/interface.nix b/src/modules/functions.translators/interface.nix index fabee307..7f9660e0 100644 --- a/src/modules/functions.translators/interface.nix +++ b/src/modules/functions.translators/interface.nix @@ -4,5 +4,6 @@ in { options.functions.translators = { makeTranslatorDefaultArgs = lib.mkOption {type = t.functionTo t.anything;}; + wrapPureTranslator = lib.mkOption {type = t.functionTo t.package;}; }; } From 37936a5fc4ec8594a32340fb531068cd3dbec3ee Mon Sep 17 00:00:00 2001 From: Yusuf Bera Ertan Date: Thu, 6 Oct 2022 21:36:42 +0300 Subject: [PATCH 09/23] fix: configFile -> dream2nixConfigFile --- src/modules/functions.translators/implementation.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/modules/functions.translators/implementation.nix b/src/modules/functions.translators/implementation.nix index d4128f50..5f59496f 100644 --- a/src/modules/functions.translators/implementation.nix +++ b/src/modules/functions.translators/implementation.nix @@ -22,7 +22,7 @@ utils pkgs dream2nixWithExternals - configFile + dream2nixConfigFile ; bin = utils.writePureShellScript @@ -41,7 +41,7 @@ --show-trace --impure --raw --expr " let dream2nix = import ${dream2nixWithExternals} { - config = ${configFile}; + config = ${dream2nixConfigFile}; }; translatorArgs = (builtins.fromJSON From 819f52348b277cff32322ac0235485b3c9cc486a Mon Sep 17 00:00:00 2001 From: Yusuf Bera Ertan Date: Thu, 6 Oct 2022 21:39:33 +0300 Subject: [PATCH 10/23] fix: correct apps path --- src/modules/functions.translators/implementation.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/functions.translators/implementation.nix b/src/modules/functions.translators/implementation.nix index 5f59496f..4a0c97c7 100644 --- a/src/modules/functions.translators/implementation.nix +++ b/src/modules/functions.translators/implementation.nix @@ -57,7 +57,7 @@ (dreamLock // { _generic = builtins.removeAttrs dreamLock._generic [ \"cyclicDependencies\" ]; }) - " | python3 ${../apps/cli/format-dream-lock.py} > out + " | python3 ${../../apps/cli/format-dream-lock.py} > out tmpOut=$(realpath out) popd mkdir -p $(dirname $outputFile) From 011e3dbd27570774b864459297c00d4ffa1c4540 Mon Sep 17 00:00:00 2001 From: Yusuf Bera Ertan Date: Thu, 6 Oct 2022 22:04:27 +0300 Subject: [PATCH 11/23] fix: only use pure translator translateBin if its a pure translator --- src/modules/interfaces.translator/implementation.nix | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/modules/interfaces.translator/implementation.nix b/src/modules/interfaces.translator/implementation.nix index 1ad225a7..a2a261d4 100644 --- a/src/modules/interfaces.translator/implementation.nix +++ b/src/modules/interfaces.translator/implementation.nix @@ -17,7 +17,10 @@ } ); translateBin = - framework.functions.translators.wrapPureTranslator - {inherit (config) subsystem name;}; + if config.translate != null + then + framework.functions.translators.wrapPureTranslator + {inherit (config) subsystem name;} + else framework.lib.mkForce config.translateBin; }; } From c90129fa5c3f0cc2bf6f96df3e0feaa8f5618772 Mon Sep 17 00:00:00 2001 From: Yusuf Bera Ertan Date: Thu, 6 Oct 2022 22:21:27 +0300 Subject: [PATCH 12/23] tests: fix unit test --- tests/unit/test_pure_translators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_pure_translators.py b/tests/unit/test_pure_translators.py index e1be637d..1f05570a 100644 --- a/tests/unit/test_pure_translators.py +++ b/tests/unit/test_pure_translators.py @@ -81,7 +81,7 @@ def test_packageName(p): @pytest.mark.parametrize("p", projects) def test_exportedPackages(p): exportedPackages = nix_ffi.eval( - f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.translateınstanced", + f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.translateInstanced", params=dict( project=p['project'], source=p['source'], From 14a56a8d4368b20942f3496558cca82c487dc6d0 Mon Sep 17 00:00:00 2001 From: Yusuf Bera Ertan Date: Thu, 6 Oct 2022 22:24:07 +0300 Subject: [PATCH 13/23] tests: fix _d2n-extended-new-subsystem --- examples/_d2n-extended-new-subsystem/builders.nix | 6 ++++-- examples/_d2n-extended-new-subsystem/translators.nix | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/examples/_d2n-extended-new-subsystem/builders.nix b/examples/_d2n-extended-new-subsystem/builders.nix index 445d7d02..e7c60a98 100644 --- a/examples/_d2n-extended-new-subsystem/builders.nix +++ b/examples/_d2n-extended-new-subsystem/builders.nix @@ -1,9 +1,11 @@ -{ +{config, ...}: let + inherit (config.pkgs) hello; +in { builders.dummy = {...}: { name = "dummy"; subsystem = "hello"; type = "pure"; - build = {hello, ...}: {...}: { + build = {...}: { packages.${hello.pname}.${hello.version} = hello; }; diff --git a/examples/_d2n-extended-new-subsystem/translators.nix b/examples/_d2n-extended-new-subsystem/translators.nix index d5dcf30f..21ed84d1 100644 --- a/examples/_d2n-extended-new-subsystem/translators.nix +++ b/examples/_d2n-extended-new-subsystem/translators.nix @@ -1,9 +1,11 @@ -{ +{config, ...}: let + inherit (config.pkgs) hello; +in { translators.dummy = {...}: { type = "pure"; name = "dummy"; subsystem = "hello"; - translate = {hello, ...}: {...}: { + translate = {...}: { result = { _generic = { subsystem = "hello"; From 0bac3a34760cea1096f51bf28106f24c4e2182e7 Mon Sep 17 00:00:00 2001 From: Yusuf Bera Ertan Date: Thu, 6 Oct 2022 22:48:12 +0300 Subject: [PATCH 14/23] tests: attemp to fix _d2n-extended --- examples/_d2n-extended/flake.nix | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/_d2n-extended/flake.nix b/examples/_d2n-extended/flake.nix index a32a625a..4e1d253c 100644 --- a/examples/_d2n-extended/flake.nix +++ b/examples/_d2n-extended/flake.nix @@ -17,7 +17,7 @@ (builtins.toFile "cargo-toml-new.nix" '' { translators.cargo-toml-new = { - imports = ["${inp.dream2nix}/src/subsystems/rust/translators/cargo-toml"]; + imports = [(attrs: import "${inp.dream2nix}/src/subsystems/rust/translators/cargo-toml" attrs.framework)]; name = "cargo-toml-new"; subsystem = "rust"; }; @@ -26,7 +26,7 @@ (builtins.toFile "brp-new.nix" '' { builders.brp-new = { - imports = ["${inp.dream2nix}/src/subsystems/rust/builders/build-rust-package"]; + imports = [(attrs: import "${inp.dream2nix}/src/subsystems/rust/builders/build-rust-package" attrs.framework)]; name = "brp-new"; subsystem = "rust"; }; @@ -35,7 +35,7 @@ (builtins.toFile "cargo-new.nix" '' { discoverers.cargo-new = { - imports = ["${inp.dream2nix}/src/subsystems/rust/discoverers/cargo"]; + imports = [(attrs: import "${inp.dream2nix}/src/subsystems/rust/discoverers/cargo" attrs.framework)]; name = "cargo-new"; subsystem = "rust"; }; @@ -44,7 +44,7 @@ (builtins.toFile "crates-io-new.nix" '' {config, ...}: { fetchers.crates-io = config.lib.mkForce { - imports = ["${inp.dream2nix}/src/fetchers/crates-io"]; + imports = [(attrs: import "${inp.dream2nix}/src/fetchers/crates-io" attrs.framework)]; }; } '') From 20c95ec1f74384aaa23a322935bacc6ac0d40ff7 Mon Sep 17 00:00:00 2001 From: Yusuf Bera Ertan Date: Fri, 7 Oct 2022 16:22:36 +0300 Subject: [PATCH 15/23] fix: avoid infinite recursion by introducing a new translateBinInstanced option --- src/apps/translate-index/default.nix | 2 +- src/apps/translate/default.nix | 2 +- .../functions.translators/implementation.nix | 2 +- .../interfaces.translator/implementation.nix | 31 +++++++++++-------- .../interfaces.translator/interface.nix | 4 +++ .../haskell/translators/hackage/default.nix | 2 +- .../nodejs/translators/npm/default.nix | 2 +- .../translators/package-json/default.nix | 2 +- .../php/translators/composer-json/default.nix | 2 +- .../php/translators/packagist/default.nix | 4 +-- .../rust/translators/cargo-toml/default.nix | 2 +- .../rust/translators/crates-io/default.nix | 4 +-- src/utils/default.nix | 2 +- 13 files changed, 35 insertions(+), 26 deletions(-) diff --git a/src/apps/translate-index/default.nix b/src/apps/translate-index/default.nix index ca815730..1b93defa 100644 --- a/src/apps/translate-index/default.nix +++ b/src/apps/translate-index/default.nix @@ -54,7 +54,7 @@ in if [ ! -e "$bin" ]; then echo "building executable for translator $translator" ${callNixWithD2N} build -o "$bin" " - dream2nix.framework.translators.$translator.translateBin + dream2nix.framework.translators.$translator.translateBinInstanced " fi done diff --git a/src/apps/translate/default.nix b/src/apps/translate/default.nix index 2409b24a..536525ff 100644 --- a/src/apps/translate/default.nix +++ b/src/apps/translate/default.nix @@ -51,7 +51,7 @@ utils.writePureShellScriptBin translateBin="$TRANSLATOR_DIR/$translator" else translateBin=$(${callNixWithD2N} build --print-out-paths --no-link " - dream2nix.framework.translators.$translator.translateBin + dream2nix.framework.translators.$translator.translateBinInstanced ") fi diff --git a/src/modules/functions.translators/implementation.nix b/src/modules/functions.translators/implementation.nix index 4a0c97c7..4593b11b 100644 --- a/src/modules/functions.translators/implementation.nix +++ b/src/modules/functions.translators/implementation.nix @@ -47,7 +47,7 @@ (builtins.fromJSON (builtins.unsafeDiscardStringContext (builtins.readFile '''$1'''))); dreamLock' = - dream2nix.framework.translatorsBySubsystem.${subsystem}.${name}.translate + dream2nix.framework.translatorsBySubsystem.${subsystem}.${name}.translateInstanced translatorArgs; # simpleTranslate2 puts dream-lock in result dreamLock = dreamLock'.result or dreamLock'; diff --git a/src/modules/interfaces.translator/implementation.nix b/src/modules/interfaces.translator/implementation.nix index a2a261d4..cb936375 100644 --- a/src/modules/interfaces.translator/implementation.nix +++ b/src/modules/interfaces.translator/implementation.nix @@ -5,22 +5,27 @@ }: { config = { translateInstanced = args: - config.translate - ( - (framework.functions.translators.makeTranslatorDefaultArgs - (config.extraArgs or {})) - // args - // (args.project.subsystemInfo or {}) - // { - tree = - args.tree or (framework.dlib.prepareSourceTree {inherit (args) source;}); - } - ); - translateBin = + if config.translate != null + then + config.translate + ( + (framework.functions.translators.makeTranslatorDefaultArgs + (config.extraArgs or {})) + // args + // (args.project.subsystemInfo or {}) + // { + tree = + args.tree or (framework.dlib.prepareSourceTree {inherit (args) source;}); + } + ) + else null; + translateBinInstanced = if config.translate != null then framework.functions.translators.wrapPureTranslator {inherit (config) subsystem name;} - else framework.lib.mkForce config.translateBin; + else if config.translateBin != null + then config.translateBin + else null; }; } diff --git a/src/modules/interfaces.translator/interface.nix b/src/modules/interfaces.translator/interface.nix index a38368a3..61117192 100644 --- a/src/modules/interfaces.translator/interface.nix +++ b/src/modules/interfaces.translator/interface.nix @@ -54,6 +54,10 @@ in { type = t.nullOr t.package; default = null; }; + translateBinInstanced = lib.mkOption { + type = t.nullOr t.package; + readOnly = true; + }; type = lib.mkOption { type = t.enum [ "ifd" diff --git a/src/subsystems/haskell/translators/hackage/default.nix b/src/subsystems/haskell/translators/hackage/default.nix index e29f90d8..1ab61837 100644 --- a/src/subsystems/haskell/translators/hackage/default.nix +++ b/src/subsystems/haskell/translators/hackage/default.nix @@ -82,7 +82,7 @@ popd # execute cabal-plan translator - ${translators.cabal-plan.translateBin} $TMPDIR/newJsonInput + ${translators.cabal-plan.translateBinInstanced} $TMPDIR/newJsonInput # finalize dream-lock. Add source and export default package # set correct package version under `packages` diff --git a/src/subsystems/nodejs/translators/npm/default.nix b/src/subsystems/nodejs/translators/npm/default.nix index 2d63a7af..b6fde3b0 100644 --- a/src/subsystems/nodejs/translators/npm/default.nix +++ b/src/subsystems/nodejs/translators/npm/default.nix @@ -62,7 +62,7 @@ popd # call package-lock translator - ${translators.package-lock.translateBin} $TMPDIR/newJsonInput + ${translators.package-lock.translateBinInstanced} $TMPDIR/newJsonInput # get resolved package version export version=$(npm view $candidate version) diff --git a/src/subsystems/nodejs/translators/package-json/default.nix b/src/subsystems/nodejs/translators/package-json/default.nix index 13ec3b79..0a8eaf24 100644 --- a/src/subsystems/nodejs/translators/package-json/default.nix +++ b/src/subsystems/nodejs/translators/package-json/default.nix @@ -60,7 +60,7 @@ jq ".source = \"$newSource\"" -c -r $jsonInput > $TMPDIR/newJsonInput popd - ${translators.package-lock.translateBin} $TMPDIR/newJsonInput + ${translators.package-lock.translateBinInstanced} $TMPDIR/newJsonInput ''; # inherit options from package-lock translator diff --git a/src/subsystems/php/translators/composer-json/default.nix b/src/subsystems/php/translators/composer-json/default.nix index 8d7ae010..087f3366 100644 --- a/src/subsystems/php/translators/composer-json/default.nix +++ b/src/subsystems/php/translators/composer-json/default.nix @@ -75,7 +75,7 @@ jq ".source = \"$newSource\"" -c -r $jsonInput > $TMPDIR/newJsonInput popd - ${translators.composer-lock.translateBin} $TMPDIR/newJsonInput + ${translators.composer-lock.translateBinInstanced} $TMPDIR/newJsonInput ''; # inherit options from composer-lock translator diff --git a/src/subsystems/php/translators/packagist/default.nix b/src/subsystems/php/translators/packagist/default.nix index e82b6765..3552e410 100644 --- a/src/subsystems/php/translators/packagist/default.nix +++ b/src/subsystems/php/translators/packagist/default.nix @@ -70,10 +70,10 @@ if [ -f $TMPDIR/source/composer.lock ] then echo 'Translating with composer-lock' - ${translators.composer-lock.translateBin} $TMPDIR/newJsonInput + ${translators.composer-lock.translateBinInstanced} $TMPDIR/newJsonInput else echo 'Translating with composer-json' - ${translators.composer-json.translateBin} $TMPDIR/newJsonInput + ${translators.composer-json.translateBinInstanced} $TMPDIR/newJsonInput fi # add main package source info to dream-lock.json diff --git a/src/subsystems/rust/translators/cargo-toml/default.nix b/src/subsystems/rust/translators/cargo-toml/default.nix index 3a21d76c..1aa710b7 100644 --- a/src/subsystems/rust/translators/cargo-toml/default.nix +++ b/src/subsystems/rust/translators/cargo-toml/default.nix @@ -46,7 +46,7 @@ popd if [ $cargoResult -eq 0 ]; then - ${translators.cargo-lock.translateBin} $TMPDIR/newJsonInput + ${translators.cargo-lock.translateBinInstanced} $TMPDIR/newJsonInput else echo "cargo failed to generate the lockfile" exit 1 diff --git a/src/subsystems/rust/translators/crates-io/default.nix b/src/subsystems/rust/translators/crates-io/default.nix index 5ff30ca2..fd6addd7 100644 --- a/src/subsystems/rust/translators/crates-io/default.nix +++ b/src/subsystems/rust/translators/crates-io/default.nix @@ -66,9 +66,9 @@ # we don't need to run cargo-toml translator if Cargo.lock exists if [ -f "$TMPDIR/source/Cargo.lock" ]; then - ${translators.cargo-lock.translateBin} $TMPDIR/newJsonInput + ${translators.cargo-lock.translateBinInstanced} $TMPDIR/newJsonInput else - ${translators.cargo-toml.translateBin} $TMPDIR/newJsonInput + ${translators.cargo-toml.translateBinInstanced} $TMPDIR/newJsonInput fi # add main package source info to dream-lock.json diff --git a/src/utils/default.nix b/src/utils/default.nix index 416131b5..e126ffe9 100644 --- a/src/utils/default.nix +++ b/src/utils/default.nix @@ -206,7 +206,7 @@ in '' dreamLockPath="${project.dreamLockPath}" - ${translator.translateBin} ${argsJsonFile} + ${translator.translateBinInstanced} ${argsJsonFile} # aggregate source hashes if [ "${l.toJSON aggregate}" == "true" ]; then From 9ae157fd184ffd1bcd0ee67b9f52b734edc2afa9 Mon Sep 17 00:00:00 2001 From: Yusuf Bera Ertan Date: Fri, 7 Oct 2022 16:24:41 +0300 Subject: [PATCH 16/23] refactor: rename *Instanced to *Final --- src/apps/translate-index/default.nix | 2 +- src/apps/translate/default.nix | 2 +- src/default.nix | 2 +- .../functions.translators/implementation.nix | 2 +- .../interfaces.translator/implementation.nix | 4 ++-- .../interfaces.translator/interface.nix | 10 ++++++++-- .../haskell/translators/hackage/default.nix | 2 +- .../nodejs/translators/npm/default.nix | 2 +- .../translators/package-json/default.nix | 2 +- .../php/translators/composer-json/default.nix | 2 +- .../php/translators/packagist/default.nix | 4 ++-- .../rust/translators/cargo-toml/default.nix | 2 +- .../rust/translators/crates-io/default.nix | 4 ++-- src/utils/default.nix | 2 +- tests/unit/test_pure_translators.py | 20 +++++++++---------- 15 files changed, 34 insertions(+), 28 deletions(-) diff --git a/src/apps/translate-index/default.nix b/src/apps/translate-index/default.nix index 1b93defa..85da1e73 100644 --- a/src/apps/translate-index/default.nix +++ b/src/apps/translate-index/default.nix @@ -54,7 +54,7 @@ in if [ ! -e "$bin" ]; then echo "building executable for translator $translator" ${callNixWithD2N} build -o "$bin" " - dream2nix.framework.translators.$translator.translateBinInstanced + dream2nix.framework.translators.$translator.translateBinFinal " fi done diff --git a/src/apps/translate/default.nix b/src/apps/translate/default.nix index 536525ff..8dab4193 100644 --- a/src/apps/translate/default.nix +++ b/src/apps/translate/default.nix @@ -51,7 +51,7 @@ utils.writePureShellScriptBin translateBin="$TRANSLATOR_DIR/$translator" else translateBin=$(${callNixWithD2N} build --print-out-paths --no-link " - dream2nix.framework.translators.$translator.translateBinInstanced + dream2nix.framework.translators.$translator.translateBinFinal ") fi diff --git a/src/default.nix b/src/default.nix index 25268421..77667870 100644 --- a/src/default.nix +++ b/src/default.nix @@ -502,7 +502,7 @@ in let l.forEach projectsPureUnresolved (proj: let translator = getTranslator proj.translator; - dreamLock'' = translator.translateInstanced { + dreamLock'' = translator.translateFinal { inherit source tree discoveredProjects; project = proj; }; diff --git a/src/modules/functions.translators/implementation.nix b/src/modules/functions.translators/implementation.nix index 4593b11b..ef396e11 100644 --- a/src/modules/functions.translators/implementation.nix +++ b/src/modules/functions.translators/implementation.nix @@ -47,7 +47,7 @@ (builtins.fromJSON (builtins.unsafeDiscardStringContext (builtins.readFile '''$1'''))); dreamLock' = - dream2nix.framework.translatorsBySubsystem.${subsystem}.${name}.translateInstanced + dream2nix.framework.translatorsBySubsystem.${subsystem}.${name}.translateFinal translatorArgs; # simpleTranslate2 puts dream-lock in result dreamLock = dreamLock'.result or dreamLock'; diff --git a/src/modules/interfaces.translator/implementation.nix b/src/modules/interfaces.translator/implementation.nix index cb936375..f23be908 100644 --- a/src/modules/interfaces.translator/implementation.nix +++ b/src/modules/interfaces.translator/implementation.nix @@ -4,7 +4,7 @@ ... }: { config = { - translateInstanced = args: + translateFinal = args: if config.translate != null then config.translate @@ -19,7 +19,7 @@ } ) else null; - translateBinInstanced = + translateBinFinal = if config.translate != null then framework.functions.translators.wrapPureTranslator diff --git a/src/modules/interfaces.translator/interface.nix b/src/modules/interfaces.translator/interface.nix index 61117192..2a99443b 100644 --- a/src/modules/interfaces.translator/interface.nix +++ b/src/modules/interfaces.translator/interface.nix @@ -46,17 +46,23 @@ in { type = t.nullOr (t.functionTo t.attrs); default = null; }; - translateInstanced = lib.mkOption { + translateFinal = lib.mkOption { type = t.nullOr (t.functionTo t.attrs); readOnly = true; + description = '' + the final translate that you should use. + ''; }; translateBin = lib.mkOption { type = t.nullOr t.package; default = null; }; - translateBinInstanced = lib.mkOption { + translateBinFinal = lib.mkOption { type = t.nullOr t.package; readOnly = true; + description = '' + the final translateBin that you should use. + ''; }; type = lib.mkOption { type = t.enum [ diff --git a/src/subsystems/haskell/translators/hackage/default.nix b/src/subsystems/haskell/translators/hackage/default.nix index 1ab61837..4d0ddda2 100644 --- a/src/subsystems/haskell/translators/hackage/default.nix +++ b/src/subsystems/haskell/translators/hackage/default.nix @@ -82,7 +82,7 @@ popd # execute cabal-plan translator - ${translators.cabal-plan.translateBinInstanced} $TMPDIR/newJsonInput + ${translators.cabal-plan.translateBinFinal} $TMPDIR/newJsonInput # finalize dream-lock. Add source and export default package # set correct package version under `packages` diff --git a/src/subsystems/nodejs/translators/npm/default.nix b/src/subsystems/nodejs/translators/npm/default.nix index b6fde3b0..4405d020 100644 --- a/src/subsystems/nodejs/translators/npm/default.nix +++ b/src/subsystems/nodejs/translators/npm/default.nix @@ -62,7 +62,7 @@ popd # call package-lock translator - ${translators.package-lock.translateBinInstanced} $TMPDIR/newJsonInput + ${translators.package-lock.translateBinFinal} $TMPDIR/newJsonInput # get resolved package version export version=$(npm view $candidate version) diff --git a/src/subsystems/nodejs/translators/package-json/default.nix b/src/subsystems/nodejs/translators/package-json/default.nix index 0a8eaf24..d908341b 100644 --- a/src/subsystems/nodejs/translators/package-json/default.nix +++ b/src/subsystems/nodejs/translators/package-json/default.nix @@ -60,7 +60,7 @@ jq ".source = \"$newSource\"" -c -r $jsonInput > $TMPDIR/newJsonInput popd - ${translators.package-lock.translateBinInstanced} $TMPDIR/newJsonInput + ${translators.package-lock.translateBinFinal} $TMPDIR/newJsonInput ''; # inherit options from package-lock translator diff --git a/src/subsystems/php/translators/composer-json/default.nix b/src/subsystems/php/translators/composer-json/default.nix index 087f3366..e8fdda8d 100644 --- a/src/subsystems/php/translators/composer-json/default.nix +++ b/src/subsystems/php/translators/composer-json/default.nix @@ -75,7 +75,7 @@ jq ".source = \"$newSource\"" -c -r $jsonInput > $TMPDIR/newJsonInput popd - ${translators.composer-lock.translateBinInstanced} $TMPDIR/newJsonInput + ${translators.composer-lock.translateBinFinal} $TMPDIR/newJsonInput ''; # inherit options from composer-lock translator diff --git a/src/subsystems/php/translators/packagist/default.nix b/src/subsystems/php/translators/packagist/default.nix index 3552e410..ab88b417 100644 --- a/src/subsystems/php/translators/packagist/default.nix +++ b/src/subsystems/php/translators/packagist/default.nix @@ -70,10 +70,10 @@ if [ -f $TMPDIR/source/composer.lock ] then echo 'Translating with composer-lock' - ${translators.composer-lock.translateBinInstanced} $TMPDIR/newJsonInput + ${translators.composer-lock.translateBinFinal} $TMPDIR/newJsonInput else echo 'Translating with composer-json' - ${translators.composer-json.translateBinInstanced} $TMPDIR/newJsonInput + ${translators.composer-json.translateBinFinal} $TMPDIR/newJsonInput fi # add main package source info to dream-lock.json diff --git a/src/subsystems/rust/translators/cargo-toml/default.nix b/src/subsystems/rust/translators/cargo-toml/default.nix index 1aa710b7..66e70d1e 100644 --- a/src/subsystems/rust/translators/cargo-toml/default.nix +++ b/src/subsystems/rust/translators/cargo-toml/default.nix @@ -46,7 +46,7 @@ popd if [ $cargoResult -eq 0 ]; then - ${translators.cargo-lock.translateBinInstanced} $TMPDIR/newJsonInput + ${translators.cargo-lock.translateBinFinal} $TMPDIR/newJsonInput else echo "cargo failed to generate the lockfile" exit 1 diff --git a/src/subsystems/rust/translators/crates-io/default.nix b/src/subsystems/rust/translators/crates-io/default.nix index fd6addd7..a26fabd3 100644 --- a/src/subsystems/rust/translators/crates-io/default.nix +++ b/src/subsystems/rust/translators/crates-io/default.nix @@ -66,9 +66,9 @@ # we don't need to run cargo-toml translator if Cargo.lock exists if [ -f "$TMPDIR/source/Cargo.lock" ]; then - ${translators.cargo-lock.translateBinInstanced} $TMPDIR/newJsonInput + ${translators.cargo-lock.translateBinFinal} $TMPDIR/newJsonInput else - ${translators.cargo-toml.translateBinInstanced} $TMPDIR/newJsonInput + ${translators.cargo-toml.translateBinFinal} $TMPDIR/newJsonInput fi # add main package source info to dream-lock.json diff --git a/src/utils/default.nix b/src/utils/default.nix index e126ffe9..604ef339 100644 --- a/src/utils/default.nix +++ b/src/utils/default.nix @@ -206,7 +206,7 @@ in '' dreamLockPath="${project.dreamLockPath}" - ${translator.translateBinInstanced} ${argsJsonFile} + ${translator.translateBinFinal} ${argsJsonFile} # aggregate source hashes if [ "${l.toJSON aggregate}" == "true" ]; then diff --git a/tests/unit/test_pure_translators.py b/tests/unit/test_pure_translators.py index 1f05570a..ae6fbc84 100644 --- a/tests/unit/test_pure_translators.py +++ b/tests/unit/test_pure_translators.py @@ -65,7 +65,7 @@ def check_format_sourceSpec(sourceSpec): @pytest.mark.parametrize("p", projects) def test_packageName(p): defaultPackage = nix_ffi.eval( - f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.translateInstanced", + f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.translateFinal", params=dict( project=p['project'], source=p['source'], @@ -81,7 +81,7 @@ def test_packageName(p): @pytest.mark.parametrize("p", projects) def test_exportedPackages(p): exportedPackages = nix_ffi.eval( - f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.translateInstanced", + f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.translateFinal", params=dict( project=p['project'], source=p['source'], @@ -97,7 +97,7 @@ def test_exportedPackages(p): @pytest.mark.parametrize("p", projects) def test_extraObjects(p): extraObjects = nix_ffi.eval( - f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.translateInstanced", + f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.translateFinal", params=dict( project=p['project'], source=p['source'], @@ -121,7 +121,7 @@ def test_extraObjects(p): @pytest.mark.parametrize("p", projects) def test_location(p): location = nix_ffi.eval( - f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.translateInstanced", + f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.translateFinal", params=dict( project=p['project'], source=p['source'], @@ -136,7 +136,7 @@ def test_location(p): @pytest.mark.parametrize("p", projects) def test_serializedRawObjects(p): serializedRawObjects = nix_ffi.eval( - f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.translateInstanced", + f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.translateFinal", params=dict( project=p['project'], source=p['source'], @@ -159,7 +159,7 @@ def test_serializedRawObjects(p): @pytest.mark.parametrize("p", projects) def test_subsystemName(p): subsystemName = nix_ffi.eval( - f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.translateInstanced", + f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.translateFinal", params=dict( project=p['project'], source=p['source'], @@ -175,7 +175,7 @@ def test_subsystemName(p): @pytest.mark.parametrize("p", projects) def test_subsystemAttrs(p): subsystemAttrs = nix_ffi.eval( - f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.translateInstanced", + f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.translateFinal", params=dict( project=p['project'], source=p['source'], @@ -191,7 +191,7 @@ def test_subsystemAttrs(p): @pytest.mark.parametrize("p", projects) def test_translatorName(p): translatorName = nix_ffi.eval( - f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.translateInstanced", + f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.translateFinal", params=dict( project=p['project'], source=p['source'], @@ -207,7 +207,7 @@ def test_translatorName(p): @pytest.mark.parametrize("p", projects) def test_extractors(p): finalObjects = nix_ffi.eval( - f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.translateInstanced", + f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.translateFinal", params=dict( project=p['project'], source=p['source'], @@ -241,7 +241,7 @@ def test_extractors(p): @pytest.mark.parametrize("p", projects) def test_keys(p): objectsByKey = nix_ffi.eval( - f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.translateInstanced", + f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.translateFinal", params=dict( project=p['project'], source=p['source'], From 22d09979c13e9d2495a3acaac5b9d54434e89e82 Mon Sep 17 00:00:00 2001 From: Yusuf Bera Ertan Date: Fri, 7 Oct 2022 16:29:12 +0300 Subject: [PATCH 17/23] refactor: replace inherit (pkgs) with a with keyword --- .../translators/debian-binary/default.nix | 16 +++------------ .../haskell/translators/hackage/default.nix | 20 +++---------------- .../nodejs/translators/npm/default.nix | 18 +++-------------- .../translators/package-json/default.nix | 16 +++------------ .../php/translators/composer-json/default.nix | 15 +++----------- .../php/translators/packagist/default.nix | 15 +++----------- .../python/translators/pip/default.nix | 16 +++------------ .../translators/racket-impure/default.nix | 19 ++++-------------- .../rust/translators/cargo-toml/default.nix | 13 +++--------- .../rust/translators/crates-io/default.nix | 17 +++------------- 10 files changed, 31 insertions(+), 134 deletions(-) diff --git a/src/subsystems/debian/translators/debian-binary/default.nix b/src/subsystems/debian/translators/debian-binary/default.nix index f86be0b0..d62bd284 100644 --- a/src/subsystems/debian/translators/debian-binary/default.nix +++ b/src/subsystems/debian/translators/debian-binary/default.nix @@ -16,26 +16,16 @@ # by the input parameter `outFile`. # The output file must contain the dream lock data encoded as json. # See /src/specifications/dream-lock-example.json - translateBin = let - inherit - (pkgs) - bash - coreutils - jq - nix - callPackage - python3 - ; - in + translateBin = utils.writePureShellScript - [ + (with pkgs; [ bash coreutils jq nix (callPackage ./aptdream {}) python3 - ] + ]) '' # accroding to the spec, the translator reads the input from a json file jsonInput=$1 diff --git a/src/subsystems/haskell/translators/hackage/default.nix b/src/subsystems/haskell/translators/hackage/default.nix index 4d0ddda2..24971301 100644 --- a/src/subsystems/haskell/translators/hackage/default.nix +++ b/src/subsystems/haskell/translators/hackage/default.nix @@ -17,23 +17,9 @@ # by the input parameter `outFile`. # The output file must contain the dream lock data encoded as json. # See /src/specifications/dream-lock-example.json - translateBin = let - inherit - (pkgs) - bash - coreutils - curl - gnutar - gzip - haskellPackages - jq - moreutils - nix - python3 - ; - in + translateBin = utils.writePureShellScript - [ + (with pkgs; [ bash coreutils curl @@ -45,7 +31,7 @@ moreutils nix python3 - ] + ]) '' # accroding to the spec, the translator reads the input from a json file jsonInput=$1 diff --git a/src/subsystems/nodejs/translators/npm/default.nix b/src/subsystems/nodejs/translators/npm/default.nix index 4405d020..eb024cdc 100644 --- a/src/subsystems/nodejs/translators/npm/default.nix +++ b/src/subsystems/nodejs/translators/npm/default.nix @@ -9,21 +9,9 @@ # the input format is specified in /specifications/translator-call-example.json # this script receives a json file including the input paths and specialArgs - translateBin = let - inherit - (pkgs) - bash - coreutils - git - jq - moreutils - nodePackages - openssh - python3 - ; - in + translateBin = utils.writePureShellScript - [ + (with pkgs; [ bash coreutils git @@ -32,7 +20,7 @@ nodePackages.npm openssh python3 - ] + ]) '' # accroding to the spec, the translator reads the input from a json file jsonInput=$1 diff --git a/src/subsystems/nodejs/translators/package-json/default.nix b/src/subsystems/nodejs/translators/package-json/default.nix index d908341b..4b27d149 100644 --- a/src/subsystems/nodejs/translators/package-json/default.nix +++ b/src/subsystems/nodejs/translators/package-json/default.nix @@ -8,26 +8,16 @@ # the input format is specified in /specifications/translator-call-example.json # this script receives a json file including the input paths and specialArgs - translateBin = let - inherit - (pkgs) - bash - coreutils - git - jq - nodePackages - openssh - ; - in + translateBin = utils.writePureShellScript - [ + (with pkgs; [ bash coreutils git jq nodePackages.npm openssh - ] + ]) '' # accroding to the spec, the translator reads the input from a json file jsonInput=$1 diff --git a/src/subsystems/php/translators/composer-json/default.nix b/src/subsystems/php/translators/composer-json/default.nix index e8fdda8d..0a07618a 100644 --- a/src/subsystems/php/translators/composer-json/default.nix +++ b/src/subsystems/php/translators/composer-json/default.nix @@ -17,24 +17,15 @@ # by the input parameter `outFile`. # The output file must contain the dream lock data encoded as json. # See /src/specifications/dream-lock-example.json - translateBin = let - inherit - (pkgs) - bash - coreutils - moreutils - jq - phpPackages - ; - in + translateBin = utils.writePureShellScript - [ + (with pkgs; [ bash coreutils moreutils jq phpPackages.composer - ] + ]) '' # accroding to the spec, the translator reads the input from a json file jsonInput=$1 diff --git a/src/subsystems/php/translators/packagist/default.nix b/src/subsystems/php/translators/packagist/default.nix index ab88b417..44e93cb2 100644 --- a/src/subsystems/php/translators/packagist/default.nix +++ b/src/subsystems/php/translators/packagist/default.nix @@ -9,24 +9,15 @@ # the input format is specified in /specifications/translator-call-example.json # this script receives a json file including the input paths and specialArgs - translateBin = let - inherit - (pkgs) - coreutils - curl - jq - git - moreutils - ; - in + translateBin = utils.writePureShellScript - [ + (with pkgs; [ coreutils curl jq git moreutils - ] + ]) '' # according to the spec, the translator reads the input from a json file jsonInput=$1 diff --git a/src/subsystems/python/translators/pip/default.nix b/src/subsystems/python/translators/pip/default.nix index 3987ceaa..29285025 100644 --- a/src/subsystems/python/translators/pip/default.nix +++ b/src/subsystems/python/translators/pip/default.nix @@ -8,26 +8,16 @@ # the input format is specified in /specifications/translator-call-example.json # this script receives a json file including the input paths and extraArgs - translateBin = let - inherit - (pkgs) - bash - coreutils - jq - nix - remarshal - toml2json - ; - in + translateBin = utils.writePureShellScript - [ + (with pkgs; [ bash coreutils jq nix remarshal toml2json - ] + ]) '' # accroding to the spec, the translator reads the input from a json file jsonInput=$1 diff --git a/src/subsystems/racket/translators/racket-impure/default.nix b/src/subsystems/racket/translators/racket-impure/default.nix index 69433757..e378710b 100644 --- a/src/subsystems/racket/translators/racket-impure/default.nix +++ b/src/subsystems/racket/translators/racket-impure/default.nix @@ -17,37 +17,26 @@ # The output file must contain the dream lock data encoded as json. # See /src/specifications/dream-lock-example.json translateBin = let - inherit - (pkgs) - bash - coreutils - fetchurl - jq - nix - racket - runCommandLocal - ; - pruned-racket-catalog = let - src = fetchurl { + src = pkgs.fetchurl { url = "https://github.com/nix-community/pruned-racket-catalog/tarball/9f11e5ea5765c8a732c5e3129ca2b71237ae2bac"; sha256 = "sha256-/n30lailqSndoqPGWcFquCpQWVQcciMiypXYLhNmFUo="; }; in - runCommandLocal "pruned-racket-catalog" {} '' + pkgs.runCommandLocal "pruned-racket-catalog" {} '' mkdir $out cd $out tar --strip-components 1 -xf ${src} ''; in utils.writePureShellScript - [ + (with pkgs; [ bash coreutils jq nix racket - ] + ]) '' # according to the spec, the translator reads the input from a json file jsonInput=$1 diff --git a/src/subsystems/rust/translators/cargo-toml/default.nix b/src/subsystems/rust/translators/cargo-toml/default.nix index 66e70d1e..80661a90 100644 --- a/src/subsystems/rust/translators/cargo-toml/default.nix +++ b/src/subsystems/rust/translators/cargo-toml/default.nix @@ -8,20 +8,13 @@ # the input format is specified in /specifications/translator-call-example.json # this script receives a json file including the input paths and specialArgs - translateBin = let - inherit - (pkgs) - coreutils - jq - rustPlatform - ; - in + translateBin = utils.writePureShellScript - [ + (with pkgs; [ coreutils jq rustPlatform.rust.cargo - ] + ]) '' # according to the spec, the translator reads the input from a json file jsonInput=$1 diff --git a/src/subsystems/rust/translators/crates-io/default.nix b/src/subsystems/rust/translators/crates-io/default.nix index a26fabd3..131f4593 100644 --- a/src/subsystems/rust/translators/crates-io/default.nix +++ b/src/subsystems/rust/translators/crates-io/default.nix @@ -9,20 +9,9 @@ # the input format is specified in /specifications/translator-call-example.json # this script receives a json file including the input paths and specialArgs - translateBin = let - inherit - (pkgs) - coreutils - curl - gnutar - gzip - jq - moreutils - rustPlatform - ; - in + translateBin = utils.writePureShellScript - [ + (with pkgs; [ coreutils curl gnutar @@ -30,7 +19,7 @@ jq moreutils rustPlatform.rust.cargo - ] + ]) '' # according to the spec, the translator reads the input from a json file jsonInput=$1 From 33b93b20e98b67bb34fad153af4673d73ddc6ffc Mon Sep 17 00:00:00 2001 From: Yusuf Bera Ertan Date: Fri, 7 Oct 2022 16:32:07 +0300 Subject: [PATCH 18/23] refactor: translateFinal -> finalTranslate to make it more consistent with naming from nixpkgs and home-manager --- src/apps/translate-index/default.nix | 2 +- src/apps/translate/default.nix | 2 +- src/default.nix | 2 +- .../functions.translators/implementation.nix | 2 +- .../interfaces.translator/implementation.nix | 4 ++-- .../interfaces.translator/interface.nix | 4 ++-- .../haskell/translators/hackage/default.nix | 2 +- .../nodejs/translators/npm/default.nix | 2 +- .../translators/package-json/default.nix | 2 +- .../php/translators/composer-json/default.nix | 2 +- .../php/translators/packagist/default.nix | 4 ++-- .../rust/translators/cargo-toml/default.nix | 2 +- .../rust/translators/crates-io/default.nix | 4 ++-- src/utils/default.nix | 2 +- tests/unit/test_pure_translators.py | 20 +++++++++---------- 15 files changed, 28 insertions(+), 28 deletions(-) diff --git a/src/apps/translate-index/default.nix b/src/apps/translate-index/default.nix index 85da1e73..440442eb 100644 --- a/src/apps/translate-index/default.nix +++ b/src/apps/translate-index/default.nix @@ -54,7 +54,7 @@ in if [ ! -e "$bin" ]; then echo "building executable for translator $translator" ${callNixWithD2N} build -o "$bin" " - dream2nix.framework.translators.$translator.translateBinFinal + dream2nix.framework.translators.$translator.finalTranslateBin " fi done diff --git a/src/apps/translate/default.nix b/src/apps/translate/default.nix index 8dab4193..6f94d2ad 100644 --- a/src/apps/translate/default.nix +++ b/src/apps/translate/default.nix @@ -51,7 +51,7 @@ utils.writePureShellScriptBin translateBin="$TRANSLATOR_DIR/$translator" else translateBin=$(${callNixWithD2N} build --print-out-paths --no-link " - dream2nix.framework.translators.$translator.translateBinFinal + dream2nix.framework.translators.$translator.finalTranslateBin ") fi diff --git a/src/default.nix b/src/default.nix index 77667870..971902d9 100644 --- a/src/default.nix +++ b/src/default.nix @@ -502,7 +502,7 @@ in let l.forEach projectsPureUnresolved (proj: let translator = getTranslator proj.translator; - dreamLock'' = translator.translateFinal { + dreamLock'' = translator.finalTranslate { inherit source tree discoveredProjects; project = proj; }; diff --git a/src/modules/functions.translators/implementation.nix b/src/modules/functions.translators/implementation.nix index ef396e11..896b1ce7 100644 --- a/src/modules/functions.translators/implementation.nix +++ b/src/modules/functions.translators/implementation.nix @@ -47,7 +47,7 @@ (builtins.fromJSON (builtins.unsafeDiscardStringContext (builtins.readFile '''$1'''))); dreamLock' = - dream2nix.framework.translatorsBySubsystem.${subsystem}.${name}.translateFinal + dream2nix.framework.translatorsBySubsystem.${subsystem}.${name}.finalTranslate translatorArgs; # simpleTranslate2 puts dream-lock in result dreamLock = dreamLock'.result or dreamLock'; diff --git a/src/modules/interfaces.translator/implementation.nix b/src/modules/interfaces.translator/implementation.nix index f23be908..5dcc0136 100644 --- a/src/modules/interfaces.translator/implementation.nix +++ b/src/modules/interfaces.translator/implementation.nix @@ -4,7 +4,7 @@ ... }: { config = { - translateFinal = args: + finalTranslate = args: if config.translate != null then config.translate @@ -19,7 +19,7 @@ } ) else null; - translateBinFinal = + finalTranslateBin = if config.translate != null then framework.functions.translators.wrapPureTranslator diff --git a/src/modules/interfaces.translator/interface.nix b/src/modules/interfaces.translator/interface.nix index 2a99443b..1b07929a 100644 --- a/src/modules/interfaces.translator/interface.nix +++ b/src/modules/interfaces.translator/interface.nix @@ -46,7 +46,7 @@ in { type = t.nullOr (t.functionTo t.attrs); default = null; }; - translateFinal = lib.mkOption { + finalTranslate = lib.mkOption { type = t.nullOr (t.functionTo t.attrs); readOnly = true; description = '' @@ -57,7 +57,7 @@ in { type = t.nullOr t.package; default = null; }; - translateBinFinal = lib.mkOption { + finalTranslateBin = lib.mkOption { type = t.nullOr t.package; readOnly = true; description = '' diff --git a/src/subsystems/haskell/translators/hackage/default.nix b/src/subsystems/haskell/translators/hackage/default.nix index 24971301..02616d61 100644 --- a/src/subsystems/haskell/translators/hackage/default.nix +++ b/src/subsystems/haskell/translators/hackage/default.nix @@ -68,7 +68,7 @@ popd # execute cabal-plan translator - ${translators.cabal-plan.translateBinFinal} $TMPDIR/newJsonInput + ${translators.cabal-plan.finalTranslateBin} $TMPDIR/newJsonInput # finalize dream-lock. Add source and export default package # set correct package version under `packages` diff --git a/src/subsystems/nodejs/translators/npm/default.nix b/src/subsystems/nodejs/translators/npm/default.nix index eb024cdc..172ceedc 100644 --- a/src/subsystems/nodejs/translators/npm/default.nix +++ b/src/subsystems/nodejs/translators/npm/default.nix @@ -50,7 +50,7 @@ popd # call package-lock translator - ${translators.package-lock.translateBinFinal} $TMPDIR/newJsonInput + ${translators.package-lock.finalTranslateBin} $TMPDIR/newJsonInput # get resolved package version export version=$(npm view $candidate version) diff --git a/src/subsystems/nodejs/translators/package-json/default.nix b/src/subsystems/nodejs/translators/package-json/default.nix index 4b27d149..dc4949d1 100644 --- a/src/subsystems/nodejs/translators/package-json/default.nix +++ b/src/subsystems/nodejs/translators/package-json/default.nix @@ -50,7 +50,7 @@ jq ".source = \"$newSource\"" -c -r $jsonInput > $TMPDIR/newJsonInput popd - ${translators.package-lock.translateBinFinal} $TMPDIR/newJsonInput + ${translators.package-lock.finalTranslateBin} $TMPDIR/newJsonInput ''; # inherit options from package-lock translator diff --git a/src/subsystems/php/translators/composer-json/default.nix b/src/subsystems/php/translators/composer-json/default.nix index 0a07618a..4b1940cc 100644 --- a/src/subsystems/php/translators/composer-json/default.nix +++ b/src/subsystems/php/translators/composer-json/default.nix @@ -66,7 +66,7 @@ jq ".source = \"$newSource\"" -c -r $jsonInput > $TMPDIR/newJsonInput popd - ${translators.composer-lock.translateBinFinal} $TMPDIR/newJsonInput + ${translators.composer-lock.finalTranslateBin} $TMPDIR/newJsonInput ''; # inherit options from composer-lock translator diff --git a/src/subsystems/php/translators/packagist/default.nix b/src/subsystems/php/translators/packagist/default.nix index 44e93cb2..2a35d542 100644 --- a/src/subsystems/php/translators/packagist/default.nix +++ b/src/subsystems/php/translators/packagist/default.nix @@ -61,10 +61,10 @@ if [ -f $TMPDIR/source/composer.lock ] then echo 'Translating with composer-lock' - ${translators.composer-lock.translateBinFinal} $TMPDIR/newJsonInput + ${translators.composer-lock.finalTranslateBin} $TMPDIR/newJsonInput else echo 'Translating with composer-json' - ${translators.composer-json.translateBinFinal} $TMPDIR/newJsonInput + ${translators.composer-json.finalTranslateBin} $TMPDIR/newJsonInput fi # add main package source info to dream-lock.json diff --git a/src/subsystems/rust/translators/cargo-toml/default.nix b/src/subsystems/rust/translators/cargo-toml/default.nix index 80661a90..0c0d6b26 100644 --- a/src/subsystems/rust/translators/cargo-toml/default.nix +++ b/src/subsystems/rust/translators/cargo-toml/default.nix @@ -39,7 +39,7 @@ popd if [ $cargoResult -eq 0 ]; then - ${translators.cargo-lock.translateBinFinal} $TMPDIR/newJsonInput + ${translators.cargo-lock.finalTranslateBin} $TMPDIR/newJsonInput else echo "cargo failed to generate the lockfile" exit 1 diff --git a/src/subsystems/rust/translators/crates-io/default.nix b/src/subsystems/rust/translators/crates-io/default.nix index 131f4593..bd8eaa58 100644 --- a/src/subsystems/rust/translators/crates-io/default.nix +++ b/src/subsystems/rust/translators/crates-io/default.nix @@ -55,9 +55,9 @@ # we don't need to run cargo-toml translator if Cargo.lock exists if [ -f "$TMPDIR/source/Cargo.lock" ]; then - ${translators.cargo-lock.translateBinFinal} $TMPDIR/newJsonInput + ${translators.cargo-lock.finalTranslateBin} $TMPDIR/newJsonInput else - ${translators.cargo-toml.translateBinFinal} $TMPDIR/newJsonInput + ${translators.cargo-toml.finalTranslateBin} $TMPDIR/newJsonInput fi # add main package source info to dream-lock.json diff --git a/src/utils/default.nix b/src/utils/default.nix index 604ef339..aad60557 100644 --- a/src/utils/default.nix +++ b/src/utils/default.nix @@ -206,7 +206,7 @@ in '' dreamLockPath="${project.dreamLockPath}" - ${translator.translateBinFinal} ${argsJsonFile} + ${translator.finalTranslateBin} ${argsJsonFile} # aggregate source hashes if [ "${l.toJSON aggregate}" == "true" ]; then diff --git a/tests/unit/test_pure_translators.py b/tests/unit/test_pure_translators.py index ae6fbc84..f304c924 100644 --- a/tests/unit/test_pure_translators.py +++ b/tests/unit/test_pure_translators.py @@ -65,7 +65,7 @@ def check_format_sourceSpec(sourceSpec): @pytest.mark.parametrize("p", projects) def test_packageName(p): defaultPackage = nix_ffi.eval( - f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.translateFinal", + f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.finalTranslate", params=dict( project=p['project'], source=p['source'], @@ -81,7 +81,7 @@ def test_packageName(p): @pytest.mark.parametrize("p", projects) def test_exportedPackages(p): exportedPackages = nix_ffi.eval( - f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.translateFinal", + f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.finalTranslate", params=dict( project=p['project'], source=p['source'], @@ -97,7 +97,7 @@ def test_exportedPackages(p): @pytest.mark.parametrize("p", projects) def test_extraObjects(p): extraObjects = nix_ffi.eval( - f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.translateFinal", + f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.finalTranslate", params=dict( project=p['project'], source=p['source'], @@ -121,7 +121,7 @@ def test_extraObjects(p): @pytest.mark.parametrize("p", projects) def test_location(p): location = nix_ffi.eval( - f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.translateFinal", + f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.finalTranslate", params=dict( project=p['project'], source=p['source'], @@ -136,7 +136,7 @@ def test_location(p): @pytest.mark.parametrize("p", projects) def test_serializedRawObjects(p): serializedRawObjects = nix_ffi.eval( - f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.translateFinal", + f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.finalTranslate", params=dict( project=p['project'], source=p['source'], @@ -159,7 +159,7 @@ def test_serializedRawObjects(p): @pytest.mark.parametrize("p", projects) def test_subsystemName(p): subsystemName = nix_ffi.eval( - f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.translateFinal", + f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.finalTranslate", params=dict( project=p['project'], source=p['source'], @@ -175,7 +175,7 @@ def test_subsystemName(p): @pytest.mark.parametrize("p", projects) def test_subsystemAttrs(p): subsystemAttrs = nix_ffi.eval( - f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.translateFinal", + f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.finalTranslate", params=dict( project=p['project'], source=p['source'], @@ -191,7 +191,7 @@ def test_subsystemAttrs(p): @pytest.mark.parametrize("p", projects) def test_translatorName(p): translatorName = nix_ffi.eval( - f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.translateFinal", + f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.finalTranslate", params=dict( project=p['project'], source=p['source'], @@ -207,7 +207,7 @@ def test_translatorName(p): @pytest.mark.parametrize("p", projects) def test_extractors(p): finalObjects = nix_ffi.eval( - f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.translateFinal", + f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.finalTranslate", params=dict( project=p['project'], source=p['source'], @@ -241,7 +241,7 @@ def test_extractors(p): @pytest.mark.parametrize("p", projects) def test_keys(p): objectsByKey = nix_ffi.eval( - f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.translateFinal", + f"framework.translatorsBySubsystem.{p['subsystem']}.{p['translator']}.finalTranslate", params=dict( project=p['project'], source=p['source'], From f94bcfff1c4b51140d81db82ded571349ac12e0d Mon Sep 17 00:00:00 2001 From: Yusuf Bera Ertan Date: Fri, 7 Oct 2022 16:37:26 +0300 Subject: [PATCH 19/23] fix: dont use with to avoid scope errors --- src/subsystems/haskell/translators/stack-lock/default.nix | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/subsystems/haskell/translators/stack-lock/default.nix b/src/subsystems/haskell/translators/stack-lock/default.nix index c1411259..be56bb33 100644 --- a/src/subsystems/haskell/translators/stack-lock/default.nix +++ b/src/subsystems/haskell/translators/stack-lock/default.nix @@ -188,8 +188,7 @@ in { }; getDependencyNames = finalObj: objectsByName: let - cabal = with finalObj; - cabalData.${name}.${version}; + cabal = cabalData.${finalObj.name}.${finalObj.version}; targetBuildDepends = cabal.library.condTreeData.build-info.targetBuildDepends or []; From e32c973455843d4259e26516e042f0739b1280bd Mon Sep 17 00:00:00 2001 From: Yusuf Bera Ertan Date: Fri, 7 Oct 2022 16:55:49 +0300 Subject: [PATCH 20/23] fix: dont use with to avoid scope errors --- src/subsystems/haskell/translators/stack-lock/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/subsystems/haskell/translators/stack-lock/default.nix b/src/subsystems/haskell/translators/stack-lock/default.nix index be56bb33..931ff343 100644 --- a/src/subsystems/haskell/translators/stack-lock/default.nix +++ b/src/subsystems/haskell/translators/stack-lock/default.nix @@ -306,7 +306,7 @@ in { { type = "http"; url = haskellUtils.getHackageUrl finalObj; - hash = with finalObj; "sha256:${all-cabal-hashes.${name}.${version}.SHA256}"; + hash = "sha256:${all-cabal-hashes.${finalObj.name}.${finalObj.version}.SHA256}"; }; }; From 033f894373c64c269e619181361893bd5ffb2cf1 Mon Sep 17 00:00:00 2001 From: Yusuf Bera Ertan Date: Fri, 7 Oct 2022 17:01:20 +0300 Subject: [PATCH 21/23] fix: remove default discoverer from discoverers by subsystem --- src/modules/discoverers/implementation.nix | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/modules/discoverers/implementation.nix b/src/modules/discoverers/implementation.nix index 13edcd9c..b9d01987 100644 --- a/src/modules/discoverers/implementation.nix +++ b/src/modules/discoverers/implementation.nix @@ -5,6 +5,9 @@ in { config = { discoverers = funcs.import_ collectedModules; - discoverersBySubsystem = funcs.structureBySubsystem config.discoverers; + discoverersBySubsystem = funcs.structureBySubsystem ( + # remove the "default" discoverer we create, as it's not subsystem specific. + builtins.removeAttrs config.discoverers ["default"] + ); }; } From 10b6663f7b691b4d87b339ad356a51e9d7842a41 Mon Sep 17 00:00:00 2001 From: Yusuf Bera Ertan Date: Fri, 7 Oct 2022 17:32:35 +0300 Subject: [PATCH 22/23] fix: make all function options non-mergeable --- src/modules/functions.combined-fetcher/interface.nix | 2 +- src/modules/functions.default-fetcher/interface.nix | 2 +- src/modules/functions.discoverers/interface.nix | 6 +++--- src/modules/functions.fetchers/interface.nix | 12 ++++++------ src/modules/functions.translators/interface.nix | 8 ++++++-- 5 files changed, 17 insertions(+), 13 deletions(-) diff --git a/src/modules/functions.combined-fetcher/interface.nix b/src/modules/functions.combined-fetcher/interface.nix index 4551042d..c09af2f5 100644 --- a/src/modules/functions.combined-fetcher/interface.nix +++ b/src/modules/functions.combined-fetcher/interface.nix @@ -4,7 +4,7 @@ in { options = { functions.combinedFetcher = l.mkOption { - type = t.functionTo t.attrs; + type = t.uniq (t.functionTo t.attrs); }; }; } diff --git a/src/modules/functions.default-fetcher/interface.nix b/src/modules/functions.default-fetcher/interface.nix index e1dbb714..248fbba0 100644 --- a/src/modules/functions.default-fetcher/interface.nix +++ b/src/modules/functions.default-fetcher/interface.nix @@ -4,7 +4,7 @@ in { options = { functions.defaultFetcher = l.mkOption { - type = t.functionTo t.attrs; + type = t.uniq (t.functionTo t.attrs); }; }; } diff --git a/src/modules/functions.discoverers/interface.nix b/src/modules/functions.discoverers/interface.nix index d716f018..8169f022 100644 --- a/src/modules/functions.discoverers/interface.nix +++ b/src/modules/functions.discoverers/interface.nix @@ -4,13 +4,13 @@ in { options.functions.discoverers = { discoverProjects = l.mkOption { - type = t.functionTo (t.listOf t.attrs); + type = t.uniq (t.functionTo (t.listOf t.attrs)); }; applyProjectSettings = l.mkOption { - type = t.functionTo (t.functionTo (t.listOf t.attrs)); + type = t.uniq (t.functionTo (t.functionTo (t.listOf t.attrs))); }; getDreamLockPath = l.mkOption { - type = t.functionTo (t.functionTo t.path); + type = t.uniq (t.functionTo (t.functionTo t.path)); }; }; } diff --git a/src/modules/functions.fetchers/interface.nix b/src/modules/functions.fetchers/interface.nix index 58be0fd3..5f9079d6 100644 --- a/src/modules/functions.fetchers/interface.nix +++ b/src/modules/functions.fetchers/interface.nix @@ -5,31 +5,31 @@ in { options = { functions.fetchers = { constructSource = l.mkOption { - type = t.functionTo t.attrs; + type = t.uniq (t.functionTo t.attrs); }; updateSource = l.mkOption { - type = t.functionTo t.attrs; + type = t.uniq (t.functionTo t.attrs); description = '' update source spec to different version ''; }; fetchSource = l.mkOption { - type = t.functionTo t.path; + type = t.uniq (t.functionTo t.path); description = '' fetch a source defined via a dream lock source spec ''; }; fetchShortcut = l.mkOption { - type = t.functionTo t.path; + type = t.uniq (t.functionTo t.path); description = '' fetch a source defined by a shortcut ''; }; parseShortcut = l.mkOption { - type = t.functionTo t.attrs; + type = t.uniq (t.functionTo t.attrs); }; translateShortcut = l.mkOption { - type = t.functionTo t.attrs; + type = t.uniq (t.functionTo t.attrs); description = '' translate shortcut to dream lock source spec ''; diff --git a/src/modules/functions.translators/interface.nix b/src/modules/functions.translators/interface.nix index 7f9660e0..19c5a183 100644 --- a/src/modules/functions.translators/interface.nix +++ b/src/modules/functions.translators/interface.nix @@ -3,7 +3,11 @@ t = lib.types; in { options.functions.translators = { - makeTranslatorDefaultArgs = lib.mkOption {type = t.functionTo t.anything;}; - wrapPureTranslator = lib.mkOption {type = t.functionTo t.package;}; + makeTranslatorDefaultArgs = lib.mkOption { + type = t.uniq (t.functionTo t.attrs); + }; + wrapPureTranslator = lib.mkOption { + type = t.uniq (t.functionTo t.package); + }; }; } From 109e32da9c37fcc6700ce0452644e088922cc3da Mon Sep 17 00:00:00 2001 From: Yusuf Bera Ertan Date: Fri, 7 Oct 2022 17:40:42 +0300 Subject: [PATCH 23/23] fix: make parseParams optional --- src/modules/functions.fetchers/implementation.nix | 2 +- src/modules/interfaces.builder/interface.nix | 2 +- src/modules/interfaces.discoverer/interface.nix | 2 +- src/modules/interfaces.fetcher/interface.nix | 7 ++++--- src/modules/interfaces.translator/interface.nix | 6 +++--- 5 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/modules/functions.fetchers/implementation.nix b/src/modules/functions.fetchers/implementation.nix index 6c6750d4..164f8e32 100644 --- a/src/modules/functions.fetchers/implementation.nix +++ b/src/modules/functions.fetchers/implementation.nix @@ -186,7 +186,7 @@ in { fetcher = fetchers."${fetcherName}"; args = - if fetcher ? parseParams + if fetcher.parseParams != null then fetcher.parseParams params else if b.length params != b.length fetcher.inputs then diff --git a/src/modules/interfaces.builder/interface.nix b/src/modules/interfaces.builder/interface.nix index 9a925820..1985ed56 100644 --- a/src/modules/interfaces.builder/interface.nix +++ b/src/modules/interfaces.builder/interface.nix @@ -17,7 +17,7 @@ in { description = "Subsystem of the builder."; }; build = lib.mkOption { - type = t.functionTo t.attrs; + type = t.uniq (t.functionTo t.attrs); default = _: {}; }; type = lib.mkOption { diff --git a/src/modules/interfaces.discoverer/interface.nix b/src/modules/interfaces.discoverer/interface.nix index e84bcaab..22f6c4c0 100644 --- a/src/modules/interfaces.discoverer/interface.nix +++ b/src/modules/interfaces.discoverer/interface.nix @@ -17,7 +17,7 @@ in { description = "Subsystem of the discoverer."; }; discover = lib.mkOption { - type = t.functionTo (t.listOf t.attrs); + type = t.uniq (t.functionTo (t.listOf t.attrs)); default = _: {}; }; }; diff --git a/src/modules/interfaces.fetcher/interface.nix b/src/modules/interfaces.fetcher/interface.nix index dba00e34..e5ca2dc6 100644 --- a/src/modules/interfaces.fetcher/interface.nix +++ b/src/modules/interfaces.fetcher/interface.nix @@ -41,14 +41,15 @@ in { ''; }; parseParams = l.mkOption { - type = t.functionTo t.attrs; + type = t.nullOr (t.uniq (t.functionTo t.attrs)); + default = null; }; outputs = l.mkOption { - type = t.functionTo ( + type = t.uniq (t.functionTo ( t.submoduleWith { modules = [outputsOptions]; } - ); + )); }; }; } diff --git a/src/modules/interfaces.translator/interface.nix b/src/modules/interfaces.translator/interface.nix index 1b07929a..3d3af146 100644 --- a/src/modules/interfaces.translator/interface.nix +++ b/src/modules/interfaces.translator/interface.nix @@ -8,7 +8,7 @@ in { default = false; }; discoverProject = lib.mkOption { - type = t.nullOr (t.functionTo (t.anything)); + type = t.nullOr (t.uniq (t.functionTo (t.anything))); default = null; }; extraArgs = lib.mkOption { @@ -43,11 +43,11 @@ in { type = t.str; }; translate = lib.mkOption { - type = t.nullOr (t.functionTo t.attrs); + type = t.nullOr (t.uniq (t.functionTo t.attrs)); default = null; }; finalTranslate = lib.mkOption { - type = t.nullOr (t.functionTo t.attrs); + type = t.nullOr (t.uniq (t.functionTo t.attrs)); readOnly = true; description = '' the final translate that you should use.