From f685e44d59c36ba7437eee6500cbbaa6050fcf07 Mon Sep 17 00:00:00 2001 From: DavHau Date: Mon, 20 Sep 2021 20:52:31 +0100 Subject: [PATCH] Add builder for nodejs - python builder support application - add version to dream.lock - allowBuiltinsFetchers config option - node2nix builder - handle github sources without hash --- flake.lock | 17 +++ flake.nix | 15 ++- specifications/generic-lock-example.json | 4 +- .../subsystems/nodejs/dream.lock.example.json | 5 + src/apps/cli.py | 16 ++- src/apps/default.nix | 3 + src/builders/default.nix | 7 ++ src/builders/nodejs/node2nix/default.nix | 72 +++++++++++ .../python/simple-builder/default.nix | 27 +++- src/config.json | 3 + src/default.nix | 35 ++++-- src/fetchers/default-fetcher.nix | 58 +++++++-- src/fetchers/default.nix | 3 + src/translators/default.nix | 29 +++-- .../nodejs/pure/npmlock2nix/default.nix | 119 ++++++++++++++++-- 15 files changed, 368 insertions(+), 45 deletions(-) create mode 100644 specifications/subsystems/nodejs/dream.lock.example.json create mode 100644 src/builders/nodejs/node2nix/default.nix create mode 100644 src/config.json diff --git a/flake.lock b/flake.lock index 1217faa8..fc75203f 100644 --- a/flake.lock +++ b/flake.lock @@ -15,6 +15,22 @@ "type": "indirect" } }, + "node2nix": { + "flake": false, + "locked": { + "lastModified": 1613594272, + "narHash": "sha256-fcnPNexhowSkatLSl+0dat5oDaWKH53Pg+VKrE8+x+Q=", + "owner": "svanderburg", + "repo": "node2nix", + "rev": "0c94281ea98f1b17532176106f90f909aa133704", + "type": "github" + }, + "original": { + "owner": "svanderburg", + "repo": "node2nix", + "type": "github" + } + }, "npmlock2nix": { "flake": false, "locked": { @@ -34,6 +50,7 @@ "root": { "inputs": { "nixpkgs": "nixpkgs", + "node2nix": "node2nix", "npmlock2nix": "npmlock2nix" } } diff --git a/flake.nix b/flake.nix index f77b4e8d..3157fbfd 100644 --- a/flake.nix +++ b/flake.nix @@ -3,10 +3,11 @@ inputs = { nixpkgs.url = "nixpkgs/nixos-unstable"; + node2nix = { url = "github:svanderburg/node2nix"; flake = false; }; npmlock2nix = { url = "github:nix-community/npmlock2nix"; flake = false; }; }; - outputs = { self, nixpkgs, npmlock2nix }: + outputs = { self, nixpkgs, node2nix, npmlock2nix }: let lib = nixpkgs.lib; @@ -20,13 +21,16 @@ overlays = [ self.overlay ]; }); + externalSourcesFor = forAllSystems (system: nixpkgsFor."${system}".runCommand "dream2nix-vendored" {} '' + mkdir -p $out/{npmlock2nix,node2nix} + cp ${npmlock2nix}/{internal.nix,LICENSE} $out/npmlock2nix/ + cp ${node2nix}/{nix/node-env.nix,LICENSE} $out/node2nix/ + ''); + dream2nixFor = forAllSystems (system: import ./src rec { pkgs = nixpkgsFor."${system}"; + externalSources = externalSourcesFor."${system}"; inherit lib; - externalSources = pkgs.runCommand "dream2nix-imported" {} '' - mkdir -p $out/npmlock2nix - cp ${npmlock2nix}/{internal.nix,LICENSE} $out/npmlock2nix/ - ''; }); in @@ -54,6 +58,7 @@ ]; shellHook = '' export NIX_PATH=nixpkgs=${nixpkgs} + export d2nExternalSources=${externalSourcesFor."${system}"} ''; }); }; diff --git a/specifications/generic-lock-example.json b/specifications/generic-lock-example.json index f8aa85d2..e6a0b9f2 100644 --- a/specifications/generic-lock-example.json +++ b/specifications/generic-lock-example.json @@ -3,11 +3,13 @@ "requests": { "url": "https://download.pypi.org/requests/2.28.0", "hash": "000000000000000000000000000000000000000", + "version": "1.2.3", "type": "fetchurl" }, "certifi": { "url": "https://download.pypi.org/certifi/2.0", "hash": "000000000000000000000000000000000000000", + "version": "2.3.4", "type": "fetchurl" } }, @@ -32,4 +34,4 @@ "certifi": "wheel" } } -} \ No newline at end of file +} diff --git a/specifications/subsystems/nodejs/dream.lock.example.json b/specifications/subsystems/nodejs/dream.lock.example.json new file mode 100644 index 00000000..3f7b0657 --- /dev/null +++ b/specifications/subsystems/nodejs/dream.lock.example.json @@ -0,0 +1,5 @@ +{ + "buildSystem": { + "nodejsVersion": 14 + } +} diff --git a/src/apps/cli.py b/src/apps/cli.py index 8800864d..59a59e79 100644 --- a/src/apps/cli.py +++ b/src/apps/cli.py @@ -13,7 +13,8 @@ with open (os.environ.get("translatorsJsonFile")) as f: def strip_hashes_from_lock(lock): for source in lock['sources'].values(): - del source['hash'] + if 'hash' in source: + del source['hash'] def order_dict(d): @@ -75,7 +76,9 @@ def translate(args): raise Exception(f"Input path '{path}' does not exist") inputFiles = list(filter(lambda p: os.path.isfile(p), inputPaths)) + inputFiles = list(map(lambda p:os.path.realpath(p), inputFiles)) inputDirectories = list(filter(lambda p: os.path.isdir(p), inputPaths)) + inputDirectories = list(map(lambda p:os.path.realpath(p), inputDirectories)) # determine output directory if os.path.isdir(args.output): @@ -162,6 +165,17 @@ def translate(args): lock['generic']['translatedBy'] = f"{subsystem}.{trans_type}.{trans_name}" lock['generic']['translatorParams'] = " ".join(sys.argv[2:]) + # clean up dependency graph + # remove empty entries + if 'dependencyGraph' in lock['generic']: + for pname, deps in lock['generic']['dependencyGraph'].copy().items(): + if not deps: + del lock['generic']['dependencyGraph'][pname] + + # re-write dream.lock + with open(output, 'w') as f: + json.dump(lock, f, indent=2) + # calculate combined hash if --combined was specified if args.combined: diff --git a/src/apps/default.nix b/src/apps/default.nix index 75086cba..5ac3ae36 100644 --- a/src/apps/default.nix +++ b/src/apps/default.nix @@ -43,6 +43,9 @@ in mkdir $target/external cp -r ${externalSources}/* $target/external/ chmod -R +w $target + + echo "Installed dream2nix successfully to '$target'." + echo "Please check/modify settings in '$target/config.json'" '' ) {}; } diff --git a/src/builders/default.nix b/src/builders/default.nix index ae2c718f..79fa1450 100644 --- a/src/builders/default.nix +++ b/src/builders/default.nix @@ -9,5 +9,12 @@ simpleBuilder = callPackage ./python/simple-builder {}; }; + + nodejs = rec { + + default = node2nix; + + node2nix = callPackage ./nodejs/node2nix {}; + }; } diff --git a/src/builders/nodejs/node2nix/default.nix b/src/builders/nodejs/node2nix/default.nix new file mode 100644 index 00000000..97aa1300 --- /dev/null +++ b/src/builders/nodejs/node2nix/default.nix @@ -0,0 +1,72 @@ +# builder imported from node2nix + +{ + externals, + node2nix ? externals.node2nix, + + lib, + pkgs, + ... +}: + +{ + fetchedSources, + dreamLock, +}: +let + mainPackageName = dreamLock.generic.mainPackage; + + nodejsVersion = dreamLock.buildSystem.nodejsVersion; + + nodejs = + pkgs."nodejs-${builtins.toString nodejsVersion}_x" + or (throw "Could not find nodejs version '${nodejsVersion}' in pkgs"); + + node2nixEnv = node2nix nodejs; + + # make node2nix compatible sources + makeSource = name: { + name = lib.head (lib.splitString "#" name); + packageName = lib.head (lib.splitString "#" name); + version = dreamLock.sources."${name}".version; + src = fetchedSources."${name}"; + dependencies = lib.forEach dreamLock.generic.dependencyGraph."${name}" or [] (dependency: + makeSource dependency + ); + }; + + callNode2Nix = funcName: args: + node2nixEnv."${funcName}" rec { + name = mainPackageName; + packageName = name; + version = dreamLock.sources."${mainPackageName}".version; + dependencies = + lib.forEach + (lib.filter + (pname: pname != mainPackageName) + (lib.attrNames dreamLock.generic.dependencyGraph) + ) + (dependency: makeSource dependency); + # buildInputs ? [] + # npmFlags ? "" + # dontNpmInstall ? false + # preRebuild ? "" + # dontStrip ? true + # unpackPhase ? "true" + # buildPhase ? "true" + # meta ? {} + production = true; + bypassCache = true; + reconstructLock = true; + src = fetchedSources."${dreamLock.generic.mainPackage}"; + } + // args; + +in +{ + + package = callNode2Nix "buildNodePackage" {}; + + shell = callNode2Nix "buildNodeShell" {}; + +} diff --git a/src/builders/python/simple-builder/default.nix b/src/builders/python/simple-builder/default.nix index 167a1cdd..e33a97b8 100644 --- a/src/builders/python/simple-builder/default.nix +++ b/src/builders/python/simple-builder/default.nix @@ -1,3 +1,5 @@ +# A very simple single derivation python builder + { lib, pkgs, @@ -11,12 +13,29 @@ let python = pkgs."${dreamLock.buildSystem.pythonAttr}"; + + buildFunc = + if dreamLock.buildSystem.application then + python.pkgs.buildPythonApplication + else + python.pkgs.buildPythonPackage; + + mainPackageName = dreamLock.generic.mainPackage; + + packageName = + if mainPackageName == null then + if dreamLock.buildSystem.application then + "application" + else + "environment" + else + mainPackageName; in -python.pkgs.buildPythonPackage { - name = "python-environment"; +buildFunc { + name = packageName; format = ""; - src = lib.attrValues fetchedSources; + src = fetchedSources."${toString (mainPackageName)}" or null; buildInputs = pkgs.pythonManylinuxPackages.manylinux1; nativeBuildInputs = [ pkgs.autoPatchelfHook python.pkgs.wheelUnpackHook ]; unpackPhase = '' @@ -32,7 +51,7 @@ python.pkgs.buildPythonPackage { runHook preInstall mkdir -p "$out/${python.sitePackages}" export PYTHONPATH="$out/${python.sitePackages}:$PYTHONPATH" - ${python}/bin/python -m pip install ./dist/*.{whl,tar.gz,zip} \ + ${python}/bin/python -m pip install ./dist/*.{whl,tar.gz,zip} $src \ --no-index \ --no-warn-script-location \ --prefix="$out" \ diff --git a/src/config.json b/src/config.json new file mode 100644 index 00000000..c54ef2d9 --- /dev/null +++ b/src/config.json @@ -0,0 +1,3 @@ +{ + "allowBuiltinFetchers": true +} diff --git a/src/default.nix b/src/default.nix index 649e07f9..e8eeaa6e 100644 --- a/src/default.nix +++ b/src/default.nix @@ -2,8 +2,10 @@ pkgs ? import {}, lib ? pkgs.lib, externalSources ? - if builtins.getEnv "d2nExternalSources" != "" then + # if called via CLI, load externals via env + if builtins ? getEnv && builtins.getEnv "d2nExternalSources" != "" then builtins.getEnv "d2nExternalSources" + # load from default dircetory else ./external, }: @@ -14,24 +16,35 @@ let callPackage = f: args: pkgs.callPackage f (args // { inherit callPackage; + inherit externals; + inherit externalSources; inherit utils; }); externals = { npmlock2nix = pkgs.callPackage "${externalSources}/npmlock2nix/internal.nix" {}; + node2nix = nodejs: pkgs.callPackage "${externalSources}/node2nix/node-env.nix" { inherit nodejs; }; }; + config = builtins.fromJSON (builtins.readFile ./config.json); + in rec { - apps = callPackage ./apps { inherit externalSources location translators; }; + # apps for CLI and installation + apps = callPackage ./apps { inherit location translators; }; + # builder implementaitons for all subsystems builders = callPackage ./builders {}; - fetchers = callPackage ./fetchers {}; + # fetcher implementations + fetchers = callPackage ./fetchers { + inherit (config) allowBuiltinFetchers; + }; - translators = callPackage ./translators { inherit externalSources externals location; }; + # the translator modules and utils for all subsystems + translators = callPackage ./translators { inherit location; }; # the location of the dream2nix framework for self references (update scripts, etc.) @@ -43,7 +56,10 @@ rec { let buildSystem = dreamLock.generic.buildSystem; in - builders."${buildSystem}".default; + if ! builders ? "${buildSystem}" then + throw "Could not find any builder for subsystem '${buildSystem}'" + else + builders."${buildSystem}".default; # detect if granular or combined fetching must be used @@ -54,24 +70,27 @@ rec { fetchers.defaultFetcher; + # automatically parse dream.lock if passed as file parseLock = lock: if builtins.isPath lock || builtins.isString lock then builtins.fromJSON (builtins.readFile lock) else lock; - + # fetch only sources and do not build fetchSources = { dreamLock, builder ? findBuilder (parseLock dreamLock), fetcher ? findFetcher (parseLock dreamLock), sourceOverrides ? oldSources: {}, + allowBuiltinFetchers ? true, }: let # if generic lock is a file, read and parse it dreamLock' = (parseLock dreamLock); fetched = fetcher { + inherit allowBuiltinFetchers; sources = dreamLock'.sources; sourcesCombinedHash = dreamLock'.generic.sourcesCombinedHash; }; @@ -98,13 +117,15 @@ rec { }; - # automatically build package defined by generic lock + # build package defined by dream.lock + # TODO: rename to riseAndShine buildPackage = { dreamLock, builder ? findBuilder (parseLock dreamLock), fetcher ? findFetcher (parseLock dreamLock), sourceOverrides ? oldSources: {}, + allowBuiltinFetchers ? true, }@args: let # if generic lock is a file, read and parse it diff --git a/src/fetchers/default-fetcher.nix b/src/fetchers/default-fetcher.nix index b43ae639..3c6538f9 100644 --- a/src/fetchers/default-fetcher.nix +++ b/src/fetchers/default-fetcher.nix @@ -11,30 +11,74 @@ { # sources attrset from generic lock sources, + allowBuiltinFetchers, ... }: + +let + githubMissingHashErrorText = pname: '' + Error: Cannot verify the integrity of the source of '${pname}' + It is a github reference with no hash providedand. + Solve this problem via any of the wollowing ways: + + - (alternative 1): allow the use of builtin fetchers (which can verify using git rev). + ``` + dream2nix.buildPackage { + ... + allowBuiltinFetchers = true; + ... + } + ``` + + - (alternative 2): add a hash to the source via override + ``` + dream2nix.buildPackage { + ... + sourceOverrides = oldSources: { + "${pname}" = oldSources."${pname}".overrideAttrs (_:{ + hash = ""; + }) + } + ... + } + ``` + + ''; +in + { # attrset: pname -> path of downloaded source fetchedSources = lib.mapAttrs (pname: source: if source.type == "github" then - fetchFromGitHub { - inherit (source) url owner repo rev; - sha256 = source.hash or null; - } + # handle when no hash is provided + if ! source ? hash then + if allowBuiltinFetchers then + builtins.fetchGit { + inherit (source) rev; + allRefs = true; + url = "https://github.com/${source.owner}/${source.repo}"; + } + else + throw githubMissingHashErrorText pname + else + fetchFromGitHub { + inherit (source) url owner repo rev; + hash = source.hash or null; + } else if source.type == "gitlab" then fetchFromGitLab { inherit (source) url owner repo rev; - sha256 = source.hash or null; + hash = source.hash or null; } else if source.type == "git" then fetchgit { inherit (source) url rev; - sha256 = source.hash or null; + hash = source.hash or null; } else if source.type == "fetchurl" then fetchurl { inherit (source) url; - sha256 = source.hash or null; + hash = source.hash or null; } else if source.type == "unknown" then "unknown" diff --git a/src/fetchers/default.nix b/src/fetchers/default.nix index 28982e72..27c8ab32 100644 --- a/src/fetchers/default.nix +++ b/src/fetchers/default.nix @@ -1,5 +1,8 @@ { callPackage, + + # config + allowBuiltinFetchers, ... }: rec { diff --git a/src/translators/default.nix b/src/translators/default.nix index 0aea3e68..e60df6d9 100644 --- a/src/translators/default.nix +++ b/src/translators/default.nix @@ -26,7 +26,12 @@ let translateBin = wrapPureTranslator [ subsystem type name ]; }; in - translatorWithBin // { inherit subsystem type name; }; + translatorWithBin // { + inherit subsystem type name; + translate = args: + translator.translate + ((getSpecialArgsDefaults translator.specialArgs or {}) // args); + }; buildSystems = dirNames ./.; @@ -45,7 +50,7 @@ let bin = pkgs.writeScriptBin "translate" '' #!${pkgs.bash}/bin/bash - jsonInputFile=$1 + jsonInputFile=$(realpath $1) outputFile=$(${pkgs.jq}/bin/jq '.outputFile' -c -r $jsonInputFile) export d2nExternalSources=${externalSources} @@ -165,19 +170,21 @@ let lib.head (lib.attrValues (lib.head (lib.attrValues (lib.head (lib.attrValues compatTranslators))))) ); + getSpecialArgsDefaults = specialArgsDef: + lib.mapAttrs + (name: def: + if def.type == "flag" then + false + else + def.default + ) + specialArgsDef; + selectTranslatorJSON = args: let translator = (selectTranslator args); data = { - SpecialArgsDefaults = - lib.mapAttrs - (name: def: - if def.type == "flag" then - false - else - def.default - ) - translator.specialArgs or {}; + SpecialArgsDefaults = getSpecialArgsDefaults (translator.specialArgs or {}); inherit (translator) subsystem type name; }; in diff --git a/src/translators/nodejs/pure/npmlock2nix/default.nix b/src/translators/nodejs/pure/npmlock2nix/default.nix index 0a5d51d6..51f04fb0 100644 --- a/src/translators/nodejs/pure/npmlock2nix/default.nix +++ b/src/translators/nodejs/pure/npmlock2nix/default.nix @@ -12,23 +12,116 @@ { inputDirectories, inputFiles, + + dev, ... }: let parsed = externals.npmlock2nix.readLockfile (builtins.elemAt inputFiles 0); + + parseGithubDepedency = dependency: + externals.npmlock2nix.parseGitHubRef dependency.version; + + getVersion = dependency: + if dependency ? from && dependency ? version then + builtins.substring 0 8 (parseGithubDepedency dependency).rev + else + dependency.version; + + pinVersions = dependencies: parentScopeDeps: + lib.mapAttrs + (pname: pdata: + let + selfScopeDeps = parentScopeDeps // (pdata.dependencies or {}); + in + pdata // { + depsExact = + if ! pdata ? requires then + [] + else + lib.forEach (lib.attrNames pdata.requires) (reqName: + "${reqName}#${getVersion selfScopeDeps."${reqName}"}" + ); + dependencies = pinVersions (pdata.dependencies or {}) selfScopeDeps; + } + ) + dependencies; + + packageLockWithPinnedVersions = pinVersions parsed.dependencies parsed.dependencies; + + # recursively collect dependencies + parseDependencies = dependencies: + lib.mapAttrsToList # returns list of lists + (pname: pdata: + if ! dev && pdata.dev or false then + [] + else + # handle github dependency + if pdata ? from && pdata ? version then + let + githubData = parseGithubDepedency pdata; + in + [ rec { + name = "${pname}#${version}"; + version = builtins.substring 0 8 githubData.rev; + owner = githubData.org; + repo = githubData.repo; + rev = githubData.rev; + type = "github"; + depsExact = pdata.depsExact; + }] + # handle http(s) dependency + else + [rec { + name = "${pname}#${version}"; + version = pdata.version; + url = pdata.resolved; + type = "fetchurl"; + hash = pdata.integrity; + depsExact = pdata.depsExact; + }] + ++ + (lib.optionals (pdata ? dependencies) + (lib.flatten (parseDependencies pdata.dependencies)) + ) + ) + dependencies; in - { - sources = builtins.mapAttrs (pname: pdata:{ - url = pdata.resolved; - type = "fetchurl"; - hash = pdata.integrity; - }) parsed.dependencies; + rec { + sources = + let + lockedSources = lib.listToAttrs ( + map + (dep: lib.nameValuePair + dep.name + ( + if dep.type == "github" then + { inherit (dep) type version owner repo rev; } + else + { inherit (dep) type version url hash; } + ) + ) + (lib.flatten (parseDependencies packageLockWithPinnedVersions)) + ); + in + # if only a package-lock.json is given, the main source is missing + lockedSources // { + "${parsed.name}" = { + type = "unknown"; + version = parsed.version; + }; + }; generic = { buildSystem = "nodejs"; producedBy = translatorName; - mainPackage = null; - dependencyGraph = null; + mainPackage = parsed.name; + dependencyGraph = + lib.listToAttrs + (map + (dep: lib.nameValuePair dep.name dep.depsExact ) + (lib.flatten (parseDependencies packageLockWithPinnedVersions)) + ); sourcesCombinedHash = null; }; @@ -42,10 +135,18 @@ inputDirectories, inputFiles, }@args: - builtins.trace (lib.attrValues args) { inputDirectories = []; inputFiles = lib.filter (f: builtins.match ".*(package-lock\\.json)" f != null) args.inputFiles; }; + + specialArgs = { + + dev = { + description = "include dependencies for development"; + type = "flag"; + }; + + }; }