From a1a3e57e0551a1eb484c4158472ebd398c5523c2 Mon Sep 17 00:00:00 2001 From: DavHau Date: Tue, 14 Sep 2021 03:00:29 +0100 Subject: [PATCH] add translator: npmlock2nix --- flake.lock | 19 ++++- flake.nix | 13 +++- src/apps/cli.py | 4 + src/apps/default.nix | 6 ++ src/default.nix | 13 +++- src/translators/default.nix | 77 ++++++++++++++----- .../nodejs/pure/npmlock2nix/default.nix | 39 ++++++++++ .../impure/pip/generate-generic-lock.py | 4 +- 8 files changed, 149 insertions(+), 26 deletions(-) create mode 100644 src/translators/nodejs/pure/npmlock2nix/default.nix diff --git a/flake.lock b/flake.lock index 307f6571..1217faa8 100644 --- a/flake.lock +++ b/flake.lock @@ -15,9 +15,26 @@ "type": "indirect" } }, + "npmlock2nix": { + "flake": false, + "locked": { + "lastModified": 1631558099, + "narHash": "sha256-xguvgtrIQHPxpc4J6EhfBnYtQDGJpt6hK1dN7KEi8R4=", + "owner": "nix-community", + "repo": "npmlock2nix", + "rev": "33eb3300561d724da64a46ab8e9a05a9cfa9264b", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "npmlock2nix", + "type": "github" + } + }, "root": { "inputs": { - "nixpkgs": "nixpkgs" + "nixpkgs": "nixpkgs", + "npmlock2nix": "npmlock2nix" } } }, diff --git a/flake.nix b/flake.nix index d14682c2..30d25775 100644 --- a/flake.nix +++ b/flake.nix @@ -3,12 +3,13 @@ inputs = { nixpkgs.url = "nixpkgs/nixos-unstable"; + npmlock2nix = { url = "github:nix-community/npmlock2nix"; flake = false; }; }; - outputs = { self, nixpkgs }: + outputs = { self, nixpkgs, npmlock2nix }: let - lib = nixpkgs.lib; + lib = nixpkgs.lib; supportedSystems = [ "x86_64-linux" ]; @@ -19,7 +20,13 @@ overlays = [ self.overlay ]; }); - dream2nixFor = forAllSystems (system: import ./src { pkgs = nixpkgsFor."${system}"; } ); + dream2nixFor = forAllSystems (system: import ./src rec { + pkgs = nixpkgsFor."${system}"; + externalSources = pkgs.runCommand "dream2nix-imported" {} '' + mkdir -p $out/npmlock2nix + cp ${npmlock2nix}/{internal.nix,LICENSE} $out/npmlock2nix/ + ''; + }); in { diff --git a/src/apps/cli.py b/src/apps/cli.py index 5193662b..30be3358 100644 --- a/src/apps/cli.py +++ b/src/apps/cli.py @@ -49,6 +49,10 @@ def translate(args): outputFile=output, ) + # remove output files if exists + if os.path.exists(output): + os.remove(output) + # dump translator arguments to json file and execute translator with tempfile.NamedTemporaryFile("w") as inputJsonFile: json.dump(translatorInput, inputJsonFile, indent=2) diff --git a/src/apps/default.nix b/src/apps/default.nix index c4cfe9e2..985ff952 100644 --- a/src/apps/default.nix +++ b/src/apps/default.nix @@ -1,5 +1,7 @@ { pkgs, + + externalSources, location, translators, }: @@ -11,6 +13,8 @@ in # the unified translator cli cli = callPackage ({ python3, writeScript, ... }: writeScript "cli" '' + export d2nExternalSources=${externalSources} + translatorsJsonFile=${translators.translatorsJsonFile} \ dream2nixSrc=${../.} \ ${python3}/bin/python ${./cli.py} "$@" @@ -35,6 +39,8 @@ in fi cp -r ${location}/* $target/ + mkdir $target/external + cp -r ${externalSources}/* $target/external/ chmod -R +w $target '' ) {}; diff --git a/src/default.nix b/src/default.nix index 62663875..c3a78e50 100644 --- a/src/default.nix +++ b/src/default.nix @@ -1,20 +1,29 @@ { pkgs ? import {}, + externalSources ? + if builtins.getEnv "d2nExternalSources" != "" then + builtins.getEnv "d2nExternalSources" + else + ./external, }: let callPackage = pkgs.callPackage; + + externals = { + npmlock2nix = callPackage "${externalSources}/npmlock2nix/internal.nix" {}; + }; in rec { - apps = callPackage ./apps { inherit location translators; }; + apps = callPackage ./apps { inherit externalSources location translators; }; builders = callPackage ./builders {}; fetchers = callPackage ./fetchers {}; - translators = callPackage ./translators {}; + translators = callPackage ./translators { inherit externalSources externals location; }; # the location of the dream2nix framework for self references (update scripts, etc.) diff --git a/src/translators/default.nix b/src/translators/default.nix index c8d24f8d..033e8099 100644 --- a/src/translators/default.nix +++ b/src/translators/default.nix @@ -1,38 +1,79 @@ -{ pkgs }: +{ + pkgs, + + externalSources, + externals, + location, +}: let lib = pkgs.lib; callPackage = pkgs.callPackage; + callTranslator = file: name: args: pkgs.callPackage file (args // { + inherit externals; + translatorName = name; + }); # every translator must provide 'bin/translate' translatorExec = translatorPkg: "${translatorPkg}/bin/translate"; + # directory names of a given directory dirNames = dir: lib.attrNames (lib.filterAttrs (name: type: type == "directory") (builtins.readDir dir)); - translators = + # wrapPureTranslator + wrapPureTranslator = translatorAttrPath: pkgs.writeScriptBin "translate" '' + #!${pkgs.bash}/bin/bash + + echo wrapPureTranslator + + jsonInputFile=$1 + outputFile=$(${pkgs.jq}/bin/jq '.outputFile' -c -r $jsonInputFile) + export d2nExternalSources=${externalSources} + + nix eval --impure --raw --expr " + builtins.toJSON ( + (import ${location} {}).translators.translatorsInternal.${ + lib.concatStringsSep "." translatorAttrPath + }.translate + (builtins.fromJSON (builtins.readFile '''$1''')) + ) + " | ${pkgs.jq}/bin/jq > $outputFile + ''; + + mkTranslatorsSet = function: lib.genAttrs (dirNames ./.) (subsystem: lib.genAttrs - (lib.filter (dir: builtins.pathExists (./. + "/${subsystem}/${dir}")) [ "impure" "ifd" "pure-nix" ]) - (transType: - lib.genAttrs (dirNames (./. + "/${subsystem}/${transType}")) (translatorName: - callPackage (./. + "/${subsystem}/${transType}/${translatorName}") {} - ) - ) + (lib.filter (dir: builtins.pathExists (./. + "/${subsystem}/${dir}")) [ "impure" "ifd" "pure" ]) + (transType: function subsystem transType) ); - # dump the list of available translators to a json file so they can be listed in the CLI - translatorsJsonFile = pkgs.writeText "translators.json" (builtins.toJSON ( - lib.genAttrs (dirNames ./.) (subsystem: - lib.genAttrs - (lib.filter (dir: builtins.pathExists (./. + "/${subsystem}/${dir}")) [ "impure" "ifd" "pure-nix" ]) - (transType: - dirNames (./. + "/${subsystem}/${transType}") - ) + + translators = mkTranslatorsSet (subsystem: type: + lib.genAttrs (dirNames (./. + "/${subsystem}/${type}")) (translatorName: + if type == "impure" then + callTranslator (./. + "/${subsystem}/${type}/${translatorName}") translatorName {} + else + wrapPureTranslator [ subsystem type translatorName ] ) - )); + ); + + translatorsInternal = mkTranslatorsSet (subsystem: type: + lib.genAttrs (dirNames (./. + "/${subsystem}/${type}")) (translatorName: + callTranslator (./. + "/${subsystem}/${type}/${translatorName}") translatorName {} + ) + ); + + translatorsJsonFile = + pkgs.writeText + "translators.json" + (builtins.toJSON + (mkTranslatorsSet (subsystem: type: + dirNames (./. + "/${subsystem}/${type}") + ) + )); in { - inherit translators translatorsJsonFile; + inherit translators translatorsInternal translatorsJsonFile; } diff --git a/src/translators/nodejs/pure/npmlock2nix/default.nix b/src/translators/nodejs/pure/npmlock2nix/default.nix new file mode 100644 index 00000000..5c3af813 --- /dev/null +++ b/src/translators/nodejs/pure/npmlock2nix/default.nix @@ -0,0 +1,39 @@ +{ + externals, + translatorName, +}: + +let + translate = + { + inputFiles, + ... + }: + let + parsed = externals.npmlock2nix.readLockfile (builtins.elemAt inputFiles 0); + in + { + sources = builtins.mapAttrs (pname: pdata:{ + url = pdata.resolved; + type = "fetchurl"; + hash = pdata.integrity; + }) parsed.dependencies; + + generic = { + buildSystem = "nodejs"; + buildSystemFormatVersion = 1; + producedBy = translatorName; + dependencyGraph = null; + sourcesCombinedHash = null; + }; + + buildSystem = { + nodejsVersion = 14; + }; + }; + +in + +{ + inherit translate; +} diff --git a/src/translators/python/impure/pip/generate-generic-lock.py b/src/translators/python/impure/pip/generate-generic-lock.py index b6159593..9e79541d 100644 --- a/src/translators/python/impure/pip/generate-generic-lock.py +++ b/src/translators/python/impure/pip/generate-generic-lock.py @@ -6,7 +6,7 @@ import sys def main(): - direcotry = sys.argv[1] + directory = sys.argv[1] output_file = sys.argv[2] packages = {} @@ -15,7 +15,7 @@ def main(): # - url # - sha256 # - format (sdist/wheel) - for path in list(glob(direcotry + '/*')): + for path in list(glob(directory + '/*')): _, _, file = path.rpartition('/') print(f"processing file: {file}")