diff --git a/flake.nix b/flake.nix index e46a0912..de78a495 100644 --- a/flake.nix +++ b/flake.nix @@ -48,6 +48,9 @@ }); devShell = forAllSystems (system: nixpkgsFor."${system}".mkShell { + buildInputs = with nixpkgsFor."${system}"; [ + nixUnstable + ]; shellHook = '' export NIX_PATH=nixpkgs=${nixpkgs} ''; diff --git a/src/apps/cli.py b/src/apps/cli.py index 2cb95ce0..1a30eba2 100644 --- a/src/apps/cli.py +++ b/src/apps/cli.py @@ -21,10 +21,23 @@ def list_translators(args): for subsystem, trans_types in translators.items(): displayed = [] for trans_type, translators_ in trans_types.items(): - for translator in translators_: - displayed.append(f"{trans_type}.{translator}") + for trans_name, translator in translators_.items(): + lines = tuple( + f"{trans_type}.{trans_name}", + ) + if translator: + lines += ( + f"\n special args:", + ) + for argName, argData in translator.items(): + lines += ( + f"\n --arg_{argName} {{value}}", + f"\n default: {argData['default']}", + f"\n examples: {', '.join(argData['examples'])}", + ) + displayed.append(''.join(lines)) nl = '\n' - out += f"\n - {subsystem}.{f'{nl} - {subsystem}.'.join(displayed)}" + out += f"\n\n - {subsystem}.{f'{nl} - {subsystem}.'.join(displayed)}" print(out) @@ -34,6 +47,12 @@ def translate(args): inputPaths = args.input + # collect special args + specialArgs = {} + for argName, argVal in vars(args).items(): + if argName.startswith("arg_"): + specialArgs[argName[4:]] = argVal + # check if all inputs exist for path in inputPaths: if not os.path.exists(path): @@ -56,6 +75,7 @@ def translate(args): outputFile=output, selector=args.translator or "", ) + translatorInput.update(specialArgs) # remove output file if exists if os.path.exists(output): @@ -69,19 +89,45 @@ def translate(args): env.update(dict( FUNC_ARGS=inputJsonFile.name )) - procBuild = sp.run( + procEval = sp.run( [ - "nix", "build", "--impure", "--expr", - f"(import {dream2nix_src} {{}}).translators.selectTranslatorBin {{}}", "-o", "translator" + "nix", "eval", "--impure", "--raw", "--expr", + f"((import {dream2nix_src} {{}}).translators.selectTranslatorJSON {{}})", ], capture_output=True, env=env ) + if procEval.returncode: + print("Selecting translator failed", file=sys.stdout) + print(procEval.stderr.decode(), file=sys.stderr) + exit(1) + + # parse data for auto selected translator + resultEval = json.loads(procEval.stdout) + subsystem = resultEval['subsystem'] + trans_type = resultEval['type'] + trans_name = resultEval['name'] + + # include default values into input data + translatorInputWithDefaults = resultEval['SpecialArgsDefaults'] + translatorInputWithDefaults.update(translatorInput) + json.dump(translatorInputWithDefaults, inputJsonFile, indent=2) + inputJsonFile.seek(0) + + # build the translator bin + procBuild = sp.run( + [ + "nix", "build", "--impure", "-o", "translator", "--expr", + f"(import {dream2nix_src} {{}}).translators.translators.{subsystem}.{trans_type}.{trans_name}.translateBin", + ], + capture_output=True, + ) if procBuild.returncode: print("Building translator failed", file=sys.stdout) print(procBuild.stderr.decode(), file=sys.stderr) exit(1) + # execute translator translatorPath = os.path.realpath("translator") os.remove("translator") sp.run( @@ -154,6 +200,9 @@ def parse_args(): list_parser.set_defaults(func=list_translators) + + # PARSER FOR TRNASLATOR + translate_parser = sub.add_parser( "translate", prog="translate", @@ -186,6 +235,13 @@ def parse_args(): nargs="+" ) + # parse special args + # (custom parameters required by individual translators) + parsed, unknown = translate_parser.parse_known_args() + for arg in unknown: + if arg.startswith("--arg_"): + translate_parser.add_argument(arg.split('=')[0]) + args = parser.parse_args() if not hasattr(args, "func"): diff --git a/src/fetchers/default.nix b/src/fetchers/default.nix index 8a0d3434..28982e72 100644 --- a/src/fetchers/default.nix +++ b/src/fetchers/default.nix @@ -1,5 +1,6 @@ { callPackage, + ... }: rec { defaultFetcher = callPackage ./default-fetcher.nix {}; diff --git a/src/translators/default.nix b/src/translators/default.nix index 3bbe3651..6ca8df16 100644 --- a/src/translators/default.nix +++ b/src/translators/default.nix @@ -11,19 +11,23 @@ let lib = pkgs.lib; + callTranslator = subsystem: type: name: file: args: let translator = callPackage file (args // { inherit externals; translatorName = name; }); + translatorWithBin = + # if the translator is a pure nix translator, + # generate a translatorBin for CLI compatibility + if translator ? translateBin then translator + else translator // { + translateBin = wrapPureTranslator [ subsystem type name ]; + }; in - # if the translator is a pure nix translator, - # generate a translatorBin for CLI compatibility - if translator ? translateBin then translator - else translator // { - translateBin = wrapPureTranslator [ subsystem type name ]; - }; + translatorWithBin // { inherit subsystem type name; }; + buildSystems = dirNames ./.; @@ -74,15 +78,17 @@ let ) ); - # json file exposing all existing translators to CLI + # json file exposing all existing translators to CLI including their special args translatorsJsonFile = - pkgs.writeText - "translators.json" - (builtins.toJSON - (mkTranslatorsSet (subsystem: type: - dirNames (./. + "/${subsystem}/${type}") + let + data = lib.mapAttrsRecursiveCond + (as: !(as ? "translateBin")) + (k: v: + v.specialArgs or {} ) - )); + translators; + in + pkgs.writeText "translators.json" (builtins.toJSON data); # filter translators by compatibility for the given input paths compatibleTranslators = @@ -134,7 +140,7 @@ let # return the correct translator bin for the given input paths - selectTranslatorBin = utils.makeCallableViaEnv ( + selectTranslator = utils.makeCallableViaEnv ( { selector, # like 'python.impure' or 'python.impure.pip' inputDirectories, # input paths to translate @@ -156,10 +162,23 @@ let - ${builtins.concatStringsSep "\n - " (inputDirectories ++ inputFiles)} '' else - (lib.head (lib.attrValues (lib.head (lib.attrValues (lib.head (lib.attrValues compatTranslators)))))).translateBin + lib.head (lib.attrValues (lib.head (lib.attrValues (lib.head (lib.attrValues compatTranslators))))) ); + selectTranslatorJSON = args: + let + translator = (selectTranslator args); + data = { + SpecialArgsDefaults = + lib.mapAttrs + (name: def: def.default) + translator.specialArgs or {}; + inherit (translator) subsystem type name; + }; + in + builtins.toJSON data; + in { - inherit translators translatorsJsonFile selectTranslatorBin; + inherit translators translatorsJsonFile selectTranslatorJSON; } diff --git a/src/translators/python/impure/pip/default.nix b/src/translators/python/impure/pip/default.nix index ce0e2765..d6399d88 100644 --- a/src/translators/python/impure/pip/default.nix +++ b/src/translators/python/impure/pip/default.nix @@ -13,6 +13,7 @@ { # the input format is specified in /specifications/translator-call-example.json + # this script receives a json file including the input paths and specialArgs translateBin = writeScriptBin "translate" '' #!${bash}/bin/bash @@ -23,29 +24,31 @@ # read the json input outputFile=$(${jq}/bin/jq '.outputFile' -c -r $jsonInput) + pythonAttr=$(${jq}/bin/jq '.pythonAttr' -c -r $jsonInput) inputDirectories=$(${jq}/bin/jq '.inputDirectories | .[]' -c -r $jsonInput) inputFiles=$(${jq}/bin/jq '.inputFiles | .[]' -c -r $jsonInput) - - # pip executable - pip=${python3.pkgs.pip}/bin/pip + # build python and pip executables + tmpBuild=$(mktemp -d) + cd $tmpBuild + nix build --impure --expr "(import {}).$pythonAttr" -o python + nix build --impure --expr "(import {}).$pythonAttr.pkgs.pip" -o pip + cd - # prepare temporary directory - tmp=translateTmp - rm -rf $tmp - mkdir $tmp + tmp=$(mktemp -d) # download files according to requirements - $pip download \ + $tmpBuild/pip/bin/pip download \ --no-cache \ --dest $tmp \ --progress-bar off \ -r ''${inputFiles/$'\n'/$' -r '} # generate the generic lock from the downloaded list of files - ${python3}/bin/python ${./generate-generic-lock.py} $tmp $outputFile + $tmpBuild/python/bin/python ${./generate-generic-lock.py} $tmp $outputFile - rm -rf $tmp + rm -rf $tmp $tmpBuild ''; @@ -59,4 +62,19 @@ inputDirectories = []; inputFiles = lib.filter (f: builtins.match ".*(requirements).*\\.txt" f != null) args.inputFiles; }; + + # define special args and provide defaults + specialArgs = { + + # the python attribute + pythonAttr = { + default = "python3${lib.elemAt (lib.splitString "." python3.version) 1}"; + description = "python version to translate for"; + examples = [ + "python27" + "python39" + "python310" + ]; + }; + }; }