diff --git a/README.md b/README.md index adc5e747..99bcc5f3 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ It focuses on the following aspects: - Risk free opt-in FOD fetching - Common UI across 2nix tools - Reduce effort to develop new 2nix solutions + - Exploration and adoption of new nix features ### Motivation 2nix tools, or in other words, tools converting instructions of other build systems to nix build instructions, are an important part of the nix/nixos ecosystem. These tools make packaging workflows easier and often allow to manage complexity that would be hard or impossible to manage without. @@ -64,7 +65,7 @@ Essential components like package update scripts or fetching and override logic Optionally, to save more storag space, individual hashes for source can be ommited and a single large FOD used instead. Due to a unified minimalistic fetching layer the risk of FOD hash breakages should be very low. -### Common UI across many 2nix tools +### Common UI across many 2nix solutions 2nix solutions which follow the dream2nix framework will have a unified UI for workflows like project initialization or code generation. This will allow quicker onboarding of new users by providing familiar workflows across different build systems. ### Reduced effort to develop new 2nix solutions diff --git a/flake.nix b/flake.nix index 46d27ffc..d14682c2 100644 --- a/flake.nix +++ b/flake.nix @@ -25,10 +25,12 @@ { overlay = curr: prev: {}; + defaultApp = forAllSystems (system: self.apps."${system}".cli); + apps = forAllSystems (system: { - translate = { + cli = { "type" = "app"; - "program" = builtins.toString (dream2nixFor."${system}".apps.translate); + "program" = builtins.toString (dream2nixFor."${system}".apps.cli); }; install = { "type" = "app"; diff --git a/src/apps/translators-cli.py b/src/apps/cli.py similarity index 55% rename from src/apps/translators-cli.py rename to src/apps/cli.py index 72cb8ae5..5193662b 100644 --- a/src/apps/translators-cli.py +++ b/src/apps/cli.py @@ -11,82 +11,37 @@ with open (os.environ.get("translatorsJsonFile")) as f: translators = json.load(f) -# TODO: detection translator automatically according to files -def auto_detect_translator(files, subsystem): - return list(translators[subsystem].keys())[0] - - def stripHashesFromLock(lock): for source in lock['sources'].values(): del source['hash'] -def parse_args(): - - parser = argparse.ArgumentParser( - prog="translate", - description="translate projects to nix", - ) - - parser.add_argument( - "-s", "--subsystem", - help="which subsystem to use, (eg: python, nodejs, ...)", - choices=translators.keys() - ) - - parser.add_argument( - "-t", "--translator", - help="which specific translator to use", - default="auto" - ) - - parser.add_argument( - "-o", "--output", - help="output file/directory (generic lock)", - default="./dream.lock" - ) - - parser.add_argument( - "-c", "--combined", - help="Store only one hash for all sources combined (smaller lock file -> larger FOD)", - action="store_true" - ) - - parser.add_argument( - "input", - help="input files containing relevant metadata", - nargs="+" - ) - - args = parser.parse_args() - - # TODO: detection subsystem automatically according to files - if not hasattr(args, "subsystem"): - print("Please specify subsystem (-s, --subsystem)", file=sys.stderr) - parser.print_help() - exit(1) - - return args +def list(args): + out = "Available translators per build system" + for subsystem, trans_types in translators.items(): + displayed = [] + for trans_type, translators_ in trans_types.items(): + for translator in translators_: + displayed.append(f"{trans_type}.{translator}") + nl = '\n' + out += f"\n - {subsystem}.{f'{nl} - {subsystem}.'.join(displayed)}" + print(out) -def main(): - args = parse_args() +def translate(args): + + dream2nix_src = os.environ.get("dream2nixSrc") - subsystem = args.subsystem files = args.input - - # determine translator - if args.translator == "auto": - translator = auto_detect_translator(files, subsystem) - else: - translator = args.translator + translator = args.translator # determine output directory if os.path.isdir(args.output): output = f"{args.output}/dream.lock" else: output = args.output + output = os.path.realpath(output) # translator arguments translatorInput = dict( @@ -95,11 +50,25 @@ def main(): ) # dump translator arguments to json file and execute translator - with tempfile.NamedTemporaryFile("w") as inputJson: - json.dump(translatorInput, inputJson, indent=2) - inputJson.seek(0) # flushes write cache + with tempfile.NamedTemporaryFile("w") as inputJsonFile: + json.dump(translatorInput, inputJsonFile, indent=2) + inputJsonFile.seek(0) # flushes write cache + procBuild = sp.run( + [ + "nix", "build", "--impure", "--expr", + f"(import {dream2nix_src} {{}}).translators.translators.{translator}", "-o", "translator" + ], + capture_output=True, + ) + if procBuild.returncode: + print("Building translator failed", file=sys.stdout) + print(procBuild.stderr.decode(), file=sys.stderr) + exit(1) + + translatorPath = os.path.realpath("translator") + os.remove("translator") sp.run( - [f"{translators[subsystem][translator]}/bin/translate", inputJson.name] + sys.argv[1:] + [f"{translatorPath}/bin/translate", inputJsonFile.name] + sys.argv[1:] ) # raise error if output wasn't produced @@ -113,7 +82,7 @@ def main(): # calculate combined hash if args.combined: - print("Start building combined sourced FOD to get output hash") + print("Start building FOD for combined sources to get output hash") # remove hashes from lock file and init sourcesCombinedHash with emtpy string stripHashesFromLock(lock) @@ -122,7 +91,6 @@ def main(): json.dump(lock, f, indent=2) # compute FOD hash of combined sources - dream2nix_src = os.environ.get("dream2nixSrc") proc = sp.run( [ "nix", "build", "--impure", "-L", "--expr", @@ -149,5 +117,71 @@ def main(): print(f"Created {output}") +def parse_args(): + + parser = argparse.ArgumentParser( + prog="nix run dream2nix --", + # description="translate projects to nix", + ) + + sub = parser.add_subparsers( + title='actions', + description='valid actions', + help='which action to execute' + ) + + list_parser = sub.add_parser( + "list", + description="list available translators" + ) + + list_parser.set_defaults(func=list) + + translate_parser = sub.add_parser( + "translate", + prog="translate", + description="translate projects to nix", + ) + + translate_parser.set_defaults(func=translate) + + translate_parser.add_argument( + "-t", "--translator", + help="select translator (list via: 'dream2nix list')", + default="auto" + ) + + translate_parser.add_argument( + "-o", "--output", + help="output file/directory for the generic lock", + default="./dream.lock" + ) + + translate_parser.add_argument( + "-c", "--combined", + help="Store only one hash for all sources combined (smaller lock file -> larger FOD)", + action="store_true" + ) + + translate_parser.add_argument( + "input", + help="input files containing relevant metadata", + nargs="+" + ) + + args = parser.parse_args() + + if not hasattr(args, "func"): + parser.print_help() + exit(1) + + args.func(args) + + +def main(): + + args = parse_args() + + if __name__ == "__main__": main() diff --git a/src/apps/default.nix b/src/apps/default.nix index 2ad19a8a..c4cfe9e2 100644 --- a/src/apps/default.nix +++ b/src/apps/default.nix @@ -9,11 +9,11 @@ in { # the unified translator cli - translate = callPackage ({ python3, writeScript, ... }: + cli = callPackage ({ python3, writeScript, ... }: writeScript "cli" '' translatorsJsonFile=${translators.translatorsJsonFile} \ dream2nixSrc=${../.} \ - ${python3}/bin/python ${./translators-cli.py} "$@" + ${python3}/bin/python ${./cli.py} "$@" '' ) {}; diff --git a/src/translators/default.nix b/src/translators/default.nix index a18b19dd..c8d24f8d 100644 --- a/src/translators/default.nix +++ b/src/translators/default.nix @@ -1,52 +1,36 @@ { pkgs }: let + lib = pkgs.lib; + callPackage = pkgs.callPackage; # every translator must provide 'bin/translate' translatorExec = translatorPkg: "${translatorPkg}/bin/translate"; - # the list of all available translators - translators = { + dirNames = dir: lib.attrNames (lib.filterAttrs (name: type: type == "directory") (builtins.readDir dir)); - python = { + translators = + lib.genAttrs (dirNames ./.) (subsystem: + lib.genAttrs + (lib.filter (dir: builtins.pathExists (./. + "/${subsystem}/${dir}")) [ "impure" "ifd" "pure-nix" ]) + (transType: + lib.genAttrs (dirNames (./. + "/${subsystem}/${transType}")) (translatorName: + callPackage (./. + "/${subsystem}/${transType}/${translatorName}") {} + ) + ) + ); - # minimal POC python translator using pip. Type: 'external' - external-pip-python36 = callPackage ./python/external-pip { python = pkgs.python36; }; - external-pip-python37 = callPackage ./python/external-pip { python = pkgs.python37; }; - external-pip-python38 = callPackage ./python/external-pip { python = pkgs.python38; }; - external-pip-python39 = callPackage ./python/external-pip { python = pkgs.python39; }; - external-pip-python310 = callPackage ./python/external-pip { python = pkgs.python310; }; - - # TODO: add more translators - - }; - }; - - # Put all translator executables in a json file. - # This will allow the cli to call the translators of different build systems - # in a standardised way - # TODO: This doesn't scale as it requires all translators being built. - # Redesign this, to call the individual translators using nix run ... - translatorsJsonFile = callPackage ({ bash, lib, runCommand, ... }: - runCommand - "translators.json" - { - buildInputs = lib.flatten - ( - lib.mapAttrsToList - (subsystem: translators: - lib.attrValues translators - ) - translators - ); - } - # 'unsafeDiscardStringContext' is safe in thix context because all store paths are declared as buildInputs - '' - #!${bash}/bin/bash - cp ${builtins.toFile "translators.json" (builtins.unsafeDiscardStringContext (builtins.toJSON translators))} $out - '' - ) {}; + # dump the list of available translators to a json file so they can be listed in the CLI + translatorsJsonFile = pkgs.writeText "translators.json" (builtins.toJSON ( + lib.genAttrs (dirNames ./.) (subsystem: + lib.genAttrs + (lib.filter (dir: builtins.pathExists (./. + "/${subsystem}/${dir}")) [ "impure" "ifd" "pure-nix" ]) + (transType: + dirNames (./. + "/${subsystem}/${transType}") + ) + ) + )); in { diff --git a/src/translators/python/external-pip/default.nix b/src/translators/python/impure/pip/default.nix similarity index 87% rename from src/translators/python/external-pip/default.nix rename to src/translators/python/impure/pip/default.nix index 99e3f8b7..fae13c1c 100644 --- a/src/translators/python/external-pip/default.nix +++ b/src/translators/python/impure/pip/default.nix @@ -1,7 +1,7 @@ { bash, jq, - python, + python3, writeScriptBin, ... }: @@ -22,7 +22,7 @@ writeScriptBin "translate" '' outputFile=$(${jq}/bin/jq '.outputFile' -c -r $jsonInput) # pip executable - pip=${python.pkgs.pip}/bin/pip + pip=${python3.pkgs.pip}/bin/pip # prepare temporary directory tmp=translateTmp @@ -37,7 +37,7 @@ writeScriptBin "translate" '' -r ''${inputFiles/$'\n'/$' -r '} # generate the generic lock from the downloaded list of files - ${python}/bin/python ${./generate-generic-lock.py} $tmp $outputFile + ${python3}/bin/python ${./generate-generic-lock.py} $tmp $outputFile rm -rf $tmp '' diff --git a/src/translators/python/external-pip/generate-generic-lock.py b/src/translators/python/impure/pip/generate-generic-lock.py similarity index 100% rename from src/translators/python/external-pip/generate-generic-lock.py rename to src/translators/python/impure/pip/generate-generic-lock.py