improve handling of translator modules

README.md

@@ -10,6 +10,7 @@ It focuses on the following aspects:
   - Risk free opt-in FOD fetching
   - Common UI across 2nix tools
   - Reduce effort to develop new 2nix solutions
+  - Exploration and adoption of new nix features
 
 ### Motivation
 2nix tools, or in other words, tools converting instructions of other build systems to nix build instructions, are an important part of the nix/nixos ecosystem. These tools make packaging workflows easier and often allow to manage complexity that would be hard or impossible to manage without.
@@ -64,7 +65,7 @@ Essential components like package update scripts or fetching and override logic
 Optionally, to save more storag space, individual hashes for source can be ommited and a single large FOD used instead.
 Due to a unified minimalistic fetching layer the risk of FOD hash breakages should be very low.
 
-### Common UI across many 2nix tools
+### Common UI across many 2nix solutions
 2nix solutions which follow the dream2nix framework will have a unified UI for workflows like project initialization or code generation. This will allow quicker onboarding of new users by providing familiar workflows across different build systems.
 
 ### Reduced effort to develop new 2nix solutions

flake.nix

@@ -25,10 +25,12 @@
       {
         overlay = curr: prev: {};
 
+        defaultApp = forAllSystems (system: self.apps."${system}".cli);
+
         apps = forAllSystems (system: {
-          translate = {
+          cli = {
             "type" = "app";
-            "program" = builtins.toString (dream2nixFor."${system}".apps.translate);
+            "program" = builtins.toString (dream2nixFor."${system}".apps.cli);
           };
           install = {
             "type" = "app";

src/apps/cli.py

@@ -11,82 +11,37 @@ with open (os.environ.get("translatorsJsonFile")) as f:
   translators = json.load(f)
 
 
-# TODO: detection translator automatically according to files
-def auto_detect_translator(files, subsystem):
-  return list(translators[subsystem].keys())[0]
-
-
 def stripHashesFromLock(lock):
   for source in lock['sources'].values():
     del source['hash']
 
 
-def parse_args():
-
-  parser = argparse.ArgumentParser(
-    prog="translate",
-    description="translate projects to nix",
-  )
-
-  parser.add_argument(
-    "-s", "--subsystem",
-    help="which subsystem to use, (eg: python, nodejs, ...)",
-    choices=translators.keys()
-  )
-
-  parser.add_argument(
-    "-t", "--translator",
-    help="which specific translator to use",
-    default="auto"
-  )
-
-  parser.add_argument(
-    "-o", "--output",
-    help="output file/directory (generic lock)",
-    default="./dream.lock"
-  )
-
-  parser.add_argument(
-    "-c", "--combined",
-    help="Store only one hash for all sources combined (smaller lock file -> larger FOD)",
-    action="store_true"
-  )
-
-  parser.add_argument(
-    "input",
-    help="input files containing relevant metadata",
-    nargs="+"
-  )
-
-  args = parser.parse_args()
-
-  # TODO: detection subsystem automatically according to files
-  if not hasattr(args, "subsystem"):
-    print("Please specify subsystem (-s, --subsystem)", file=sys.stderr)
-    parser.print_help()
-    exit(1)
-
-  return args
+def list(args):
+  out = "Available translators per build system"
+  for subsystem, trans_types in translators.items():
+    displayed = []
+    for trans_type, translators_ in trans_types.items():
+      for translator in translators_:
+        displayed.append(f"{trans_type}.{translator}")
+    nl = '\n'
+    out += f"\n  - {subsystem}.{f'{nl}  - {subsystem}.'.join(displayed)}"
+  print(out)
 
 
-def main():
 
-  args = parse_args()
+def translate(args):
+
+  dream2nix_src = os.environ.get("dream2nixSrc")
 
-  subsystem = args.subsystem
   files = args.input
-
-  # determine translator
-  if args.translator == "auto":
-    translator = auto_detect_translator(files, subsystem)
-  else:
-    translator = args.translator
+  translator = args.translator
 
   # determine output directory
   if os.path.isdir(args.output):
     output = f"{args.output}/dream.lock"
   else:
     output = args.output
+  output = os.path.realpath(output)
 
   # translator arguments
   translatorInput = dict(
@@ -95,11 +50,25 @@ def main():
   )
 
   # dump translator arguments to json file and execute translator
-  with tempfile.NamedTemporaryFile("w") as inputJson:
-    json.dump(translatorInput, inputJson, indent=2)
-    inputJson.seek(0) # flushes write cache
+  with tempfile.NamedTemporaryFile("w") as inputJsonFile:
+    json.dump(translatorInput, inputJsonFile, indent=2)
+    inputJsonFile.seek(0) # flushes write cache
+    procBuild = sp.run(
+      [
+        "nix", "build", "--impure", "--expr",
+        f"(import {dream2nix_src} {{}}).translators.translators.{translator}", "-o", "translator"
+      ],
+      capture_output=True,
+    )
+    if procBuild.returncode:
+      print("Building translator failed", file=sys.stdout)
+      print(procBuild.stderr.decode(), file=sys.stderr)
+      exit(1)
+
+    translatorPath = os.path.realpath("translator")
+    os.remove("translator")
     sp.run(
-      [f"{translators[subsystem][translator]}/bin/translate", inputJson.name] + sys.argv[1:]
+      [f"{translatorPath}/bin/translate", inputJsonFile.name] + sys.argv[1:]
     )
 
   # raise error if output wasn't produced
@@ -113,7 +82,7 @@ def main():
   # calculate combined hash
   if args.combined:
 
-    print("Start building combined sourced FOD to get output hash")
+    print("Start building FOD for combined sources to get output hash")
 
     # remove hashes from lock file and init sourcesCombinedHash with emtpy string
     stripHashesFromLock(lock)
@@ -122,7 +91,6 @@ def main():
       json.dump(lock, f, indent=2)
 
     # compute FOD hash of combined sources
-    dream2nix_src = os.environ.get("dream2nixSrc")
     proc = sp.run(
       [
         "nix", "build", "--impure", "-L", "--expr",
@@ -149,5 +117,71 @@ def main():
   print(f"Created {output}")
 
 
+def parse_args():
+
+  parser = argparse.ArgumentParser(
+    prog="nix run dream2nix --",
+    # description="translate projects to nix",
+  )
+
+  sub = parser.add_subparsers(
+    title='actions',
+    description='valid actions',
+    help='which action to execute'
+  )
+
+  list_parser = sub.add_parser(
+    "list",
+    description="list available translators"
+  )
+
+  list_parser.set_defaults(func=list)
+
+  translate_parser = sub.add_parser(
+    "translate",
+    prog="translate",
+    description="translate projects to nix",
+  )
+
+  translate_parser.set_defaults(func=translate)
+
+  translate_parser.add_argument(
+    "-t", "--translator",
+    help="select translator (list via: 'dream2nix list')",
+    default="auto"
+  )
+
+  translate_parser.add_argument(
+    "-o", "--output",
+    help="output file/directory for the generic lock",
+    default="./dream.lock"
+  )
+
+  translate_parser.add_argument(
+    "-c", "--combined",
+    help="Store only one hash for all sources combined (smaller lock file -> larger FOD)",
+    action="store_true"
+  )
+
+  translate_parser.add_argument(
+    "input",
+    help="input files containing relevant metadata",
+    nargs="+"
+  )
+
+  args = parser.parse_args()
+
+  if not hasattr(args, "func"):
+    parser.print_help()
+    exit(1)
+
+  args.func(args)
+
+
+def main():
+
+  args = parse_args()
+
+
 if __name__ == "__main__":
   main()

src/apps/default.nix

@@ -9,11 +9,11 @@ in
 {
 
   # the unified translator cli
-  translate = callPackage ({ python3, writeScript, ... }:
+  cli = callPackage ({ python3, writeScript, ... }:
     writeScript "cli" ''
       translatorsJsonFile=${translators.translatorsJsonFile} \
       dream2nixSrc=${../.} \
-        ${python3}/bin/python ${./translators-cli.py} "$@"
+        ${python3}/bin/python ${./cli.py} "$@"
     ''
   ) {};
 

src/translators/default.nix

@@ -1,52 +1,36 @@
 { pkgs }: 
 let
 
+  lib = pkgs.lib;
+
   callPackage = pkgs.callPackage;
 
   # every translator must provide 'bin/translate'
   translatorExec = translatorPkg: "${translatorPkg}/bin/translate";
 
-  # the list of all available translators
-  translators = {
+  dirNames = dir: lib.attrNames (lib.filterAttrs (name: type: type == "directory") (builtins.readDir dir));
 
-    python = {
+  translators =
+    lib.genAttrs (dirNames ./.) (subsystem:
+      lib.genAttrs
+        (lib.filter (dir: builtins.pathExists (./. + "/${subsystem}/${dir}")) [ "impure" "ifd" "pure-nix" ])
+        (transType:
+          lib.genAttrs (dirNames (./. + "/${subsystem}/${transType}")) (translatorName:
+            callPackage (./. + "/${subsystem}/${transType}/${translatorName}") {}
+        )
+      )
+    );
 
-      # minimal POC python translator using pip. Type: 'external'
-      external-pip-python36 = callPackage ./python/external-pip { python = pkgs.python36; };
-      external-pip-python37 = callPackage ./python/external-pip { python = pkgs.python37; };
-      external-pip-python38 = callPackage ./python/external-pip { python = pkgs.python38; };
-      external-pip-python39 = callPackage ./python/external-pip { python = pkgs.python39; };
-      external-pip-python310 = callPackage ./python/external-pip { python = pkgs.python310; };
-
-      # TODO: add more translators
-
-    };
-  };
-
-  # Put all translator executables in a json file.
-  # This will allow the cli to call the translators of different build systems
-  # in a standardised way
-  # TODO: This doesn't scale as it requires all translators being built.
-  #       Redesign this, to call the individual translators using nix run ... 
-  translatorsJsonFile = callPackage ({ bash, lib, runCommand, ... }:
-    runCommand
-      "translators.json"
-      {
-        buildInputs = lib.flatten 
-          (
-            lib.mapAttrsToList
-              (subsystem: translators:
-                lib.attrValues translators
-              )
-              translators
-          );
-      }
-      # 'unsafeDiscardStringContext' is safe in thix context because all store paths are declared as buildInputs
-      ''
-        #!${bash}/bin/bash
-        cp ${builtins.toFile "translators.json" (builtins.unsafeDiscardStringContext (builtins.toJSON translators))} $out
-      ''
-  ) {};
+  # dump the list of available translators to a json file so they can be listed in the CLI
+  translatorsJsonFile = pkgs.writeText "translators.json" (builtins.toJSON (
+    lib.genAttrs (dirNames ./.) (subsystem:
+      lib.genAttrs 
+        (lib.filter (dir: builtins.pathExists (./. + "/${subsystem}/${dir}")) [ "impure" "ifd" "pure-nix" ])
+        (transType:
+          dirNames (./. + "/${subsystem}/${transType}")
+        )
+    )
+  ));
 
 in
 {

src/translators/python/impure/pip/default.nix

@@ -1,7 +1,7 @@
 {
   bash,
   jq,
-  python,
+  python3,
   writeScriptBin,
   ...
 }:
@@ -22,7 +22,7 @@ writeScriptBin "translate" ''
   outputFile=$(${jq}/bin/jq '.outputFile' -c -r $jsonInput)
 
   # pip executable
-  pip=${python.pkgs.pip}/bin/pip
+  pip=${python3.pkgs.pip}/bin/pip
 
   # prepare temporary directory
   tmp=translateTmp
@@ -37,7 +37,7 @@ writeScriptBin "translate" ''
     -r ''${inputFiles/$'\n'/$' -r '}
 
   # generate the generic lock from the downloaded list of files
-  ${python}/bin/python ${./generate-generic-lock.py} $tmp $outputFile
+  ${python3}/bin/python ${./generate-generic-lock.py} $tmp $outputFile
 
   rm -rf $tmp
 ''