CLI improvements

- rename command package -> add
  - improve handling of dream2nix config
  - improve purity of cli execution
  - automatically find git repo root
  - improve package update cli

src/apps/cli/cli.py

@@ -1,10 +1,10 @@
 from cleo import Application
 
-from commands.package import PackageCommand
+from commands.add import AddCommand
 from commands.update import UpdateCommand
 
-application = Application("package")
-application.add(PackageCommand())
+application = Application("dream2nix")
+application.add(AddCommand())
 application.add(UpdateCommand())
 
 if __name__ == '__main__':

src/apps/cli/commands/add.py

@@ -6,30 +6,35 @@ import sys
 import tempfile
 
 import networkx as nx
-from cleo import Command, option
+from cleo import Command, argument, option
 
-from utils import dream2nix_src, checkLockJSON, callNixFunction, buildNixFunction, buildNixAttribute, \
+from utils import config, dream2nix_src, checkLockJSON, callNixFunction, buildNixFunction, buildNixAttribute, \
   list_translators_for_source, strip_hashes_from_lock
 
 
-class PackageCommand(Command):
+class AddCommand(Command):
 
   description = (
-    "Package a software project using nix"
+    f"Add a package to {config['repoName']}"
   )
 
-  name = "package"
+  name = "add"
+
+  arguments = [
+    argument(
+      "source",
+      "source of the package, can be a path, tarball URL, or flake-style spec")
+  ]
 
   options = [
-    option(
-      "source",
-      None,
-      "source of the package, can be a path or flake-like spec",
-      flag=False,
-      multiple=False
-    ),
     option("translator", None, "which translator to use", flag=False),
-    option("output", None, "output file/directory for the dream-lock.json", flag=False),
+    option("target", None, "target file/directory for the dream-lock.json", flag=False),
+    option(
+      "--packages-root",
+      None,
+      "Put package under a new directory inside packages-root",
+      flag=False
+    ),
     option(
       "combined",
       None,
@@ -60,47 +65,32 @@ class PackageCommand(Command):
       )
     }
 
-    # ensure output directory
-    output = self.option("output")
-    if not output:
-      output = './.'
-    if not os.path.isdir(output):
-      os.mkdir(output)
-    filesToCreate = ['dream-lock.json']
-
-    existingFiles = set(os.listdir(output))
-    if not self.option('no-default-nix') \
-        and not 'default.nix' in existingFiles:
-      if self.confirm(
-          'Create a default.nix for debugging purposes',
-          default=True):
-        filesToCreate.append('default.nix')
-
-    if self.option('force'):
-      for f in filesToCreate:
-        if os.path.isfile(f):
-          os.remove(f)
+    # ensure packages-root
+    if self.option("packages-root"):
+      packages_root = self.option("packages-root")
+    elif config['packagesDir']:
+      packages_root = config['packagesDir']
     else:
-      
-      if any(f in existingFiles for f in filesToCreate):
-        print(
-          f"output directory {output} already contains a 'default.nix' "
-          "or 'dream-lock.json'. Delete first, or user '--force'.",
-          file=sys.stderr,
-        )
-        exit(1)
-    output = os.path.realpath(output)
-    outputDreamLock = f"{output}/dream-lock.json"
-    outputDefaultNix = f"{output}/default.nix"
+      packages_root = './.'
+    if not os.path.isdir(packages_root):
+      print(
+        f"Packages direcotry {packages_root} does not exist. Please create.",
+        file = sys.stderr,
+      )
 
     # verify source
-    source = self.option("source")
-    if not source:
+    source = self.argument("source")
+    if not source and not config['packagesDir']:
       source = os.path.realpath('./.')
       print(
         f"Source not specified. Defaulting to current directory: {source}",
         file=sys.stderr,
       )
+    # else:
+    #   print(
+    #     f"Source not specified. Defaulting to current directory: {source}",
+    #     file=sys.stderr,
+    #   )
     # check if source is valid fetcher spec
     sourceSpec = {}
     # handle source shortcuts
@@ -235,38 +225,102 @@ class PackageCommand(Command):
               if specified_extra_args[arg_name]:
                 break
 
-    # arguments for calling the translator nix module
-    translator_input = dict(
-      inputFiles=[],
-      inputDirectories=[source],
-      outputFile=outputDreamLock,
-    )
-    translator_input.update(specified_extra_args)
-
     # build the translator bin
     t = translator
     translator_path = buildNixAttribute(
       f"translators.translators.{t['subsystem']}.{t['type']}.{t['name']}.translateBin"
     )
 
-    # dump translator arguments to json file and execute translator
-    print("\nTranslating upstream metadata")
-    with tempfile.NamedTemporaryFile("w") as input_json_file:
-      json.dump(translator_input, input_json_file, indent=2)
-      input_json_file.seek(0) # flushes write cache
+    # direct outputs of translator to temporary file
+    with tempfile.NamedTemporaryFile("r") as output_temp_file:
 
-      # execute translator
-      sp.run(
-        [f"{translator_path}/bin/run", input_json_file.name]
+      # arguments for calling the translator nix module
+      translator_input = dict(
+        inputFiles=[],
+        inputDirectories=[source],
+        outputFile=output_temp_file.name,
       )
+      translator_input.update(specified_extra_args)
 
-    # raise error if output wasn't produced
-    if not os.path.isfile(outputDreamLock):
-      raise Exception(f"Translator failed to create dream-lock.json")
+      # dump translator arguments to json file and execute translator
+      print("\nTranslating project metadata")
+      with tempfile.NamedTemporaryFile("w") as input_json_file:
+        json.dump(translator_input, input_json_file, indent=2)
+        input_json_file.seek(0) # flushes write cache
 
-    # read produced lock file
-    with open(outputDreamLock) as f:
-      lock = json.load(f)
+        # execute translator
+        sp.run(
+          [f"{translator_path}/bin/run", input_json_file.name]
+        )
+
+      # raise error if output wasn't produced
+      if not output_temp_file.read():
+        raise Exception(f"Translator failed to create dream-lock.json")
+
+      # read produced lock file
+      with open(output_temp_file.name) as f:
+        lock = json.load(f)
+
+
+    # get package name and version from lock
+    mainPackageName = lock['_generic']['mainPackageName']
+    mainPackageVersion = lock['_generic']['mainPackageVersion']
+
+    # calculate output directory
+    mainPackageDirName = mainPackageName.strip('@').replace('/', '-')
+
+    # verify / change main package dir name
+    def update_name(mainPackageDirName):
+      print(f"Current package attribute name is: {mainPackageDirName}")
+      new_name = self.ask(
+        "Specify new attribute name or leave empty to keep current:"
+      )
+      if new_name:
+        return new_name
+      return mainPackageDirName
+
+    mainPackageDirName = update_name(mainPackageDirName)
+
+    if self.option('target'):
+      if self.option('target').startswith('/'):
+        output = self.option('target')
+      else:
+        output = f"{packages_root}/{self.option('target')}"
+    else:
+      output = f"{packages_root}/{mainPackageDirName}"
+
+    # collect files to create
+    filesToCreate = ['dream-lock.json']
+    if not os.path.isdir(output):
+      os.mkdir(output)
+    existingFiles = set(os.listdir(output))
+    if not self.option('no-default-nix')\
+        and not 'default.nix' in existingFiles\
+        and not config['packagesDir']:
+      if self.confirm(
+          'Create a default.nix for debugging purposes',
+          default=True):
+        filesToCreate.append('default.nix')
+
+    # overwrite existing files only if --force is set
+    if self.option('force'):
+      for f in filesToCreate:
+        if os.path.isfile(f):
+          os.remove(f)
+    # raise error if any file exists already
+    else:
+      if any(f in existingFiles for f in filesToCreate):
+        print(
+          f"output directory {output} already contains a 'default.nix' "
+          "or 'dream-lock.json'. Resolve via one of these:\n"
+          "  - use --force to overwrite files\n"
+          "  - use --target to specify another target dir",
+          file=sys.stderr,
+        )
+        exit(1)
+    output = os.path.realpath(output)
+    outputDreamLock = f"{output}/dream-lock.json"
+    outputDefaultNix = f"{output}/default.nix"
 
     # write translator information to lock file
     combined = self.option('combined')
@@ -281,8 +335,6 @@ class PackageCommand(Command):
     ])
 
     # add main package source
-    mainPackageName = lock['_generic']['mainPackageName']
-    mainPackageVersion = lock['_generic']['mainPackageVersion']
     mainSource = sourceSpec.copy()
     if not mainSource:
       mainSource = dict(

src/apps/cli/commands/update.py

@@ -4,31 +4,42 @@ import subprocess as sp
 import sys
 import tempfile
 
-from cleo import Command, option
+from cleo import Command, argument, option
 
-from utils import buildNixFunction, callNixFunction
+from utils import config, buildNixFunction, callNixFunction
 
 
 class UpdateCommand(Command):
   description = (
-    "Update an existing dream2nix based package"
+    f"Update an existing package in {config['repoName']}"
   )
 
   name = "update"
 
+  arguments = [
+    argument(
+      "name",
+      "name of the package or path containing a dream-lock.json",
+    ),
+  ]
+
   options = [
-    option("dream-lock", None, "dream-lock.json file or its parent directory", flag=False, value_required=True),
     option("updater", None, "name of the updater module to use", flag=False),
-    option("new-version", None, "new package version", flag=False),
+    option("to-version", None, "target package version", flag=False),
   ]
 
   def handle(self):
     if self.io.is_interactive():
       self.line(f"\n{self.description}\n")
 
-    dreamLockFile = os.path.abspath(self.option("dream-lock"))
-    if not dreamLockFile.endswith('dream-lock.json'):
-      dreamLockFile = os.path.abspath(dreamLockFile + "/dream-lock.json")
+    if config['packagesDir'] and '/' not in self.argument("name"):
+      dreamLockFile =\
+        os.path.abspath(
+          f"{config['packagesDir']}/{self.argument('name')}/dream-lock.json")
+    else:
+      dreamLockFile = os.path.abspath(self.argument("name"))
+      if not dreamLockFile.endswith('dream-lock.json'):
+        dreamLockFile = os.path.abspath(dreamLockFile + "/dream-lock.json")
 
     # parse dream lock
     with open(dreamLockFile) as f:
@@ -44,9 +55,11 @@ class UpdateCommand(Command):
           file=sys.stderr,
         )
         exit(1)
+    print(f"updater module is: {updater}")
 
     # find new version
-    version = self.option('new-version')
+    old_version = lock['_generic']['mainPackageVersion']
+    version = self.option('to-version')
     if not version:
       update_script = buildNixFunction(
         "updaters.makeUpdateScript",
@@ -55,7 +68,7 @@ class UpdateCommand(Command):
       )
       update_proc = sp.run([f"{update_script}/bin/run"], capture_output=True)
       version = update_proc.stdout.decode().strip()
-    print(f"Updating to version {version}")
+    print(f"Updating from version {old_version} to {version}")
 
     cli_py = os.path.abspath(f"{__file__}/../../cli.py")
     # delete the hash
@@ -73,8 +86,8 @@ class UpdateCommand(Command):
       tmpDreamLock.seek(0)  # flushes write cache
       sp.run(
         [
-          sys.executable, f"{cli_py}", "package", "--force", "--source", tmpDreamLock.name,
-          "--output", os.path.abspath(os.path.dirname(dreamLockFile))
+          sys.executable, f"{cli_py}", "add", tmpDreamLock.name, "--force",
+          "--target", os.path.abspath(os.path.dirname(dreamLockFile))
         ]
         + lock['_generic']['translatorParams'].split()
       )

src/apps/cli/default.nix

@@ -1,13 +1,16 @@
 {
   # from dream2nix
+  configFile,
   dream2nixWithExternals,
   fetchers,
+  nix,
   translators,
+  utils,
 
   # from nixpkgs
+  gitMinimal,
   lib,
   python3,
-  writeScript,
   ...
 }:
 
@@ -19,11 +22,25 @@ let
 
 in
 {
-  program = writeScript "cli" ''
-    dream2nixSrc=${dream2nixWithExternals} \
-    fetcherNames="${b.toString (lib.attrNames fetchers.fetchers)}" \
-      ${cliPython}/bin/python ${./.}/cli.py "$@"
-  '';
+  program = 
+    let
+      script = utils.writePureShellScript
+        [
+          gitMinimal
+          nix
+        ]
+        ''
+          # escape the temp dir created by writePureShellScript
+          cd - > /dev/null
+
+          # run the cli
+          dream2nixConfig=${configFile} \
+          dream2nixSrc=${dream2nixWithExternals} \
+          fetcherNames="${b.toString (lib.attrNames fetchers.fetchers)}" \
+            ${cliPython}/bin/python ${./.}/cli.py "$@"
+        '';
+    in
+      "${script}/bin/run";
 
   templateDefaultNix =
     {

src/apps/cli/utils.py

@@ -8,6 +8,25 @@ from jsonschema import validate
 
 dream2nix_src = os.environ.get("dream2nixSrc")
 
+def find_repo_root():
+  proc = sp.run(
+    ['git', 'rev-parse', '--show-toplevel'],
+    capture_output=True,
+  )
+  if proc.returncode:
+    print(proc.stderr.decode(), file=sys.stderr)
+    print(
+      f"\nProbably not inside git repo {config['repoName']}\n"
+      f"Please clone the repo first.",
+      file=sys.stderr
+    )
+    exit(1)
+  return proc.stdout.decode().strip()
+
+with open(os.environ.get("dream2nixConfig")) as f:
+  config = json.load(f)
+  if config['repoName'] and config ['packagesDir']:
+    config['packagesDir'] = f"{find_repo_root()}/{config['packagesDir']}"
 
 def checkLockJSON(lock):
   lock_schema_raw=open(dream2nix_src+"/specifications/dream-lock-schema.json").read()

src/apps/default.nix

@@ -2,6 +2,7 @@
   lib,
   pkgs,
 
+  # dream2nix
   callPackageDream,
   translators,
   ...
@@ -25,12 +26,12 @@ rec {
       }
     ) apps;
 
-  # the unified translator cli
+  # the dream2nix cli
   cli = callPackageDream (import ./cli) {};
 
   # the contribute cli
   contribute = callPackageDream (import ./contribute) {};
 
-  # install the framework to a specified location by copying the code
+  # instrall the framework to a specified location by copying the code
   install = callPackageDream (import ./install) {};
 }

src/default.nix

@@ -9,6 +9,7 @@
 
   # the dream2nix cli depends on some nix 2.4 features
   nix ? pkgs.writeScriptBin "nix" ''
+    #!${pkgs.bash}/bin/bash
     ${pkgs.nixUnstable}/bin/nix --option experimental-features "nix-command flakes" "$@"
   '',
 
@@ -52,11 +53,14 @@ let
 
   config = (import ./utils/config.nix).loadConfig args.config or {};
 
+  configFile = pkgs.writeText "dream2nix-config.json" (b.toJSON config);
+
   # like pkgs.callPackage, but includes all the dream2nix modules
   callPackageDream = f: args: pkgs.callPackage f (args // {
     inherit builders;
     inherit callPackageDream;
     inherit config;
+    inherit configFile;
     inherit externals;
     inherit externalSources;
     inherit fetchers;

src/lib.nix

@@ -6,9 +6,6 @@
   nixpkgsSrc ? <nixpkgs>,
   lib ? (import nixpkgsSrc {}).lib,
 
-  # default to empty dream2nix config
-  config ? {},
-
   externalSources,
 
   externalPaths,
@@ -19,9 +16,7 @@ let
 
   b = builtins;
 
-  config = (import ./utils/config.nix).loadConfig args.config or {};
-
-  dream2nixForSystem = system: pkgs:
+  dream2nixForSystem = config: system: pkgs:
     import ./default.nix
       { inherit config externalPaths externalSources pkgs; };
 
@@ -81,6 +76,8 @@ let
     }@argsInit:
     let
 
+      config = (import ./utils/config.nix).loadConfig argsInit.config or {};
+
       overridesDirs' = config.overridesDirs;
 
       allPkgs = makeNixpkgs pkgs systems;
@@ -88,7 +85,7 @@ let
       forAllSystems = f:
         lib.mapAttrs f allPkgs;
 
-      dream2nixFor = forAllSystems dream2nixForSystem;
+      dream2nixFor = forAllSystems (dream2nixForSystem config);
     in
       {
         riseAndShine = riseAndShineArgs:
@@ -135,7 +132,7 @@ let
       allPkgs = makeNixpkgs pkgs systems;
 
       dream2nixFor =
-        lib.mapAttrs dream2nixForSystem allPkgs;
+        lib.mapAttrs (dream2nixForSystem {}) allPkgs;
 
       allBuilderOutputs =
         lib.mapAttrs

src/utils/config.nix

@@ -23,9 +23,11 @@ let
       defaults =  {
         overridesDirs = [];
         packagesDir = null;
+        repoName = "this repo";
       };
     in
       defaults // config;
+
 in
 {
   inherit loadConfig;