From ec63d19cf516c397730e4f23826335b293b2e459 Mon Sep 17 00:00:00 2001
From: DavHau <hsngrmpf+github@gmail.com>
Date: Tue, 16 Nov 2021 17:02:22 +0700
Subject: [PATCH 1/8] add.py: refactor spaghetti into functions

---
 src/apps/cli/commands/add.py | 706 ++++++++++++++++++-----------------
 1 file changed, 372 insertions(+), 334 deletions(-)

diff --git a/src/apps/cli/commands/add.py b/src/apps/cli/commands/add.py
index a6b5b650..28cc152c 100644
--- a/src/apps/cli/commands/add.py
+++ b/src/apps/cli/commands/add.py
@@ -58,14 +58,89 @@ class AddCommand(Command):
       self.line(f"\n{self.description}\n")
 
     # parse extra args
+    specified_extra_args = self.parse_extra_args()
+
+    # ensure packages-root
+    packages_root = self.find_packages_root()
+
+    # get source path and spec
+    source, sourceSpec = self.parse_source()
+
+    # select translator
+    translator = self.select_translator(source)
+
+    # raise error if any specified extra arg is unknown
+    specified_extra_args = self.declare_extra_args(specified_extra_args, translator)
+
+    # do the translation and produce dream lock
+    lock = self.run_translate(source, specified_extra_args, translator)
+
+    # get package name and version from lock
+    mainPackageName = lock['_generic']['mainPackageName']
+    mainPackageVersion = lock['_generic']['mainPackageVersion']
+
+    # calculate output directory and attribute name
+    mainPackageDirName = self.define_attribute_name(mainPackageName)
+
+    # calculate output files
+    filesToCreate, output = self.calc_outputs(mainPackageDirName, packages_root)
+    outputDreamLock = f"{output}/dream-lock.json"
+    outputDefaultNix = f"{output}/default.nix"
+
+    # add translator information to lock
+    self.extend_with_translator_info(lock, specified_extra_args, translator)
+
+    # add main package source
+    self.add_main_source(lock, mainPackageName, mainPackageVersion, sourceSpec)
+
+    # clean up dependency graph
+    if 'dependencies' in lock['_generic']:
+      self.postprocess_dep_graph(lock)
+
+    # calculate combined hash if --combined was specified
+    if self.option('combined'):
+      self.aggregate_hashes(lock, outputDreamLock)
+
+    # validate dream lock format
+    checkLockJSON(lock)
+
+    # format dream lock
+    lockStr = self.format_lock_str(lock)
+
+    # save dream lock file
+    with open(outputDreamLock, 'w') as f:
+      f.write(lockStr)
+    print(f"Created {output}/dream-lock.json")
+
+    # create default.nix
+    if 'default.nix' in filesToCreate:
+      self.create_default_nix(lock, output, outputDefaultNix, source)
+
+    # add new package to git
+    if config['isRepo']:
+      sp.run(["git", "add", "-N", output])
+
+  def parse_extra_args(self):
     specified_extra_args = {
       arg[0]: arg[1] for arg in map(
         lambda e: e.split('='),
         self.option("arg"),
       )
     }
+    return specified_extra_args
 
-    # ensure packages-root
+  def create_default_nix(self, lock, output, outputDefaultNix, source):
+    template = callNixFunction(
+      'apps.apps.cli.templateDefaultNix',
+      dream2nixLocationRelative=os.path.relpath(dream2nix_src, output),
+      dreamLock=lock,
+      sourcePathRelative=os.path.relpath(source, os.path.dirname(outputDefaultNix))
+    )
+    with open(outputDefaultNix, 'w') as defaultNix:
+      defaultNix.write(template)
+      print(f"Created {output}/default.nix")
+
+  def find_packages_root(self):
     if self.option("packages-root"):
       packages_root = self.option("packages-root")
     elif config['packagesDir']:
@@ -75,52 +150,268 @@ class AddCommand(Command):
     if not os.path.isdir(packages_root):
       print(
         f"Packages direcotry {packages_root} does not exist. Please create.",
-        file = sys.stderr,
-      )
-
-    # verify source
-    source = self.argument("source")
-    if not source and not config['packagesDir']:
-      source = os.path.realpath('./.')
-      print(
-        f"Source not specified. Defaulting to current directory: {source}",
         file=sys.stderr,
       )
-    # else:
-    #   print(
-    #     f"Source not specified. Defaulting to current directory: {source}",
-    #     file=sys.stderr,
-    #   )
-    # check if source is valid fetcher spec
-    sourceSpec = {}
-    # handle source shortcuts
-    if source.partition(':')[0].split('+')[0] in os.environ.get("fetcherNames", None).split()\
-        or source.startswith('http'):
-      print(f"fetching source for '{source}'")
-      sourceSpec =\
-        callNixFunction("fetchers.translateShortcut", shortcut=source)
-      source =\
-        buildNixFunction("fetchers.fetchShortcut", shortcut=source, extract=True)
-    # handle source paths
-    else:
-      # check if source path exists
-      if not os.path.exists(source):
-        print(f"Input source '{source}' does not exist", file=sys.stdout)
-        exit(1)
-      source = os.path.realpath(source)
-      # handle source from dream-lock.json
-      if source.endswith('dream-lock.json'):
-        print(f"fetching source defined via existing dream-lock.json")
-        with open(source) as f:
-          sourceDreamLock = json.load(f)
-        sourceMainPackageName = sourceDreamLock['_generic']['mainPackageName']
-        sourceMainPackageVersion = sourceDreamLock['_generic']['mainPackageVersion']
-        sourceSpec =\
-          sourceDreamLock['sources'][sourceMainPackageName][sourceMainPackageVersion]
-        source = \
-          buildNixFunction("fetchers.fetchSource", source=sourceSpec, extract=True)
+    return packages_root
 
-    # select translator
+  def format_lock_str(self, lock):
+    lockStr = json.dumps(lock, indent=2, sort_keys=True)
+    lockStr = lockStr \
+      .replace("[\n          ", "[ ") \
+      .replace("\"\n        ]", "\" ]") \
+      .replace(",\n          ", ", ")
+    return lockStr
+
+  def aggregate_hashes(self, lock, outputDreamLock):
+    print("Building FOD of combined sources to retrieve output hash")
+    # remove hashes from lock file and init sourcesCombinedHash with empty string
+    strip_hashes_from_lock(lock)
+    lock['_generic']['sourcesCombinedHash'] = ""
+    with open(outputDreamLock, 'w') as f:
+      json.dump(lock, f, indent=2)
+    # compute FOD hash of combined sources
+    proc = sp.run(
+      [
+        "nix", "build", "--impure", "-L", "--expr",
+        f"(import {dream2nix_src} {{}}).fetchSources {{ dreamLock = {outputDreamLock}; }}"
+      ],
+      capture_output=True,
+    )
+    # read the output hash from the failed build log
+    match = re.search(r"FOD_PATH=(.*=)", proc.stderr.decode())
+    if not match:
+      print(proc.stderr.decode())
+      print(proc.stdout.decode())
+      raise Exception("Could not find FOD hash in FOD log")
+    hash = match.groups()[0]
+    print(f"Computed FOD hash: {hash}")
+    # store the hash in the lock
+    lock['_generic']['sourcesCombinedHash'] = hash
+
+  def postprocess_dep_graph(self, lock):
+    depGraph = lock['_generic']['dependencies']
+    # remove empty entries
+    if 'dependencies' in lock['_generic']:
+      for pname, deps in depGraph.copy().items():
+        if not deps:
+          del depGraph[pname]
+    # mark cyclic dependencies
+    edges = set()
+    for pname, versions in depGraph.items():
+      for version, deps in versions.items():
+        for dep in deps:
+          edges.add(((pname, version), tuple(dep)))
+    G = nx.DiGraph(sorted(list(edges)))
+    cycle_count = 0
+    removed_edges = []
+    for pname, versions in depGraph.items():
+      for version in versions.keys():
+        key = (pname, version)
+        try:
+          while True:
+            cycle = nx.find_cycle(G, key)
+            cycle_count += 1
+            node_from, node_to = cycle[-1][0], cycle[-1][1]
+            G.remove_edge(node_from, node_to)
+            removed_edges.append((node_from, node_to))
+        except nx.NetworkXNoCycle:
+          continue
+    lock['cyclicDependencies'] = {}
+    if removed_edges:
+      cycles_text = 'Detected Cyclic dependencies:'
+      for node, removed in removed_edges:
+        n_name, n_ver = node[0], node[1]
+        r_name, r_ver = removed[0], removed[1]
+        cycles_text += \
+          f"\n  {n_name}#{n_ver} -> {r_name}#{r_ver}"
+        if n_name not in lock['cyclicDependencies']:
+          lock['cyclicDependencies'][n_name] = {}
+        if n_ver not in lock['cyclicDependencies'][n_name]:
+          lock['cyclicDependencies'][n_name][n_ver] = []
+        lock['cyclicDependencies'][n_name][n_ver].append(removed)
+      print(cycles_text)
+
+  def add_main_source(self, lock, mainPackageName, mainPackageVersion, sourceSpec):
+    mainSource = sourceSpec.copy()
+    if not mainSource:
+      mainSource = dict(
+        type="unknown",
+      )
+    if mainPackageName not in lock['sources']:
+      lock['sources'][mainPackageName] = {
+        mainPackageVersion: mainSource
+      }
+    else:
+      lock['sources'][mainPackageName][mainPackageVersion] = mainSource
+
+  def extend_with_translator_info(self, lock, specified_extra_args, translator):
+    t = translator
+    lock['_generic']['translatedBy'] = f"{t['subsystem']}.{t['type']}.{t['name']}"
+    lock['_generic']['translatorParams'] = " ".join(
+      [
+        '--translator',
+        f"{translator['subsystem']}.{translator['type']}.{translator['name']}",
+      ] + (
+        ["--combined"] if self.option('combined') else []
+      ) + [
+        f"--arg {n}={v}" for n, v in specified_extra_args.items()
+      ])
+
+  def calc_outputs(self, mainPackageDirName, packages_root):
+    if self.option('target'):
+      if self.option('target').startswith('/'):
+        output = self.option('target')
+      else:
+        output = f"{packages_root}/{self.option('target')}"
+    else:
+      output = f"{packages_root}/{mainPackageDirName}"
+    # collect files to create
+    filesToCreate = ['dream-lock.json']
+    if not os.path.isdir(output):
+      os.mkdir(output)
+    existingFiles = set(os.listdir(output))
+    if not self.option('no-default-nix') \
+            and not 'default.nix' in existingFiles \
+            and not config['packagesDir']:
+      if self.confirm(
+              'Create a default.nix for debugging purposes',
+              default=True):
+        filesToCreate.append('default.nix')
+    # overwrite existing files only if --force is set
+    if self.option('force'):
+      for f in filesToCreate:
+        if os.path.isfile(f):
+          os.remove(f)
+    # raise error if any file exists already
+    else:
+      if any(f in existingFiles for f in filesToCreate):
+        print(
+          f"output directory {output} already contains a 'default.nix' "
+          "or 'dream-lock.json'. Resolve via one of these:\n"
+          "  - use --force to overwrite files\n"
+          "  - use --target to specify another target dir",
+          file=sys.stderr,
+        )
+        exit(1)
+    output = os.path.realpath(output)
+    return filesToCreate, output
+
+  def define_attribute_name(self, mainPackageName):
+    mainPackageDirName = mainPackageName.strip('@').replace('/', '-')
+
+    # verify / change main package dir name
+    def update_name(mainPackageDirName):
+      print(f"Current package attribute name is: {mainPackageDirName}")
+      new_name = self.ask(
+        "Specify new attribute name or leave empty to keep current:"
+      )
+      if new_name:
+        return new_name
+      return mainPackageDirName
+
+    mainPackageDirName = update_name(mainPackageDirName)
+    return mainPackageDirName
+
+  def run_translate(self, source, specified_extra_args, translator):
+    # build the translator bin
+    t = translator
+    translator_path = buildNixAttribute(
+      f"translators.translators.{t['subsystem']}.{t['type']}.{t['name']}.translateBin"
+    )
+    # direct outputs of translator to temporary file
+    with tempfile.NamedTemporaryFile("r") as output_temp_file:
+      # arguments for calling the translator nix module
+      translator_input = dict(
+        inputFiles=[],
+        inputDirectories=[source],
+        outputFile=output_temp_file.name,
+      )
+      translator_input.update(specified_extra_args)
+
+      # dump translator arguments to json file and execute translator
+      print("\nTranslating project metadata")
+      with tempfile.NamedTemporaryFile("w") as input_json_file:
+        json.dump(translator_input, input_json_file, indent=2)
+        input_json_file.seek(0)  # flushes write cache
+
+        # execute translator
+        sp.run(
+          [f"{translator_path}/bin/run", input_json_file.name]
+        )
+
+      # raise error if output wasn't produced
+      if not output_temp_file.read():
+        raise Exception(f"Translator failed to create dream-lock.json")
+
+      # read produced lock file
+      with open(output_temp_file.name) as f:
+        lock = json.load(f)
+    return lock
+
+  def declare_extra_args(self, specified_extra_args, translator):
+    unknown_extra_args = set(specified_extra_args.keys()) - set(translator['extraArgs'].keys())
+    if unknown_extra_args:
+      print(
+        f"Invalid extra args for translator '{translator['name']}': "
+        f" {', '.join(unknown_extra_args)}"
+        "\nPlease remove these parameters",
+        file=sys.stderr
+      )
+      exit(1)
+    # transform flags to bool
+    for argName, argVal in specified_extra_args.copy().items():
+      if translator['extraArgs'][argName]['type'] == 'flag':
+        if argVal.lower() in ('yes', 'y', 'true'):
+          specified_extra_args[argName] = True
+        elif argVal.lower() in ('no', 'n', 'false'):
+          specified_extra_args[argName] = False
+        else:
+          print(
+            f"Invalid value {argVal} for argument {argName}",
+            file=sys.stderr
+          )
+    specified_extra_args = \
+      {k: (bool(v) if translator['extraArgs'][k]['type'] == 'flag' else v) \
+       for k, v in specified_extra_args.items()}
+    # on non-interactive session, assume defaults for unspecified extra args
+    if not self.io.is_interactive():
+      specified_extra_args.update(
+        {n: (True if v['type'] == 'flag' else v['default']) \
+         for n, v in translator['extraArgs'].items() \
+         if n not in specified_extra_args and 'default' in v}
+      )
+    unspecified_extra_args = \
+      {n: v for n, v in translator['extraArgs'].items() \
+       if n not in specified_extra_args}
+    # raise error if any extra arg unspecified in non-interactive session
+    if unspecified_extra_args:
+      if not self.io.is_interactive():
+        print(
+          f"Please specify the following extra arguments required by translator '{translator['name']}' :\n" \
+          ', '.join(unspecified_extra_args.keys()),
+          file=sys.stderr
+        )
+        exit(1)
+      # interactively retrieve answers for unspecified extra arguments
+      else:
+        print(f"\nThe translator '{translator['name']}' requires additional options")
+        for arg_name, arg in unspecified_extra_args.items():
+          print('')
+          if arg['type'] == 'flag':
+            print(f"Please specify '{arg_name}'")
+            specified_extra_args[arg_name] = self.confirm(f"{arg['description']}:", False)
+          else:
+            print(f"Please specify '{arg_name}': {arg['description']}")
+            print(f"Example values: " + ', '.join(arg['examples']))
+            if 'default' in arg:
+              print(f"Leave empty for default ({arg['default']})")
+            while True:
+              specified_extra_args[arg_name] = self.ask(f"{arg_name}:", arg.get('default'))
+              if specified_extra_args[arg_name]:
+                break
+    return specified_extra_args
+
+  def select_translator(self, source):
     translatorsSorted = sorted(
       list_translators_for_source(source),
       key=lambda t: (
@@ -153,302 +444,49 @@ class AddCommand(Command):
           ))[0]
         elif len(translator) == 1:
           translator = list(filter(
-            lambda t:  [t['name']] == translator,
+            lambda t: [t['name']] == translator,
             translatorsSorted,
           ))[0]
       except IndexError:
         print(f"Could not find translator '{'.'.join(translator)}'", file=sys.stderr)
         exit(1)
+    return translator
 
-    # raise error if any specified extra arg is unknown
-    unknown_extra_args = set(specified_extra_args.keys()) - set(translator['extraArgs'].keys())
-    if unknown_extra_args:
+  def parse_source(self):
+    # verify source
+    source = self.argument("source")
+    if not source and not config['packagesDir']:
+      source = os.path.realpath('./.')
       print(
-        f"Invalid extra args for translator '{translator['name']}': "
-        f" {', '.join(unknown_extra_args)}"
-        "\nPlease remove these parameters",
-        file=sys.stderr
+        f"Source not specified. Defaulting to current directory: {source}",
+        file=sys.stderr,
       )
-      exit(1)
-
-    # transform flags to bool
-    for argName, argVal in specified_extra_args.copy().items():
-      if translator['extraArgs'][argName]['type'] == 'flag':
-        if argVal.lower() in ('yes', 'y', 'true'):
-          specified_extra_args[argName] = True
-        elif argVal.lower() in ('no', 'n', 'false'):
-          specified_extra_args[argName] = False
-        else:
-          print(
-            f"Invalid value {argVal} for argument {argName}",
-            file=sys.stderr
-          )
-
-    specified_extra_args =\
-      {k: (bool(v) if translator['extraArgs'][k]['type'] == 'flag' else v ) \
-          for k, v in specified_extra_args.items()}
-
-    # on non-interactive session, assume defaults for unspecified extra args
-    if not self.io.is_interactive():
-      specified_extra_args.update(
-        {n: (True if v['type'] == 'flag' else v['default']) \
-          for n, v in translator['extraArgs'].items() \
-          if n not in specified_extra_args and 'default' in v}
-      )
-    unspecified_extra_args = \
-      {n: v for n, v in translator['extraArgs'].items() \
-        if n not in specified_extra_args}
-    # raise error if any extra arg unspecified in non-interactive session
-    if unspecified_extra_args:
-      if not self.io.is_interactive():
-        print(
-          f"Please specify the following extra arguments required by translator '{translator['name']}' :\n" \
-            ', '.join(unspecified_extra_args.keys()),
-          file=sys.stderr
-        )
+    # check if source is a valid fetcher spec
+    sourceSpec = {}
+    # handle source shortcuts
+    if source.partition(':')[0].split('+')[0] in os.environ.get("fetcherNames", None).split() \
+            or source.startswith('http'):
+      print(f"fetching source for '{source}'")
+      sourceSpec = \
+        callNixFunction("fetchers.translateShortcut", shortcut=source)
+      source = \
+        buildNixFunction("fetchers.fetchShortcut", shortcut=source, extract=True)
+    # handle source paths
+    else:
+      # check if source path exists
+      if not os.path.exists(source):
+        print(f"Input source '{source}' does not exist", file=sys.stdout)
         exit(1)
-      # interactively retrieve answers for unspecified extra arguments
-      else:
-        print(f"\nThe translator '{translator['name']}' requires additional options")
-        for arg_name, arg in unspecified_extra_args.items():
-          print('')
-          if arg['type'] == 'flag':
-            print(f"Please specify '{arg_name}'")
-            specified_extra_args[arg_name] = self.confirm(f"{arg['description']}:", False)
-          else:
-            print(f"Please specify '{arg_name}': {arg['description']}")
-            print(f"Example values: " + ', '.join(arg['examples']))
-            if 'default' in arg:
-              print(f"Leave empty for default ({arg['default']})")
-            while True:
-              specified_extra_args[arg_name] = self.ask(f"{arg_name}:", arg.get('default'))
-              if specified_extra_args[arg_name]:
-                break
-
-    # build the translator bin
-    t = translator
-    translator_path = buildNixAttribute(
-      f"translators.translators.{t['subsystem']}.{t['type']}.{t['name']}.translateBin"
-    )
-
-    # direct outputs of translator to temporary file
-    with tempfile.NamedTemporaryFile("r") as output_temp_file:
-
-      # arguments for calling the translator nix module
-      translator_input = dict(
-        inputFiles=[],
-        inputDirectories=[source],
-        outputFile=output_temp_file.name,
-      )
-      translator_input.update(specified_extra_args)
-
-      # dump translator arguments to json file and execute translator
-      print("\nTranslating project metadata")
-      with tempfile.NamedTemporaryFile("w") as input_json_file:
-        json.dump(translator_input, input_json_file, indent=2)
-        input_json_file.seek(0) # flushes write cache
-
-        # execute translator
-        sp.run(
-          [f"{translator_path}/bin/run", input_json_file.name]
-        )
-
-      # raise error if output wasn't produced
-      if not output_temp_file.read():
-        raise Exception(f"Translator failed to create dream-lock.json")
-
-      # read produced lock file
-      with open(output_temp_file.name) as f:
-        lock = json.load(f)
-
-
-    # get package name and version from lock
-    mainPackageName = lock['_generic']['mainPackageName']
-    mainPackageVersion = lock['_generic']['mainPackageVersion']
-
-    # calculate output directory
-    mainPackageDirName = mainPackageName.strip('@').replace('/', '-')
-
-    # verify / change main package dir name
-    def update_name(mainPackageDirName):
-      print(f"Current package attribute name is: {mainPackageDirName}")
-      new_name = self.ask(
-        "Specify new attribute name or leave empty to keep current:"
-      )
-      if new_name:
-        return new_name
-      return mainPackageDirName
-
-    mainPackageDirName = update_name(mainPackageDirName)
-
-    if self.option('target'):
-      if self.option('target').startswith('/'):
-        output = self.option('target')
-      else:
-        output = f"{packages_root}/{self.option('target')}"
-    else:
-      output = f"{packages_root}/{mainPackageDirName}"
-
-    # collect files to create
-    filesToCreate = ['dream-lock.json']
-    if not os.path.isdir(output):
-      os.mkdir(output)
-    existingFiles = set(os.listdir(output))
-    if not self.option('no-default-nix')\
-        and not 'default.nix' in existingFiles\
-        and not config['packagesDir']:
-      if self.confirm(
-          'Create a default.nix for debugging purposes',
-          default=True):
-        filesToCreate.append('default.nix')
-
-    # overwrite existing files only if --force is set
-    if self.option('force'):
-      for f in filesToCreate:
-        if os.path.isfile(f):
-          os.remove(f)
-    # raise error if any file exists already
-    else:
-      if any(f in existingFiles for f in filesToCreate):
-        print(
-          f"output directory {output} already contains a 'default.nix' "
-          "or 'dream-lock.json'. Resolve via one of these:\n"
-          "  - use --force to overwrite files\n"
-          "  - use --target to specify another target dir",
-          file=sys.stderr,
-        )
-        exit(1)
-    output = os.path.realpath(output)
-    outputDreamLock = f"{output}/dream-lock.json"
-    outputDefaultNix = f"{output}/default.nix"
-
-    # write translator information to lock file
-    combined = self.option('combined')
-    lock['_generic']['translatedBy'] = f"{t['subsystem']}.{t['type']}.{t['name']}"
-    lock['_generic']['translatorParams'] = " ".join([
-      '--translator',
-      f"{translator['subsystem']}.{translator['type']}.{translator['name']}",
-    ] + (
-      ["--combined"] if combined else []
-    ) + [
-      f"--arg {n}={v}" for n, v in specified_extra_args.items()
-    ])
-
-    # add main package source
-    mainSource = sourceSpec.copy()
-    if not mainSource:
-      mainSource = dict(
-        type="unknown",
-      )
-    if mainPackageName not in lock['sources']:
-      lock['sources'][mainPackageName] = {
-        mainPackageVersion: mainSource
-      }
-    else:
-      lock['sources'][mainPackageName][mainPackageVersion] = mainSource
-
-    # clean up dependency graph
-    # remove empty entries
-    if 'dependencies' in lock['_generic']:
-      depGraph = lock['_generic']['dependencies']
-      if 'dependencies' in lock['_generic']:
-        for pname, deps in depGraph.copy().items():
-          if not deps:
-            del depGraph[pname]
-
-      # remove cyclic dependencies
-      edges = set()
-      for pname, versions in depGraph.items():
-        for version, deps in versions.items():
-          for dep in deps:
-            edges.add(((pname, version), tuple(dep)))
-      G = nx.DiGraph(sorted(list(edges)))
-      cycle_count = 0
-      removed_edges = []
-      for pname, versions in depGraph.items():
-        for version in versions.keys():
-          key = (pname, version)
-          try:
-            while True:
-              cycle = nx.find_cycle(G, key)
-              cycle_count += 1
-              # remove_dependecy(indexed_pkgs, G, cycle[-1][0], cycle[-1][1])
-              node_from, node_to = cycle[-1][0], cycle[-1][1]
-              G.remove_edge(node_from, node_to)
-              removed_edges.append((node_from, node_to))
-          except nx.NetworkXNoCycle:
-            continue
-      lock['cyclicDependencies'] = {}
-      if removed_edges:
-        cycles_text = 'Detected Cyclic dependencies:'
-        for node, removed in removed_edges:
-          n_name, n_ver = node[0], node[1]
-          r_name, r_ver = removed[0], removed[1]
-          cycles_text +=\
-            f"\n  {n_name}#{n_ver} -> {r_name}#{r_ver}"
-          if n_name not in lock['cyclicDependencies']:
-            lock['cyclicDependencies'][n_name] = {}
-          if n_ver not in lock['cyclicDependencies'][n_name]:
-            lock['cyclicDependencies'][n_name][n_ver] = []
-          lock['cyclicDependencies'][n_name][n_ver].append(removed)
-        print(cycles_text)
-
-    # calculate combined hash if --combined was specified
-    if combined:
-
-      print("Building FOD of combined sources to retrieve output hash")
-
-      # remove hashes from lock file and init sourcesCombinedHash with empty string
-      strip_hashes_from_lock(lock)
-      lock['_generic']['sourcesCombinedHash'] = ""
-      with open(outputDreamLock, 'w') as f:
-        json.dump(lock, f, indent=2)
-
-      # compute FOD hash of combined sources
-      proc = sp.run(
-        [
-          "nix", "build", "--impure", "-L", "--expr",
-          f"(import {dream2nix_src} {{}}).fetchSources {{ dreamLock = {outputDreamLock}; }}"
-        ],
-        capture_output=True,
-      )
-
-      # read the output hash from the failed build log
-      match = re.search(r"FOD_PATH=(.*=)", proc.stderr.decode())
-      if not match:
-        print(proc.stderr.decode())
-        print(proc.stdout.decode())
-        raise Exception("Could not find FOD hash in FOD log")
-      hash = match.groups()[0]
-      print(f"Computed FOD hash: {hash}")
-
-      # store the hash in the lock
-      lock['_generic']['sourcesCombinedHash'] = hash
-
-    # re-write dream-lock.json
-    checkLockJSON(lock)
-    lockStr = json.dumps(lock, indent=2, sort_keys = True)
-    lockStr = lockStr\
-      .replace("[\n          ", "[ ")\
-      .replace("\"\n        ]", "\" ]")\
-      .replace(",\n          ", ", ")
-    with open(outputDreamLock, 'w') as f:
-      f.write(lockStr)
-
-    # create default.nix
-    template = callNixFunction(
-      'apps.apps.cli.templateDefaultNix',
-      dream2nixLocationRelative=os.path.relpath(dream2nix_src, output),
-      dreamLock = lock,
-      sourcePathRelative = os.path.relpath(source, os.path.dirname(outputDefaultNix))
-    )
-    # with open(f"{dream2nix_src}/apps/cli2/templateDefault.nix") as template:
-    if 'default.nix' in filesToCreate:
-      with open(outputDefaultNix, 'w') as defaultNix:
-        defaultNix.write(template)
-        print(f"Created {output}/default.nix")
-
-    print(f"Created {output}/dream-lock.json")
-
-    if config['isRepo']:
-      sp.run(["git", "add", "-N", output])
+      source = os.path.realpath(source)
+      # handle source from dream-lock.json
+      if source.endswith('dream-lock.json'):
+        print(f"fetching source defined via existing dream-lock.json")
+        with open(source) as f:
+          sourceDreamLock = json.load(f)
+        sourceMainPackageName = sourceDreamLock['_generic']['mainPackageName']
+        sourceMainPackageVersion = sourceDreamLock['_generic']['mainPackageVersion']
+        sourceSpec = \
+          sourceDreamLock['sources'][sourceMainPackageName][sourceMainPackageVersion]
+        source = \
+          buildNixFunction("fetchers.fetchSource", source=sourceSpec, extract=True)
+    return source, sourceSpec

From ff9e30b24270a7a367ee4de8d5f44465132b03ef Mon Sep 17 00:00:00 2001
From: Akshat Agarwal <humancalico@disroot.org>
Date: Thu, 11 Nov 2021 13:41:17 +0530
Subject: [PATCH 2/8] add go support using gomod2nix

Co-authored-by: DavHau <hsngrmpf+github@gmail.com>
---
 flake.lock                                    | 17 +++++
 flake.nix                                     |  6 +-
 src/builders/go/gomod2nix/default.nix         | 21 ++++++
 .../go/impure/gomod2nix/default.nix           | 68 +++++++++++++++++++
 .../go/impure/gomod2nix/translate.nix         | 66 ++++++++++++++++++
 5 files changed, 177 insertions(+), 1 deletion(-)
 create mode 100644 src/builders/go/gomod2nix/default.nix
 create mode 100644 src/translators/go/impure/gomod2nix/default.nix
 create mode 100644 src/translators/go/impure/gomod2nix/translate.nix

diff --git a/flake.lock b/flake.lock
index 77b306e5..949038a1 100644
--- a/flake.lock
+++ b/flake.lock
@@ -1,5 +1,21 @@
 {
   "nodes": {
+    "gomod2nix": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1627572165,
+        "narHash": "sha256-MFpwnkvQpauj799b4QTBJQFEddbD02+Ln5k92QyHOSk=",
+        "owner": "tweag",
+        "repo": "gomod2nix",
+        "rev": "67f22dd738d092c6ba88e420350ada0ed4992ae8",
+        "type": "github"
+      },
+      "original": {
+        "owner": "tweag",
+        "repo": "gomod2nix",
+        "type": "github"
+      }
+    },
     "mach-nix": {
       "flake": false,
       "locked": {
@@ -81,6 +97,7 @@
     },
     "root": {
       "inputs": {
+        "gomod2nix": "gomod2nix",
         "mach-nix": "mach-nix",
         "nix-parsec": "nix-parsec",
         "nixpkgs": "nixpkgs",
diff --git a/flake.nix b/flake.nix
index e94e53b5..87a1ecc8 100644
--- a/flake.nix
+++ b/flake.nix
@@ -3,7 +3,10 @@
 
   inputs = {
     nixpkgs.url = "nixpkgs/nixos-unstable";
-    
+
+    # required for builder go/gomod2nix
+    gomod2nix = { url = "github:tweag/gomod2nix"; flake = false; };
+
     # required for translator nodejs/pure/package-lock
     nix-parsec = { url = "github:nprindle/nix-parsec"; flake = false; };
 
@@ -19,6 +22,7 @@
 
   outputs = {
     self,
+    gomod2nix,
     mach-nix,
     nix-parsec,
     nixpkgs,
diff --git a/src/builders/go/gomod2nix/default.nix b/src/builders/go/gomod2nix/default.nix
new file mode 100644
index 00000000..3a833261
--- /dev/null
+++ b/src/builders/go/gomod2nix/default.nix
@@ -0,0 +1,21 @@
+{
+  lib,
+  pkgs,
+  externals,
+  ...
+}:
+
+{
+  fetchedSources,
+  dreamLock,
+}:
+let
+  gomod2nixTOML = fetchedSources.mapAttrs
+    dependencyObject.goName;
+in
+externals.gomod2nixBuilder rec {
+  pname = dreamLock.generic.mainPackage;
+  version = dreamLock.sources."${pname}".version;
+  src = fetchedSources."${pname}";
+  modules = ./gomod2nix.toml;
+}
diff --git a/src/translators/go/impure/gomod2nix/default.nix b/src/translators/go/impure/gomod2nix/default.nix
new file mode 100644
index 00000000..8869d49a
--- /dev/null
+++ b/src/translators/go/impure/gomod2nix/default.nix
@@ -0,0 +1,68 @@
+{
+  # dream2nix utils
+  utils,
+  dream2nixWithExternals,
+
+  bash,
+  coreutils,
+  jq,
+  lib,
+  nix,
+  writeScriptBin,
+  ...
+}:
+
+{
+
+  # the input format is specified in /specifications/translator-call-example.json
+  # this script receives a json file including the input paths and specialArgs
+  translateBin = utils.writePureShellScript
+    [
+      bash
+      coreutils
+      jq
+      nix
+    ]
+    ''
+      # accroding to the spec, the translator reads the input from a json file
+      jsonInput=$1
+
+      # read the json input
+      outputFile=$(${jq}/bin/jq '.outputFile' -c -r $jsonInput)
+      inputDirectory=$(${jq}/bin/jq '.inputDirectories | .[0]' -c -r $jsonInput)
+
+      tmpBuild=$(mktemp -d)
+      cd $tmpBuild
+      cp -r $inputDirectory/* .
+      chmod -R +w .
+      # This should be in sync with gomod2nix version in flake.lock
+      nix run github:tweag/gomod2nix/67f22dd738d092c6ba88e420350ada0ed4992ae8
+
+      nix eval --show-trace --impure --raw --expr "import ${./translate.nix} ${dream2nixWithExternals} ./." > $outputFile
+    '';
+
+
+  # From a given list of paths, this function returns all paths which can be processed by this translator.
+  # This allows the framework to detect if the translator is compatible with the given inputs
+  # to automatically select the right translator.
+  compatiblePaths =
+    {
+      inputDirectories,
+      inputFiles,
+    }@args:
+    {
+      inputDirectories = lib.filter
+        (utils.containsMatchingFile [ ''go\.sum'' ''go\.mod'' ])
+        args.inputDirectories;
+
+      inputFiles = [];
+    };
+
+
+  # If the translator requires additional arguments, specify them here.
+  # There are only two types of arguments:
+  #   - string argument (type = "argument")
+  #   - boolean flag (type = "flag")
+  # String arguments contain a default value and examples. Flags do not.
+  extraArgs = {};
+}
diff --git a/src/translators/go/impure/gomod2nix/translate.nix b/src/translators/go/impure/gomod2nix/translate.nix
new file mode 100644
index 00000000..521c6fe3
--- /dev/null
+++ b/src/translators/go/impure/gomod2nix/translate.nix
@@ -0,0 +1,66 @@
+dream2nixWithExternals:
+cwd:
+let
+  dream2nix = import dream2nixWithExternals { };
+  b = builtins;
+  parsed = b.fromTOML (builtins.readFile "${cwd}/gomod2nix.toml");
+  pkgs = import <nixpkgs> { };
+  lib = pkgs.lib;
+  serializePackages = inputData:
+    lib.mapAttrsToList
+      (goName: depAttrs: depAttrs // { inherit goName; })
+      parsed;
+  translated = dream2nix.utils.simpleTranslate "gomod2nix" rec {
+
+    inputData = parsed;
+
+    mainPackageName =
+      let
+        firstLine = (b.elemAt (lib.splitString "\n" (b.readFile "${cwd}/go.mod")) 0);
+      in
+        lib.last (lib.splitString "/" (b.elemAt (lib.splitString " " firstLine) 1));
+
+    mainPackageVersion = "unknown";
+
+    subsystemName = "go";
+
+    subsystemAttrs = { };
+
+    inherit serializePackages;
+
+    mainPackageDependencies =
+      lib.forEach
+        (serializePackages parsed)
+        (dep: {
+            name = getName dep;
+            version = getVersion dep;
+        });
+
+    getOriginalID = dependencyObject:
+      null;
+
+    getName = dependencyObject:
+      dependencyObject.goName;
+
+    getVersion = dependencyObject:
+      lib.removePrefix "v" dependencyObject.sumVersion;
+
+    getDependencies = dependencyObject: getDepByNameVer: dependenciesByOriginalID:
+      [];
+
+    getSourceType = dependencyObject: "git";
+
+    sourceConstructors = {
+      git = dependencyObject:
+        {
+          type = "git";
+          version = getVersion dependencyObject;
+          hash = dependencyObject.fetch.sha256;
+          url = dependencyObject.fetch.url;
+          rev = dependencyObject.fetch.rev;
+        };
+    };
+
+  };
+in
+  dream2nix.utils.dreamLock.toJSON translated

From 3b5b9096e138d2b1fe420dd7833fb19950d1dcaa Mon Sep 17 00:00:00 2001
From: DavHau <hsngrmpf+github@gmail.com>
Date: Wed, 17 Nov 2021 11:23:31 +0700
Subject: [PATCH 3/8] hercules ci + improvements

 - yarn-lock: add nodejs version argument
 - add impure tests
 - add ci.nix for hercules
---
 ci.nix                                        | 17 ++++++
 flake.nix                                     | 13 +++-
 src/apps/cli/commands/add.py                  | 61 ++++++++++---------
 src/apps/cli/commands/update.py               |  2 +-
 src/apps/cli/default.nix                      | 33 +++++-----
 src/default.nix                               |  8 ++-
 src/templates/translators/impure.nix          |  6 +-
 src/translators/default.nix                   | 55 ++++++++++++++---
 .../nodejs/impure/package-json/default.nix    |  8 +--
 .../nodejs/pure/yarn-lock/default.nix         | 40 +++++++-----
 src/utils/default.nix                         | 12 ++--
 src/utils/dream-lock.nix                      |  6 +-
 tests/{ => eval}/defaut.nix                   |  0
 tests/impure/default.nix                      | 34 +++++++++++
 tests/impure/test_package-lock.nix            | 27 ++++++++
 tests/impure/test_yarn-lock-prettier.nix      | 27 ++++++++
 checks.nix => tests/pure/default.nix          | 14 +++--
 17 files changed, 268 insertions(+), 95 deletions(-)
 create mode 100644 ci.nix
 rename tests/{ => eval}/defaut.nix (100%)
 create mode 100644 tests/impure/default.nix
 create mode 100644 tests/impure/test_package-lock.nix
 create mode 100644 tests/impure/test_yarn-lock-prettier.nix
 rename checks.nix => tests/pure/default.nix (74%)

diff --git a/ci.nix b/ci.nix
new file mode 100644
index 00000000..de47ba28
--- /dev/null
+++ b/ci.nix
@@ -0,0 +1,17 @@
+let
+  b = builtins;
+  flakeCompatSrc = b.fetchurl "https://raw.githubusercontent.com/edolstra/flake-compat/12c64ca55c1014cdc1b16ed5a804aa8576601ff2/default.nix";
+  flake = (import flakeCompatSrc { src = ./.; }).defaultNix;
+  pkgs = import flake.inputs.nixpkgs {};
+  recurseIntoAll = b.mapAttrs (name: val: pkgs.recurseIntoAttrs val);
+
+in
+{
+  inherit flake;
+}
+
+// (recurseIntoAll {
+
+  checks = flake.checks.x86_64-linux;
+
+})
diff --git a/flake.nix b/flake.nix
index e94e53b5..3704a53a 100644
--- a/flake.nix
+++ b/flake.nix
@@ -3,7 +3,7 @@
 
   inputs = {
     nixpkgs.url = "nixpkgs/nixos-unstable";
-    
+
     # required for translator nodejs/pure/package-lock
     nix-parsec = { url = "github:nprindle/nix-parsec"; flake = false; };
 
@@ -127,7 +127,14 @@
 
         # all apps including cli, install, etc.
         apps = forAllSystems (system: pkgs:
-          dream2nixFor."${system}".apps.flakeApps
+          dream2nixFor."${system}".apps.flakeApps // {
+            tests-impure = {
+              type = "app";
+              program =
+                b.toString
+                  (dream2nixFor."${system}".callPackageDream ./tests/impure {});
+            };
+          }
         );
 
         # a dev shell for working on dream2nix
@@ -151,7 +158,7 @@
           '';
         });
 
-        checks = forAllSystems (system: pkgs: import ./checks.nix {
+        checks = forAllSystems (system: pkgs: import ./tests/pure {
           inherit lib pkgs;
           dream2nix = dream2nixFor."${system}";
         });
diff --git a/src/apps/cli/commands/add.py b/src/apps/cli/commands/add.py
index 28cc152c..60bb0321 100644
--- a/src/apps/cli/commands/add.py
+++ b/src/apps/cli/commands/add.py
@@ -23,14 +23,17 @@ class AddCommand(Command):
   arguments = [
     argument(
       "source",
-      "source of the package, can be a path, tarball URL, or flake-style spec")
+      "Sources of the packages. Can be a paths, tarball URLs, or flake-style specs",
+      # multiple=True
+    )
   ]
 
   options = [
     option("translator", None, "which translator to use", flag=False),
     option("target", None, "target file/directory for the dream-lock.json", flag=False),
+    option("attribute-name", None, "attribute name for new new pakcage", flag=False),
     option(
-      "--packages-root",
+      "packages-root",
       None,
       "Put package under a new directory inside packages-root",
       flag=False
@@ -63,17 +66,9 @@ class AddCommand(Command):
     # ensure packages-root
     packages_root = self.find_packages_root()
 
-    # get source path and spec
-    source, sourceSpec = self.parse_source()
 
-    # select translator
-    translator = self.select_translator(source)
-
-    # raise error if any specified extra arg is unknown
-    specified_extra_args = self.declare_extra_args(specified_extra_args, translator)
-
-    # do the translation and produce dream lock
-    lock = self.run_translate(source, specified_extra_args, translator)
+    lock, sourceSpec, specified_extra_args, translator =\
+      self.translate_from_source(specified_extra_args, self.argument("source"))
 
     # get package name and version from lock
     mainPackageName = lock['_generic']['mainPackageName']
@@ -114,12 +109,23 @@ class AddCommand(Command):
 
     # create default.nix
     if 'default.nix' in filesToCreate:
-      self.create_default_nix(lock, output, outputDefaultNix, source)
+      self.create_default_nix(lock, output, outputDefaultNix, sources[0])
 
     # add new package to git
     if config['isRepo']:
       sp.run(["git", "add", "-N", output])
 
+  def translate_from_source(self, specified_extra_args, source):
+    # get source path and spec
+    source, sourceSpec = self.parse_source(source)
+    # select translator
+    translator = self.select_translator(source)
+    # raise error if any specified extra arg is unknown
+    specified_extra_args = self.declare_extra_args(specified_extra_args, translator)
+    # do the translation and produce dream lock
+    lock = self.run_translate(source, specified_extra_args, translator)
+    return lock, sourceSpec, specified_extra_args, translator
+
   def parse_extra_args(self):
     specified_extra_args = {
       arg[0]: arg[1] for arg in map(
@@ -297,20 +303,20 @@ class AddCommand(Command):
     return filesToCreate, output
 
   def define_attribute_name(self, mainPackageName):
-    mainPackageDirName = mainPackageName.strip('@').replace('/', '-')
+    attributeName = self.option('attribute-name')
+    if attributeName:
+      return attributeName
+
+    attributeName = mainPackageName.strip('@').replace('/', '-')
 
     # verify / change main package dir name
-    def update_name(mainPackageDirName):
-      print(f"Current package attribute name is: {mainPackageDirName}")
-      new_name = self.ask(
-        "Specify new attribute name or leave empty to keep current:"
-      )
-      if new_name:
-        return new_name
-      return mainPackageDirName
-
-    mainPackageDirName = update_name(mainPackageDirName)
-    return mainPackageDirName
+    print(f"Current package attribute name is: {attributeName}")
+    new_name = self.ask(
+      "Specify new attribute name or leave empty to keep current:"
+    )
+    if new_name:
+      attributeName = newName
+    return attributeName
 
   def run_translate(self, source, specified_extra_args, translator):
     # build the translator bin
@@ -336,7 +342,7 @@ class AddCommand(Command):
 
         # execute translator
         sp.run(
-          [f"{translator_path}/bin/run", input_json_file.name]
+          [f"{translator_path}", input_json_file.name]
         )
 
       # raise error if output wasn't produced
@@ -452,9 +458,8 @@ class AddCommand(Command):
         exit(1)
     return translator
 
-  def parse_source(self):
+  def parse_source(self, source):
     # verify source
-    source = self.argument("source")
     if not source and not config['packagesDir']:
       source = os.path.realpath('./.')
       print(
diff --git a/src/apps/cli/commands/update.py b/src/apps/cli/commands/update.py
index 7e69b483..20d5ddb4 100644
--- a/src/apps/cli/commands/update.py
+++ b/src/apps/cli/commands/update.py
@@ -66,7 +66,7 @@ class UpdateCommand(Command):
         dreamLock=dreamLockFile,
         updater=updater,
       )
-      update_proc = sp.run([f"{update_script}/bin/run"], capture_output=True)
+      update_proc = sp.run([f"{update_script}"], capture_output=True)
       version = update_proc.stdout.decode().strip()
     print(f"Updating from version {old_version} to {version}")
 
diff --git a/src/apps/cli/default.nix b/src/apps/cli/default.nix
index 5e8be7fa..cd5330ab 100644
--- a/src/apps/cli/default.nix
+++ b/src/apps/cli/default.nix
@@ -22,25 +22,22 @@ let
 
 in
 {
-  program = 
-    let
-      script = utils.writePureShellScript
-        [
-          gitMinimal
-          nix
-        ]
-        ''
-          # escape the temp dir created by writePureShellScript
-          cd - > /dev/null
+  program =
+    utils.writePureShellScript
+      [
+        gitMinimal
+        nix
+      ]
+      ''
+        # escape the temp dir created by writePureShellScript
+        cd - > /dev/null
 
-          # run the cli
-          dream2nixConfig=${configFile} \
-          dream2nixSrc=${dream2nixWithExternals} \
-          fetcherNames="${b.toString (lib.attrNames fetchers.fetchers)}" \
-            ${cliPython}/bin/python ${./.}/cli.py "$@"
-        '';
-    in
-      "${script}/bin/run";
+        # run the cli
+        dream2nixConfig=${configFile} \
+        dream2nixSrc=${dream2nixWithExternals} \
+        fetcherNames="${b.toString (lib.attrNames fetchers.fetchers)}" \
+          ${cliPython}/bin/python ${./.}/cli.py "$@"
+      '';
 
   templateDefaultNix =
     {
diff --git a/src/default.nix b/src/default.nix
index 9dac74b3..69e42d6b 100644
--- a/src/default.nix
+++ b/src/default.nix
@@ -57,6 +57,7 @@ let
 
   # like pkgs.callPackage, but includes all the dream2nix modules
   callPackageDream = f: args: pkgs.callPackage f (args // {
+    inherit apps;
     inherit builders;
     inherit callPackageDream;
     inherit config;
@@ -172,7 +173,7 @@ let
             fetchedSources;
       };
 
-  
+
   makeDreamLockForSource =
     {
       source,
@@ -330,7 +331,7 @@ let
 
 
   # produce outputs for a dream-lock or a source
-  riseAndShine = 
+  riseAndShine =
     {
       dreamLock ? null,
       builder ? null,
@@ -403,12 +404,13 @@ let
 
     in
       builderOutputs;
-   
+
 in
 {
   inherit
     apps
     builders
+    callPackageDream
     dream2nixWithExternals
     fetchers
     fetchSources
diff --git a/src/templates/translators/impure.nix b/src/templates/translators/impure.nix
index d954dff3..b2468cf0 100644
--- a/src/templates/translators/impure.nix
+++ b/src/templates/translators/impure.nix
@@ -12,7 +12,7 @@
 
 {
 
-  # A derivation which outputs an executable at `/bin/run`.
+  # A derivation which outputs a single executable at `$out`.
   # The executable will be called by dream2nix for translation
   # The input format is specified in /specifications/translator-call-example.json.
   # The first arg `$1` will be a json file containing the input parameters
@@ -57,10 +57,10 @@
       # examples:
       #   - ''.*requirements.*\.txt''
       #   - ''.*package-lock\.json''
-      inputDirectories = lib.filter 
+      inputDirectories = lib.filter
         (utils.containsMatchingFile [ ''TODO: regex1'' ''TODO: regex2'' ])
         args.inputDirectories;
-      
+
       inputFiles = [];
     };
 
diff --git a/src/translators/default.nix b/src/translators/default.nix
index 98b14201..e7c67d9f 100644
--- a/src/translators/default.nix
+++ b/src/translators/default.nix
@@ -10,14 +10,14 @@
   dream2nixWithExternals,
   utils,
   ...
-}: 
+}:
 let
 
   b = builtins;
 
   lib = pkgs.lib;
 
-  callTranslator = subsystem: type: name: file: args: 
+  callTranslator = subsystem: type: name: file: args:
     let
       translator = callPackageDream file (args // {
         inherit externals;
@@ -37,7 +37,7 @@ let
           translator.translate
             ((getextraArgsDefaults translator.extraArgs or {}) // args);
       };
-      
+
 
   subsystems = utils.dirNames ./.;
 
@@ -59,10 +59,10 @@ let
           nix eval --show-trace --impure --raw --expr "
               let
                 dream2nix = import ${dream2nixWithExternals} {};
-                dreamLock = 
+                dreamLock =
                   dream2nix.translators.translators.${
                     lib.concatStringsSep "." translatorAttrPath
-                  }.translate 
+                  }.translate
                     (builtins.fromJSON (builtins.readFile '''$1'''));
               in
                 dream2nix.utils.dreamLock.toJSON
@@ -95,23 +95,62 @@ let
   # flat list of all translators
   translatorsList = lib.collect (v: v ? translateBin) translators;
 
-  # json file exposing all existing translators to CLI including their special args
+  # returns the list of translators including their special args
+  # and adds a flag `compatible` to each translator indicating
+  # if the translator is compatible to all given paths
   translatorsForInput =
     {
       inputDirectories,
       inputFiles,
     }@args:
     lib.forEach translatorsList
-      (t: {
+      (t: rec {
         inherit (t)
           name
           extraArgs
           subsystem
           type
         ;
-        compatible = t.compatiblePaths args == args;
+        compatiblePaths = t.compatiblePaths args;
+        compatible = compatiblePaths == args;
       });
 
+  # also includes subdirectories of the given paths up to a certain depth
+  # to check for translator compatibility
+  translatorsForInputRecursive =
+    {
+      inputDirectories,
+      inputFiles,
+      depth ? 2,
+    }:
+    let
+      listDirsRec = dir: depth:
+        let
+          subDirs = (utils.listDirs dir);
+        in
+          if depth == 0 then
+            subDirs
+          else
+            subDirs
+            ++
+            (lib.flatten
+              (map
+                (subDir: listDirsRec subDir (depth -1))
+                subDirs));
+
+      dirsToCheck =
+        lib.flatten
+          (map
+            (inputDir: listDirsRec inputDir depth)
+            inputDirectories);
+
+    in
+      translatorsForInput {
+        inputDirectories = dirsToCheck;
+        inherit inputFiles;
+      };
+
+
   # pupulates a translators special args with defaults
   getextraArgsDefaults = extraArgsDef:
     lib.mapAttrs
diff --git a/src/translators/nodejs/impure/package-json/default.nix b/src/translators/nodejs/impure/package-json/default.nix
index e181673f..4a85eb94 100644
--- a/src/translators/nodejs/impure/package-json/default.nix
+++ b/src/translators/nodejs/impure/package-json/default.nix
@@ -35,8 +35,8 @@
       cat package-lock.json
 
       jq ".inputDirectories[0] = \"$(pwd)\"" -c -r $jsonInput > ./newJsonInput
-      
-      ${translators.translators.nodejs.pure.package-lock.translateBin}/bin/run $(realpath ./newJsonInput)
+
+      ${translators.translators.nodejs.pure.package-lock.translateBin} $(realpath ./newJsonInput)
     '';
 
 
@@ -49,10 +49,10 @@
       inputFiles,
     }@args:
     {
-      inputDirectories = lib.filter 
+      inputDirectories = lib.filter
         (utils.containsMatchingFile [ ''.*package.json'' ])
         args.inputDirectories;
-      
+
       inputFiles = [];
     };
 
diff --git a/src/translators/nodejs/pure/yarn-lock/default.nix b/src/translators/nodejs/pure/yarn-lock/default.nix
index aaa96a02..8e5ded56 100644
--- a/src/translators/nodejs/pure/yarn-lock/default.nix
+++ b/src/translators/nodejs/pure/yarn-lock/default.nix
@@ -2,6 +2,7 @@
   lib,
 
   externals,
+  nodejs,
   translatorName,
   utils,
   ...
@@ -16,14 +17,15 @@
       # extraArgs
       name,
       noDev,
+      nodejs,
       peer,
       ...
-    }:
+    }@args:
 
     let
       b = builtins;
       dev = ! noDev;
-  
+
       sourceDir = lib.elemAt inputDirectories 0;
       yarnLock = utils.readTextFile "${sourceDir}/yarn.lock";
       packageJSON = b.fromJSON (b.readFile "${sourceDir}/package.json");
@@ -43,7 +45,7 @@
               ${lib.substring failureOffset 50 tryParse.value.str}
             '';
     in
-    
+
     utils.simpleTranslate translatorName rec {
 
       inputData = parsedLock;
@@ -60,9 +62,7 @@
 
       subsystemName = "nodejs";
 
-      subsystemAttrs = {
-        nodejsVersion = 14;
-      };
+      subsystemAttrs = { nodejsVersion = args.nodejs; };
 
       mainPackageDependencies =
         lib.mapAttrsToList
@@ -116,13 +116,13 @@
         in
           lib.forEach
             dependencies
-            (dependency: 
+            (dependency:
               builtins.head (
                 lib.mapAttrsToList
                   (name: versionSpec:
                     let
                       yarnName = "${name}@${versionSpec}";
-                      depObject = dependenciesByOriginalID."${yarnName}"; 
+                      depObject = dependenciesByOriginalID."${yarnName}";
                       version = depObject.version;
                     in
                       if ! dependenciesByOriginalID ? ${yarnName} then
@@ -153,7 +153,7 @@
                   && lib.hasInfix "codeload.github.com/" dObj.resolved
 
               || lib.hasInfix "@git+" dObj.yarnName
-              
+
               # example:
               # "jest-image-snapshot@https://github.com/machard/jest-image-snapshot#machard-patch-1":
               #   version "4.2.0"
@@ -175,7 +175,7 @@
           else
             "http";
 
-      
+
       sourceConstructors = {
         git = dependencyObject:
           if utils.identifyGitUrl dependencyObject.resolved then
@@ -215,13 +215,13 @@
         path = dependencyObject:
           if lib.hasInfix "@link:" dependencyObject.yarnName then
             {
-              version = dependencyObject.version;     
+              version = dependencyObject.version;
               path =
                 lib.last (lib.splitString "@link:" dependencyObject.yarnName);
             }
           else if lib.hasInfix "@file:" dependencyObject.yarnName then
             {
-              version = dependencyObject.version;     
+              version = dependencyObject.version;
               path =
               lib.last (lib.splitString "@file:" dependencyObject.yarnName);
             }
@@ -231,7 +231,7 @@
         http = dependencyObject:
           {
             type = "http";
-            version = dependencyObject.version;  
+            version = dependencyObject.version;
             hash =
               if dependencyObject ? integrity then
                 dependencyObject.integrity
@@ -256,7 +256,7 @@
 
 
     };
-      
+
 
   # From a given list of paths, this function returns all paths which can be processed by this translator.
   # This allows the framework to detect if the translator is compatible with the given inputs
@@ -267,7 +267,7 @@
       inputFiles,
     }@args:
     {
-      inputDirectories = lib.filter 
+      inputDirectories = lib.filter
         (utils.containsMatchingFile [ ''.*yarn\.lock'' ''.*package.json'' ])
         args.inputDirectories;
 
@@ -297,6 +297,16 @@
       type = "flag";
     };
 
+    nodejs = {
+      description = "nodejs version to use for building";
+      default = lib.elemAt (lib.splitString "." nodejs.version) 0;
+      examples = [
+        "14"
+        "16"
+      ];
+      type = "argument";
+    };
+
     peer = {
       description = "Include peer dependencies";
       type = "flag";
diff --git a/src/utils/default.nix b/src/utils/default.nix
index c62575c5..25ad901f 100644
--- a/src/utils/default.nix
+++ b/src/utils/default.nix
@@ -5,7 +5,7 @@
   lib,
   nix,
   runCommand,
-  writeScriptBin,
+  writeScript,
 
   # dream2nix inputs
   callPackageDream,
@@ -15,7 +15,7 @@
 let
 
   b = builtins;
-  
+
   dreamLockUtils = callPackageDream ./dream-lock.nix {};
 
   overrideUtils = callPackageDream ./override.nix {};
@@ -58,6 +58,8 @@ rec {
 
   listFiles = path: lib.attrNames (lib.filterAttrs (n: v: v == "regular") (builtins.readDir path));
 
+  listDirs = path: lib.attrNames (lib.filterAttrs (n: v: v == "directory") (builtins.readDir path));
+
   # directory names of a given directory
   dirNames = dir: lib.attrNames (lib.filterAttrs (name: type: type == "directory") (builtins.readDir dir));
 
@@ -87,7 +89,7 @@ rec {
       b.readFile hashFile;
 
   # builder to create a shell script that has it's own PATH
-  writePureShellScript = availablePrograms: script: writeScriptBin "run" ''
+  writePureShellScript = availablePrograms: script: writeScript "script.sh" ''
     #!${bash}/bin/bash
     set -Eeuo pipefail
 
@@ -118,7 +120,7 @@ rec {
           ''
           + old.postFetch;
       });
-  
+
   sanitizeDerivationName = name:
     lib.replaceStrings [ "@" "/" ] [ "__at__" "__slash__" ] name;
 
@@ -165,7 +167,7 @@ rec {
 
   satisfiesSemver = poetry2nixSemver.satisfiesSemver;
 
-  # like nixpkgs recursiveUpdateUntil, but the depth of the 
+  # like nixpkgs recursiveUpdateUntil, but the depth of the
   recursiveUpdateUntilDepth = depth: lhs: rhs:
     lib.recursiveUpdateUntil (path: l: r: (b.length path) > depth) lhs rhs;
 
diff --git a/src/utils/dream-lock.nix b/src/utils/dream-lock.nix
index 4245baae..712922ef 100644
--- a/src/utils/dream-lock.nix
+++ b/src/utils/dream-lock.nix
@@ -9,7 +9,7 @@ let
 
   b = builtins;
 
-  readDreamLock = 
+  readDreamLock =
     {
       dreamLock,
     }@args:
@@ -49,7 +49,7 @@ let
         b.filter
           (dep: ! b.elem dep cyclicDependencies."${pname}"."${version}" or [])
           dependencyGraph."${pname}"."${version}" or [];
-      
+
       getCyclicDependencies = pname: version:
         cyclicDependencies."${pname}"."${version}" or [];
 
@@ -100,7 +100,7 @@ let
     getSubDreamLock = dreamLock: name: version:
       let
         lock = (readDreamLock { inherit dreamLock; }).lock;
-      
+
       in
         lock // {
           _generic = lock._generic // {
diff --git a/tests/defaut.nix b/tests/eval/defaut.nix
similarity index 100%
rename from tests/defaut.nix
rename to tests/eval/defaut.nix
diff --git a/tests/impure/default.nix b/tests/impure/default.nix
new file mode 100644
index 00000000..bbe861d5
--- /dev/null
+++ b/tests/impure/default.nix
@@ -0,0 +1,34 @@
+{
+  lib,
+
+  # dream2nix
+  callPackageDream,
+  utils,
+  ...
+}:
+let
+
+  l = lib // builtins;
+
+  allTestFiles =
+    l.attrNames
+      (l.filterAttrs
+        (name: type: type == "regular" && l.hasPrefix "test_" name)
+        (l.readDir ./.));
+
+  allTests =
+    l.map
+      (file: callPackageDream ("${./.}/${file}") {})
+      allTestFiles;
+
+  executeAll = utils.writePureShellScript
+    []
+    ''
+      for test in ${toString allTests}; do
+        $test
+      done
+    '';
+
+
+in
+  executeAll
diff --git a/tests/impure/test_package-lock.nix b/tests/impure/test_package-lock.nix
new file mode 100644
index 00000000..9a1421f2
--- /dev/null
+++ b/tests/impure/test_package-lock.nix
@@ -0,0 +1,27 @@
+{
+  lib,
+
+  # dream2nix
+  apps,
+  utils,
+  ...
+}:
+let
+
+  l = lib // builtins;
+
+  cli = apps.cli.program;
+
+in
+utils.writePureShellScript
+[]
+''
+  ${cli} add github:mattermost/mattermost-webapp/v6.1.0 \
+    --no-default-nix \
+    --translator package-lock \
+    --attribute-name mattermost-webapp \
+    --arg name="{automatic}" \
+    --arg noDev=false \
+    --arg nodejs=14 \
+    --arg peer=false
+''
diff --git a/tests/impure/test_yarn-lock-prettier.nix b/tests/impure/test_yarn-lock-prettier.nix
new file mode 100644
index 00000000..e1dd7752
--- /dev/null
+++ b/tests/impure/test_yarn-lock-prettier.nix
@@ -0,0 +1,27 @@
+{
+  lib,
+
+  # dream2nix
+  apps,
+  utils,
+  ...
+}:
+let
+
+  l = lib // builtins;
+
+  cli = apps.cli.program;
+
+in
+utils.writePureShellScript
+[]
+''
+  ${cli} add github:prettier/prettier/2.4.1 \
+    --no-default-nix \
+    --translator yarn-lock \
+    --attribute-name prettier \
+    --arg name="{automatic}" \
+    --arg noDev=false \
+    --arg nodejs=14 \
+    --arg peer=false
+''
diff --git a/checks.nix b/tests/pure/default.nix
similarity index 74%
rename from checks.nix
rename to tests/pure/default.nix
index 2fb1bd1b..fc2ab832 100644
--- a/checks.nix
+++ b/tests/pure/default.nix
@@ -28,9 +28,15 @@ let
         url = "https://github.com/prettier/prettier/tarball/2.4.1";
         sha256 = "19b37qakhlsnr2n5bgv83aih5npgzbad1d2p2rs3zbq5syqbxdyi";
       };
-      cmds = outputs: [
-        "${outputs.defaultPackage}/bin/prettier --version | grep -q 2.4.1 && mkdir $out"
-      ];
+      cmds = outputs:
+        let
+          prettier = outputs.defaultPackage.overrideAttrs (old: {
+            dontBuild = true;
+          });
+        in
+          [
+            "${prettier}/bin/prettier --version | grep -q 2.4.1 && mkdir $out"
+          ];
     };
   };
 
@@ -38,6 +44,6 @@ let
     lib.mapAttrs
       (name: args: makeTest (args // { inherit name; }))
       projects;
-  
+
 in
 allTests

From 2f4ac1e6c29a9c1295d7215c4c6c26c5f832aac0 Mon Sep 17 00:00:00 2001
From: DavHau <hsngrmpf+github@gmail.com>
Date: Wed, 17 Nov 2021 11:45:04 +0700
Subject: [PATCH 4/8] add tests for gomod2nix

---
 tests/impure/test_go.nix           | 23 +++++++++++++++++++++++
 tests/impure/test_package-lock.nix |  3 +--
 2 files changed, 24 insertions(+), 2 deletions(-)
 create mode 100644 tests/impure/test_go.nix

diff --git a/tests/impure/test_go.nix b/tests/impure/test_go.nix
new file mode 100644
index 00000000..689250cd
--- /dev/null
+++ b/tests/impure/test_go.nix
@@ -0,0 +1,23 @@
+{
+  lib,
+
+  # dream2nix
+  apps,
+  utils,
+  ...
+}:
+let
+
+  l = lib // builtins;
+
+  cli = apps.cli.program;
+
+in
+utils.writePureShellScript
+[]
+''
+  ${cli} add github:tweag/gomod2nix/67f22dd738d092c6ba88e420350ada0ed4992ae8 \
+    --no-default-nix \
+    --translator gomod2nix \
+    --attribute-name gomod2nix
+''
diff --git a/tests/impure/test_package-lock.nix b/tests/impure/test_package-lock.nix
index 9a1421f2..22ce7a1d 100644
--- a/tests/impure/test_package-lock.nix
+++ b/tests/impure/test_package-lock.nix
@@ -22,6 +22,5 @@ utils.writePureShellScript
     --attribute-name mattermost-webapp \
     --arg name="{automatic}" \
     --arg noDev=false \
-    --arg nodejs=14 \
-    --arg peer=false
+    --arg nodejs=14
 ''

From a156f7e705f16ec2c222ae7104d2cef367308880 Mon Sep 17 00:00:00 2001
From: DavHau <hsngrmpf+github@gmail.com>
Date: Wed, 17 Nov 2021 12:12:49 +0700
Subject: [PATCH 5/8] add github actions

---
 .github/workflows/tests.yml | 32 ++++++++++++++++++++++++++++++++
 .gitignore                  |  1 +
 2 files changed, 33 insertions(+)
 create mode 100644 .github/workflows/tests.yml

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
new file mode 100644
index 00000000..30c95b55
--- /dev/null
+++ b/.github/workflows/tests.yml
@@ -0,0 +1,32 @@
+name: "Test dream2nix"
+on:
+  pull_request:
+  push:
+
+jobs:
+
+  pure-tests:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2.4.0
+    - uses: cachix/install-nix-action@v15
+      with:
+        install_url: https://nixos-nix-install-tests.cachix.org/serve/w659aglf1hfvkj5wj696q9x8r19p6b7k/install
+        install_options: '--tarball-url-prefix https://nixos-nix-install-tests.cachix.org/serve'
+        extra_nix_config: |
+          access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}
+
+    - run: nix flake check
+
+  impure-tests:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2.4.0
+    - uses: cachix/install-nix-action@v15
+      with:
+        install_url: https://nixos-nix-install-tests.cachix.org/serve/w659aglf1hfvkj5wj696q9x8r19p6b7k/install
+        install_options: '--tarball-url-prefix https://nixos-nix-install-tests.cachix.org/serve'
+        extra_nix_config: |
+          access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}
+
+    - run: nix run .#tests-impure
diff --git a/.gitignore b/.gitignore
index 0a5c8687..a5bef45b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
 .*/
+!.github/
 result
 interpreter

From 87157ecc8b0d1c4b5370187df350cfcdf056356e Mon Sep 17 00:00:00 2001
From: DavHau <hsngrmpf+github@gmail.com>
Date: Wed, 17 Nov 2021 12:18:22 +0700
Subject: [PATCH 6/8] fix NIX_PATH impurity in writePureShellScript

---
 src/utils/default.nix | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/utils/default.nix b/src/utils/default.nix
index 25ad901f..6ddfcd8d 100644
--- a/src/utils/default.nix
+++ b/src/utils/default.nix
@@ -4,6 +4,7 @@
   fetchzip,
   lib,
   nix,
+  pkgs,
   runCommand,
   writeScript,
 
@@ -94,6 +95,8 @@ rec {
     set -Eeuo pipefail
 
     export PATH="${lib.makeBinPath availablePrograms}"
+    export NIX_PATH=nixpkgs=${pkgs.path}
+
     tmpdir=$(${coreutils}/bin/mktemp -d)
     cd $tmpdir
 

From a2f5f0bb674261eda4b03c26606c944821bf216b Mon Sep 17 00:00:00 2001
From: DavHau <hsngrmpf+github@gmail.com>
Date: Wed, 17 Nov 2021 12:25:03 +0700
Subject: [PATCH 7/8] disable hercules-ci

Runs outdated nix version which cannot fetch git submodules
---
 ci.nix | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/ci.nix b/ci.nix
index de47ba28..f2558898 100644
--- a/ci.nix
+++ b/ci.nix
@@ -6,12 +6,17 @@ let
   recurseIntoAll = b.mapAttrs (name: val: pkgs.recurseIntoAttrs val);
 
 in
+# {
+#   inherit flake;
+# }
+
+# // (recurseIntoAll {
+
+#   checks = flake.checks.x86_64-linux;
+
+# })
+
+# hercules ci's nix version cannot fetch submodules and crashes
 {
-  inherit flake;
+  inherit (pkgs) hello;
 }
-
-// (recurseIntoAll {
-
-  checks = flake.checks.x86_64-linux;
-
-})

From 06289a42bc05908be09127aeeb194d1a86c1c4b6 Mon Sep 17 00:00:00 2001
From: DavHau <hsngrmpf+github@gmail.com>
Date: Wed, 17 Nov 2021 12:38:07 +0700
Subject: [PATCH 8/8] push to cachix

---
 .github/workflows/tests.yml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 30c95b55..f357195e 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -15,6 +15,10 @@ jobs:
         install_options: '--tarball-url-prefix https://nixos-nix-install-tests.cachix.org/serve'
         extra_nix_config: |
           access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}
+    - uses: cachix/cachix-action@v10
+      with:
+        name: nix-community
+        signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
 
     - run: nix flake check
 
@@ -28,5 +32,9 @@ jobs:
         install_options: '--tarball-url-prefix https://nixos-nix-install-tests.cachix.org/serve'
         extra_nix_config: |
           access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}
+    - uses: cachix/cachix-action@v10
+      with:
+        name: nix-community
+        signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
 
     - run: nix run .#tests-impure