automatic unit tests for pure translators

- translators can now enable unit tests via `generateUnitTestsForProjects`

src/apps/cli/nix_ffi.py

@@ -9,6 +9,7 @@ dream2nix_src = os.environ.get("dream2nixSrc")
 def nix(*args, **kwargs):
   return sp.run(["nix", "--option", "experimental-features", "nix-command flakes"] + list(args), capture_output=True, **kwargs)
 
+# TODO: deprecate and replace all usage with `eval()` (see below).
 def callNixFunction(function_path, **kwargs):
   with tempfile.NamedTemporaryFile("w") as input_json_file:
     json.dump(dict(**kwargs), input_json_file, indent=2)
@@ -37,6 +38,48 @@ def callNixFunction(function_path, **kwargs):
   # parse result data
   return json.loads(proc.stdout)
 
+def eval(attr_path, wrapper_code=None, **kwargs):
+  if wrapper_code == None:
+    # dummy wrapper code
+    wrapper_code = "{result}: result"
+
+  is_function_call = len(kwargs) > 0
+
+  with tempfile.NamedTemporaryFile("w") as input_json_file:
+    json.dump(dict(**kwargs), input_json_file, indent=2)
+    input_json_file.seek(0) # flushes write cache
+    env = os.environ.copy()
+    env.update(dict(
+      FUNC_ARGS=input_json_file.name
+    ))
+    with tempfile.NamedTemporaryFile("w") as wrapper_code_file:
+      wrapper_code_file.write(wrapper_code)
+      wrapper_code_file.seek(0) # flushes write cache
+      proc = nix(
+        "eval", "--show-trace", "--impure", "--raw", "--expr",
+        f'''
+          let
+            d2n = (import {dream2nix_src} {{}});
+            wrapper = import {wrapper_code_file.name};
+            result =
+              if "{is_function_call}" == "True"
+              then d2n.utils.callViaEnv d2n.{attr_path}
+              else d2n.{attr_path};
+          in
+            builtins.toJSON (
+              wrapper {{ inherit result; }}
+            )
+        ''',
+        env=env
+      )
+  if proc.returncode:
+    print(f"Failed evaluating '{attr_path}'", file=sys.stderr)
+    print(proc.stderr.decode(), file=sys.stderr)
+    exit(1)
+
+  # parse result data
+  return json.loads(proc.stdout)
+
 
 def buildNixFunction(function_path, **kwargs):
   with tempfile.NamedTemporaryFile("w") as input_json_file:

src/default.nix

@@ -485,10 +485,17 @@
       l.forEach projectsPureUnresolved
       (proj: let
         translator = getTranslator proj.subsystem proj.translator;
-        dreamLock' = translator.translate {
+        dreamLock'' = translator.translate {
           inherit source tree discoveredProjects;
           project = proj;
         };
+
+        /*
+         simpleTranslate2 exposes result via `.result` in order to allow for
+         unit testing via `.inputs`.
+         */
+        dreamLock' = dreamLock''.result or dreamLock'';
+
         dreamLock =
           dreamLock'
           // {

src/lib/simpleTranslate2.nix

@@ -78,6 +78,22 @@
       subsystemAttrs ? {},
       translatorName,
     }: let
+      inputs = {
+        inherit
+          defaultPackage
+          exportedPackages
+          extractors
+          extraDependencies
+          extraObjects
+          keys
+          location
+          serializedRawObjects
+          subsystemName
+          subsystemAttrs
+          translatorName
+          ;
+      };
+
       allDependencies =
         l.foldl'
         (result: finalObj:
@@ -217,28 +233,35 @@
           ))
           {}
           cyclesList;
-    in
-      {
-        decompressed = true;
 
-        _generic = {
-          inherit
-            defaultPackage
-            location
-            ;
-          packages = exportedPackages;
-          subsystem = subsystemName;
-          sourcesAggregatedHash = null;
-        };
+      data =
+        {
+          decompressed = true;
 
-        # build system specific attributes
-        _subsystem = subsystemAttrs;
+          _generic = {
+            inherit
+              defaultPackage
+              location
+              ;
+            packages = exportedPackages;
+            subsystem = subsystemName;
+            sourcesAggregatedHash = null;
+          };
 
-        inherit cyclicDependencies sources;
-      }
-      // {dependencies = dependencyGraph;};
-  in
-    dreamLockData;
+          # build system specific attributes
+          _subsystem = subsystemAttrs;
+
+          inherit cyclicDependencies sources;
+        }
+        // {dependencies = dependencyGraph;};
+    in {
+      inherit data;
+      inherit inputs;
+    };
+  in {
+    result = dreamLockData.data;
+    inputs = dreamLockData.inputs;
+  };
 in {
   inherit simpleTranslate2;
 }

src/templates/translators/impure.nix

@@ -6,12 +6,13 @@
   # The executable will be called by dream2nix for translation
   # The input format is specified in /specifications/translator-call-example.json.
   # The first arg `$1` will be a json file containing the input parameters
-  # like defined in /specifications/translator-call-example.json and the
+  # like defined in /src/specifications/translator-call-example.json and the
   # additional arguments required according to extraArgs
   #
   # The program is expected to create a file at the location specified
   # by the input parameter `outFile`.
   # The output file must contain the dream lock data encoded as json.
+  # See /src/specifications/dream-lock-example.json
   translateBin = {
     # dream2nix utils
     utils,
@@ -39,10 +40,11 @@
 
       # TODO:
       # read input files/dirs and produce a json file at $outputFile
-      # containing the dream lock similar to /specifications/dream-lock-example.json
+      # containing the dream lock similar to /src/specifications/dream-lock-example.json
     '';
 
   # If the translator requires additional arguments, specify them here.
+  # When users run the CLI, they will be asked to specify these arguments.
   # There are only two types of arguments:
   #   - string argument (type = "argument")
   #   - boolean flag (type = "flag")

src/templates/translators/pure.nix

@@ -4,6 +4,23 @@
 }:
 
 {
+
+  /*
+    Automatically generate unit tests for this translator using project sources
+    from the specified list.
+
+    !!! Your first action should be adding a project here. This will simplify
+    your work because you will be able to use `nix run .#tests-unit` to
+    test your implementation for correctness.
+  */
+  generateUnitTestsForProjects = [
+    (builtins.fetchTarball {
+      url = "";
+      sha256 = "";
+    })
+  ];
+
+  # translate from a given source and a project specification to a dream-lock.
   translate =
     {
       translatorName,
@@ -219,11 +236,12 @@
         });
 
   # If the translator requires additional arguments, specify them here.
-  # When users run the CLI, they will be asked to specify these arguments.
+  # Users will be able to set these arguments via `settings`.
   # There are only two types of arguments:
   #   - string argument (type = "argument")
   #   - boolean flag (type = "flag")
   # String arguments contain a default value and examples. Flags do not.
+  # Flags are false by default.
   extraArgs = {
 
     # Example: boolean option

src/translators/nodejs/pure/yarn-lock/default.nix

@@ -97,7 +97,7 @@
             ? resolved
             && lib.hasInfix "codeload.github.com/" dObj.resolved)
           || (lib.hasInfix "@git+" dObj.yarnName
-            || lib.hasPrefix "git+" dObj.resolved)
+            || lib.hasPrefix "git+" (dObj.resolved or ""))
           # example:
           # "jest-image-snapshot@https://github.com/machard/jest-image-snapshot#machard-patch-1":
           #   version "4.2.0"
@@ -140,7 +140,7 @@
         name = rawObj: finalObj:
           if
             (lib.hasInfix "@git+" rawObj.yarnName
-              || lib.hasPrefix "git+" rawObj.resolved)
+              || lib.hasPrefix "git+" (rawObj.resolved or ""))
           then lib.head (lib.splitString "@git+" rawObj.yarnName)
           # Example:
           # @matrix-org/olm@https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.3.tgz
@@ -311,6 +311,13 @@ in {
 
   inherit translate;
 
+  generateUnitTestsForProjects = [
+    (builtins.fetchTarball {
+      url = "https://github.com/prettier/prettier/tarball/66585d9f250f11569456421e66a2407397e98f69";
+      sha256 = "14fqwavb04b1ws1s58cmwq7wqj9xif4pv166ab23qpgnq57629yy";
+    })
+  ];
+
   # If the translator requires additional arguments, specify them here.
   # There are only two types of arguments:
   #   - string argument (type = "argument")

tests/unit/default.nix

@@ -1,6 +1,7 @@
 {
   self,
   lib,
+  coreutils,
   nix,
   python3,
   utils,
@@ -8,12 +9,19 @@
   ...
 }: let
   l = lib // builtins;
+
+  pythonEnv = python3.withPackages (ps:
+    with ps; [
+      pytest
+      pytest-xdist
+    ]);
 in
   utils.writePureShellScript
   [
+    coreutils
     nix
   ]
   ''
     export dream2nixSrc=${dream2nixWithExternals}
-    ${python3.pkgs.pytest}/bin/pytest ${self}/tests/unit -v "$@"
+    ${pythonEnv}/bin/pytest ${self}/tests/unit -n $(nproc) -v "$@"
   ''

tests/unit/test_pure_translators.py

@@ -0,0 +1,311 @@
+import nix_ffi
+import os
+import pytest
+
+
+def get_projects_to_test():
+  tests = nix_ffi.eval(
+    'translators.translators',
+    wrapper_code = '''
+      {result}: let
+        l = lib // builtins;
+        lib = (import <nixpkgs> {}).lib;
+        mapAttrsToList = f: attrs: l.attrValues (l.mapAttrs f attrs);
+      in
+        l.flatten (l.flatten (l.flatten
+          (mapAttrsToList
+          (subsystem: types:
+            mapAttrsToList
+            (type: names:
+              mapAttrsToList
+              (name: translator:
+                l.map
+                  (source: {
+                    source = l.toString source;
+                    translator = name;
+                    inherit subsystem type;
+                  })
+                  (translator.generateUnitTestsForProjects or []))
+              names)
+            types)
+          result)))
+    ''',
+  )
+  result = []
+  for test in tests:
+    if test['type'] == 'all':
+      continue
+    result.append(dict(
+      project = dict(
+        name="test",
+        relPath="",
+        translator=test['translator'],
+        subsystemInfo={},
+      ),
+      translator=test['translator'],
+      source = test['source'],
+      subsystem = test['subsystem'],
+      type = test['type'],
+    ))
+  return result
+
+
+projects = get_projects_to_test()
+
+
+def check_format_dependencies(dependencies):
+  assert isinstance(dependencies, list)
+  for dep in dependencies:
+    assert set(dep.keys()) == {'name', 'version'}
+    assert isinstance(dep['name'], str)
+    assert len(dep['name']) > 0
+    assert isinstance(dep['version'], str)
+    assert len(dep['version']) > 0
+
+def check_format_sourceSpec(sourceSpec):
+  assert isinstance(sourceSpec, dict)
+  assert 'type' in sourceSpec
+
+@pytest.mark.parametrize("p", projects)
+def test_packageName(p):
+  defaultPackage = nix_ffi.eval(
+    f"translators.translators.{p['subsystem']}.{p['type']}.{p['translator']}.translate",
+    params=dict(
+      project=p['project'],
+      source=p['source'],
+    ),
+    wrapper_code = '''
+      {result}:
+      result.inputs.defaultPackage
+    ''',
+  )
+  assert isinstance(defaultPackage, str)
+  assert len(defaultPackage) > 0
+
+@pytest.mark.parametrize("p", projects)
+def test_exportedPackages(p):
+  exportedPackages = nix_ffi.eval(
+    f"translators.translators.{p['subsystem']}.{p['type']}.{p['translator']}.translate",
+    params=dict(
+      project=p['project'],
+      source=p['source'],
+    ),
+    wrapper_code = '''
+      {result}:
+      result.inputs.exportedPackages
+    ''',
+  )
+  assert isinstance(exportedPackages, dict)
+  assert len(exportedPackages) > 0
+
+@pytest.mark.parametrize("p", projects)
+def test_extraDependencies(p):
+  extraDependencies = nix_ffi.eval(
+    f"translators.translators.{p['subsystem']}.{p['type']}.{p['translator']}.translate",
+    params=dict(
+      project=p['project'],
+      source=p['source'],
+    ),
+    wrapper_code = '''
+      {result}:
+      result.inputs.extraDependencies
+    ''',
+  )
+  assert isinstance(extraDependencies, list)
+  for extra_dep in extraDependencies:
+    assert set(extra_dep.keys()) == {"dependencies", "name", "version"}
+    assert isinstance(extra_dep['name'], str)
+    assert len(extra_dep['name']) > 0
+    assert isinstance(extra_dep['version'], str)
+    assert len(extra_dep['version']) > 0
+    check_format_dependencies(extra_dep['dependencies'])
+
+@pytest.mark.parametrize("p", projects)
+def test_extraObjects(p):
+  extraObjects = nix_ffi.eval(
+    f"translators.translators.{p['subsystem']}.{p['type']}.{p['translator']}.translate",
+    params=dict(
+      project=p['project'],
+      source=p['source'],
+    ),
+    wrapper_code = '''
+      {result}:
+      result.inputs.extraObjects
+    ''',
+  )
+  assert isinstance(extraObjects, list)
+  for extra_obj in extraObjects:
+    assert set(extra_obj.keys()) == \
+      {'name', 'version', 'dependencies', 'sourceSpec'}
+    assert isinstance(extra_obj['name'], str)
+    assert len(extra_obj['name']) > 0
+    assert isinstance(extra_obj['version'], str)
+    assert len(extra_obj['version']) > 0
+    check_format_dependencies(extra_obj['dependencies'])
+    check_format_sourceSpec(extra_obj['sourceSpec'])
+
+@pytest.mark.parametrize("p", projects)
+def test_location(p):
+  location = nix_ffi.eval(
+    f"translators.translators.{p['subsystem']}.{p['type']}.{p['translator']}.translate",
+    params=dict(
+      project=p['project'],
+      source=p['source'],
+    ),
+    wrapper_code = '''
+      {result}:
+      result.inputs.location
+    ''',
+  )
+  assert isinstance(location, str)
+
+@pytest.mark.parametrize("p", projects)
+def test_serializedRawObjects(p):
+  serializedRawObjects = nix_ffi.eval(
+    f"translators.translators.{p['subsystem']}.{p['type']}.{p['translator']}.translate",
+    params=dict(
+      project=p['project'],
+      source=p['source'],
+    ),
+    wrapper_code = '''
+      {result}:
+
+      result.inputs.serializedRawObjects
+    ''',
+  )
+  assert isinstance(serializedRawObjects, list)
+  for raw_obj in serializedRawObjects:
+    assert isinstance(raw_obj, dict)
+
+@pytest.mark.parametrize("p", projects)
+def test_subsystemName(p):
+  subsystemName = nix_ffi.eval(
+    f"translators.translators.{p['subsystem']}.{p['type']}.{p['translator']}.translate",
+    params=dict(
+      project=p['project'],
+      source=p['source'],
+    ),
+    wrapper_code = '''
+      {result}:
+      result.inputs.subsystemName
+    ''',
+  )
+  assert isinstance(subsystemName, str)
+  assert len(subsystemName) > 0
+
+@pytest.mark.parametrize("p", projects)
+def test_subsystemAttrs(p):
+  subsystemAttrs = nix_ffi.eval(
+    f"translators.translators.{p['subsystem']}.{p['type']}.{p['translator']}.translate",
+    params=dict(
+      project=p['project'],
+      source=p['source'],
+    ),
+    wrapper_code = '''
+      {result}:
+      result.inputs.subsystemAttrs
+    ''',
+  )
+  assert isinstance(subsystemAttrs, dict)
+
+@pytest.mark.parametrize("p", projects)
+def test_translatorName(p):
+  translatorName = nix_ffi.eval(
+    f"translators.translators.{p['subsystem']}.{p['type']}.{p['translator']}.translate",
+    params=dict(
+      project=p['project'],
+      source=p['source'],
+    ),
+    wrapper_code = '''
+      {result}:
+      result.inputs.translatorName
+    ''',
+  )
+  assert isinstance(translatorName, str)
+  assert len(translatorName) > 0
+
+@pytest.mark.parametrize("p", projects)
+def test_extractors(p):
+  finalObjects = nix_ffi.eval(
+    f"translators.translators.{p['subsystem']}.{p['type']}.{p['translator']}.translate",
+    params=dict(
+      project=p['project'],
+      source=p['source'],
+    ),
+    wrapper_code = '''
+      {result}:
+      let
+        l = builtins;
+        inputs = result.inputs;
+        rawObjects = inputs.serializedRawObjects;
+
+        finalObjects =
+          l.map
+          (rawObj: let
+            finalObj =
+              l.mapAttrs
+              (key: extractFunc: extractFunc rawObj finalObj)
+              inputs.extractors;
+          in
+            finalObj)
+          rawObjects;
+      in
+        finalObjects ++ (inputs.extraObjects or [])
+    ''',
+  )
+  assert isinstance(finalObjects, list)
+  assert len(finalObjects) > 0
+  for finalObj in finalObjects:
+    assert set(finalObj.keys()) == \
+      {'name', 'version', 'sourceSpec', 'dependencies'}
+    check_format_dependencies(finalObj['dependencies'])
+    check_format_sourceSpec(finalObj['sourceSpec'])
+
+@pytest.mark.parametrize("p", projects)
+def test_keys(p):
+  objectsByKey = nix_ffi.eval(
+    f"translators.translators.{p['subsystem']}.{p['type']}.{p['translator']}.translate",
+    params=dict(
+      project=p['project'],
+      source=p['source'],
+    ),
+    wrapper_code = '''
+      {result}:
+      let
+        l = builtins;
+        inputs = result.inputs;
+        rawObjects = inputs.serializedRawObjects;
+
+        finalObjects =
+          l.map
+          (rawObj: let
+            finalObj =
+              {inherit rawObj;}
+              // l.mapAttrs
+              (key: extractFunc: extractFunc rawObj finalObj)
+              inputs.extractors;
+          in
+            finalObj)
+          rawObjects;
+
+        objectsByKey =
+          l.mapAttrs
+          (key: keyFunc:
+            l.foldl'
+            (merged: finalObj:
+              merged
+              // {"${keyFunc finalObj.rawObj finalObj}" = finalObj;})
+            {}
+            (finalObjects ++ (inputs.extraObjects or [])))
+          inputs.keys;
+      in
+        objectsByKey
+    ''',
+  )
+  assert isinstance(objectsByKey, dict)
+  for key_name, objects in objectsByKey.items():
+    for finalObj in objects.values():
+      assert set(finalObj.keys()) == \
+        {'name', 'version', 'sourceSpec', 'dependencies', 'rawObj'}
+      check_format_dependencies(finalObj['dependencies'])
+      check_format_sourceSpec(finalObj['sourceSpec'])