simpleTranslate2: improve unit tests. Be more lazy

src/apps/cli/nix_ffi.py

@@ -41,7 +41,7 @@ def callNixFunction(function_path, **kwargs):
 def eval(attr_path, wrapper_code=None, **kwargs):
   if wrapper_code == None:
     # dummy wrapper code
-    wrapper_code = "{result}: result"
+    wrapper_code = "{result, ...}: result"
 
   is_function_call = len(kwargs) > 0
 
@@ -59,15 +59,21 @@ def eval(attr_path, wrapper_code=None, **kwargs):
         "eval", "--show-trace", "--impure", "--raw", "--expr",
         f'''
           let
+            b = builtins;
             d2n = (import {dream2nix_src} {{}});
-            wrapper = import {wrapper_code_file.name};
-            result =
+            result' =
               if "{is_function_call}" == "True"
               then d2n.utils.callViaEnv d2n.{attr_path}
               else d2n.{attr_path};
+            result = (d2n.callPackageDream
+              {wrapper_code_file.name}
+              {{ result = result'; }});
           in
-            builtins.toJSON (
-              wrapper {{ inherit result; }}
+            b.toJSON (
+              # remove override attributes added by callPackage
+              if b.isAttrs result
+              then b.removeAttrs result ["override" "overrideDerivation"]
+              else result
             )
         ''',
         env=env

src/lib/default.nix

@@ -53,7 +53,7 @@
   modules = import ./modules.nix {inherit config dlib lib;};
 
   simpleTranslate2 =
-    (import ./simpleTranslate2.nix {inherit dlib lib;}).simpleTranslate2;
+    import ./simpleTranslate2.nix {inherit dlib lib;};
 
   parseUtils = import ./parsing.nix {inherit lib;};
 

src/lib/simpleTranslate2.nix

@@ -5,7 +5,88 @@
 }: let
   l = lib // builtins;
 
-  simpleTranslate2 = func: let
+  expectedFields = [
+    "name"
+    "version"
+    "sourceSpec"
+    "dependencies"
+  ];
+
+  mkFinalObjects = rawObjects: extractors:
+    l.map
+    (rawObj: let
+      finalObj =
+        {inherit rawObj;}
+        // l.mapAttrs
+        (key: extractFunc: extractFunc rawObj finalObj)
+        extractors;
+    in
+      finalObj)
+    rawObjects;
+
+  # checks validity of all objects by iterating over them
+  mkValidatedFinalObjects = finalObjects: translatorName: extraObjects:
+    l.map
+    (finalObj:
+      if l.any (field: ! finalObj ? "${field}") expectedFields
+      then
+        throw
+        ''
+          Translator ${translatorName} failed.
+          The following object does not contain all required fields:
+          Object:
+            ${l.toJSON finalObj}
+          Missing fields:
+            ${l.subtractLists expectedFields (l.attrNames finalObj)}
+        ''
+      # TODO: validate sourceSpec as well
+      else finalObj)
+    (finalObjects ++ extraObjects);
+
+  mkExportedFinalObjects = finalObjects: exportedPackages:
+    l.filter
+    (finalObj:
+      exportedPackages.${finalObj.name} or null == finalObj.version)
+    finalObjects;
+
+  mkRelevantFinalObjects = exportedFinalObjects: allDependencies:
+    l.genericClosure {
+      startSet =
+        l.map
+        (finalObj:
+          finalObj
+          // {key = "${finalObj.name}#${finalObj.version}";})
+        exportedFinalObjects;
+      operator = finalObj:
+        l.map
+        (c:
+          allDependencies.${c.name}.${c.version}
+          // {key = "${c.name}#${c.version}";})
+        finalObj.dependencies;
+    };
+
+  /*
+   format:
+   {
+     foo = {
+       "1.0.0" = finalObj
+     }
+   }
+   */
+  makeDependencies = finalObjects:
+    l.foldl'
+    (result: finalObj:
+      lib.recursiveUpdate
+      result
+      {
+        "${finalObj.name}" = {
+          "${finalObj.version}" = finalObj;
+        };
+      })
+    {}
+    finalObjects;
+
+  translate = func: let
     final =
       func
       {
@@ -14,43 +95,7 @@
 
     rawObjects = final.serializedRawObjects;
 
-    expectedFields = [
-      "name"
-      "version"
-      "sourceSpec"
-      "dependencies"
-    ];
-
-    finalObjects' =
-      l.map
-      (rawObj: let
-        finalObj =
-          {inherit rawObj;}
-          // l.mapAttrs
-          (key: extractFunc: extractFunc rawObj finalObj)
-          final.extractors;
-      in
-        finalObj)
-      rawObjects;
-
-    # checks validity of all objects by iterating over them
-    finalObjects =
-      l.map
-      (finalObj:
-        if l.any (field: ! finalObj ? "${field}") expectedFields
-        then
-          throw
-          ''
-            Translator ${final.translatorName} failed.
-            The following object does not contain all required fields:
-            Object:
-              ${l.toJSON finalObj}
-            Missing fields:
-              ${l.subtractLists expectedFields (l.attrNames finalObj)}
-          ''
-        # TODO: validate sourceSpec as well
-        else finalObj)
-      (finalObjects' ++ (final.extraObjects or []));
+    finalObjects' = mkFinalObjects rawObjects final.extractors;
 
     objectsByKey =
       l.mapAttrs
@@ -92,49 +137,19 @@
           ;
       };
 
-      /*
-       format:
-       {
-         foo = {
-           "1.0.0" = finalObj
-         }
-       }
-       */
-      makeDependencies = finalObjects:
-        l.foldl'
-        (result: finalObj:
-          lib.recursiveUpdate
-          result
-          {
-            "${finalObj.name}" = {
-              "${finalObj.version}" = finalObj;
-            };
-          })
-        {}
-        finalObjects;
+      finalObjects =
+        mkValidatedFinalObjects
+        finalObjects'
+        translatorName
+        (final.extraObjects or []);
 
       allDependencies = makeDependencies finalObjects;
 
       exportedFinalObjects =
-        l.filter
-        (finalObj:
-          exportedPackages.${finalObj.name} or null == finalObj.version)
-        finalObjects;
+        mkExportedFinalObjects finalObjects exportedPackages;
 
-      relevantFinalObjects = l.genericClosure {
-        startSet =
-          l.map
-          (finalObj:
-            finalObj
-            // {key = "${finalObj.name}#${finalObj.version}";})
-          exportedFinalObjects;
-        operator = finalObj:
-          l.map
-          (c:
-            allDependencies.${c.name}.${c.version}
-            // {key = "${c.name}#${c.version}";})
-          finalObj.dependencies;
-      };
+      relevantFinalObjects =
+        mkRelevantFinalObjects exportedFinalObjects allDependencies;
 
       relevantDependencies = makeDependencies relevantFinalObjects;
 
@@ -275,5 +290,11 @@
     inputs = dreamLockData.inputs;
   };
 in {
-  inherit simpleTranslate2;
+  inherit
+    translate
+    mkFinalObjects
+    mkExportedFinalObjects
+    mkRelevantFinalObjects
+    makeDependencies
+    ;
 }

src/subsystems/haskell/translators/stack-lock/default.nix

@@ -239,7 +239,7 @@ in {
             && (objectsByName ? ${name}))
         depNames;
     in
-      dlib.simpleTranslate2
+      dlib.simpleTranslate2.translate
       ({objectsByKey, ...}: rec {
         inherit translatorName;
 

src/subsystems/nodejs/translators/yarn-lock/default.nix

@@ -46,7 +46,7 @@
 
     workspacesPackageJson = nodejsUtils.getWorkspacePackageJson tree workspaces;
   in
-    dlib.simpleTranslate2
+    dlib.simpleTranslate2.translate
     ({objectsByKey, ...}: let
       makeWorkspaceExtraObject = workspace: let
         json = workspacesPackageJson."${workspace}";

src/subsystems/rust/translators/cargo-lock/default.nix

@@ -184,7 +184,7 @@ in {
         or (l.warn "no version found in Cargo.toml for ${name}, defaulting to unknown" "unknown");
     };
   in
-    dlib.simpleTranslate2
+    dlib.simpleTranslate2.translate
     ({...}: rec {
       inherit translatorName;
 

src/templates/translators/pure.nix

@@ -135,7 +135,7 @@ in
 
     in
 
-      dlib.simpleTranslate2
+      dlib.simpleTranslate2.translate
         ({
           objectsByKey,
           ...

tests/examples/default.nix

@@ -40,7 +40,7 @@ in
   ]
   ''
     if [ -z ''${1+x} ]; then
-      parallel -j$(nproc) -a <(ls ${examples}) ${testScript}
+      parallel --halt now,fail=1 -j$(nproc) -a <(ls ${examples}) ${testScript}
     else
       arg1=$1
       shift

tests/unit/test_pure_translators copy.py

@@ -0,0 +1,281 @@
+import nix_ffi
+import os
+import pytest
+
+
+def get_projects_to_test():
+  tests = nix_ffi.eval(
+    'subsystems.allTranslators',
+    wrapper_code = '''
+      {result, ...}: let
+        lib = (import <nixpkgs> {}).lib;
+        l = lib // builtins;
+      in
+        l.flatten (
+          l.map
+          (
+            translator:
+              l.map
+                (source: {
+                  source = l.toString source;
+                  translator = translator.name;
+                  inherit (translator) subsystem type;
+                })
+                (translator.generateUnitTestsForProjects or [])
+          )
+          result
+        )
+    ''',
+  )
+  result = []
+  for test in tests:
+    if test['type'] == 'all':
+      continue
+    result.append(dict(
+      project = dict(
+        name="test",
+        relPath="",
+        translator=test['translator'],
+        subsystemInfo={},
+      ),
+      translator=test['translator'],
+      source = test['source'],
+      subsystem = test['subsystem'],
+      type = test['type'],
+    ))
+  return result
+
+
+projects = get_projects_to_test()
+
+
+def check_format_dependencies(dependencies):
+  assert isinstance(dependencies, list)
+  for dep in dependencies:
+    assert set(dep.keys()) == {'name', 'version'}
+    assert isinstance(dep['name'], str)
+    assert len(dep['name']) > 0
+    assert isinstance(dep['version'], str)
+    assert len(dep['version']) > 0
+
+def check_format_sourceSpec(sourceSpec):
+  assert isinstance(sourceSpec, dict)
+  assert 'type' in sourceSpec
+
+@pytest.mark.parametrize("p", projects)
+def test_packageName(p):
+  defaultPackage = nix_ffi.eval(
+    f"subsystems.{p['subsystem']}.translators.{p['translator']}.translate",
+    params=dict(
+      project=p['project'],
+      source=p['source'],
+    ),
+    wrapper_code = '''
+      {result, ...}:
+      result.inputs.defaultPackage
+    ''',
+  )
+  assert isinstance(defaultPackage, str)
+  assert len(defaultPackage) > 0
+
+# @pytest.mark.parametrize("p", projects)
+# def test_exportedPackages(p):
+#   exportedPackages = nix_ffi.eval(
+#     f"subsystems.{p['subsystem']}.translators.{p['translator']}.translate",
+#     params=dict(
+#       project=p['project'],
+#       source=p['source'],
+#     ),
+#     wrapper_code = '''
+#       {result, ...}:
+#       result.inputs.exportedPackages
+#     ''',
+#   )
+#   assert isinstance(exportedPackages, dict)
+#   assert len(exportedPackages) > 0
+
+@pytest.mark.parametrize("p", projects)
+def test_extraObjects(p):
+  extraObjects = nix_ffi.eval(
+    f"subsystems.{p['subsystem']}.translators.{p['translator']}.translate",
+    params=dict(
+      project=p['project'],
+      source=p['source'],
+    ),
+    wrapper_code = '''
+      {result, ...}:
+      result.inputs.extraObjects
+    ''',
+  )
+  assert isinstance(extraObjects, list)
+  for extra_obj in extraObjects:
+    assert set(extra_obj.keys()) == \
+      {'name', 'version', 'dependencies', 'sourceSpec'}
+    assert isinstance(extra_obj['name'], str)
+    assert len(extra_obj['name']) > 0
+    assert isinstance(extra_obj['version'], str)
+    assert len(extra_obj['version']) > 0
+    check_format_dependencies(extra_obj['dependencies'])
+    check_format_sourceSpec(extra_obj['sourceSpec'])
+
+@pytest.mark.parametrize("p", projects)
+def test_location(p):
+  location = nix_ffi.eval(
+    f"subsystems.{p['subsystem']}.translators.{p['translator']}.translate",
+    params=dict(
+      project=p['project'],
+      source=p['source'],
+    ),
+    wrapper_code = '''
+      {result, ...}:
+      result.inputs.location
+    ''',
+  )
+  assert isinstance(location, str)
+
+@pytest.mark.parametrize("p", projects)
+def test_serializedRawObjects(p):
+  serializedRawObjects = nix_ffi.eval(
+    f"subsystems.{p['subsystem']}.translators.{p['translator']}.translate",
+    params=dict(
+      project=p['project'],
+      source=p['source'],
+    ),
+    wrapper_code = '''
+      {result, ...}:
+
+      result.inputs.serializedRawObjects
+    ''',
+  )
+  assert isinstance(serializedRawObjects, list)
+  for raw_obj in serializedRawObjects:
+    assert isinstance(raw_obj, dict)
+
+@pytest.mark.parametrize("p", projects)
+def test_subsystemName(p):
+  subsystemName = nix_ffi.eval(
+    f"subsystems.{p['subsystem']}.translators.{p['translator']}.translate",
+    params=dict(
+      project=p['project'],
+      source=p['source'],
+    ),
+    wrapper_code = '''
+      {result, ...}:
+      result.inputs.subsystemName
+    ''',
+  )
+  assert isinstance(subsystemName, str)
+  assert len(subsystemName) > 0
+
+@pytest.mark.parametrize("p", projects)
+def test_subsystemAttrs(p):
+  subsystemAttrs = nix_ffi.eval(
+    f"subsystems.{p['subsystem']}.translators.{p['translator']}.translate",
+    params=dict(
+      project=p['project'],
+      source=p['source'],
+    ),
+    wrapper_code = '''
+      {result, ...}:
+      result.inputs.subsystemAttrs
+    ''',
+  )
+  assert isinstance(subsystemAttrs, dict)
+
+@pytest.mark.parametrize("p", projects)
+def test_translatorName(p):
+  translatorName = nix_ffi.eval(
+    f"subsystems.{p['subsystem']}.translators.{p['translator']}.translate",
+    params=dict(
+      project=p['project'],
+      source=p['source'],
+    ),
+    wrapper_code = '''
+      {result, ...}:
+      result.inputs.translatorName
+    ''',
+  )
+  assert isinstance(translatorName, str)
+  assert len(translatorName) > 0
+
+# @pytest.mark.parametrize("p", projects)
+# def test_extractors(p):
+#   finalObjects = nix_ffi.eval(
+#     f"subsystems.{p['subsystem']}.translators.{p['translator']}.translate",
+#     params=dict(
+#       project=p['project'],
+#       source=p['source'],
+#     ),
+#     wrapper_code = '''
+#       {result, ...}:
+#       let
+#         l = builtins;
+#         inputs = result.inputs;
+#         rawObjects = inputs.serializedRawObjects;
+
+#         finalObjects = mkFinalObjects rawObjects inputs.extractors;
+#         allDependencies = makeDependencies finalObjects;
+#         exportedFinalObjects =
+#           mkExportedFinalObjects finalObjects exportedPackages;
+#         relevantFinalObjects =
+#           mkRelevantFinalObjects exportedFinalObjects allDependencies;
+#       in
+#         relevantFinalObjects ++ (inputs.extraObjects or [])
+#     ''',
+#   )
+#   assert isinstance(finalObjects, list)
+#   assert len(finalObjects) > 0
+#   for finalObj in finalObjects:
+#     assert set(finalObj.keys()) == \
+#       {'name', 'version', 'sourceSpec', 'dependencies'}
+#     check_format_dependencies(finalObj['dependencies'])
+#     check_format_sourceSpec(finalObj['sourceSpec'])
+
+@pytest.mark.parametrize("p", projects)
+def test_keys(p):
+  objectsByKey = nix_ffi.eval(
+    f"subsystems.{p['subsystem']}.translators.{p['translator']}.translate",
+    params=dict(
+      project=p['project'],
+      source=p['source'],
+    ),
+    wrapper_code = '''
+      {result, ...}:
+      let
+        l = builtins;
+        inputs = result.inputs;
+        rawObjects = inputs.serializedRawObjects;
+
+        finalObjects =
+          l.map
+          (rawObj: let
+            finalObj =
+              {inherit rawObj;}
+              // l.mapAttrs
+              (key: extractFunc: extractFunc rawObj finalObj)
+              inputs.extractors;
+          in
+            finalObj)
+          rawObjects;
+
+        objectsByKey =
+          l.mapAttrs
+          (key: keyFunc:
+            l.foldl'
+            (merged: finalObj:
+              merged
+              // {"${keyFunc finalObj.rawObj finalObj}" = finalObj;})
+            {}
+            (finalObjects))
+          inputs.keys;
+      in
+        objectsByKey
+    ''',
+  )
+  assert isinstance(objectsByKey, dict)
+  for key_name, objects in objectsByKey.items():
+    for finalObj in objects.values():
+      assert set(finalObj.keys()) == \
+        {'name', 'version', 'sourceSpec', 'dependencies', 'rawObj'}
+      check_format_dependencies(finalObj['dependencies'])
+      check_format_sourceSpec(finalObj['sourceSpec'])

tests/unit/test_pure_translators.py

@@ -7,7 +7,7 @@ def get_projects_to_test():
   tests = nix_ffi.eval(
     'subsystems.allTranslators',
     wrapper_code = '''
-      {result}: let
+      {result, ...}: let
         lib = (import <nixpkgs> {}).lib;
         l = lib // builtins;
       in
@@ -71,7 +71,7 @@ def test_packageName(p):
       source=p['source'],
     ),
     wrapper_code = '''
-      {result}:
+      {result, ...}:
       result.inputs.defaultPackage
     ''',
   )
@@ -87,7 +87,7 @@ def test_exportedPackages(p):
       source=p['source'],
     ),
     wrapper_code = '''
-      {result}:
+      {result, ...}:
       result.inputs.exportedPackages
     ''',
   )
@@ -103,7 +103,7 @@ def test_extraObjects(p):
       source=p['source'],
     ),
     wrapper_code = '''
-      {result}:
+      {result, ...}:
       result.inputs.extraObjects
     ''',
   )
@@ -127,7 +127,7 @@ def test_location(p):
       source=p['source'],
     ),
     wrapper_code = '''
-      {result}:
+      {result, ...}:
       result.inputs.location
     ''',
   )
@@ -142,9 +142,13 @@ def test_serializedRawObjects(p):
       source=p['source'],
     ),
     wrapper_code = '''
-      {result}:
-
-      result.inputs.serializedRawObjects
+      {result, lib, ...}:
+      let
+        len = lib.length result.inputs.serializedRawObjects;
+      in
+        # for performance reasons check only first/last 10 items of the list
+        (lib.sublist 0 10 result.inputs.serializedRawObjects)
+        ++ (lib.sublist (lib.max (len - 10) 0) len result.inputs.serializedRawObjects)
     ''',
   )
   assert isinstance(serializedRawObjects, list)
@@ -160,7 +164,7 @@ def test_subsystemName(p):
       source=p['source'],
     ),
     wrapper_code = '''
-      {result}:
+      {result, ...}:
       result.inputs.subsystemName
     ''',
   )
@@ -176,7 +180,8 @@ def test_subsystemAttrs(p):
       source=p['source'],
     ),
     wrapper_code = '''
-      {result}:
+      {result, ...}:
+      builtins.trace result.inputs.subsystemAttrs
       result.inputs.subsystemAttrs
     ''',
   )
@@ -191,7 +196,7 @@ def test_translatorName(p):
       source=p['source'],
     ),
     wrapper_code = '''
-      {result}:
+      {result, ...}:
       result.inputs.translatorName
     ''',
   )
@@ -207,30 +212,27 @@ def test_extractors(p):
       source=p['source'],
     ),
     wrapper_code = '''
-      {result}:
+      {result, dlib, ...}:
       let
         l = builtins;
         inputs = result.inputs;
         rawObjects = inputs.serializedRawObjects;
+        s = dlib.simpleTranslate2;
 
-        finalObjects =
-          l.map
-          (rawObj: let
-            finalObj =
-              l.mapAttrs
-              (key: extractFunc: extractFunc rawObj finalObj)
-              inputs.extractors;
-          in
-            finalObj)
-          rawObjects;
+        finalObjects = s.mkFinalObjects rawObjects inputs.extractors;
+        allDependencies = s.makeDependencies finalObjects;
+        exportedFinalObjects =
+          s.mkExportedFinalObjects finalObjects inputs.exportedPackages;
+        relevantFinalObjects =
+          s.mkRelevantFinalObjects exportedFinalObjects allDependencies;
       in
-        finalObjects ++ (inputs.extraObjects or [])
+        relevantFinalObjects ++ (inputs.extraObjects or [])
     ''',
   )
   assert isinstance(finalObjects, list)
   assert len(finalObjects) > 0
   for finalObj in finalObjects:
-    assert set(finalObj.keys()) == \
+    assert (set(finalObj.keys()) - {'rawObj', 'key'}) == \
       {'name', 'version', 'sourceSpec', 'dependencies'}
     check_format_dependencies(finalObj['dependencies'])
     check_format_sourceSpec(finalObj['sourceSpec'])
@@ -244,23 +246,19 @@ def test_keys(p):
       source=p['source'],
     ),
     wrapper_code = '''
-      {result}:
+      {result, dlib, ...}:
       let
         l = builtins;
         inputs = result.inputs;
         rawObjects = inputs.serializedRawObjects;
+        s = dlib.simpleTranslate2;
 
-        finalObjects =
-          l.map
-          (rawObj: let
-            finalObj =
-              {inherit rawObj;}
-              // l.mapAttrs
-              (key: extractFunc: extractFunc rawObj finalObj)
-              inputs.extractors;
-          in
-            finalObj)
-          rawObjects;
+        finalObjects = s.mkFinalObjects rawObjects inputs.extractors;
+        allDependencies = s.makeDependencies finalObjects;
+        exportedFinalObjects =
+          s.mkExportedFinalObjects finalObjects inputs.exportedPackages;
+        relevantFinalObjects =
+          s.mkRelevantFinalObjects exportedFinalObjects allDependencies;
 
         objectsByKey =
           l.mapAttrs
@@ -270,7 +268,7 @@ def test_keys(p):
               merged
               // {"${keyFunc finalObj.rawObj finalObj}" = finalObj;})
             {}
-            (finalObjects))
+            relevantFinalObjects)
           inputs.keys;
       in
         objectsByKey