diff --git a/src/apps/cli/nix_ffi.py b/src/apps/cli/nix_ffi.py index a36e3b67..9631ffad 100644 --- a/src/apps/cli/nix_ffi.py +++ b/src/apps/cli/nix_ffi.py @@ -41,7 +41,7 @@ def callNixFunction(function_path, **kwargs): def eval(attr_path, wrapper_code=None, **kwargs): if wrapper_code == None: # dummy wrapper code - wrapper_code = "{result}: result" + wrapper_code = "{result, ...}: result" is_function_call = len(kwargs) > 0 @@ -59,15 +59,21 @@ def eval(attr_path, wrapper_code=None, **kwargs): "eval", "--show-trace", "--impure", "--raw", "--expr", f''' let + b = builtins; d2n = (import {dream2nix_src} {{}}); - wrapper = import {wrapper_code_file.name}; - result = + result' = if "{is_function_call}" == "True" then d2n.utils.callViaEnv d2n.{attr_path} else d2n.{attr_path}; + result = (d2n.callPackageDream + {wrapper_code_file.name} + {{ result = result'; }}); in - builtins.toJSON ( - wrapper {{ inherit result; }} + b.toJSON ( + # remove override attributes added by callPackage + if b.isAttrs result + then b.removeAttrs result ["override" "overrideDerivation"] + else result ) ''', env=env diff --git a/src/lib/default.nix b/src/lib/default.nix index 32a87599..147c7649 100644 --- a/src/lib/default.nix +++ b/src/lib/default.nix @@ -53,7 +53,7 @@ modules = import ./modules.nix {inherit config dlib lib;}; simpleTranslate2 = - (import ./simpleTranslate2.nix {inherit dlib lib;}).simpleTranslate2; + import ./simpleTranslate2.nix {inherit dlib lib;}; parseUtils = import ./parsing.nix {inherit lib;}; diff --git a/src/lib/simpleTranslate2.nix b/src/lib/simpleTranslate2.nix index ee6b6ed0..6ebaee2c 100644 --- a/src/lib/simpleTranslate2.nix +++ b/src/lib/simpleTranslate2.nix @@ -5,7 +5,88 @@ }: let l = lib // builtins; - simpleTranslate2 = func: let + expectedFields = [ + "name" + "version" + "sourceSpec" + "dependencies" + ]; + + mkFinalObjects = rawObjects: extractors: + l.map + (rawObj: let + finalObj = + {inherit rawObj;} + // l.mapAttrs + (key: extractFunc: extractFunc rawObj finalObj) + extractors; + in + finalObj) + rawObjects; + + # checks validity of all objects by iterating over them + mkValidatedFinalObjects = finalObjects: translatorName: extraObjects: + l.map + (finalObj: + if l.any (field: ! finalObj ? "${field}") expectedFields + then + throw + '' + Translator ${translatorName} failed. + The following object does not contain all required fields: + Object: + ${l.toJSON finalObj} + Missing fields: + ${l.subtractLists expectedFields (l.attrNames finalObj)} + '' + # TODO: validate sourceSpec as well + else finalObj) + (finalObjects ++ extraObjects); + + mkExportedFinalObjects = finalObjects: exportedPackages: + l.filter + (finalObj: + exportedPackages.${finalObj.name} or null == finalObj.version) + finalObjects; + + mkRelevantFinalObjects = exportedFinalObjects: allDependencies: + l.genericClosure { + startSet = + l.map + (finalObj: + finalObj + // {key = "${finalObj.name}#${finalObj.version}";}) + exportedFinalObjects; + operator = finalObj: + l.map + (c: + allDependencies.${c.name}.${c.version} + // {key = "${c.name}#${c.version}";}) + finalObj.dependencies; + }; + + /* + format: + { + foo = { + "1.0.0" = finalObj + } + } + */ + makeDependencies = finalObjects: + l.foldl' + (result: finalObj: + lib.recursiveUpdate + result + { + "${finalObj.name}" = { + "${finalObj.version}" = finalObj; + }; + }) + {} + finalObjects; + + translate = func: let final = func { @@ -14,43 +95,7 @@ rawObjects = final.serializedRawObjects; - expectedFields = [ - "name" - "version" - "sourceSpec" - "dependencies" - ]; - - finalObjects' = - l.map - (rawObj: let - finalObj = - {inherit rawObj;} - // l.mapAttrs - (key: extractFunc: extractFunc rawObj finalObj) - final.extractors; - in - finalObj) - rawObjects; - - # checks validity of all objects by iterating over them - finalObjects = - l.map - (finalObj: - if l.any (field: ! finalObj ? "${field}") expectedFields - then - throw - '' - Translator ${final.translatorName} failed. - The following object does not contain all required fields: - Object: - ${l.toJSON finalObj} - Missing fields: - ${l.subtractLists expectedFields (l.attrNames finalObj)} - '' - # TODO: validate sourceSpec as well - else finalObj) - (finalObjects' ++ (final.extraObjects or [])); + finalObjects' = mkFinalObjects rawObjects final.extractors; objectsByKey = l.mapAttrs @@ -92,49 +137,19 @@ ; }; - /* - format: - { - foo = { - "1.0.0" = finalObj - } - } - */ - makeDependencies = finalObjects: - l.foldl' - (result: finalObj: - lib.recursiveUpdate - result - { - "${finalObj.name}" = { - "${finalObj.version}" = finalObj; - }; - }) - {} - finalObjects; + finalObjects = + mkValidatedFinalObjects + finalObjects' + translatorName + (final.extraObjects or []); allDependencies = makeDependencies finalObjects; exportedFinalObjects = - l.filter - (finalObj: - exportedPackages.${finalObj.name} or null == finalObj.version) - finalObjects; + mkExportedFinalObjects finalObjects exportedPackages; - relevantFinalObjects = l.genericClosure { - startSet = - l.map - (finalObj: - finalObj - // {key = "${finalObj.name}#${finalObj.version}";}) - exportedFinalObjects; - operator = finalObj: - l.map - (c: - allDependencies.${c.name}.${c.version} - // {key = "${c.name}#${c.version}";}) - finalObj.dependencies; - }; + relevantFinalObjects = + mkRelevantFinalObjects exportedFinalObjects allDependencies; relevantDependencies = makeDependencies relevantFinalObjects; @@ -275,5 +290,11 @@ inputs = dreamLockData.inputs; }; in { - inherit simpleTranslate2; + inherit + translate + mkFinalObjects + mkExportedFinalObjects + mkRelevantFinalObjects + makeDependencies + ; } diff --git a/src/subsystems/haskell/translators/stack-lock/default.nix b/src/subsystems/haskell/translators/stack-lock/default.nix index b3e33900..905c26bd 100644 --- a/src/subsystems/haskell/translators/stack-lock/default.nix +++ b/src/subsystems/haskell/translators/stack-lock/default.nix @@ -239,7 +239,7 @@ in { && (objectsByName ? ${name})) depNames; in - dlib.simpleTranslate2 + dlib.simpleTranslate2.translate ({objectsByKey, ...}: rec { inherit translatorName; diff --git a/src/subsystems/nodejs/translators/yarn-lock/default.nix b/src/subsystems/nodejs/translators/yarn-lock/default.nix index 92ddae8a..45d8fc81 100644 --- a/src/subsystems/nodejs/translators/yarn-lock/default.nix +++ b/src/subsystems/nodejs/translators/yarn-lock/default.nix @@ -46,7 +46,7 @@ workspacesPackageJson = nodejsUtils.getWorkspacePackageJson tree workspaces; in - dlib.simpleTranslate2 + dlib.simpleTranslate2.translate ({objectsByKey, ...}: let makeWorkspaceExtraObject = workspace: let json = workspacesPackageJson."${workspace}"; diff --git a/src/subsystems/rust/translators/cargo-lock/default.nix b/src/subsystems/rust/translators/cargo-lock/default.nix index d307cf27..a07b39bb 100644 --- a/src/subsystems/rust/translators/cargo-lock/default.nix +++ b/src/subsystems/rust/translators/cargo-lock/default.nix @@ -184,7 +184,7 @@ in { or (l.warn "no version found in Cargo.toml for ${name}, defaulting to unknown" "unknown"); }; in - dlib.simpleTranslate2 + dlib.simpleTranslate2.translate ({...}: rec { inherit translatorName; diff --git a/src/templates/translators/pure.nix b/src/templates/translators/pure.nix index 6d11b980..9a59a68c 100644 --- a/src/templates/translators/pure.nix +++ b/src/templates/translators/pure.nix @@ -135,7 +135,7 @@ in in - dlib.simpleTranslate2 + dlib.simpleTranslate2.translate ({ objectsByKey, ... diff --git a/tests/examples/default.nix b/tests/examples/default.nix index c32fd078..432a1bc3 100644 --- a/tests/examples/default.nix +++ b/tests/examples/default.nix @@ -40,7 +40,7 @@ in ] '' if [ -z ''${1+x} ]; then - parallel -j$(nproc) -a <(ls ${examples}) ${testScript} + parallel --halt now,fail=1 -j$(nproc) -a <(ls ${examples}) ${testScript} else arg1=$1 shift diff --git a/tests/unit/test_pure_translators copy.py b/tests/unit/test_pure_translators copy.py new file mode 100644 index 00000000..9c5705c2 --- /dev/null +++ b/tests/unit/test_pure_translators copy.py @@ -0,0 +1,281 @@ +import nix_ffi +import os +import pytest + + +def get_projects_to_test(): + tests = nix_ffi.eval( + 'subsystems.allTranslators', + wrapper_code = ''' + {result, ...}: let + lib = (import {}).lib; + l = lib // builtins; + in + l.flatten ( + l.map + ( + translator: + l.map + (source: { + source = l.toString source; + translator = translator.name; + inherit (translator) subsystem type; + }) + (translator.generateUnitTestsForProjects or []) + ) + result + ) + ''', + ) + result = [] + for test in tests: + if test['type'] == 'all': + continue + result.append(dict( + project = dict( + name="test", + relPath="", + translator=test['translator'], + subsystemInfo={}, + ), + translator=test['translator'], + source = test['source'], + subsystem = test['subsystem'], + type = test['type'], + )) + return result + + +projects = get_projects_to_test() + + +def check_format_dependencies(dependencies): + assert isinstance(dependencies, list) + for dep in dependencies: + assert set(dep.keys()) == {'name', 'version'} + assert isinstance(dep['name'], str) + assert len(dep['name']) > 0 + assert isinstance(dep['version'], str) + assert len(dep['version']) > 0 + +def check_format_sourceSpec(sourceSpec): + assert isinstance(sourceSpec, dict) + assert 'type' in sourceSpec + +@pytest.mark.parametrize("p", projects) +def test_packageName(p): + defaultPackage = nix_ffi.eval( + f"subsystems.{p['subsystem']}.translators.{p['translator']}.translate", + params=dict( + project=p['project'], + source=p['source'], + ), + wrapper_code = ''' + {result, ...}: + result.inputs.defaultPackage + ''', + ) + assert isinstance(defaultPackage, str) + assert len(defaultPackage) > 0 + +# @pytest.mark.parametrize("p", projects) +# def test_exportedPackages(p): +# exportedPackages = nix_ffi.eval( +# f"subsystems.{p['subsystem']}.translators.{p['translator']}.translate", +# params=dict( +# project=p['project'], +# source=p['source'], +# ), +# wrapper_code = ''' +# {result, ...}: +# result.inputs.exportedPackages +# ''', +# ) +# assert isinstance(exportedPackages, dict) +# assert len(exportedPackages) > 0 + +@pytest.mark.parametrize("p", projects) +def test_extraObjects(p): + extraObjects = nix_ffi.eval( + f"subsystems.{p['subsystem']}.translators.{p['translator']}.translate", + params=dict( + project=p['project'], + source=p['source'], + ), + wrapper_code = ''' + {result, ...}: + result.inputs.extraObjects + ''', + ) + assert isinstance(extraObjects, list) + for extra_obj in extraObjects: + assert set(extra_obj.keys()) == \ + {'name', 'version', 'dependencies', 'sourceSpec'} + assert isinstance(extra_obj['name'], str) + assert len(extra_obj['name']) > 0 + assert isinstance(extra_obj['version'], str) + assert len(extra_obj['version']) > 0 + check_format_dependencies(extra_obj['dependencies']) + check_format_sourceSpec(extra_obj['sourceSpec']) + +@pytest.mark.parametrize("p", projects) +def test_location(p): + location = nix_ffi.eval( + f"subsystems.{p['subsystem']}.translators.{p['translator']}.translate", + params=dict( + project=p['project'], + source=p['source'], + ), + wrapper_code = ''' + {result, ...}: + result.inputs.location + ''', + ) + assert isinstance(location, str) + +@pytest.mark.parametrize("p", projects) +def test_serializedRawObjects(p): + serializedRawObjects = nix_ffi.eval( + f"subsystems.{p['subsystem']}.translators.{p['translator']}.translate", + params=dict( + project=p['project'], + source=p['source'], + ), + wrapper_code = ''' + {result, ...}: + + result.inputs.serializedRawObjects + ''', + ) + assert isinstance(serializedRawObjects, list) + for raw_obj in serializedRawObjects: + assert isinstance(raw_obj, dict) + +@pytest.mark.parametrize("p", projects) +def test_subsystemName(p): + subsystemName = nix_ffi.eval( + f"subsystems.{p['subsystem']}.translators.{p['translator']}.translate", + params=dict( + project=p['project'], + source=p['source'], + ), + wrapper_code = ''' + {result, ...}: + result.inputs.subsystemName + ''', + ) + assert isinstance(subsystemName, str) + assert len(subsystemName) > 0 + +@pytest.mark.parametrize("p", projects) +def test_subsystemAttrs(p): + subsystemAttrs = nix_ffi.eval( + f"subsystems.{p['subsystem']}.translators.{p['translator']}.translate", + params=dict( + project=p['project'], + source=p['source'], + ), + wrapper_code = ''' + {result, ...}: + result.inputs.subsystemAttrs + ''', + ) + assert isinstance(subsystemAttrs, dict) + +@pytest.mark.parametrize("p", projects) +def test_translatorName(p): + translatorName = nix_ffi.eval( + f"subsystems.{p['subsystem']}.translators.{p['translator']}.translate", + params=dict( + project=p['project'], + source=p['source'], + ), + wrapper_code = ''' + {result, ...}: + result.inputs.translatorName + ''', + ) + assert isinstance(translatorName, str) + assert len(translatorName) > 0 + +# @pytest.mark.parametrize("p", projects) +# def test_extractors(p): +# finalObjects = nix_ffi.eval( +# f"subsystems.{p['subsystem']}.translators.{p['translator']}.translate", +# params=dict( +# project=p['project'], +# source=p['source'], +# ), +# wrapper_code = ''' +# {result, ...}: +# let +# l = builtins; +# inputs = result.inputs; +# rawObjects = inputs.serializedRawObjects; + +# finalObjects = mkFinalObjects rawObjects inputs.extractors; +# allDependencies = makeDependencies finalObjects; +# exportedFinalObjects = +# mkExportedFinalObjects finalObjects exportedPackages; +# relevantFinalObjects = +# mkRelevantFinalObjects exportedFinalObjects allDependencies; +# in +# relevantFinalObjects ++ (inputs.extraObjects or []) +# ''', +# ) +# assert isinstance(finalObjects, list) +# assert len(finalObjects) > 0 +# for finalObj in finalObjects: +# assert set(finalObj.keys()) == \ +# {'name', 'version', 'sourceSpec', 'dependencies'} +# check_format_dependencies(finalObj['dependencies']) +# check_format_sourceSpec(finalObj['sourceSpec']) + +@pytest.mark.parametrize("p", projects) +def test_keys(p): + objectsByKey = nix_ffi.eval( + f"subsystems.{p['subsystem']}.translators.{p['translator']}.translate", + params=dict( + project=p['project'], + source=p['source'], + ), + wrapper_code = ''' + {result, ...}: + let + l = builtins; + inputs = result.inputs; + rawObjects = inputs.serializedRawObjects; + + finalObjects = + l.map + (rawObj: let + finalObj = + {inherit rawObj;} + // l.mapAttrs + (key: extractFunc: extractFunc rawObj finalObj) + inputs.extractors; + in + finalObj) + rawObjects; + + objectsByKey = + l.mapAttrs + (key: keyFunc: + l.foldl' + (merged: finalObj: + merged + // {"${keyFunc finalObj.rawObj finalObj}" = finalObj;}) + {} + (finalObjects)) + inputs.keys; + in + objectsByKey + ''', + ) + assert isinstance(objectsByKey, dict) + for key_name, objects in objectsByKey.items(): + for finalObj in objects.values(): + assert set(finalObj.keys()) == \ + {'name', 'version', 'sourceSpec', 'dependencies', 'rawObj'} + check_format_dependencies(finalObj['dependencies']) + check_format_sourceSpec(finalObj['sourceSpec']) diff --git a/tests/unit/test_pure_translators.py b/tests/unit/test_pure_translators.py index f7f4bc93..5c27eb24 100644 --- a/tests/unit/test_pure_translators.py +++ b/tests/unit/test_pure_translators.py @@ -7,7 +7,7 @@ def get_projects_to_test(): tests = nix_ffi.eval( 'subsystems.allTranslators', wrapper_code = ''' - {result}: let + {result, ...}: let lib = (import {}).lib; l = lib // builtins; in @@ -71,7 +71,7 @@ def test_packageName(p): source=p['source'], ), wrapper_code = ''' - {result}: + {result, ...}: result.inputs.defaultPackage ''', ) @@ -87,7 +87,7 @@ def test_exportedPackages(p): source=p['source'], ), wrapper_code = ''' - {result}: + {result, ...}: result.inputs.exportedPackages ''', ) @@ -103,7 +103,7 @@ def test_extraObjects(p): source=p['source'], ), wrapper_code = ''' - {result}: + {result, ...}: result.inputs.extraObjects ''', ) @@ -127,7 +127,7 @@ def test_location(p): source=p['source'], ), wrapper_code = ''' - {result}: + {result, ...}: result.inputs.location ''', ) @@ -142,9 +142,13 @@ def test_serializedRawObjects(p): source=p['source'], ), wrapper_code = ''' - {result}: - - result.inputs.serializedRawObjects + {result, lib, ...}: + let + len = lib.length result.inputs.serializedRawObjects; + in + # for performance reasons check only first/last 10 items of the list + (lib.sublist 0 10 result.inputs.serializedRawObjects) + ++ (lib.sublist (lib.max (len - 10) 0) len result.inputs.serializedRawObjects) ''', ) assert isinstance(serializedRawObjects, list) @@ -160,7 +164,7 @@ def test_subsystemName(p): source=p['source'], ), wrapper_code = ''' - {result}: + {result, ...}: result.inputs.subsystemName ''', ) @@ -176,7 +180,8 @@ def test_subsystemAttrs(p): source=p['source'], ), wrapper_code = ''' - {result}: + {result, ...}: + builtins.trace result.inputs.subsystemAttrs result.inputs.subsystemAttrs ''', ) @@ -191,7 +196,7 @@ def test_translatorName(p): source=p['source'], ), wrapper_code = ''' - {result}: + {result, ...}: result.inputs.translatorName ''', ) @@ -207,30 +212,27 @@ def test_extractors(p): source=p['source'], ), wrapper_code = ''' - {result}: + {result, dlib, ...}: let l = builtins; inputs = result.inputs; rawObjects = inputs.serializedRawObjects; + s = dlib.simpleTranslate2; - finalObjects = - l.map - (rawObj: let - finalObj = - l.mapAttrs - (key: extractFunc: extractFunc rawObj finalObj) - inputs.extractors; - in - finalObj) - rawObjects; + finalObjects = s.mkFinalObjects rawObjects inputs.extractors; + allDependencies = s.makeDependencies finalObjects; + exportedFinalObjects = + s.mkExportedFinalObjects finalObjects inputs.exportedPackages; + relevantFinalObjects = + s.mkRelevantFinalObjects exportedFinalObjects allDependencies; in - finalObjects ++ (inputs.extraObjects or []) + relevantFinalObjects ++ (inputs.extraObjects or []) ''', ) assert isinstance(finalObjects, list) assert len(finalObjects) > 0 for finalObj in finalObjects: - assert set(finalObj.keys()) == \ + assert (set(finalObj.keys()) - {'rawObj', 'key'}) == \ {'name', 'version', 'sourceSpec', 'dependencies'} check_format_dependencies(finalObj['dependencies']) check_format_sourceSpec(finalObj['sourceSpec']) @@ -244,23 +246,19 @@ def test_keys(p): source=p['source'], ), wrapper_code = ''' - {result}: + {result, dlib, ...}: let l = builtins; inputs = result.inputs; rawObjects = inputs.serializedRawObjects; + s = dlib.simpleTranslate2; - finalObjects = - l.map - (rawObj: let - finalObj = - {inherit rawObj;} - // l.mapAttrs - (key: extractFunc: extractFunc rawObj finalObj) - inputs.extractors; - in - finalObj) - rawObjects; + finalObjects = s.mkFinalObjects rawObjects inputs.extractors; + allDependencies = s.makeDependencies finalObjects; + exportedFinalObjects = + s.mkExportedFinalObjects finalObjects inputs.exportedPackages; + relevantFinalObjects = + s.mkRelevantFinalObjects exportedFinalObjects allDependencies; objectsByKey = l.mapAttrs @@ -270,7 +268,7 @@ def test_keys(p): merged // {"${keyFunc finalObj.rawObj finalObj}" = finalObj;}) {} - (finalObjects)) + relevantFinalObjects) inputs.keys; in objectsByKey