Merge pull request #27 from DavHau/dev

Simplified interface for pure translators

src/apps/cli/cli.py

@@ -136,16 +136,20 @@ class PackageCommand(Command):
       ))[0]
     else:
       translator = translator.split('.')
-      if len(translator) == 3:
-        translator = list(filter(
-          lambda t: [t['subsystem'], t['type'], t['name']] == translator,
-          translatorsSorted,
-        ))[0]
-      elif len(translator) == 1:
-        translator = list(filter(
-          lambda t:  [t['name']] == translator,
-          translatorsSorted,
-        ))[0]
+      try:
+        if len(translator) == 3:
+          translator = list(filter(
+            lambda t: [t['subsystem'], t['type'], t['name']] == translator,
+            translatorsSorted,
+          ))[0]
+        elif len(translator) == 1:
+          translator = list(filter(
+            lambda t:  [t['name']] == translator,
+            translatorsSorted,
+          ))[0]
+      except IndexError:
+        print(f"Could not find translator '{'.'.join(translator)}'", file=sys.stderr)
+        exit(1)
 
     # raise error if any specified extra arg is unknown
     unknown_extra_args = set(specified_extra_args.keys()) - set(translator['specialArgs'].keys())

src/translators/nodejs/pure/npmlock2nix/default.nix

@@ -1,171 +0,0 @@
-{
-  lib,
-
-  externals,
-  translatorName,
-  utils,
-  ...
-}:
-
-{
-  translate =
-    {
-      inputDirectories,
-      inputFiles,
-
-      dev,
-      ...
-    }:
-    let
-      packageLock =
-        if inputDirectories != [] then
-          "${lib.elemAt inputDirectories 0}/package-lock.json"
-        else
-          lib.elemAt inputFiles 0;
-
-      parsed = externals.npmlock2nix.readLockfile packageLock;
-
-      parseGithubDependency = dependency:
-        externals.npmlock2nix.parseGitHubRef dependency.version;
-
-      getVersion = dependency:
-        if dependency ? from && dependency ? version then
-          builtins.substring 0 8 (parseGithubDependency dependency).rev
-        else
-          dependency.version;
-
-      pinVersions = dependencies: parentScopeDeps:
-        lib.mapAttrs
-          (pname: pdata:
-            let
-              selfScopeDeps = parentScopeDeps // (pdata.dependencies or {});
-            in
-              pdata // {
-                depsExact =
-                  if ! pdata ? requires then
-                    []
-                  else
-                    lib.forEach (lib.attrNames pdata.requires) (reqName:
-                      "${reqName}#${getVersion selfScopeDeps."${reqName}"}"
-                    );
-                dependencies = pinVersions (pdata.dependencies or {}) selfScopeDeps;
-              }
-          )
-          dependencies;
-      
-      packageLockWithPinnedVersions = pinVersions parsed.dependencies parsed.dependencies;
-
-      # recursively collect dependencies
-      parseDependencies = dependencies:
-        lib.mapAttrsToList  # returns list of lists
-          (pname: pdata:
-            if ! dev && pdata.dev or false then
-              []
-            else
-              # handle github dependency
-              if pdata ? from && pdata ? version then
-                let
-                  githubData = parseGithubDependency pdata;
-                in
-                [ rec {
-                  name = "${pname}#${version}";
-                  version = builtins.substring 0 8 githubData.rev;
-                  owner = githubData.org;
-                  repo = githubData.repo;
-                  rev = githubData.rev;
-                  type = "github";
-                  depsExact = pdata.depsExact;
-                }]
-              # handle http(s) dependency
-              else
-                [rec {
-                  name = "${pname}#${version}";
-                  version = pdata.version;
-                  url = pdata.resolved;
-                  type = "fetchurl";
-                  hash = pdata.integrity;
-                  depsExact = pdata.depsExact;
-                }]
-            ++
-            (lib.optionals (pdata ? dependencies)
-              (lib.flatten (parseDependencies pdata.dependencies))
-            )
-          )
-          dependencies;
-    in
-
-    # the dream lock
-    rec {
-      sources =
-        let
-          lockedSources = lib.listToAttrs (
-            map
-              (dep: lib.nameValuePair
-                dep.name
-                (
-                  if dep.type == "github" then
-                    { inherit (dep) type version owner repo rev; }
-                  else
-                    { inherit (dep) type version url hash; }
-                )
-              )
-              (lib.flatten (parseDependencies packageLockWithPinnedVersions))
-          );
-        in
-          # if only a package-lock.json is given, the main source is missing
-          lockedSources // {
-            "${parsed.name}" = {
-              type = "unknown";
-              version = parsed.version;
-            };
-          };
-
-      generic = {
-        buildSystem = "nodejs";
-        producedBy = translatorName;
-        mainPackage = parsed.name;
-        dependencyGraph =
-          {
-            "${parsed.name}" =
-              lib.mapAttrsToList
-                (pname: pdata: "${pname}#${getVersion pdata}")
-                (lib.filterAttrs
-                  (pname: pdata: ! (pdata.dev or false) || dev)
-                  parsed.dependencies);
-          }
-          //
-          lib.listToAttrs 
-            (map
-              (dep: lib.nameValuePair dep.name dep.depsExact)
-              (lib.flatten (parseDependencies packageLockWithPinnedVersions))
-            );
-        sourcesCombinedHash = null;
-      };
-
-      buildSystem = {
-        nodejsVersion = 14;
-      };
-    };
-
-  compatiblePaths =
-    {
-      inputDirectories,
-      inputFiles,
-    }@args:
-    {
-      inputDirectories = lib.filter 
-        (utils.containsMatchingFile [ ''.*package-lock\.json'' ''.*package.json'' ])
-        args.inputDirectories;
-
-      inputFiles = [];
-    };
-
-  specialArgs = {
-
-    dev = {
-      description = "include dependencies for development";
-      type = "flag";
-    };
-
-  };
-}

src/translators/nodejs/pure/package-lock/default.nix

@@ -0,0 +1,161 @@
+{
+  lib,
+  nodejs,
+
+  externals,
+  translatorName,
+  utils,
+  ...
+}:
+
+{
+  translate =
+    {
+      inputDirectories,
+      inputFiles,
+
+      dev,
+      nodejs,
+      ...
+    }@args:
+    let
+      packageLock =
+        if inputDirectories != [] then
+          "${lib.elemAt inputDirectories 0}/package-lock.json"
+        else
+          lib.elemAt inputFiles 0;
+
+      parsed = externals.npmlock2nix.readLockfile packageLock;
+
+      parseGithubDependency = dependency:
+        externals.npmlock2nix.parseGitHubRef dependency.version;
+
+      getVersion = dependencyObject:
+          if dependencyObject ? from && dependencyObject ? version then
+            builtins.substring 0 8 (parseGithubDependency dependencyObject).rev
+          else
+            dependencyObject.version;
+      
+      pinVersions = dependencies: parentScopeDeps:
+        lib.mapAttrs
+          (pname: pdata:
+            let
+              selfScopeDeps = parentScopeDeps // (pdata.dependencies or {});
+            in
+              pdata // {
+                depsExact =
+                  if ! pdata ? requires then
+                    []
+                  else
+                    lib.forEach (lib.attrNames pdata.requires) (reqName: {
+                      name = reqName;
+                      version = getVersion selfScopeDeps."${reqName}";
+                    });
+                dependencies = pinVersions (pdata.dependencies or {}) selfScopeDeps;
+              }
+          )
+          dependencies;
+      
+      packageLockWithPinnedVersions = pinVersions parsed.dependencies parsed.dependencies;
+
+    in
+
+      utils.simpleTranslate translatorName {
+        # values
+        inputData = packageLockWithPinnedVersions;
+        mainPackageName = parsed.name;
+        mainPackageVersion = parsed.version;
+        mainPackageDependencies =
+          lib.mapAttrsToList
+            (pname: pdata:
+              { name = pname; version = getVersion pdata; })
+            (lib.filterAttrs
+              (pname: pdata: ! (pdata.dev or false) || dev)
+              parsed.dependencies);
+        buildSystemName = "nodejs";
+        buildSystemAttrs = { nodejsVersion = args.nodejs; };
+
+        # functions
+        serializePackages = inputData:
+          let
+            serialize = inputData:
+              lib.mapAttrsToList  # returns list of lists
+                (pname: pdata:
+                  [ (pdata // { inherit pname; }) ]
+                  ++
+                  (lib.optionals (pdata ? dependencies)
+                    (lib.flatten (serialize pdata.dependencies))))
+                inputData;
+          in
+            lib.filter
+              (pdata:
+                dev || ! (pdata.dev or false))
+              (lib.flatten (serialize inputData));
+
+        getName = dependencyObject: dependencyObject.pname;
+
+        inherit getVersion;
+
+        getSourceType = dependencyObject:
+          if dependencyObject ? from && dependencyObject ? version then
+            "github"
+          else
+            "fetchurl";
+        
+        sourceConstructors = {
+          github = dependencyObject:
+            let
+              githubData = parseGithubDependency dependencyObject;
+            in
+              rec {
+                version = builtins.substring 0 8 githubData.rev;
+                owner = githubData.org;
+                repo = githubData.repo;
+                rev = githubData.rev;
+              };
+
+          fetchurl = dependencyObject:
+            rec {
+              version = dependencyObject.version;
+              url = dependencyObject.resolved;
+              hash = dependencyObject.integrity;
+            };
+        };
+
+        getDependencies = dependencyObject: getDepByNameVer: getDepByOriginalID:
+          dependencyObject.depsExact;
+      };
+
+
+  compatiblePaths =
+    {
+      inputDirectories,
+      inputFiles,
+    }@args:
+    {
+      inputDirectories = lib.filter 
+        (utils.containsMatchingFile [ ''.*package-lock\.json'' ''.*package.json'' ])
+        args.inputDirectories;
+
+      inputFiles = [];
+    };
+
+  specialArgs = {
+
+    dev = {
+      description = "include dependencies for development";
+      type = "flag";
+    };
+
+    nodejs = {
+      description = "specify nodejs version";
+      default = lib.elemAt (lib.splitString "." nodejs.version) 0;
+      examples = [
+        "14"
+        "16"
+      ];
+      type = "argument";
+    };
+
+  };
+}

src/translators/nodejs/pure/yarn-lock/default.nix

@@ -19,16 +19,13 @@
       peer,
       ...
     }:
+
     let
       b = builtins;
       yarnLock = utils.readTextFile "${lib.elemAt inputDirectories 0}/yarn.lock";
       packageJSON = b.fromJSON (b.readFile "${lib.elemAt inputDirectories 0}/package.json");
-      parser = import ./parser.nix { inherit lib; inherit (externals) nix-parsec;};
+      parser = import ../yarn-lock/parser.nix { inherit lib; inherit (externals) nix-parsec;};
       tryParse = parser.parseLock yarnLock;
-
-      mainPackageName = packageJSON.name;
-      mainPackageKey = "${mainPackageName}#${packageJSON.version}";
-      
       parsedLock =
         if tryParse.type == "success" then
           lib.foldAttrs (n: a: n // a) {} tryParse.value
@@ -37,117 +34,126 @@
             failureOffset = tryParse.value.offset;
           in
             throw "parser failed at: \n${lib.substring failureOffset 50 tryParse.value.str}";
-      nameFromLockName = lockName:
-        let
-          version = lib.last (lib.splitString "@" lockName);
-        in
-          lib.removeSuffix "@${version}" lockName;
-      sources = lib.mapAttrs' (dependencyName: dependencyAttrs:
-        let
-          name = nameFromLockName dependencyName;
-        in
-          lib.nameValuePair ("${name}#${dependencyAttrs.version}") (
-          if lib.hasInfix "@github:" dependencyName
-              || lib.hasInfix "codeload.github.com/" dependencyAttrs.resolved then
-            let
-               gitUrlInfos = lib.splitString "/" dependencyAttrs.resolved;
-            in
-            {
-              type = "github";
-              rev = lib.elemAt gitUrlInfos 6;
-              owner = lib.elemAt gitUrlInfos 3;
-              repo = lib.elemAt gitUrlInfos 4;
-              version = dependencyAttrs.version;
-            }
-          else if lib.hasInfix "@link:" dependencyName then
-            {
-              version = dependencyAttrs.version;     
-              path = lib.last (lib.splitString "@link:" dependencyName);
-              type = "path";
-            }
-          else
-            {
-              version = dependencyAttrs.version;  
-              hash =
-                if dependencyAttrs ? integrity then
-                  dependencyAttrs.integrity
-                else
-                  throw "Missing integrity for ${dependencyName}";
-              url = lib.head (lib.splitString "#" dependencyAttrs.resolved);
-              type = "fetchurl";
-            }
-          )) parsedLock;
-      dependencyGraph =
-        (lib.mapAttrs'
-          (dependencyName: dependencyAttrs:
-            let
-              name = nameFromLockName dependencyName;
-              dependencies = 
-                dependencyAttrs.dependencies or []
-                ++ (lib.optionals optional (dependencyAttrs.optionalDependencies or []));
-              graph = lib.forEach dependencies (dependency:
-                builtins.head (
-                  lib.mapAttrsToList
-                    (name: value:
-                      let
-                        yarnName = "${name}@${value}";
-                        version = parsedLock."${yarnName}".version;
-                      in
-                      "${name}#${version}"
-                    )
-                    dependency
-                )
-              );
-            in
-              lib.nameValuePair ("${name}#${dependencyAttrs.version}") graph
-          )
-          parsedLock
-        )
-        //
-        {
-          "${mainPackageName}" =
-            lib.mapAttrsToList
-              (depName: depSemVer:
-                let
-                  depYarnKey = "${depName}@${depSemVer}";
-                  dependencyAttrs =
-                    if ! parsedLock ? "${depYarnKey}" then
-                      throw "Cannot find entry for top level dependency: '${depYarnKey}'"
-                    else
-                      parsedLock."${depYarnKey}";
-                in
-                  "${depName}#${dependencyAttrs.version}"
-              )
-              (
-                packageJSON.dependencies or {}
-                //
-                (lib.optionalAttrs dev (packageJSON.devDependencies or {}))
-                //
-                (lib.optionalAttrs peer (packageJSON.peerDependencies or {}))
-              );
-        };
-
-
     in
-    # TODO: produce dream lock like in /specifications/dream-lock-example.json
-      
-    rec {
-      inherit sources;
+    
+    utils.simpleTranslate translatorName rec {
 
-      generic = {
-        buildSystem = "nodejs";
-        producedBy = translatorName;
-        mainPackage = mainPackageName;
-        inherit dependencyGraph;
-        sourcesCombinedHash = null;
-      };
+      inputData = parsedLock;
+      mainPackageName = packageJSON.name;
+      mainPackageVersion = packageJSON.version;
+      buildSystemName = "nodejs";
 
-      # build system specific attributes
-      buildSystem = {
-
-        # example
+      buildSystemAttrs = {
         nodejsVersion = 14;
       };
+
+      mainPackageDependencies =
+        lib.mapAttrsToList
+          (depName: depSemVer:
+            let
+              depYarnKey = "${depName}@${depSemVer}";
+              dependencyAttrs =
+                if ! inputData ? "${depYarnKey}" then
+                  throw "Cannot find entry for top level dependency: '${depYarnKey}'"
+                else
+                  inputData."${depYarnKey}";
+            in
+              {
+                name = depName;
+                version = dependencyAttrs.version;
+              }
+          )
+          (
+            packageJSON.dependencies or {}
+            //
+            (lib.optionalAttrs dev (packageJSON.devDependencies or {}))
+            //
+            (lib.optionalAttrs peer (packageJSON.peerDependencies or {}))
+          );
+
+      serializePackages = inputData:
+        lib.mapAttrsToList
+          (yarnName: depAttrs: depAttrs // { inherit yarnName; })
+          parsedLock;
+
+      getOriginalID = dependencyObject:
+        dependencyObject.yarnName;
+
+      getName = dependencyObject:
+        let
+          version = lib.last (lib.splitString "@" dependencyObject.yarnName);
+        in
+          lib.removeSuffix "@${version}" dependencyObject.yarnName;
+
+      getVersion = dependencyObject:
+          dependencyObject.version;
+
+      getDependencies = dependencyObject: getDepByNameVer: getDepByOriginalID:
+        let
+          dependencies =
+            dependencyObject.dependencies or []
+            ++ (lib.optionals optional (dependencyObject.optionalDependencies or []));
+        in
+          lib.forEach
+            dependencies
+            (dependency: 
+              builtins.head (
+                lib.mapAttrsToList
+                  (name: value:
+                    let
+                      yarnName = "${name}@${value}";
+                      depObject = getDepByOriginalID yarnName; 
+                      version = depObject.version;
+                    in
+                      { inherit name version; }
+                  )
+                  dependency
+              )
+            );
+
+      getSourceType = dependencyObject:
+        if lib.hasInfix "@github:" dependencyObject.yarnName
+            || lib.hasInfix "codeload.github.com/" dependencyObject.resolved then
+          "github"
+        else if lib.hasInfix "@link:" dependencyObject.yarnName then
+          "path"
+        else
+          "fetchurl";
+
+      
+      sourceConstructors = {
+        github = dependencyObject:
+          let
+            gitUrlInfos = lib.splitString "/" dependencyObject.resolved;
+          in
+          {
+            type = "github";
+            rev = lib.elemAt gitUrlInfos 6;
+            owner = lib.elemAt gitUrlInfos 3;
+            repo = lib.elemAt gitUrlInfos 4;
+            version = dependencyObject.version;
+          };
+
+        path = dependencyObject:
+          {
+            version = dependencyObject.version;     
+            path = lib.last (lib.splitString "@link:" dependencyObject.yarnName);
+            type = "path";
+          };
+
+        fetchurl = dependencyObject:
+          {
+            type = "fetchurl";
+            version = dependencyObject.version;  
+            hash =
+              if dependencyObject ? integrity then
+                dependencyObject.integrity
+              else
+                throw "Missing integrity for ${dependencyObject.yarnName}";
+            url = lib.head (lib.splitString "#" dependencyObject.resolved);
+          };
+      };
+
     };
       
 

src/utils/default.nix

@@ -13,15 +13,23 @@ let
 
   b = builtins;
 
-  overrideUtils = callPackageDream ./overrideUtils.nix {};
+  overrideUtils = callPackageDream ./override.nix {};
+
+  dreamLockUtils = callPackageDream ./dream-lock.nix {};
+
+  translatorUtils = callPackageDream ./translator.nix {};
 
 in
 
+dreamLockUtils
+//
 overrideUtils
 //
+translatorUtils
+//
 rec {
 
-  readDreamLock = callPackageDream ./readDreamLock.nix {};
+  
 
   readTextFile = file: lib.replaceStrings [ "\r\n" ] [ "\n" ] (b.readFile file);
 

src/utils/dream-lock.nix

@@ -0,0 +1,28 @@
+{
+  lib,
+  ...
+}:
+let
+
+  b = builtins;
+
+  readDreamLock = 
+    {
+      dreamLock,
+    }@args:
+    let
+      lock =
+        if b.isPath dreamLock then
+          b.fromJSON (b.readFile dreamLock)
+        else
+          dreamLock;
+      mainPackage = lock.generic.mainPackage;
+      dependencyGraph = lock.generic.dependencyGraph;
+    in
+      lock;
+
+in
+  {
+    inherit readDreamLock;
+  }
+

src/utils/readDreamLock.nix

@@ -1,25 +0,0 @@
-{
-  lib,
-  ...
-}:
-let
-  b = builtins;
-in
-{
-  dreamLock,
-}@args:
-let
-
-  lock =
-    if b.isPath dreamLock then
-      b.fromJSON (b.readFile dreamLock)
-    else
-      dreamLock;
-
-  mainPackage = lock.generic.mainPackage;
-
-  dependencyGraph = lock.generic.dependencyGraph;
-
-in
-lock
-

src/utils/translator.nix

@@ -0,0 +1,105 @@
+{
+  lib,
+  ...
+}:
+let
+
+  b = builtins;
+
+  simpleTranslate = translatorName:
+    {
+      # values
+      inputData,
+      mainPackageName,
+      mainPackageVersion,
+      mainPackageDependencies,
+      buildSystemName,
+      buildSystemAttrs,
+
+      # functions
+      serializePackages,
+      getName,
+      getVersion,
+      getSourceType,
+      sourceConstructors,
+      getDependencies ? null,
+      getOriginalID ? null,
+      mainPackageSource ? { type = "unknown"; },
+    }:
+    let
+  
+      serializedPackagesList = serializePackages inputData;
+
+      allDependencies = b.foldl'
+        (result: pkgData: lib.recursiveUpdate result {
+          "${getName pkgData}" = {
+            "${getVersion pkgData}" = pkgData;
+          };
+        })
+        {}
+        serializedPackagesList;
+
+      allDependenciesByOriginalID = b.foldl'
+        (result: pkgData: lib.recursiveUpdate result {
+          "${getOriginalID pkgData}" = pkgData;
+        })
+        {}
+        serializedPackagesList;
+
+      sources = b.foldl'
+        (result: pkgData: lib.recursiveUpdate result {
+          "${getName pkgData}#${getVersion pkgData}" =
+            let
+              type = getSourceType pkgData;
+            in
+              (sourceConstructors."${type}" pkgData)
+              // { inherit type; };
+        })
+        {}
+        serializedPackagesList;
+      
+      dependencyGraph =
+        {
+          "${mainPackageName}#${mainPackageVersion}" =
+            lib.forEach mainPackageDependencies
+              (dep: "${dep.name}#${dep.version}");
+        }
+        //
+        lib.listToAttrs
+          (lib.forEach
+            serializedPackagesList
+            (pkgData: lib.nameValuePair
+              "${getName pkgData}#${getVersion pkgData}"
+              (b.map
+                (depNameVer: "${depNameVer.name}#${depNameVer.version}")
+                (getDependencies pkgData getDepByNameVer getDepByOriginalID))));
+      
+      getDepByOriginalID = id:
+        allDependenciesByOriginalID."${id}";
+      
+      getDepByNameVer = name: version:
+        allDependencies."${name}"."${version}";
+
+    in
+      {
+          inherit sources;
+
+          generic =
+            {
+              buildSystem = buildSystemName;
+              mainPackage = "${mainPackageName}#${mainPackageVersion}";
+              sourcesCombinedHash = null;
+              translator = translatorName;
+            }
+            //
+            (lib.optionalAttrs (getDependencies != null) { inherit dependencyGraph; });
+
+          # build system specific attributes
+          buildSystem = buildSystemAttrs;
+      };
+
+in
+  {
+    inherit simpleTranslate;
+  }
+