Merge branch 'dev' into go

ci.nix

@@ -0,0 +1,17 @@
+let
+  b = builtins;
+  flakeCompatSrc = b.fetchurl "https://raw.githubusercontent.com/edolstra/flake-compat/12c64ca55c1014cdc1b16ed5a804aa8576601ff2/default.nix";
+  flake = (import flakeCompatSrc { src = ./.; }).defaultNix;
+  pkgs = import flake.inputs.nixpkgs {};
+  recurseIntoAll = b.mapAttrs (name: val: pkgs.recurseIntoAttrs val);
+
+in
+{
+  inherit flake;
+}
+
+// (recurseIntoAll {
+
+  checks = flake.checks.x86_64-linux;
+
+})

flake.nix

@@ -131,7 +131,14 @@
 
         # all apps including cli, install, etc.
         apps = forAllSystems (system: pkgs:
-          dream2nixFor."${system}".apps.flakeApps
+          dream2nixFor."${system}".apps.flakeApps // {
+            tests-impure = {
+              type = "app";
+              program =
+                b.toString
+                  (dream2nixFor."${system}".callPackageDream ./tests/impure {});
+            };
+          }
         );
 
         # a dev shell for working on dream2nix
@@ -155,7 +162,7 @@
           '';
         });
 
-        checks = forAllSystems (system: pkgs: import ./checks.nix {
+        checks = forAllSystems (system: pkgs: import ./tests/pure {
           inherit lib pkgs;
           dream2nix = dream2nixFor."${system}";
         });

src/apps/cli/commands/add.py

@@ -23,14 +23,17 @@ class AddCommand(Command):
   arguments = [
     argument(
       "source",
-      "source of the package, can be a path, tarball URL, or flake-style spec")
+      "Sources of the packages. Can be a paths, tarball URLs, or flake-style specs",
+      # multiple=True
+    )
   ]
 
   options = [
     option("translator", None, "which translator to use", flag=False),
     option("target", None, "target file/directory for the dream-lock.json", flag=False),
+    option("attribute-name", None, "attribute name for new new pakcage", flag=False),
     option(
-      "--packages-root",
+      "packages-root",
       None,
       "Put package under a new directory inside packages-root",
       flag=False
@@ -63,17 +66,9 @@ class AddCommand(Command):
     # ensure packages-root
     packages_root = self.find_packages_root()
 
-    # get source path and spec
-    source, sourceSpec = self.parse_source()
 
-    # select translator
-    translator = self.select_translator(source)
-
-    # raise error if any specified extra arg is unknown
-    specified_extra_args = self.declare_extra_args(specified_extra_args, translator)
-
-    # do the translation and produce dream lock
-    lock = self.run_translate(source, specified_extra_args, translator)
+    lock, sourceSpec, specified_extra_args, translator =\
+      self.translate_from_source(specified_extra_args, self.argument("source"))
 
     # get package name and version from lock
     mainPackageName = lock['_generic']['mainPackageName']
@@ -114,12 +109,23 @@ class AddCommand(Command):
 
     # create default.nix
     if 'default.nix' in filesToCreate:
-      self.create_default_nix(lock, output, outputDefaultNix, source)
+      self.create_default_nix(lock, output, outputDefaultNix, sources[0])
 
     # add new package to git
     if config['isRepo']:
       sp.run(["git", "add", "-N", output])
 
+  def translate_from_source(self, specified_extra_args, source):
+    # get source path and spec
+    source, sourceSpec = self.parse_source(source)
+    # select translator
+    translator = self.select_translator(source)
+    # raise error if any specified extra arg is unknown
+    specified_extra_args = self.declare_extra_args(specified_extra_args, translator)
+    # do the translation and produce dream lock
+    lock = self.run_translate(source, specified_extra_args, translator)
+    return lock, sourceSpec, specified_extra_args, translator
+
   def parse_extra_args(self):
     specified_extra_args = {
       arg[0]: arg[1] for arg in map(
@@ -297,20 +303,20 @@ class AddCommand(Command):
     return filesToCreate, output
 
   def define_attribute_name(self, mainPackageName):
-    mainPackageDirName = mainPackageName.strip('@').replace('/', '-')
+    attributeName = self.option('attribute-name')
+    if attributeName:
+      return attributeName
+
+    attributeName = mainPackageName.strip('@').replace('/', '-')
 
     # verify / change main package dir name
-    def update_name(mainPackageDirName):
-      print(f"Current package attribute name is: {mainPackageDirName}")
-      new_name = self.ask(
-        "Specify new attribute name or leave empty to keep current:"
-      )
-      if new_name:
-        return new_name
-      return mainPackageDirName
-
-    mainPackageDirName = update_name(mainPackageDirName)
-    return mainPackageDirName
+    print(f"Current package attribute name is: {attributeName}")
+    new_name = self.ask(
+      "Specify new attribute name or leave empty to keep current:"
+    )
+    if new_name:
+      attributeName = newName
+    return attributeName
 
   def run_translate(self, source, specified_extra_args, translator):
     # build the translator bin
@@ -336,7 +342,7 @@ class AddCommand(Command):
 
         # execute translator
         sp.run(
-          [f"{translator_path}/bin/run", input_json_file.name]
+          [f"{translator_path}", input_json_file.name]
         )
 
       # raise error if output wasn't produced
@@ -452,9 +458,8 @@ class AddCommand(Command):
         exit(1)
     return translator
 
-  def parse_source(self):
+  def parse_source(self, source):
     # verify source
-    source = self.argument("source")
     if not source and not config['packagesDir']:
       source = os.path.realpath('./.')
       print(

src/apps/cli/commands/update.py

@@ -66,7 +66,7 @@ class UpdateCommand(Command):
         dreamLock=dreamLockFile,
         updater=updater,
       )
-      update_proc = sp.run([f"{update_script}/bin/run"], capture_output=True)
+      update_proc = sp.run([f"{update_script}"], capture_output=True)
       version = update_proc.stdout.decode().strip()
     print(f"Updating from version {old_version} to {version}")
 

src/apps/cli/default.nix

@@ -22,25 +22,22 @@ let
 
 in
 {
-  program = 
-    let
-      script = utils.writePureShellScript
-        [
-          gitMinimal
-          nix
-        ]
-        ''
-          # escape the temp dir created by writePureShellScript
-          cd - > /dev/null
+  program =
+    utils.writePureShellScript
+      [
+        gitMinimal
+        nix
+      ]
+      ''
+        # escape the temp dir created by writePureShellScript
+        cd - > /dev/null
 
-          # run the cli
-          dream2nixConfig=${configFile} \
-          dream2nixSrc=${dream2nixWithExternals} \
-          fetcherNames="${b.toString (lib.attrNames fetchers.fetchers)}" \
-            ${cliPython}/bin/python ${./.}/cli.py "$@"
-        '';
-    in
-      "${script}/bin/run";
+        # run the cli
+        dream2nixConfig=${configFile} \
+        dream2nixSrc=${dream2nixWithExternals} \
+        fetcherNames="${b.toString (lib.attrNames fetchers.fetchers)}" \
+          ${cliPython}/bin/python ${./.}/cli.py "$@"
+      '';
 
   templateDefaultNix =
     {

src/default.nix

@@ -57,6 +57,7 @@ let
 
   # like pkgs.callPackage, but includes all the dream2nix modules
   callPackageDream = f: args: pkgs.callPackage f (args // {
+    inherit apps;
     inherit builders;
     inherit callPackageDream;
     inherit config;
@@ -172,7 +173,7 @@ let
             fetchedSources;
       };
 
-  
+
   makeDreamLockForSource =
     {
       source,
@@ -330,7 +331,7 @@ let
 
 
   # produce outputs for a dream-lock or a source
-  riseAndShine = 
+  riseAndShine =
     {
       dreamLock ? null,
       builder ? null,
@@ -403,12 +404,13 @@ let
 
     in
       builderOutputs;
-   
+
 in
 {
   inherit
     apps
     builders
+    callPackageDream
     dream2nixWithExternals
     fetchers
     fetchSources

src/templates/translators/impure.nix

@@ -12,7 +12,7 @@
 
 {
 
-  # A derivation which outputs an executable at `/bin/run`.
+  # A derivation which outputs a single executable at `$out`.
   # The executable will be called by dream2nix for translation
   # The input format is specified in /specifications/translator-call-example.json.
   # The first arg `$1` will be a json file containing the input parameters
@@ -57,10 +57,10 @@
       # examples:
       #   - ''.*requirements.*\.txt''
       #   - ''.*package-lock\.json''
-      inputDirectories = lib.filter 
+      inputDirectories = lib.filter
         (utils.containsMatchingFile [ ''TODO: regex1'' ''TODO: regex2'' ])
         args.inputDirectories;
-      
+
       inputFiles = [];
     };
 

src/translators/default.nix

@@ -10,14 +10,14 @@
   dream2nixWithExternals,
   utils,
   ...
-}: 
+}:
 let
 
   b = builtins;
 
   lib = pkgs.lib;
 
-  callTranslator = subsystem: type: name: file: args: 
+  callTranslator = subsystem: type: name: file: args:
     let
       translator = callPackageDream file (args // {
         inherit externals;
@@ -37,7 +37,7 @@ let
           translator.translate
             ((getextraArgsDefaults translator.extraArgs or {}) // args);
       };
-      
+
 
   subsystems = utils.dirNames ./.;
 
@@ -59,10 +59,10 @@ let
           nix eval --show-trace --impure --raw --expr "
               let
                 dream2nix = import ${dream2nixWithExternals} {};
-                dreamLock = 
+                dreamLock =
                   dream2nix.translators.translators.${
                     lib.concatStringsSep "." translatorAttrPath
-                  }.translate 
+                  }.translate
                     (builtins.fromJSON (builtins.readFile '''$1'''));
               in
                 dream2nix.utils.dreamLock.toJSON
@@ -95,23 +95,62 @@ let
   # flat list of all translators
   translatorsList = lib.collect (v: v ? translateBin) translators;
 
-  # json file exposing all existing translators to CLI including their special args
+  # returns the list of translators including their special args
+  # and adds a flag `compatible` to each translator indicating
+  # if the translator is compatible to all given paths
   translatorsForInput =
     {
       inputDirectories,
       inputFiles,
     }@args:
     lib.forEach translatorsList
-      (t: {
+      (t: rec {
         inherit (t)
           name
           extraArgs
           subsystem
           type
         ;
-        compatible = t.compatiblePaths args == args;
+        compatiblePaths = t.compatiblePaths args;
+        compatible = compatiblePaths == args;
       });
 
+  # also includes subdirectories of the given paths up to a certain depth
+  # to check for translator compatibility
+  translatorsForInputRecursive =
+    {
+      inputDirectories,
+      inputFiles,
+      depth ? 2,
+    }:
+    let
+      listDirsRec = dir: depth:
+        let
+          subDirs = (utils.listDirs dir);
+        in
+          if depth == 0 then
+            subDirs
+          else
+            subDirs
+            ++
+            (lib.flatten
+              (map
+                (subDir: listDirsRec subDir (depth -1))
+                subDirs));
+
+      dirsToCheck =
+        lib.flatten
+          (map
+            (inputDir: listDirsRec inputDir depth)
+            inputDirectories);
+
+    in
+      translatorsForInput {
+        inputDirectories = dirsToCheck;
+        inherit inputFiles;
+      };
+
+
   # pupulates a translators special args with defaults
   getextraArgsDefaults = extraArgsDef:
     lib.mapAttrs

src/translators/nodejs/impure/package-json/default.nix

@@ -35,8 +35,8 @@
       cat package-lock.json
 
       jq ".inputDirectories[0] = \"$(pwd)\"" -c -r $jsonInput > ./newJsonInput
-      
-      ${translators.translators.nodejs.pure.package-lock.translateBin}/bin/run $(realpath ./newJsonInput)
+
+      ${translators.translators.nodejs.pure.package-lock.translateBin} $(realpath ./newJsonInput)
     '';
 
 
@@ -49,10 +49,10 @@
       inputFiles,
     }@args:
     {
-      inputDirectories = lib.filter 
+      inputDirectories = lib.filter
         (utils.containsMatchingFile [ ''.*package.json'' ])
         args.inputDirectories;
-      
+
       inputFiles = [];
     };
 

src/translators/nodejs/pure/yarn-lock/default.nix

@@ -2,6 +2,7 @@
   lib,
 
   externals,
+  nodejs,
   translatorName,
   utils,
   ...
@@ -16,14 +17,15 @@
       # extraArgs
       name,
       noDev,
+      nodejs,
       peer,
       ...
-    }:
+    }@args:
 
     let
       b = builtins;
       dev = ! noDev;
-  
+
       sourceDir = lib.elemAt inputDirectories 0;
       yarnLock = utils.readTextFile "${sourceDir}/yarn.lock";
       packageJSON = b.fromJSON (b.readFile "${sourceDir}/package.json");
@@ -43,7 +45,7 @@
               ${lib.substring failureOffset 50 tryParse.value.str}
             '';
     in
-    
+
     utils.simpleTranslate translatorName rec {
 
       inputData = parsedLock;
@@ -60,9 +62,7 @@
 
       subsystemName = "nodejs";
 
-      subsystemAttrs = {
-        nodejsVersion = 14;
-      };
+      subsystemAttrs = { nodejsVersion = args.nodejs; };
 
       mainPackageDependencies =
         lib.mapAttrsToList
@@ -116,13 +116,13 @@
         in
           lib.forEach
             dependencies
-            (dependency: 
+            (dependency:
               builtins.head (
                 lib.mapAttrsToList
                   (name: versionSpec:
                     let
                       yarnName = "${name}@${versionSpec}";
-                      depObject = dependenciesByOriginalID."${yarnName}"; 
+                      depObject = dependenciesByOriginalID."${yarnName}";
                       version = depObject.version;
                     in
                       if ! dependenciesByOriginalID ? ${yarnName} then
@@ -153,7 +153,7 @@
                   && lib.hasInfix "codeload.github.com/" dObj.resolved
 
               || lib.hasInfix "@git+" dObj.yarnName
-              
+
               # example:
               # "jest-image-snapshot@https://github.com/machard/jest-image-snapshot#machard-patch-1":
               #   version "4.2.0"
@@ -175,7 +175,7 @@
           else
             "http";
 
-      
+
       sourceConstructors = {
         git = dependencyObject:
           if utils.identifyGitUrl dependencyObject.resolved then
@@ -215,13 +215,13 @@
         path = dependencyObject:
           if lib.hasInfix "@link:" dependencyObject.yarnName then
             {
-              version = dependencyObject.version;     
+              version = dependencyObject.version;
               path =
                 lib.last (lib.splitString "@link:" dependencyObject.yarnName);
             }
           else if lib.hasInfix "@file:" dependencyObject.yarnName then
             {
-              version = dependencyObject.version;     
+              version = dependencyObject.version;
               path =
               lib.last (lib.splitString "@file:" dependencyObject.yarnName);
             }
@@ -231,7 +231,7 @@
         http = dependencyObject:
           {
             type = "http";
-            version = dependencyObject.version;  
+            version = dependencyObject.version;
             hash =
               if dependencyObject ? integrity then
                 dependencyObject.integrity
@@ -256,7 +256,7 @@
 
 
     };
-      
+
 
   # From a given list of paths, this function returns all paths which can be processed by this translator.
   # This allows the framework to detect if the translator is compatible with the given inputs
@@ -267,7 +267,7 @@
       inputFiles,
     }@args:
     {
-      inputDirectories = lib.filter 
+      inputDirectories = lib.filter
         (utils.containsMatchingFile [ ''.*yarn\.lock'' ''.*package.json'' ])
         args.inputDirectories;
 
@@ -297,6 +297,16 @@
       type = "flag";
     };
 
+    nodejs = {
+      description = "nodejs version to use for building";
+      default = lib.elemAt (lib.splitString "." nodejs.version) 0;
+      examples = [
+        "14"
+        "16"
+      ];
+      type = "argument";
+    };
+
     peer = {
       description = "Include peer dependencies";
       type = "flag";

src/utils/default.nix

@@ -5,7 +5,7 @@
   lib,
   nix,
   runCommand,
-  writeScriptBin,
+  writeScript,
 
   # dream2nix inputs
   callPackageDream,
@@ -15,7 +15,7 @@
 let
 
   b = builtins;
-  
+
   dreamLockUtils = callPackageDream ./dream-lock.nix {};
 
   overrideUtils = callPackageDream ./override.nix {};
@@ -58,6 +58,8 @@ rec {
 
   listFiles = path: lib.attrNames (lib.filterAttrs (n: v: v == "regular") (builtins.readDir path));
 
+  listDirs = path: lib.attrNames (lib.filterAttrs (n: v: v == "directory") (builtins.readDir path));
+
   # directory names of a given directory
   dirNames = dir: lib.attrNames (lib.filterAttrs (name: type: type == "directory") (builtins.readDir dir));
 
@@ -87,7 +89,7 @@ rec {
       b.readFile hashFile;
 
   # builder to create a shell script that has it's own PATH
-  writePureShellScript = availablePrograms: script: writeScriptBin "run" ''
+  writePureShellScript = availablePrograms: script: writeScript "script.sh" ''
     #!${bash}/bin/bash
     set -Eeuo pipefail
 
@@ -118,7 +120,7 @@ rec {
           ''
           + old.postFetch;
       });
-  
+
   sanitizeDerivationName = name:
     lib.replaceStrings [ "@" "/" ] [ "__at__" "__slash__" ] name;
 
@@ -165,7 +167,7 @@ rec {
 
   satisfiesSemver = poetry2nixSemver.satisfiesSemver;
 
-  # like nixpkgs recursiveUpdateUntil, but the depth of the 
+  # like nixpkgs recursiveUpdateUntil, but the depth of the
   recursiveUpdateUntilDepth = depth: lhs: rhs:
     lib.recursiveUpdateUntil (path: l: r: (b.length path) > depth) lhs rhs;
 

src/utils/dream-lock.nix

@@ -9,7 +9,7 @@ let
 
   b = builtins;
 
-  readDreamLock = 
+  readDreamLock =
     {
       dreamLock,
     }@args:
@@ -49,7 +49,7 @@ let
         b.filter
           (dep: ! b.elem dep cyclicDependencies."${pname}"."${version}" or [])
           dependencyGraph."${pname}"."${version}" or [];
-      
+
       getCyclicDependencies = pname: version:
         cyclicDependencies."${pname}"."${version}" or [];
 
@@ -100,7 +100,7 @@ let
     getSubDreamLock = dreamLock: name: version:
       let
         lock = (readDreamLock { inherit dreamLock; }).lock;
-      
+
       in
         lock // {
           _generic = lock._generic // {

tests/impure/default.nix

@@ -0,0 +1,34 @@
+{
+  lib,
+
+  # dream2nix
+  callPackageDream,
+  utils,
+  ...
+}:
+let
+
+  l = lib // builtins;
+
+  allTestFiles =
+    l.attrNames
+      (l.filterAttrs
+        (name: type: type == "regular" && l.hasPrefix "test_" name)
+        (l.readDir ./.));
+
+  allTests =
+    l.map
+      (file: callPackageDream ("${./.}/${file}") {})
+      allTestFiles;
+
+  executeAll = utils.writePureShellScript
+    []
+    ''
+      for test in ${toString allTests}; do
+        $test
+      done
+    '';
+
+
+in
+  executeAll

tests/impure/test_package-lock.nix

@@ -0,0 +1,27 @@
+{
+  lib,
+
+  # dream2nix
+  apps,
+  utils,
+  ...
+}:
+let
+
+  l = lib // builtins;
+
+  cli = apps.cli.program;
+
+in
+utils.writePureShellScript
+[]
+''
+  ${cli} add github:mattermost/mattermost-webapp/v6.1.0 \
+    --no-default-nix \
+    --translator package-lock \
+    --attribute-name mattermost-webapp \
+    --arg name="{automatic}" \
+    --arg noDev=false \
+    --arg nodejs=14 \
+    --arg peer=false
+''

tests/impure/test_yarn-lock-prettier.nix

@@ -0,0 +1,27 @@
+{
+  lib,
+
+  # dream2nix
+  apps,
+  utils,
+  ...
+}:
+let
+
+  l = lib // builtins;
+
+  cli = apps.cli.program;
+
+in
+utils.writePureShellScript
+[]
+''
+  ${cli} add github:prettier/prettier/2.4.1 \
+    --no-default-nix \
+    --translator yarn-lock \
+    --attribute-name prettier \
+    --arg name="{automatic}" \
+    --arg noDev=false \
+    --arg nodejs=14 \
+    --arg peer=false
+''

tests/pure/default.nix

@@ -28,9 +28,15 @@ let
         url = "https://github.com/prettier/prettier/tarball/2.4.1";
         sha256 = "19b37qakhlsnr2n5bgv83aih5npgzbad1d2p2rs3zbq5syqbxdyi";
       };
-      cmds = outputs: [
-        "${outputs.defaultPackage}/bin/prettier --version | grep -q 2.4.1 && mkdir $out"
-      ];
+      cmds = outputs:
+        let
+          prettier = outputs.defaultPackage.overrideAttrs (old: {
+            dontBuild = true;
+          });
+        in
+          [
+            "${prettier}/bin/prettier --version | grep -q 2.4.1 && mkdir $out"
+          ];
     };
   };
 
@@ -38,6 +44,6 @@ let
     lib.mapAttrs
       (name: args: makeTest (args // { inherit name; }))
       projects;
-  
+
 in
 allTests