Merge pull request #215 from nix-community/nodejs

Imrovements for: nodejs, translators, indexers

src/apps/translate-index/default.nix

@@ -3,30 +3,36 @@
   translate,
   coreutils,
   jq,
+  parallel,
+  writeScript,
   ...
-}:
+}: let
-utils.writePureShellScriptBin
+  script = writeScript "run-translate" ''
-"translate-index"
+    ${translate}/bin/translate $1 $targetDir
-[coreutils translate jq]
+  '';
-''
+in
-  cd $WORKDIR
+  utils.writePureShellScriptBin
+  "translate-index"
+  [coreutils translate jq parallel]
+  ''
+    cd $WORKDIR
 
-  usage="usage:
+    usage="usage:
-    $0 INDEX_PATH TARGET_DIR"
+      $0 INDEX_PATH TARGET_DIR"
 
-  if [ "$#" -ne 2 ]; then
+    if [ "$#" -ne 2 ]; then
-    echo "error: wrong number of arguments"
+      echo "error: wrong number of arguments"
-    echo "$usage"
+      echo "$usage"
-    exit 1
+      exit 1
-  fi
+    fi
 
-  index=''${1:?"error: please pass index file path"}
+    index=''${1:?"error: please pass index file path"}
-  index=$(realpath $index)
+    index=$(realpath $index)
-  targetDir=''${2:?"error: please pass a target directory"}
+    targetDir=''${2:?"error: please pass a target directory"}
-  targetDir=$(realpath $targetDir)
+    targetDir=$(realpath $targetDir)
 
-  for shortcut in $(jq '.[]' -c -r $index); do
+    export targetDir
-    translate "$shortcut" $targetDir \
+
-      || echo "failed to translate \"$shortcut\""
+    parallel --halt now,fail=1 -j$(nproc) --delay 1 -a <(jq '.[]' -c -r $index) ${script}
-  done
+
-''
+  ''

src/fetchers/default-fetcher.nix

@@ -46,8 +46,8 @@
             source =
               source
               // {
-                pname = name;
+                pname = source.pname or name;
-                inherit version;
+                version = source.version or version;
               };
           }
         else throw "unsupported source type '${source.type}'")

src/indexers/libraries-io/default.nix

@@ -39,7 +39,7 @@ libraries.io also supports other interesting popularity metrics:
 
       outFile=$(jq '.outputFile' -c -r $input)
 
-      echo "loadein api key"
+      echo "loading api key"
       if [ -z ''${API_KEY+x} ]; then
         echo "Please define env variable API_KEY for libaries.io api key"
         exit 1
@@ -62,6 +62,7 @@ libraries.io also supports other interesting popularity metrics:
 
       rm -f $outFile
       for page in $(seq 1 $numPages); do
+        echo "requesting page $page"
         url="https://libraries.io/api/search?page=$page&sort=dependents_count&per_page=100&platforms=$platformQuery&api_key=$apiKey"
         curl -k "$url" | jq "$jqQuery" -r >> $outFile
       done

src/specifications/dream-lock-schema.json

@@ -97,8 +97,8 @@
                   },
                   "then": {
                     "properties": {
-                      "rootName": { "type": "string" },
+                      "rootName": { "type": ["string", "null"] },
-                      "rootVersion": { "type": "string" },
+                      "rootVersion": { "type": ["string", "null"] },
                       "path": { "type": "string" },
                       "type": { "type": "string" },
                       "dir": { "type": "string" }

src/subsystems/nodejs/builders/granular/default.nix

@@ -473,6 +473,9 @@
           elif [ -n "$runBuild" ] && [ "$(jq '.scripts.build' ./package.json)" != "null" ]; then
             npm run build
           else
+            if [ "$(jq '.scripts.preinstall' ./package.json)" != "null" ]; then
+              npm --production --offline --nodedir=$nodeSources run preinstall
+            fi
             if [ "$(jq '.scripts.install' ./package.json)" != "null" ]; then
               npm --production --offline --nodedir=$nodeSources run install
             fi

src/subsystems/nodejs/translators/package-json/default.nix

@@ -47,11 +47,31 @@
       cd ./$relPath
       rm -rf package-lock.json
 
-      if [ "$(jq '.project.subsystemInfo.noDev' -c -r $jsonInput)" == "true" ]; then
+      hasInstallScript=false
+      if [ "$(jq 'has("scripts")' -c -r package.json)" == "false" ]; then
+        :
+      elif [ "$(jq '.scripts | has("preinstall")' -c -r package.json)" == "true" ]; then
+        hasInstallScript=true
+      elif [ "$(jq '.scripts | has("install")' -c -r package.json)" == "true" ]; then
+        hasInstallScript=true
+      elif [ "$(jq '.scripts | has("postinstall")' -c -r package.json)" == "true" ]; then
+        hasInstallScript=true
+      else
+        :
+      fi
+
+      echo "translating in temp dir: $(pwd)"
+
+      if [ "$hasInstallScript" == "false" ]; then
+        echo "package.json does not define an [pre|post]install script -> omitting dev dependencies"
+      fi
+
+      if [ "$hasInstallScript" == "false" ] \
+          || [ "$(jq '.project.subsystemInfo.noDev' -c -r $jsonInput)" == "true" ]; then
         echo "excluding dev dependencies"
         jq '.devDependencies = {}' ./package.json > package.json.mod
         mv package.json.mod package.json
-        npm install --package-lock-only --production $npmArgs
+        npm install --package-lock-only --omit=dev $npmArgs
       else
         npm install --package-lock-only $npmArgs
       fi

src/subsystems/nodejs/translators/package-lock/default.nix

@@ -25,7 +25,15 @@
   } @ args: let
     b = builtins;
 
-    dev = ! noDev;
+    hasInstallScript =
+      (packageJson ? scripts.preinstall)
+      || (packageJson ? scripts.install)
+      || (packageJson ? scripts.postinstall);
+
+    noDev =
+      if ! hasInstallScript
+      then true
+      else args.noDev;
     name = project.name;
     tree = args.tree.getNodeFromPath project.relPath;
     relPath = project.relPath;
@@ -70,11 +78,23 @@
       then let
         path = getPath dependencyObject;
       in
-        (
+        if ! (l.pathExists "${source}/${path}/package.json")
-          b.fromJSON
+        then
-          (b.readFile "${source}/${path}/package.json")
+          throw ''
-        )
+            The lock file references a sub-package residing at '${source}/${path}',
-        .version
+            but that directory doesn't exist or doesn't contain a package.json
+
+            The reason might be that devDependencies are not included in this package release.
+            Possible solutions:
+              - get full package source via git and translate from there
+              - disable devDependencies by passing `noDev` to the translator
+          ''
+        else
+          (
+            b.fromJSON
+            (b.readFile "${source}/${path}/package.json")
+          )
+          .version
       else if lib.hasPrefix "https://" dependencyObject.version
       then "unknown"
       else dependencyObject.version;
@@ -153,7 +173,7 @@
           version = getVersion pdata;
         })
         (lib.filterAttrs
-          (pname: pdata: ! (pdata.dev or false) || dev)
+          (pname: pdata: ! (pdata.dev or false) || ! noDev)
           parsedDependencies);
 
       subsystemName = "nodejs";
@@ -186,7 +206,7 @@
       in
         lib.filter
         (pdata:
-          dev || ! (pdata.dev or false))
+          ! noDev || ! (pdata.dev or false))
         (lib.flatten (serialize inputData));
 
       getName = dependencyObject: dependencyObject.pname;

src/subsystems/nodejs/translators/utils.nix

@@ -2,8 +2,7 @@
   l = lib // builtins;
 in rec {
   getPackageJsonDeps = packageJson: noDev:
-    packageJson.dependencies
+    (packageJson.dependencies or {})
-    or {}
     // (lib.optionalAttrs (! noDev) (packageJson.devDependencies or {}));
 
   getWorkspaceLockFile = tree: project: fname: let
@@ -17,9 +16,9 @@ in rec {
       (tree.getNodeFromPath "${dirRelPath}/package.json").jsonContent;
 
     hasNoDependencies =
-      ! packageJson ? dependencies
+      ((packageJson.dependencies or {}) == {})
-      && ! packageJson ? devDependencies
+      && ((packageJson.devDependencies or {}) == {})
-      && ! packageJson ? workspaces;
+      && (! packageJson ? workspaces);
   in
     if hasNoDependencies
     then null

src/utils/default.nix

@@ -232,10 +232,16 @@ in
             | python3 ${../apps/cli/format-dream-lock.py} \
             | sponge $dreamLockPath
 
+          # validate dream-lock.json against jsonschema
+          ${python3.pkgs.jsonschema}/bin/jsonschema \
+            --instance $dreamLockPath \
+            --output pretty \
+            ${../specifications/dream-lock-schema.json}
+
           # add dream-lock.json to git
           if git rev-parse --show-toplevel &>/dev/null; then
             echo "adding file to git: $dreamLockPath"
-            git add $dreamLockPath
+            git add $dreamLockPath || :
           fi
         '';
     in

src/utils/dream-lock.nix

@@ -11,7 +11,9 @@
   replaceRootSources = {
     dreamLock,
     newSourceRoot,
-  }: let
+  } @ args: let
+    dreamLockLoaded = utils.readDreamLock {dreamLock = args.dreamLock;};
+    iface = dreamLockLoaded.interface;
     patchVersion = version: source:
       if
         source.type
@@ -24,10 +26,18 @@
           dir = source.relPath;
         }
       else source;
+
     patchedSources =
-      l.mapAttrs
+      l.recursiveUpdate
-      (_: versions: l.mapAttrs patchVersion versions)
+      {
-      dreamLock.sources;
+        "${iface.defaultPackageName}"."${iface.defaultPackageVersion}" =
+          newSourceRoot;
+      }
+      (
+        l.mapAttrs
+        (_: versions: l.mapAttrs patchVersion versions)
+        dreamLock.sources
+      );
   in
     dreamLock // {sources = patchedSources;};
 

src/utils/index.nix

@@ -39,7 +39,12 @@ in rec {
     dreamLocks = findDreamLocks tree;
     makePackagesForDreamLock = dreamLock:
       (dream2nixInterface.makeOutputsForDreamLock {
-        inherit dreamLock;
+        inherit
+          dreamLock
+          inject
+          packageOverrides
+          sourceOverrides
+          ;
       })
       .packages;
   in