Merge pull request #42 from DavHau/dev

Yarn translator and nodejs builder improvements
2025-01-03 20:06:15 +03:00 · 2021-11-03 10:54:22 +07:00 · 2021-11-03 10:54:22 +07:00 · 12d837e086
commit 12d837e086
parent ddc1fdde6d b1cb420434
12 changed files with 210 additions and 132 deletions
--- a/flake.nix
+++ b/flake.nix
@ -96,10 +96,21 @@
        # System independent dream2nix api.
        # Similar to drem2nixFor but will require 'system(s)' or 'pkgs' as an argument.
        # Produces flake-like output schema.
-        lib = import ./src/lib.nix {
+        lib = (import ./src/lib.nix {
          inherit externalSources overridesDir lib;
          nixpkgsSrc = "${nixpkgs}";
-        };
+        })
+        # system specific dream2nix library
+        // (forAllSystems (system: pkgs:
+          import ./src {
+            inherit
+              externalSources
+              lib
+              overridesDir
+              pkgs
+            ;
+          }
+        ));

        # the dream2nix cli to be used with 'nix run dream2nix'
        defaultApp = forAllSystems (system: pkgs: self.apps."${system}".cli);
--- a/overrides/nodejs/default.nix
+++ b/overrides/nodejs/default.nix
@ -8,6 +8,43 @@ let
 in

 {
+  degit = {
+    run-build = {
+      installScript = ''
+        npm run build
+        cp help.md ./dist
+      '';
+    };
+  };
+
+  esbuild = {
+    "add-binary-0.12.17" = {
+      _condition = pkg: pkg.version == "0.12.17";
+      ESBUILD_BINARY_PATH =
+        let
+          esbuild = pkgs.buildGoModule rec {
+            pname = "esbuild";
+            version = "0.12.17";
+
+            src = pkgs.fetchFromGitHub {
+              owner = "evanw";
+              repo = "esbuild";
+              rev = "v${version}";
+              sha256 = "sha256-wZOBjNOgGmwIQNCrhzwGPmI/fW/yZiDqq8l4oSDTvZs=";
+            };
+
+            vendorSha256 = "sha256-2ABWPqhK2Cf4ipQH7XvRrd+ZscJhYPc3SV2cGT0apdg=";
+          };
+        in
+          "${esbuild}/bin/esbuild";
+    };
+  };
+
+  geckodriver = {
+    add-binary = {
+      GECKODRIVER_FILEPATH = "${pkgs.geckodriver}/bin/geckodriver";
+    };
+  };

  gifsicle = {
    add-binary = {
@ -72,4 +109,5 @@ in
      '';
    };
  };
+
 }
--- a/src/apps/cli/commands/package.py
+++ b/src/apps/cli/commands/package.py
@ -278,9 +278,12 @@ class PackageCommand(Command):
      mainSource = dict(
        type="unknown",
      )
-    lock['sources'][mainPackageName] = {
-      mainPackageVersion: mainSource
-    }
+    if mainPackageName not in lock['sources']:
+      lock['sources'][mainPackageName] = {
+        mainPackageVersion: mainSource
+      }
+    else:
+      lock['sources'][mainPackageName][mainPackageVersion] = mainSource

    # clean up dependency graph
    # remove empty entries
--- a/src/builders/nodejs/granular/default.nix
+++ b/src/builders/nodejs/granular/default.nix
@ -113,6 +113,11 @@ let
        
          pname = utils.sanitizeDerivationName name;

+          # only run build on the main package
+          runBuild =
+            packageName == mainPackageName
+                && version == mainPackageVersion;
+
          inherit dependenciesJson nodeDeps nodeSources version;

          src = getSource name version;
@ -241,12 +246,12 @@ let
                  if [[ $module == @* ]]; then
                    for submodule in $(ls $dep/lib/node_modules/$module); do
                      mkdir -p $nodeModules/$packageName/node_modules/$module
-                      echo -e "creating link: $dep/lib/node_modules/$module/$submodule\n  -> $nodeModules/$packageName/node_modules/$module/$submodule"
+                      echo "installing: $module/$submodule"
                      ln -s $dep/lib/node_modules/$module/$submodule $nodeModules/$packageName/node_modules/$module/$submodule
                    done
                  else
                    mkdir -p $nodeModules/$packageName/node_modules/
-                    echo -e "creating link: $dep/lib/node_modules/$module\n  -> $nodeModules/$packageName/node_modules/$module"
+                    echo "installing: $module"
                    ln -s $dep/lib/node_modules/$module $nodeModules/$packageName/node_modules/$module
                  fi
                done
@ -270,6 +275,8 @@ let
              else
                echo "$installScript" | bash
              fi
+            elif [ -n "$runBuild" ] && [ "$(jq '.scripts.build' ./package.json)" != "null" ]; then
+              npm run build
            elif [ "$(jq '.scripts.postinstall' ./package.json)" != "null" ]; then
              npm --production --offline --nodedir=$nodeSources run postinstall
            fi
@ -279,14 +286,15 @@ let

          # Symlinks executables and manual pages to correct directories
          d2nPostInstallPhase = ''
-            # Create symlink to the deployed executable folder, if applicable
+            
+            echo "Symlinking exectuables to /bin"
            if [ -d "$nodeModules/.bin" ]
            then
              chmod +x $nodeModules/.bin/*
              ln -s $nodeModules/.bin $out/bin
            fi

-            # Create symlinks to the deployed manual page folders, if applicable
+            echo "Symlinking manual pages"
            if [ -d "$nodeModules/$packageName/man" ]
            then
              mkdir -p $out/share
--- a/src/builders/nodejs/granular/fix-package.py
+++ b/src/builders/nodejs/granular/fix-package.py
@ -5,11 +5,13 @@ import sys


 with open(os.environ.get('dependenciesJsonPath')) as f:
-  actual_deps = json.load(f)
+  available_deps = json.load(f)

 with open('package.json') as f:
  package_json = json.load(f)

+out = os.environ.get('out')
+
 changed = False

 # fail if platform incompatible
@ -38,43 +40,26 @@ if package_json['version'] != version:
  changed = True
  package_json['version'] = version

-# delete devDependencies
-# We rely on dream.lock having the correct dependencies specified
-if 'devDependencies' in package_json:
-  print(
-    f"package.json: removed all devDependencies",
-    file=sys.stderr
-  )
-  changed = True
-  del package_json['devDependencies']
-
-# delete peerDependencies
-# We rely on dream.lock instead
-if 'peerDependencies' in package_json:
-  print(
-    f"package.json: removed all peerDependencies",
-    file=sys.stderr
-  )
-  changed = True
-  del package_json['peerDependencies']

 # pinpoint exact versions
 # This is mostly needed to replace git references with exact versions,
 # as NPM install will otherwise re-fetch these
 if 'dependencies' in package_json:
  for pname, version in package_json['dependencies'].items():
-    if actual_deps[pname] != package_json['dependencies'][pname]:
-      package_json['dependencies'][pname] = actual_deps[pname]
+    if 'bundledDependencies' in package_json\
+        and pname in package_json['bundledDependencies']:
+      continue
+    if available_deps[pname] != package_json['dependencies'][pname]:
+      package_json['dependencies'][pname] = available_deps[pname]
      changed = True
      print(
-        f"package.json: Pinning version '{version}' to '{actual_deps[pname]}'"
+        f"package.json: Pinning version '{version}' to '{available_deps[pname]}'"
        f" for dependency '{pname}'",
        file=sys.stderr
      )

 # create symlinks for executables (bin entries from package.json)
 if 'bin' in package_json:
-  out = os.environ.get('out')
  bin = package_json['bin']

  if isinstance(bin, str):
@ -99,7 +84,6 @@ if 'bin' in package_json:
      dest = os.path.relpath(relpath, sourceDir)
      os.symlink(dest, source)

-
 # write changes to package.json
 if changed:
  with open('package.json', 'w') as f:
--- a/src/default.nix
+++ b/src/default.nix
@ -248,8 +248,7 @@ let
              inherit fetchedSources;
            };

-      in
-        builder ( builderArgs // {
+        outputs = builder ( builderArgs // {

          inherit
            buildPackageWithOtherBuilder
@ -269,6 +268,9 @@ let

        });

+      in
+        outputs;
+

  # produce outputs for a dream.lock or a source
  riseAndShine = 
@ -331,8 +333,9 @@ let
        builder = builder';
        builderArgs = (args.builderArgs or {}) // {
          packageOverrides =
-            args.packageOverrides or {}
-            // dreamOverrides."${dreamLock.generic.buildSystem}" or {};
+            lib.recursiveUpdate
+              (dreamOverrides."${dreamLock.generic.buildSystem}" or {})
+              (args.packageOverrides or {});
        };
      };

--- a/src/fetchers/http/default.nix
+++ b/src/fetchers/http/default.nix
@ -22,15 +22,19 @@
      });

      fetched = hash:
-        if lib.stringLength hash == 40 then
-          fetchurl {
-            inherit url;
-            sha1 = hash;
-          }
-        else
-          fetchurl {
-            inherit url hash;
-          };
+        let drv = 
+          if lib.stringLength hash == 40 then
+            fetchurl {
+              inherit url;
+              sha1 = hash;
+            }
+          else
+            fetchurl {
+              inherit url hash;
+            };
+        in drv.overrideAttrs (old: {
+          name = lib.strings.sanitizeDerivationName old.name;
+        });

    };
 }
--- a/src/translators/nodejs/impure/package-json/default.nix
+++ b/src/translators/nodejs/impure/package-json/default.nix
@ -49,10 +49,6 @@
      inputFiles,
    }@args:
    {
-      # TODO: insert regex here that matches valid input file names
-      # examples:
-      #   - ''.*requirements.*\.txt''
-      #   - ''.*package-lock\.json''
      inputDirectories = lib.filter 
        (utils.containsMatchingFile [ ''.*package.json'' ])
        args.inputDirectories;
@ -60,31 +56,5 @@
      inputFiles = [];
    };

-
-  # If the translator requires additional arguments, specify them here.
-  # There are only two types of arguments:
-  #   - string argument (type = "argument")
-  #   - boolean flag (type = "flag")
-  # String arguments contain a default value and examples. Flags do not.
-  extraArgs = {
-
-    # # Example: boolean option
-    # # Flags always default to 'false' if not specified by the user
-    # dev-dependenices = {
-    #   description = "Include dev dependencies";
-    #   type = "flag";
-    # };
-
-    # # Example: string option
-    # the-answer = {
-    #   default = "42";
-    #   description = "The Answer to the Ultimate Question of Life";
-    #   examples = [
-    #     "0"
-    #     "1234"
-    #   ];
-    #   type = "argument";
-    # };
-
-  };
+  extraArgs = {};
 }
--- a/src/translators/nodejs/pure/package-lock/default.nix
+++ b/src/translators/nodejs/pure/package-lock/default.nix
@ -91,10 +91,21 @@
            serialize = inputData:
              lib.mapAttrsToList  # returns list of lists
                (pname: pdata:
-                  [ (pdata // { inherit pname; }) ]
+                  [ (pdata // {
+                      inherit pname;
+                      depsExact =
+                        lib.filter
+                          (req:
+                            (! (pdata.dependencies."${req.name}".bundled or false)))
+                          pdata.depsExact or {};
+                    }) ]
                  ++
                  (lib.optionals (pdata ? dependencies)
-                    (lib.flatten (serialize pdata.dependencies))))
+                    (lib.flatten
+                      (serialize
+                        (lib.filterAttrs
+                          (pname: data: ! data.bundled or false)
+                          pdata.dependencies)))))
                inputData;
          in
            lib.filter
@ -146,7 +157,7 @@
  extraArgs = {

    noDev = {
-      description = "Whether to exclude development dependencies";
+      description = "Exclude development dependencies";
      type = "flag";
    };

--- a/src/translators/nodejs/pure/yarn-lock/default.nix
+++ b/src/translators/nodejs/pure/yarn-lock/default.nix
@ -16,7 +16,7 @@
      # extraArgs
      name,
      noDev,
-      optional,
+      noOptional,
      peer,
      ...
    }:
@ -24,9 +24,14 @@
    let
      b = builtins;
      dev = ! noDev;
-      yarnLock = utils.readTextFile "${lib.elemAt inputDirectories 0}/yarn.lock";
-      packageJSON = b.fromJSON (b.readFile "${lib.elemAt inputDirectories 0}/package.json");
-      parser = import ../yarn-lock/parser.nix { inherit lib; inherit (externals) nix-parsec;};
+      optional = ! noOptional;
+  
+      sourceDir = lib.elemAt inputDirectories 0;
+      yarnLock = utils.readTextFile "${sourceDir}/yarn.lock";
+      packageJSON = b.fromJSON (b.readFile "${sourceDir}/package.json");
+      parser = import ../yarn-lock/parser.nix
+        { inherit lib; inherit (externals) nix-parsec;};
+
      tryParse = parser.parseLock yarnLock;
      parsedLock =
        if tryParse.type == "success" then
@ -35,7 +40,10 @@
          let
            failureOffset = tryParse.value.offset;
          in
-            throw "parser failed at: \n${lib.substring failureOffset 50 tryParse.value.str}";
+            throw ''
+              parser failed at:
+              ${lib.substring failureOffset 50 tryParse.value.str}
+            '';
    in
    
    utils.simpleTranslate translatorName rec {
@ -45,7 +53,10 @@
      mainPackageName =
        packageJSON.name or
          (if name != null then name else
-            throw "Could not identify package name. Please specify extra argument 'name'");
+            throw (
+              "Could not identify package name. "
+              + "Please specify extra argument 'name'"
+            ));

      mainPackageVersion = packageJSON.version or "unknown";

@ -88,10 +99,13 @@
        dependencyObject.yarnName;

      getName = dependencyObject:
-        let
-          version = lib.last (lib.splitString "@" dependencyObject.yarnName);
-        in
-          lib.removeSuffix "@${version}" dependencyObject.yarnName;
+        if lib.hasInfix "@git+" dependencyObject.yarnName then
+          lib.head (lib.splitString "@git+" dependencyObject.yarnName)
+        else
+          let
+            version = lib.last (lib.splitString "@" dependencyObject.yarnName);
+          in
+            lib.removeSuffix "@${version}" dependencyObject.yarnName;

      getVersion = dependencyObject:
          dependencyObject.version;
@ -116,7 +130,8 @@
                      if ! dependenciesByOriginalID ? ${yarnName} then
                        # handle missing lock file entry
                        let
-                          versionMatch = b.match ''.*\^([[:digit:]|\.]+)'' versionSpec;
+                          versionMatch =
+                            b.match ''.*\^([[:digit:]|\.]+)'' versionSpec;
                        in
                          {
                            inherit name;
@ -131,15 +146,21 @@

      getSourceType = dependencyObject:
        if lib.hasInfix "@github:" dependencyObject.yarnName
-            || 
-            (dependencyObject ? resolved
-              && lib.hasInfix "codeload.github.com/" dependencyObject.resolved  ) then
+            || (dependencyObject ? resolved
+                && lib.hasInfix
+                  "codeload.github.com/"
+                  dependencyObject.resolved)
+            || (lib.hasInfix "@git+" dependencyObject.yarnName) then
          if dependencyObject ? integrity then
-            b.trace "Warning: Using git despite integrity exists for ${getName dependencyObject}"
+            b.trace (
+              "Warning: Using git despite integrity exists for"
+              + "${getName dependencyObject}"
+            )
              "git"
          else
            "git"
-        else if lib.hasInfix "@link:" dependencyObject.yarnName then
+        else if lib.hasInfix "@link:" dependencyObject.yarnName
+            || lib.hasInfix "@file:" dependencyObject.yarnName then
          "path"
        else
          "http";
@ -147,23 +168,38 @@
      
      sourceConstructors = {
        git = dependencyObject:
-          let
-            gitUrlInfos = lib.splitString "/" dependencyObject.resolved;
-            rev = lib.elemAt gitUrlInfos 6;
-            owner = lib.elemAt gitUrlInfos 3;
-            repo = lib.elemAt gitUrlInfos 4;
-            version = dependencyObject.version;
-          in
-          {
-            url = "https://github.com/${owner}/${repo}";
-            inherit rev version;
-          };
+          if utils.identifyGitUrl dependencyObject.resolved then
+            (utils.parseGitUrl dependencyObject.resolved) // {
+              version = dependencyObject.version;
+            }
+          else
+            let
+              githubUrlInfos = lib.splitString "/" dependencyObject.resolved;
+              owner = lib.elemAt githubUrlInfos 3;
+              repo = lib.elemAt githubUrlInfos 4;
+              rev = lib.elemAt githubUrlInfos 6;
+              version = dependencyObject.version;
+            in
+            {
+              url = "https://github.com/${owner}/${repo}";
+              inherit rev version;
+            };

        path = dependencyObject:
-          {
-            version = dependencyObject.version;     
-            path = lib.last (lib.splitString "@link:" dependencyObject.yarnName);
-          };
+          if lib.hasInfix "@link:" dependencyObject.yarnName then
+            {
+              version = dependencyObject.version;     
+              path =
+                lib.last (lib.splitString "@link:" dependencyObject.yarnName);
+            }
+          else if lib.hasInfix "@file:" dependencyObject.yarnName then
+            {
+              version = dependencyObject.version;     
+              path =
+              lib.last (lib.splitString "@file:" dependencyObject.yarnName);
+            }
+          else
+            throw "unknown path format ${b.toJSON dependencyObject}";

        http = dependencyObject:
          {
@ -174,7 +210,8 @@
                dependencyObject.integrity
              else
                let
-                  hash = lib.last (lib.splitString "#" dependencyObject.resolved);
+                  hash =
+                    lib.last (lib.splitString "#" dependencyObject.resolved);
                in
                  if lib.stringLength hash == 40 then hash
              else
@ -227,17 +264,17 @@
    };

    noDev = {
-      description = "Whether to exclude development dependencies";
+      description = "Exclude development dependencies";
      type = "flag";
    };

-    optional = {
-      description = "Whether to include optional dependencies";
+    noOptional = {
+      description = "Exclude optional dependencies";
      type = "flag";
    };

    peer = {
-      description = "Whether to include peer dependencies";
+      description = "Include peer dependencies";
      type = "flag";
    };

--- a/src/utils/dream-lock.nix
+++ b/src/utils/dream-lock.nix
@ -116,7 +116,8 @@ let
    let
      key = "${pname}#${version}";
    in
-      if fetchedSources ? "${key}" then
+      if fetchedSources ? "${key}"
+          && fetchedSources."${key}" != "unknown" then
        fetchedSources."${key}"
      else
        throw ''
@ -152,22 +153,17 @@ let
        lock = (readDreamLock { inherit dreamLock; }).lock;

        oldDependencyGraph = lock.generic.dependencyGraph;
-        
+
        newDependencyGraph =
          lib.mapAttrs
            (key: deps:
              let
-                nameVer = utils.keyToNameVersion key;
+                oldDeps = oldDependencyGraph."${key}" or [];
              in
-                deps
+                (oldDeps
                ++
-                (if inject ? "${nameVer.name}"."${nameVer.version}" then
-                  (b.map
-                    utils.nameVersionToKey
-                    inject."${nameVer.name}"."${nameVer.version}")
-                else
-                  []))
-            oldDependencyGraph;
+                lib.filter (dep: ! b.elem dep oldDeps) deps))
+            (oldDependencyGraph // inject);

      in
        lib.recursiveUpdate lock {
--- a/src/utils/override.nix
+++ b/src/utils/override.nix
@ -36,9 +36,9 @@ let
    '';

  applyOverridesToPackage = conditionalOverrides: pkg: pname:
-    if ! conditionalOverrides ? "${pname}" then
-      pkg
-    else
+    # if ! conditionalOverrides ? "${pname}" then
+    #   pkg
+    # else
      
      let

@ -52,7 +52,20 @@ let
        # filter the overrides by the package name and conditions
        overridesToApply =
          let
-            overridesForPackage = conditionalOverrides."${pname}";
+            regexOverrides =
+              lib.filterAttrs
+                (name: data:
+                  lib.hasPrefix "^" name
+                  &&
+                  b.match name pname != null)
+                conditionalOverrides;
+
+            overridesForPackage =
+              b.foldl'
+                (overrides: new: overrides // new)
+                conditionalOverrides."${pname}" or {}
+                (lib.attrValues regexOverrides);
+
            overridesListForPackage =
              lib.mapAttrsToList
                (_name: data: data // { inherit _name; })