importNpmLock.buildNodeModules: init

`importNpmLock.buildNodeModules` returns a derivation with a pre-built `node_modules` directory, as imported by `importNpmLock`.
This is to be used together with `importNpmLock.hooks.linkNodeModulesHook` to facilitate `nix-shell`/`nix develop` based development workflows:

```nix
pkgs.mkShell {
  packages = [
    importNpmLock.hooks.linkNodeModulesHook
    nodejs
  ];

  npmDeps = importNpmLock.buildNodeModules {
    npmRoot = ./.;
    inherit nodejs;
  };
}
```
will create a development shell where a `node_modules` directory is created & packages symlinked to the Nix store when activated.

This code is adapted from https://github.com/adisbladis/buildNodeModules

doc/languages-frameworks/javascript.section.md

@@ -287,6 +287,43 @@ buildNpmPackage {
 }
 ```
 
+#### importNpmLock.buildNodeModules {#javascript-buildNpmPackage-importNpmLock.buildNodeModules}
+
+`importNpmLock.buildNodeModules` returns a derivation with a pre-built `node_modules` directory, as imported by `importNpmLock`.
+
+This is to be used together with `importNpmLock.hooks.linkNodeModulesHook` to facilitate `nix-shell`/`nix develop` based development workflows.
+
+It accepts an argument with the following attributes:
+
+`npmRoot` (Path; optional)
+: Path to package directory containing the source tree. If not specified, the `package` and `packageLock` arguments must both be specified.
+
+`package` (Attrset; optional)
+: Parsed contents of `package.json`, as returned by `lib.importJSON ./my-package.json`. If not specified, the `package.json` in `npmRoot` is used.
+
+`packageLock` (Attrset; optional)
+: Parsed contents of `package-lock.json`, as returned `lib.importJSON ./my-package-lock.json`. If not specified, the `package-lock.json` in `npmRoot` is used.
+
+`derivationArgs` (`mkDerivation` attrset; optional)
+: Arguments passed to `stdenv.mkDerivation`
+
+For example:
+
+```nix
+pkgs.mkShell {
+  packages = [
+    importNpmLock.hooks.linkNodeModulesHook
+    nodejs
+  ];
+
+  npmDeps = importNpmLock.buildNodeModules {
+    npmRoot = ./.;
+    inherit nodejs;
+  };
+}
+```
+will create a development shell where a `node_modules` directory is created & packages symlinked to the Nix store when activated.
+
 ### corepack {#javascript-corepack}
 
 This package puts the corepack wrappers for pnpm and yarn in your PATH, and they will honor the `packageManager` setting in the `package.json`.

pkgs/build-support/node/import-npm-lock/default.nix

@@ -52,11 +52,16 @@ let
       else null
     );
 
+  cleanModule = lib.flip removeAttrs [
+    "link" # Remove link not to symlink directories. These have been processed to store paths already.
+    "funding" # Remove funding to get rid sponsorship nag in build output
+  ];
+
   # Manage node_modules outside of the store with hooks
   hooks = callPackages ./hooks { };
 
 in
-{
+lib.fix (self: {
   importNpmLock =
     { npmRoot ? null
     , package ? importJSON (npmRoot + "/package.json")
@@ -94,10 +99,8 @@ in
                   fetcherOpts = fetcherOpts.${modulePath} or {};
                 };
               in
-              (removeAttrs module [
+              cleanModule module
-                "link"
+              // lib.optionalAttrs (src != null) {
-                "funding"
-              ]) // lib.optionalAttrs (src != null) {
                 resolved = "file:${src}";
               } // lib.optionalAttrs (module ? dependencies) {
                 dependencies = mapLockDependencies module.dependencies;
@@ -133,8 +136,52 @@ in
       cp "$packageLockPath" $out/package-lock.json
     '';
 
+  # Build node modules from package.json & package-lock.json
+  buildNodeModules =
+    { npmRoot ? null
+    , package ? importJSON (npmRoot + "/package.json")
+    , packageLock ? importJSON (npmRoot + "/package-lock.json")
+    , nodejs
+    , derivationArgs ? { }
+    }:
+    stdenv.mkDerivation ({
+      pname = derivationArgs.pname or "${getName package}-node-modules";
+      version = derivationArgs.version or getVersion package;
+
+      dontUnpack = true;
+
+      npmDeps = self.importNpmLock {
+        inherit npmRoot package packageLock;
+      };
+
+      package = toJSON package;
+      packageLock = toJSON packageLock;
+
+      installPhase = ''
+        runHook preInstall
+        mkdir $out
+        cp package.json $out/
+        cp package-lock.json $out/
+        [[ -d node_modules ]] && mv node_modules $out/
+        runHook postInstall
+      '';
+    } // derivationArgs // {
+      nativeBuildInputs = [
+        nodejs
+        nodejs.passthru.python
+        hooks.npmConfigHook
+      ] ++ derivationArgs.nativeBuildInputs or [ ];
+
+      passAsFile = [ "package" "packageLock" ] ++ derivationArgs.passAsFile or [ ];
+
+      postPatch = ''
+        cp --no-preserve=mode "$packagePath" package.json
+        cp --no-preserve=mode "$packageLockPath" package-lock.json
+      '' + derivationArgs.postPatch or "";
+    });
+
   inherit hooks;
-  inherit (hooks) npmConfigHook;
+  inherit (hooks) npmConfigHook linkNodeModulesHook;
 
   __functor = self: self.importNpmLock;
-}
+})

pkgs/build-support/node/import-npm-lock/hooks/default.nix

@@ -10,4 +10,14 @@
         storePrefix = builtins.storeDir;
       };
     } ./npm-config-hook.sh;
+
+  linkNodeModulesHook = makeSetupHook
+    {
+      name = "node-modules-hook.sh";
+      substitutions = {
+        nodejs = lib.getExe nodejs;
+        script = ./link-node-modules.js;
+        storePrefix = builtins.storeDir;
+      };
+    } ./link-node-modules-hook.sh;
 }

pkgs/build-support/node/import-npm-lock/hooks/link-node-modules-hook.sh

@@ -0,0 +1,31 @@
+linkNodeModulesHook() {
+    echo "Executing linkNodeModulesHook"
+    runHook preShellHook
+
+    if [ -n "${npmRoot-}" ]; then
+      pushd "$npmRoot"
+    fi
+
+    @nodejs@ @script@ @storePrefix@ "${npmDeps}/node_modules"
+    if test -f node_modules/.bin; then
+        export PATH=$(readlink -f node_modules/.bin):$PATH
+    fi
+
+    if [ -n "${npmRoot-}" ]; then
+      popd
+    fi
+
+    runHook postShellHook
+    echo "Finished executing linkNodeModulesShellHook"
+}
+
+if [ -z "${dontLinkNodeModules:-}" ] && [ -z "${shellHook-}" ]; then
+    echo "Using linkNodeModulesHook shell hook"
+    shellHook=linkNodeModulesHook
+fi
+
+
+if [ -z "${dontLinkNodeModules:-}" ]; then
+    echo "Using linkNodeModulesHook preConfigure hook"
+    preConfigureHooks+=(linkNodeModulesHook)
+fi

pkgs/build-support/node/import-npm-lock/hooks/link-node-modules.js

@@ -0,0 +1,96 @@
+#!/usr/bin/env node
+const fs = require("fs");
+const path = require("path");
+
+async function asyncFilter(arr, pred) {
+  const filtered = [];
+  for (const elem of arr) {
+    if (await pred(elem)) {
+      filtered.push(elem);
+    }
+  }
+  return filtered;
+}
+
+// Get a list of all _unmanaged_ files in node_modules.
+// This means every file in node_modules that is _not_ a symlink to the Nix store.
+async function getUnmanagedFiles(storePrefix, files) {
+  return await asyncFilter(files, async (file) => {
+    const filePath = path.join("node_modules", file);
+
+    // Is file a symlink
+    const stat = await fs.promises.lstat(filePath);
+    if (!stat.isSymbolicLink()) {
+      return true;
+    }
+
+    // Is file in the store
+    const linkTarget = await fs.promises.readlink(filePath);
+    return !linkTarget.startsWith(storePrefix);
+  });
+}
+
+async function main() {
+  const args = process.argv.slice(2);
+  const storePrefix = args[0];
+  const sourceModules = args[1];
+
+  // Ensure node_modules exists
+  try {
+    await fs.promises.mkdir("node_modules");
+  } catch (err) {
+    if (err.code !== "EEXIST") {
+      throw err;
+    }
+  }
+
+  const files = await fs.promises.readdir("node_modules");
+
+  // Get deny list of files that we don't manage.
+  // We do manage nix store symlinks, but not other files.
+  // For example: If a .vite was present in both our
+  // source node_modules and the non-store node_modules we don't want to overwrite
+  // the non-store one.
+  const unmanaged = await getUnmanagedFiles(storePrefix, files);
+  const managed = new Set(files.filter((file) => ! unmanaged.includes(file)));
+
+  const sourceFiles = await fs.promises.readdir(sourceModules);
+  await Promise.all(
+    sourceFiles.map(async (file) => {
+      const sourcePath = path.join(sourceModules, file);
+      const targetPath = path.join("node_modules", file);
+
+      // Skip file if it's not a symlink to a store path
+      if (unmanaged.includes(file)) {
+        console.log(`'${targetPath}' exists, cowardly refusing to link.`);
+        return;
+      }
+
+      // Don't unlink this file, we just wrote it.
+      managed.delete(file);
+
+      // Link to a temporary dummy path and rename.
+      // This is to get some degree of atomicity.
+      try {
+        await fs.promises.symlink(sourcePath, targetPath + "-nix-hook-temp");
+      } catch (err) {
+        if (err.code !== "EEXIST") {
+          throw err;
+        }
+
+        await fs.promises.unlink(targetPath + "-nix-hook-temp");
+        await fs.promises.symlink(sourcePath, targetPath + "-nix-hook-temp");
+      }
+      await fs.promises.rename(targetPath + "-nix-hook-temp", targetPath);
+    })
+  );
+
+  // Clean up store symlinks not included in this generation of node_modules
+  await Promise.all(
+    Array.from(managed).map((file) =>
+      fs.promises.unlink(path.join("node_modules", file)),
+    )
+  );
+}
+
+main();