From 9c7ff7277cdc8839199849ca759ceba306f98b44 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Thu, 8 Aug 2024 14:26:26 +1200 Subject: [PATCH] importNpmLock.buildNodeModules: init `importNpmLock.buildNodeModules` returns a derivation with a pre-built `node_modules` directory, as imported by `importNpmLock`. This is to be used together with `importNpmLock.hooks.linkNodeModulesHook` to facilitate `nix-shell`/`nix develop` based development workflows: ```nix pkgs.mkShell { packages = [ importNpmLock.hooks.linkNodeModulesHook nodejs ]; npmDeps = importNpmLock.buildNodeModules { npmRoot = ./.; inherit nodejs; }; } ``` will create a development shell where a `node_modules` directory is created & packages symlinked to the Nix store when activated. This code is adapted from https://github.com/adisbladis/buildNodeModules --- .../javascript.section.md | 37 +++++++ .../node/import-npm-lock/default.nix | 61 ++++++++++-- .../node/import-npm-lock/hooks/default.nix | 10 ++ .../hooks/link-node-modules-hook.sh | 31 ++++++ .../hooks/link-node-modules.js | 96 +++++++++++++++++++ 5 files changed, 228 insertions(+), 7 deletions(-) create mode 100644 pkgs/build-support/node/import-npm-lock/hooks/link-node-modules-hook.sh create mode 100644 pkgs/build-support/node/import-npm-lock/hooks/link-node-modules.js diff --git a/doc/languages-frameworks/javascript.section.md b/doc/languages-frameworks/javascript.section.md index 8eb3d1ff39a4..8fa0f9a5aaa5 100644 --- a/doc/languages-frameworks/javascript.section.md +++ b/doc/languages-frameworks/javascript.section.md @@ -287,6 +287,43 @@ buildNpmPackage { } ``` +#### importNpmLock.buildNodeModules {#javascript-buildNpmPackage-importNpmLock.buildNodeModules} + +`importNpmLock.buildNodeModules` returns a derivation with a pre-built `node_modules` directory, as imported by `importNpmLock`. + +This is to be used together with `importNpmLock.hooks.linkNodeModulesHook` to facilitate `nix-shell`/`nix develop` based development workflows. + +It accepts an argument with the following attributes: + +`npmRoot` (Path; optional) +: Path to package directory containing the source tree. If not specified, the `package` and `packageLock` arguments must both be specified. + +`package` (Attrset; optional) +: Parsed contents of `package.json`, as returned by `lib.importJSON ./my-package.json`. If not specified, the `package.json` in `npmRoot` is used. + +`packageLock` (Attrset; optional) +: Parsed contents of `package-lock.json`, as returned `lib.importJSON ./my-package-lock.json`. If not specified, the `package-lock.json` in `npmRoot` is used. + +`derivationArgs` (`mkDerivation` attrset; optional) +: Arguments passed to `stdenv.mkDerivation` + +For example: + +```nix +pkgs.mkShell { + packages = [ + importNpmLock.hooks.linkNodeModulesHook + nodejs + ]; + + npmDeps = importNpmLock.buildNodeModules { + npmRoot = ./.; + inherit nodejs; + }; +} +``` +will create a development shell where a `node_modules` directory is created & packages symlinked to the Nix store when activated. + ### corepack {#javascript-corepack} This package puts the corepack wrappers for pnpm and yarn in your PATH, and they will honor the `packageManager` setting in the `package.json`. diff --git a/pkgs/build-support/node/import-npm-lock/default.nix b/pkgs/build-support/node/import-npm-lock/default.nix index b63e5237dc3e..48057ec8891e 100644 --- a/pkgs/build-support/node/import-npm-lock/default.nix +++ b/pkgs/build-support/node/import-npm-lock/default.nix @@ -52,11 +52,16 @@ let else null ); + cleanModule = lib.flip removeAttrs [ + "link" # Remove link not to symlink directories. These have been processed to store paths already. + "funding" # Remove funding to get rid sponsorship nag in build output + ]; + # Manage node_modules outside of the store with hooks hooks = callPackages ./hooks { }; in -{ +lib.fix (self: { importNpmLock = { npmRoot ? null , package ? importJSON (npmRoot + "/package.json") @@ -94,10 +99,8 @@ in fetcherOpts = fetcherOpts.${modulePath} or {}; }; in - (removeAttrs module [ - "link" - "funding" - ]) // lib.optionalAttrs (src != null) { + cleanModule module + // lib.optionalAttrs (src != null) { resolved = "file:${src}"; } // lib.optionalAttrs (module ? dependencies) { dependencies = mapLockDependencies module.dependencies; @@ -133,8 +136,52 @@ in cp "$packageLockPath" $out/package-lock.json ''; + # Build node modules from package.json & package-lock.json + buildNodeModules = + { npmRoot ? null + , package ? importJSON (npmRoot + "/package.json") + , packageLock ? importJSON (npmRoot + "/package-lock.json") + , nodejs + , derivationArgs ? { } + }: + stdenv.mkDerivation ({ + pname = derivationArgs.pname or "${getName package}-node-modules"; + version = derivationArgs.version or getVersion package; + + dontUnpack = true; + + npmDeps = self.importNpmLock { + inherit npmRoot package packageLock; + }; + + package = toJSON package; + packageLock = toJSON packageLock; + + installPhase = '' + runHook preInstall + mkdir $out + cp package.json $out/ + cp package-lock.json $out/ + [[ -d node_modules ]] && mv node_modules $out/ + runHook postInstall + ''; + } // derivationArgs // { + nativeBuildInputs = [ + nodejs + nodejs.passthru.python + hooks.npmConfigHook + ] ++ derivationArgs.nativeBuildInputs or [ ]; + + passAsFile = [ "package" "packageLock" ] ++ derivationArgs.passAsFile or [ ]; + + postPatch = '' + cp --no-preserve=mode "$packagePath" package.json + cp --no-preserve=mode "$packageLockPath" package-lock.json + '' + derivationArgs.postPatch or ""; + }); + inherit hooks; - inherit (hooks) npmConfigHook; + inherit (hooks) npmConfigHook linkNodeModulesHook; __functor = self: self.importNpmLock; -} +}) diff --git a/pkgs/build-support/node/import-npm-lock/hooks/default.nix b/pkgs/build-support/node/import-npm-lock/hooks/default.nix index 5990371def91..0140ebf2d470 100644 --- a/pkgs/build-support/node/import-npm-lock/hooks/default.nix +++ b/pkgs/build-support/node/import-npm-lock/hooks/default.nix @@ -10,4 +10,14 @@ storePrefix = builtins.storeDir; }; } ./npm-config-hook.sh; + + linkNodeModulesHook = makeSetupHook + { + name = "node-modules-hook.sh"; + substitutions = { + nodejs = lib.getExe nodejs; + script = ./link-node-modules.js; + storePrefix = builtins.storeDir; + }; + } ./link-node-modules-hook.sh; } diff --git a/pkgs/build-support/node/import-npm-lock/hooks/link-node-modules-hook.sh b/pkgs/build-support/node/import-npm-lock/hooks/link-node-modules-hook.sh new file mode 100644 index 000000000000..12f2f8507fa5 --- /dev/null +++ b/pkgs/build-support/node/import-npm-lock/hooks/link-node-modules-hook.sh @@ -0,0 +1,31 @@ +linkNodeModulesHook() { + echo "Executing linkNodeModulesHook" + runHook preShellHook + + if [ -n "${npmRoot-}" ]; then + pushd "$npmRoot" + fi + + @nodejs@ @script@ @storePrefix@ "${npmDeps}/node_modules" + if test -f node_modules/.bin; then + export PATH=$(readlink -f node_modules/.bin):$PATH + fi + + if [ -n "${npmRoot-}" ]; then + popd + fi + + runHook postShellHook + echo "Finished executing linkNodeModulesShellHook" +} + +if [ -z "${dontLinkNodeModules:-}" ] && [ -z "${shellHook-}" ]; then + echo "Using linkNodeModulesHook shell hook" + shellHook=linkNodeModulesHook +fi + + +if [ -z "${dontLinkNodeModules:-}" ]; then + echo "Using linkNodeModulesHook preConfigure hook" + preConfigureHooks+=(linkNodeModulesHook) +fi diff --git a/pkgs/build-support/node/import-npm-lock/hooks/link-node-modules.js b/pkgs/build-support/node/import-npm-lock/hooks/link-node-modules.js new file mode 100644 index 000000000000..79e247eb4acb --- /dev/null +++ b/pkgs/build-support/node/import-npm-lock/hooks/link-node-modules.js @@ -0,0 +1,96 @@ +#!/usr/bin/env node +const fs = require("fs"); +const path = require("path"); + +async function asyncFilter(arr, pred) { + const filtered = []; + for (const elem of arr) { + if (await pred(elem)) { + filtered.push(elem); + } + } + return filtered; +} + +// Get a list of all _unmanaged_ files in node_modules. +// This means every file in node_modules that is _not_ a symlink to the Nix store. +async function getUnmanagedFiles(storePrefix, files) { + return await asyncFilter(files, async (file) => { + const filePath = path.join("node_modules", file); + + // Is file a symlink + const stat = await fs.promises.lstat(filePath); + if (!stat.isSymbolicLink()) { + return true; + } + + // Is file in the store + const linkTarget = await fs.promises.readlink(filePath); + return !linkTarget.startsWith(storePrefix); + }); +} + +async function main() { + const args = process.argv.slice(2); + const storePrefix = args[0]; + const sourceModules = args[1]; + + // Ensure node_modules exists + try { + await fs.promises.mkdir("node_modules"); + } catch (err) { + if (err.code !== "EEXIST") { + throw err; + } + } + + const files = await fs.promises.readdir("node_modules"); + + // Get deny list of files that we don't manage. + // We do manage nix store symlinks, but not other files. + // For example: If a .vite was present in both our + // source node_modules and the non-store node_modules we don't want to overwrite + // the non-store one. + const unmanaged = await getUnmanagedFiles(storePrefix, files); + const managed = new Set(files.filter((file) => ! unmanaged.includes(file))); + + const sourceFiles = await fs.promises.readdir(sourceModules); + await Promise.all( + sourceFiles.map(async (file) => { + const sourcePath = path.join(sourceModules, file); + const targetPath = path.join("node_modules", file); + + // Skip file if it's not a symlink to a store path + if (unmanaged.includes(file)) { + console.log(`'${targetPath}' exists, cowardly refusing to link.`); + return; + } + + // Don't unlink this file, we just wrote it. + managed.delete(file); + + // Link to a temporary dummy path and rename. + // This is to get some degree of atomicity. + try { + await fs.promises.symlink(sourcePath, targetPath + "-nix-hook-temp"); + } catch (err) { + if (err.code !== "EEXIST") { + throw err; + } + + await fs.promises.unlink(targetPath + "-nix-hook-temp"); + await fs.promises.symlink(sourcePath, targetPath + "-nix-hook-temp"); + } + await fs.promises.rename(targetPath + "-nix-hook-temp", targetPath); + }) + ); + + // Clean up store symlinks not included in this generation of node_modules + await Promise.all( + Array.from(managed).map((file) => + fs.promises.unlink(path.join("node_modules", file)), + ) + ); +} + +main();