From 1beb1eaab56e719e938c0b1759c9df91b29639be Mon Sep 17 00:00:00 2001 From: DavHau Date: Fri, 29 Jul 2022 15:17:32 +0200 Subject: [PATCH 1/9] feat: improve nodejs translators/builders - also execute preinstall script during package installation - disable devDependencies if no instal script present by default --- .../nodejs/builders/granular/default.nix | 3 ++ .../translators/package-json/default.nix | 22 ++++++++++-- .../translators/package-lock/default.nix | 36 ++++++++++++++----- 3 files changed, 51 insertions(+), 10 deletions(-) diff --git a/src/subsystems/nodejs/builders/granular/default.nix b/src/subsystems/nodejs/builders/granular/default.nix index a6c37b65..2fb10bee 100644 --- a/src/subsystems/nodejs/builders/granular/default.nix +++ b/src/subsystems/nodejs/builders/granular/default.nix @@ -473,6 +473,9 @@ elif [ -n "$runBuild" ] && [ "$(jq '.scripts.build' ./package.json)" != "null" ]; then npm run build else + if [ "$(jq '.scripts.preinstall' ./package.json)" != "null" ]; then + npm --production --offline --nodedir=$nodeSources run preinstall + fi if [ "$(jq '.scripts.install' ./package.json)" != "null" ]; then npm --production --offline --nodedir=$nodeSources run install fi diff --git a/src/subsystems/nodejs/translators/package-json/default.nix b/src/subsystems/nodejs/translators/package-json/default.nix index 02dd3cda..adfd8979 100644 --- a/src/subsystems/nodejs/translators/package-json/default.nix +++ b/src/subsystems/nodejs/translators/package-json/default.nix @@ -47,11 +47,29 @@ cd ./$relPath rm -rf package-lock.json - if [ "$(jq '.project.subsystemInfo.noDev' -c -r $jsonInput)" == "true" ]; then + hasInstallScript=false + if [ "$(jq 'has("scripts")' -c -r package.json)" == "false" ]; then + : + elif [ "$(jq '.scripts | has("preinstall")' -c -r package.json)" == "true" ]; then + hasInstallScript=true + elif [ "$(jq '.scripts | has("install")' -c -r package.json)" == "true" ]; then + hasInstallScript=true + elif [ "$(jq '.scripts | has("postinstall")' -c -r package.json)" == "true" ]; then + hasInstallScript=true + else + : + fi + + if [ "$hasInstallScript" == "false" ]; then + echo "package.json does not define an [pre|post]install script -> omitting dev dependencies" + fi + + if [ "$hasInstallScript" == "false" ] \ + || [ "$(jq '.project.subsystemInfo.noDev' -c -r $jsonInput)" == "true" ]; then echo "excluding dev dependencies" jq '.devDependencies = {}' ./package.json > package.json.mod mv package.json.mod package.json - npm install --package-lock-only --production $npmArgs + npm install --package-lock-only --omit=dev $npmArgs else npm install --package-lock-only $npmArgs fi diff --git a/src/subsystems/nodejs/translators/package-lock/default.nix b/src/subsystems/nodejs/translators/package-lock/default.nix index cf5117f1..7a3cac5d 100644 --- a/src/subsystems/nodejs/translators/package-lock/default.nix +++ b/src/subsystems/nodejs/translators/package-lock/default.nix @@ -25,7 +25,15 @@ } @ args: let b = builtins; - dev = ! noDev; + hasInstallScript = + (packageJson ? scripts.preinstall) + || (packageJson ? scripts.install) + || (packageJson ? scripts.postinstall); + + noDev = + if ! hasInstallScript + then true + else args.noDev; name = project.name; tree = args.tree.getNodeFromPath project.relPath; relPath = project.relPath; @@ -70,11 +78,23 @@ then let path = getPath dependencyObject; in - ( - b.fromJSON - (b.readFile "${source}/${path}/package.json") - ) - .version + if ! (l.pathExists "${source}/${path}/package.json") + then + throw '' + The lock file references a sub-package residing at '${source}/${path}', + but that directory doesn't exist or doesn't contain a package.json + + The reason might be that devDependencies are not included in this package release. + Possible solutions: + - get full package source via git and translate from there + - disable devDependencies by passing `noDev` to the translator + '' + else + ( + b.fromJSON + (b.readFile "${source}/${path}/package.json") + ) + .version else if lib.hasPrefix "https://" dependencyObject.version then "unknown" else dependencyObject.version; @@ -153,7 +173,7 @@ version = getVersion pdata; }) (lib.filterAttrs - (pname: pdata: ! (pdata.dev or false) || dev) + (pname: pdata: ! (pdata.dev or false) || ! noDev) parsedDependencies); subsystemName = "nodejs"; @@ -186,7 +206,7 @@ in lib.filter (pdata: - dev || ! (pdata.dev or false)) + ! noDev || ! (pdata.dev or false)) (lib.flatten (serialize inputData)); getName = dependencyObject: dependencyObject.pname; From c6723cb1a5a05ec69cb9c75c2a27ec6090b949cd Mon Sep 17 00:00:00 2001 From: DavHau Date: Fri, 29 Jul 2022 16:02:14 +0200 Subject: [PATCH 2/9] nodejs fix: empty dependencies not detected sometimes --- src/subsystems/nodejs/translators/utils.nix | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/subsystems/nodejs/translators/utils.nix b/src/subsystems/nodejs/translators/utils.nix index cff701e9..7bc0f683 100644 --- a/src/subsystems/nodejs/translators/utils.nix +++ b/src/subsystems/nodejs/translators/utils.nix @@ -2,8 +2,7 @@ l = lib // builtins; in rec { getPackageJsonDeps = packageJson: noDev: - packageJson.dependencies - or {} + (packageJson.dependencies or {}) // (lib.optionalAttrs (! noDev) (packageJson.devDependencies or {})); getWorkspaceLockFile = tree: project: fname: let @@ -17,9 +16,9 @@ in rec { (tree.getNodeFromPath "${dirRelPath}/package.json").jsonContent; hasNoDependencies = - ! packageJson ? dependencies - && ! packageJson ? devDependencies - && ! packageJson ? workspaces; + ((packageJson.dependencies or {}) == {}) + && ((packageJson.devDependencies or {}) == {}) + && (! packageJson ? workspaces); in if hasNoDependencies then null From 5ddce49dceda9d8294f4f3ffb7b9175d2e1112e2 Mon Sep 17 00:00:00 2001 From: DavHau Date: Fri, 29 Jul 2022 17:05:14 +0200 Subject: [PATCH 3/9] fix: indexers: always add main source to dream-lock.json --- src/utils/dream-lock.nix | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/utils/dream-lock.nix b/src/utils/dream-lock.nix index b1a8714f..e766527f 100644 --- a/src/utils/dream-lock.nix +++ b/src/utils/dream-lock.nix @@ -11,7 +11,9 @@ replaceRootSources = { dreamLock, newSourceRoot, - }: let + } @ args: let + dreamLockLoaded = utils.readDreamLock {dreamLock = args.dreamLock;}; + iface = dreamLockLoaded.interface; patchVersion = version: source: if source.type @@ -24,10 +26,18 @@ dir = source.relPath; } else source; + patchedSources = - l.mapAttrs - (_: versions: l.mapAttrs patchVersion versions) - dreamLock.sources; + l.recursiveUpdate + { + "${iface.defaultPackageName}"."${iface.defaultPackageVersion}" = + newSourceRoot; + } + ( + l.mapAttrs + (_: versions: l.mapAttrs patchVersion versions) + dreamLock.sources + ); in dreamLock // {sources = patchedSources;}; From ab7edc429618cb2108a21911227bd483f6b19e27 Mon Sep 17 00:00:00 2001 From: DavHau Date: Fri, 29 Jul 2022 17:06:00 +0200 Subject: [PATCH 4/9] fix: indexers: respect more parameters - inject - packageOverrides - sourceOverrides --- src/subsystems/nodejs/translators/package-json/default.nix | 2 ++ src/utils/index.nix | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/subsystems/nodejs/translators/package-json/default.nix b/src/subsystems/nodejs/translators/package-json/default.nix index adfd8979..2e1cbfa5 100644 --- a/src/subsystems/nodejs/translators/package-json/default.nix +++ b/src/subsystems/nodejs/translators/package-json/default.nix @@ -60,6 +60,8 @@ : fi + echo "translating in temp dir: $(pwd)" + if [ "$hasInstallScript" == "false" ]; then echo "package.json does not define an [pre|post]install script -> omitting dev dependencies" fi diff --git a/src/utils/index.nix b/src/utils/index.nix index 63ba702b..66e08835 100644 --- a/src/utils/index.nix +++ b/src/utils/index.nix @@ -39,7 +39,12 @@ in rec { dreamLocks = findDreamLocks tree; makePackagesForDreamLock = dreamLock: (dream2nixInterface.makeOutputsForDreamLock { - inherit dreamLock; + inherit + dreamLock + inject + packageOverrides + sourceOverrides + ; }) .packages; in From 33fdde023350d18debcfbbe9e00909ad4d0b30d6 Mon Sep 17 00:00:00 2001 From: DavHau Date: Fri, 29 Jul 2022 21:23:42 +0200 Subject: [PATCH 5/9] feat: indexer: use gnuparallel to speed up index translation --- src/apps/translate-index/default.nix | 50 ++++++++++++++++------------ 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/src/apps/translate-index/default.nix b/src/apps/translate-index/default.nix index 15824a97..3654efbd 100644 --- a/src/apps/translate-index/default.nix +++ b/src/apps/translate-index/default.nix @@ -3,30 +3,36 @@ translate, coreutils, jq, + parallel, + writeScript, ... -}: -utils.writePureShellScriptBin -"translate-index" -[coreutils translate jq] -'' - cd $WORKDIR +}: let + script = writeScript "run-translate" '' + ${translate}/bin/translate $1 $targetDir + ''; +in + utils.writePureShellScriptBin + "translate-index" + [coreutils translate jq parallel] + '' + cd $WORKDIR - usage="usage: - $0 INDEX_PATH TARGET_DIR" + usage="usage: + $0 INDEX_PATH TARGET_DIR" - if [ "$#" -ne 2 ]; then - echo "error: wrong number of arguments" - echo "$usage" - exit 1 - fi + if [ "$#" -ne 2 ]; then + echo "error: wrong number of arguments" + echo "$usage" + exit 1 + fi - index=''${1:?"error: please pass index file path"} - index=$(realpath $index) - targetDir=''${2:?"error: please pass a target directory"} - targetDir=$(realpath $targetDir) + index=''${1:?"error: please pass index file path"} + index=$(realpath $index) + targetDir=''${2:?"error: please pass a target directory"} + targetDir=$(realpath $targetDir) - for shortcut in $(jq '.[]' -c -r $index); do - translate "$shortcut" $targetDir \ - || echo "failed to translate \"$shortcut\"" - done -'' + export targetDir + + parallel --halt now,fail=1 -j$(nproc) --delay 1 -a <(jq '.[]' -c -r $index) ${script} + + '' From 33b8f4e78ec923d7628017b21adf945d3f155645 Mon Sep 17 00:00:00 2001 From: DavHau Date: Fri, 29 Jul 2022 21:24:56 +0200 Subject: [PATCH 6/9] fix: don't fail on translation if `git add` doesn't work --- src/utils/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/default.nix b/src/utils/default.nix index c351a084..42a6f89a 100644 --- a/src/utils/default.nix +++ b/src/utils/default.nix @@ -235,7 +235,7 @@ in # add dream-lock.json to git if git rev-parse --show-toplevel &>/dev/null; then echo "adding file to git: $dreamLockPath" - git add $dreamLockPath + git add $dreamLockPath || : fi ''; in From b1e15df1a1bda3a72e0972fad7147d0d7c1e18bc Mon Sep 17 00:00:00 2001 From: DavHau Date: Fri, 29 Jul 2022 21:25:30 +0200 Subject: [PATCH 7/9] fix: fetching: only override sourceSpec version and name if not already specified --- src/fetchers/default-fetcher.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fetchers/default-fetcher.nix b/src/fetchers/default-fetcher.nix index 14815d68..49df1f8e 100644 --- a/src/fetchers/default-fetcher.nix +++ b/src/fetchers/default-fetcher.nix @@ -46,8 +46,8 @@ source = source // { - pname = name; - inherit version; + pname = source.pname or name; + version = source.version or version; }; } else throw "unsupported source type '${source.type}'") From 56fae02e3a3a99d9e957965a0b1e396b092376fa Mon Sep 17 00:00:00 2001 From: DavHau Date: Fri, 29 Jul 2022 21:59:20 +0200 Subject: [PATCH 8/9] feat: verify dream-lock.json against jsonschema --- src/specifications/dream-lock-schema.json | 4 ++-- src/utils/default.nix | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/specifications/dream-lock-schema.json b/src/specifications/dream-lock-schema.json index 098bc638..bdb964cc 100644 --- a/src/specifications/dream-lock-schema.json +++ b/src/specifications/dream-lock-schema.json @@ -97,8 +97,8 @@ }, "then": { "properties": { - "rootName": { "type": "string" }, - "rootVersion": { "type": "string" }, + "rootName": { "type": ["string", "null"] }, + "rootVersion": { "type": ["string", "null"] }, "path": { "type": "string" }, "type": { "type": "string" }, "dir": { "type": "string" } diff --git a/src/utils/default.nix b/src/utils/default.nix index 42a6f89a..0c06cb08 100644 --- a/src/utils/default.nix +++ b/src/utils/default.nix @@ -232,6 +232,12 @@ in | python3 ${../apps/cli/format-dream-lock.py} \ | sponge $dreamLockPath + # validate dream-lock.json against jsonschema + ${python3.pkgs.jsonschema}/bin/jsonschema \ + --instance $dreamLockPath \ + --output pretty \ + ${../specifications/dream-lock-schema.json} + # add dream-lock.json to git if git rev-parse --show-toplevel &>/dev/null; then echo "adding file to git: $dreamLockPath" From f3aec937d3d9ea443e2ba1e922e11b8cfdae8f9d Mon Sep 17 00:00:00 2001 From: DavHau Date: Fri, 29 Jul 2022 22:33:34 +0200 Subject: [PATCH 9/9] indexers: libraries-io: improve verbosity --- src/indexers/libraries-io/default.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/indexers/libraries-io/default.nix b/src/indexers/libraries-io/default.nix index 990bbcd9..45f7c766 100644 --- a/src/indexers/libraries-io/default.nix +++ b/src/indexers/libraries-io/default.nix @@ -39,7 +39,7 @@ libraries.io also supports other interesting popularity metrics: outFile=$(jq '.outputFile' -c -r $input) - echo "loadein api key" + echo "loading api key" if [ -z ''${API_KEY+x} ]; then echo "Please define env variable API_KEY for libaries.io api key" exit 1 @@ -62,6 +62,7 @@ libraries.io also supports other interesting popularity metrics: rm -f $outFile for page in $(seq 1 $numPages); do + echo "requesting page $page" url="https://libraries.io/api/search?page=$page&sort=dependents_count&per_page=100&platforms=$platformQuery&api_key=$apiKey" curl -k "$url" | jq "$jqQuery" -r >> $outFile done