diff --git a/src/apps/translate-index/default.nix b/src/apps/translate-index/default.nix index 15824a97..3654efbd 100644 --- a/src/apps/translate-index/default.nix +++ b/src/apps/translate-index/default.nix @@ -3,30 +3,36 @@ translate, coreutils, jq, + parallel, + writeScript, ... -}: -utils.writePureShellScriptBin -"translate-index" -[coreutils translate jq] -'' - cd $WORKDIR +}: let + script = writeScript "run-translate" '' + ${translate}/bin/translate $1 $targetDir + ''; +in + utils.writePureShellScriptBin + "translate-index" + [coreutils translate jq parallel] + '' + cd $WORKDIR - usage="usage: - $0 INDEX_PATH TARGET_DIR" + usage="usage: + $0 INDEX_PATH TARGET_DIR" - if [ "$#" -ne 2 ]; then - echo "error: wrong number of arguments" - echo "$usage" - exit 1 - fi + if [ "$#" -ne 2 ]; then + echo "error: wrong number of arguments" + echo "$usage" + exit 1 + fi - index=''${1:?"error: please pass index file path"} - index=$(realpath $index) - targetDir=''${2:?"error: please pass a target directory"} - targetDir=$(realpath $targetDir) + index=''${1:?"error: please pass index file path"} + index=$(realpath $index) + targetDir=''${2:?"error: please pass a target directory"} + targetDir=$(realpath $targetDir) - for shortcut in $(jq '.[]' -c -r $index); do - translate "$shortcut" $targetDir \ - || echo "failed to translate \"$shortcut\"" - done -'' + export targetDir + + parallel --halt now,fail=1 -j$(nproc) --delay 1 -a <(jq '.[]' -c -r $index) ${script} + + '' diff --git a/src/fetchers/default-fetcher.nix b/src/fetchers/default-fetcher.nix index 14815d68..49df1f8e 100644 --- a/src/fetchers/default-fetcher.nix +++ b/src/fetchers/default-fetcher.nix @@ -46,8 +46,8 @@ source = source // { - pname = name; - inherit version; + pname = source.pname or name; + version = source.version or version; }; } else throw "unsupported source type '${source.type}'") diff --git a/src/indexers/libraries-io/default.nix b/src/indexers/libraries-io/default.nix index 990bbcd9..45f7c766 100644 --- a/src/indexers/libraries-io/default.nix +++ b/src/indexers/libraries-io/default.nix @@ -39,7 +39,7 @@ libraries.io also supports other interesting popularity metrics: outFile=$(jq '.outputFile' -c -r $input) - echo "loadein api key" + echo "loading api key" if [ -z ''${API_KEY+x} ]; then echo "Please define env variable API_KEY for libaries.io api key" exit 1 @@ -62,6 +62,7 @@ libraries.io also supports other interesting popularity metrics: rm -f $outFile for page in $(seq 1 $numPages); do + echo "requesting page $page" url="https://libraries.io/api/search?page=$page&sort=dependents_count&per_page=100&platforms=$platformQuery&api_key=$apiKey" curl -k "$url" | jq "$jqQuery" -r >> $outFile done diff --git a/src/specifications/dream-lock-schema.json b/src/specifications/dream-lock-schema.json index 098bc638..bdb964cc 100644 --- a/src/specifications/dream-lock-schema.json +++ b/src/specifications/dream-lock-schema.json @@ -97,8 +97,8 @@ }, "then": { "properties": { - "rootName": { "type": "string" }, - "rootVersion": { "type": "string" }, + "rootName": { "type": ["string", "null"] }, + "rootVersion": { "type": ["string", "null"] }, "path": { "type": "string" }, "type": { "type": "string" }, "dir": { "type": "string" } diff --git a/src/subsystems/nodejs/builders/granular/default.nix b/src/subsystems/nodejs/builders/granular/default.nix index a6c37b65..2fb10bee 100644 --- a/src/subsystems/nodejs/builders/granular/default.nix +++ b/src/subsystems/nodejs/builders/granular/default.nix @@ -473,6 +473,9 @@ elif [ -n "$runBuild" ] && [ "$(jq '.scripts.build' ./package.json)" != "null" ]; then npm run build else + if [ "$(jq '.scripts.preinstall' ./package.json)" != "null" ]; then + npm --production --offline --nodedir=$nodeSources run preinstall + fi if [ "$(jq '.scripts.install' ./package.json)" != "null" ]; then npm --production --offline --nodedir=$nodeSources run install fi diff --git a/src/subsystems/nodejs/translators/package-json/default.nix b/src/subsystems/nodejs/translators/package-json/default.nix index 02dd3cda..2e1cbfa5 100644 --- a/src/subsystems/nodejs/translators/package-json/default.nix +++ b/src/subsystems/nodejs/translators/package-json/default.nix @@ -47,11 +47,31 @@ cd ./$relPath rm -rf package-lock.json - if [ "$(jq '.project.subsystemInfo.noDev' -c -r $jsonInput)" == "true" ]; then + hasInstallScript=false + if [ "$(jq 'has("scripts")' -c -r package.json)" == "false" ]; then + : + elif [ "$(jq '.scripts | has("preinstall")' -c -r package.json)" == "true" ]; then + hasInstallScript=true + elif [ "$(jq '.scripts | has("install")' -c -r package.json)" == "true" ]; then + hasInstallScript=true + elif [ "$(jq '.scripts | has("postinstall")' -c -r package.json)" == "true" ]; then + hasInstallScript=true + else + : + fi + + echo "translating in temp dir: $(pwd)" + + if [ "$hasInstallScript" == "false" ]; then + echo "package.json does not define an [pre|post]install script -> omitting dev dependencies" + fi + + if [ "$hasInstallScript" == "false" ] \ + || [ "$(jq '.project.subsystemInfo.noDev' -c -r $jsonInput)" == "true" ]; then echo "excluding dev dependencies" jq '.devDependencies = {}' ./package.json > package.json.mod mv package.json.mod package.json - npm install --package-lock-only --production $npmArgs + npm install --package-lock-only --omit=dev $npmArgs else npm install --package-lock-only $npmArgs fi diff --git a/src/subsystems/nodejs/translators/package-lock/default.nix b/src/subsystems/nodejs/translators/package-lock/default.nix index cf5117f1..7a3cac5d 100644 --- a/src/subsystems/nodejs/translators/package-lock/default.nix +++ b/src/subsystems/nodejs/translators/package-lock/default.nix @@ -25,7 +25,15 @@ } @ args: let b = builtins; - dev = ! noDev; + hasInstallScript = + (packageJson ? scripts.preinstall) + || (packageJson ? scripts.install) + || (packageJson ? scripts.postinstall); + + noDev = + if ! hasInstallScript + then true + else args.noDev; name = project.name; tree = args.tree.getNodeFromPath project.relPath; relPath = project.relPath; @@ -70,11 +78,23 @@ then let path = getPath dependencyObject; in - ( - b.fromJSON - (b.readFile "${source}/${path}/package.json") - ) - .version + if ! (l.pathExists "${source}/${path}/package.json") + then + throw '' + The lock file references a sub-package residing at '${source}/${path}', + but that directory doesn't exist or doesn't contain a package.json + + The reason might be that devDependencies are not included in this package release. + Possible solutions: + - get full package source via git and translate from there + - disable devDependencies by passing `noDev` to the translator + '' + else + ( + b.fromJSON + (b.readFile "${source}/${path}/package.json") + ) + .version else if lib.hasPrefix "https://" dependencyObject.version then "unknown" else dependencyObject.version; @@ -153,7 +173,7 @@ version = getVersion pdata; }) (lib.filterAttrs - (pname: pdata: ! (pdata.dev or false) || dev) + (pname: pdata: ! (pdata.dev or false) || ! noDev) parsedDependencies); subsystemName = "nodejs"; @@ -186,7 +206,7 @@ in lib.filter (pdata: - dev || ! (pdata.dev or false)) + ! noDev || ! (pdata.dev or false)) (lib.flatten (serialize inputData)); getName = dependencyObject: dependencyObject.pname; diff --git a/src/subsystems/nodejs/translators/utils.nix b/src/subsystems/nodejs/translators/utils.nix index cff701e9..7bc0f683 100644 --- a/src/subsystems/nodejs/translators/utils.nix +++ b/src/subsystems/nodejs/translators/utils.nix @@ -2,8 +2,7 @@ l = lib // builtins; in rec { getPackageJsonDeps = packageJson: noDev: - packageJson.dependencies - or {} + (packageJson.dependencies or {}) // (lib.optionalAttrs (! noDev) (packageJson.devDependencies or {})); getWorkspaceLockFile = tree: project: fname: let @@ -17,9 +16,9 @@ in rec { (tree.getNodeFromPath "${dirRelPath}/package.json").jsonContent; hasNoDependencies = - ! packageJson ? dependencies - && ! packageJson ? devDependencies - && ! packageJson ? workspaces; + ((packageJson.dependencies or {}) == {}) + && ((packageJson.devDependencies or {}) == {}) + && (! packageJson ? workspaces); in if hasNoDependencies then null diff --git a/src/utils/default.nix b/src/utils/default.nix index c351a084..0c06cb08 100644 --- a/src/utils/default.nix +++ b/src/utils/default.nix @@ -232,10 +232,16 @@ in | python3 ${../apps/cli/format-dream-lock.py} \ | sponge $dreamLockPath + # validate dream-lock.json against jsonschema + ${python3.pkgs.jsonschema}/bin/jsonschema \ + --instance $dreamLockPath \ + --output pretty \ + ${../specifications/dream-lock-schema.json} + # add dream-lock.json to git if git rev-parse --show-toplevel &>/dev/null; then echo "adding file to git: $dreamLockPath" - git add $dreamLockPath + git add $dreamLockPath || : fi ''; in diff --git a/src/utils/dream-lock.nix b/src/utils/dream-lock.nix index b1a8714f..e766527f 100644 --- a/src/utils/dream-lock.nix +++ b/src/utils/dream-lock.nix @@ -11,7 +11,9 @@ replaceRootSources = { dreamLock, newSourceRoot, - }: let + } @ args: let + dreamLockLoaded = utils.readDreamLock {dreamLock = args.dreamLock;}; + iface = dreamLockLoaded.interface; patchVersion = version: source: if source.type @@ -24,10 +26,18 @@ dir = source.relPath; } else source; + patchedSources = - l.mapAttrs - (_: versions: l.mapAttrs patchVersion versions) - dreamLock.sources; + l.recursiveUpdate + { + "${iface.defaultPackageName}"."${iface.defaultPackageVersion}" = + newSourceRoot; + } + ( + l.mapAttrs + (_: versions: l.mapAttrs patchVersion versions) + dreamLock.sources + ); in dreamLock // {sources = patchedSources;}; diff --git a/src/utils/index.nix b/src/utils/index.nix index 63ba702b..66e08835 100644 --- a/src/utils/index.nix +++ b/src/utils/index.nix @@ -39,7 +39,12 @@ in rec { dreamLocks = findDreamLocks tree; makePackagesForDreamLock = dreamLock: (dream2nixInterface.makeOutputsForDreamLock { - inherit dreamLock; + inherit + dreamLock + inject + packageOverrides + sourceOverrides + ; }) .packages; in