mirror of
https://github.com/nix-community/dream2nix.git
synced 2024-09-11 14:35:30 +03:00
Merge pull request #293 from nix-community/fix-index-apps
Improve indexer architecture
This commit is contained in:
commit
e0eefe6d08
@ -1,15 +1,26 @@
|
||||
{
|
||||
callNixWithD2N,
|
||||
utils,
|
||||
translate,
|
||||
coreutils,
|
||||
jq,
|
||||
parallel,
|
||||
python3,
|
||||
writeScript,
|
||||
...
|
||||
}: let
|
||||
script = writeScript "run-translate" ''
|
||||
${translate}/bin/translate $1 $targetDir || echo "Failed to translate $1"
|
||||
'';
|
||||
script =
|
||||
utils.writePureShellScript
|
||||
[coreutils translate jq python3]
|
||||
''
|
||||
job_nr=$2
|
||||
time=$(date +%s)
|
||||
runtime=$(($time - $start_time))
|
||||
average_runtime=$(python3 -c "print($runtime / $job_nr)")
|
||||
total_remaining_time=$(python3 -c "print($average_runtime * ($num_jobs - $job_nr))")
|
||||
echo "starting job nr. $job_nr; average job runtime: $average_runtime sec; remaining time: $total_remaining_time sec"
|
||||
translate $1 $targetDir || echo "Failed to translate $1"
|
||||
'';
|
||||
in
|
||||
utils.writePureShellScriptBin
|
||||
"translate-index"
|
||||
@ -31,6 +42,26 @@ in
|
||||
|
||||
export targetDir
|
||||
|
||||
parallel --halt now,fail=1 -j$(nproc) --delay 1 -a <(jq '.[]' -c -r $index) ${script}
|
||||
export num_jobs=$(jq 'length' -c -r $index)
|
||||
seq $num_jobs > $TMPDIR/job_numbers
|
||||
|
||||
JOBS=''${JOBS:-$(nproc)}
|
||||
|
||||
# build translator executables
|
||||
export TRANSLATOR_DIR=$TMPDIR/translators
|
||||
for translator in $(jq '.[] | .translator' -c -r libraries-io/index.json); do
|
||||
bin="$TRANSLATOR_DIR/$translator"
|
||||
if [ ! -e "$bin" ]; then
|
||||
echo "building executable for translator $translator"
|
||||
${callNixWithD2N} build -o "$bin" "
|
||||
dream2nix.framework.translatorInstances.$translator.translateBin
|
||||
"
|
||||
fi
|
||||
done
|
||||
|
||||
export start_time=$(date +%s)
|
||||
parallel --halt now,fail=1 -j$JOBS --link -a <(jq '.[]' -c -r $index) -a $TMPDIR/job_numbers ${script}
|
||||
|
||||
runtime=$(($(date +%s) - $start_time))
|
||||
echo "FINISHED! Executed $num_jobs jobs in $runtime seconds"
|
||||
''
|
||||
|
@ -46,9 +46,14 @@ utils.writePureShellScriptBin
|
||||
|
||||
echo -e "\nTranslating:: $name (translator: $translator) (lock path: $dreamLockPath)"
|
||||
|
||||
translateBin=$(${callNixWithD2N} build --print-out-paths --no-link "
|
||||
dream2nix.framework.translatorInstances.$translator.translateBin
|
||||
")
|
||||
# allow pre-built translator executables to avoid the `nix build` on each run
|
||||
if [ -n "$TRANSLATOR_DIR" ]; then
|
||||
translateBin="$TRANSLATOR_DIR/$translator"
|
||||
else
|
||||
translateBin=$(${callNixWithD2N} build --print-out-paths --no-link "
|
||||
dream2nix.framework.translatorInstances.$translator.translateBin
|
||||
")
|
||||
fi
|
||||
|
||||
echo "
|
||||
{
|
||||
|
@ -20,6 +20,7 @@
|
||||
moreutils,
|
||||
nodePackages,
|
||||
openssh,
|
||||
python3,
|
||||
writeScriptBin,
|
||||
...
|
||||
}:
|
||||
@ -32,6 +33,7 @@
|
||||
moreutils
|
||||
nodePackages.npm
|
||||
openssh
|
||||
python3
|
||||
]
|
||||
''
|
||||
# accroding to the spec, the translator reads the input from a json file
|
||||
@ -42,6 +44,9 @@
|
||||
name=$(jq '.project.name' -c -r $jsonInput)
|
||||
version=$(jq '.project.version' -c -r $jsonInput)
|
||||
npmArgs=$(jq '.project.subsystemInfo.npmArgs' -c -r $jsonInput)
|
||||
if [ "$npmArgs" == "null" ]; then
|
||||
npmArgs=
|
||||
fi
|
||||
|
||||
if [ "$version" = "null" ]; then
|
||||
candidate="$name"
|
||||
@ -49,7 +54,6 @@
|
||||
candidate="$name@$version"
|
||||
fi
|
||||
|
||||
|
||||
pushd $TMPDIR
|
||||
newSource=$(pwd)
|
||||
|
||||
@ -62,26 +66,14 @@
|
||||
# call package-lock translator
|
||||
${subsystems.nodejs.translators.package-lock.translateBin} $TMPDIR/newJsonInput
|
||||
|
||||
# generate source info for main package
|
||||
url=$(npm view $candidate dist.tarball)
|
||||
hash=$(npm view $candidate dist.integrity)
|
||||
echo "
|
||||
{
|
||||
\"type\": \"http\",
|
||||
\"url\": \"$url\",
|
||||
\"hash\": \"$hash\"
|
||||
}
|
||||
" > $TMPDIR/sourceInfo.json
|
||||
# get resolved package version
|
||||
export version=$(npm view $candidate version)
|
||||
|
||||
# set correct package version under `packages`
|
||||
cat $outputFile \
|
||||
| python3 ${./fixup-dream-lock.py} $TMPDIR/sourceInfo.json \
|
||||
| sponge $outputFile
|
||||
|
||||
# add main package source info to dream-lock.json
|
||||
${apps.callNixWithD2N} eval --json "
|
||||
with dream2nix.utils.dreamLock;
|
||||
replaceRootSources {
|
||||
dreamLock = l.fromJSON (l.readFile \"$outputFile\");
|
||||
newSourceRoot = l.fromJSON (l.readFile \"$TMPDIR/sourceInfo.json\");
|
||||
}
|
||||
" \
|
||||
| sponge "$outputFile"
|
||||
'';
|
||||
|
||||
# inherit options from package-lock translator
|
||||
|
14
src/subsystems/nodejs/translators/npm/fixup-dream-lock.py
Normal file
14
src/subsystems/nodejs/translators/npm/fixup-dream-lock.py
Normal file
@ -0,0 +1,14 @@
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
lock = json.load(sys.stdin)
|
||||
version = os.environ.get('version')
|
||||
|
||||
# set default package version correctly
|
||||
defaultPackage = lock['_generic']['defaultPackage']
|
||||
lock['_generic']['packages'] = {
|
||||
defaultPackage: version
|
||||
}
|
||||
|
||||
print(json.dumps(lock, indent=2))
|
@ -34,7 +34,7 @@
|
||||
|
||||
translatorUtils = callPackageDream ./translator.nix {};
|
||||
|
||||
indexUtils = callPackageDream ./index.nix {};
|
||||
indexUtils = callPackageDream ./index {};
|
||||
|
||||
poetry2nixSemver = import "${externalSources.poetry2nix}/semver.nix" {
|
||||
inherit lib;
|
||||
|
59
src/utils/index/build-script.py
Normal file
59
src/utils/index/build-script.py
Normal file
@ -0,0 +1,59 @@
|
||||
import json
|
||||
import sys
|
||||
import subprocess as sp
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def store_error(attrPath, category, text, name=None):
|
||||
with open(f"errors/{attrPath.replace('/', '--')}", 'w') as f:
|
||||
json.dump(
|
||||
dict(
|
||||
attrPath=attrPath,
|
||||
during=category,
|
||||
error=text,
|
||||
name=name,
|
||||
),
|
||||
f,
|
||||
)
|
||||
|
||||
|
||||
input = json.loads(sys.argv[1])
|
||||
attr = input['attr']
|
||||
attrPath = '.'.join(input['attrPath'])
|
||||
|
||||
# handle eval error
|
||||
if "error" in input:
|
||||
error = input['error']
|
||||
print(
|
||||
f"Evaluation failed. attr: {attr} attrPath: {attrPath}\n"
|
||||
"Error:\n{error}",
|
||||
file=sys.stderr
|
||||
)
|
||||
store_error(attrPath, 'eval', error)
|
||||
# try to build package
|
||||
else:
|
||||
name = input['name']
|
||||
drvPath = input['drvPath']
|
||||
print(
|
||||
f"Building {name} attr: {attr} attrPath: {attrPath} "
|
||||
f"drvPath: ({drvPath})",
|
||||
file=sys.stderr
|
||||
)
|
||||
try:
|
||||
proc = sp.run(
|
||||
['nix', 'build', '-L', drvPath],
|
||||
capture_output=True,
|
||||
check=True,
|
||||
)
|
||||
print(
|
||||
f"Finished {name}. attr: {attr} attrPath: {attrPath}",
|
||||
file=sys.stderr
|
||||
)
|
||||
# handle build error
|
||||
except sp.CalledProcessError as error:
|
||||
Path('errors').mkdir(exist_ok=True)
|
||||
print(
|
||||
f"Error while building {name}. attr: {attr} attrPath: {attrPath}",
|
||||
file=sys.stderr
|
||||
)
|
||||
store_error(attrPath, 'build', error.stderr.decode(), name)
|
@ -80,6 +80,7 @@ in rec {
|
||||
'';
|
||||
in
|
||||
mkApp script;
|
||||
|
||||
mkTranslateApp = name:
|
||||
mkApp (
|
||||
pkgs.writers.writeBash "translate-${name}" ''
|
||||
@ -88,27 +89,41 @@ in rec {
|
||||
${name}/index.json ${name}/locks
|
||||
''
|
||||
);
|
||||
mkCiJobApp = name: input:
|
||||
|
||||
mkCiAppWith = commands:
|
||||
mkApp (
|
||||
utils.writePureShellScript
|
||||
(with pkgs; [
|
||||
coreutils
|
||||
git
|
||||
gnugrep
|
||||
openssh
|
||||
])
|
||||
''
|
||||
mainBranch=$(git branch | grep -E '(master)|(main)')
|
||||
git branch data || :
|
||||
git checkout data
|
||||
flake=$(cat flake.nix)
|
||||
flakeLock=$(cat flake.lock)
|
||||
set -x
|
||||
git fetch origin data || :
|
||||
git checkout origin/data || :
|
||||
git branch -D data || :
|
||||
git checkout -b data
|
||||
# the flake should always be the one from the current main branch
|
||||
git checkout $mainBranch flake.nix
|
||||
git checkout $mainBranch flake.lock
|
||||
${(mkIndexApp name input).program}
|
||||
${(mkTranslateApp name).program}
|
||||
rm -rf ./*
|
||||
echo "$flake" > flake.nix
|
||||
echo "$flakeLock" > flake.lock
|
||||
${commands}
|
||||
git add .
|
||||
git commit "automatic update - $(date --rfc-3339=seconds)"
|
||||
git commit -m "automatic update - $(date --rfc-3339=seconds)"
|
||||
''
|
||||
);
|
||||
|
||||
mkCiJobApp = name: input:
|
||||
mkCiAppWith
|
||||
''
|
||||
${(mkIndexApp name input).program}
|
||||
${(mkTranslateApp name).program}
|
||||
'';
|
||||
|
||||
translateApps = l.listToAttrs (
|
||||
l.map
|
||||
(
|
||||
@ -119,6 +134,7 @@ in rec {
|
||||
)
|
||||
indexNames
|
||||
);
|
||||
|
||||
indexApps = l.listToAttrs (
|
||||
l.mapAttrsToList
|
||||
(
|
||||
@ -129,6 +145,7 @@ in rec {
|
||||
)
|
||||
indexes
|
||||
);
|
||||
|
||||
ciJobApps = l.listToAttrs (
|
||||
l.mapAttrsToList
|
||||
(
|
||||
@ -140,6 +157,50 @@ in rec {
|
||||
indexes
|
||||
);
|
||||
|
||||
ciJobAllApp =
|
||||
mkCiAppWith
|
||||
''
|
||||
${lib.concatStringsSep "\n" (l.mapAttrsToList (_: app: app.program) indexApps)}
|
||||
${lib.concatStringsSep "\n" (l.mapAttrsToList (_: app: app.program) translateApps)}
|
||||
'';
|
||||
|
||||
buildAllApp = let
|
||||
buildScript =
|
||||
pkgs.writers.writePython3 "build-job" {}
|
||||
./build-script.py;
|
||||
statsScript =
|
||||
pkgs.writers.writePython3 "build-job" {}
|
||||
./make-stats.py;
|
||||
in
|
||||
mkApp (
|
||||
utils.writePureShellScript
|
||||
(with pkgs; [
|
||||
coreutils
|
||||
git
|
||||
parallel
|
||||
nix
|
||||
nix-eval-jobs
|
||||
])
|
||||
''
|
||||
rm -rf ./errors
|
||||
mkdir -p ./errors
|
||||
JOBS=''${JOBS:-$(nproc)}
|
||||
EVAL_JOBS=''${EVAL_JOBS:-1}
|
||||
LIMIT=''${LIMIT:-0}
|
||||
if [ "$LIMIT" -gt "0" ]; then
|
||||
limit="head -n $LIMIT"
|
||||
else
|
||||
limit="cat"
|
||||
fi
|
||||
echo "settings: JOBS $JOBS; EVAL_JOBS: $EVAL_JOBS; LIMIT $LIMIT"
|
||||
parallel --halt now,fail=1 -j$JOBS --link \
|
||||
-a <(nix-eval-jobs --gc-roots-dir $TMPDIR/gcroot --flake "$(realpath .)#packages.x86_64-linux" --workers $EVAL_JOBS --max-memory-size 3000 | $limit) \
|
||||
${buildScript}
|
||||
${statsScript}
|
||||
rm -r ./errors
|
||||
''
|
||||
);
|
||||
|
||||
mkIndexOutputs = name: let
|
||||
src = "${toString source}/${name}/locks";
|
||||
in
|
||||
@ -167,7 +228,9 @@ in rec {
|
||||
outputs = {
|
||||
packages = allPackages;
|
||||
apps =
|
||||
indexApps
|
||||
{ci-job-all = ciJobAllApp;}
|
||||
// {build-all = buildAllApp;}
|
||||
// indexApps
|
||||
// translateApps
|
||||
// ciJobApps;
|
||||
};
|
32
src/utils/index/make-stats.py
Normal file
32
src/utils/index/make-stats.py
Normal file
@ -0,0 +1,32 @@
|
||||
import json
|
||||
import os
|
||||
|
||||
error_files = os.listdir('errors')
|
||||
eval_errors = 0
|
||||
build_errors = 0
|
||||
all_errors = {}
|
||||
|
||||
for file in error_files:
|
||||
with open(file) as f:
|
||||
error = json.load(f)
|
||||
# add error to all_errors
|
||||
all_errors[error['attrPath']] = error
|
||||
# count error types
|
||||
if error['category'] == 'eval':
|
||||
eval_errors += 1
|
||||
else:
|
||||
build_errors += 1
|
||||
|
||||
num_errors = eval_errors + build_errors
|
||||
|
||||
stats = dict(
|
||||
errors=num_errors,
|
||||
errors_eval=eval_errors,
|
||||
errors_build=build_errors,
|
||||
)
|
||||
|
||||
with open("errors.json", 'w') as f:
|
||||
json.dump(all_errors, f)
|
||||
|
||||
with open('stats.json', 'w') as f:
|
||||
json.dump(stats, f)
|
Loading…
Reference in New Issue
Block a user