mirror of
https://github.com/nix-community/dream2nix.git
synced 2024-09-11 14:35:30 +03:00
chore: performance optimization for index translation
- allow translator binary to be passed as translator to the `translate`app - this allows pre-evaluating/pre-building all translator executables once - add more debugging statements for performance monitoring
This commit is contained in:
parent
25fd6e1a63
commit
1d182f0808
@ -1,15 +1,26 @@
|
||||
{
|
||||
callNixWithD2N,
|
||||
utils,
|
||||
translate,
|
||||
coreutils,
|
||||
jq,
|
||||
parallel,
|
||||
python3,
|
||||
writeScript,
|
||||
...
|
||||
}: let
|
||||
script = writeScript "run-translate" ''
|
||||
${translate}/bin/translate $1 $targetDir || echo "Failed to translate $1"
|
||||
'';
|
||||
script =
|
||||
utils.writePureShellScript
|
||||
[coreutils translate jq python3]
|
||||
''
|
||||
job_nr=$2
|
||||
time=$(date +%s)
|
||||
runtime=$(($time - $start_time))
|
||||
average_runtime=$(python3 -c "print($runtime / $job_nr)")
|
||||
total_remaining_time=$(python3 -c "print($average_runtime * ($num_jobs - $job_nr))")
|
||||
echo "starting job nr. $job_nr; average job runtime: $average_runtime sec; remaining time: $total_remaining_time sec"
|
||||
translate $1 $targetDir || echo "Failed to translate $1"
|
||||
'';
|
||||
in
|
||||
utils.writePureShellScriptBin
|
||||
"translate-index"
|
||||
@ -31,6 +42,26 @@ in
|
||||
|
||||
export targetDir
|
||||
|
||||
parallel --halt now,fail=1 -j$(nproc) --delay 1 -a <(jq '.[]' -c -r $index) ${script}
|
||||
export num_jobs=$(jq 'length' -c -r $index)
|
||||
seq $num_jobs > $TMPDIR/job_numbers
|
||||
|
||||
JOBS=''${JOBS:-$(nproc)}
|
||||
|
||||
# build translator executables
|
||||
export TRANSLATOR_DIR=$TMPDIR/translators
|
||||
for translator in $(jq '.[] | .translator' -c -r libraries-io/index.json); do
|
||||
bin="$TRANSLATOR_DIR/$translator"
|
||||
if [ ! -e "$bin" ]; then
|
||||
echo "building executable for translator $translator"
|
||||
${callNixWithD2N} build -o "$bin" "
|
||||
dream2nix.framework.translatorInstances.$translator.translateBin
|
||||
"
|
||||
fi
|
||||
done
|
||||
|
||||
export start_time=$(date +%s)
|
||||
parallel --halt now,fail=1 -j$JOBS --link -a <(jq '.[]' -c -r $index) -a $TMPDIR/job_numbers ${script}
|
||||
|
||||
runtime=$(($(date +%s) - $start_time))
|
||||
echo "FINISHED! Executed $num_jobs jobs in $runtime seconds"
|
||||
''
|
||||
|
@ -46,9 +46,14 @@ utils.writePureShellScriptBin
|
||||
|
||||
echo -e "\nTranslating:: $name (translator: $translator) (lock path: $dreamLockPath)"
|
||||
|
||||
translateBin=$(${callNixWithD2N} build --print-out-paths --no-link "
|
||||
dream2nix.framework.translatorInstances.$translator.translateBin
|
||||
")
|
||||
# allow pre-built translator executables to avoid the `nix build` on each run
|
||||
if [ -n "$TRANSLATOR_DIR" ]; then
|
||||
translateBin="$TRANSLATOR_DIR/$translator"
|
||||
else
|
||||
translateBin=$(${callNixWithD2N} build --print-out-paths --no-link "
|
||||
dream2nix.framework.translatorInstances.$translator.translateBin
|
||||
")
|
||||
fi
|
||||
|
||||
echo "
|
||||
{
|
||||
|
Loading…
Reference in New Issue
Block a user