nix-community/dream2nix
1
1
Fork 0
mirror of https://github.com/nix-community/dream2nix.git synced 2024-12-25 15:33:20 +03:00
Code Issues Projects Releases Wiki Activity

indexers: log translation errors

Browse Source
This commit is contained in:
DavHau 2022-10-07 17:21:59 +02:00
parent de60de001e
commit d9b0d15eef
2 changed files with 24 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
src/apps/translate-index/default.nix
View File

@ -13,18 +13,27 @@
utils.writePureShellScript
[coreutils translate jq python3]
''
jobJson=$1
job_nr=$2
time=$(date +%s)
runtime=$(($time - $start_time))
average_runtime=$(python3 -c "print($runtime / $job_nr)")
total_remaining_time=$(python3 -c "print($average_runtime * ($num_jobs - $job_nr))")
echo "starting job nr. $job_nr; average job runtime: $average_runtime sec; remaining time: $total_remaining_time sec"
translate $1 $targetDir || echo "Failed to translate $1"
# if job fails, store error in ./translation-errors/$jobId.log
translate $jobJson $targetDir &> $TMPDIR/log \
|| (
echo "Failed to translate $1"
mkdir -p ./translation-errors
jobId=$(jq '.id' -c -r <(echo "$jobJson"))
cp $TMPDIR/log ./translation-errors/$jobId.log
)
'';
in
utils.writePureShellScriptBin
"translate-index"
[coreutils translate jq parallel]
[coreutils translate jq parallel python3]
''
set -e
usage="usage:
@ -65,4 +74,6 @@ in
runtime=$(($(date +%s) - $start_time))
echo "FINISHED! Executed $num_jobs jobs in $runtime seconds"
python3 ${./summarize-stats.py} translation-errors.json
''

11
src/apps/translate-index/summarize-stats.py Normal file
View File

@ -0,0 +1,11 @@
import json
import os
import sys
failed_proj_ids = list(os.listdir('translation-errors'))
if failed_proj_ids:
print("saving list of failed projects in ./translation-errors.json")
print("failure logs can be found in ./translation-errors/")
with open(sys.argv[1], 'w') as f:
json.dump(failed_proj_ids, f, indent=2)