chore: performance optimization for index translation

- allow translator binary to be passed as translator to the `translate`app
- this allows pre-evaluating/pre-building all translator executables once
- add more debugging statements for performance monitoring

src/apps/translate-index/default.nix

@@ -1,15 +1,26 @@
 {
+  callNixWithD2N,
   utils,
   translate,
   coreutils,
   jq,
   parallel,
+  python3,
   writeScript,
   ...
 }: let
-  script = writeScript "run-translate" ''
-    ${translate}/bin/translate $1 $targetDir || echo "Failed to translate $1"
-  '';
+  script =
+    utils.writePureShellScript
+    [coreutils translate jq python3]
+    ''
+      job_nr=$2
+      time=$(date +%s)
+      runtime=$(($time - $start_time))
+      average_runtime=$(python3 -c "print($runtime / $job_nr)")
+      total_remaining_time=$(python3 -c "print($average_runtime * ($num_jobs - $job_nr))")
+      echo "starting job nr. $job_nr; average job runtime: $average_runtime sec; remaining time: $total_remaining_time sec"
+      translate $1 $targetDir || echo "Failed to translate $1"
+    '';
 in
   utils.writePureShellScriptBin
   "translate-index"
@@ -31,6 +42,26 @@ in
 
     export targetDir
 
-    parallel --halt now,fail=1 -j$(nproc) --delay 1 -a <(jq '.[]' -c -r $index) ${script}
+    export num_jobs=$(jq 'length' -c -r $index)
+    seq $num_jobs > $TMPDIR/job_numbers
 
+    JOBS=''${JOBS:-$(nproc)}
+
+    # build translator executables
+    export TRANSLATOR_DIR=$TMPDIR/translators
+    for translator in $(jq '.[] | .translator' -c -r libraries-io/index.json); do
+      bin="$TRANSLATOR_DIR/$translator"
+      if [ ! -e "$bin" ]; then
+        echo "building executable for translator $translator"
+        ${callNixWithD2N} build -o "$bin" "
+          dream2nix.framework.translatorInstances.$translator.translateBin
+        "
+      fi
+    done
+
+    export start_time=$(date +%s)
+    parallel --halt now,fail=1 -j$JOBS --link -a <(jq '.[]' -c -r $index) -a $TMPDIR/job_numbers ${script}
+
+    runtime=$(($(date +%s) - $start_time))
+    echo "FINISHED! Executed $num_jobs jobs in $runtime seconds"
   ''

src/apps/translate/default.nix

@@ -46,9 +46,14 @@ utils.writePureShellScriptBin
 
   echo -e "\nTranslating:: $name (translator: $translator) (lock path: $dreamLockPath)"
 
-  translateBin=$(${callNixWithD2N} build --print-out-paths --no-link "
-    dream2nix.framework.translatorInstances.$translator.translateBin
-  ")
+  # allow pre-built translator executables to avoid the `nix build` on each run
+  if [ -n "$TRANSLATOR_DIR" ]; then
+    translateBin="$TRANSLATOR_DIR/$translator"
+  else
+    translateBin=$(${callNixWithD2N} build --print-out-paths --no-link "
+      dream2nix.framework.translatorInstances.$translator.translateBin
+    ")
+  fi
 
   echo "
     {