chore: indexers: improve build-all app

src/utils/index/build-script.py

@@ -0,0 +1,59 @@
+import json
+import sys
+import subprocess as sp
+from pathlib import Path
+
+
+def store_error(attrPath, category, text, name=None):
+    with open(f"errors/{attrPath.replace('/', '--')}", 'w') as f:
+        json.dump(
+            dict(
+                attrPath=attrPath,
+                during=category,
+                error=text,
+                name=name,
+            ),
+            f,
+        )
+
+
+input = json.loads(sys.argv[1])
+attr = input['attr']
+attrPath = '.'.join(input['attrPath'])
+
+# handle eval error
+if "error" in input:
+    error = input['error']
+    print(
+        f"Evaluation failed. attr: {attr} attrPath: {attrPath}\n"
+        "Error:\n{error}",
+        file=sys.stderr
+    )
+    store_error(attrPath, 'eval', error)
+# try to build package
+else:
+    name = input['name']
+    drvPath = input['drvPath']
+    print(
+        f"Building {name} attr: {attr} attrPath: {attrPath} "
+        f"drvPath: ({drvPath})",
+        file=sys.stderr
+    )
+    try:
+        proc = sp.run(
+            ['nix', 'build', '-L', drvPath],
+            capture_output=True,
+            check=True,
+        )
+        print(
+            f"Finished {name}. attr: {attr} attrPath: {attrPath}",
+            file=sys.stderr
+        )
+    # handle build error
+    except sp.CalledProcessError as error:
+        Path('errors').mkdir(exist_ok=True)
+        print(
+            f"Error while building {name}. attr: {attr} attrPath: {attrPath}",
+            file=sys.stderr
+        )
+        store_error(attrPath, 'build', error.stderr.decode(), name)

src/utils/index/default.nix

@@ -167,37 +167,10 @@ in rec {
     buildAllApp = let
       buildScript =
         pkgs.writers.writePython3 "build-job" {}
-        ''
-          import sys
-          import subprocess as sp
-          from pathlib import path
-
-          input = json.load(open(sys.argv[1]))
-          attr = input['attr']
-          attrPath = '.'.join(input['attrPath'])
-          if "error" in input:
-            print(
-              f"Evaluation failed. attr: {attr} attrPath: {attrPath}\nError:\n{error}",
-              file=sys.stderr
-            )
-          else:
-            name = input['name']
-            drvPath = input['drvPath']
-            print(
-              f"Building {name}. attr: {attr} attrPath: {attrPath}\nError:\n{error}",
-              file=sys.stderr
-            )
-            try:
-              proc = sp.run(
-                ['nix', 'build', drvPath],
-                capture_output = true,
-                check=True,
-              )
-            except sp.CalledProcessError as error:
-              Path('errors').mkdir(exist_ok=True)
-              with open(f'errors/{name}') as f:
-                f.write(error.stderr)
-        '';
+        ./build-script.py;
+      statsScript =
+        pkgs.writers.writePython3 "build-job" {}
+        ./make-stats.py;
     in
       mkApp (
         utils.writePureShellScript
@@ -209,10 +182,22 @@ in rec {
           nix-eval-jobs
         ])
         ''
+          rm -rf ./errors
+          mkdir -p ./errors
           JOBS=''${JOBS:-$(nproc)}
+          EVAL_JOBS=''${EVAL_JOBS:-1}
+          LIMIT=''${LIMIT:-0}
+          if [ "$LIMIT" -gt "0" ]; then
+            limit="head -n $LIMIT"
+          else
+            limit="cat"
+          fi
+          echo "settings: JOBS $JOBS; EVAL_JOBS: $EVAL_JOBS; LIMIT $LIMIT"
           parallel --halt now,fail=1 -j$JOBS --link \
-            -a <(nix-eval-jobs --gc-roots-dir $(pwd)/gcroot --flake $(realpath .)#packages.x86_64-linux --workers $JOBS) \
+            -a <(nix-eval-jobs --gc-roots-dir $TMPDIR/gcroot --flake "$(realpath .)#packages.x86_64-linux" --workers $EVAL_JOBS --max-memory-size 3000 | $limit) \
             ${buildScript}
+          ${statsScript}
+          rm -r ./errors
         ''
       );
 

src/utils/index/make-stats.py

@@ -0,0 +1,32 @@
+import json
+import os
+
+error_files = os.listdir('errors')
+eval_errors = 0
+build_errors = 0
+all_errors = {}
+
+for file in error_files:
+    with open(file) as f:
+        error = json.load(f)
+    # add error to all_errors
+    all_errors[error['attrPath']] = error
+    # count error types
+    if error['category'] == 'eval':
+        eval_errors += 1
+    else:
+        build_errors += 1
+
+num_errors = eval_errors + build_errors
+
+stats = dict(
+    errors=num_errors,
+    errors_eval=eval_errors,
+    errors_build=build_errors,
+)
+
+with open("errors.json", 'w') as f:
+    json.dump(all_errors, f)
+
+with open('stats.json', 'w') as f:
+    json.dump(stats, f)