mirror of
https://github.com/nix-community/dream2nix.git
synced 2024-10-26 15:23:14 +03:00
chore: indexers: improve build-all app
This commit is contained in:
parent
3450ce358e
commit
e2cc514c47
59
src/utils/index/build-script.py
Normal file
59
src/utils/index/build-script.py
Normal file
@ -0,0 +1,59 @@
|
||||
import json
|
||||
import sys
|
||||
import subprocess as sp
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def store_error(attrPath, category, text, name=None):
|
||||
with open(f"errors/{attrPath.replace('/', '--')}", 'w') as f:
|
||||
json.dump(
|
||||
dict(
|
||||
attrPath=attrPath,
|
||||
during=category,
|
||||
error=text,
|
||||
name=name,
|
||||
),
|
||||
f,
|
||||
)
|
||||
|
||||
|
||||
input = json.loads(sys.argv[1])
|
||||
attr = input['attr']
|
||||
attrPath = '.'.join(input['attrPath'])
|
||||
|
||||
# handle eval error
|
||||
if "error" in input:
|
||||
error = input['error']
|
||||
print(
|
||||
f"Evaluation failed. attr: {attr} attrPath: {attrPath}\n"
|
||||
"Error:\n{error}",
|
||||
file=sys.stderr
|
||||
)
|
||||
store_error(attrPath, 'eval', error)
|
||||
# try to build package
|
||||
else:
|
||||
name = input['name']
|
||||
drvPath = input['drvPath']
|
||||
print(
|
||||
f"Building {name} attr: {attr} attrPath: {attrPath} "
|
||||
f"drvPath: ({drvPath})",
|
||||
file=sys.stderr
|
||||
)
|
||||
try:
|
||||
proc = sp.run(
|
||||
['nix', 'build', '-L', drvPath],
|
||||
capture_output=True,
|
||||
check=True,
|
||||
)
|
||||
print(
|
||||
f"Finished {name}. attr: {attr} attrPath: {attrPath}",
|
||||
file=sys.stderr
|
||||
)
|
||||
# handle build error
|
||||
except sp.CalledProcessError as error:
|
||||
Path('errors').mkdir(exist_ok=True)
|
||||
print(
|
||||
f"Error while building {name}. attr: {attr} attrPath: {attrPath}",
|
||||
file=sys.stderr
|
||||
)
|
||||
store_error(attrPath, 'build', error.stderr.decode(), name)
|
@ -167,37 +167,10 @@ in rec {
|
||||
buildAllApp = let
|
||||
buildScript =
|
||||
pkgs.writers.writePython3 "build-job" {}
|
||||
''
|
||||
import sys
|
||||
import subprocess as sp
|
||||
from pathlib import path
|
||||
|
||||
input = json.load(open(sys.argv[1]))
|
||||
attr = input['attr']
|
||||
attrPath = '.'.join(input['attrPath'])
|
||||
if "error" in input:
|
||||
print(
|
||||
f"Evaluation failed. attr: {attr} attrPath: {attrPath}\nError:\n{error}",
|
||||
file=sys.stderr
|
||||
)
|
||||
else:
|
||||
name = input['name']
|
||||
drvPath = input['drvPath']
|
||||
print(
|
||||
f"Building {name}. attr: {attr} attrPath: {attrPath}\nError:\n{error}",
|
||||
file=sys.stderr
|
||||
)
|
||||
try:
|
||||
proc = sp.run(
|
||||
['nix', 'build', drvPath],
|
||||
capture_output = true,
|
||||
check=True,
|
||||
)
|
||||
except sp.CalledProcessError as error:
|
||||
Path('errors').mkdir(exist_ok=True)
|
||||
with open(f'errors/{name}') as f:
|
||||
f.write(error.stderr)
|
||||
'';
|
||||
./build-script.py;
|
||||
statsScript =
|
||||
pkgs.writers.writePython3 "build-job" {}
|
||||
./make-stats.py;
|
||||
in
|
||||
mkApp (
|
||||
utils.writePureShellScript
|
||||
@ -209,10 +182,22 @@ in rec {
|
||||
nix-eval-jobs
|
||||
])
|
||||
''
|
||||
rm -rf ./errors
|
||||
mkdir -p ./errors
|
||||
JOBS=''${JOBS:-$(nproc)}
|
||||
EVAL_JOBS=''${EVAL_JOBS:-1}
|
||||
LIMIT=''${LIMIT:-0}
|
||||
if [ "$LIMIT" -gt "0" ]; then
|
||||
limit="head -n $LIMIT"
|
||||
else
|
||||
limit="cat"
|
||||
fi
|
||||
echo "settings: JOBS $JOBS; EVAL_JOBS: $EVAL_JOBS; LIMIT $LIMIT"
|
||||
parallel --halt now,fail=1 -j$JOBS --link \
|
||||
-a <(nix-eval-jobs --gc-roots-dir $(pwd)/gcroot --flake $(realpath .)#packages.x86_64-linux --workers $JOBS) \
|
||||
-a <(nix-eval-jobs --gc-roots-dir $TMPDIR/gcroot --flake "$(realpath .)#packages.x86_64-linux" --workers $EVAL_JOBS --max-memory-size 3000 | $limit) \
|
||||
${buildScript}
|
||||
${statsScript}
|
||||
rm -r ./errors
|
||||
''
|
||||
);
|
||||
|
||||
|
32
src/utils/index/make-stats.py
Normal file
32
src/utils/index/make-stats.py
Normal file
@ -0,0 +1,32 @@
|
||||
import json
|
||||
import os
|
||||
|
||||
error_files = os.listdir('errors')
|
||||
eval_errors = 0
|
||||
build_errors = 0
|
||||
all_errors = {}
|
||||
|
||||
for file in error_files:
|
||||
with open(file) as f:
|
||||
error = json.load(f)
|
||||
# add error to all_errors
|
||||
all_errors[error['attrPath']] = error
|
||||
# count error types
|
||||
if error['category'] == 'eval':
|
||||
eval_errors += 1
|
||||
else:
|
||||
build_errors += 1
|
||||
|
||||
num_errors = eval_errors + build_errors
|
||||
|
||||
stats = dict(
|
||||
errors=num_errors,
|
||||
errors_eval=eval_errors,
|
||||
errors_build=build_errors,
|
||||
)
|
||||
|
||||
with open("errors.json", 'w') as f:
|
||||
json.dump(all_errors, f)
|
||||
|
||||
with open('stats.json', 'w') as f:
|
||||
json.dump(stats, f)
|
Loading…
Reference in New Issue
Block a user