mirror of
https://github.com/Mic92/nix-fast-build.git
synced 2024-10-05 13:47:08 +03:00
first commit
This commit is contained in:
commit
e3b29e5f80
18
default.nix
Normal file
18
default.nix
Normal file
@ -0,0 +1,18 @@
|
||||
{ python3, makeWrapper, nix, nix-eval-jobs, nix-output-monitor, lib }:
|
||||
let
|
||||
path = lib.makeBinPath [ nix nix-eval-jobs nix-output-monitor ];
|
||||
in
|
||||
python3.pkgs.buildPythonApplication {
|
||||
pname = "nix-ci-build";
|
||||
version = "0.1.0";
|
||||
format = "pyproject";
|
||||
src = ./.;
|
||||
buildInputs = with python3.pkgs; [ setuptools ];
|
||||
nativeBuildInputs = [ makeWrapper ];
|
||||
preFixup = ''
|
||||
makeWrapperArgs+=(--prefix PATH : ${path})
|
||||
'';
|
||||
shellHook = ''
|
||||
export PATH=${path}:$PATH
|
||||
'';
|
||||
}
|
64
flake.lock
Normal file
64
flake.lock
Normal file
@ -0,0 +1,64 @@
|
||||
{
|
||||
"nodes": {
|
||||
"flake-parts": {
|
||||
"inputs": {
|
||||
"nixpkgs-lib": "nixpkgs-lib"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1693611461,
|
||||
"narHash": "sha256-aPODl8vAgGQ0ZYFIRisxYG5MOGSkIczvu2Cd8Gb9+1Y=",
|
||||
"owner": "hercules-ci",
|
||||
"repo": "flake-parts",
|
||||
"rev": "7f53fdb7bdc5bb237da7fefef12d099e4fd611ca",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "hercules-ci",
|
||||
"repo": "flake-parts",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1694032533,
|
||||
"narHash": "sha256-I8cfCV/4JNJJ8KHOTxTU1EphKT8ARSb4s9pq99prYV0=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "efd23a1c9ae8c574e2ca923c2b2dc336797f4cc4",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "NixOS",
|
||||
"ref": "nixpkgs-unstable",
|
||||
"repo": "nixpkgs",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixpkgs-lib": {
|
||||
"locked": {
|
||||
"dir": "lib",
|
||||
"lastModified": 1693471703,
|
||||
"narHash": "sha256-0l03ZBL8P1P6z8MaSDS/MvuU8E75rVxe5eE1N6gxeTo=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "3e52e76b70d5508f3cec70b882a29199f4d1ee85",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"dir": "lib",
|
||||
"owner": "NixOS",
|
||||
"ref": "nixos-unstable",
|
||||
"repo": "nixpkgs",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"root": {
|
||||
"inputs": {
|
||||
"flake-parts": "flake-parts",
|
||||
"nixpkgs": "nixpkgs"
|
||||
}
|
||||
}
|
||||
},
|
||||
"root": "root",
|
||||
"version": 7
|
||||
}
|
24
flake.nix
Normal file
24
flake.nix
Normal file
@ -0,0 +1,24 @@
|
||||
{
|
||||
description = "Evaluate and build in parallel";
|
||||
|
||||
inputs = {
|
||||
nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
|
||||
flake-parts.url = "github:hercules-ci/flake-parts";
|
||||
};
|
||||
|
||||
outputs = inputs@{ flake-parts, ... }:
|
||||
flake-parts.lib.mkFlake { inherit inputs; } ({ lib, ... }: {
|
||||
systems = [
|
||||
"aarch64-linux"
|
||||
"x86_64-linux"
|
||||
"riscv64-linux"
|
||||
|
||||
"x86_64-darwin"
|
||||
"aarch64-darwin"
|
||||
];
|
||||
perSystem = { pkgs, self', ... }: {
|
||||
packages.nix-ci-builds = pkgs.callPackage ./default.nix {};
|
||||
packages.default = self'.packages.nix-ci-builds;
|
||||
};
|
||||
});
|
||||
}
|
297
nix_ci_build/__init__.py
Normal file
297
nix_ci_build/__init__.py
Normal file
@ -0,0 +1,297 @@
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from contextlib import ExitStack, contextmanager
|
||||
from dataclasses import dataclass, field
|
||||
from tempfile import TemporaryDirectory
|
||||
from typing import IO, Any, Iterator, NoReturn
|
||||
|
||||
|
||||
def die(msg: str) -> NoReturn:
|
||||
print(msg, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Options:
|
||||
flake: str = ""
|
||||
options: list[str] = field(default_factory=list)
|
||||
systems: set[str] = field(default_factory=set)
|
||||
max_jobs: int = 0
|
||||
retries: int = 0
|
||||
verbose: bool = False
|
||||
|
||||
|
||||
def run_nix(args: list[str]) -> subprocess.CompletedProcess[str]:
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
["nix"] + args, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
|
||||
)
|
||||
except FileNotFoundError:
|
||||
die("nix not found in PATH")
|
||||
return proc
|
||||
|
||||
|
||||
def current_system() -> str:
|
||||
proc = run_nix(["eval", "--impure", "--raw", "--expr", "builtins.currentSystem"])
|
||||
if proc.returncode != 0:
|
||||
die(f"Failed to determine current system: {proc.stderr}")
|
||||
return proc.stdout.strip()
|
||||
|
||||
|
||||
def max_jobs() -> int:
|
||||
proc = run_nix(["show-config", "max-jobs"])
|
||||
if proc.returncode != 0:
|
||||
die(f"Failed to determine number of CPUs: {proc.stderr}")
|
||||
return int(proc.stdout.strip())
|
||||
|
||||
|
||||
def parse_args(args: list[str]) -> Options:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"-f",
|
||||
"--flake",
|
||||
default=".#checks",
|
||||
help="Flake url to evaluate/build (default: .#checks",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-j",
|
||||
"--max-jobs",
|
||||
type=int,
|
||||
default=max_jobs(),
|
||||
help="Maximum number of build jobs to run in parallel (0 for unlimited)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--option", help="Nix option to set", action="append", default=[]
|
||||
)
|
||||
parser.add_argument(
|
||||
"--systems",
|
||||
help="Comma-separated list of systems to build for (default: current system)",
|
||||
default=current_system(),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--retries",
|
||||
type=int,
|
||||
default=0,
|
||||
help="Number of times to retry failed builds",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--verbose",
|
||||
action="store_true",
|
||||
help="Print verbose output",
|
||||
)
|
||||
a = parser.parse_args(args)
|
||||
systems = set(a.systems.split(","))
|
||||
return Options(
|
||||
flake=a.flake,
|
||||
options=a.option,
|
||||
max_jobs=a.max_jobs,
|
||||
verbose=a.verbose,
|
||||
systems=systems,
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def nix_eval_jobs(opts: Options) -> Iterator[subprocess.Popen[str]]:
|
||||
with TemporaryDirectory() as d:
|
||||
args = [
|
||||
"nix-eval-jobs",
|
||||
"--gc-roots-dir",
|
||||
d,
|
||||
"--force-recurse",
|
||||
"--flake",
|
||||
opts.flake,
|
||||
] + opts.options
|
||||
print("$ " + " ".join(args))
|
||||
with subprocess.Popen(args, text=True, stdout=subprocess.PIPE) as proc:
|
||||
try:
|
||||
yield proc
|
||||
finally:
|
||||
proc.kill()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def nix_build(
|
||||
installable: str, stdout: IO[Any] | None, opts: Options
|
||||
) -> Iterator[subprocess.Popen]:
|
||||
log_format = "raw"
|
||||
args = [
|
||||
"nix",
|
||||
"build",
|
||||
installable,
|
||||
"--log-format",
|
||||
log_format,
|
||||
"--keep-going",
|
||||
] + opts.options
|
||||
if opts.verbose:
|
||||
print("$ " + " ".join(args))
|
||||
with subprocess.Popen(args, text=True, stderr=stdout) as proc:
|
||||
try:
|
||||
yield proc
|
||||
finally:
|
||||
proc.kill()
|
||||
|
||||
|
||||
@dataclass
|
||||
class Build:
|
||||
attr: str
|
||||
drv_path: str
|
||||
proc: subprocess.Popen[str]
|
||||
retries: int
|
||||
rc: int | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class EvalError:
|
||||
attr: str
|
||||
error: str
|
||||
|
||||
|
||||
def wait_for_any_build(builds: list[Build]) -> Build:
|
||||
while True:
|
||||
for i, build in enumerate(builds):
|
||||
rc = build.proc.poll()
|
||||
if rc is not None:
|
||||
del builds[i]
|
||||
build.rc = rc
|
||||
return build
|
||||
time.sleep(0.05)
|
||||
|
||||
|
||||
def drain_builds(
|
||||
builds: list[Build], stdout: IO[Any] | None, stack: ExitStack, opts: Options
|
||||
) -> list[Build]:
|
||||
build = wait_for_any_build(builds)
|
||||
if build.rc != 0:
|
||||
print(f"build {build.attr} exited with {build.rc}", file=sys.stderr)
|
||||
if build.retries < opts.retries:
|
||||
print(f"retrying build {build.attr} [{build.retries + 1}/{opts.retries}]")
|
||||
builds.append(
|
||||
create_build(
|
||||
build.attr, build.drv_path, stdout, stack, opts, build.retries + 1
|
||||
)
|
||||
)
|
||||
else:
|
||||
return [build]
|
||||
return []
|
||||
|
||||
|
||||
def create_build(
|
||||
attr: str,
|
||||
drv_path: str,
|
||||
stdout: IO[Any] | None,
|
||||
exit_stack: ExitStack,
|
||||
opts: Options,
|
||||
retries: int = 0,
|
||||
) -> Build:
|
||||
nix_build_proc = exit_stack.enter_context(nix_build(drv_path + "^*", stdout, opts))
|
||||
return Build(attr, drv_path, nix_build_proc, retries=retries)
|
||||
|
||||
|
||||
class Pipe:
|
||||
def __init__(self) -> None:
|
||||
fds = os.pipe()
|
||||
self.read_file = os.fdopen(fds[0], "rb")
|
||||
self.write_file = os.fdopen(fds[1], "wb")
|
||||
|
||||
def __enter__(self) -> "Pipe":
|
||||
return self
|
||||
|
||||
def __exit__(self, _exc_type: Any, _exc_value: Any, _traceback: Any) -> None:
|
||||
self.read_file.close()
|
||||
self.write_file.close()
|
||||
|
||||
|
||||
def stop_gracefully(proc: subprocess.Popen, timeout: int = 1) -> None:
|
||||
proc.terminate()
|
||||
try:
|
||||
proc.wait(timeout=timeout)
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def nix_output_monitor(fd: int) -> Iterator[subprocess.Popen]:
|
||||
proc = subprocess.Popen(["nom"], stdin=fd)
|
||||
try:
|
||||
yield proc
|
||||
finally:
|
||||
stop_gracefully(proc)
|
||||
|
||||
|
||||
def run_builds(stack: ExitStack, opts: Options) -> int:
|
||||
eval_error = []
|
||||
build_failures = []
|
||||
drv_paths = set()
|
||||
proc = stack.enter_context(nix_eval_jobs(opts))
|
||||
assert proc.stdout
|
||||
pipe = stack.enter_context(Pipe())
|
||||
nom_proc: subprocess.Popen | None = None
|
||||
stdout = pipe.write_file
|
||||
builds = []
|
||||
for line in proc.stdout:
|
||||
if nom_proc is None:
|
||||
nom_proc = stack.enter_context(nix_output_monitor(pipe.read_file.fileno()))
|
||||
if opts.verbose:
|
||||
print(line, end="")
|
||||
try:
|
||||
job = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
die(f"Failed to parse line of nix-eval-jobs output: {line}")
|
||||
error = job.get("error")
|
||||
attr = job.get("attr", "unknown-flake-attribute")
|
||||
if error:
|
||||
eval_error.append(EvalError(attr, error))
|
||||
continue
|
||||
system = job.get("system")
|
||||
if system and system not in opts.systems:
|
||||
continue
|
||||
drv_path = job.get("drvPath")
|
||||
if not drv_path:
|
||||
die(f"nix-eval-jobs did not return a drvPath: {line}")
|
||||
while len(builds) >= opts.max_jobs and opts.max_jobs != 0:
|
||||
build_failures += drain_builds(builds, stdout, stack, opts)
|
||||
print(f" building {attr}")
|
||||
if drv_path in drv_paths:
|
||||
continue
|
||||
drv_paths.add(drv_path)
|
||||
builds.append(create_build(attr, drv_path, stdout, stack, opts))
|
||||
|
||||
while builds:
|
||||
build_failures += drain_builds(builds, stdout, stack, opts)
|
||||
|
||||
if nom_proc is not None:
|
||||
stop_gracefully(nom_proc)
|
||||
|
||||
eval_rc = proc.wait()
|
||||
if eval_rc != 0:
|
||||
print(
|
||||
f"nix-eval-jobs exited with {eval_rc}, check logs for details",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
for error in eval_error:
|
||||
print(f"{error.attr}: {error.error}", file=sys.stderr)
|
||||
|
||||
for build in build_failures:
|
||||
print(f"{build.attr}: build failed with {build.rc}", file=sys.stderr)
|
||||
|
||||
if len(build_failures) > 0 or len(eval_error) > 0 or eval_rc != 0:
|
||||
return 1
|
||||
else:
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> None:
|
||||
opts = parse_args(sys.argv[1:])
|
||||
rc = 0
|
||||
with ExitStack() as stack:
|
||||
rc = run_builds(stack, opts)
|
||||
sys.exit(rc)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
73
pyproject.toml
Normal file
73
pyproject.toml
Normal file
@ -0,0 +1,73 @@
|
||||
[build-system]
|
||||
requires = ["setuptools"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "nix-ci-build"
|
||||
description = "Evaluate and build in parallel"
|
||||
version = "2.10.1"
|
||||
authors = [{ name = "Jörg Thalheim", email = "joerg@thalheim.io" }]
|
||||
license = { text = "MIT" }
|
||||
classifiers = [
|
||||
"Development Status :: 5 - Production/Stable",
|
||||
"Environment :: Console",
|
||||
"Topic :: Utilities",
|
||||
"Intended Audience :: Developers",
|
||||
"Programming Language :: Python :: 3.6",
|
||||
]
|
||||
|
||||
[project.urls]
|
||||
Homepage = "https://github.com/Mic92/nix-ci-build"
|
||||
|
||||
[project.scripts]
|
||||
nix-ci-build = "nix_ci_build:main"
|
||||
|
||||
[tool.setuptools.packages]
|
||||
find = {}
|
||||
|
||||
[tool.setuptools.package-data]
|
||||
nixpkgs_review = ["nix/*.nix"]
|
||||
|
||||
|
||||
[tool.ruff]
|
||||
line-length = 88
|
||||
|
||||
select = ["E", "F", "I", "U", "N"]
|
||||
ignore = ["E501"]
|
||||
|
||||
[tool.black]
|
||||
line-length = 88
|
||||
target-version = ['py39']
|
||||
include = '\.pyi?$'
|
||||
exclude = '''
|
||||
/(
|
||||
\.git
|
||||
| \.hg
|
||||
| \.mypy_cache
|
||||
| \.tox
|
||||
| \.venv
|
||||
| _build
|
||||
| buck-out
|
||||
| build
|
||||
| dist
|
||||
# The following are specific to Black, you probably don't want those.
|
||||
| blib2to3
|
||||
| tests/data
|
||||
| profiling
|
||||
)/
|
||||
'''
|
||||
|
||||
[tool.mypy]
|
||||
python_version = "3.10"
|
||||
warn_redundant_casts = true
|
||||
disallow_untyped_calls = true
|
||||
disallow_untyped_defs = true
|
||||
no_implicit_optional = true
|
||||
|
||||
[[tool.mypy.overrides]]
|
||||
module = "setuptools.*"
|
||||
ignore_missing_imports = true
|
||||
|
||||
[[tool.mypy.overrides]]
|
||||
module = "pytest.*"
|
||||
ignore_missing_imports = true
|
Loading…
Reference in New Issue
Block a user