first commit

.envrc

@@ -0,0 +1 @@
+use flake

README.md

@@ -0,0 +1,3 @@
+# nix-ci-build
+
+nix-eval-jobs + 

default.nix

@@ -0,0 +1,18 @@
+{ python3, makeWrapper, nix, nix-eval-jobs, nix-output-monitor, lib }:
+let
+  path = lib.makeBinPath [ nix nix-eval-jobs nix-output-monitor ];
+in
+python3.pkgs.buildPythonApplication {
+  pname = "nix-ci-build";
+  version = "0.1.0";
+  format = "pyproject";
+  src = ./.;
+  buildInputs = with python3.pkgs; [ setuptools ];
+  nativeBuildInputs = [ makeWrapper ];
+  preFixup = ''
+    makeWrapperArgs+=(--prefix PATH : ${path})
+  '';
+  shellHook = ''
+    export PATH=${path}:$PATH
+  '';
+}

flake.lock

@@ -0,0 +1,64 @@
+{
+  "nodes": {
+    "flake-parts": {
+      "inputs": {
+        "nixpkgs-lib": "nixpkgs-lib"
+      },
+      "locked": {
+        "lastModified": 1693611461,
+        "narHash": "sha256-aPODl8vAgGQ0ZYFIRisxYG5MOGSkIczvu2Cd8Gb9+1Y=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "7f53fdb7bdc5bb237da7fefef12d099e4fd611ca",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1694032533,
+        "narHash": "sha256-I8cfCV/4JNJJ8KHOTxTU1EphKT8ARSb4s9pq99prYV0=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "efd23a1c9ae8c574e2ca923c2b2dc336797f4cc4",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixpkgs-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs-lib": {
+      "locked": {
+        "dir": "lib",
+        "lastModified": 1693471703,
+        "narHash": "sha256-0l03ZBL8P1P6z8MaSDS/MvuU8E75rVxe5eE1N6gxeTo=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "3e52e76b70d5508f3cec70b882a29199f4d1ee85",
+        "type": "github"
+      },
+      "original": {
+        "dir": "lib",
+        "owner": "NixOS",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "flake-parts": "flake-parts",
+        "nixpkgs": "nixpkgs"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -0,0 +1,24 @@
+{
+  description = "Evaluate and build in parallel";
+
+  inputs = {
+    nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
+    flake-parts.url = "github:hercules-ci/flake-parts";
+  };
+
+  outputs = inputs@{ flake-parts, ... }:
+    flake-parts.lib.mkFlake { inherit inputs; } ({ lib, ... }: {
+      systems = [
+        "aarch64-linux"
+        "x86_64-linux"
+        "riscv64-linux"
+
+        "x86_64-darwin"
+        "aarch64-darwin"
+      ];
+      perSystem = { pkgs, self', ... }: {
+        packages.nix-ci-builds = pkgs.callPackage ./default.nix {};
+        packages.default = self'.packages.nix-ci-builds;
+      };
+    });
+}

nix_ci_build/__init__.py

@@ -0,0 +1,297 @@
+import argparse
+import json
+import os
+import subprocess
+import sys
+import time
+from contextlib import ExitStack, contextmanager
+from dataclasses import dataclass, field
+from tempfile import TemporaryDirectory
+from typing import IO, Any, Iterator, NoReturn
+
+
+def die(msg: str) -> NoReturn:
+    print(msg, file=sys.stderr)
+    sys.exit(1)
+
+
+@dataclass
+class Options:
+    flake: str = ""
+    options: list[str] = field(default_factory=list)
+    systems: set[str] = field(default_factory=set)
+    max_jobs: int = 0
+    retries: int = 0
+    verbose: bool = False
+
+
+def run_nix(args: list[str]) -> subprocess.CompletedProcess[str]:
+    try:
+        proc = subprocess.run(
+            ["nix"] + args, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
+        )
+    except FileNotFoundError:
+        die("nix not found in PATH")
+    return proc
+
+
+def current_system() -> str:
+    proc = run_nix(["eval", "--impure", "--raw", "--expr", "builtins.currentSystem"])
+    if proc.returncode != 0:
+        die(f"Failed to determine current system: {proc.stderr}")
+    return proc.stdout.strip()
+
+
+def max_jobs() -> int:
+    proc = run_nix(["show-config", "max-jobs"])
+    if proc.returncode != 0:
+        die(f"Failed to determine number of CPUs: {proc.stderr}")
+    return int(proc.stdout.strip())
+
+
+def parse_args(args: list[str]) -> Options:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-f",
+        "--flake",
+        default=".#checks",
+        help="Flake url to evaluate/build (default: .#checks",
+    )
+    parser.add_argument(
+        "-j",
+        "--max-jobs",
+        type=int,
+        default=max_jobs(),
+        help="Maximum number of build jobs to run in parallel (0 for unlimited)",
+    )
+    parser.add_argument(
+        "--option", help="Nix option to set", action="append", default=[]
+    )
+    parser.add_argument(
+        "--systems",
+        help="Comma-separated list of systems to build for (default: current system)",
+        default=current_system(),
+    )
+    parser.add_argument(
+        "--retries",
+        type=int,
+        default=0,
+        help="Number of times to retry failed builds",
+    )
+    parser.add_argument(
+        "--verbose",
+        action="store_true",
+        help="Print verbose output",
+    )
+    a = parser.parse_args(args)
+    systems = set(a.systems.split(","))
+    return Options(
+        flake=a.flake,
+        options=a.option,
+        max_jobs=a.max_jobs,
+        verbose=a.verbose,
+        systems=systems,
+    )
+
+
+@contextmanager
+def nix_eval_jobs(opts: Options) -> Iterator[subprocess.Popen[str]]:
+    with TemporaryDirectory() as d:
+        args = [
+            "nix-eval-jobs",
+            "--gc-roots-dir",
+            d,
+            "--force-recurse",
+            "--flake",
+            opts.flake,
+        ] + opts.options
+        print("$ " + " ".join(args))
+        with subprocess.Popen(args, text=True, stdout=subprocess.PIPE) as proc:
+            try:
+                yield proc
+            finally:
+                proc.kill()
+
+
+@contextmanager
+def nix_build(
+    installable: str, stdout: IO[Any] | None, opts: Options
+) -> Iterator[subprocess.Popen]:
+    log_format = "raw"
+    args = [
+        "nix",
+        "build",
+        installable,
+        "--log-format",
+        log_format,
+        "--keep-going",
+    ] + opts.options
+    if opts.verbose:
+        print("$ " + " ".join(args))
+    with subprocess.Popen(args, text=True, stderr=stdout) as proc:
+        try:
+            yield proc
+        finally:
+            proc.kill()
+
+
+@dataclass
+class Build:
+    attr: str
+    drv_path: str
+    proc: subprocess.Popen[str]
+    retries: int
+    rc: int | None = None
+
+
+@dataclass
+class EvalError:
+    attr: str
+    error: str
+
+
+def wait_for_any_build(builds: list[Build]) -> Build:
+    while True:
+        for i, build in enumerate(builds):
+            rc = build.proc.poll()
+            if rc is not None:
+                del builds[i]
+                build.rc = rc
+                return build
+        time.sleep(0.05)
+
+
+def drain_builds(
+    builds: list[Build], stdout: IO[Any] | None, stack: ExitStack, opts: Options
+) -> list[Build]:
+    build = wait_for_any_build(builds)
+    if build.rc != 0:
+        print(f"build {build.attr} exited with {build.rc}", file=sys.stderr)
+        if build.retries < opts.retries:
+            print(f"retrying build {build.attr} [{build.retries + 1}/{opts.retries}]")
+            builds.append(
+                create_build(
+                    build.attr, build.drv_path, stdout, stack, opts, build.retries + 1
+                )
+            )
+        else:
+            return [build]
+    return []
+
+
+def create_build(
+    attr: str,
+    drv_path: str,
+    stdout: IO[Any] | None,
+    exit_stack: ExitStack,
+    opts: Options,
+    retries: int = 0,
+) -> Build:
+    nix_build_proc = exit_stack.enter_context(nix_build(drv_path + "^*", stdout, opts))
+    return Build(attr, drv_path, nix_build_proc, retries=retries)
+
+
+class Pipe:
+    def __init__(self) -> None:
+        fds = os.pipe()
+        self.read_file = os.fdopen(fds[0], "rb")
+        self.write_file = os.fdopen(fds[1], "wb")
+
+    def __enter__(self) -> "Pipe":
+        return self
+
+    def __exit__(self, _exc_type: Any, _exc_value: Any, _traceback: Any) -> None:
+        self.read_file.close()
+        self.write_file.close()
+
+ 
+def stop_gracefully(proc: subprocess.Popen, timeout: int = 1) -> None:
+    proc.terminate()
+    try:
+        proc.wait(timeout=timeout)
+    except subprocess.TimeoutExpired:
+        proc.kill()
+
+
+@contextmanager
+def nix_output_monitor(fd: int) -> Iterator[subprocess.Popen]:
+    proc = subprocess.Popen(["nom"], stdin=fd)
+    try:
+        yield proc
+    finally:
+        stop_gracefully(proc)
+
+
+def run_builds(stack: ExitStack, opts: Options) -> int:
+    eval_error = []
+    build_failures = []
+    drv_paths = set()
+    proc = stack.enter_context(nix_eval_jobs(opts))
+    assert proc.stdout
+    pipe = stack.enter_context(Pipe())
+    nom_proc: subprocess.Popen | None = None
+    stdout = pipe.write_file
+    builds = []
+    for line in proc.stdout:
+        if nom_proc is None:
+            nom_proc = stack.enter_context(nix_output_monitor(pipe.read_file.fileno()))
+        if opts.verbose:
+            print(line, end="")
+        try:
+            job = json.loads(line)
+        except json.JSONDecodeError:
+            die(f"Failed to parse line of nix-eval-jobs output: {line}")
+        error = job.get("error")
+        attr = job.get("attr", "unknown-flake-attribute")
+        if error:
+            eval_error.append(EvalError(attr, error))
+            continue
+        system = job.get("system")
+        if system and system not in opts.systems:
+            continue
+        drv_path = job.get("drvPath")
+        if not drv_path:
+            die(f"nix-eval-jobs did not return a drvPath: {line}")
+        while len(builds) >= opts.max_jobs and opts.max_jobs != 0:
+            build_failures += drain_builds(builds, stdout, stack, opts)
+        print(f"  building {attr}")
+        if drv_path in drv_paths:
+            continue
+        drv_paths.add(drv_path)
+        builds.append(create_build(attr, drv_path, stdout, stack, opts))
+
+    while builds:
+        build_failures += drain_builds(builds, stdout, stack, opts)
+
+    if nom_proc is not None:
+        stop_gracefully(nom_proc)
+
+    eval_rc = proc.wait()
+    if eval_rc != 0:
+        print(
+            f"nix-eval-jobs exited with {eval_rc}, check logs for details",
+            file=sys.stderr,
+        )
+
+    for error in eval_error:
+        print(f"{error.attr}: {error.error}", file=sys.stderr)
+
+    for build in build_failures:
+        print(f"{build.attr}: build failed with {build.rc}", file=sys.stderr)
+
+    if len(build_failures) > 0 or len(eval_error) > 0 or eval_rc != 0:
+        return 1
+    else:
+        return 0
+
+
+def main() -> None:
+    opts = parse_args(sys.argv[1:])
+    rc = 0
+    with ExitStack() as stack:
+        rc = run_builds(stack, opts)
+    sys.exit(rc)
+
+
+if __name__ == "__main__":
+    main()

pyproject.toml

@@ -0,0 +1,73 @@
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "nix-ci-build"
+description = "Evaluate and build in parallel"
+version = "2.10.1"
+authors = [{ name = "Jörg Thalheim", email = "joerg@thalheim.io" }]
+license = { text = "MIT" }
+classifiers = [
+  "Development Status :: 5 - Production/Stable",
+  "Environment :: Console",
+  "Topic :: Utilities",
+  "Intended Audience :: Developers",
+  "Programming Language :: Python :: 3.6",
+]
+
+[project.urls]
+Homepage = "https://github.com/Mic92/nix-ci-build"
+
+[project.scripts]
+nix-ci-build = "nix_ci_build:main"
+
+[tool.setuptools.packages]
+find = {}
+
+[tool.setuptools.package-data]
+nixpkgs_review = ["nix/*.nix"]
+
+
+[tool.ruff]
+line-length = 88
+
+select = ["E", "F", "I", "U", "N"]
+ignore = ["E501"]
+
+[tool.black]
+line-length = 88
+target-version = ['py39']
+include = '\.pyi?$'
+exclude = '''
+/(
+    \.git
+  | \.hg
+  | \.mypy_cache
+  | \.tox
+  | \.venv
+  | _build
+  | buck-out
+  | build
+  | dist
+  # The following are specific to Black, you probably don't want those.
+  | blib2to3
+  | tests/data
+  | profiling
+)/
+'''
+
+[tool.mypy]
+python_version = "3.10"
+warn_redundant_casts = true
+disallow_untyped_calls = true
+disallow_untyped_defs = true
+no_implicit_optional = true
+
+[[tool.mypy.overrides]]
+module = "setuptools.*"
+ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+module = "pytest.*"
+ignore_missing_imports = true