git.iko.soy/jibb
on c4e28d2 help sign in
jibb/tools/check
316 lines 12 kB history raw
1#!/usr/bin/env python3
2"""Run the repo's checks: all of them, a group, or some by name.
4 tools/check every check: the eval group, then ship
5 tools/check --group eval the checks that need only urbit and Python
6 tools/check --group ship the checks that need the fake ~zod
7 tools/check golden 'type-*' only the checks these names or globs match
8 tools/check --list the checks, their groups and commands
9 tools/check -j 3 at most 3 eval checks at once
11CHECKS below is the one list of the repo's checks: docs/development.md
12sends you here before pushing, and `nix flake check` runs this tool too
13(see flake.nix), so a new check goes in the list and nowhere else.
15The eval checks run under `urbit eval` without a ship, several at once
16(-j, default: as many as there are CPUs), in the order of the list, so
17the slowest come first. The ship checks then run one at a time, in
18order, on the ship tools/ship talks to: $JIBB_PIER, or /root/urbit/zod
19(tools/ship also reads $JIBB_CLICK and $JIBB_SHIP_LOG). When that ship
20isn't running, tools/check starts it first and stops it again at the end.
22Each check is one command, run from the repo root, and passes when it
23exits 0. Output is kept and printed only for a check that fails (all of
24it with -v); --logs DIR keeps every check's output in DIR/<name>.log.
26Exit status: 0 when every check ran and passed, 1 when a check failed,
272 when the checks couldn't run (an unknown name, a ship that won't start).
28"""
30import argparse
31import concurrent.futures
32import fnmatch
33import os
34import re
35import shutil
36import signal
37import subprocess
38import sys
39import tempfile
40import threading
41import time
43ROOT = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
44SHIP = os.path.join('tools', 'ship')
47class Check:
48 def __init__(self, name, group, command):
49 self.name = name
50 self.group = group # 'eval' or 'ship'
51 self.command = command # argv, run from ROOT
53 def shown(self):
54 return ' '.join(c if re.match(r'^[\w./=-]+$', c) else "'%s'" % c
55 for c in self.command)
58def tool(name, group='eval', *args):
59 """A check that runs tools/<name> and passes when it exits 0."""
60 base = os.path.basename(name)
61 return Check(base, group, [os.path.join('tools', name)] + list(args))
64# Every check, in the order they start: the eval group, slowest first,
65# then the ship group, in the order they must run. Times are from one
66# run on four CPUs (2026-11-03).
68# arms-1 to arms-4 run the test- arms of every tests/lib suite under
69# eval, in shards of about equal time (tools/test-arms --list --shard
70# 2/4 shows the second). A new suite in tests/lib joins a shard by
71# itself; a slow one belongs in COSTS in tools/test-arms.
72ARM_SHARDS = 4
74CHECKS = [
75 tool('testdata/selftest'), # about 115 s
76 tool('golden'), # about 60 to 100 s
77] + [
78 Check('arms-%d' % i, 'eval', # 55 to 80 s each
79 ['tools/test-arms', '--shard', '%d/%d' % (i, ARM_SHARDS)])
80 for i in range(1, ARM_SHARDS + 1)
81] + [
82 Check('doc-blocks', 'eval', ['tools/doc-blocks', 'docs/spec']),
83 # The ship group. ship-selftest leaves its fixture files on the
84 # jibb-dev desk, and ship-test, which syncs the desk to the repo
85 # first, puts the real ones back, so it must come after.
86 tool('testdata/ship-selftest', 'ship'),
87 Check('ship-test', 'ship', [SHIP, 'test']),
88 Check('check-commit', 'ship', [SHIP, 'check-commit']),
89 tool('clay-fixtures', 'ship'),
92GROUPS = ('eval', 'ship')
95# ---------------------------------------------------------------- running
98class Result:
99 def __init__(self, check):
100 self.check = check
101 self.ok = False
102 self.status = None
103 self.why = ''
104 self.output = ''
105 self.seconds = 0.0
108_lock = threading.Lock()
109_running = set() # process groups to kill if we're interrupted
112def say(msg=''):
113 with _lock:
114 print(msg, flush=True)
117def run(check, timeout, env):
118 """Run one check; a Result."""
119 res = Result(check)
120 start = time.time()
121 with tempfile.TemporaryFile() as out:
122 try:
123 proc = subprocess.Popen(
124 check.command, cwd=ROOT, env=env, stdin=subprocess.DEVNULL,
125 stdout=out, stderr=subprocess.STDOUT,
126 start_new_session=True)
127 except OSError as e:
128 res.why = 'could not run %s: %s' % (check.command[0], e)
129 return res
130 with _lock:
131 _running.add(proc.pid)
132 try:
133 res.status = proc.wait(timeout=timeout)
134 except subprocess.TimeoutExpired:
135 kill(proc.pid)
136 proc.wait()
137 res.why = 'timed out after %d s' % timeout
138 finally:
139 with _lock:
140 _running.discard(proc.pid)
141 kill(proc.pid) # anything it left running, such as urbit eval
142 out.seek(0)
143 res.output = out.read().decode('utf-8', 'replace')
144 res.seconds = time.time() - start
145 if res.why:
146 return res
147 if res.status != 0:
148 res.why = 'exit status %d' % res.status
149 else:
150 res.ok = True
151 return res
154def kill(pgid):
155 try:
156 os.killpg(pgid, signal.SIGKILL)
157 except (ProcessLookupError, PermissionError):
158 pass
161def kill_all(*_):
162 with _lock:
163 pids = list(_running)
164 for pid in pids:
165 kill(pid)
168def report(res, verbose, logs, limit=300):
169 mark = 'PASS' if res.ok else 'FAIL'
170 line = '%s %-34s %5d s' % (mark, res.check.name, round(res.seconds))
171 if not res.ok:
172 line += ' (%s)' % res.why
173 lines = res.output.rstrip('\n').split('\n') if res.output.strip() else []
174 if logs:
175 with open(os.path.join(logs, res.check.name + '.log'), 'w') as f:
176 f.write('$ %s\n%s' % (res.check.shown(), res.output))
177 if res.ok and not verbose:
178 say(line)
179 return
180 shown = lines
181 if not verbose and len(lines) > limit:
182 shown = ['(%d lines cut; --logs DIR keeps them all)'
183 % (len(lines) - limit)] + lines[-limit:]
184 body = [' $ ' + res.check.shown()] + [' ' + l for l in shown]
185 say('\n'.join([line] + body))
188# ---------------------------------------------------------------- the ship
191def ship(env, *args, **kw):
192 return subprocess.run([SHIP] + list(args), cwd=ROOT, env=env, **kw)
195def ship_running(env):
196 """Whether tools/ship's ship runs: status exits 3 when it doesn't."""
197 res = ship(env, 'status', stdout=subprocess.DEVNULL,
198 stderr=subprocess.DEVNULL)
199 return res.returncode != 3
202# ---------------------------------------------------------------- main
205def select(names, group):
206 """The checks to run, in list order; exits 2 on a name nothing matches."""
207 chosen = [c for c in CHECKS if group in (None, c.group)]
208 if not names:
209 return chosen
210 picked = set()
211 for name in names:
212 hits = [c.name for c in chosen if fnmatch.fnmatchcase(c.name, name)]
213 if not hits:
214 where = ' in the %s group' % group if group else ''
215 print('check: no check %s%s; tools/check --list lists them'
216 % (name, where), file=sys.stderr)
217 sys.exit(2)
218 picked.update(hits)
219 return [c for c in chosen if c.name in picked]
222def main(argv):
223 ap = argparse.ArgumentParser(
224 prog='tools/check', description=__doc__.split('\n\n')[0],
225 formatter_class=argparse.RawDescriptionHelpFormatter,
226 epilog='\n\n'.join(__doc__.split('\n\n')[1:]))
227 ap.add_argument('names', nargs='*', metavar='NAME',
228 help='checks to run, by name or glob (default: all)')
229 ap.add_argument('--group', choices=GROUPS,
230 help='only the checks in this group')
231 ap.add_argument('--list', action='store_true',
232 help='list the checks and exit')
233 ap.add_argument('-j', '--jobs', type=int, default=os.cpu_count() or 1,
234 metavar='N', help='eval checks to run at once '
235 '(default: the number of CPUs, %(default)s)')
236 ap.add_argument('--timeout', type=int, default=2400, metavar='SECONDS',
237 help='seconds each check may take (default %(default)s)')
238 ap.add_argument('--logs', metavar='DIR',
239 help="keep every check's output in DIR/<name>.log")
240 ap.add_argument('-v', '--verbose', action='store_true',
241 help="print every check's output, not only a failure's")
242 args = ap.parse_args(argv)
244 checks = select(args.names, args.group)
245 if args.list:
246 for c in checks:
247 print('%-34s %-5s %s' % (c.name, c.group, c.shown()))
248 return 0
249 if args.jobs < 1:
250 ap.error('-j must be at least 1')
251 if args.logs:
252 os.makedirs(args.logs, exist_ok=True)
254 env = dict(os.environ, PYTHONDONTWRITEBYTECODE='1')
255 signal.signal(signal.SIGTERM, lambda *_: (kill_all(), sys.exit(143)))
256 started = time.time()
257 results = []
258 unable = False
259 try:
260 evals = [c for c in checks if c.group == 'eval']
261 if evals:
262 say('running %d eval check%s, %d at a time'
263 % (len(evals), '' if len(evals) == 1 else 's', args.jobs))
264 with concurrent.futures.ThreadPoolExecutor(args.jobs) as pool:
265 futures = [pool.submit(run, c, args.timeout, env)
266 for c in evals]
267 for f in concurrent.futures.as_completed(futures):
268 res = f.result()
269 results.append(res)
270 report(res, args.verbose, args.logs)
271 ships = [c for c in checks if c.group == 'ship']
272 if ships:
273 started_ship = False
274 if not ship_running(env):
275 say('starting the ship for %d ship check%s'
276 % (len(ships), '' if len(ships) == 1 else 's'))
277 start = ship(env, 'start')
278 if start.returncode != 0:
279 say('check: the ship would not start, so the ship '
280 'checks did not run')
281 unable = True
282 ships = []
283 started_ship = not unable
284 else:
285 say('running %d ship check%s on the running ship'
286 % (len(ships), '' if len(ships) == 1 else 's'))
287 try:
288 for c in ships:
289 res = run(c, args.timeout, env)
290 results.append(res)
291 report(res, args.verbose, args.logs)
292 finally:
293 if started_ship:
294 say('stopping the ship tools/check started')
295 ship(env, 'stop', '--force')
296 except KeyboardInterrupt:
297 kill_all()
298 print('check: interrupted', file=sys.stderr)
299 return 130
301 failed = [r for r in results if not r.ok]
302 order = {c.name: i for i, c in enumerate(CHECKS)}
303 say('')
304 say('%d passed, %d failed%s in %d s' % (
305 len(results) - len(failed), len(failed),
306 ', ship checks not run' if unable else '',
307 round(time.time() - started)))
308 for r in sorted(failed, key=lambda r: order[r.check.name]):
309 say(' failed: %s (%s)' % (r.check.name, r.why))
310 if failed:
311 return 1
312 return 2 if unable else 0
315if __name__ == '__main__':
316 sys.exit(main(sys.argv[1:]))