@@ -0,0 +1,385 @@
1+#!/usr/bin/env python3
2+"""Run the repo's checks: all of them, a group, or some by name.
3+
4+ tools/check every check: the eval group, then ship
5+ tools/check --group eval the checks that need only urbit and Python
6+ tools/check --group ship the checks that need the fake ~zod
7+ tools/check golden 'type-*' only the checks these names or globs match
8+ tools/check --list the checks, their groups and commands
9+ tools/check -j 3 at most 3 eval checks at once
10+
11+CHECKS below is the one list of the repo's checks: docs/development.md
12+sends you here before pushing, and `nix flake check` runs this tool too
13+(see flake.nix), so a new check goes in the list and nowhere else.
14+
15+The eval checks run under `urbit eval` without a ship, several at once
16+(-j, default: as many as there are CPUs), in the order of the list, so
17+the slowest come first. The ship checks then run one at a time, in
18+order, on the ship tools/ship talks to: $JIBB_PIER, or /root/urbit/zod
19+(tools/ship also reads $JIBB_CLICK and $JIBB_SHIP_LOG). When that ship
20+isn't running, tools/check starts it first and stops it again at the end.
21+
22+Each check is one command, run from the repo root. It passes when it
23+exits 0 and, for the few that print their verdict rather than exit with
24+it, when its output matches what the list expects. Output is kept and
25+printed only for a check that fails (all of it with -v); --logs DIR
26+keeps every check's output in DIR/<name>.log.
27+
28+Exit status: 0 when every check ran and passed, 1 when a check failed,
29+2 when the checks couldn't run (an unknown name, a ship that won't start).
30+"""
31+
32+import argparse
33+import concurrent.futures
34+import fnmatch
35+import os
36+import re
37+import shutil
38+import signal
39+import subprocess
40+import sys
41+import tempfile
42+import threading
43+import time
44+
45+ROOT = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
46+SHIP = os.path.join('tools', 'ship')
47+
48+
49+class Check:
50+ def __init__(self, name, group, command, expect=None):
51+ self.name = name
52+ self.group = group # 'eval' or 'ship'
53+ self.command = command # argv, run from ROOT
54+ self.expect = expect # a regex stdout must match in full
55+
56+ def shown(self):
57+ return ' '.join(c if re.match(r'^[\w./=-]+$', c) else "'%s'" % c
58+ for c in self.command)
59+
60+
61+def tool(name, group='eval', *args):
62+ """A check that runs tools/<name> and passes when it exits 0."""
63+ base = os.path.basename(name)
64+ return Check(base, group, [os.path.join('tools', name)] + list(args))
65+
66+
67+def hoon(name, *args, expect):
68+ """A check that runs tools/hoon and compares what it prints."""
69+ return Check(name, 'eval', ['tools/hoon'] + list(args), expect=expect)
70+
71+
72+# Every check, in the order they start: the eval group, slowest first,
73+# then the ship group, in the order they must run. Times are from one
74+# run of every check, three at a time on four CPUs (2026-11-02).
75+CHECKS = [
76+ tool('type-todo-full-test'), # about 580 s
77+ tool('type-deferred-prepared-test'), # about 410 s
78+ tool('golden'),
79+ tool('importer-test'),
80+ tool('type-import-default-test'),
81+ tool('type-import-test'),
82+ tool('type-import-overlap-test'),
83+ tool('type-import-structural-test'),
84+ tool('type-union-source-test'),
85+ tool('type-union-alias-overlap-test'),
86+ tool('type-full-overlap-test'),
87+ tool('type-tlon-test'),
88+ tool('type-import-diagnostic-test'),
89+ tool('type-mark-test'),
90+ tool('type-poke-test'),
91+ tool('type-api-paths-test'),
92+ tool('type-api-evidence-test'),
93+ tool('type-hole-guards-test'),
94+ tool('type-list-alias-test'),
95+ tool('type-generic-alias-test'),
96+ tool('type-positional-face-guards-test'),
97+ tool('type-qath-guards-test'),
98+ tool('type-todo-partial-test'),
99+ tool('type-todo-api-test'),
100+ tool('type-direct-gate-test'),
101+ tool('type-page-main-test'),
102+ tool('type-buttons-test'),
103+ tool('type-doc-examples-test'),
104+ tool('type-theme-test'),
105+ tool('type-program-main-test'),
106+ tool('type-program-init-test'),
107+ tool('type-program-update-test'),
108+ tool('type-program-watch-test'),
109+ tool('type-store-test'),
110+ tool('type-generic-recursion-test'),
111+ tool('type-generic-construction-test'),
112+ tool('type-generic-pattern-test'),
113+ tool('type-deferred-test'),
114+ tool('type-deferred-resolver-test'),
115+ tool('type-deferred-context-test'),
116+ tool('type-cell-branch-test'),
117+ tool('type-closed-gate-guards-test'),
118+ tool('type-constructor-fit-guards-test'),
119+ tool('type-nested-union-guards-test'),
120+ tool('type-empty-test'),
121+ tool('type-gate-free-guards-test'),
122+ tool('type-gate-free-source-test'),
123+ tool('type-sub-guards-test'),
124+ tool('type-ask-guards-test'),
125+ tool('type-request-guards-test'),
126+ tool('type-proj-guards-test'),
127+ tool('type-arithmetic-guards-test'),
128+ tool('type-module-scope-test'),
129+ # These print their verdict and exit 0 either way.
130+ hoon('check-spans-test', 'check-spans-test', expect=r'<<>>\n'),
131+ hoon('itype-unify-test', 'itype-unify-test', expect=r'<<>>\n'),
132+ hoon('zing-hoon-reference', '-f',
133+ 'tools/testdata/zing-hoon-reference.hoon',
134+ expect=r"'Hoon flattening reference: \d+ cases passed'\n"),
135+ hoon('lead-late-hoon-reference', '-f',
136+ 'tools/testdata/lead-late-hoon-reference.hoon',
137+ expect=r"'Hoon lead/late reference: \d+ comparisons passed'\n"),
138+ hoon('succ-hoon-reference', '-f',
139+ 'tools/testdata/succ-hoon-reference.hoon',
140+ expect=r"'Hoon successor reference: \d+ comparisons passed'\n"),
141+ tool('testdata/selftest'),
142+ Check('doc-blocks', 'eval', ['tools/doc-blocks', 'docs/spec']),
143+ Check('gen-tests', 'eval', [SHIP, 'gen-tests', '--check']),
144+ tool('testdata/clay-fixtures-selftest'),
145+ # The ship group. ship-selftest leaves its fixture files on the
146+ # jibb-dev desk, and ship-test, which syncs the desk to the repo
147+ # first, puts the real ones back, so it must come after.
148+ tool('testdata/ship-selftest', 'ship'),
149+ Check('ship-test', 'ship', [SHIP, 'test']),
150+ Check('check-commit', 'ship', [SHIP, 'check-commit']),
151+ tool('clay-fixtures', 'ship'),
152+]
153+
154+GROUPS = ('eval', 'ship')
155+
156+
157+# ---------------------------------------------------------------- running
158+
159+
160+class Result:
161+ def __init__(self, check):
162+ self.check = check
163+ self.ok = False
164+ self.status = None
165+ self.why = ''
166+ self.output = ''
167+ self.seconds = 0.0
168+
169+
170+_lock = threading.Lock()
171+_running = set() # process groups to kill if we're interrupted
172+
173+
174+def say(msg=''):
175+ with _lock:
176+ print(msg, flush=True)
177+
178+
179+def run(check, timeout, env):
180+ """Run one check; a Result."""
181+ res = Result(check)
182+ start = time.time()
183+ with tempfile.TemporaryFile() as out, tempfile.TemporaryFile() as err:
184+ try:
185+ proc = subprocess.Popen(
186+ check.command, cwd=ROOT, env=env, stdin=subprocess.DEVNULL,
187+ stdout=out,
188+ stderr=err if check.expect is not None else subprocess.STDOUT,
189+ start_new_session=True)
190+ except OSError as e:
191+ res.why = 'could not run %s: %s' % (check.command[0], e)
192+ return res
193+ with _lock:
194+ _running.add(proc.pid)
195+ try:
196+ res.status = proc.wait(timeout=timeout)
197+ except subprocess.TimeoutExpired:
198+ kill(proc.pid)
199+ proc.wait()
200+ res.why = 'timed out after %d s' % timeout
201+ finally:
202+ with _lock:
203+ _running.discard(proc.pid)
204+ kill(proc.pid) # anything it left running, such as urbit eval
205+ out.seek(0)
206+ err.seek(0)
207+ stdout = out.read().decode('utf-8', 'replace')
208+ stderr = err.read().decode('utf-8', 'replace')
209+ res.seconds = time.time() - start
210+ res.output = stdout + stderr
211+ if res.why:
212+ return res
213+ if res.status != 0:
214+ res.why = 'exit status %d' % res.status
215+ elif check.expect is not None and \
216+ not re.fullmatch(check.expect, stdout):
217+ res.why = "it didn't print the verdict that means a pass"
218+ else:
219+ res.ok = True
220+ return res
221+
222+
223+def kill(pgid):
224+ try:
225+ os.killpg(pgid, signal.SIGKILL)
226+ except (ProcessLookupError, PermissionError):
227+ pass
228+
229+
230+def kill_all(*_):
231+ with _lock:
232+ pids = list(_running)
233+ for pid in pids:
234+ kill(pid)
235+
236+
237+def report(res, verbose, logs, limit=300):
238+ mark = 'PASS' if res.ok else 'FAIL'
239+ line = '%s %-34s %5d s' % (mark, res.check.name, round(res.seconds))
240+ if not res.ok:
241+ line += ' (%s)' % res.why
242+ lines = res.output.rstrip('\n').split('\n') if res.output.strip() else []
243+ if logs:
244+ with open(os.path.join(logs, res.check.name + '.log'), 'w') as f:
245+ f.write('$ %s\n%s' % (res.check.shown(), res.output))
246+ if res.ok and not verbose:
247+ say(line)
248+ return
249+ shown = lines
250+ if not verbose and len(lines) > limit:
251+ shown = ['(%d lines cut; --logs DIR keeps them all)'
252+ % (len(lines) - limit)] + lines[-limit:]
253+ body = [' $ ' + res.check.shown()] + [' ' + l for l in shown]
254+ say('\n'.join([line] + body))
255+
256+
257+# ---------------------------------------------------------------- the ship
258+
259+
260+def ship(env, *args, **kw):
261+ return subprocess.run([SHIP] + list(args), cwd=ROOT, env=env, **kw)
262+
263+
264+def ship_running(env):
265+ """Whether tools/ship's ship runs: status exits 3 when it doesn't."""
266+ res = ship(env, 'status', stdout=subprocess.DEVNULL,
267+ stderr=subprocess.DEVNULL)
268+ return res.returncode != 3
269+
270+
271+# ---------------------------------------------------------------- main
272+
273+
274+def select(names, group):
275+ """The checks to run, in list order; exits 2 on a name nothing matches."""
276+ chosen = [c for c in CHECKS if group in (None, c.group)]
277+ if not names:
278+ return chosen
279+ picked = set()
280+ for name in names:
281+ hits = [c.name for c in chosen if fnmatch.fnmatchcase(c.name, name)]
282+ if not hits:
283+ where = ' in the %s group' % group if group else ''
284+ print('check: no check %s%s; tools/check --list lists them'
285+ % (name, where), file=sys.stderr)
286+ sys.exit(2)
287+ picked.update(hits)
288+ return [c for c in chosen if c.name in picked]
289+
290+
291+def main(argv):
292+ ap = argparse.ArgumentParser(
293+ prog='tools/check', description=__doc__.split('\n\n')[0],
294+ formatter_class=argparse.RawDescriptionHelpFormatter,
295+ epilog='\n\n'.join(__doc__.split('\n\n')[1:]))
296+ ap.add_argument('names', nargs='*', metavar='NAME',
297+ help='checks to run, by name or glob (default: all)')
298+ ap.add_argument('--group', choices=GROUPS,
299+ help='only the checks in this group')
300+ ap.add_argument('--list', action='store_true',
301+ help='list the checks and exit')
302+ ap.add_argument('-j', '--jobs', type=int, default=os.cpu_count() or 1,
303+ metavar='N', help='eval checks to run at once '
304+ '(default: the number of CPUs, %(default)s)')
305+ ap.add_argument('--timeout', type=int, default=2400, metavar='SECONDS',
306+ help='seconds each check may take (default %(default)s)')
307+ ap.add_argument('--logs', metavar='DIR',
308+ help="keep every check's output in DIR/<name>.log")
309+ ap.add_argument('-v', '--verbose', action='store_true',
310+ help="print every check's output, not only a failure's")
311+ args = ap.parse_args(argv)
312+
313+ checks = select(args.names, args.group)
314+ if args.list:
315+ for c in checks:
316+ print('%-34s %-5s %s' % (c.name, c.group, c.shown()))
317+ return 0
318+ if args.jobs < 1:
319+ ap.error('-j must be at least 1')
320+ if args.logs:
321+ os.makedirs(args.logs, exist_ok=True)
322+
323+ env = dict(os.environ, PYTHONDONTWRITEBYTECODE='1')
324+ signal.signal(signal.SIGTERM, lambda *_: (kill_all(), sys.exit(143)))
325+ started = time.time()
326+ results = []
327+ unable = False
328+ try:
329+ evals = [c for c in checks if c.group == 'eval']
330+ if evals:
331+ say('running %d eval check%s, %d at a time'
332+ % (len(evals), '' if len(evals) == 1 else 's', args.jobs))
333+ with concurrent.futures.ThreadPoolExecutor(args.jobs) as pool:
334+ futures = [pool.submit(run, c, args.timeout, env)
335+ for c in evals]
336+ for f in concurrent.futures.as_completed(futures):
337+ res = f.result()
338+ results.append(res)
339+ report(res, args.verbose, args.logs)
340+ ships = [c for c in checks if c.group == 'ship']
341+ if ships:
342+ started_ship = False
343+ if not ship_running(env):
344+ say('starting the ship for %d ship check%s'
345+ % (len(ships), '' if len(ships) == 1 else 's'))
346+ start = ship(env, 'start')
347+ if start.returncode != 0:
348+ say('check: the ship would not start, so the ship '
349+ 'checks did not run')
350+ unable = True
351+ ships = []
352+ started_ship = not unable
353+ else:
354+ say('running %d ship check%s on the running ship'
355+ % (len(ships), '' if len(ships) == 1 else 's'))
356+ try:
357+ for c in ships:
358+ res = run(c, args.timeout, env)
359+ results.append(res)
360+ report(res, args.verbose, args.logs)
361+ finally:
362+ if started_ship:
363+ say('stopping the ship tools/check started')
364+ ship(env, 'stop', '--force')
365+ except KeyboardInterrupt:
366+ kill_all()
367+ print('check: interrupted', file=sys.stderr)
368+ return 130
369+
370+ failed = [r for r in results if not r.ok]
371+ order = {c.name: i for i, c in enumerate(CHECKS)}
372+ say('')
373+ say('%d passed, %d failed%s in %d s' % (
374+ len(results) - len(failed), len(failed),
375+ ', ship checks not run' if unable else '',
376+ round(time.time() - started)))
377+ for r in sorted(failed, key=lambda r: order[r.check.name]):
378+ say(' failed: %s (%s)' % (r.check.name, r.why))
379+ if failed:
380+ return 1
381+ return 2 if unable else 0
382+
383+
384+if __name__ == '__main__':
385+ sys.exit(main(sys.argv[1:]))