@@ -282,8 +282,15 @@ def golden_program(lib_path, cases):
282282 ' '.join(ue.cord_literal(p) for p in parts))
283283 internal_expr = '(rap 3 `(list @t)`~[%s])' % (
284284 ' '.join(ue.cord_literal(p) for p in internal_parts))
285− harness = harness.replace('|^ %+ rap 3',
286− '=/ prepared-prelude (prepare-prelude-with-internal:lib %s %s)\n|^ %%+ rap 3' % (prelude_expr, internal_expr))
285+ # Under mule: when preparing the prelude crashes, the cases that
286+ # need it prepare it again under their own mule and fail with the
287+ # trace, and the other cases still run.
288+ harness = harness.replace(
289+ '|^ %+ rap 3',
290+ '=/ prepared-prelude\n'
291+ ' =/ prep (mule |.((prepare-prelude-with-internal:lib %s %s)))\n'
292+ ' ?:(?=(%%& -.prep) p.prep ~)\n'
293+ '|^ %%+ rap 3' % (prelude_expr, internal_expr))
287294 prog.add(harness.replace('{count}', str(len(cases)))
288295 .replace('{runs}', runs)
289296 .replace('{types-expr}', types_expr if any(c.want_types for c in cases) else "''")
@@ -360,9 +367,28 @@ def eval_crash(prog, res, case):
360367 case.outputs['types'] = out
361368
362369
363−def run_batch(lib_path, cases, opts, top=True):
364− """Run cases in one eval. If the eval itself dies, which mule can't
365− stop (out of memory, a timeout), run each case alone to find which."""
370+def run_cases(lib_path, cases, opts):
371+ """Run the cases in one eval. If the eval itself dies, which mule
372+ can't stop (out of memory, a timeout), split them in two and run each
373+ half, at most opts.jobs evals at once, until the cases that kill it
374+ run alone."""
375+ pool = concurrent.futures.ThreadPoolExecutor(opts.jobs)
376+ try:
377+ pending = {pool.submit(run_batch, lib_path, cases, opts, True)}
378+ while pending:
379+ done, pending = concurrent.futures.wait(
380+ pending, return_when=concurrent.futures.FIRST_COMPLETED)
381+ for f in done:
382+ for part in f.result():
383+ pending.add(pool.submit(run_batch, lib_path, part, opts,
384+ False))
385+ finally:
386+ # On an error or ^C, drop the evals that haven't started.
387+ pool.shutdown(wait=False, cancel_futures=True)
388+
389+
390+def run_batch(lib_path, cases, opts, top):
391+ """Run cases in one eval; the halves to run again if the eval died."""
366392 prog = golden_program(lib_path, cases)
367393 if opts.save_program and top:
368394 with open(opts.save_program, 'wb') as f:
@@ -370,7 +396,7 @@ def run_batch(lib_path, cases, opts, top=True):
370396 res = ue.run_eval(prog, jam=True, timeout=opts.timeout)
371397 if res.ok:
372398 decode(prog, res.noun, cases)
373− return
399+ return []
374400 if res.is_compile_error():
375401 lib = ue.display_path(lib_path)
376402 signature = ("error: the golden harness doesn't compile "
@@ -387,13 +413,11 @@ def run_batch(lib_path, cases, opts, top=True):
387413 raise ue.ToolError(report + ('\n\n' + hint if hint else ''))
388414 if len(cases) == 1:
389415 eval_crash(prog, res, cases[0])
390− return
391− warn('%s while running %d cases at once; running each alone to find '
416+ return []
417+ half = len(cases) // 2
418+ warn('%s while running %d cases at once; running each half to find '
392419 'which' % (res.describe(), len(cases)))
393− with concurrent.futures.ThreadPoolExecutor(opts.jobs) as pool:
394− for f in [pool.submit(run_batch, lib_path, [c], opts, False)
395− for c in cases]:
396− f.result()
420+ return [cases[:half], cases[half:]]
397421
398422
399423# Output
@@ -522,8 +546,8 @@ def main(argv):
522546 ap.add_argument('--timeout', type=float, default=600, metavar='SECONDS',
523547 help='how long one urbit eval may run (default: 600)')
524548 ap.add_argument('--jobs', type=int, default=2, metavar='N',
525− help='evals at once when cases are rerun one by one '
526− 'after the eval dies (default: 2)')
549+ help='evals at once when the cases are split up to find '
550+ 'which one kills the eval (default: 2)')
527551 ap.add_argument('--save-program', metavar='FILE',
528552 help='also write the generated Hoon program to FILE')
529553 opts = ap.parse_args(argv)
@@ -548,7 +572,7 @@ def main(argv):
548572 len(cases), '' if len(cases) == 1 else 's'),
549573 file=sys.stderr, flush=True)
550574 start = time.time()
551− run_batch(os.path.abspath(opts.lib), cases, opts)
575+ run_cases(os.path.abspath(opts.lib), cases, opts)
552576 took = time.time() - start
553577 except ue.ToolError as e:
554578 print(str(e), file=sys.stderr)