@@ -282,8 +282,15 @@ def golden_program(lib_path, cases):
282 ' '.join(ue.cord_literal(p) for p in parts))
282 ' '.join(ue.cord_literal(p) for p in parts))
283 internal_expr = '(rap 3 `(list @t)`~[%s])' % (
283 internal_expr = '(rap 3 `(list @t)`~[%s])' % (
284 ' '.join(ue.cord_literal(p) for p in internal_parts))
284 ' '.join(ue.cord_literal(p) for p in internal_parts))
285+ # Under mule: when preparing the prelude crashes, the cases that
286+ # need it prepare it again under their own mule and fail with the
287+ # trace, and the other cases still run.
285− harness = harness.replace('|^ %+ rap 3',
288+ harness = harness.replace(
290+ '=/ prepared-prelude\n'
291+ ' =/ prep (mule |.((prepare-prelude-with-internal:lib %s %s)))\n'
292+ ' ?:(?=(%%& -.prep) p.prep ~)\n'
286− '=/ prepared-prelude (prepare-prelude-with-internal:lib %s %s)\n|^ %%+ rap 3' % (prelude_expr, internal_expr))
293+ '|^ %%+ rap 3' % (prelude_expr, internal_expr))
287 prog.add(harness.replace('{count}', str(len(cases)))
294 prog.add(harness.replace('{count}', str(len(cases)))
288 .replace('{runs}', runs)
295 .replace('{runs}', runs)
289 .replace('{types-expr}', types_expr if any(c.want_types for c in cases) else "''")
296 .replace('{types-expr}', types_expr if any(c.want_types for c in cases) else "''")
@@ -360,9 +367,28 @@ def eval_crash(prog, res, case):
360 case.outputs['types'] = out
367 case.outputs['types'] = out
363−def run_batch(lib_path, cases, opts, top=True):
370+def run_cases(lib_path, cases, opts):
364− """Run cases in one eval. If the eval itself dies, which mule can't
371+ """Run the cases in one eval. If the eval itself dies, which mule
365− stop (out of memory, a timeout), run each case alone to find which."""
372+ can't stop (out of memory, a timeout), split them in two and run each
373+ half, at most opts.jobs evals at once, until the cases that kill it
375+ pool = concurrent.futures.ThreadPoolExecutor(opts.jobs)
377+ pending = {pool.submit(run_batch, lib_path, cases, opts, True)}
379+ done, pending = concurrent.futures.wait(
380+ pending, return_when=concurrent.futures.FIRST_COMPLETED)
382+ for part in f.result():
383+ pending.add(pool.submit(run_batch, lib_path, part, opts,
386+ # On an error or ^C, drop the evals that haven't started.
387+ pool.shutdown(wait=False, cancel_futures=True)
390+def run_batch(lib_path, cases, opts, top):
391+ """Run cases in one eval; the halves to run again if the eval died."""
366 prog = golden_program(lib_path, cases)
392 prog = golden_program(lib_path, cases)
367 if opts.save_program and top:
393 if opts.save_program and top:
368 with open(opts.save_program, 'wb') as f:
394 with open(opts.save_program, 'wb') as f:
@@ -370,7 +396,7 @@ def run_batch(lib_path, cases, opts, top=True):
370 res = ue.run_eval(prog, jam=True, timeout=opts.timeout)
396 res = ue.run_eval(prog, jam=True, timeout=opts.timeout)
371 if res.ok:
397 if res.ok:
372 decode(prog, res.noun, cases)
398 decode(prog, res.noun, cases)
373− return
399+ return []
374 if res.is_compile_error():
400 if res.is_compile_error():
375 lib = ue.display_path(lib_path)
401 lib = ue.display_path(lib_path)
376 signature = ("error: the golden harness doesn't compile "
402 signature = ("error: the golden harness doesn't compile "
@@ -387,13 +413,11 @@ def run_batch(lib_path, cases, opts, top=True):
387 raise ue.ToolError(report + ('\n\n' + hint if hint else ''))
413 raise ue.ToolError(report + ('\n\n' + hint if hint else ''))
388 if len(cases) == 1:
414 if len(cases) == 1:
389 eval_crash(prog, res, cases[0])
415 eval_crash(prog, res, cases[0])
390− return
416+ return []
417+ half = len(cases) // 2
391− warn('%s while running %d cases at once; running each alone to find '
418+ warn('%s while running %d cases at once; running each half to find '
392 'which' % (res.describe(), len(cases)))
419 'which' % (res.describe(), len(cases)))
393− with concurrent.futures.ThreadPoolExecutor(opts.jobs) as pool:
420+ return [cases[:half], cases[half:]]
394− for f in [pool.submit(run_batch, lib_path, [c], opts, False)
@@ -522,8 +546,8 @@ def main(argv):
522 ap.add_argument('--timeout', type=float, default=600, metavar='SECONDS',
546 ap.add_argument('--timeout', type=float, default=600, metavar='SECONDS',
523 help='how long one urbit eval may run (default: 600)')
547 help='how long one urbit eval may run (default: 600)')
524 ap.add_argument('--jobs', type=int, default=2, metavar='N',
548 ap.add_argument('--jobs', type=int, default=2, metavar='N',
525− help='evals at once when cases are rerun one by one '
549+ help='evals at once when the cases are split up to find '
526− 'after the eval dies (default: 2)')
550+ 'which one kills the eval (default: 2)')
527 ap.add_argument('--save-program', metavar='FILE',
551 ap.add_argument('--save-program', metavar='FILE',
528 help='also write the generated Hoon program to FILE')
552 help='also write the generated Hoon program to FILE')
529 opts = ap.parse_args(argv)
553 opts = ap.parse_args(argv)
@@ -548,7 +572,7 @@ def main(argv):
548 len(cases), '' if len(cases) == 1 else 's'),
572 len(cases), '' if len(cases) == 1 else 's'),
549 file=sys.stderr, flush=True)
573 file=sys.stderr, flush=True)
550 start = time.time()
574 start = time.time()
551− run_batch(os.path.abspath(opts.lib), cases, opts)
575+ run_cases(os.path.abspath(opts.lib), cases, opts)
552 took = time.time() - start
576 took = time.time() - start
553 except ue.ToolError as e:
577 except ue.ToolError as e:
554 print(str(e), file=sys.stderr)
578 print(str(e), file=sys.stderr)