2"""Run the Jibb desks on a fake ship: start it, sync desks, run the tests.
4 tools/ship start start the fake ship in the background
5 tools/ship stop stop it
6 tools/ship status running or not; desks, revisions, mounts
7 tools/ship sync my-app assemble a desk from its folders, |commit
8 tools/ship test sync the jibb-dev test desk, run -test
9 tools/ship gen-tests write tests/lib/jibb.hoon from tests/golden
10 tools/ship check-commit prove that |commit stores .jibb files
11 tools/ship scry /cx/=my-app=/ui/about/jibb
12 tools/ship hoon '(add 2 2)' evaluate Hoon on the ship, as dojo would
13 tools/ship run FILE run a thread file (a strand) on the ship
14 tools/ship poke hood kiln-commit '[%my-app |]'
15 tools/ship commit|mount|unmount DESK
16 tools/ship log the end of the ship's log
18The ship is a fake ~zod at /root/urbit/zod (--pier or $JIBB_PIER), run
19with -t in its own session, its output going to <pier>.log (--log or
20$JIBB_SHIP_LOG); a start moves the previous log to <pier>.log.1.
22Commands reach the ship over its conn.sock, as khan-eval threads: click
23(/root/src/tools/pkg/click/click, --click or $JIBB_CLICK) formats and
24jams each request, and this script sends it and reads the whole reply.
25It doesn't let click do the sending, because click reads the reply with
26`nc -U -W 1`, which stops after the first 16 KB, so bigger replies came
27back cut off and decoded to garbage. click's -i quoting escapes only
28single quotes and joins the lines with gaps, so threads are written
29without comments and their backslashes are doubled before click sees
32Exit status: 0 on success, 1 when something failed (a commit, a test, a
33check), 2 when the command couldn't run (a usage error, no ship).
49sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)))
52ROOT = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
53PIER = os.environ.get('JIBB_PIER', '/root/urbit/zod')
54CLICK = os.environ.get('JIBB_CLICK', '/root/src/tools/pkg/click/click')
55STUB = os.path.join(ROOT, 'tools', 'testdata', 'ship-stub.hoon')
56GOLDEN = os.path.join(ROOT, 'tests', 'golden')
57TESTS_LIB = os.path.join(ROOT, 'tests', 'lib', 'jibb.hoon')
59# Desks the repo builds, and the folders each is assembled from, in
60# order. Every desk gets every package under vendor/. A folder is
61# (repo folder, desk folder, folders left out).
63 'my-app': [('jibb-dev', '', ['tests']), ('my-app', '', [])],
64 'jibb': [('jibb', '', [])],
65 'jibb-dev': [('jibb-dev', '', []), ('tests', 'tests', [])],
67# The test desk has no sys.kelvin of its own; Clay refuses a commit
68# without one, so it gets the %jibb desk's.
69EXTRA = {'jibb-dev': {'sys.kelvin': os.path.join('jibb', 'sys.kelvin')}}
71# Marks whose stored noun is the file's bytes, as a cord. sync checks
72# these files by hash; files with other marks, such as desk.bill and
73# the golden .txt files, are checked by converting the repo's copy on
74# the ship and comparing the values. (%txt is not one: it stores a
75# wain, and vere's +to-wain jet keeps no trace of a final newline.)
76BYTE_MARKS = {'hoon', 'jibb', 'js'}
78# vere's conn.sock closes a connection as soon as it has written the
79# reply, so a reply bigger than the socket's buffer (about 200 KB) is
80# cut off. Threads that may print a lot cut their output to this.
83KNOT = re.compile(r'^[a-z0-9\-.~_]+$')
84TERM = re.compile(r'^[a-z][a-z0-9\-]*$')
86# Log lines that come from the ship's own business, not from what we
87# asked: docket and kiln failing to reach the network on a fake ship.
89 r'^(http: fail \(\d+, \d+\)|docket: (fetching|thread failed|'
90 r'retry)|%retry-too-many|kiln: activation failed|thread-fail$|'
91 r'cancelled$|eyre: replacing existing binding|mdns: )')
95 """Something failed: report it, exit 1."""
98class Unable(Exception):
99 """The command can't run: report it, exit 2."""
103 print(msg, flush=True)
110 """Print a warning once, however often the code finds it."""
111 if msg not in _warned:
113 print('ship: ' + msg, file=sys.stderr, flush=True)
116# ---------------------------------------------------------------- nouns
118# A noun is an int (an atom) or a 2-tuple (a cell).
122 """Decode a jammed noun, without recursion."""
123 total = len(data) * 8
127 raise ValueError('cue: ran off the end of the jam')
128 return (data[i >> 3] >> (i & 7)) & 1
134 raise ValueError('cue: ran off the end of the jam')
135 v = int.from_bytes(data[i >> 3:(i + k + 7) >> 3], 'little')
136 return (v >> (i & 7)) & ((1 << k) - 1)
140 while bit(i + c) == 0:
144 b = (1 << (c - 1)) | bits(i + c + 1, c - 1)
145 return c + c + b, bits(i + c + c, b)
153 length, val = rub(i + 1)
156 elif bit(i + 1) == 0:
157 stack.append([start, False, None])
161 length, ref = rub(i + 2)
178 """An atom's bytes, low byte first, as Hoon cords are."""
179 return a.to_bytes((a.bit_length() + 7) // 8, 'little')
183 return atom_bytes(a).decode('utf-8', 'replace')
187 return int.from_bytes(s.encode(), 'little')
191 """A Hoon list as a Python list."""
193 while isinstance(n, tuple):
200 return bytes(items(n)).decode('utf-8', 'replace')
204 return '/' + '/'.join(text(s) for s in items(n))
208 """A Hoon cord literal holding exactly these bytes."""
215 elif 0x20 <= b < 0x7f:
218 out.append('\\%02x' % b)
223def hoon_path(segments):
224 """A Hoon path literal. A segment that isn't a term, such as
225 odd.name_1, goes in as a knot, [~.odd.name_1]: written plain, the
226 parser would read the . as the end of a coin."""
227 return '/' + '/'.join(s if TERM.match(s) else '[~.%s]' % s
228 for s in segments) if segments else '/'
231# ---------------------------------------------------------------- tangs
235 """A tank on one line, as Hoon's +ram:re prints it."""
236 if isinstance(tank, int):
241 if tag in ('rose', 'palm'):
244 p, (q, (r, s)) = spec
245 sep, opn, cls = tape(p), tape(q) + tape(r), tape(s)
248 sep, opn, cls = tape(p), tape(q), tape(r)
249 return opn + sep.join(ram(k) for k in items(kids)) + cls
255 for tank in items(tang):
257 out.append(ram(tank))
258 except (TypeError, ValueError):
259 out.append(repr(tank))
263# ---------------------------------------------------------------- the ship
267 run = os.path.join(pier, '.run')
268 if os.access(run, os.X_OK):
270 found = shutil.which('urbit')
272 raise Unable('no urbit binary: neither %s nor urbit on PATH' % run)
277 def __init__(self, pier, log, click):
278 self.pier = os.path.realpath(pier)
279 self.log = log or self.pier + '.log'
281 self.sock = os.path.join(self.pier, '.urb', 'conn.sock')
286 """(king pid, serf pid) of the urbit running this pier, or None."""
288 for pid in os.listdir('/proc'):
289 if not pid.isdigit():
292 with open('/proc/%s/cmdline' % pid, 'rb') as f:
293 args = [a.decode('utf-8', 'replace')
294 for a in f.read().split(b'\0') if a]
297 if len(args) < 2 or self.pier not in [
298 os.path.realpath(a) if a.startswith('/') else a
301 if args[1] == 'work':
303 elif args[1] not in ('eval', 'dock', 'info', 'pack', 'meld'):
308 king, serf = self.processes()
312 if not self.running():
313 raise Unable('the ship at %s is not running; start it with '
314 'tools/ship start' % self.pier)
315 if not os.path.exists(self.sock):
316 raise Unable('the ship is running but has no %s yet; wait for '
317 'it to come up' % self.sock)
323 return os.path.getsize(self.log)
327 def log_since(self, offset):
328 """The log's lines from a byte offset on, without colours."""
330 with open(self.log, 'rb') as f:
335 data = re.sub(rb'\x1b\[[0-9;?]*[A-Za-z]', b'', data)
336 return data.decode('utf-8', 'replace').replace('\r', '').split('\n')
340 def start(self, timeout):
342 say('the ship is already running (pid %d)' % self.running())
343 self.wait_live(time.time() + timeout, None, 0)
345 if not os.path.isdir(os.path.join(self.pier, '.urb')):
346 raise Unable('no pier at %s' % self.pier)
347 binary = find_urbit(self.pier)
348 if os.path.exists(self.log) and os.path.getsize(self.log):
349 os.replace(self.log, self.log + '.1')
350 logf = open(self.log, 'ab')
351 logf.write(('tools/ship: starting %s -t %s at %s\n' % (
353 datetime.datetime.now().isoformat(timespec='seconds'))).encode())
355 proc = subprocess.Popen(
356 [binary, '-t', self.pier], stdin=subprocess.DEVNULL,
357 stdout=logf, stderr=subprocess.STDOUT,
358 cwd=os.path.dirname(self.pier), start_new_session=True)
360 say('starting the ship at %s (pid %d), logging to %s'
361 % (self.pier, proc.pid, self.log))
362 self.wait_live(time.time() + timeout, proc, 0)
364 def wait_live(self, deadline, proc, offset):
365 """Wait for the pier to say it's live and for a thread to run."""
366 live = re.compile(r'pier \(\d+\): live')
368 if proc is not None and proc.poll() is not None:
369 tail = '\n'.join(self.log_since(offset)[-30:])
370 raise Unable('the ship exited with status %d; the end of '
371 'its log:\n%s' % (proc.returncode, tail))
372 if any(live.search(l) for l in self.log_since(offset)) or \
374 if os.path.exists(self.sock):
376 our = self.thread(OUR, timeout=60)
377 say('the ship is live: %s' % text(our))
379 except (Fail, OSError) as e:
380 if time.time() > deadline:
381 raise Unable('the ship is up but its threads '
383 if time.time() > deadline:
384 raise Unable('the ship did not come up in time; see %s'
388 def stop(self, timeout, force):
389 king, serf = self.processes()
390 if not king and not serf:
391 say('the ship is not running')
393 target = king or serf
394 say('stopping the ship (pid %d)' % target)
395 os.kill(target, signal.SIGTERM)
396 deadline = time.time() + timeout
397 while time.time() < deadline:
398 if not self.running():
399 say('the ship has stopped')
403 raise Fail('the ship is still running after %ds; tools/ship '
404 'stop --force kills it' % timeout)
405 for pid in self.processes():
407 os.kill(pid, signal.SIGKILL)
409 say('killed the ship')
413 def thread(self, hoon, timeout=600):
414 """Run a strand on the ship with khan-eval; the noun of its vase.
416 hoon is a whole strand, `=/ m (strand ,vase) ...`, in the
417 subject khan-eval gives it: strandio, spider and zuse.
419 if not os.path.exists(self.sock):
420 raise Unable('no %s: is the ship running?' % self.sock)
421 frame = self.request(hoon)
422 reply = self.call(frame, timeout)
423 noun = cue(reply[5:])
425 _rid, (kind, rest) = noun
426 except (TypeError, ValueError):
427 raise Fail('a strange reply from the ship: %.200r' % (noun,))
428 if kind == term('avow'):
433 raise Fail('the thread failed: %%%s\n%s' % (
434 text(mote), '\n'.join(' ' + l for l in tang_lines(tang))))
435 raise Fail('the ship refused the request: %%%s %.200r'
436 % (text(kind), rest))
438 def request(self, hoon):
439 """The request for a khan-eval thread, as click formats and jams it."""
440 prepared = prepare(hoon)
441 with tempfile.TemporaryDirectory(prefix='jibb-ship-') as tmp:
442 src = os.path.join(tmp, 'thread.hoon')
443 out = os.path.join(tmp, 'request.jam')
444 with open(src, 'w', encoding='ascii') as f:
446 res = subprocess.run(
447 [self.click, '-k', '-j', '-b', find_urbit(self.pier),
448 '-i', src, '-o', out, self.pier],
451 with open(out, 'rb') as f:
455 size = int.from_bytes(frame[1:5], 'little') if len(frame) >= 5 else -1
456 if len(frame) < 6 or frame[0] != 0 or size != len(frame) - 5:
457 err = [l for l in res.stderr.decode('utf-8', 'replace')
458 .replace('\r', '').split('\n')
459 if l and not re.match(r'^(loom|lite|eval \()', l)]
460 raise Fail('click could not jam the request:\n ' +
461 '\n '.join(err[-20:] or ['(no output)']))
464 def call(self, frame, timeout):
465 """Send a newt frame to conn.sock and read one whole frame back."""
466 s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
467 s.settimeout(timeout)
473 while need is None or len(buf) < need:
474 chunk = s.recv(1 << 20)
478 if need is None and len(buf) >= 5:
479 need = 5 + int.from_bytes(buf[1:5], 'little')
480 except socket.timeout:
481 raise Fail('no reply from the ship in %ds' % timeout)
485 raise Fail('the ship closed the connection without replying')
487 raise Fail('the reply was cut off after %d of its %d bytes: '
488 'vere closes conn.sock as soon as it has written a '
489 'reply, so only what fits in the socket buffer gets '
490 'through; ask for less' % (len(buf), need))
495 """Hoon as click -i needs it: no comments, backslashes doubled, and
496 every byte that isn't printable ASCII written as a \\XX escape, all
497 inside the cord click wraps the thread in."""
499 for line in hoon.split('\n'):
500 line = strip_comment(line).rstrip()
504 for ch in '\n'.join(lines).encode('utf-8'):
507 elif ch == 0x0a or 0x20 <= ch < 0x7f:
510 out.append('\\%02x' % ch)
511 return ''.join(out) + '\n'
514def strip_comment(line):
515 """A line of Hoon without its :: comment, minding quotes."""
527 elif line.startswith('::', i):
533# ---------------------------------------------------------------- threads
535# Each is a strand, filled in by fill(): @@NAME@@ is replaced. No
536# comments here, since click joins the lines (see prepare()).
540;< =bowl:spider bind:m get-bowl
541(pure:m !>((scot %p our.bowl)))
544# Mount points, from Clay's %hill, and each desk with its revision.
547;< =bowl:spider bind:m get-bowl
548;< ~ bind:m (send-raw-card [%pass /boat %arvo %c %boat ~])
549;< res=[wire sign-arvo] bind:m take-sign-arvo
550=/ sig=sign-arvo +.res
551=/ mon=(list @tas) ?.(?=([%clay %hill *] sig) ~ p.sig)
552=/ our (scot %p our.bowl)
553=/ now (scot %da now.bowl)
554=/ dez=(list desk) ~(tap in .^((set desk) %cd /[our]/[%$]/[now]))
559[d ud:.^(cass:clay %cw /[our]/[d]/[now])]
562# |new-desk @@DESK@@, as gen/hood/new-desk.hoon makes it: the %noun,
563# %hoon, %txt and %kelvin marks and sys.kelvin, from %base.
566;< =bowl:spider bind:m get-bowl
567=/ bas=path /(scot %p our.bowl)/base/(scot %da now.bowl)
569 ~[/mar/noun/hoon /mar/hoon/hoon /mar/txt/hoon /mar/kelvin/hoon /sys/kelvin]
570=/ pages=(map path page:clay)
571 %- ~(gas by *(map path page:clay))
574 [p (rear p) .^(* %cx (weld bas p))]
575;< ~ bind:m (poke-our %hood %helm-pass !>((new-desk:cloy %@@DESK@@ ~ pages)))
582;< =bowl:spider bind:m get-bowl
583=/ pax=path (en-beam [our.bowl %@@DESK@@ da+now.bowl] /)
584;< ~ bind:m (poke-our %hood %kiln-mount !>([pax %@@DESK@@]))
591;< ~ bind:m (poke-our %hood %kiln-unmount !>(`term`%@@DESK@@))
595# |commit %@@DESK@@, giving the revision before it.
598;< =bowl:spider bind:m get-bowl
599=/ rev=@ud ud:.^(cass:clay %cw /(scot %p our.bowl)/@@DESK@@/(scot %da now.bowl))
600;< ~ bind:m (poke-our %hood %kiln-commit !>([%@@DESK@@ |]))
604# Change a desk with @@ACT@@ and wait for the change to land: [%done
605# revision] when the desk gets a new revision, [%crud term tang] when
606# Clay rejects it, [%none ~] when nothing happens in @@SECS@@ seconds.
607# The change is a |commit (COMMIT_ACT) or deletions (DELETE_ACT), and
608# Clay names a rejected one by its task, @@MOTE@@: %into or %info.
609# Clay's error reaches only dill's %logs watchers (dill no longer
610# prints errors, and -t sends drum's output nowhere), so the thread
611# watches %logs while it waits.
614;< =bowl:spider bind:m get-bowl
615=/ rev=@ud ud:.^(cass:clay %cw /(scot %p our.bowl)/@@DESK@@/(scot %da now.bowl))
616=/ until=@da (add now.bowl (mul ~s1 @@SECS@@))
621 ;< ~ bind:m (send-raw-card [%pass /jibb/logs %arvo %d %logs ~])
622 ;< ~ bind:m (send-raw-card [%pass /jibb/next %arvo %c %warp our.bowl %@@DESK@@ ~])
624 ?. rest (pure:(strand ,~) ~)
625 (send-raw-card [%pass /jibb/wait %arvo %b %rest until])
627;< ~ bind:m (send-raw-card [%pass /jibb/logs %arvo %d %logs `~])
628;< ~ bind:m (send-raw-card [%pass /jibb/next %arvo %c %warp our.bowl %@@DESK@@ `[%next %w ud+rev /]])
629;< ~ bind:m (send-raw-card [%pass /jibb/wait %arvo %b %wait until])
633;< [wir=wire sig=sign-arvo] bind:m take-sign-arvo
634?: ?=([%jibb %logs ~] wir)
635 ?. ?=([%dill %logs %crud %@@MOTE@@ *] sig)
637 =/ cud ;;([@tas tang] +>+.sig)
638 (finish [%crud -.cud (scag 300 +.cud)] &)
639?: ?=([%jibb %next ~] wir)
640 ;< now=@da bind:m get-time
641 =/ new=@ud ud:.^(cass:clay %cw /(scot %p our.bowl)/@@DESK@@/(scot %da now))
642 (finish [%done new] &)
643?: ?=([%jibb %wait ~] wir)
649COMMIT_ACT = '(poke-our %hood %kiln-commit !>([%@@DESK@@ |]))'
651# Delete the files @@PATHS@@ from %@@DESK@@ through Clay, with a %info
652# task, as a |commit does for files deleted from the mount.
653DELETE_ACT = ('(send-raw-card [%pass /jibb/info %arvo %c %info %@@DESK@@ %& '
654 '(turn `(list path)`@@PATHS@@ |=(p=path [p %del ~]))])')
656# |merge %@@DESK@@ our %@@FROM@@, =gem %@@GERM@@
659;< =bowl:spider bind:m get-bowl
660=/ arg [%@@DESK@@ our.bowl %@@FROM@@ da+now.bowl %@@GERM@@]
661;< ~ bind:m (poke-our %hood %kiln-merge !>(arg))
665# A page of the desk's manifest, for comparing it with the repo: the
666# revision; whether +to-wain ends the last line at a final newline, as
667# vere's jet does, or gives an empty line after it, as the Hoon does;
668# how many files the desk has; for each file sent with its content,
669# whether the desk's mark, converting that content from %mime, gives
670# the stored value; and for the files @@FROM@@ to @@FROM@@ + @@COUNT@@,
671# the path with [%cord hash] for a cord, [%wain hash] for a %txt file's
672# lines, each with a newline after it, or [%other 0]. A hash is the
673# first 8 bytes of the sha-256. now is fixed across pages.
676;< =bowl:spider bind:m get-bowl
678=/ pre=path /(scot %p our.bowl)/@@DESK@@/(scot %da now)
679=/ all=(list path) .^((list path) %ct pre)
680=/ conv=(list [path ?])
681 %+ turn `(list [path @t])`@@CONV@@
683 ?. .^(? %cu (weld pre pax)) [pax |]
684 =/ have .^(* %cx (weld pre pax))
685 =/ tub .^(tube:clay %cc (weld pre /mime/[(rear pax)]))
686 =/ res (mule |.((tub !>([/text/plain (as-octs:mimes:html c)]))))
687 [pax &(?=(%& -.res) =(q.p.res have))]
688=/ page=(list [path @tas @])
689 %+ turn (swag [@@FROM@@ @@COUNT@@] all)
691 =/ have .^(* %cx (weld pre pax))
693 ?@ have [%cord (end [3 8] (shax have))]
694 ?. =(%txt (rear pax)) [%other 0]
696 (end [3 8] (shax (rap 3 (turn ;;(wain have) |=(l=@t (cat 3 l '\0a'))))))
697=/ jet=? =(1 (lent (to-wain:format 'a\0a')))
698(pure:m !>([ud:.^(cass:clay %cw pre) now jet (lent all) conv page]))
701# -test /=@@DESK@@=/tests, through the %test thread on %base.
704;< =bowl:spider bind:m get-bowl
705=/ pax=path /(scot %p our.bowl)/@@DESK@@/(scot %da now.bowl)/tests
706;< res=thread-result bind:m (await-thread %test !>([~ pax]))
708 %& (pure:m !>([%& ;;(? q.p.res)]))
709 %| (pure:m !>([%| p.res]))
714 """A number as a Hoon @ud literal: 150.000."""
715 return '{:,}'.format(n).replace(',', '.')
718# +fit: printed lines, cut to what a reply can carry (see REPLY_BYTES).
722 =| [n=@ud out=(list @t)]
725 =. n (add n +((met 3 i.lines)))
729 (crip "... and {(a-co:co (lent lines))} more lines, cut to fit the reply")
730 $(lines t.lines, out [i.lines out])
731""".replace('@@BYTES@@', hoon_ud(REPLY_BYTES))
733# A Hoon expression, evaluated as dojo would with our, now and eny,
734# printed as dojo prints it.
737;< =bowl:spider bind:m get-bowl
745(pure:m !>((fit (turn (wash [0 @@WIDTH@@] (sell vax)) crip))))
748# A strand's result vase, printed as dojo prints it.
754(pure:m !>((fit (turn (wash [0 @@WIDTH@@] (sell vax)) crip))))
757# A poke, as dojo's :agent &mark value.
760;< =bowl:spider bind:m get-bowl
763;< ~ bind:m (poke-our %@@AGENT@@ %@@MARK@@ !>(@@VALUE@@))
768def fill(template, **kw):
770 for k, v in kw.items():
771 out = out.replace('@@%s@@' % k, v)
772 left = re.findall(r'@@[A-Z]+@@', out)
774 raise RuntimeError('unfilled template slots: %s' % left)
779 if not TERM.match(name):
780 raise Unable('%s is not a desk name: desks are terms, lowercase '
781 'letters, digits and hyphens, starting with a letter'
786# ---------------------------------------------------------------- desks
790 """A desk file's Clay path from its Unix path: mar/txt.hoon is
792 segs = rel.split('/')
793 name, ext = segs[-1].rsplit('.', 1)
794 return segs[:-1] + [name, ext]
797def unix_problem(rel):
798 """Why vere would leave this file out of the desk, or None."""
799 segs = rel.split('/')
801 if '.' not in name or name.endswith('.'):
802 return 'it has no extension, so no mark'
803 if name.endswith('~'):
804 return 'it ends in ~'
806 if not KNOT.match(s):
807 return ('%r is not a knot: Clay paths are lowercase letters, '
808 'digits and - . ~ _' % s)
812def mark_exists(files, mark):
813 """Whether the desk has a mark: Clay looks for mark a-b-c at
814 /mar/a-b-c.hoon, then /mar/a/b-c.hoon, and so on."""
815 parts = mark.split('-')
816 for i in range(len(parts)):
817 rel = '/'.join(['mar'] + parts[:i] + ['-'.join(parts[i:])]) + '.hoon'
824 """A desk's files: desk-relative Unix path -> (bytes, where from)."""
826 def __init__(self, desk):
830 def add(self, rel, data, src):
831 old = self.files.get(rel)
832 if old is not None and old[0] != data:
833 raise Unable('%s/%s comes from both %s and %s, which differ'
834 % (self.desk, rel, old[1], src))
835 self.files[rel] = (data, src)
837 def replace(self, rel, data, src):
838 self.files[rel] = (data, src)
840 def add_tree(self, folder, into, skip=(), drop=None):
841 if not os.path.isdir(folder):
842 raise Unable('no folder %s' % show(folder))
843 for dirpath, dirnames, filenames in os.walk(folder):
844 rel_dir = os.path.relpath(dirpath, folder)
845 rel_dir = '' if rel_dir == '.' else rel_dir
846 dirnames[:] = sorted(
847 d for d in dirnames if not d.startswith('.')
848 and os.path.join(rel_dir, d) not in skip)
849 for fn in sorted(filenames):
850 if fn.startswith('.'):
852 rel = os.path.join(rel_dir, fn)
853 if drop and drop(rel):
855 desk_rel = os.path.join(into, rel) if into else rel
856 with open(os.path.join(dirpath, fn), 'rb') as f:
857 self.add(desk_rel, f.read(), show(os.path.join(dirpath, fn)))
861 path = os.path.abspath(path)
862 rel = os.path.relpath(path, ROOT)
863 return path if rel.startswith('..') else rel
866def assemble(desk, lib=None, golden=None):
867 """The files of one of the repo's desks, checked as vere and Clay
868 will see them. lib replaces /lib/jibb.hoon; golden, for the
869 jibb-dev test desk, is the folder of golden cases. When either is
870 overridden, exclude other /tests/lib programs so they don't run
871 against an unrelated library or pollute the custom case results."""
872 if desk not in DESKS:
873 raise Unable('no desk %s here; the repo builds %s'
874 % (desk, ', '.join(sorted(DESKS))))
875 # Override runs are harness tests, not tests of the repository's
876 # importer, checker, or other test files against the replacement lib.
877 isolated = (lib is not None or
878 (golden is not None and
879 os.path.abspath(golden) != os.path.abspath(GOLDEN)))
881 vendor = os.path.join(ROOT, 'vendor')
882 for pkg in sorted(os.listdir(vendor)):
883 if os.path.isdir(os.path.join(vendor, pkg)) and not pkg.startswith('.'):
884 a.add_tree(os.path.join(vendor, pkg), '')
885 for folder, into, skip in DESKS[desk]:
886 if folder == 'tests':
887 golden = golden or GOLDEN
888 if os.path.isdir(os.path.join(ROOT, 'tests')):
889 a.add_tree(os.path.join(ROOT, 'tests'), 'tests',
890 skip=['golden'], drop=lambda rel:
891 rel == os.path.join('lib', 'jibb.hoon') or
892 (isolated and rel.startswith('lib' + os.sep) and
893 rel.endswith('.hoon')))
894 if os.path.isdir(golden):
895 a.add_tree(golden, os.path.join('tests', 'golden'))
896 a.add(os.path.join('tests', 'lib', 'jibb.hoon'),
897 generate_tests(golden).encode(),
898 'tools/ship gen-tests --golden %s' % show(golden))
900 a.add_tree(os.path.join(ROOT, folder), into, skip=skip)
901 for rel, src in EXTRA.get(desk, {}).items():
902 with open(os.path.join(ROOT, src), 'rb') as f:
903 a.add(rel, f.read(), src)
905 if 'lib/jibb.hoon' not in a.files:
906 raise Unable('desk %s has no lib/jibb.hoon to replace' % desk)
907 with open(lib, 'rb') as f:
908 a.replace('lib/jibb.hoon', f.read(), show(lib))
913def check_assembly(a):
915 for rel in sorted(a.files):
916 why = unix_problem(rel)
917 if why is None and not mark_exists(a.files, rel.rsplit('.', 1)[1]):
918 why = 'the desk has no mark for .%s files' % rel.rsplit('.', 1)[1]
921 if rel.startswith('tests/golden/'):
922 warn('leaving %s out of %s: %s' % (a.files[rel][1], a.desk, why))
925 problems.append('%s (%s): %s' % (rel, a.files[rel][1], why))
926 if 'sys.kelvin' not in a.files:
927 problems.append('no sys.kelvin: Clay refuses a commit without one')
929 raise Unable('desk %s would not commit as it is:\n %s'
930 % (a.desk, '\n '.join(problems)))
933# ---------------------------------------------------------------- gen-tests
936:: tests/lib/jibb.hoon: the golden tests, run on a ship by -test
938:: Generated by tools/ship gen-tests from the cases in
939:: @@SOURCE@@/. Don't edit it: run tools/ship gen-tests again
940:: after changing the cases.
942:: Each test- arm runs golden:jibb on one case's source.jibb, with the
943:: file set to /tests/golden/<case>/source/jibb or the logical path in
944:: path.txt, and compares the outputs with tokens.txt and tree.txt.
945:: Each file holds the output plus one newline. An expect-types: line
946:: or types.txt opts a case into golden-types and its types.txt output.
947:: Files come in through /*; a missing expected output fails its case.
948:: tools/ship test syncs the desk and runs -test /=jibb-dev=/tests.
950:: The .txt files reach Clay through the %txt mark, as lines, so the
951:: test compares lines: the file's with those of the output plus a
952:: newline. A %txt file can't show whether it ends in a newline
953:: (vere's +to-wain jet ends the last line there, with no empty line
954:: after it), so only tools/golden, and tools/ship test's own warning,
955:: catch an expected file without its final newline.
959TESTS_HELPERS = r"""::
960:: +check: run golden:jibb on a case and compare both outputs
963 |= [case=@ta src=@t tokens=(unit wain) tree=(unit wain)]
965 (check-at case /tests/golden/[case]/source/jibb src tokens tree)
968 |= [case=@ta at=path src=@t tokens=(unit wain) tree=(unit wain)]
970 =/ out=[tokens=@t tree=@t] (golden:jibb at src)
972 (compare (file case "tokens.txt") tokens tokens.out)
973 (compare (file case "tree.txt") tree tree.out)
976 |= [case=@ta name=tape]
978 "tests/golden/{(trip case)}/{name}"
980:: +compare: a file's lines against those of an output plus a newline
983 |= [name=tape want=(unit wain) got=@t]
985 =/ have=wain (to-wain:format (cat 3 got '\0a'))
987 :- leaf+"{name} is missing; tools/golden --update writes it"
988 (turn have |=(l=@t leaf+" | {(trip l)}"))
991 =/ n=@ud (first-difference u.want have)
992 :~ leaf+"{name} differs from the output at line {(a-co:co +(n))}:"
993 leaf+" expected: {(line-at u.want n)}"
994 leaf+" actual: {(line-at have n)}"
1004 $(a t.a, b t.b, n +(n))
1007 |= [lines=wain n=@ud]
1009 ?: (gte n (lent lines))
1010 "(the end of the file)"
1011 ?: =('' (snag n lines))
1013 (trip (snag n lines))
1018def golden_cases(golden):
1019 """[(case name, folder)] for each case folder holding a source.jibb."""
1020 if not os.path.isdir(golden):
1023 for name in sorted(os.listdir(golden)):
1024 folder = os.path.join(golden, name)
1025 if not os.path.isdir(folder) or name.startswith('.'):
1027 if not os.path.isfile(os.path.join(folder, 'source.jibb')):
1028 warn('skipping %s: it has no source.jibb' % show(folder))
1030 if not KNOT.match(name):
1031 warn('skipping %s: %r is not a knot, so it has no Clay path '
1032 '(lowercase letters, digits and - . ~ _)' % (show(folder), name))
1034 cases.append((name, folder))
1038def arm_names(names):
1039 """A test- arm name for each case: its name with what a term can't
1040 hold made -, and a number added to tell clashes apart."""
1041 out, used = [], set()
1043 base = 'test-' + re.sub(r'[^a-z0-9-]', '-', name)
1047 arm = '%s-%d' % (base, n)
1053def hoon_list(items):
1054 """Render a Hoon list, including the empty list, which is just ~."""
1055 return '~[' + ' '.join(items) + ']' if items else '~'
1058def generate_tests(golden):
1059 cases = golden_cases(golden)
1063 has_module_types = False
1064 for _, folder in cases:
1066 sur = golden_case.sur_imports(folder)
1067 marks = golden_case.mark_imports(folder)
1068 modules = golden_case.module_fixtures(folder)
1069 has_imports |= bool(sur or marks)
1070 has_marks |= bool(marks)
1071 has_modules |= bool(modules)
1072 has_module_types |= bool(golden_case.wants_modules(folder) and
1073 golden_case.wants_types(folder))
1074 except ValueError as e:
1075 raise Unable(str(e))
1076 lines = [fill(TESTS_HEADER, SOURCE=show(golden)).rstrip('\n'),
1077 '/+ jibb, jibb-import' if has_imports or has_module_types else '/+ jibb']
1078 if has_module_types:
1079 lines.append('/* prelude-source %jibb /lib/jibb/prelude/jibb')
1080 lines.append('/* internal-source %jibb /lib/jibb/internal/jibb')
1082 for i, ((name, folder), arm) in enumerate(
1083 zip(cases, arm_names([c[0] for c in cases])), 1):
1084 base = '/tests/golden/%s' % name
1085 lines.append('/* src-%d %%jibb %s/source/jibb' % (i, base))
1086 # a knot literal: a case's name may hold . and _, which a
1087 # path literal in Hoon can't, though /* takes them
1089 logical = golden_case.source_path(folder)
1090 imports = golden_case.sur_imports(folder)
1091 marks = golden_case.mark_imports(folder)
1092 modules = golden_case.module_fixtures(folder)
1093 want_modules = golden_case.wants_modules(folder)
1094 except ValueError as e:
1095 raise Unable(str(e))
1096 args = ['~.%s' % name]
1098 args.append(logical)
1099 args.append('src-%d' % i)
1100 for kind, face in (('tokens', 'tok'), ('tree', 'tre')):
1101 if os.path.isfile(os.path.join(folder, kind + '.txt')):
1102 lines.append('/* %s-%d %%txt %s/%s/txt'
1103 % (face, i, base, kind))
1104 args.append('`%s-%d' % (face, i))
1107 call = '(%s %s)' % ('check-at' if logical else 'check', ' '.join(args))
1108 module_fixtures = []
1109 for j, (module, at, _) in enumerate(modules, 1):
1110 face = 'mod-%d-%d' % (i, j)
1111 lines.append('/* %s %%jibb %s/modules%s/jibb' %
1112 (face, base, module[:-5]))
1113 module_fixtures.append('[%s %s %s]' % (hoon_cord(module.encode()), at, face))
1114 if golden_case.wants_types(folder):
1115 if os.path.isfile(os.path.join(folder, 'types.txt')):
1116 lines.append('/* typ-%d %%txt %s/types/txt' % (i, base))
1117 expected_types = '`typ-%d' % i
1119 expected_types = '~'
1120 path = logical or '~[%%tests %%golden ~.%s %%source %%jibb]' % name
1123 for j, (module, mold, _) in enumerate(imports, 1):
1124 face = 'imp-%d-%d' % (i, j)
1125 lines.append('/* %s %%txt %s/sur/%s/txt' % (face, base, module))
1126 fixtures.append('[%%%s %%%s %s]' % (module, mold, face))
1128 for j, (mark, _) in enumerate(marks, 1):
1129 face = 'mark-%d-%d' % (i, j)
1130 lines.append('/* %s %%txt %s/mar/%s/txt' % (face, base, mark))
1131 mark_fixtures.append('[%%%s %s]' % (mark, face))
1132 call = '(weld %s (check-module-types ~.%s %s src-%d %s %s %s %s))' % (
1133 call, name, path, i, expected_types, hoon_list(fixtures),
1134 hoon_list(mark_fixtures), hoon_list(module_fixtures))
1137 for j, (module, mold, _) in enumerate(imports, 1):
1138 face = 'imp-%d-%d' % (i, j)
1139 lines.append('/* %s %%txt %s/sur/%s/txt' % (face, base, module))
1140 fixtures.append('[%%%s %%%s %s]' % (module, mold, face))
1142 for j, (mark, _) in enumerate(marks, 1):
1143 face = 'mark-%d-%d' % (i, j)
1144 lines.append('/* %s %%txt %s/mar/%s/txt' % (face, base, mark))
1145 mark_fixtures.append('[%%%s %s]' % (mark, face))
1146 call = '(weld %s (check-mark-types ~.%s %s src-%d %s %s %s))' % (
1147 call, name, path, i, expected_types, hoon_list(fixtures),
1148 hoon_list(mark_fixtures))
1151 for j, (module, mold, _) in enumerate(imports, 1):
1152 face = 'imp-%d-%d' % (i, j)
1153 lines.append('/* %s %%txt %s/sur/%s/txt' % (face, base, module))
1154 fixtures.append('[%%%s %%%s %s]' % (module, mold, face))
1155 call = '(weld %s (check-import-types ~.%s %s src-%d %s ~[%s]))' % (
1156 call, name, path, i, expected_types, ' '.join(fixtures))
1158 call = '(weld %s (check-types ~.%s %s src-%d %s))' % (
1159 call, name, path, i, expected_types)
1161 if os.path.isfile(os.path.join(folder, 'module-trees.txt')):
1162 lines.append('/* mtre-%d %%txt %s/module-trees/txt' % (i, base))
1163 expected_modules = '`mtre-%d' % i
1165 expected_modules = '~'
1166 call = '(weld %s (check-module-trees ~.%s %s ~[%s]))' % (
1167 call, name, expected_modules, ' '.join(module_fixtures))
1168 arms.append('++ %s\n %s' % (arm, call))
1169 if has_module_types:
1170 lines.append('=/ prepared-prelude (prepare-prelude-with-internal:jibb prelude-source internal-source)')
1173 lines.append(':: one arm per case')
1177 if any(golden_case.wants_types(folder) for _, folder in cases):
1178 types_helper = '''::
1180 |= [case=@ta at=path src=@t want=(unit wain)]
1182 (compare (file case "types.txt") want (golden-types:jibb at src))
1185 types_helper += '''::
1186++ check-import-types
1187 |= [case=@ta at=path src=@t want=(unit wain) fixtures=(list [module=term mold=term source=wain])]
1189 =/ result (mule |.((import-types-result at src fixtures)))
1191 [leaf+"fixture compilation or type checking crashed" p.result]
1192 (compare (file case "types.txt") want p.result)
1194++ import-types-result
1195 |= [at=path src=@t fixtures=(list [module=term mold=term source=wain])]
1197 =/ external=(list [module=term name=term result=*])
1199 |= fixture=[module=term mold=term source=wain]
1200 ~_ leaf+"/sur/{(trip module.fixture)}.txt fixture"
1201 ?~ source.fixture !!
1202 =/ text=@t (rap 3 (turn source.fixture |=(line=@t (cat 3 line '\\0a'))))
1203 =/ cor=type -:(ride -:!>(..zuse) text)
1204 [module.fixture mold.fixture (read-named:jibb-import module.fixture mold.fixture cor)]
1205 (golden-types-imports:jibb at src external)
1208 types_helper += '''::
1210 |= [case=@ta at=path src=@t want=(unit wain) fixtures=(list [module=term mold=term source=wain]) marks=(list [name=term source=wain])]
1212 =/ result (mule |.((mark-types-result at src fixtures marks)))
1214 [leaf+"fixture compilation or type checking crashed" p.result]
1215 (compare (file case "types.txt") want p.result)
1218 |= [at=path src=@t fixtures=(list [module=term mold=term source=wain]) marks=(list [name=term source=wain])]
1220 =/ external=(list [module=term name=term result=*])
1222 |= fixture=[module=term mold=term source=wain]
1223 ~_ leaf+"/sur/{(trip module.fixture)}.txt fixture"
1224 ?~ source.fixture !!
1225 =/ text=@t (rap 3 (turn source.fixture |=(line=@t (cat 3 line '\\0a'))))
1226 =/ cor=type -:(ride -:!>(..zuse) text)
1227 [module.fixture mold.fixture (read-named:jibb-import module.fixture mold.fixture cor)]
1228 =/ mark-external=(list [name=term result=*])
1230 |= fixture=[name=term source=wain]
1231 ~_ leaf+"/mar/{(trip name.fixture)}.txt fixture"
1232 ?~ source.fixture !!
1233 =/ text=@t (rap 3 (turn source.fixture |=(line=@t (cat 3 line '\\0a'))))
1234 =/ cor=type -:(ride -:!>(..zuse) text)
1235 [name.fixture (read-mark:jibb-import name.fixture cor)]
1236 (golden-types-with-marks:jibb at src external mark-external)
1239 types_helper += '''::
1240++ check-module-trees
1241 |= [case=@ta want=(unit wain) fixtures=(list [label=@t at=path source=@t])]
1243 =/ result (mule |.((module-trees-result fixtures)))
1245 [leaf+"module fixture parsing crashed" p.result]
1246 (compare (file case "module-trees.txt") want p.result)
1248++ module-trees-result
1249 |= fixtures=(list [label=@t at=path source=@t])
1255 |= fixture=[label=@t at=path source=@t]
1256 ~_ leaf+(trip label.fixture)
1257 =/ out (golden:jibb at.fixture source.fixture)
1258 (rap 3 ~[label.fixture '\\0a' tree.out])
1261 if has_module_types:
1262 types_helper += '''::
1263++ check-module-types
1264 |= [case=@ta at=path src=@t want=(unit wain) fixtures=(list [module=term mold=term source=wain]) marks=(list [name=term source=wain]) modules=(list [label=@t at=path source=@t])]
1266 =/ result (mule |.((module-types-result at src fixtures marks modules)))
1268 [leaf+"fixture compilation or type checking crashed" p.result]
1269 (compare (file case "types.txt") want p.result)
1271++ module-types-result
1272 |= [at=path src=@t fixtures=(list [module=term mold=term source=wain]) marks=(list [name=term source=wain]) modules=(list [label=@t at=path source=@t])]
1274 =/ external=(list [module=term name=term result=*])
1276 |= fixture=[module=term mold=term source=wain]
1277 ~_ leaf+"/sur/{(trip module.fixture)}.txt fixture"
1278 ?~ source.fixture !!
1279 =/ text=@t (rap 3 (turn source.fixture |=(line=@t (cat 3 line '\\0a'))))
1280 =/ cor=type -:(ride -:!>(..zuse) text)
1281 [module.fixture mold.fixture (read-named:jibb-import module.fixture mold.fixture cor)]
1282 =/ mark-external=(list [name=term result=*])
1284 |= fixture=[name=term source=wain]
1285 ~_ leaf+"/mar/{(trip name.fixture)}.txt fixture"
1286 ?~ source.fixture !!
1287 =/ text=@t (rap 3 (turn source.fixture |=(line=@t (cat 3 line '\\0a'))))
1288 =/ cor=type -:(ride -:!>(..zuse) text)
1289 [name.fixture (read-mark:jibb-import name.fixture cor)]
1290 =/ sources=(list [path=path source=@t])
1291 (turn modules |=(item=[label=@t at=path source=@t] [at.item source.item]))
1293 (golden-types-with-prelude-and-internal:jibb at src external mark-external sources prelude-source internal-source)
1294 (golden-types-with-prepared-prelude:jibb at src external mark-external sources prepared-prelude)
1296 return '\n'.join(lines) + '\n' + TESTS_HELPERS.replace('\n--\n', '\n' + types_helper + '--\n')
1299# ---------------------------------------------------------------- sync
1302def desks_info(ship):
1303 """(mount points, {desk: revision})."""
1304 mon, dez = ship.thread(DESKS_INFO)
1305 return ({text(m) for m in items(mon)},
1306 {text(d): rev for d, rev in items(dez)})
1309def ensure_desk(ship, desk, quiet=False):
1310 """Create the desk if it's missing and mount it; the mount folder."""
1311 mounts, revs = desks_info(ship)
1312 mount = os.path.join(ship.pier, desk)
1313 if desk not in revs:
1314 ship.thread(fill(NEW_DESK, DESK=desk))
1316 say('%s: created the desk, as |new-desk does' % desk)
1317 if desk in mounts and not os.path.isdir(mount):
1318 ship.thread(fill(UNMOUNT, DESK=desk))
1319 mounts.discard(desk)
1320 if desk not in mounts:
1321 ship.thread(fill(MOUNT, DESK=desk))
1322 deadline = time.time() + 30
1323 while not os.path.isfile(os.path.join(mount, 'sys.kelvin')):
1324 if time.time() > deadline:
1325 raise Fail('%s: mounted, but %s/sys.kelvin never appeared'
1329 say('%s: mounted at %s' % (desk, mount))
1333def mirror(mount, files):
1334 """Make the mount folder hold exactly these files (dotfiles aside,
1335 which vere ignores); the paths added, changed and deleted."""
1336 added, changed, deleted = [], [], []
1337 for rel, (data, _src) in sorted(files.items()):
1338 path = os.path.join(mount, rel)
1340 with open(path, 'rb') as f:
1342 except FileNotFoundError:
1346 os.makedirs(os.path.dirname(path), exist_ok=True)
1347 with open(path, 'wb') as f:
1349 (added if old is None else changed).append(rel)
1350 for dirpath, dirnames, filenames in os.walk(mount, topdown=False):
1351 rel_dir = os.path.relpath(dirpath, mount)
1352 rel_dir = '' if rel_dir == '.' else rel_dir
1353 if any(s.startswith('.') for s in rel_dir.split(os.sep) if s):
1355 for fn in filenames:
1356 rel = os.path.join(rel_dir, fn)
1357 if fn.startswith('.') or rel in files:
1359 os.remove(os.path.join(dirpath, fn))
1361 if rel_dir and not os.listdir(dirpath):
1363 return added, changed, sorted(deleted)
1366def commit(ship, desk):
1367 """|commit the desk; the revision before."""
1368 return ship.thread(fill(COMMIT, DESK=desk))
1371MANIFEST_PAGE = 2000 # files per reply; see REPLY_BYTES
1372CONV_BYTES = 65536 # bigger files with other marks are only looked for
1375def short_hash(data):
1376 """The first 8 bytes of the sha-256, as the atom the ship gives."""
1377 return int.from_bytes(hashlib.sha256(data).digest()[:8], 'little')
1380def txt_lines(data, jet):
1381 """The wain the %txt mark stores for a file: lines, without the
1382 empty one after a final newline if jet, as vere's +to-wain does."""
1385 lines = data.split(b'\n')
1386 if jet and data.endswith(b'\n'):
1388 # a line is an atom, which can't end in zero bytes
1389 return [l.rstrip(b'\0') for l in lines]
1392def verify(ship, desk, files):
1393 """Compare the desk with the files; (revision, problems, extra), extra
1394 being the paths of the desk's files that aren't among the files.
1396 The ship sends its manifest, a short hash of each file, and Python
1397 compares: files whose mark stores a cord by the hash of their
1398 bytes, %txt files by the hash of their lines, and the rest, which
1399 are few and small (desk.bill, sys.kelvin, desk.docket-0), by
1400 sending their content to be converted by the desk's own mark.
1402 by_path = {'/' + '/'.join(clay_path(rel)): rel for rel in files}
1404 for pax, rel in sorted(by_path.items()):
1405 mark = clay_path(rel)[-1]
1406 data = files[rel][0]
1407 if mark not in BYTE_MARKS and mark != 'txt' and \
1408 len(data) <= CONV_BYTES:
1409 conv.append('[%s %s]' % (hoon_path(clay_path(rel)),
1411 now, start, total, desk_files = 'now.bowl', 0, None, {}
1412 while total is None or start < total:
1413 noun = ship.thread(fill(
1414 MANIFEST, DESK=desk, NOW=now, FROM=hoon_ud(start),
1415 COUNT=hoon_ud(MANIFEST_PAGE),
1416 CONV='~[%s]' % ' '.join(conv) if conv else '~'))
1417 rev, (da, (jet, (total, (conv_res, page)))) = noun
1418 now = '`@da``@`%s' % hoon_ud(da)
1419 for pax, (kind, val) in items(page):
1420 desk_files[path_text(pax)] = (text(kind), val)
1421 start += MANIFEST_PAGE
1424 extra = sorted(set(desk_files) - set(by_path))
1426 problems.append('%s is on the desk but not in the repo' % pax)
1427 converted = {path_text(p): ok == 0 for p, ok in items(conv_res)}
1428 for pax, rel in sorted(by_path.items()):
1429 data, src = files[rel]
1430 mark = clay_path(rel)[-1]
1431 if pax not in desk_files:
1432 problems.append('%s is missing from the desk' % pax)
1434 kind, val = desk_files[pax]
1435 if mark in BYTE_MARKS:
1436 same = kind == 'cord' and val == short_hash(data.rstrip(b'\0'))
1438 same = kind == 'wain' and val == short_hash(
1439 b''.join(l + b'\n' for l in txt_lines(data, jet)))
1440 elif pax in converted:
1441 same = converted[pax]
1443 continue # too big to send; it's there
1445 problems.append('%s differs from %s' % (pax, src))
1446 return rev, problems, extra
1449def sync(ship, desk, lib=None, golden=None, timeout=60, quiet=False):
1450 """Assemble a desk, copy it into its mount and |commit; the revision.
1452 The desk is checked against the repo first: when they already match
1453 there's nothing to commit. Files on the desk that the repo doesn't
1454 have are deleted through Clay first, since a |commit deletes only
1455 the files vere has seen in the mount, and vere forgets them when
1456 the ship restarts. Then the commit runs in a thread that waits for
1457 the new revision, or for Clay's error.
1459 started = time.time()
1460 a = assemble(desk, lib=lib, golden=golden)
1462 mount = ensure_desk(ship, desk, quiet=quiet)
1463 rev, problems, extra = verify(ship, desk, a.files)
1466 rev = delete_and_wait(ship, desk, extra, timeout)
1467 done.append('deleted %d from the desk in revision %d'
1468 % (len(extra), rev))
1469 added, changed, deleted = mirror(mount, a.files)
1470 if len(problems) > len(extra):
1471 rev = commit_and_wait(ship, desk, timeout)
1472 done.append('committed revision %d' % rev)
1474 rev, problems, _ = verify(ship, desk, a.files)
1476 raise Fail('%s: after |commit, at revision %d, the desk still '
1477 "doesn't match the repo:\n %s"
1478 % (desk, rev, '\n '.join(problems[:20])))
1479 what = '; '.join(done)
1481 what = 'nothing to commit: already at revision %d' % rev
1483 say('%s: %d files; copied +%d ~%d -%d; %s; the desk matches the '
1484 'repo (%.1fs)' % (desk, len(a.files), len(added), len(changed),
1485 len(deleted), what, time.time() - started))
1489def commit_and_wait(ship, desk, timeout):
1490 """|commit, and wait for the new revision; raise Clay's error."""
1491 return change_and_wait(ship, desk, fill(COMMIT_ACT, DESK=desk), 'into',
1492 'the commit', timeout)
1495# click -i still passes the formatted thread as one argument to click-format.
1496# Keep well below Linux's per-argument limit, including click's escaping.
1497DELETE_REQUEST_BYTES = 60000
1498DELETE_BATCH_PATHS = 1024
1501def click_argument_bytes(hoon):
1502 """Bytes in click-format's single command argument, including its NUL."""
1503 src = prepare(hoon).rstrip('\n')
1504 return len(src.replace("'", "\\'").replace('\n', ' ').encode('ascii')) + 1
1507def delete_action(desk, paths):
1508 return fill(DELETE_ACT, DESK=desk, PATHS='~[%s]' % ' '.join(
1509 hoon_path(p.split('/')[1:]) for p in paths))
1512def delete_batches(desk, paths, timeout):
1513 """Preflight bounded deletion requests before changing the desk."""
1514 def partition(batch):
1515 act = delete_action(desk, batch)
1516 request = fill(CHANGE_WAIT, ACT=act, MOTE='info', DESK=desk,
1517 SECS=hoon_ud(timeout))
1518 if click_argument_bytes(request) <= DELETE_REQUEST_BYTES:
1519 return [(batch, act)]
1521 raise Unable('a Clay deletion path is too long for click')
1522 mid = len(batch) // 2
1523 return partition(batch[:mid]) + partition(batch[mid:])
1526 for start in range(0, len(paths), DELETE_BATCH_PATHS):
1527 batches.extend(partition(paths[start:start + DELETE_BATCH_PATHS]))
1528 # Preserve the existing empty-list behavior for direct callers.
1529 return batches or partition([])
1532def delete_and_wait(ship, desk, paths, timeout):
1533 """Delete files through Clay, waiting for each resulting revision."""
1534 batches = delete_batches(desk, paths, timeout)
1535 for batch, act in batches:
1536 rev = change_and_wait(ship, desk, act, 'info',
1537 'the deletion of %d files' % len(batch), timeout)
1541def change_and_wait(ship, desk, act, mote, what, timeout):
1542 """Change the desk with act, a CHANGE_WAIT action whose rejection
1543 Clay names %mote, and wait for the new revision; raise Clay's error.
1544 what names the change in errors."""
1545 noun = ship.thread(fill(CHANGE_WAIT, ACT=act, MOTE=mote, DESK=desk,
1546 SECS=hoon_ud(timeout)),
1547 timeout=timeout + 60)
1554 raise Fail('%s: Clay rejected %s: %%%s\n %s' % (
1555 desk, what, text(mote), '\n '.join(tang_lines(tang))))
1556 raise Fail('%s: %s changed nothing in %ds' % (desk, what, timeout))
1559# ---------------------------------------------------------------- scry
1562def parse_scry(spec, care=None):
1563 """A scry path as dojo writes it, /cx/=my-app=/ui/about/jibb, into
1564 (care, Hoon for the path). = stands for our, the desk %base and
1565 now, in the first three places after the care."""
1566 segs = spec.split('/')
1567 if not spec.startswith('/'):
1568 raise Unable('a scry path starts with /: %s' % spec)
1571 if not segs or not re.match(r'^[a-z]{2}$', segs[0]):
1572 raise Unable('a scry path starts with the vane and care, as '
1573 '/cx/=my-app=/desk/bill: %s' % spec)
1574 care, segs = segs[0], segs[1:]
1575 # =desk= is three segments: our, desk, now
1577 if segs and segs[0].startswith('=') and segs[0].endswith('=') and \
1579 segs = ['=', segs[0][1:-1], '='] + segs[1:]
1580 defaults = ['(scot %p our.bowl)', '%base', '(scot %da now)']
1581 for i, s in enumerate(segs):
1582 if i < 3 and s in ('=', ''):
1583 out.append('[%s]' % defaults[i])
1585 continue # a trailing or doubled /
1589 out.append('[~.%s]' % s)
1591 out.append('[%s]' % hoon_cord(s.encode()))
1592 return care, '/' + '/'.join(out)
1595# now is the time the path's = stands for: the ship's now, or for the
1596# later parts of a big --raw read, the time of the first.
1599;< =bowl:spider bind:m get-bowl
1601=/ val .^(@@TYPE@@ %@@CARE@@ @@PATH@@)
1605# --raw: [%atom size now part], part being @@SIZE@@ bytes from
1606# @@OFFSET@@ on, or [%cell ~].
1609?^ raw (pure:m !>([%cell ~]))
1610(pure:m !>([%atom (met 3 raw) now (cut 3 [@@OFFSET@@ @@SIZE@@] raw)]))
1613SCRY_PRINT = FIT + r"""
1614(pure:m !>((fit (turn (wash [0 @@WIDTH@@] (sell !>(val))) crip))))
1618def cmd_scry(ship, args):
1620 care, pax = parse_scry(args.path)
1622 noun = ship.thread(fill(SCRY, TYPE=args.type, CARE=care, PATH=pax,
1623 NOW='now.bowl', RESULT=fill(
1624 SCRY_PRINT, WIDTH=hoon_ud(args.width))))
1625 for line in items(noun):
1628 # an atom comes in parts, all read at the time of the first
1629 now, offset, parts, size = 'now.bowl', 0, [], None
1630 while size is None or offset < size:
1631 noun = ship.thread(fill(SCRY, TYPE=args.type, CARE=care, PATH=pax,
1632 NOW=now, RESULT=fill(
1633 SCRY_RAW, OFFSET=hoon_ud(offset),
1634 SIZE=hoon_ud(REPLY_BYTES))))
1635 if text(noun[0]) != 'atom':
1636 raise Fail('the result is a cell, not an atom; drop --raw')
1637 size, (da, part) = noun[1]
1638 now = '`@da``@`%s' % hoon_ud(da)
1639 want = min(REPLY_BYTES, size - offset)
1640 parts.append(atom_bytes(part).ljust(want, b'\0'))
1642 sys.stdout.buffer.write(b''.join(parts))
1646# ---------------------------------------------------------------- hoon, run, poke
1649def cmd_hoon(ship, args):
1653 with open(args.file, encoding='utf-8') as f:
1656 raise Unable('give an expression, or -f FILE')
1657 noun = ship.thread(fill(EXPR, EXPR=indent(expr, 2), WIDTH=hoon_ud(args.width)))
1658 for line in items(noun):
1662def cmd_run(ship, args):
1664 with open(args.file, encoding='utf-8') as f:
1666 noun = ship.thread(fill(RUN_FILE, THREAD=indent(thread, 2),
1667 WIDTH=hoon_ud(args.width)), timeout=args.timeout)
1668 for line in items(noun):
1672def cmd_poke(ship, args):
1674 for name in (args.agent, args.mark):
1675 if not TERM.match(name):
1676 raise Unable('%s is not a term' % name)
1677 ship.thread(fill(POKE, AGENT=args.agent, MARK=args.mark,
1679 say('poked %%%s with %%%s' % (args.agent, args.mark))
1682def indent(text_, n):
1683 return '\n'.join((' ' * n + l) if l.strip() else l
1684 for l in text_.strip('\n').split('\n'))
1687# ---------------------------------------------------------------- status, log
1690def cmd_status(ship, args):
1691 king, serf = ship.processes()
1692 if not king and not serf:
1693 say('the ship at %s is not running' % ship.pier)
1695 say('the ship at %s is running: pid %s (serf %s), log %s'
1696 % (ship.pier, king, serf, ship.log))
1698 our = ship.thread(OUR, timeout=30)
1699 mounts, revs = desks_info(ship)
1700 except (Fail, OSError) as e:
1701 say('but it does not answer: %s' % e)
1703 say('%s is live; its desks:' % text(our))
1704 width = max([len(d) for d in revs] + [10])
1705 for d in sorted(revs):
1706 mount = os.path.join(ship.pier, d)
1707 say((' %%%-*s revision %-4d %s' % (
1708 width, d, revs[d], 'mounted at ' + mount if d in mounts
1713def cmd_log(ship, args):
1714 lines = ship.log_since(0)
1715 if lines and lines[-1] == '':
1718 lines = [l for l in lines if not NOISE.match(l)]
1719 for line in lines[-args.lines:]:
1723def cmd_commit(ship, args):
1726 rev = commit_and_wait(ship, desk_term(args.desk), args.timeout)
1728 if 'changed nothing' not in str(e):
1730 say('|commit %%%s: no new revision in %ds; nothing to commit?'
1731 % (args.desk, args.timeout))
1733 say('|commit %%%s: committed revision %d' % (args.desk, rev))
1736def cmd_mount(ship, args):
1738 mount = ensure_desk(ship, desk_term(args.desk))
1739 say('%s is mounted at %s' % (args.desk, mount))
1742def cmd_unmount(ship, args):
1744 ship.thread(fill(UNMOUNT, DESK=desk_term(args.desk)))
1745 say('|unmount %%%s sent' % args.desk)
1748def cmd_sync(ship, args):
1749 lib = STUB if args.stub else args.lib
1750 for desk in args.desks:
1751 sync(ship, desk, lib=lib, golden=args.golden, timeout=args.timeout)
1754def cmd_gen_tests(ship, args):
1755 golden = args.golden or GOLDEN
1756 out = args.out or TESTS_LIB
1757 content = generate_tests(golden)
1759 with open(out, encoding='utf-8') as f:
1761 except FileNotFoundError:
1763 n = len(golden_cases(golden))
1766 say('%s is out of date with %s; run tools/ship gen-tests'
1767 % (show(out), show(golden)))
1769 say('%s is up to date: %d cases' % (show(out), n))
1772 say('%s is up to date: %d cases' % (show(out), n))
1774 os.makedirs(os.path.dirname(os.path.abspath(out)), exist_ok=True)
1775 with open(out, 'w', encoding='utf-8') as f:
1777 say('wrote %s: %d cases from %s' % (show(out), n, show(golden)))
1781# ---------------------------------------------------------------- test
1783TEST_LINE = re.compile(r'^(OK|FAILED|CRASHED|built) +(/\S*)')
1786def settle(ship, offset, quiet=0.5, limit=5.0):
1787 """Wait for the log to stop growing, so a thread's output is in."""
1788 deadline = time.time() + limit
1789 size = ship.log_size()
1790 while time.time() < deadline:
1792 now = ship.log_size()
1796 return [l for l in ship.log_since(offset) if l.strip() and not NOISE.match(l)]
1799def cmd_test(ship, args):
1801 lib = STUB if args.stub else args.lib
1802 golden = args.golden or GOLDEN
1803 started = time.time()
1804 if args.golden is None:
1805 content = generate_tests(golden)
1807 with open(TESTS_LIB, encoding='utf-8') as f:
1808 stale = f.read() != content
1809 except FileNotFoundError:
1812 os.makedirs(os.path.dirname(TESTS_LIB), exist_ok=True)
1813 with open(TESTS_LIB, 'w', encoding='utf-8') as f:
1815 say('updated %s from %s' % (show(TESTS_LIB), show(golden)))
1816 for _name, folder in golden_cases(golden):
1817 for fn in ('tokens.txt', 'tree.txt', 'types.txt', 'module-trees.txt'):
1818 data = _read(os.path.join(folder, fn))
1819 if data and not data.endswith(b'\n'):
1820 warn("%s has no final newline: the ship can't see that (the "
1821 "%%txt mark keeps only lines, and vere's +to-wain jet "
1822 "drops a final newline), but tools/golden fails it"
1823 % show(os.path.join(folder, fn)))
1824 if not args.no_sync:
1825 sync(ship, 'jibb-dev', lib=lib, golden=golden, timeout=args.timeout)
1826 synced = time.time()
1827 offset = ship.log_size()
1828 flag, rest = ship.thread(fill(RUN_TEST, DESK='jibb-dev'),
1829 timeout=args.timeout)
1830 lines = settle(ship, offset)
1833 counts = {'OK': 0, 'FAILED': 0, 'CRASHED': 0}
1835 m = TEST_LINE.match(line)
1836 if m and m.group(1) in counts and '/test-' in m.group(2):
1837 counts[m.group(1)] += 1
1838 built_failed = any(re.match(r'^FAILED +\S+ \(build\)', l) for l in lines)
1839 took = 'sync %.1fs, -test %.1fs' % (synced - started, time.time() - synced)
1842 say('-test itself failed: %%%s' % text(mote))
1843 for line in tang_lines(tang):
1845 say('FAIL (%s)' % took)
1848 summary = '%d OK, %d FAILED, %d CRASHED' % (
1849 counts['OK'], counts['FAILED'], counts['CRASHED'])
1851 summary += '; a test file failed to build'
1852 say('%s: %s (%s)' % ('PASS' if ok else 'FAIL', summary, took))
1853 return 0 if ok else 1
1856# ---------------------------------------------------------------- check-commit
1858# Each file's value as a cord, read with %cx.
1861;< =bowl:spider bind:m get-bowl
1862=/ pre=path (en-beam [our.bowl %@@DESK@@ da+now.bowl] /)
1863=/ paz=(list path) @@PATHS@@
1867?. .^(? %cu (weld pre p))
1869[p %& .^(@t %cx (weld pre p))]
1872# Each %jibb file through the desk's own tubes: to %mime and back, to
1873# %txt and back. Also whether +to-wain ends the last line at a final
1874# newline, as vere's jet does, rather than giving an empty line after
1875# it, as the Hoon in zuse does.
1878;< =bowl:spider bind:m get-bowl
1879=/ bek=beak [our.bowl %@@DESK@@ da+now.bowl]
1880=/ pre=path (en-beam bek /)
1881=/ paz=(list path) @@PATHS@@
1882;< to-mime=tube:clay bind:m (build-tube bek %jibb %mime)
1883;< of-mime=tube:clay bind:m (build-tube bek %mime %jibb)
1884;< to-txt=tube:clay bind:m (build-tube bek %jibb %txt)
1885;< of-txt=tube:clay bind:m (build-tube bek %txt %jibb)
1886=/ jet=? =(1 (lent (to-wain:format 'a\0a')))
1891=/ src=@t .^(@t %cx (weld pre p))
1892=/ mym=mime !<(mime (to-mime !>(src)))
1893=/ wan=wain !<(wain (to-txt !>(src)))
1894=/ bam=@t !<(@t (of-mime !>(mym)))
1895=/ bat=@t !<(@t (of-txt !>(wan)))
1896[p p.mym p.q.mym q.q.mym (of-wain:format wan) (lent wan) bam bat]
1899# Overwrite a desk with a commit that has no parents, as |new-desk,
1900# =hard & does. It gets what |new-desk puts in a new desk, %base's
1901# %noun, %hoon, %txt and %kelvin marks and sys.kelvin; then the marks
1902# of another desk, if asked; then some %jibb files. Clay won't commit
1903# a desk without sys.kelvin, nor sys.kelvin without the %kelvin mark.
1906;< =bowl:spider bind:m get-bowl
1907=/ bas=path /(scot %p our.bowl)/base/(scot %da now.bowl)
1908=/ src=path /(scot %p our.bowl)/@@FROM@@/(scot %da now.bowl)
1910 ~[/mar/noun/hoon /mar/hoon/hoon /mar/txt/hoon /mar/kelvin/hoon /sys/kelvin]
1911=/ pages=(map path page:clay)
1912 %- ~(gas by *(map path page:clay))
1914 (turn new |=(p=path [p (rear p) .^(* %cx (weld bas p))]))
1915 (turn `(list path)`@@MARKS@@ |=(p=path [p (rear p) .^(* %cx (weld src p))]))
1916 `(list [path page:clay])`@@FILES@@
1918;< ~ bind:m (poke-our %hood %helm-pass !>((new-desk:cloy %@@DESK@@ ~ pages)))
1922CHECK_OURS = 'jibb-check-ours'
1923CHECK_THEIRS = 'jibb-check-theirs'
1925# The file the merge checks edit: the start of worked example 3.
1929++ main (program init update view subs)
1931+$ model [todos=(list item.todo) draft=@t error=(unit @t)]
1936 [%got todos=(list item.todo)]
1937 [%acked err=(unit @t)]
1941 'ui/check-commit/unclosed-string.jibb':
1942 b'++ main view\n\n++ view\n text "Hello, wor',
1943 'ui/check-commit/unfinished-rune.jibb':
1944 b'++ update |= [=msg m=model]\n ?- msg\n [%typed t]\n ?-\n',
1945 'ui/check-commit/cut-utf8.jibb':
1946 b'++ title "Caf\xc3',
1947 'ui/check-commit/empty.jibb': b'',
1950# What kiln prints when a merge ends.
1951KILN_SAID = re.compile(r'^(merged with strategy|failed to merge with '
1952 r'strategy|auto merge failed on strategy)')
1955def paths_hoon(rels):
1956 return '~[%s]' % ' '.join(hoon_path(clay_path(r)) for r in rels) \
1960def read_files(ship, desk, rels):
1961 """{rel: bytes or None} for files on a desk, read with %cx."""
1962 noun = ship.thread(fill(READ_FILES, DESK=desk, PATHS=paths_hoon(rels)))
1964 for rel, (_p, (flag, val)) in zip(rels, items(noun)):
1965 out[rel] = atom_bytes(val) if flag == 0 else None
1969def wait_until(what, test, timeout):
1970 deadline = time.time() + timeout
1975 if time.time() > deadline:
1976 raise Fail('timed out waiting for %s' % what)
1980def reset_desk(ship, desk, files=None, marks_from=None):
1981 """Overwrite a desk with what |new-desk gives, the marks of another
1982 desk if asked, and these files ({rel: bytes}, all %jibb)."""
1983 pages = ' '.join('[%s %%jibb %s]' % (hoon_path(clay_path(rel)),
1985 for rel, data in sorted((files or {}).items()))
1987 RESET_DESK, DESK=desk, FROM=marks_from or 'base',
1988 MARKS='.^((list path) %ct (weld src /mar))' if marks_from else '~',
1989 FILES='~[%s]' % pages if pages else '~'))
1992def merge_and_wait(ship, desk, source, germ, timeout):
1993 """|merge %desk our %source, =gem %germ, and wait for kiln to say
1994 how it went: (merged, the lines the ship printed)."""
1995 offset = ship.log_size()
1996 ship.thread(fill(MERGE, DESK=desk, FROM=source, GERM=germ))
1997 said = wait_until('kiln to finish |merge %%%s our %%%s' % (desk, source),
1998 lambda: [l for l in ship.log_since(offset)
1999 if KILN_SAID.match(l)], timeout)
2000 return said[0].startswith('merged with'), settle(ship, offset)
2003def change_line(data, n, new):
2004 lines = data.split(b'\n')
2006 return b'\n'.join(lines)
2009def commit_edit(ship, desk, rel, data, timeout):
2010 """Write a file into a desk's mount and |commit it; wait for it."""
2011 mount = os.path.join(ship.pier, desk)
2012 path = os.path.join(mount, rel)
2013 os.makedirs(os.path.dirname(path), exist_ok=True)
2014 with open(path, 'wb') as f:
2016 commit_and_wait(ship, desk, timeout)
2017 got = read_files(ship, desk, [rel])[rel]
2019 raise Fail('%s: after |commit, /%s reads back as %r' % (desk, rel, got))
2022def without_final_newline(data):
2023 return data[:-1] if data.endswith(b'\n') else data
2026def cmd_check_commit(ship, args):
2029 started = time.time()
2031 def check(name, fn):
2037 notes = str(e).split('\n')
2039 results.append((name, ok))
2040 say('%s %s (%.1fs)' % ('PASS' if ok else 'FAIL', name, time.time() - t0))
2044 a = assemble('my-app')
2045 ui = sorted(rel for rel in a.files
2046 if rel.startswith('ui/') and rel.endswith('.jibb'))
2049 rev = sync(ship, 'my-app', timeout=args.timeout, quiet=True)
2050 return ['my-app is at revision %d with the repo\'s %d files, %d of '
2051 'them under /ui' % (rev, len(a.files), len(ui))]
2053 def byte_for_byte():
2054 got = read_files(ship, 'my-app', ui)
2057 want = a.files[rel][0]
2058 if got[rel] is None:
2059 bad.append('/%s is not on the desk' % rel)
2060 elif got[rel] != want:
2061 bad.append('/%s: %d bytes on the desk, %d in %s'
2062 % (rel, len(got[rel]), len(want), a.files[rel][1]))
2064 notes.append('%s: %d bytes, identical' % (
2065 '/' + '/'.join(clay_path(rel)), len(want)))
2067 raise Fail('\n'.join(bad))
2071 jet, res = ship.thread(fill(CONVERT, DESK='my-app',
2072 PATHS=paths_hoon(ui)))
2075 for rel, res in zip(ui, items(res)):
2076 want = a.files[rel][0]
2077 _p, (mite, (size, (octs, (joined, (count, (bam, bat)))))) = res
2078 lines = txt_lines(want, jet)
2079 back = b'\n'.join(lines)
2081 if path_text(mite) != '/text/x-jibb':
2082 bad.append('%s: to %%mime gives the type %s'
2083 % (name, path_text(mite)))
2084 if size != len(want) or atom_bytes(octs) != want:
2085 bad.append('%s: to %%mime changes the bytes' % name)
2086 if atom_bytes(bam) != want:
2087 bad.append('%s: %%mime back to %%jibb changes it' % name)
2088 if count != len(lines) or atom_bytes(joined) != back:
2089 bad.append('%s: to %%txt gives %d lines, not the file\'s %d'
2090 % (name, count, len(lines)))
2091 if atom_bytes(bat) != back:
2092 bad.append('%s: %%txt back to %%jibb gives something else '
2093 'than the lines joined' % name)
2094 lost += back != want
2096 raise Fail('\n'.join(bad))
2097 notes = ['%d files: to %%mime (/text/x-jibb) and back unchanged; to '
2098 '%%txt, one cord per line, and back' % len(ui)]
2101 'but: %%txt keeps no final newline, so %d of the %d files '
2102 'come back from %%txt without theirs. The mark is right; '
2103 "vere's +to-wain jet ends the last line at a final newline, "
2104 'where the Hoon in zuse gives an empty line after it. %%hoon '
2105 'files do the same.' % (lost, len(ui)))
2109 mount = ensure_desk(ship, 'my-app', quiet=True)
2110 rels = sorted(HALF_WRITTEN)
2114 path = os.path.join(mount, rel)
2115 os.makedirs(os.path.dirname(path), exist_ok=True)
2116 with open(path, 'wb') as f:
2117 f.write(HALF_WRITTEN[rel])
2118 rev = commit_and_wait(ship, 'my-app', args.timeout)
2119 got = read_files(ship, 'my-app', rels)
2120 notes.append('|commit made revision %d' % rev)
2122 if got[rel] != HALF_WRITTEN[rel]:
2123 raise Fail('/%s committed, but reads back as %r'
2125 notes.append('/%s committed and reads back the same: %r'
2126 % (rel, HALF_WRITTEN[rel][-24:]))
2128 shutil.rmtree(os.path.join(mount, 'ui', 'check-commit'),
2130 rev = commit_and_wait(ship, 'my-app', args.timeout)
2131 left = [r for r, v in read_files(ship, 'my-app', rels).items()
2134 raise Fail('deleting them and |commit left %s' % ', '.join(left))
2135 notes.append('deleted them again: |commit made revision %d' % rev)
2138 def merges_by_line():
2139 rel = 'ui/merge.jibb'
2141 ours = change_line(base, 5, b'+$ model [todos=(list item.todo) '
2142 b'draft=@t error=(unit @t) note=@t]')
2143 theirs = change_line(base, 9, b' [%toggled id=@ud on=?]')
2144 merged = change_line(ours, 9, b' [%toggled id=@ud on=?]')
2146 prepare_pair(ship, rel, base, args.timeout)
2147 commit_edit(ship, CHECK_OURS, rel, ours, args.timeout)
2148 commit_edit(ship, CHECK_THEIRS, rel, theirs, args.timeout)
2149 notes.append('%%%s changed line 5 and %%%s line 9 of /%s, each with '
2150 '|commit' % (CHECK_OURS, CHECK_THEIRS, rel))
2151 cmd = '|merge %%%s our %%%s' % (CHECK_OURS, CHECK_THEIRS)
2152 ok, said = merge_and_wait(ship, CHECK_OURS, CHECK_THEIRS, 'auto',
2155 if not any('auto merge failed on strategy %fine' in l
2157 raise Fail('%s failed:\n %s' % (cmd, '\n '.join(said[:20])))
2159 "but: %s, with kiln's default =gem %%auto, stops at "
2160 '"auto merge failed on strategy %%fine": kiln goes on to '
2161 '%%meet only on a %%bad-fine-merge error, and this Clay '
2162 'reports %%merge-failed' % cmd)
2163 cmd += ', =gem %mate'
2164 ok, said = merge_and_wait(ship, CHECK_OURS, CHECK_THEIRS, 'mate',
2167 raise Fail('%s failed:\n %s' % (cmd, '\n '.join(said[:20])))
2168 got = read_files(ship, CHECK_OURS, [rel])[rel]
2170 notes.append('%s: both changes, byte for byte' % cmd)
2171 elif got == without_final_newline(merged):
2172 notes.append('%s: both changes, line for line' % cmd)
2174 'but: the merged file has lost its final newline: Clay '
2175 'merges through %txt (grad %txt), which keeps none; see '
2176 'the %txt note above')
2178 raise Fail('%s did not give both changes; /%s is now:\n%s'
2180 return notes + ['the ship said: ' + l for l in said
2181 if KILN_SAID.match(l)]
2183 def conflicts_by_line():
2184 rel = 'ui/merge.jibb'
2186 ours = change_line(base, 7, b' [%typed text=@t at=@da]')
2187 theirs = change_line(base, 7, b' [%typed words=@t]')
2188 prepare_pair(ship, rel, base, args.timeout)
2189 commit_edit(ship, CHECK_OURS, rel, ours, args.timeout)
2190 commit_edit(ship, CHECK_THEIRS, rel, theirs, args.timeout)
2191 ok, said = merge_and_wait(ship, CHECK_OURS, CHECK_THEIRS, 'mate',
2193 got = read_files(ship, CHECK_OURS, [rel])[rel]
2194 if ok or got != ours or not any('mate-conflict' in l for l in said):
2195 raise Fail('a %%mate merge of two changes to line 7 should fail '
2196 'with a conflict and leave %%%s alone; the ship said:'
2197 '\n %s' % (CHECK_OURS, '\n '.join(said[:20])))
2198 return ['both desks changed line 7: |merge %%%s our %%%s, =gem %%mate '
2199 'fails and leaves the file alone' % (CHECK_OURS, CHECK_THEIRS)] \
2200 + ['the ship said: ' + l for l in said
2201 if KILN_SAID.match(l) or l == 'mate-conflict']
2204 mounts, revs = desks_info(ship)
2205 for desk in (CHECK_OURS, CHECK_THEIRS):
2207 reset_desk(ship, desk)
2209 ship.thread(fill(UNMOUNT, DESK=desk))
2210 rev = sync(ship, 'my-app', timeout=args.timeout, quiet=True)
2211 return ['%%%s and %%%s hold only what |new-desk gives and are '
2212 'unmounted (Clay keeps desks); my-app matches the repo at '
2213 'revision %d' % (CHECK_OURS, CHECK_THEIRS, rev)]
2215 check('sync my-app: |commit copies the repo onto the desk', synced)
2216 check('every /ui .jibb file reads back with %cx byte for byte',
2218 check('%jibb converts to %mime and back, and to %txt and back',
2220 check('half-written .jibb files still commit', half_written)
2221 check('a three-way merge of a .jibb file merges by line', merges_by_line)
2222 check('changes to the same line conflict', conflicts_by_line)
2223 check('clean up', clean_up)
2224 failed = [n for n, ok in results if not ok]
2225 say('%s: %d of %d checks passed (%.1fs)' % (
2226 'FAIL' if failed else 'PASS', len(results) - len(failed),
2227 len(results), time.time() - started))
2228 return 1 if failed else 0
2231def prepare_pair(ship, rel, base, timeout):
2232 """Reset the merge checks' desks: ours holds my-app's marks and the
2233 file, in a commit without parents; theirs takes it with a merge, so
2234 that ours's commit is the base of the next merge between them."""
2235 _mounts, revs = desks_info(ship)
2236 reset_desk(ship, CHECK_OURS, {rel: base}, marks_from='my-app')
2237 germ = 'only-that' if revs.get(CHECK_THEIRS) else 'init'
2238 ok, said = merge_and_wait(ship, CHECK_THEIRS, CHECK_OURS, germ, timeout)
2239 if not ok or read_files(ship, CHECK_THEIRS, [rel])[rel] != base:
2240 raise Fail('|merge %%%s our %%%s, =gem %%%s did not give it the file:'
2241 '\n %s' % (CHECK_THEIRS, CHECK_OURS, germ,
2242 '\n '.join(said[:20])))
2243 for desk in (CHECK_OURS, CHECK_THEIRS):
2244 mount = ensure_desk(ship, desk, quiet=True)
2245 wait_until('the mount of %s to hold the file' % desk,
2246 lambda: _read(os.path.join(mount, rel)) == base, timeout)
2251 with open(path, 'rb') as f:
2257# ---------------------------------------------------------------- main
2261 my-app vendor/* + jibb-dev (without any tests/) + my-app
2262 jibb vendor/* + jibb
2263 jibb-dev vendor/* + jibb-dev + tests, the test desk: tests/golden from
2264 --golden, tests/lib/jibb.hoon generated from it, and sys.kelvin
2266A sync creates the desk if it's missing (as |new-desk does), mounts it at
2267<pier>/<desk>, makes the mount hold exactly the desk's files, runs
2268|commit, and waits until the desk matches the repo, checking every file.
2270Exit status: 0 on success, 1 when something failed, 2 when the command
2271couldn't run. status exits 3 when the ship isn't running.
2276 ap = argparse.ArgumentParser(
2277 prog='tools/ship', description=__doc__.split('\n\n')[0],
2278 epilog=EPILOG, formatter_class=argparse.RawDescriptionHelpFormatter)
2279 ap.add_argument('--pier', default=PIER,
2280 help='the pier (default %(default)s, or $JIBB_PIER)')
2281 ap.add_argument('--log', default=os.environ.get('JIBB_SHIP_LOG'),
2282 help="the ship's log (default <pier>.log, or $JIBB_SHIP_LOG)")
2283 ap.add_argument('--click', default=CLICK,
2284 help='click (default %(default)s, or $JIBB_CLICK)')
2285 sub = ap.add_subparsers(dest='command', metavar='command')
2288 p = sub.add_parser('start', help='start the ship in the background')
2289 p.add_argument('--timeout', type=int, default=600)
2290 p = sub.add_parser('stop', help='stop the ship')
2291 p.add_argument('--timeout', type=int, default=120)
2292 p.add_argument('--force', action='store_true',
2293 help='kill it if it does not stop in time')
2294 sub.add_parser('status', help='whether it runs; its desks and mounts')
2295 p = sub.add_parser('log', help="the end of the ship's log")
2296 p.add_argument('-n', '--lines', type=int, default=40)
2297 p.add_argument('--all', action='store_true',
2298 help="keep docket's and kiln's network noise")
2300 p = sub.add_parser('sync', help='assemble desks from their folders and '
2302 p.add_argument('desks', nargs='+', metavar='desk')
2303 g = p.add_mutually_exclusive_group()
2304 g.add_argument('--stub', action='store_true',
2305 help='use tools/testdata/ship-stub.hoon as /lib/jibb.hoon')
2306 g.add_argument('--lib', help='use this file as /lib/jibb.hoon')
2307 p.add_argument('--golden', help='the golden cases for the jibb-dev desk '
2308 '(default tests/golden)')
2309 p.add_argument('--timeout', type=int, default=120,
2310 help='seconds to wait for the commit')
2312 p = sub.add_parser('test', help='sync the jibb-dev desk and run -test '
2314 g = p.add_mutually_exclusive_group()
2315 g.add_argument('--stub', action='store_true',
2316 help='use tools/testdata/ship-stub.hoon as /lib/jibb.hoon')
2317 g.add_argument('--lib', help='use this file as /lib/jibb.hoon')
2318 p.add_argument('--golden', help='test these golden cases instead of '
2319 'tests/golden (and leave tests/lib/jibb.hoon alone)')
2320 p.add_argument('--no-sync', action='store_true',
2321 help='run -test on the desk as it is')
2322 p.add_argument('--timeout', type=int, default=3600)
2324 p = sub.add_parser('gen-tests', help='write tests/lib/jibb.hoon from '
2326 p.add_argument('--golden', help='the cases (default tests/golden)')
2327 p.add_argument('--out', help='the file (default tests/lib/jibb.hoon)')
2328 p.add_argument('--check', action='store_true',
2329 help="don't write; exit 1 if the file is out of date")
2331 p = sub.add_parser('check-commit', help='prove that |commit stores '
2332 '.jibb files, and leave the desks clean')
2333 p.add_argument('--timeout', type=int, default=120)
2335 p = sub.add_parser('scry', help='scry, as /cx/=my-app=/ui/about/jibb')
2336 p.add_argument('path')
2337 p.add_argument('--type', default='*', help='the mold (default *)')
2338 p.add_argument('--raw', action='store_true',
2339 help="write an atom's bytes, as a file's")
2340 p.add_argument('--width', type=int, default=100)
2342 p = sub.add_parser('hoon', help='evaluate a Hoon expression, with our, '
2344 p.add_argument('expr', nargs='?')
2345 p.add_argument('-f', '--file', help='read the expression from a file')
2346 p.add_argument('--width', type=int, default=100)
2348 p = sub.add_parser('run', help='run a thread: a file holding a strand')
2349 p.add_argument('file')
2350 p.add_argument('--timeout', type=int, default=600)
2351 p.add_argument('--width', type=int, default=100)
2353 p = sub.add_parser('poke', help="poke an agent, as dojo's :agent &mark "
2355 p.add_argument('agent')
2356 p.add_argument('mark')
2357 p.add_argument('value')
2359 p = sub.add_parser('commit', help='|commit a desk, and wait for the '
2360 'new revision or the error')
2361 p.add_argument('desk')
2362 p.add_argument('--timeout', type=int, default=10)
2363 for name, what in (('mount', '|mount a desk, creating it if missing'),
2364 ('unmount', '|unmount a desk')):
2365 p = sub.add_parser(name, help=what)
2366 p.add_argument('desk')
2368 args = ap.parse_args(argv)
2369 ship = Ship(args.pier, args.log, args.click)
2371 'start': lambda: ship.start(args.timeout),
2372 'stop': lambda: ship.stop(args.timeout, args.force),
2373 'status': lambda: cmd_status(ship, args),
2374 'log': lambda: cmd_log(ship, args),
2375 'sync': lambda: cmd_sync(ship, args),
2376 'test': lambda: cmd_test(ship, args),
2377 'gen-tests': lambda: cmd_gen_tests(ship, args),
2378 'check-commit': lambda: cmd_check_commit(ship, args),
2379 'scry': lambda: cmd_scry(ship, args),
2380 'hoon': lambda: cmd_hoon(ship, args),
2381 'run': lambda: cmd_run(ship, args),
2382 'poke': lambda: cmd_poke(ship, args),
2383 'commit': lambda: cmd_commit(ship, args),
2384 'mount': lambda: cmd_mount(ship, args),
2385 'unmount': lambda: cmd_unmount(ship, args),
2388 return handlers[args.command]() or 0
2390 print('ship: %s' % e, file=sys.stderr)
2393 print('ship: %s' % e, file=sys.stderr)
2395 except KeyboardInterrupt:
2399if __name__ == '__main__':
2400 sys.exit(main(sys.argv[1:]))
keysesc to close
- j k
- move down, up
- enter
- open
- esc
- back, or up a level
- 1 2 3 4
- readme, files, commits, refs
- /
- filter
- y
- copy the clone command
- c
- compare the chosen commit with another (or shift-click both)
- ?
- show or hide this list