From 68491420d2a350e948e3a17ed3b8529665bba790 Mon Sep 17 00:00:00 2001 From: Fang Date: Thu, 4 Jun 2020 17:12:35 +0200 Subject: [PATCH 01/32] eyre: refactor %delete to reuse timeout logic Turns +on-channel-timeout into +discard-channel, which cleans up the entirety of the channel, based on its current state. This allows us to simplify the %delete channel request into a simple function call. --- pkg/arvo/sys/vane/eyre.hoon | 78 +++++++++++-------------------------- 1 file changed, 22 insertions(+), 56 deletions(-) diff --git a/pkg/arvo/sys/vane/eyre.hoon b/pkg/arvo/sys/vane/eyre.hoon index 43c3d75967..62e402cc12 100644 --- a/pkg/arvo/sys/vane/eyre.hoon +++ b/pkg/arvo/sys/vane/eyre.hoon @@ -1486,50 +1486,10 @@ $(requests t.requests) :: %delete - =/ unitsession - (~(get by session.channel-state.state) channel-id) - :: - ?~ unitsession - $(requests t.requests) - :: - =/ session u.unitsession - =. session.channel-state.state - (~(del by session.channel-state.state) channel-id) - :: + =^ moves state + (discard-channel channel-id |) =. gall-moves - %+ weld gall-moves - :: - :: produce a list of moves which cancels every gall subscription - :: - %+ turn ~(tap by subscriptions.session) - |= [channel-wire=wire ship=@p app=term =path duc=^duct] - ^- move - :: - [duc %pass channel-wire [%g %deal [our ship] app %leave ~]] - :: - ?: ?=([%& *] state.session) - =. gall-moves - :_ gall-moves - :: - ^- move - ?> ?=([%& *] state.session) - :^ duct.p.state.session %pass /channel/timeout/[channel-id] - [%b %rest date.p.state.session] - :: - $(requests t.requests) - :: - ?> ?=([%| *] state.session) - =. duct-to-key.channel-state.state - (~(del by duct-to-key.channel-state.state) p.state.session) - :: - ?~ heartbeat.session $(requests t.requests) - =. gall-moves - %+ snoc gall-moves - %^ cancel-heartbeat-move - channel-id - date.u.heartbeat.session - duct.u.heartbeat.session - :: + (weld gall-moves moves) $(requests t.requests) :: == @@ -1671,10 +1631,12 @@ channel(heartbeat (some [heartbeat-time duct])) == (snoc http-moves (set-heartbeat-move channel-id heartbeat-time)) - :: +on-channel-timeout: we received a wake to clear an old session + :: +discard-channel: remove a channel from state :: - ++ on-channel-timeout - |= channel-id=@t + :: cleans up state, timers, and gall subscriptions of the channel + :: + ++ discard-channel + |= [channel-id=@t expired=?] ^- [(list move) server-state] :: =/ usession=(unit channel) @@ -1686,6 +1648,10 @@ :_ %_ state session.channel-state (~(del by session.channel-state.state) channel-id) + :: + duct-to-key.channel-state + ?. ?=(%| -.state.session) duct-to-key.channel-state.state + (~(del by duct-to-key.channel-state.state) p.state.session) == =/ heartbeat-cancel=(list move) ?~ heartbeat.session ~ @@ -1694,7 +1660,13 @@ date.u.heartbeat.session duct.u.heartbeat.session == + =/ expire-cancel=(list move) + ?: expired ~ + ?. ?=(%& -.state.session) ~ + =, p.state.session + [(cancel-timeout-move channel-id date duct)]~ %+ weld heartbeat-cancel + %+ weld expire-cancel :: produce a list of moves which cancels every gall subscription :: %+ turn ~(tap by subscriptions.session) @@ -2108,13 +2080,7 @@ [(zing (flop moves)) http-server-gate] :: discard channel state, and cancel any active gall subscriptions :: - =^ mov server-state.ax (on-channel-timeout:by-channel channel-id) - :: cancel channel timer - :: - =/ channel (~(got by session.channel-state) channel-id) - =? mov ?=([%& *] state.channel) - :_ mov - (cancel-timeout-move:by-channel channel-id p.state.channel) + =^ mov server-state.ax (discard-channel:by-channel channel-id |) $(moves [mov moves], inactive t.inactive) :: :: %vega: notifies us of a completed kernel upgrade @@ -2362,10 +2328,10 @@ ?> ?=([%b %wake *] sign) ?^ error.sign [[duct %slip %d %flog %crud %wake u.error.sign]~ http-server-gate] - =/ on-channel-timeout - on-channel-timeout:by-channel:(per-server-event event-args) + =/ discard-channel + discard-channel:by-channel:(per-server-event event-args) =^ moves server-state.ax - (on-channel-timeout i.t.t.wire) + (discard-channel i.t.t.wire &) [moves http-server-gate] :: %heartbeat From b54dc7cd345bf430d44dbdc9dd316698de67f8e1 Mon Sep 17 00:00:00 2001 From: Fang Date: Thu, 4 Jun 2020 22:06:13 +0200 Subject: [PATCH 02/32] eyre, zuse: expire channels with their sessions Associates channels with the authentication sessions that opened them, and deletes the channel when its associated session expires. Also updates the debug dashboard to display channel counts per session. --- bin/solid.pill | 4 +- pkg/arvo/app/dbug.hoon | 5 +- pkg/arvo/sys/vane/eyre.hoon | 71 +++++++++++++++++++------ pkg/arvo/sys/zuse.hoon | 3 ++ pkg/interface/dbug/src/js/views/eyre.js | 2 +- 5 files changed, 64 insertions(+), 21 deletions(-) diff --git a/bin/solid.pill b/bin/solid.pill index f10d602213..e6a2605f75 100644 --- a/bin/solid.pill +++ b/bin/solid.pill @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:dfaee098f2dca396c17aa1ddf1d2755f5f7d8639864b0e970fce0290d67008cc -size 20306202 +oid sha256:9f7988e3ffc99ecd8cb66daf469bff3b9cb41ca82895a91cae7d4f7d59bb48a7 +size 20308420 diff --git a/pkg/arvo/app/dbug.hoon b/pkg/arvo/app/dbug.hoon index 90124315bd..a45e8d6a2b 100644 --- a/pkg/arvo/app/dbug.hoon +++ b/pkg/arvo/app/dbug.hoon @@ -360,12 +360,13 @@ :- %a %+ turn %+ sort ~(tap by sessions:auth-state:v-eyre) - |= [[@uv a=@da] [@uv b=@da]] - (gth a b) + |= [[@uv a=session:eyre] [@uv b=session:eyre]] + (gth expiry-time.a expiry-time.b) |= [cookie=@uv session:eyre] %- pairs :~ 'cookie'^s+(end 3 4 (rsh 3 2 (scot %x (shax cookie)))) 'expiry'^(time expiry-time) + 'channels'^(numb ~(wyt in channels)) == :: :: /eyre/channels.json diff --git a/pkg/arvo/sys/vane/eyre.hoon b/pkg/arvo/sys/vane/eyre.hoon index 62e402cc12..64281742ce 100644 --- a/pkg/arvo/sys/vane/eyre.hoon +++ b/pkg/arvo/sys/vane/eyre.hoon @@ -969,7 +969,7 @@ =/ first-session=? =(~ sessions.authentication-state.state) =/ expires-at=@da (add now session-timeout) =. sessions.authentication-state.state - (~(put by sessions.authentication-state.state) session expires-at) + (~(put by sessions.authentication-state.state) session [expires-at ~]) :: =/ cookie-line=@t (session-cookie-string session &) @@ -1026,15 +1026,33 @@ (handle-response response) :: delete the requesting session, or all sessions if so specified :: - =. sessions.authentication-state.state - =; all=? - ?: all ~ - (~(del by sessions.authentication-state.state) u.session-id) - ?~ body.request | - =- ?=(^ -) - %+ get-header:http 'all' - (fall (rush q.u.body.request yquy:de-purl:html) ~) - (handle-response response) + =^ channels=(list @t) sessions.authentication-state.state + =* sessions sessions.authentication-state.state + =/ all=? + ?~ body.request | + =- ?=(^ -) + %+ get-header:http 'all' + (fall (rush q.u.body.request yquy:de-purl:html) ~) + ?. all + :_ (~(del by sessions) u.session-id) + %~ tap in + channels:(~(gut by sessions) u.session-id *session) + :_ ~ + %~ tap in + %+ roll ~(val by sessions) + |= [session all=(set @t)] + (~(uni in all) channels) + :: close all affected channels, then send the response + :: + =| moves=(list move) + |- ^- (quip move server-state) + ?~ channels + =^ moz state + (handle-response response) + [(weld moves moz) state] + =^ moz state + (discard-channel:by-channel i.channels |) + $(moves (weld moves moz), channels t.channels) :: +session-id-from-request: attempt to find a session cookie :: ++ session-id-from-request @@ -1325,6 +1343,16 @@ :: =. duct-to-key.channel-state.state (~(put by duct-to-key.channel-state.state) duct channel-id) + :: associate this channel with the session cookie + :: + =. sessions.authentication-state.state + =/ session-id=(unit @uv) + (session-id-from-request:authentication request) + ?~ session-id sessions.authentication-state.state + %+ ~(jab by sessions.authentication-state.state) + u.session-id + |= =session + session(channels (~(put in channels.session) channel-id)) :: initialize sse heartbeat :: =/ heartbeat-time=@da (add now ~s20) @@ -1796,7 +1824,9 @@ :: tough luck, we don't create/revive sessions here :: no-op - :_ (~(put by sessions) u.session-id (add now session-timeout)) + :_ %+ ~(jab by sessions) u.session-id + |= =session + session(expiry-time (add now session-timeout)) =- response-header.http-event(headers -) %^ set-header:http 'set-cookie' (session-cookie-string u.session-id &) @@ -2363,11 +2393,9 @@ =* sessions sessions.authentication-state.server-state.ax =. sessions.authentication-state.server-state.ax %- ~(gas by *(map @uv session)) - %+ murn ~(tap in sessions) + %+ skip ~(tap in sessions) |= [cookie=@uv session] - ^- (unit [@uv session]) - ?: (lth expiry-time now) ~ - `[cookie expiry-time] + (lth expiry-time now) :: if there's any cookies left, set a timer for the next expected expiry :: ^- [(list move) _http-server-gate] @@ -2399,7 +2427,18 @@ ++ load => |% +$ axle-2019-10-6 - [date=%~2019.10.6 =server-state] + [date=%~2019.10.6 server-state=server-state-2019-10-6] + :: + +$ server-state-2019-10-6 + $: bindings=(list [=binding =duct =action]) + connections=(map duct outstanding-connection) + authentication-state=sessions=(map @uv @da) + =channel-state + domains=(set turf) + =http-config + ports=[insecure=@ud secure=(unit @ud)] + outgoing-duct=duct + == -- |= old=$%(axle axle-2019-10-6) ^+ ..^$ diff --git a/pkg/arvo/sys/zuse.hoon b/pkg/arvo/sys/zuse.hoon index 6c4a2c11b1..99750fa3c9 100644 --- a/pkg/arvo/sys/zuse.hoon +++ b/pkg/arvo/sys/zuse.hoon @@ -1213,6 +1213,9 @@ :: to properly handle cookie expiration as a security mechanism. :: expiry-time=@da + :: channels: channels opened by this session + :: + channels=(set @t) :: :: TODO: We should add a system for individual capabilities; we should :: mint some sort of long lived cookie for mobile apps which only has diff --git a/pkg/interface/dbug/src/js/views/eyre.js b/pkg/interface/dbug/src/js/views/eyre.js index 88e4bde911..738a685cf1 100644 --- a/pkg/interface/dbug/src/js/views/eyre.js +++ b/pkg/interface/dbug/src/js/views/eyre.js @@ -146,7 +146,7 @@ export class Eyre extends Component { const sessionItems = props.authentication.map(s => { return (
- {`${s.cookie} expires ${msToDa(s.expiry)}`} + {`${s.cookie} expires ${msToDa(s.expiry)}, uses ${s.channels} channel(s)`}
); }); From a4785458d156eed0a0d32c0a2b6d294acae1527c Mon Sep 17 00:00:00 2001 From: Fang Date: Wed, 10 Jun 2020 20:37:12 +0200 Subject: [PATCH 03/32] eyre: don't lose redirect upon failing login If we failed the password check, the login page served to us would never include any redirect details, even if they were there in the original request. Now we simply (attempt to) parse out the redirect field a little earlier. --- pkg/arvo/sys/vane/eyre.hoon | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/arvo/sys/vane/eyre.hoon b/pkg/arvo/sys/vane/eyre.hoon index f5304226e7..5cb26cdf35 100644 --- a/pkg/arvo/sys/vane/eyre.hoon +++ b/pkg/arvo/sys/vane/eyre.hoon @@ -950,12 +950,13 @@ ?~ parsed (return-static-data-on-duct 400 'text/html' (login-page ~ our)) :: + =/ redirect=(unit @t) (get-header:http 'redirect' u.parsed) ?~ password=(get-header:http 'password' u.parsed) - (return-static-data-on-duct 400 'text/html' (login-page ~ our)) + (return-static-data-on-duct 400 'text/html' (login-page redirect our)) :: check that the password is correct :: ?. =(u.password code) - (return-static-data-on-duct 400 'text/html' (login-page ~ our)) + (return-static-data-on-duct 400 'text/html' (login-page redirect our)) :: mint a unique session cookie :: =/ session=@uv @@ -981,7 +982,7 @@ =- out(moves [- moves.out]) [duct %pass /sessions/expire %b %wait expires-at] :: - ?~ redirect=(get-header:http 'redirect' u.parsed) + ?~ redirect %- handle-response :* %start :- status-code=200 From e35bb4b72a945716f61351f805029386d45819af Mon Sep 17 00:00:00 2001 From: Fang Date: Thu, 18 Jun 2020 21:48:35 +0200 Subject: [PATCH 04/32] pill: update --- bin/solid.pill | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/solid.pill b/bin/solid.pill index e6a2605f75..073e2fbbeb 100644 --- a/bin/solid.pill +++ b/bin/solid.pill @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9f7988e3ffc99ecd8cb66daf469bff3b9cb41ca82895a91cae7d4f7d59bb48a7 -size 20308420 +oid sha256:2c0c3e34876875722a8505d8e98843738460d6d8b3ab8872274a876f7859ae71 +size 17336122 From d8aed4d4afa365fcec2d0123d2c699073b706cee Mon Sep 17 00:00:00 2001 From: Philip Monk Date: Mon, 22 Jun 2020 17:21:59 -0700 Subject: [PATCH 05/32] noun: add functions to count size of noun Adds a few functions to count the size of nouns in the current road. Since this marks the nouns (high bit of refcount), you need to "discount" them immediately after to unmark them. Parallel functions exist for the counting the size of a hashtable. It would nice to hook this up to a hint, but these are useful to have available to run in the debugger or by inserting callsites as necessary. It's also possible to hook them up to the +jam jet gated on a special value. --- pkg/urbit/include/noun/allocate.h | 18 ++++ pkg/urbit/include/noun/hashtable.h | 10 ++ pkg/urbit/noun/allocate.c | 150 +++++++++++++++++++++++++++ pkg/urbit/noun/hashtable.c | 160 +++++++++++++++++++++++++++++ 4 files changed, 338 insertions(+) diff --git a/pkg/urbit/include/noun/allocate.h b/pkg/urbit/include/noun/allocate.h index 0afff67213..934b70328b 100644 --- a/pkg/urbit/include/noun/allocate.h +++ b/pkg/urbit/include/noun/allocate.h @@ -465,6 +465,24 @@ c3_w u3a_mark_road(FILE* fil_u); + /* u3a_count_noun(): count size of noun. + */ + c3_w + u3a_count_noun(u3_noun som); + + /* u3a_discount_noun(): clean up after counting a noun. + */ + c3_w + u3a_discount_noun(u3_noun som); + + /* u3a_count_ptr(): count a pointer for gc. Produce size. */ + c3_w + u3a_count_ptr(void* ptr_v); + + /* u3a_discount_ptr(): discount a pointer for gc. Produce size. */ + c3_w + u3a_discount_ptr(void* ptr_v); + /* u3a_idle(): measure free-lists in [rod_u] */ c3_w diff --git a/pkg/urbit/include/noun/hashtable.h b/pkg/urbit/include/noun/hashtable.h index 793519d52d..a6457ad951 100644 --- a/pkg/urbit/include/noun/hashtable.h +++ b/pkg/urbit/include/noun/hashtable.h @@ -139,6 +139,16 @@ c3_w u3h_mark(u3p(u3h_root) har_p); + /* u3h_count(): count hashtable for gc. + */ + c3_w + u3h_count(u3p(u3h_root) har_p); + + /* u3h_discount(): discount hashtable for gc. + */ + c3_w + u3h_discount(u3p(u3h_root) har_p); + /* u3h_walk_with(): traverse hashtable with key, value fn and data * argument; RETAINS. */ diff --git a/pkg/urbit/noun/allocate.c b/pkg/urbit/noun/allocate.c index 13706bba20..ea44ef1684 100644 --- a/pkg/urbit/noun/allocate.c +++ b/pkg/urbit/noun/allocate.c @@ -1721,6 +1721,156 @@ u3a_mark_noun(u3_noun som) } } +/* u3a_count_noun(): count size of pointer. +*/ +c3_w +u3a_count_ptr(void* ptr_v) +{ + if ( _(u3a_is_north(u3R)) ) { + if ( !((ptr_v >= u3a_into(u3R->rut_p)) && + (ptr_v < u3a_into(u3R->hat_p))) ) + { + return 0; + } + } + else { + if ( !((ptr_v >= u3a_into(u3R->hat_p)) && + (ptr_v < u3a_into(u3R->rut_p))) ) + { + return 0; + } + } + { + u3a_box* box_u = u3a_botox(ptr_v); + c3_w siz_w; + + c3_ws use_ws = (c3_ws)box_u->use_w; + + if ( use_ws == 0 ) { + fprintf(stderr, "%p is bogus\r\n", ptr_v); + siz_w = 0; + } + else { + c3_assert(use_ws != 0); + + if ( use_ws < 0 ) { + siz_w = 0; + } + else { + use_ws = -use_ws; + siz_w = box_u->siz_w; + } + box_u->use_w = (c3_w)use_ws; + } + return siz_w; + } +} + +/* u3a_count_noun(): count size of noun. +*/ +c3_w +u3a_count_noun(u3_noun som) +{ + c3_w siz_w = 0; + + while ( 1 ) { + if ( _(u3a_is_senior(u3R, som)) ) { + return siz_w; + } + else { + c3_w* dog_w = u3a_to_ptr(som); + c3_w new_w = u3a_count_ptr(dog_w); + + if ( 0 == new_w ) { + return siz_w; + } + else { + siz_w += new_w; + if ( _(u3du(som)) ) { + siz_w += u3a_count_noun(u3h(som)); + som = u3t(som); + } + else return siz_w; + } + } + } +} + +/* u3a_discount_ptr(): clean up after counting a pointer. +*/ +c3_w +u3a_discount_ptr(void* ptr_v) +{ + if ( _(u3a_is_north(u3R)) ) { + if ( !((ptr_v >= u3a_into(u3R->rut_p)) && + (ptr_v < u3a_into(u3R->hat_p))) ) + { + return 0; + } + } + else { + if ( !((ptr_v >= u3a_into(u3R->hat_p)) && + (ptr_v < u3a_into(u3R->rut_p))) ) + { + return 0; + } + } + u3a_box* box_u = u3a_botox(ptr_v); + c3_w siz_w; + + c3_ws use_ws = (c3_ws)box_u->use_w; + + if ( use_ws == 0 ) { + fprintf(stderr, "%p is bogus\r\n", ptr_v); + siz_w = 0; + } + else { + c3_assert(use_ws != 0); + + if ( use_ws < 0 ) { + use_ws = -use_ws; + siz_w = box_u->siz_w; + } + else { + siz_w = 0; + } + box_u->use_w = (c3_w)use_ws; + } + + return siz_w; +} + +/* u3a_discount_noun(): clean up after counting a noun. +*/ +c3_w +u3a_discount_noun(u3_noun som) +{ + c3_w siz_w = 0; + + while ( 1 ) { + if ( _(u3a_is_senior(u3R, som)) ) { + return siz_w; + } + else { + c3_w* dog_w = u3a_to_ptr(som); + c3_w new_w = u3a_discount_ptr(dog_w); + + if ( 0 == new_w ) { + return siz_w; + } + else { + siz_w += new_w; + if ( _(u3du(som)) ) { + siz_w += u3a_discount_noun(u3h(som)); + som = u3t(som); + } + else return siz_w; + } + } + } +} + + /* u3a_print_memory: print memory amount. */ void diff --git a/pkg/urbit/noun/hashtable.c b/pkg/urbit/noun/hashtable.c index 95a6f0d4e5..3eb7152beb 100644 --- a/pkg/urbit/noun/hashtable.c +++ b/pkg/urbit/noun/hashtable.c @@ -941,3 +941,163 @@ u3h_mark(u3p(u3h_root) har_p) return tot_w; } + +/* _ch_count_buck(): count bucket for gc. +*/ +c3_w +_ch_count_buck(u3h_buck* hab_u) +{ + c3_w tot_w = 0; + c3_w i_w; + + for ( i_w = 0; i_w < hab_u->len_w; i_w++ ) { + tot_w += u3a_count_noun(u3h_slot_to_noun(hab_u->sot_w[i_w])); + } + tot_w += u3a_count_ptr(hab_u); + + return tot_w; +} + +/* _ch_count_node(): count node for gc. +*/ +c3_w +_ch_count_node(u3h_node* han_u, c3_w lef_w) +{ + c3_w tot_w = 0; + c3_w len_w = _ch_popcount(han_u->map_w); + c3_w i_w; + + lef_w -= 5; + + for ( i_w = 0; i_w < len_w; i_w++ ) { + c3_w sot_w = han_u->sot_w[i_w]; + + if ( _(u3h_slot_is_noun(sot_w)) ) { + u3_noun kev = u3h_slot_to_noun(sot_w); + + tot_w += u3a_count_noun(kev); + } + else { + void* hav_v = u3h_slot_to_node(sot_w); + + if ( 0 == lef_w ) { + tot_w += _ch_count_buck(hav_v); + } else { + tot_w += _ch_count_node(hav_v, lef_w); + } + } + } + + tot_w += u3a_count_ptr(han_u); + + return tot_w; +} + +/* u3h_count(): count hashtable for gc. +*/ +c3_w +u3h_count(u3p(u3h_root) har_p) +{ + c3_w tot_w = 0; + u3h_root* har_u = u3to(u3h_root, har_p); + c3_w i_w; + + for ( i_w = 0; i_w < 64; i_w++ ) { + c3_w sot_w = har_u->sot_w[i_w]; + + if ( _(u3h_slot_is_noun(sot_w)) ) { + u3_noun kev = u3h_slot_to_noun(sot_w); + + tot_w += u3a_count_noun(kev); + } + else if ( _(u3h_slot_is_node(sot_w)) ) { + u3h_node* han_u = u3h_slot_to_node(sot_w); + + tot_w += _ch_count_node(han_u, 25); + } + } + + tot_w += u3a_count_ptr(har_u); + + return tot_w; +} + +/* _ch_discount_buck(): discount bucket for gc. +*/ +c3_w +_ch_discount_buck(u3h_buck* hab_u) +{ + c3_w tot_w = 0; + c3_w i_w; + + for ( i_w = 0; i_w < hab_u->len_w; i_w++ ) { + tot_w += u3a_discount_noun(u3h_slot_to_noun(hab_u->sot_w[i_w])); + } + tot_w += u3a_discount_ptr(hab_u); + + return tot_w; +} + +/* _ch_discount_node(): discount node for gc. +*/ +c3_w +_ch_discount_node(u3h_node* han_u, c3_w lef_w) +{ + c3_w tot_w = 0; + c3_w len_w = _ch_popcount(han_u->map_w); + c3_w i_w; + + lef_w -= 5; + + for ( i_w = 0; i_w < len_w; i_w++ ) { + c3_w sot_w = han_u->sot_w[i_w]; + + if ( _(u3h_slot_is_noun(sot_w)) ) { + u3_noun kev = u3h_slot_to_noun(sot_w); + + tot_w += u3a_discount_noun(kev); + } + else { + void* hav_v = u3h_slot_to_node(sot_w); + + if ( 0 == lef_w ) { + tot_w += _ch_discount_buck(hav_v); + } else { + tot_w += _ch_discount_node(hav_v, lef_w); + } + } + } + + tot_w += u3a_discount_ptr(han_u); + + return tot_w; +} + +/* u3h_discount(): discount hashtable for gc. +*/ +c3_w +u3h_discount(u3p(u3h_root) har_p) +{ + c3_w tot_w = 0; + u3h_root* har_u = u3to(u3h_root, har_p); + c3_w i_w; + + for ( i_w = 0; i_w < 64; i_w++ ) { + c3_w sot_w = har_u->sot_w[i_w]; + + if ( _(u3h_slot_is_noun(sot_w)) ) { + u3_noun kev = u3h_slot_to_noun(sot_w); + + tot_w += u3a_discount_noun(kev); + } + else if ( _(u3h_slot_is_node(sot_w)) ) { + u3h_node* han_u = u3h_slot_to_node(sot_w); + + tot_w += _ch_discount_node(han_u, 25); + } + } + + tot_w += u3a_discount_ptr(har_u); + + return tot_w; +} From fa6748a7e1a8789374c77322e586b6612cab893c Mon Sep 17 00:00:00 2001 From: Philip Monk Date: Mon, 22 Jun 2020 18:53:32 -0700 Subject: [PATCH 06/32] jets: cap memo cache and remove peek, play, and fond jets With these changes, about 90% less memory and 15% less time is needed to compile hoon.hoon. The produced noun is within 3% of the same size, which suggests this results in little if any duplication. These are three of the four most commonly hit +ut jets. The other is +nest, which cannot be un-memoized without taking much longer to compile (it didn't finish in my test). These four jets combined for 2.3 million out of the 2.4 million cache entries, the other +ut jets combine for less than 100k, and literal ~+ accounted for about 50k entries. This also caps the memo cache at 50k entries. Even with these jets not memoized, the memo cache grows to 357k entries and 122 MB. Capping at 50k entries has no effect on time and reduces memory usage of the hash table to about 25MB. Entries are reclaimed with the clock algorithm, which seems to be sufficient for this use. --- pkg/urbit/jets/f/ut_fond.c | 34 ---------------------------------- pkg/urbit/jets/f/ut_peek.c | 34 ---------------------------------- pkg/urbit/jets/f/ut_play.c | 31 ------------------------------- pkg/urbit/jets/tree.c | 21 --------------------- pkg/urbit/noun/manage.c | 2 +- 5 files changed, 1 insertion(+), 121 deletions(-) delete mode 100644 pkg/urbit/jets/f/ut_fond.c delete mode 100644 pkg/urbit/jets/f/ut_peek.c delete mode 100644 pkg/urbit/jets/f/ut_play.c diff --git a/pkg/urbit/jets/f/ut_fond.c b/pkg/urbit/jets/f/ut_fond.c deleted file mode 100644 index 588b899314..0000000000 --- a/pkg/urbit/jets/f/ut_fond.c +++ /dev/null @@ -1,34 +0,0 @@ -/* j/6/find.c -** -*/ -#include "all.h" - -u3_noun -u3wfu_fond(u3_noun cor) -{ - u3_noun sut, way, hyp, van; - - if ( (c3n == u3r_mean(cor, u3x_sam_2, &way, - u3x_sam_3, &hyp, - u3x_con, &van, - 0)) || - (u3_none == (sut = u3r_at(u3x_sam, van))) ) - { - return u3m_bail(c3__fail); - } - else { - c3_m fun_m = 141 + c3__fond + ((!!u3r_at(u3qfu_van_vet, van)) << 8); - u3_noun key = u3z_key_3(fun_m, sut, way, hyp); - u3_weak pro = u3z_find(key); - - if ( u3_none != pro ) { - u3z(key); - return pro; - } - else { - pro = u3n_nock_on(u3k(cor), u3k(u3x_at(u3x_bat, cor))); - return u3z_save(key, pro); - } - } -} - diff --git a/pkg/urbit/jets/f/ut_peek.c b/pkg/urbit/jets/f/ut_peek.c deleted file mode 100644 index cd9ba31783..0000000000 --- a/pkg/urbit/jets/f/ut_peek.c +++ /dev/null @@ -1,34 +0,0 @@ -/* j/6/peek.c -** -*/ -#include "all.h" - -u3_noun -u3wfu_peek(u3_noun cor) -{ - u3_noun sut, way, axe, van; - - if ( (c3n == u3r_mean(cor, u3x_sam_2, &way, - u3x_sam_3, &axe, - u3x_con, &van, - 0)) || - (c3n == u3ud(axe)) || - (u3_none == (sut = u3r_at(u3x_sam, van))) ) - { - return u3m_bail(c3__fail); - } - else { - c3_m fun_m = 141 + c3__peek + ((!!u3r_at(u3qfu_van_vet, van)) << 8); - u3_noun key = u3z_key_3(fun_m, sut, way, axe); - u3_weak pro = u3z_find(key); - - if ( u3_none != pro ) { - u3z(key); - return pro; - } - else { - pro = u3n_nock_on(u3k(cor), u3k(u3x_at(u3x_bat, cor))); - return u3z_save(key, pro); - } - } -} diff --git a/pkg/urbit/jets/f/ut_play.c b/pkg/urbit/jets/f/ut_play.c deleted file mode 100644 index a11b0fae77..0000000000 --- a/pkg/urbit/jets/f/ut_play.c +++ /dev/null @@ -1,31 +0,0 @@ -/* j/6/play.c -** -*/ -#include "all.h" - -u3_noun -u3wfu_play(u3_noun cor) -{ - u3_noun sut, gen, van; - - if ( (c3n == u3r_mean(cor, u3x_sam, &gen, u3x_con, &van, 0)) || - (u3_none == (sut = u3r_at(u3x_sam, van))) ) - { - return u3m_bail(c3__fail); - } - else { - c3_m fun_m = 141 + c3__play; - u3_noun vrf = u3r_at(u3qfu_van_vrf, van); - u3_noun key = u3z_key_3(fun_m, vrf, sut, gen); - u3_weak pro = u3z_find(key); - - if ( u3_none != pro ) { - u3z(key); - return pro; - } - else { - pro = u3n_nock_on(u3k(cor), u3k(u3x_at(u3x_bat, cor))); - return u3z_save(key, pro); - } - } -} diff --git a/pkg/urbit/jets/tree.c b/pkg/urbit/jets/tree.c index d25b06ead5..b3a5dcd1e6 100644 --- a/pkg/urbit/jets/tree.c +++ b/pkg/urbit/jets/tree.c @@ -477,11 +477,6 @@ static c3_c* _141_pen_loot_ha[] = { "d83e5e47f712870aba815d79943d287cbefdc00640409464b30bf755115d4a1a", 0 }; - static u3j_harm _141_pen__ut_fond_a[] = {{".2", u3wfu_fond}, {}}; - static c3_c* _141_pen__ut_fond_ha[] = { - "0da0cc79c938eb06515a5cc24a17b82cd60a50c0f1a02e2c68e5d1cf71c96054", - 0 - }; static u3j_harm _141_pen__ut_fish_a[] = {{".2", u3wfu_fish}, {}}; static c3_c* _141_pen__ut_fish_ha[] = { "2fd315436f48351002d9aa8c137649ca95b01fd57dba09db53d7235f84a284bf", @@ -516,16 +511,6 @@ static c3_c* _141_pen_loot_ha[] = { { "nest-in", 7, 0, _141_pen__ut_nest_in_d, _141_pen__ut_nest_in_ha }, {} }; - static u3j_harm _141_pen__ut_peek_a[] = {{".2", u3wfu_peek}, {}}; - static c3_c* _141_pen__ut_peek_ha[] = { - "904ff7359e89d1886f884c4409f104269cdb8dfb4683f116ff00bc98a4720df7", - 0 - }; - static u3j_harm _141_pen__ut_play_a[] = {{".2", u3wfu_play}, {}}; - static c3_c* _141_pen__ut_play_ha[] = { - "bdc5c072632f7133b4c64c465b1b214d7465b0c1163842b121b7369aba1b9b03", - 0 - }; static u3j_harm _141_pen__ut_rest_a[] = {{".2", u3wfu_rest}, {}}; static c3_c* _141_pen__ut_rest_ha[] = { "2e2d15f3efca0a4bf8ce08cca48c54d1d5a7204e2b0525137f59c3e7b037d2fd", @@ -535,14 +520,11 @@ static c3_c* _141_pen_loot_ha[] = { static u3j_core _141_pen__ut_d[] = { { "crop", 7, _141_pen__ut_crop_a, 0, _141_pen__ut_crop_ha }, - { "fond", 7, _141_pen__ut_fond_a, 0, _141_pen__ut_fond_ha }, { "fish", 7, _141_pen__ut_fish_a, 0, _141_pen__ut_fish_ha }, { "fuse", 7, _141_pen__ut_fuse_a, 0, _141_pen__ut_fuse_ha }, { "mint", 7, _141_pen__ut_mint_a, 0, _141_pen__ut_mint_ha }, { "mull", 7, _141_pen__ut_mull_a, 0, _141_pen__ut_mull_ha }, { "nest", 7, 0, _141_pen__ut_nest_d, _141_pen__ut_nest_ha }, - { "peek", 7, _141_pen__ut_peek_a, 0, _141_pen__ut_peek_ha }, - { "play", 7, _141_pen__ut_play_a, 0, _141_pen__ut_play_ha }, { "rest", 7, _141_pen__ut_rest_a, 0, _141_pen__ut_rest_ha }, {} }; @@ -575,7 +557,6 @@ static u3j_hood _141_pen__ut_ho[] = { { "fine", 49086 }, { "fire", 4 }, { "fish", 6006 }, - { "fond", 12283 }, { "fund", 6014 }, // XX +funk is not part of +ut, and this hook appears to be unused // remove from here and the +ut hint @@ -591,8 +572,6 @@ static u3j_hood _141_pen__ut_ho[] = { { "mull", 24020 }, { "nest", 92 }, { "peel", 1526 }, - { "play", 3006 }, - { "peek", 1532 }, { "repo", 22 }, { "rest", 6102 }, { "tack", 6007 }, diff --git a/pkg/urbit/noun/manage.c b/pkg/urbit/noun/manage.c index 663d16c29e..b4c1968cbb 100644 --- a/pkg/urbit/noun/manage.c +++ b/pkg/urbit/noun/manage.c @@ -496,7 +496,7 @@ _pave_south(c3_w* mem_w, c3_w siz_w, c3_w len_w) static void _pave_parts(void) { - u3R->cax.har_p = u3h_new(); + u3R->cax.har_p = u3h_new_cache(50000); u3R->jed.war_p = u3h_new(); u3R->jed.cod_p = u3h_new(); u3R->jed.han_p = u3h_new(); From 6c3b7aeef50d3e4888310e972503796c9ccda08a Mon Sep 17 00:00:00 2001 From: Philip Monk Date: Mon, 22 Jun 2020 19:11:40 -0700 Subject: [PATCH 07/32] jam: add commented-out functionality to count size of atom This is a convenient way to count memory usage of noun by simplying running `(jam 1.337 noun-1 noun-2 ... ~)`. This should be a hint, but for debugging this is sufficient. --- pkg/urbit/jets/e/jam.c | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/pkg/urbit/jets/e/jam.c b/pkg/urbit/jets/e/jam.c index 2ce9faa785..1abf18a5be 100644 --- a/pkg/urbit/jets/e/jam.c +++ b/pkg/urbit/jets/e/jam.c @@ -6,6 +6,40 @@ u3_noun u3qe_jam(u3_atom a) { +#if 0 + if (c3y == u3du(a) && 1337 == u3h(a)) { + c3_w siz_w, tot_w = 0; + u3_noun som; + for ( som = u3t(a); c3y == u3du(som); som = u3t(som) ) { + siz_w = u3a_count_noun(u3h(som)); + tot_w += siz_w; + if ( 0 == siz_w ) { + u3l_log("item: B/0\r\n"); + } + else { + u3a_print_memory(stderr, "item", siz_w); + } + } + if ( u3_blip != som ) { + u3l_log("forgot to terminate list!\r\n"); + } + c3_w mem_w = u3h_count(u3R->cax.har_p); + + for ( som = u3t(a); c3y == u3du(som); som = u3t(som) ) u3a_discount_noun(u3h(som)); + u3h_discount(u3R->cax.har_p); + + u3a_print_memory(stderr, "total", tot_w); + u3a_print_memory(stderr, "memoization cache", mem_w); + u3h_root* har_u = u3to(u3h_root, u3R->cax.har_p); + u3l_log("memoization entries: %d\r\n", har_u->use_w); + c3_w diff = (u3R->hat_p < u3R->cap_p) ? + (u3R->cap_p - u3R->hat_p) : + (u3R->hat_p - u3R->cap_p); + u3a_print_memory(stderr, "unused free", diff); + return tot_w; + } +#endif + c3_w bit_w, *sal_w; c3_w* wor_w = u3s_jam_fib(a, &bit_w); c3_w len_w = bit_w >> 5; From 71fa03457cff302b254cac52247f13cf7cb6681d Mon Sep 17 00:00:00 2001 From: Philip Monk Date: Wed, 24 Jun 2020 13:42:51 -0700 Subject: [PATCH 08/32] jets: restore fond/play/peek hooks --- pkg/urbit/jets/tree.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/urbit/jets/tree.c b/pkg/urbit/jets/tree.c index b3a5dcd1e6..f92ca5a483 100644 --- a/pkg/urbit/jets/tree.c +++ b/pkg/urbit/jets/tree.c @@ -557,6 +557,7 @@ static u3j_hood _141_pen__ut_ho[] = { { "fine", 49086 }, { "fire", 4 }, { "fish", 6006 }, + { "fond", 12283 }, { "fund", 6014 }, // XX +funk is not part of +ut, and this hook appears to be unused // remove from here and the +ut hint @@ -572,6 +573,8 @@ static u3j_hood _141_pen__ut_ho[] = { { "mull", 24020 }, { "nest", 92 }, { "peel", 1526 }, + { "play", 3006 }, + { "peek", 1532 }, { "repo", 22 }, { "rest", 6102 }, { "tack", 6007 }, From 361e41e78d5e065412cada2aff7e90d334c3325b Mon Sep 17 00:00:00 2001 From: Philip Monk Date: Wed, 24 Jun 2020 17:08:50 -0700 Subject: [PATCH 09/32] noun: add -C to control memo cache size --- pkg/urbit/daemon/main.c | 13 ++++++++++--- pkg/urbit/include/vere/vere.h | 1 + pkg/urbit/noun/manage.c | 3 ++- pkg/urbit/vere/pier.c | 7 +++++-- pkg/urbit/worker/main.c | 4 +++- 5 files changed, 21 insertions(+), 7 deletions(-) diff --git a/pkg/urbit/daemon/main.c b/pkg/urbit/daemon/main.c index 02f69acf8a..657d1c2ac5 100644 --- a/pkg/urbit/daemon/main.c +++ b/pkg/urbit/daemon/main.c @@ -93,10 +93,11 @@ _main_getopt(c3_i argc, c3_c** argv) u3_Host.ops_u.tex = c3n; u3_Host.ops_u.tra = c3n; u3_Host.ops_u.veb = c3n; + u3_Host.ops_u.hap_w = 50000; u3_Host.ops_u.kno_w = DefaultKernel; while ( -1 != (ch_i=getopt(argc, argv, - "G:J:B:K:A:H:I:w:u:e:F:k:p:LljacdgqstvxPDRS")) ) + "G:J:B:K:A:H:I:C:w:u:e:F:k:p:LljacdgqstvxPDRS")) ) { switch ( ch_i ) { case 'J': { @@ -123,6 +124,12 @@ _main_getopt(c3_i argc, c3_c** argv) u3_Host.ops_u.jin_c = strdup(optarg); break; } + case 'C': { + if ( c3n == _main_readw(optarg, 1000000000, &u3_Host.ops_u.hap_w) ) { + return c3n; + } + break; + } case 'e': { u3_Host.ops_u.eth_c = strdup(optarg); break; @@ -367,9 +374,9 @@ u3_ve_usage(c3_i argc, c3_c** argv) "where ship_name is a @p phonetic representation of an urbit address\n", "without the leading '~', and options is some subset of the following:\n", "\n", - // XX find a way to re-enable - // "-A dir Use dir for initial galaxy sync\n", + "-A dir Use dir for initial clay sync\n", "-B pill Bootstrap from this pill\n", + "-C limit Set memo cache max size; 0 means uncapped\n", "-c pier Create a new urbit in pier/\n", "-D Recompute from events\n", "-d Daemon mode; implies -t\n", diff --git a/pkg/urbit/include/vere/vere.h b/pkg/urbit/include/vere/vere.h index f8e5452197..41a539b803 100644 --- a/pkg/urbit/include/vere/vere.h +++ b/pkg/urbit/include/vere/vere.h @@ -559,6 +559,7 @@ c3_o gab; // -g, test garbage collection c3_c* dns_c; // -H, ames bootstrap domain c3_c* jin_c; // -I, inject raw event + c3_w hap_w; // -C, cap memo cache c3_c* lit_c; // -J, ivory (fastboot) kernel c3_o tra; // -j, json trace c3_w kno_w; // -K, kernel version diff --git a/pkg/urbit/noun/manage.c b/pkg/urbit/noun/manage.c index b4c1968cbb..f5127be741 100644 --- a/pkg/urbit/noun/manage.c +++ b/pkg/urbit/noun/manage.c @@ -10,6 +10,7 @@ #include #include "all.h" +#include "vere/vere.h" // XX stack-overflow recovery should be gated by -a // @@ -496,7 +497,7 @@ _pave_south(c3_w* mem_w, c3_w siz_w, c3_w len_w) static void _pave_parts(void) { - u3R->cax.har_p = u3h_new_cache(50000); + u3R->cax.har_p = u3h_new_cache(u3_Host.ops_u.hap_w); u3R->jed.war_p = u3h_new(); u3R->jed.cod_p = u3h_new(); u3R->jed.han_p = u3h_new(); diff --git a/pkg/urbit/vere/pier.c b/pkg/urbit/vere/pier.c index fff3f15581..0c5b114422 100644 --- a/pkg/urbit/vere/pier.c +++ b/pkg/urbit/vere/pier.c @@ -1039,11 +1039,12 @@ _pier_work_create(u3_pier* pir_u) /* spawn new process and connect to it */ { - c3_c* arg_c[5]; + c3_c* arg_c[6]; c3_c* bin_c = u3_Host.wrk_c; c3_c* pax_c = pir_u->pax_c; c3_c key_c[256]; c3_c wag_c[11]; + c3_c hap_c[11]; c3_i err_i; sprintf(key_c, "%" PRIx64 ":%" PRIx64 ":%" PRIx64 ":%" PRIx64 "", @@ -1053,12 +1054,14 @@ _pier_work_create(u3_pier* pir_u) pir_u->key_d[3]); sprintf(wag_c, "%u", pir_u->wag_w); + sprintf(hap_c, "%u", u3_Host.ops_u.hap_w); arg_c[0] = bin_c; // executable arg_c[1] = pax_c; // path to checkpoint directory arg_c[2] = key_c; // disk key arg_c[3] = wag_c; // runtime config - arg_c[4] = 0; + arg_c[4] = hap_c; // hash table size + arg_c[5] = 0; uv_pipe_init(u3L, &god_u->inn_u.pyp_u, 0); uv_pipe_init(u3L, &god_u->out_u.pyp_u, 0); diff --git a/pkg/urbit/worker/main.c b/pkg/urbit/worker/main.c index d3148773b1..064029a6c1 100644 --- a/pkg/urbit/worker/main.c +++ b/pkg/urbit/worker/main.c @@ -1015,8 +1015,9 @@ main(c3_i argc, c3_c* argv[]) c3_c* dir_c = argv[1]; c3_c* key_c = argv[2]; c3_c* wag_c = argv[3]; + c3_c* hap_c = argv[4]; - c3_assert(4 == argc); + c3_assert(5 == argc); memset(&u3V, 0, sizeof(u3V)); memset(&u3_Host.tra_u, 0, sizeof(u3_Host.tra_u)); @@ -1035,6 +1036,7 @@ main(c3_i argc, c3_c* argv[]) */ { sscanf(wag_c, "%" SCNu32, &u3C.wag_w); + sscanf(hap_c, "%" SCNu32, &u3_Host.ops_u.hap_w); } /* load pier directory From f9d02263eeee9d7b356f79b05642d2987e5ddf22 Mon Sep 17 00:00:00 2001 From: Philip Monk Date: Wed, 24 Jun 2020 17:11:46 -0700 Subject: [PATCH 10/32] jets: use appropriate macro --- pkg/urbit/jets/e/jam.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pkg/urbit/jets/e/jam.c b/pkg/urbit/jets/e/jam.c index 1abf18a5be..9b0502d0dc 100644 --- a/pkg/urbit/jets/e/jam.c +++ b/pkg/urbit/jets/e/jam.c @@ -32,10 +32,7 @@ u3qe_jam(u3_atom a) u3a_print_memory(stderr, "memoization cache", mem_w); u3h_root* har_u = u3to(u3h_root, u3R->cax.har_p); u3l_log("memoization entries: %d\r\n", har_u->use_w); - c3_w diff = (u3R->hat_p < u3R->cap_p) ? - (u3R->cap_p - u3R->hat_p) : - (u3R->hat_p - u3R->cap_p); - u3a_print_memory(stderr, "unused free", diff); + u3a_print_memory(stderr, "unused free", u3a_open(u3R)); return tot_w; } #endif From 627df6d41ffead77fb4c6211a799afd4fc651bdd Mon Sep 17 00:00:00 2001 From: Philip Monk Date: Wed, 24 Jun 2020 18:36:40 -0700 Subject: [PATCH 11/32] ci: add travis as trusted user --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 6fbfc591e2..0a575fc1c5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,7 @@ jobs: nix: 2.3.6 before_install: - git lfs pull + - echo "trusted-users = root travis" | sudo tee -a /etc/nix/nix.conf && sudo pkill nix-daemon install: - nix-env -iA cachix -f https://cachix.org/api/v1/install script: From cbb80e9e7d1cfa320df23ec7eb3fcf3698a89cbf Mon Sep 17 00:00:00 2001 From: Philip Monk Date: Wed, 24 Jun 2020 19:09:03 -0700 Subject: [PATCH 12/32] vere: bump version to 0.10.6 --- pkg/urbit/configure | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/urbit/configure b/pkg/urbit/configure index ebf59d23d1..9b518592e3 100755 --- a/pkg/urbit/configure +++ b/pkg/urbit/configure @@ -2,7 +2,7 @@ set -e -URBIT_VERSION="0.10.5" +URBIT_VERSION="0.10.6" deps=" \ curl gmp sigsegv argon2 ed25519 ent h2o scrypt sni uv murmur3 secp256k1 \ From 48ae088bbdf28d3ce450f3da595826ea0d31b587 Mon Sep 17 00:00:00 2001 From: Liam Fitzgerald Date: Fri, 19 Jun 2020 19:17:01 +1000 Subject: [PATCH 13/32] links-js: calculate header unread count correctly --- pkg/interface/src/apps/links/app.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/interface/src/apps/links/app.js b/pkg/interface/src/apps/links/app.js index 6703f9d761..5ce2b5cce3 100644 --- a/pkg/interface/src/apps/links/app.js +++ b/pkg/interface/src/apps/links/app.js @@ -55,8 +55,8 @@ export class LinksApp extends Component { const seen = props.linksSeen ? props.linksSeen : {}; const totalUnseen = _.reduce( - seen, - (acc, links) => acc + _.reduce(links, (total, hasSeen) => total + (hasSeen ? 0 : 1), 0), + links, + (acc, collection) => acc + collection.unseenCount, 0 ); From 9aa4d96fb5c32a246434b7cfe2b97952db944b82 Mon Sep 17 00:00:00 2001 From: Liam Fitzgerald Date: Fri, 19 Jun 2020 19:22:44 +1000 Subject: [PATCH 14/32] chat: refine image scaling --- pkg/interface/src/apps/chat/components/lib/message.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/interface/src/apps/chat/components/lib/message.js b/pkg/interface/src/apps/chat/components/lib/message.js index ce1d4bd89d..ead51131f5 100644 --- a/pkg/interface/src/apps/chat/components/lib/message.js +++ b/pkg/interface/src/apps/chat/components/lib/message.js @@ -88,7 +88,8 @@ export class Message extends Component { src={letter.url} style={{ height: 'min(250px, 20vh)', - maxWidth: '80vw' + maxWidth: 'calc(100% - 36px - 1.5rem)', + objectFit: 'contain' }} > ); From f27037d2de16b13020157bf89d15a6f35fe41789 Mon Sep 17 00:00:00 2001 From: Liam Fitzgerald Date: Sat, 20 Jun 2020 02:54:37 +1000 Subject: [PATCH 15/32] links: filter unread count on selected groups --- pkg/interface/src/apps/links/app.js | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/pkg/interface/src/apps/links/app.js b/pkg/interface/src/apps/links/app.js index 5ce2b5cce3..ed9e1dba1e 100644 --- a/pkg/interface/src/apps/links/app.js +++ b/pkg/interface/src/apps/links/app.js @@ -54,9 +54,18 @@ export class LinksApp extends Component { const seen = props.linksSeen ? props.linksSeen : {}; + const selectedGroups = props.selectedGroups ? props.selectedGroups : []; + + const selGroupPaths = selectedGroups.map(g => g[0]); const totalUnseen = _.reduce( links, - (acc, collection) => acc + collection.unseenCount, + (acc, collection, path) => { + if(selGroupPaths.length > 0 + && !selGroupPaths.includes(associations.link?.[path]?.['group-path'])) { + return acc; + } + return acc + collection.unseenCount; + }, 0 ); @@ -68,7 +77,6 @@ export class LinksApp extends Component { const invites = props.invites ? props.invites : {}; - const selectedGroups = props.selectedGroups ? props.selectedGroups : []; const listening = props.linkListening; From 2713083386dc9e1402a7658ed7eef8a82fa5577a Mon Sep 17 00:00:00 2001 From: Philip Monk Date: Thu, 25 Jun 2020 17:23:03 -0700 Subject: [PATCH 16/32] libsigsegv: disable stack vma check This patches libsigsegv to not check the stack vma on Linux, since that involves reading procfs, and we make very heavy use of sigsegv. This eliminates most of urbit's performance discrepancy between Linux and MacOS. These are the benchmarks used; note this is a local MBP vs a cloud Linux server, and the MBP is almost certainly faster hardware. We take two benchmarks, one of which decrements 10 million times and the other simply allocates 125MB of memory. These are the results: cpu-heavy == =/ n 10.000.000 |-(?~(n n $(n (dec n)))) mem-heavy == =a (bex 1.000.000.008) macos, cpu-heavy: 6 seconds macos, mem-heavy: 1 second linux-before, cpu-heavy: 30 seconds linux-before, mem-heavy: 160 seconds linux-after, cpu-heavy 9 seconds linux-after, mem-heavy 1.3 seconds This represents a 3x speedup for the cpu-heavy operation and a 120x speedup for the memory-heavy operation. This check was used to try to distinguish stack overflow from other forms of segmentation fault. In the comments in src/handler-unix.c, it describes three heuristics it uses, depending on what's available from the OS. In the linux-i386 case, all three are availble, so we simply disable the slow one. This correctly recognizes stack overflow if you simply alloca(10000000000). --- nix/nixcrpkgs/pkgs/libsigsegv/builder.sh | 15 +++++++++++++++ nix/pkgs/urbit/default.nix | 7 ++++++- nix/pkgs/urbit/libsigsegv_fix.patch | 10 ++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 nix/pkgs/urbit/libsigsegv_fix.patch diff --git a/nix/nixcrpkgs/pkgs/libsigsegv/builder.sh b/nix/nixcrpkgs/pkgs/libsigsegv/builder.sh index 78d3fb9de6..39d7ddc016 100644 --- a/nix/nixcrpkgs/pkgs/libsigsegv/builder.sh +++ b/nix/nixcrpkgs/pkgs/libsigsegv/builder.sh @@ -2,6 +2,21 @@ source $setup tar -xf $src +cd libsigsegv-$version +patch -p1 << 'HEREDOC' +--- a/src/fault-linux-i386.h 2020-06-25 23:46:02.099235491 +0000 ++++ b/src/fault-linux-i386.h 2020-06-25 23:45:48.679156892 +0000 +@@ -18,6 +18,7 @@ + + #include "fault-posix-ucontext.h" + ++#define HAVE_STACKVMA 0 + #if defined __x86_64__ + /* 64 bit registers */ + +HEREDOC +cd .. + mkdir build cd build diff --git a/nix/pkgs/urbit/default.nix b/nix/pkgs/urbit/default.nix index ada88ba965..8a11072e2b 100644 --- a/nix/pkgs/urbit/default.nix +++ b/nix/pkgs/urbit/default.nix @@ -16,9 +16,14 @@ let exe = ''${meta.bin} ${pkgs.lib.strings.concatStringsSep " " meta.flags}''; }; + sigseg = + pkgs.libsigsegv.overrideAttrs (oldAttrs: rec { + patches = [ ./libsigsegv_fix.patch ]; + }); + deps = with pkgs; - [ curl gmp libsigsegv ncurses openssl zlib lmdb ]; + [ curl gmp sigseg ncurses openssl zlib lmdb ]; vendor = [ argon2 softfloat3 ed25519 ent ge-additions libaes_siv h2o scrypt uv murmur3 secp256k1 sni ivory-header ca-header ]; diff --git a/nix/pkgs/urbit/libsigsegv_fix.patch b/nix/pkgs/urbit/libsigsegv_fix.patch new file mode 100644 index 0000000000..a3b630860d --- /dev/null +++ b/nix/pkgs/urbit/libsigsegv_fix.patch @@ -0,0 +1,10 @@ +--- a/src/fault-linux-i386.h 2020-06-25 23:46:02.099235491 +0000 ++++ b/src/fault-linux-i386.h 2020-06-25 23:45:48.679156892 +0000 +@@ -18,6 +18,7 @@ + + #include "fault-posix-ucontext.h" + ++#define HAVE_STACKVMA 0 + #if defined __x86_64__ + /* 64 bit registers */ + From 271abcd3e76cc0a3d7d37b4efdfe3d73c62a1e0b Mon Sep 17 00:00:00 2001 From: Philip Monk Date: Fri, 26 Jun 2020 11:17:02 -0700 Subject: [PATCH 17/32] vere: bump version to 0.10.7 --- pkg/urbit/configure | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/urbit/configure b/pkg/urbit/configure index 9b518592e3..8ae0f453cf 100755 --- a/pkg/urbit/configure +++ b/pkg/urbit/configure @@ -2,7 +2,7 @@ set -e -URBIT_VERSION="0.10.6" +URBIT_VERSION="0.10.7" deps=" \ curl gmp sigsegv argon2 ed25519 ent h2o scrypt sni uv murmur3 secp256k1 \ From 42de99902408bf6053f36a3c8ac7c32080ee2416 Mon Sep 17 00:00:00 2001 From: Philip Monk Date: Fri, 26 Jun 2020 19:21:09 -0700 Subject: [PATCH 18/32] kiln: don't implicitly create syncs Also silence some spurious errors. --- bin/solid.pill | 4 ++-- pkg/arvo/lib/hood/kiln.hoon | 14 ++++++++++++-- pkg/arvo/sys/vane/clay.hoon | 4 +++- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/bin/solid.pill b/bin/solid.pill index f608c8ca19..dbd85a26f0 100644 --- a/bin/solid.pill +++ b/bin/solid.pill @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0b727bafa4d479d0cb6df411b7f414132a2c1d05fd415ec4d606cb43747dfc58 -size 16662916 +oid sha256:9210b8244f5cd76654c3e8546f77dfa88a500e14ba0e593ededef318fd9d0f32 +size 16948743 diff --git a/pkg/arvo/lib/hood/kiln.hoon b/pkg/arvo/lib/hood/kiln.hoon index 28c9e8f1f0..6a0cbd955f 100644 --- a/pkg/arvo/lib/hood/kiln.hoon +++ b/pkg/arvo/lib/hood/kiln.hoon @@ -329,7 +329,10 @@ ++ poke-syncs :: print sync config |= ~ =< abet %- spam - :- [%leaf "OTAs from {}"] + :- :- %leaf + ?~ ota + "OTAs disabled" + "OTAs from {} on {}" ?: =(0 ~(wyt by syn)) [%leaf "no syncs configured"]~ %+ turn ~(tap in ~(key by syn)) @@ -443,7 +446,8 @@ ?>(?=(%wake +<.sign-arvo) +>.sign-arvo) [%ota *] abet:(take:update t.wire sign-arvo) * - ?+ +<.sign-arvo ~|([%kiln-bad-take-card +<.sign-arvo wire] !!) + ?+ +<.sign-arvo + ((slog leaf+"kiln: strange card {<+<.sign-arvo wire>}" ~) abet) %done %+ done wire ?>(?=(%done +<.sign-arvo) +>.sign-arvo) %mere %+ take-mere wire @@ -472,6 +476,8 @@ her=(slav %p i.t.way) sud=(slav %tas i.t.t.way) == + ?. (~(has by syn) hos) + abet abet:abet:(mere:(auto hos) mes) :: ++ take-writ-find-ship :: @@ -482,6 +488,8 @@ her=(slav %p i.t.way) sud=(slav %tas i.t.t.way) == + ?. (~(has by syn) hos) + abet abet:abet:(take-find-ship:(auto hos) rot) :: ++ take-writ-sync :: @@ -492,6 +500,8 @@ her=(slav %p i.t.way) sud=(slav %tas i.t.t.way) == + ?. (~(has by syn) hos) + abet abet:abet:(writ:(auto hos) rot) :: ++ take-wake-autocommit diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index 149c9cb894..50127e9142 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -4365,7 +4365,9 @@ :: %boon !! %lost !! - %writ !! + %writ + %- (slog leaf+"clay: strange writ (expected on upgrade to Fusion)" ~) + [~ ..^$] :: %done ?~ error=error.q.hin From 7266b3f5c3ddde60b15427a8aa35e0fe97bfae18 Mon Sep 17 00:00:00 2001 From: Ted Blackman Date: Sun, 28 Jun 2020 00:59:02 -0400 Subject: [PATCH 19/32] hoon: re-fix +slab; /tests: fix clay tests --- pkg/arvo/sys/hoon.hoon | 4 ++-- pkg/arvo/tests/sys/vane/clay.hoon | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/arvo/sys/hoon.hoon b/pkg/arvo/sys/hoon.hoon index 2a288cb22a..824b3a2cd2 100644 --- a/pkg/arvo/sys/hoon.hoon +++ b/pkg/arvo/sys/hoon.hoon @@ -12024,8 +12024,8 @@ :: ++ slab :: test if contains |= [way=?(%read %rite %both) cog=@tas typ=type] - ?= [%| *] - (~(find ut typ) way ~[cog]) + ?= [%& *] + (~(fond ut typ) way ~[cog]) :: ++ slap |= {vax/vase gen/hoon} ^- vase :: untyped vase .* diff --git a/pkg/arvo/tests/sys/vane/clay.hoon b/pkg/arvo/tests/sys/vane/clay.hoon index 6ae08e19d2..76b0f43dea 100644 --- a/pkg/arvo/tests/sys/vane/clay.hoon +++ b/pkg/arvo/tests/sys/vane/clay.hoon @@ -153,7 +153,7 @@ =/ [res=vase nub=state:ford:fusion] (build-file:ford /lib/strandio/hoon) ;: weld %- expect - !>((slab %get-our -.res)) + !>((slab %read %get-our -.res)) :: %+ expect-eq !> %- ~(gas in *(set path)) From f6171042aeb708b09ddd5446a60716d58b225997 Mon Sep 17 00:00:00 2001 From: Ted Blackman Date: Sun, 28 Jun 2020 01:13:42 -0400 Subject: [PATCH 20/32] arvo: use date instead of kelvin --- bin/solid.pill | 4 ++-- pkg/arvo/sys/arvo.hoon | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/bin/solid.pill b/bin/solid.pill index dbd85a26f0..a400e5b067 100644 --- a/bin/solid.pill +++ b/bin/solid.pill @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9210b8244f5cd76654c3e8546f77dfa88a500e14ba0e593ededef318fd9d0f32 -size 16948743 +oid sha256:19d6cb7e5a323590455d641161b911190b3dcd1a0f601bef1304b6b340fa0dc9 +size 16662878 diff --git a/pkg/arvo/sys/arvo.hoon b/pkg/arvo/sys/arvo.hoon index a24f1d412c..f9b34e9764 100644 --- a/pkg/arvo/sys/arvo.hoon +++ b/pkg/arvo/sys/arvo.hoon @@ -1050,14 +1050,14 @@ :: +dram: upgrade $pram, compatible :: +$ pram - $~ [%345 nub=*debt:le:part] - $% [%345 nub=debt:le:part] + $~ [%~2020.6.28 nub=*debt:le:part] + $% [%~2020.6.28 nub=debt:le:part] (list ovum) == ++ dram |= ram=pram ^- pram - ?: ?=([%345 *] ram) + ?: ?=([%~2020.6.28 *] ram) ram ~| ram+ram =| nub=debt:le:part @@ -1067,7 +1067,7 @@ |= ovo=ovum ^- move [[p.ovo]~ %give %& %noun q.ovo] - [%345 nub(run [plan]~)] + [%~2020.6.28 nub(run [plan]~)] :: $prim: old $pram :: +drum: upgrade $prim to $pram, incompatible :: @@ -1094,7 +1094,7 @@ vanes (turn nyf |=({a/@tas b/vise} [a [b *worm]])) == =. ram (dram ram) - ?> ?=([%345 *] ram) + ?> ?=([%~2020.6.28 *] ram) =/ run plot:(spam:(lead now ~) [//arvo %vega ~]) (leap now (then:(lead now `nub.ram) run)) :: +peek: external inspect @@ -1323,7 +1323,7 @@ :* our now eny - 345+nub + [%~2020.6.28 nub] bud (turn vanes |=([label=@tas =vane] [label vase.vane])) == From d7b9889b4ad3d5ff337744fc30d01c0a9f3fb54a Mon Sep 17 00:00:00 2001 From: Ted Blackman Date: Sun, 28 Jun 2020 05:06:16 -0400 Subject: [PATCH 21/32] -test: support other desks (full beams in args) --- nix/ops/test/builder.sh | 2 +- pkg/arvo/ted/test.hoon | 43 ++++++++++++++++++++--------------------- 2 files changed, 22 insertions(+), 23 deletions(-) diff --git a/nix/ops/test/builder.sh b/nix/ops/test/builder.sh index 68f03fa8ea..2746797abf 100644 --- a/nix/ops/test/builder.sh +++ b/nix/ops/test/builder.sh @@ -24,7 +24,7 @@ herb ./ship -p hood -d '+hood/mass' # Run the unit tests and then print scrollback herb ./ship -d '~& ~ ~& %test-unit-start ~' -herb ./ship -d '####-test /' +herb ./ship -d '####-test %/tests' herb ./ship -d '~& ~ ~& %test-unit-end ~' # Start and run the test app diff --git a/pkg/arvo/ted/test.hoon b/pkg/arvo/ted/test.hoon index 1c1563b225..d9f428ea7f 100644 --- a/pkg/arvo/ted/test.hoon +++ b/pkg/arvo/ted/test.hoon @@ -73,43 +73,42 @@ =((end 3 5 a) 'test-') :: ++ find-test-files - =| fiz=(set [=spur test=(unit term)]) + =| fiz=(set [=beam test=(unit term)]) =/ m (strand ,_fiz) - |= [bek=beak paz=(list path)] + |= bez=(list beam) ^- form:m =* loop $ - ?~ paz + ?~ bez (pure:m fiz) - =/ xap=path (flop i.paz) - ;< hav=? bind:m (check-for-file:strandio bek hoon+xap) + ;< hav=? bind:m (check-for-file:strandio -.i.bez hoon+s.i.bez) ?: hav - loop(paz t.paz, fiz (~(put in fiz) [hoon+xap ~])) - ;< fez=(list path) bind:m (list-tree:strandio bek xap) + loop(bez t.bez, fiz (~(put in fiz) [i.bez(s hoon+s.i.bez) ~])) + ;< fez=(list path) bind:m (list-tree:strandio i.bez) ?. =(~ fez) - =/ foz (turn fez |=(path [(flop +<) ~])) - loop(paz t.paz, fiz (~(gas in fiz) foz)) - ~| bad-test-path+i.paz - =/ tex=term =-(?>(((sane %tas) -) -) (head xap)) - =/ xup=path (tail xap) - ;< hov=? bind:m (check-for-file:strandio bek hoon+xup) + =/ foz (turn fez |=(path [[-.i.bez (flop +<)] ~])) + loop(bez t.bez, fiz (~(gas in fiz) foz)) + ~| bad-test-beam+i.bez + =/ tex=term =-(?>(((sane %tas) -) -) (head s.i.bez)) + =/ xup=path (tail s.i.bez) + ;< hov=? bind:m (check-for-file:strandio i.bez(s hoon+xup)) ?. hov - ~|(no-tests-at-path+paz !!) - loop(paz t.paz, fiz (~(put in fiz) [hoon+xup `tex])) + ~|(no-tests-at-path+i.bez !!) + loop(bez t.bez, fiz (~(put in fiz) [[-.i.bez hoon+xup] `tex])) -- ^- thread:spider |= arg=vase =/ m (strand ,vase) ^- form:m -=/ paz=(list path) (turn !<((list path) arg) |=(path [%tests +<])) -;< bek=beak bind:m get-beak:strandio -;< fiz=(set [=spur test=(unit term)]) bind:m (find-test-files bek paz) -=> .(fiz ~(tap in fiz)) +=/ bez=(list beam) + (turn !<((list path) arg) |=(p=path (need (de-beam:format p)))) +;< fiz=(set [=beam test=(unit term)]) bind:m (find-test-files bez) +=> .(fiz (sort ~(tap in fiz) aor)) =| test-arms=(map path (list test-arm)) |- ^- form:m =* gather-tests $ ?^ fiz - ~> %slog.0^leaf+"test: building {(spud (flop spur.i.fiz))}" - ;< cor=vase bind:m (build-file:strandio bek spur.i.fiz) + ~> %slog.0^leaf+"test: building {(spud (flop s.beam.i.fiz))}" + ;< cor=vase bind:m (build-file:strandio beam.i.fiz) =/ arms=(list test-arm) (get-test-arms cor) =? arms ?=(^ test.i.fiz) |- ^+ arms @@ -117,7 +116,7 @@ ?: =(name.i.arms u.test.i.fiz) [i.arms]~ $(arms t.arms) - =. test-arms (~(put by test-arms) (flop (tail spur.i.fiz)) arms) + =. test-arms (~(put by test-arms) (flop (tail s.beam.i.fiz)) arms) gather-tests(fiz t.fiz) %- pure:m !> ^= ok %+ roll (resolve-test-paths test-arms) From 3b8790cc04fa5ef35dba8a53c564c4cae68fbb42 Mon Sep 17 00:00:00 2001 From: Logan Allen Date: Fri, 26 Jun 2020 15:51:55 -0400 Subject: [PATCH 22/32] chat-store: responded to comments, cleaned up --- pkg/arvo/app/chat-store.hoon | 85 +++++++++++++++++++----------------- 1 file changed, 45 insertions(+), 40 deletions(-) diff --git a/pkg/arvo/app/chat-store.hoon b/pkg/arvo/app/chat-store.hoon index 26c3fdf8da..0466d98f74 100644 --- a/pkg/arvo/app/chat-store.hoon +++ b/pkg/arvo/app/chat-store.hoon @@ -5,22 +5,20 @@ |% +$ card card:agent:gall +$ versioned-state - $% state-zero - state-one - state-two - state-three + $% state-0 + state-1 + state-2 == :: -+$ state-zero [%0 =inbox:store] -+$ state-one [%1 =inbox:store] -+$ state-two [%2 =inbox:store] -+$ state-three [%3 =inbox:store] ++$ state-0 [%0 =inbox:store] ++$ state-1 [%1 =inbox:store] ++$ state-2 [%2 =inbox:store] +$ admin-action $% [%trim ~] == -- :: -=| state-three +=| state-2 =* state - :: %- agent:dbug @@ -38,19 +36,25 @@ ++ on-save !>(state) ++ on-load |= old-vase=vase + ^- (quip card _this) + |^ =/ old !<(versioned-state old-vase) - ?: ?=(%3 -.old) - [~ this(state old)] - ?: ?=(%2 -.old) - :_ this(state [%3 inbox.old]) - [%pass /trim %agent [our.bowl %chat-store] %poke %noun !>([%trim ~])]~ - =/ reversed-inbox=inbox:store - %- ~(run by inbox.old) + =? old ?=(%0 -.old) + (old-to-2 inbox.old) + =? old ?=(%1 -.old) + (old-to-2 inbox.old) + :_ this(state [%2 inbox.old]) + [%pass /trim %agent [our.bowl %chat-store] %poke %noun !>([%trim ~])]~ + :: + ++ old-to-2 + |= =inbox:store + ^- state-2 + :- %2 + %- ~(run by inbox) |= =mailbox:store ^- mailbox:store [config.mailbox (flop envelopes.mailbox)] - :_ this(state [%3 reversed-inbox]) - [%pass /trim %agent [our.bowl %chat-store] %poke %noun !>([%trim ~])]~ + -- :: ++ on-poke ~/ %chat-store-poke @@ -61,7 +65,7 @@ ?+ mark (on-poke:def mark vase) %json (poke-json:cc !<(json vase)) %chat-action (poke-chat-action:cc !<(action:store vase)) - %noun (poke-noun:cc !<(admin-action vase)) + %noun [~ (poke-noun:cc !<(admin-action vase))] == [cards this] :: @@ -160,28 +164,29 @@ :: ++ poke-noun |= nou=admin-action - ^- (quip card _state) + ^- _state ~& %trimming-chat-store - :- ~ - :- %3 - %- ~(urn by inbox) - |= [=path mailbox:store] - ^- mailbox:store - =/ [a=* out=(list envelope:store)] - %+ roll envelopes - |= $: =envelope:store - o=[[hav=(set serial:store) curr=@] out=(list envelope:store)] - == - ?: (~(has in hav.o) uid.envelope) - [[hav.o curr.o] out.o] - :- - ^- [(set serial:store) @] - [(~(put in hav.o) uid.envelope) +(curr.o)] - ^- (list envelope:store) - [envelope(number curr.o) out.o] - =/ len (lent out) - ~& [path [%old (lent envelopes)] [%new len]] - [[len len] (flop out)] + %_ state + inbox + %- ~(urn by inbox) + |= [=path mailbox:store] + ^- mailbox:store + =/ [a=* out=(list envelope:store)] + %+ roll envelopes + |= $: =envelope:store + o=[[hav=(set serial:store) curr=@] out=(list envelope:store)] + == + ?: (~(has in hav.o) uid.envelope) + [[hav.o curr.o] out.o] + :- + ^- [(set serial:store) @] + [(~(put in hav.o) uid.envelope) +(curr.o)] + ^- (list envelope:store) + [envelope(number curr.o) out.o] + =/ len (lent out) + ~& [path [%old (lent envelopes)] [%new len]] + [[len len] (flop out)] + == :: ++ poke-json |= jon=json From 043b8a4cc0bb64167d8ce5d31f796baf78a70444 Mon Sep 17 00:00:00 2001 From: Ted Blackman Date: Mon, 29 Jun 2020 02:43:52 -0400 Subject: [PATCH 23/32] goad: don't crash on pre-OTA sign --- pkg/arvo/app/goad.hoon | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/arvo/app/goad.hoon b/pkg/arvo/app/goad.hoon index 1f55f7b01c..5e34f2151e 100644 --- a/pkg/arvo/app/goad.hoon +++ b/pkg/arvo/app/goad.hoon @@ -24,6 +24,7 @@ |= [wir=wire sin=sign-arvo] ?+ wir (on-arvo:def wir sin) [%clay ~] `this + [%behn ~] `this :: backcompat == :: ++ on-agent on-agent:def From 29a62346331647e466d5d1f792fd871db4a71b06 Mon Sep 17 00:00:00 2001 From: Ted Blackman Date: Mon, 29 Jun 2020 05:15:25 -0400 Subject: [PATCH 24/32] invite-view: reinstate as potato --- pkg/arvo/app/invite-view.hoon | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 pkg/arvo/app/invite-view.hoon diff --git a/pkg/arvo/app/invite-view.hoon b/pkg/arvo/app/invite-view.hoon new file mode 100644 index 0000000000..666ed54730 --- /dev/null +++ b/pkg/arvo/app/invite-view.hoon @@ -0,0 +1,17 @@ +/+ default-agent +^- agent:gall +|_ =bowl:gall ++* this . + def ~(. (default-agent this %|) bowl) + step `step:agent:gall`[~ this] +++ on-init on-init:def +++ on-save on-save:def +++ on-load on-load:def +++ on-poke |=(* step) +++ on-watch on-watch:def +++ on-leave |=(* step) +++ on-peek |=(* ~) +++ on-agent |=(* step) +++ on-arvo |=(* step) +++ on-fail on-fail:def +-- From 7611c2fc0801b119979a9c210f718f870a857fcd Mon Sep 17 00:00:00 2001 From: Ted Blackman Date: Mon, 29 Jun 2020 05:26:41 -0400 Subject: [PATCH 25/32] landscape/img/codeeval.png: typo in filename --- .../app/landscape/img/{codeval.png => codeeval.png} | Bin 1 file changed, 0 insertions(+), 0 deletions(-) rename pkg/arvo/app/landscape/img/{codeval.png => codeeval.png} (100%) diff --git a/pkg/arvo/app/landscape/img/codeval.png b/pkg/arvo/app/landscape/img/codeeval.png similarity index 100% rename from pkg/arvo/app/landscape/img/codeval.png rename to pkg/arvo/app/landscape/img/codeeval.png From ede710582046ccc07316072525993e4ce105ac8e Mon Sep 17 00:00:00 2001 From: Ted Blackman Date: Mon, 29 Jun 2020 11:01:23 -0400 Subject: [PATCH 26/32] clay: flop syntax error trace --- pkg/arvo/sys/vane/clay.hoon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index 50127e9142..20b62773ce 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -771,7 +771,7 @@ ^- pile =/ [=hair res=(unit [=pile =nail])] ((pile-rule pax) [1 1] tex) ?^ res pile.u.res - %- mean + %- mean %- flop =/ lyn p.hair =/ col q.hair :~ leaf+"syntax error at [{} {}] in {}" From a958cb9288bc7a01322e60644aac0455eb31b9e7 Mon Sep 17 00:00:00 2001 From: Isaac Visintainer Date: Sun, 28 Jun 2020 21:12:46 -0600 Subject: [PATCH 27/32] metadata-store: add cleanup utility, use on-poke:def instead of no-op --- pkg/arvo/app/metadata-store.hoon | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/pkg/arvo/app/metadata-store.hoon b/pkg/arvo/app/metadata-store.hoon index 8e52aaa41c..e50bdcdb23 100644 --- a/pkg/arvo/app/metadata-store.hoon +++ b/pkg/arvo/app/metadata-store.hoon @@ -61,9 +61,27 @@ ^- (quip card _this) ?> (team:title our.bowl src.bowl) =^ cards state - ?: ?=(%metadata-action mark) + ?+ mark (on-poke:def mark vase) + %metadata-action (poke-metadata-action:mc !<(metadata-action vase)) - (on-poke:def mark vase) + %noun + =/ val=(each [%cleanup path] tang) + (mule |.(!<([%cleanup path] vase))) + ?. ?=(%& -.val) + (on-poke:def mark vase) + =/ group=path +.p.val + =/ res=(set resource) (~(get ju group-indices) group) + =. group-indices (~(del by group-indices) group) + :- ~ + %+ roll ~(tap in res) + |= [r=resource out=_state] + =. resource-indices.out (~(del by resource-indices.out) r) + =. app-indices.out + %- ~(del ju app-indices.out) + [app-name.r group app-path.r] + =. associations.out (~(del by associations.out) group r) + out + == [cards this] :: ++ on-watch From 40db8ea580f7aa2b5104c37e9ac9e8804a32033b Mon Sep 17 00:00:00 2001 From: Philip Monk Date: Mon, 29 Jun 2020 13:25:10 -0700 Subject: [PATCH 28/32] kiln: make otas continue even if they failed to apply --- pkg/arvo/lib/hood/kiln.hoon | 60 +++++++++++++++++++++---------------- pkg/arvo/sys/vane/clay.hoon | 2 +- 2 files changed, 35 insertions(+), 27 deletions(-) diff --git a/pkg/arvo/lib/hood/kiln.hoon b/pkg/arvo/lib/hood/kiln.hoon index 6a0cbd955f..e3bfb0fe2d 100644 --- a/pkg/arvo/lib/hood/kiln.hoon +++ b/pkg/arvo/lib/hood/kiln.hoon @@ -240,6 +240,7 @@ ?+ i.t.t.t.wire ~&([%strange-ota-take t.t.t.wire] ..abet) %find (take-find sign-arvo) %sync (take-sync sign-arvo) + %download (take-download sign-arvo) %merge-home (take-merge-home sign-arvo) %merge-kids (take-merge-kids sign-arvo) == @@ -255,20 +256,39 @@ == :: ++ take-sync + |= =sign-arvo + ?> ?=(%writ +<.sign-arvo) + ?> ?=(^ ota) + ?~ p.sign-arvo + =. ..abet (render-ket "OTA cancelled (1), retrying" ~) + (poke-internal `[ship desk]:u.ota) + =. ..abet (render-ket "downloading OTA update" ~) + =? aeon.u.ota ?=($w p.p.u.p.sign-arvo) + ud:;;(cass:clay q.q.r.u.p.sign-arvo) + %: emit + %pass (make-wire /download) %arvo %c + %warp ship.u.ota desk.u.ota `[%sing %v ud+aeon.u.ota /] + == + :: + ++ take-download |= =sign-arvo ^+ ..abet ?> ?=(%writ +<.sign-arvo) ?> ?=(^ ota) ?~ p.sign-arvo - =. ..abet (render-ket "OTA cancelled, retrying" ~) + =. ..abet (render-ket "OTA cancelled (2), retrying" ~) (poke-internal `[ship desk]:u.ota) - =? aeon.u.ota ?=($w p.p.u.p.sign-arvo) - ud:;;(cass:clay q.q.r.u.p.sign-arvo) + =. ..abet (render-ket "finished downloading OTA" ~) + =. aeon.u.ota +(aeon.u.ota) =/ =germ (get-germ %home) - =. ..abet (render-ket "beginning OTA to %home" ~) - %: emit - %pass (make-wire /merge-home) %arvo %c - %merg %home ship.u.ota desk.u.ota ud+aeon.u.ota germ + =. ..abet (render-ket "applying OTA to %home" ~) + %- emil + :~ :* %pass (make-wire /merge-home) %arvo %c + %merg %home ship.u.ota desk.u.ota ud+(dec aeon.u.ota) germ + == + :* %pass (make-wire /sync) %arvo %c + %warp ship.u.ota desk.u.ota `[%sing %z ud+aeon.u.ota /] + == == :: ++ take-merge-home @@ -282,20 +302,14 @@ (poke-internal `[ship desk]:u.ota) :: ?: ?=(%| -.p.sign-arvo) - =. ..abet - =/ =tape "OTA to %home failed, waiting for next revision" - (render-ket tape `p.p.sign-arvo) - =. aeon.u.ota +(aeon.u.ota) - %: emit - %pass (make-wire /sync) %arvo %c - %warp ship.u.ota desk.u.ota `[%sing %z ud+aeon.u.ota /] - == + =/ =tape "OTA to %home failed, waiting for next revision" + (render-ket tape `p.p.sign-arvo) =. ..abet (render-ket "OTA to %home succeeded" ~) - =. ..abet (render-ket "beginning OTA to %kids" ~) + =. ..abet (render-ket "applying OTA to %kids" ~) =/ =germ (get-germ %kids) %: emit %pass (make-wire /merge-kids) %arvo %c - %merg %kids ship.u.ota desk.u.ota ud+aeon.u.ota germ + %merg %kids ship.u.ota desk.u.ota ud+(dec aeon.u.ota) germ == :: ++ take-merge-kids @@ -308,15 +322,9 @@ (render-ket tape `p.p.sign-arvo) (poke-internal `[ship desk]:u.ota) :: - =. ..abet - ?- -.p.sign-arvo - %& (render-ket "OTA to %kids succeeded" ~) - %| (render-ket "OTA to %kids failed" `p.p.sign-arvo) - == - =. aeon.u.ota +(aeon.u.ota) - %: emit - %pass (make-wire /sync) %arvo %c - %warp ship.u.ota desk.u.ota `[%sing %z ud+aeon.u.ota /] + ?- -.p.sign-arvo + %& (render-ket "OTA to %kids succeeded" ~) + %| (render-ket "OTA to %kids failed" `p.p.sign-arvo) == -- :: diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index 20b62773ce..e5cbb91bea 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -2733,7 +2733,7 @@ :: traverse updated state and sanity check :: =+ ~| :* %bad-foreign-update - [gar=gar let=let.nako nut=(turn nut head) nat=(turn nat head)] + [gar=gar.nako let=let.nako nut=(turn nut head) nat=(turn nat head)] [hitdom=hit.dom letdom=let.dom] == ?: =(0 let.nako) From e7e2c07d6ff4c55048254dcace78111353c99898 Mon Sep 17 00:00:00 2001 From: Philip Monk Date: Mon, 29 Jun 2020 14:30:33 -0700 Subject: [PATCH 29/32] clay: remove scaffolding --- pkg/arvo/app/chat-store.hoon | 2 +- pkg/arvo/sys/vane/clay.hoon | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/arvo/app/chat-store.hoon b/pkg/arvo/app/chat-store.hoon index 0466d98f74..70daf9ecfe 100644 --- a/pkg/arvo/app/chat-store.hoon +++ b/pkg/arvo/app/chat-store.hoon @@ -184,7 +184,7 @@ ^- (list envelope:store) [envelope(number curr.o) out.o] =/ len (lent out) - ~& [path [%old (lent envelopes)] [%new len]] + ~? !=(len (lent envelopes)) [path [%old (lent envelopes)] [%new len]] [[len len] (flop out)] == :: diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index e5cbb91bea..1389d79f8a 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -4056,7 +4056,7 @@ |= [=desk data=(map path lobe)] ^- reef-cache ~> %slog.0^leaf+"clay: building reef on {}" - ?: !=(%homer desk) + ?: =(%home desk) [!>(..ride) !>(..is) !>(..zuse)] |^ =/ [home=? hoon=vase] From 137e4428f617c13f28ed31e520eff98d251ed3e9 Mon Sep 17 00:00:00 2001 From: Philip Monk Date: Mon, 29 Jun 2020 16:14:45 -0700 Subject: [PATCH 30/32] release: urbit-os-v1.0.24 --- bin/solid.pill | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/solid.pill b/bin/solid.pill index a400e5b067..bc6fa88e92 100644 --- a/bin/solid.pill +++ b/bin/solid.pill @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:19d6cb7e5a323590455d641161b911190b3dcd1a0f601bef1304b6b340fa0dc9 -size 16662878 +oid sha256:581c9b713aa12126f3a618d75bf4254bb431f633fbfc76941663391d3d13cdb0 +size 16742515 From ad4e2c0c9d90003190d7b96216930d7f0eb8a00a Mon Sep 17 00:00:00 2001 From: Fang Date: Tue, 30 Jun 2020 01:19:44 +0200 Subject: [PATCH 31/32] chat fe: don't embed webms as images --- pkg/interface/src/apps/chat/components/lib/message.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/interface/src/apps/chat/components/lib/message.js b/pkg/interface/src/apps/chat/components/lib/message.js index ce1d4bd89d..c58cf5e89c 100644 --- a/pkg/interface/src/apps/chat/components/lib/message.js +++ b/pkg/interface/src/apps/chat/components/lib/message.js @@ -72,7 +72,7 @@ export class Message extends Component { ); } else if ('url' in letter) { let imgMatch = - /(jpg|img|png|gif|tiff|jpeg|JPG|IMG|PNG|TIFF|GIF|webp|WEBP|webm|WEBM|svg|SVG)$/ + /(jpg|img|png|gif|tiff|jpeg|JPG|IMG|PNG|TIFF|GIF|webp|WEBP|svg|SVG)$/ .exec(letter.url); const youTubeRegex = new RegExp(String(/(?:https?:\/\/(?:[a-z]+.)?)/.source) // protocol + /(?:youtu\.?be(?:\.com)?\/)(?:embed\/)?/.source // short and long-links From 9686f3483a06f12c7bf3cc5a5fc479524cca9d37 Mon Sep 17 00:00:00 2001 From: Matilde Park Date: Mon, 29 Jun 2020 22:48:17 -0400 Subject: [PATCH 32/32] interface: update production build for typescript --- pkg/interface/config/webpack.prod.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/interface/config/webpack.prod.js b/pkg/interface/config/webpack.prod.js index e19e15940e..3e00ad14d2 100644 --- a/pkg/interface/config/webpack.prod.js +++ b/pkg/interface/config/webpack.prod.js @@ -11,7 +11,7 @@ module.exports = { module: { rules: [ { - test: /\.js?$/, + test: /\.(j|t)sx?$/, use: { loader: 'babel-loader', options: { @@ -39,7 +39,7 @@ module.exports = { ] }, resolve: { - extensions: ['.js'] + extensions: ['.js', '.ts', '.tsx'] }, devtool: 'inline-source-map', // devServer: { @@ -58,7 +58,7 @@ module.exports = { output: { filename: 'index.js', chunkFilename: 'index.js', - path: path.resolve(urbitrc.URBIT_PIERS[0] + '/app/landscape/', 'js'), + path: path.resolve(__dirname, '../../arvo/app/landscape/js'), publicPath: '/' }, optimization: {