From f851803d3c350c6bba1bd8c738bf605d66592789 Mon Sep 17 00:00:00 2001 From: Liam Fitzgerald Date: Thu, 26 Nov 2020 15:42:23 +1000 Subject: [PATCH 1/3] pull-hook: do not crash on dangling sync If the %kick handler of the inner-core crashes, then we never get to resubscribe, thus leaving a dangling entry in the state with no corresponding subscription. Updates the pull-hook-action pokes to not crash when given a dangling entry. --- pkg/arvo/lib/pull-hook.hoon | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/pkg/arvo/lib/pull-hook.hoon b/pkg/arvo/lib/pull-hook.hoon index cb3b5e6f73..3c5ef03b3b 100644 --- a/pkg/arvo/lib/pull-hook.hoon +++ b/pkg/arvo/lib/pull-hook.hoon @@ -304,33 +304,35 @@ =. tracking (~(put by tracking) resource ship) :_ state - ~[(watch-resource resource /)] + (watch-resource resource /) :: ++ remove |= =resource - :- ~[(leave-resource resource)] + :- (leave-resource resource) state(tracking (~(del by tracking) resource)) -- :: ++ leave-resource |= rid=resource - ^- card - =/ =ship - (~(got by tracking) rid) + ^- (list card) + =/ ship=(unit ship) + (~(get by tracking) rid) + ?~ ship ~ =/ =wire (make-wire pull+resource+(en-path:resource rid)) - [%pass wire %agent [ship push-hook-name.config] %leave ~] + [%pass wire %agent [u.ship push-hook-name.config] %leave ~]~ ++ watch-resource |= [rid=resource pax=path] - ^- card - =/ =ship - (~(got by tracking) rid) + ^- (list card) + =/ ship=(unit ship) + (~(get by tracking) rid) + ?~ ship ~ =/ =path (welp resource+(en-path:resource rid) pax) =/ =wire (make-wire pull+path) - [%pass wire %agent [ship push-hook-name.config] %watch path] + [%pass wire %agent [u.ship push-hook-name.config] %watch path]~ :: ++ make-wire |= =wire From f73a96ba904ec12f31cdd7f0dab209db1c7ee885 Mon Sep 17 00:00:00 2001 From: Liam Fitzgerald Date: Thu, 26 Nov 2020 16:09:14 +1000 Subject: [PATCH 2/3] pull-hook: backstop crashed kick handler Virtualises the kick handler for the inner door in lib/pull-hook, so that crashes in the handler do not cause a dangling resource with no subscription. Additionally, failed kicks now cause the sync to be dropped into a failed-kicks map in the state, where we attempt to recover from the failure in the on-load. --- pkg/arvo/lib/pull-hook.hoon | 101 +++++++++++++++++++++++++++++++----- 1 file changed, 89 insertions(+), 12 deletions(-) diff --git a/pkg/arvo/lib/pull-hook.hoon b/pkg/arvo/lib/pull-hook.hoon index 3c5ef03b3b..78566ad3c2 100644 --- a/pkg/arvo/lib/pull-hook.hoon +++ b/pkg/arvo/lib/pull-hook.hoon @@ -49,13 +49,21 @@ inner-state=vase == :: ++$ base-state-1 + $: base-state-0 + failed-kicks=(map resource ship) + == +:: +$ state-0 [%0 base-state-0] :: +$ state-1 [%1 base-state-0] :: ++$ state-2 [%2 base-state-1] +:: +$ versioned-state $% state-0 state-1 + state-2 == :: ++ default @@ -141,7 +149,7 @@ ++ agent |* =config |= =(pull-hook config) - =| state-1 + =| state-2 =* state - ^- agent:gall =< @@ -155,6 +163,7 @@ =^ cards pull-hook on-init:og [cards this] + :: ++ on-load |= =old=vase =/ old @@ -162,10 +171,16 @@ =| cards=(list card:agent:gall) |^ ?- -.old - %1 + %2 =^ og-cards pull-hook (on-load:og inner-state.old) - [(weld cards og-cards) this(state old)] + =. state old + =^ retry-cards state + retry-failed-kicks + :_ this + :(weld cards og-cards retry-cards) + :: + %1 $(old [%2 +.old ~]) :: %0 %_ $ @@ -175,6 +190,22 @@ (weld cards (missing-subscriptions tracking.old)) == == + :: + ++ retry-failed-kicks + =| acc-cards=(list card) + =/ failures=(list [rid=resource =ship]) + ~(tap by failed-kicks) + =. tracking + (~(uni by tracking) failed-kicks) + =. failed-kicks ~ + |- ^- (quip card _state) + ?~ failures + [acc-cards state] + =, failures + =^ crds state + (handle-kick:hc i) + $(failures t, acc-cards (weld acc-cards crds)) + :: ++ missing-subscriptions |= tracking=(map resource ship) ^- (list card:agent:gall) @@ -232,15 +263,9 @@ (de-path:resource t.t.t.t.wire) ?+ -.sign (on-agent:def wire sign) %kick - =/ pax=(unit path) - (on-pull-kick:og rid) - ?^ pax - :_ this - ~[(watch-resource:hc rid u.pax)] - =. tracking - (~(del by tracking) rid) - :_ this - ~[give-update] + =^ cards state + (handle-kick:hc rid src.bowl) + [cards this] :: %watch-ack ?~ p.sign @@ -286,6 +311,58 @@ -- |_ =bowl:gall +* og ~(. pull-hook bowl) + ++ mule-scry + |= [ref=* raw=*] + =/ pax=(unit path) + ((soft path) raw) + ?~ pax ~ + ?. ?=([@ @ @ @ *] u.pax) ~ + =/ ship + (slaw %p i.t.u.pax) + =/ ved + (slay i.t.t.t.u.pax) + =/ dat + ?~ ved now.bowl + =/ cas=(unit case) + ((soft case) p.u.ved) + ?~ cas now.bowl + ?: ?=(%da -.u.cas) + p.u.cas + now.bowl + :: catch bad gall scries early + ?: ?& =((end 3 1 i.u.pax) %g) + ?| !=(`our.bowl ship) + !=(dat now.bowl) + == + == + ~ + ``.^(* u.pax) + ++ handle-kick + |= [rid=resource =ship] + ^- (quip card _state) + =/ res=toon + (mock [|.((on-pull-kick:og rid)) %9 2 %0 1] mule-scry) + =/ pax=(unit path) + !< (unit path) + :- -:!>(*(unit path)) + ?:(?=(%0 -.res) p.res ~) + =? failed-kicks !?=(%0 -.res) + =/ tang + :+ leaf+"failed kick handler, please report" + leaf+"{} in {(trip dap.bowl)}" + ?: ?=(%2 -.res) + p.res + ?> ?=(%1 -.res) + (turn `(list *)`p.res (cork path smyt)) + %- (slog tang) + (~(put by failed-kicks) rid ship) + ?^ pax + :_ state + (watch-resource rid u.pax) + =. tracking + (~(del by tracking) rid) + :_ state + ~[give-update] :: ++ poke-hook-action |= =action From e8ba12c7e342cc8be9ecb2d779ebb1cae53bd403 Mon Sep 17 00:00:00 2001 From: Liam Fitzgerald Date: Fri, 4 Dec 2020 11:49:10 +1000 Subject: [PATCH 3/3] pull-hook: L review --- pkg/arvo/lib/pull-hook.hoon | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/arvo/lib/pull-hook.hoon b/pkg/arvo/lib/pull-hook.hoon index 78566ad3c2..606c057ae0 100644 --- a/pkg/arvo/lib/pull-hook.hoon +++ b/pkg/arvo/lib/pull-hook.hoon @@ -158,6 +158,7 @@ og ~(. pull-hook bowl) hc ~(. +> bowl) def ~(. (default-agent this %|) bowl) + :: ++ on-init ^- [(list card:agent:gall) agent:gall] =^ cards pull-hook @@ -311,6 +312,7 @@ -- |_ =bowl:gall +* og ~(. pull-hook bowl) + :: ++ mule-scry |= [ref=* raw=*] =/ pax=(unit path)