From 67efb2ad7ca8a6f36d7d77cd77ccca61c4523b2d Mon Sep 17 00:00:00 2001 From: Ted Blackman Date: Tue, 2 Aug 2022 23:03:34 +0300 Subject: [PATCH 1/8] ames: don't start pump timers on recork --- pkg/arvo/sys/vane/ames.hoon | 40 ++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/pkg/arvo/sys/vane/ames.hoon b/pkg/arvo/sys/vane/ames.hoon index e583634f5c..73b7e690c8 100644 --- a/pkg/arvo/sys/vane/ames.hoon +++ b/pkg/arvo/sys/vane/ames.hoon @@ -764,7 +764,7 @@ [%hear =message-num =ack-meat] [%near =naxplanation] [%prod ~] - [%wake ~] + [%wake recork=?] == :: $message-pump-gift: effect from |message-pump :: @@ -795,7 +795,7 @@ $% [%hear =message-num =fragment-num] [%done =message-num lag=@dr] [%halt ~] - [%wake current=message-num] + [%wake recork=? current=message-num] [%prod ~] == :: $packet-pump-gift: effect from |packet-pump @@ -1835,7 +1835,9 @@ :: =/ =channel [[our her.u.res] now channel-state -.u.state] :: - abet:(on-wake:(make-peer-core u.state channel) bone.u.res error) + =< abet + %- on-wake:(make-peer-core u.state channel) + [recork=& bone.u.res error] -- :: +on-init: first boot; subscribe to our info from jael :: @@ -2451,8 +2453,15 @@ :: +on-wake: handle timer expiration :: ++ on-wake - |= [=bone error=(unit tang)] + |= [recork=? =bone error=(unit tang)] ^+ peer-core + :: ignore spurious pump timers for %cork's + :: + :: This is here to fix canaries that had spurious pump timers. + :: + ?: &(!recork (~(has in closing.peer-state) bone)) + ~> %slog.0^leaf/"ames: dropping pump wake while recorking {}" + peer-core :: if we previously errored out, print and reset timer for later :: :: This really shouldn't happen, but if it does, make sure we @@ -2521,7 +2530,7 @@ peer-core :: maybe resend some timed out packets :: - (run-message-pump bone %wake ~) + (run-message-pump bone %wake recork) :: +send-shut-packet: fire encrypted packet at rcvr and maybe sponsors :: ++ send-shut-packet @@ -2902,12 +2911,15 @@ |= task=message-pump-task ^+ [gifts state] :: - =~ (dispatch-task task) - feed-packets - (run-packet-pump %halt ~) - assert - [(flop gifts) state] - == + =. message-pump (dispatch-task task) + =. message-pump feed-packets + :: don't set new pump timer if triggered by a recork timer + :: + =? message-pump =([%wake recork=&] task) + (run-packet-pump %halt ~) + :: + =. message-pump assert + [(flop gifts) state] :: +dispatch-task: perform task-specific processing :: ++ dispatch-task @@ -2917,7 +2929,7 @@ ?- -.task %prod (run-packet-pump %prod ~) %memo (on-memo message-blob.task) - %wake (run-packet-pump %wake current.state) + %wake (run-packet-pump %wake recork.task current.state) %hear ?- -.ack-meat.task %& @@ -3159,7 +3171,7 @@ ?- -.task %hear (on-hear [message-num fragment-num]:task) %done (on-done message-num.task) - %wake (on-wake current.task) + %wake (on-wake recork.task current.task) %prod on-prod %halt set-wake == @@ -3186,7 +3198,7 @@ :: +on-wake: handle packet timeout :: ++ on-wake - |= current=message-num + |= [recork=? current=message-num] ^+ packet-pump :: assert temporal coherence :: From e5e4c8602d7bfe96083685073aa84204d33f4e9d Mon Sep 17 00:00:00 2001 From: Ted Blackman Date: Tue, 2 Aug 2022 23:15:43 +0300 Subject: [PATCH 2/8] ames: properly set recork flag in +on-wake --- pkg/arvo/sys/vane/ames.hoon | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/arvo/sys/vane/ames.hoon b/pkg/arvo/sys/vane/ames.hoon index 73b7e690c8..74c0bc68c3 100644 --- a/pkg/arvo/sys/vane/ames.hoon +++ b/pkg/arvo/sys/vane/ames.hoon @@ -1812,15 +1812,15 @@ (request-attestation u.ship) :: |^ - ?. ?=([%recork ~] wire) (handle-single-wire wire) + ?. ?=([%recork ~] wire) (handle-single-wire wire recork=|) =/ wires=(list ^wire) ~(tap in corks.ames-state) |- ^+ event-core ?^ wires - $(wires t.wires, event-core (handle-single-wire i.wires)) + $(wires t.wires, event-core (handle-single-wire i.wires recork=&)) (emit duct %pass /recork %b %wait `@da`(add now ~d1)) :: ++ handle-single-wire - |= =^wire + |= [=^wire recork=?] ^+ event-core =/ res=(unit [her=ship =bone]) (parse-pump-timer-wire wire) ?~ res @@ -1837,7 +1837,7 @@ :: =< abet %- on-wake:(make-peer-core u.state channel) - [recork=& bone.u.res error] + [recork bone.u.res error] -- :: +on-init: first boot; subscribe to our info from jael :: From b860bef066922352f04adc1cdf6fc912eeff3493 Mon Sep 17 00:00:00 2001 From: fang Date: Tue, 2 Aug 2022 23:23:48 +0200 Subject: [PATCH 3/8] ames: correctly avoid setting pump timer The condition got butchered during refactor: instead of avoiding the creation of pump timers during recork wake, it was setting them _exclusively_ during recork wake. --- pkg/arvo/sys/vane/ames.hoon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/arvo/sys/vane/ames.hoon b/pkg/arvo/sys/vane/ames.hoon index 74c0bc68c3..ef70aef2b9 100644 --- a/pkg/arvo/sys/vane/ames.hoon +++ b/pkg/arvo/sys/vane/ames.hoon @@ -2915,7 +2915,7 @@ =. message-pump feed-packets :: don't set new pump timer if triggered by a recork timer :: - =? message-pump =([%wake recork=&] task) + =? message-pump !=([%wake recork=&] task) (run-packet-pump %halt ~) :: =. message-pump assert From 7c01ee9b35e0e57d690150b173538481e05de1fa Mon Sep 17 00:00:00 2001 From: fang Date: Wed, 3 Aug 2022 19:37:53 +0200 Subject: [PATCH 4/8] ames: account for corked flows during %stir Also patches the behn scry in %stir handling to be correct for latest behn. --- pkg/arvo/sys/vane/ames.hoon | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/arvo/sys/vane/ames.hoon b/pkg/arvo/sys/vane/ames.hoon index ef70aef2b9..aa075a0123 100644 --- a/pkg/arvo/sys/vane/ames.hoon +++ b/pkg/arvo/sys/vane/ames.hoon @@ -1482,9 +1482,10 @@ =/ snds=(list (list [ship bone message-pump-state])) %+ turn states |= [=ship peer-state] - %+ turn ~(tap by snd) + %+ murn ~(tap by snd) |= [=bone =message-pump-state] - [ship bone message-pump-state] + ?: (~(has in closing) bone) ~ + `[ship bone message-pump-state] =/ next-wakes %+ turn `(list [ship bone message-pump-state])`(zing snds) |= [=ship =bone message-pump-state] @@ -1498,7 +1499,7 @@ %- silt ;; (list [@da ^duct]) =< q.q %- need %- need - (rof ~ %b [[our %timers da+now] /]) + (rof ~ %bx [[our %$ da+now] /debug/timers]) =/ to-stir %+ skip next-real-wakes |= [=ship =bone =@da] From e16c4997241d244fb31d33c07f1a55736a9d4f56 Mon Sep 17 00:00:00 2001 From: fang Date: Wed, 3 Aug 2022 20:33:04 +0200 Subject: [PATCH 5/8] ames: make recork timer faster This will aid visibility and debugging. We'll probably want to make this slower again later. --- pkg/arvo/sys/vane/ames.hoon | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/arvo/sys/vane/ames.hoon b/pkg/arvo/sys/vane/ames.hoon index aa075a0123..20d0445a17 100644 --- a/pkg/arvo/sys/vane/ames.hoon +++ b/pkg/arvo/sys/vane/ames.hoon @@ -867,7 +867,7 @@ =^ moves adult-gate (call:adult-core duct dud task) %- molt ~> %slog.0^leaf/"ames: init daily recork timer" - :_(moves [duct %pass /recork %b %wait `@da`(add now ~d1)]) + :_(moves [duct %pass /recork %b %wait `@da`(add now ~m20)]) :: %born: set .unix-duct and start draining .queued-events :: ?: ?=(%born -.task) @@ -954,7 +954,7 @@ [moves adult-gate] %- molt ~> %slog.0^leaf/"ames: init daily recork timer" - :_(moves [duct %pass /recork %b %wait `@da`(add now ~d1)]) + :_(moves [duct %pass /recork %b %wait `@da`(add now ~m20)]) :: set timer to drain next event :: =. moves :_(moves [duct %pass /larva %b %wait now]) @@ -1818,7 +1818,7 @@ |- ^+ event-core ?^ wires $(wires t.wires, event-core (handle-single-wire i.wires recork=&)) - (emit duct %pass /recork %b %wait `@da`(add now ~d1)) + (emit duct %pass /recork %b %wait `@da`(add now ~m20)) :: ++ handle-single-wire |= [=^wire recork=?] @@ -2836,7 +2836,7 @@ :: receiving the OTA, so we set up a timer to retry in one day. :: %- %+ trace msg.veb - |.("old publisher, resend %cork on bone={} in ~d1") + |.("old publisher, resend %cork on bone={} in ~m20") =/ =wire (make-pump-timer-wire her.channel target-bone) =. corks.ames-state (~(put in corks.ames-state) wire) peer-core From e6453bae3c0b2f52ac24e8b8c51ec8ed095ad2a2 Mon Sep 17 00:00:00 2001 From: fang Date: Wed, 3 Aug 2022 20:46:54 +0200 Subject: [PATCH 6/8] ames: more detailed "future msg" trace This additional information may come in handy when debugging this crash we shouldn't be seeing. --- pkg/arvo/sys/vane/ames.hoon | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/arvo/sys/vane/ames.hoon b/pkg/arvo/sys/vane/ames.hoon index 20d0445a17..335ba307ba 100644 --- a/pkg/arvo/sys/vane/ames.hoon +++ b/pkg/arvo/sys/vane/ames.hoon @@ -2996,7 +2996,13 @@ ^+ message-pump :: unsent messages from the future should never get acked :: - ~| [message-num next.state] + ~| :* bone=bone + mnum=message-num + next=next.state + unsent-messages=~(wyt to unsent-messages.state) + unsent-fragments=~(wyt to unsent-fragments.state) + any-live=!=(~ live.packet-pump-state.state) + == ?> (lth message-num next.state) :: ignore duplicate message acks :: From 7af527e605ee5d40dcfa6dc735b314e7be724c0c Mon Sep 17 00:00:00 2001 From: fang Date: Wed, 3 Aug 2022 20:52:03 +0200 Subject: [PATCH 7/8] ames: correctly measure queue size --- pkg/arvo/sys/vane/ames.hoon | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/arvo/sys/vane/ames.hoon b/pkg/arvo/sys/vane/ames.hoon index 335ba307ba..44373dc85f 100644 --- a/pkg/arvo/sys/vane/ames.hoon +++ b/pkg/arvo/sys/vane/ames.hoon @@ -2999,8 +2999,8 @@ ~| :* bone=bone mnum=message-num next=next.state - unsent-messages=~(wyt to unsent-messages.state) - unsent-fragments=~(wyt to unsent-fragments.state) + unsent-messages=~(wyt in unsent-messages.state) + unsent-fragments=~(wyt in unsent-fragments.state) any-live=!=(~ live.packet-pump-state.state) == ?> (lth message-num next.state) From 9433d3354dc35e0b62feeb808097bdd78930bdaf Mon Sep 17 00:00:00 2001 From: fang Date: Wed, 3 Aug 2022 21:03:49 +0200 Subject: [PATCH 8/8] ames: unsent-fragments is a list, not a tree Measure accordingly. --- pkg/arvo/sys/vane/ames.hoon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/arvo/sys/vane/ames.hoon b/pkg/arvo/sys/vane/ames.hoon index 44373dc85f..2d8fc5ee3d 100644 --- a/pkg/arvo/sys/vane/ames.hoon +++ b/pkg/arvo/sys/vane/ames.hoon @@ -3000,7 +3000,7 @@ mnum=message-num next=next.state unsent-messages=~(wyt in unsent-messages.state) - unsent-fragments=~(wyt in unsent-fragments.state) + unsent-fragments=(lent unsent-fragments.state) any-live=!=(~ live.packet-pump-state.state) == ?> (lth message-num next.state)