clay: sync files using remote scry

When clay wants to download blobs from a foreign ship, it attempts doing
this using the new remote scry protocol.

If it doesn't receive a response within ~m1, it falls back to using the
old ames-based syncing instead. We remember this "prefer ames" state for
the specific ship for an hour, after which we'll begin trying the scry
flow again.

Compiles, but untested. Some TODOs and REVIEWs remaining herein.

We intentionally leave the dist-upgraded flag in state to avoid
cluttering the diff here. The next commit will remove it.
This commit is contained in:
fang 2022-02-15 00:51:54 +01:00
parent c9871ad2ef
commit c02274dfdd
No known key found for this signature in database
GPG Key ID: EB035760C1BBA972

View File

@ -170,6 +170,7 @@
pud=(unit [=desk =yoki]) :: pending update pud=(unit [=desk =yoki]) :: pending update
:: REMOVE on next upgrade :: REMOVE on next upgrade
dist-upgraded=_| :: are we in dist yet? dist-upgraded=_| :: are we in dist yet?
sad=(map ship @da) :: scry known broken
== :: == ::
:: ::
:: Object store. :: Object store.
@ -235,8 +236,9 @@
+$ update-state +$ update-state
$: =duct $: =duct
=rave =rave
scry=(unit @da) :: timeout timer if scry
have=(map lobe blob) have=(map lobe blob)
need=(list lobe) need=(list [=path =lobe])
nako=(qeu (unit nako)) nako=(qeu (unit nako))
busy=_| busy=_|
== ==
@ -292,7 +294,7 @@
$>(%what waif) :: $>(%what waif) ::
== :: == ::
$: %a :: to %ames $: %a :: to %ames
$>(%plea task:ames) :: $>(?(%plea %keen %yawn) task:ames) ::
== :: == ::
$: %b :: to %behn $: %b :: to %behn
$> $? %drip :: $> $? %drip ::
@ -332,6 +334,7 @@
$> $? %boon :: response $> $? %boon :: response
%done :: (n)ack %done :: (n)ack
%lost :: lost boon %lost :: lost boon
%tune :: scry response
== :: == ::
gift:ames :: gift:ames ::
== :: == ::
@ -1320,6 +1323,20 @@
=/ =path [%question desk (scot %ud index) ~] =/ =path [%question desk (scot %ud index) ~]
(emit duct %pass wire %a %plea ship %c path `riff-any`[%1 riff]) (emit duct %pass wire %a %plea ship %c path `riff-any`[%1 riff])
:: ::
++ send-over-scry
|= [kind=@ta =duct =ship index=@ud =desk =mood]
^- [timeout=@da _..send-over-scry]
=/ =time (add now ~m1)
=/ =wire /[kind]/(scot %p ship)/[desk]/(scot %ud index)
=/ =path
=, mood
[(cat 3 %c care) (scot %p ship) desk (scot case) path]
:- time
%- emil
:~ [hen %pass wire %a %keen path]
[hen %pass wire %b %wait time]
==
::
++ foreign-capable ++ foreign-capable
|= =rave |= =rave
|^ |^
@ -1354,6 +1371,27 @@
(run-if-future rove.wov |=(@da (bait hen +<))) (run-if-future rove.wov |=(@da (bait hen +<)))
|- ^+ +>+.$ |- ^+ +>+.$
=/ =rave (rove-to-rave rove.wov) =/ =rave (rove-to-rave rove.wov)
:: if it is a single request, and
:: :ship's remote scry isn't known to be broken,
:: or we learned it was broken more than an hour ago,
::
?: ?& ?=(%sing -.rave)
?| !(~(has by sad) her)
(lth now (add ~h1 (~(got by sad) her)))
== ==
:: send request as remote scry
::TODO can be deduplicated with the below?
::
=* inx nix.u.ref
=^ time=@da +>+.$
=< ?>(?=(^ ref) .)
(send-over-scry %warp-index hen her inx syd mood.rave)
%= +>+.$
nix.u.ref +(nix.u.ref)
bom.u.ref (~(put by bom.u.ref) inx [hen rave `time ~ ~ ~ |])
fod.u.ref (~(put by fod.u.ref) hen inx)
==
::
=. rave =. rave
?. ?=([%sing %v *] rave) rave ?. ?=([%sing %v *] rave) rave
[%many %| [%ud let.dom] case.mood.rave path.mood.rave] [%many %| [%ud let.dom] case.mood.rave path.mood.rave]
@ -1367,7 +1405,7 @@
(send-over-ames hen her inx syd `rave) (send-over-ames hen her inx syd `rave)
%= +>+.$ %= +>+.$
nix.u.ref +(nix.u.ref) nix.u.ref +(nix.u.ref)
bom.u.ref (~(put by bom.u.ref) inx [hen rave ~ ~ ~ |]) bom.u.ref (~(put by bom.u.ref) inx [hen rave ~ ~ ~ ~ |])
fod.u.ref (~(put by fod.u.ref) hen inx) fod.u.ref (~(put by fod.u.ref) hen inx)
== ==
:: ::
@ -1457,7 +1495,7 @@
?~ yen ?~ yen
=. lab.dom (~(put by lab.dom) bel yon) =. lab.dom (~(put by lab.dom) bel yon)
..park ..park
:: an aeon is bound to this label, :: an aeon is bound to this label,
:: but it is the same as the existing one, so we no-op :: but it is the same as the existing one, so we no-op
:: ::
?: =(u.yen yon) ?: =(u.yen yon)
@ -2992,11 +3030,24 @@
:: ::
?~ nux=(~(get by fod.u.ref) hen) ?~ nux=(~(get by fod.u.ref) hen)
..cancel-request(ref `(unit rind)`ref) :: XX TMI ..cancel-request(ref `(unit rind)`ref) :: XX TMI
=/ sat (~(got by bom.u.ref) u.nux)
=: fod.u.ref (~(del by fod.u.ref) hen) =: fod.u.ref (~(del by fod.u.ref) hen)
bom.u.ref (~(del by bom.u.ref) u.nux) bom.u.ref (~(del by bom.u.ref) u.nux)
== ==
%. [hen her u.nux [syd ~]] :: cancel the request as appropriate
send-over-ames(ref `(unit rind)`ref) :: XX TMI ::
?~ scry.sat
%. [hen her u.nux [syd ~]]
send-over-ames(ref `(unit rind)`ref) :: XX TMI
=/ =path
~| [%strange-scried-request rave.sat]
?> ?=(%sing -.rave.sat)
=, mood.rave.sat
[(cat 3 %c care) (scot %p her) syd (scot case) path]
%- emil
:~ [hen %pass / %a %yawn path]
[hen %pass / %b %rest u.scry.sat]
==
:: ::
:: Handles a request. :: Handles a request.
:: ::
@ -3019,6 +3070,19 @@
..start-request ..start-request
(duce for u.new-sub) (duce for u.new-sub)
:: ::
::TODO should this try-fill-sub or w/e in case its now in our cache?
++ retry-with-ames
|= inx=@ud
^+ ..retry-with-ames
~| [%strange-retry-no-request her syd inx]
?> ?=(^ ref)
=/ sat=update-state (~(got by bom.u.ref) inx)
=. ..retry-with-ames
=< ?>(?=(^ ref) .)
(send-over-ames hen her inx syd `rave.sat)
=. bom.u.ref (~(put by bom.u.ref) inx sat(scry ~))
..retry-with-ames
::
:: Called when a foreign ship answers one of our requests. :: Called when a foreign ship answers one of our requests.
:: ::
:: If it's a `%many` request, process in +take-foreign-update :: If it's a `%many` request, process in +take-foreign-update
@ -3147,6 +3211,12 @@
[u.ruv |] [u.ruv |]
=/ done=? | =/ done=? |
=. hen duct.sat =. hen duct.sat
:: if the request was done over scry, clear the timeout timer
::
=? ..foreign-update ?=(^ scry.sat)
=< ?>(?=(^ ref) .)
(emit hen %pass / %b %rest u.scry.sat)
=. scry.sat ~
|% |%
++ abet ++ abet
^+ ..foreign-update ^+ ..foreign-update
@ -3169,27 +3239,27 @@
work work
?> ?=(%nako p.r.u.rut) ?> ?=(%nako p.r.u.rut)
=/ nako ;;(nako q.r.u.rut) =/ nako ;;(nako q.r.u.rut)
=/ missing (missing-blobs nako) =. need.sat (welp need.sat (missing-blobs nako))
=. need.sat `(list lobe)`(welp need.sat ~(tap in missing))
=. nako.sat (~(put to nako.sat) ~ nako) =. nako.sat (~(put to nako.sat) ~ nako)
work work
:: ::
++ missing-blobs ++ missing-blobs
|= =nako |= =nako
^- (set lobe) ^- (list [path lobe])
=/ yakis ~(tap in lar.nako) =/ yakis ~(tap in lar.nako)
|- ^- (set lobe) |- ^- (list [path lobe])
=* yaki-loop $ =* yaki-loop $
?~ yakis ?~ yakis
~ ~
=/ lobes=(list [=path =lobe]) ~(tap by q.i.yakis) =/ lobes=(list [=path =lobe]) ~(tap by q.i.yakis)
|- ^- (set lobe) |- ^- (list [path lobe])
=* blob-loop $ =* blob-loop $
?~ lobes ?~ lobes
yaki-loop(yakis t.yakis) yaki-loop(yakis t.yakis)
?: (~(has by lat.ran) lobe.i.lobes) ?: (~(has by lat.ran) lobe.i.lobes)
blob-loop(lobes t.lobes) blob-loop(lobes t.lobes)
(~(put in blob-loop(lobes t.lobes)) lobe.i.lobes) :- i.lobes
blob-loop(lobes t.lobes)
:: ::
:: Receive backfill response :: Receive backfill response
:: ::
@ -3202,7 +3272,10 @@
!(~(has by lat.ran) q.q.blob) !(~(has by lat.ran) q.q.blob)
!(~(has by have.sat) q.q.blob) !(~(has by have.sat) q.q.blob)
== ==
[q.q.blob need.sat] ::NOTE we have already fallen back to the ames case,
:: so we do not need the real path here anymore.
::REVIEW right? or is this considered too fragile?
[[/ q.q.blob] need.sat]
:: We can't put a blob in lat.ran if its parent isn't already :: We can't put a blob in lat.ran if its parent isn't already
:: there. Unions are in reverse order so we don't overwrite :: there. Unions are in reverse order so we don't overwrite
:: existing blobs. :: existing blobs.
@ -3244,19 +3317,34 @@
:: another desk updating concurrently, or a previous update on this :: another desk updating concurrently, or a previous update on this
:: same desk). :: same desk).
:: ::
?: ?| (~(has by lat.ran) i.need.sat) ?: ?| (~(has by lat.ran) lobe.i.need.sat)
(~(has by have.sat) i.need.sat) (~(has by have.sat) lobe.i.need.sat)
== ==
$(need.sat t.need.sat) $(need.sat t.need.sat)
:: Otherwise, fetch the next blob :: Otherwise, fetch the next blob
:: ::
=/ =fill [%0 syd i.need.sat] =^ time=(unit @da) ..foreign-update
=/ =wire /back-index/(scot %p her)/[syd]/(scot %ud inx)
=/ =path [%backfill syd (scot %ud inx) ~]
=. ..foreign-update
=< ?>(?=(^ ref) .) =< ?>(?=(^ ref) .)
:: if it is a single request, and
:: :ship's remote scry isn't known to be broken,
:: or we learned it was broken more than an hour ago,
::
?: ?& ?=(%sing -.rave.sat)
?| !(~(has by sad) her)
(lth now (add ~h1 (~(got by sad) her)))
== ==
:: make the request over remote scry
::
=/ =mood [%x case.mood.rave.sat path.i.need.sat]
[`- +]:(send-over-scry %back-index hen her inx syd mood)
:: otherwise, request over ames
::
:- ~
=/ =wire /back-index/(scot %p her)/[syd]/(scot %ud inx)
=/ =path [%backfill syd (scot %ud inx) ~]
=/ =fill [%0 syd lobe.i.need.sat]
(emit hen %pass wire %a %plea her %c path fill) (emit hen %pass wire %a %plea her %c path fill)
..abet(busy.sat &) ..abet(busy.sat &, scry.sat time)
:: ::
:: When we get a %w foreign update, store this in our state. :: When we get a %w foreign update, store this in our state.
:: ::
@ -4587,16 +4675,53 @@
++ load ++ load
=> |% => |%
+$ raft-any +$ raft-any
$% [%10 raft-10] $% [%11 raft-11]
[%10 raft-10]
[%9 raft-9] [%9 raft-9]
[%8 raft-8] [%8 raft-8]
[%7 raft-7] [%7 raft-7]
[%6 raft-6] [%6 raft-6]
== ==
+$ raft-10 raft +$ raft-11 raft
+$ raft-10
$: rom=room
hoy=(map ship rung-10)
ran=rang
mon=(map term beam)
hez=(unit duct)
cez=(map @ta crew)
pud=(unit [=desk =yoki])
dist-upgraded=_|
==
+$ rung-10
$: rus=(map desk rede-10)
==
+$ rede-10
$: lim=@da
ref=(unit rind-10)
qyx=cult
dom=dome
per=regs
pew=regs
fiz=melt
==
+$ rind-10
$: nix=@ud
bom=(map @ud update-state-10)
fod=(map duct @ud)
haw=(map mood (unit cage))
==
+$ update-state-10
$: =duct
=rave
have=(map lobe blob)
need=(list lobe)
nako=(qeu (unit nako))
busy=_|
==
+$ raft-9 +$ raft-9
$: rom=room :: domestic $: rom=room :: domestic
hoy=(map ship rung) :: foreign hoy=(map ship rung-10) :: foreign
ran=rang :: hashes ran=rang :: hashes
mon=(map term beam) :: mount points mon=(map term beam) :: mount points
hez=(unit duct) :: sync duct hez=(unit duct) :: sync duct
@ -4637,7 +4762,7 @@
== ==
+$ rede-8 +$ rede-8
$: lim=@da $: lim=@da
ref=(unit rind) ref=(unit rind-10)
qyx=cult qyx=cult
dom=dome-8 dom=dome-8
per=regs per=regs
@ -4668,7 +4793,7 @@
== ==
+$ rede-7 +$ rede-7
$: lim=@da $: lim=@da
ref=(unit rind) ref=(unit rind-10)
qyx=cult qyx=cult
dom=dome-8 dom=dome-8
per=regs per=regs
@ -4705,7 +4830,7 @@
== ==
+$ rede-6 +$ rede-6
$: lim=@da $: lim=@da
ref=(unit rind) ref=(unit rind-10)
qyx=cult qyx=cult
dom=dome-6 dom=dome-6
per=regs per=regs
@ -4715,11 +4840,12 @@
-- --
|= old=raft-any |= old=raft-any
|^ |^
=? old ?=(%6 -.old) 7+(raft-6-to-7 +.old) =? old ?=(%6 -.old) 7+(raft-6-to-7 +.old)
=? old ?=(%7 -.old) 8+(raft-7-to-8 +.old) =? old ?=(%7 -.old) 8+(raft-7-to-8 +.old)
=? old ?=(%8 -.old) 9+(raft-8-to-9 +.old) =? old ?=(%8 -.old) 9+(raft-8-to-9 +.old)
=? old ?=(%9 -.old) 10+(raft-9-to-10 +.old) =? old ?=(%9 -.old) 10+(raft-9-to-10 +.old)
?> ?=(%10 -.old) =? old ?=(%10 -.old) 11+(raft-10-to-11 +.old)
?> ?=(%11 -.old)
..^^$(ruf +.old) ..^^$(ruf +.old)
:: +raft-6-to-7: delete stale ford caches (they could all be invalid) :: +raft-6-to-7: delete stale ford caches (they could all be invalid)
:: ::
@ -4778,7 +4904,7 @@
|= =rung-8 |= =rung-8
%- ~(run by rus.rung-8) %- ~(run by rus.rung-8)
|= =rede-8 |= =rede-8
^- rede ^- rede-10
=/ dom dom.rede-8 =/ dom dom.rede-8
rede-8(dom [ank.dom let.dom hit.dom lab.dom mim.dom *ford-cache]) rede-8(dom [ank.dom let.dom hit.dom lab.dom mim.dom *ford-cache])
== ==
@ -4787,6 +4913,26 @@
|= raf=raft-9 |= raf=raft-9
^- raft-10 ^- raft-10
raf(pud [pud.raf dist-upgraded=|]) raf(pud [pud.raf dist-upgraded=|])
:: +raft-10-to-11: remove dist flag, add scry availability tracker
::
++ raft-10-to-11
|= raf=raft-10
^- raft-11
=- raf(hoy -, dist-upgraded [dist-upgraded.raf ~])
%- ~(run by hoy.raf)
|= =rung-10
%- ~(run by rus.rung-10)
|= =rede-10
^- rede
=- rede-10(ref -)
?~ ref.rede-10 ~
=- ref.rede-10(bom.u -)
%- ~(run by bom.u.ref.rede-10)
|= update-state-10
^- update-state
::NOTE putting / paths is fine, the path is only used
:: when dealing with scry responses
[duct rave ~ have (turn need (lead /)) nako busy]
-- --
:: ::
++ scry :: inspect ++ scry :: inspect
@ -4911,16 +5057,36 @@
:: TODO better error handling :: TODO better error handling
!! !!
:: ::
%boon ?(%boon %tune)
=+ ;; res=(unit rand) payload.hin
::
=/ her=ship (slav %p i.t.tea) =/ her=ship (slav %p i.t.tea)
=/ =desk (slav %tas i.t.t.tea) =/ =desk (slav %tas i.t.t.tea)
=/ index=@ud (slav %ud i.t.t.t.tea) =/ index=@ud (slav %ud i.t.t.t.tea)
:: ::
=^ mos ruf =^ mos ruf
=; res=(unit rand)
=/ den ((de now rof hen ruf) her desk)
abet:(take-foreign-answer:den index res)
?: ?=(%boon +<.hin) ;;((unit rand) payload.hin)
%+ bind data.hin
|= =(cask)
^- rand
::REVIEW this feels a bit dumb, but the alternative
:: is reconstructing it from ref:den instead,
:: which is also weird in its own ways.
=+ (need (de-omen path.hin))
=/ =care ;;(care ?@(vis (rsh 3 vis) car.vis))
[[care r.bem q.bem] s.bem cask]
[mos ..^$]
::
%wake
=/ her=ship (slav %p i.t.tea)
=/ =desk (slav %tas i.t.t.tea)
=/ index=@ud (slav %ud i.t.t.t.tea)
~& [%clay %scry-broken her]
=^ mos ruf
=. sad.ruf (~(put by sad.ruf) her now)
=/ den ((de now rof hen ruf) her desk) =/ den ((de now rof hen ruf) her desk)
abet:(take-foreign-answer:den index res) abet:(retry-with-ames:den index)
[mos ..^$] [mos ..^$]
== ==
:: ::
@ -4940,13 +5106,16 @@
:: TODO better error handling :: TODO better error handling
!! !!
:: ::
%boon ?(%boon %tune)
=+ ;; =blob payload.hin
::
=/ her=ship (slav %p i.t.tea) =/ her=ship (slav %p i.t.tea)
=/ =desk (slav %tas i.t.t.tea) =/ =desk (slav %tas i.t.t.tea)
=/ index=@ud (slav %ud i.t.t.t.tea) =/ index=@ud (slav %ud i.t.t.t.tea)
:: ::
=/ =blob
?: ?=(%boon +<.hin) ;;(blob payload.hin)
?~ data.hin ~|(%now-what !!) ::TODO
[%direct (page-to-lobe u.data.hin) u.data.hin]
::
=^ mos ruf =^ mos ruf
=/ den ((de now rof hen ruf) her desk) =/ den ((de now rof hen ruf) her desk)
abet:abet:(take-backfill:(foreign-update:den index) blob) abet:abet:(take-backfill:(foreign-update:den index) blob)
@ -5018,6 +5187,7 @@
:: handled in the wire dispatcher :: handled in the wire dispatcher
:: ::
%boon !! %boon !!
%tune !!
%lost !! %lost !!
%onto !! %onto !!
%unto !! %unto !!