urbit/shrub
1
1
Fork 0
mirror of https://github.com/urbit/shrub.git synced 2024-12-19 00:13:12 +03:00
Code Issues Packages Projects Releases Wiki Activity

clay: convert all blobs to pages

Browse Source 
This converts the blob store from having deltas, directs, and
tombstones, to just having direct pages.  This simplifies a lot of code,
since we don't have to constantly ensure that deltas always have their
parent available.

This removes the hardcoded text diff logic from clay, which was
previously required for bootstrapping.

Over the wire, we handle both old and new requests and responses
transparently, so communication is normal in both directions across
ships which do or do not have this change.
This commit is contained in:
Philip Monk 2022-04-13 00:15:31 -07:00
parent 72fa850912
commit 8805e0ee85
3 changed files with 173 additions and 389 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
pkg/arvo/sys/lull.hoon
View File

@ -787,16 +787,6 @@
+$ ankh (axal [p=lobe q=cage]) :: fs node (new) +$ ankh (axal [p=lobe q=cage]) :: fs node (new)
+$ beam [[p=ship q=desk r=case] s=path] :: global name +$ beam [[p=ship q=desk r=case] s=path] :: global name
+$ beak [p=ship q=desk r=case] :: path prefix +$ beak [p=ship q=desk r=case] :: path prefix
+$ blob :: fs blob
$% [%delta p=lobe q=lobe r=page] :: delta on q
[%direct p=lobe q=page] :: immediate
[%dead p=lobe ~] :: tombstone
== ::
+$ blub :: over-the-wire blob
$% [%delta p=lobe q=[p=mark q=lobe] r=page] :: delta on q
[%direct p=lobe q=page] :: immediate
[%dead p=lobe ~] :: tombstone
== ::
+$ cable :: lib/sur/mark ref +$ cable :: lib/sur/mark ref
$: face=(unit term) :: $: face=(unit term) ::
file-path=term :: file-path=term ::
@ -841,7 +831,6 @@
%meet-that :: hers if conflict %meet-that :: hers if conflict
== :: == ::
+$ lobe @uvI :: blob ref +$ lobe @uvI :: blob ref
+$ maki [p=@ta q=@ta r=@ta s=path] ::
+$ miso :: ankh delta +$ miso :: ankh delta
$% [%del ~] :: delete $% [%del ~] :: delete
[%ins p=cage] :: insert [%ins p=cage] :: insert
@ -870,10 +859,9 @@
+$ norm (axal ?) :: tombstone policy +$ norm (axal ?) :: tombstone policy
+$ open $-(path vase) :: get prelude +$ open $-(path vase) :: get prelude
+$ page (cask *) :: untyped cage +$ page (cask *) :: untyped cage
+$ plop blub :: unvalidated blob
+$ rang :: repository +$ rang :: repository
$: hut=(map tako yaki) :: changes $: hut=(map tako yaki) :: changes
lat=(map lobe blob) :: data lat=(map lobe page) :: data
== :: == ::
+$ rant :: response to request +$ rant :: response to request
$: p=[p=care q=case r=desk] :: clade release book $: p=[p=care q=case r=desk] :: clade release book

528
pkg/arvo/sys/vane/clay.hoon
View File

@ -144,10 +144,17 @@
mut=(list (trel path lobe cage)) :: mutations mut=(list (trel path lobe cage)) :: mutations
== :: == ::
:: ::
:: Over-the-wire backfill request :: Over-the-wire backfill request/response
:: ::
+$ fill +$ fill
$% [%0 =desk =lobe] $% [%0 =desk =lobe]
[%1 =desk =lobe]
==
+$ fell
$% [%direct p=lobe q=page]
[%delta p=lobe q=[p=mark q=lobe] r=page]
[%dead p=lobe ~]
[%1 peg=(unit page)]
== ==
:: ::
:: Ford cache :: Ford cache
@ -160,10 +167,6 @@
tubes=(map mars [res=tube dez=(set [dir=? =path])]) tubes=(map mars [res=tube dez=(set [dir=? =path])])
== ==
:: ::
:: Hash of a blob, for lookup in the object store (lat.ran)
::
+$ lobe @uvI :: blob ref
::
:: New desk data. :: New desk data.
:: ::
:: Sent to other ships to update them about a particular desk. Includes a map :: Sent to other ships to update them about a particular desk. Includes a map
@ -174,7 +177,7 @@
$: gar=(map aeon tako) :: new ids $: gar=(map aeon tako) :: new ids
let=aeon :: next id let=aeon :: next id
lar=(set yaki) :: new commits lar=(set yaki) :: new commits
bar=(set plop) :: new content bar=~ :: new content
== :: == ::
:: ::
:: ::
@ -200,15 +203,6 @@
pud=(unit [=desk =yoki]) :: pending update pud=(unit [=desk =yoki]) :: pending update
== :: == ::
:: ::
:: Object store.
::
:: Maps of commit hashes to commits and content hashes to content.
::
+$ rang ::
$: hut=(map tako yaki) ::
lat=(map lobe blob) ::
== ::
::
:: Unvalidated response to a request. :: Unvalidated response to a request.
:: ::
:: Like a +$rant, but with a page of data rather than a cage of it. :: Like a +$rant, but with a page of data rather than a cage of it.
@ -263,8 +257,7 @@
+$ update-state +$ update-state
$: =duct $: =duct
=rave =rave
have=(map lobe blob) need=(list lobe)
need=[let=(list lobe) old=(list lobe)]
nako=(qeu (unit nako)) nako=(qeu (unit nako))
busy=_| busy=_|
== ==
@ -431,42 +424,14 @@
:: ::
[deletes changes] [deletes changes]
:: ::
:: Safely add items to an object store. ++ fell-to-page
:: |= =fell
:: It's important that first argument is the old and second is the new. ^- (unit page)
:: The rule is that if one side has a tombstone, we want to use the ?- -.fell
:: other side. Additionally, if the old one was %direct, we don't want %dead ~
:: to overwrite that with a %delta, since that could create loops where %direct `q.fell
:: they didn't exist. %delta ~
:: %1 peg.fell
++ uni-blobs
|= [old=(map lobe blob) new=(map lobe blob)]
^- (map lobe blob)
%- (~(uno by old) new)
|= [=lobe o=blob n=blob]
^- blob
?- -.o
%direct o
%delta o
%dead n
==
::
++ blob-to-blub
|= =blob
^- blub
?- -.blob
%dead blob
%direct blob
%delta blob(q [%blub q.blob])
==
::
++ blub-to-blob
|= =blub
^- blob
?- -.blub
%dead blub
%direct blub
%delta blub(q q.q.blub)
== ==
-- => -- =>
~% %clay + ~ ~% %clay + ~
@ -529,7 +494,7 @@
$: =ankh $: =ankh
deletes=(set path) deletes=(set path)
changes=(map path (each page lobe)) changes=(map path (each page lobe))
file-store=(map lobe blob) file-store=(map lobe page)
=ford-cache =ford-cache
== ==
-- --
@ -564,11 +529,11 @@
=. cycle.nub (~(put in cycle.nub) vale+path) =. cycle.nub (~(put in cycle.nub) vale+path)
::~> %slog.0^leaf/"ford: read file {(spud path)}" ::~> %slog.0^leaf/"ford: read file {(spud path)}"
?^ change=(~(get by changes) path) ?^ change=(~(get by changes) path)
=^ page nub =/ page
?: ?=(%& -.u.change) ?: ?=(%& -.u.change)
[p.u.change nub] p.u.change
~| %ugly-lobe^p.u.change^path ~| %tombstoned-file^path^p.u.change
(lobe-to-page p.u.change) (~(got by file-store) p.u.change)
=^ cage nub (validate-page path page) =^ cage nub (validate-page path page)
[cage nub] [cage nub]
?< (~(has in deletes) path) ?< (~(has in deletes) path)
@ -787,38 +752,6 @@
:: ~> %slog.0^leaf/"ford: make tube {<a>} -> {<b>}" :: ~> %slog.0^leaf/"ford: make tube {<a>} -> {<b>}"
:_(nub =>([..zuse gat=gat] |=(v=vase (slam gat v)))) :_(nub =>([..zuse gat=gat] |=(v=vase (slam gat v))))
:: ::
++ lobe-to-page
|= =lobe
^- [page state]
=/ =blob (~(got by file-store) lobe)
|- ^- [page state]
?- -.blob
%dead ~|(lobe-to-page-dead+lobe !!)
%direct [q.blob nub]
%delta
=/ [=parent=^lobe diff=page] [q r]:blob
=^ parent-page nub $(blob (~(got by file-store) parent-lobe))
=^ =cage nub (run-pact parent-page diff)
[[p q.q]:cage nub]
==
::
++ lobe-to-unit-page
|= =lobe
^- [(unit page) state]
=/ =blob (~(got by file-store) lobe)
|- ^- [(unit page) state]
?- -.blob
%dead [~ nub]
%direct [`q.blob nub]
%delta
=/ [=parent=^lobe diff=page] [q r]:blob
=^ parent-page nub $(blob (~(got by file-store) parent-lobe))
?~ parent-page
[~ nub]
=^ =cage nub (run-pact u.parent-page diff)
[`[p q.q]:cage nub]
==
::
++ validate-page ++ validate-page
|= [=path =page] |= [=path =page]
^- [cage state] ^- [cage state]
@ -1385,7 +1318,7 @@
(send-over-ames hen her inx syd `rave) (send-over-ames hen her inx syd `rave)
%= +>+.$ %= +>+.$
nix.u.ref +(nix.u.ref) nix.u.ref +(nix.u.ref)
bom.u.ref (~(put by bom.u.ref) inx [hen rave ~ [~ ~] ~ |]) bom.u.ref (~(put by bom.u.ref) inx [hen rave ~ ~ |])
fod.u.ref (~(put by fod.u.ref) hen inx) fod.u.ref (~(put by fod.u.ref) hen inx)
== ==
:: ::
@ -1560,7 +1493,7 @@
|= [updated=? =yoki =rang] |= [updated=? =yoki =rang]
^+ ..park ^+ ..park
=: hut.ran (~(uni by hut.rang) hut.ran) =: hut.ran (~(uni by hut.rang) hut.ran)
lat.ran (uni-blobs lat.ran lat.rang) lat.ran (~(uni by lat.rang) lat.ran)
== ==
=/ new-data=(map path (each page lobe)) =/ new-data=(map path (each page lobe))
?- -.yoki ?- -.yoki
@ -1569,7 +1502,7 @@
== ==
?. %- ~(all in new-data) :: use +all:in so we get the key ?. %- ~(all in new-data) :: use +all:in so we get the key
|= [=path tum=(each page lobe)] |= [=path tum=(each page lobe)]
?: |(?=(%& -.tum) !=(%dead -:(~(got by lat.ran) p.tum))) ?: |(?=(%& -.tum) (~(has by lat.ran) p.tum))
& &
(mean leaf/"clay: commit failed, file tombstoned: {<path>} {<`@uv`p.tum>}" ~) (mean leaf/"clay: commit failed, file tombstoned: {<path>} {<`@uv`p.tum>}" ~)
!! !!
@ -1611,11 +1544,11 @@
:: ::
=^ change-cages ford-cache.args (checkout-changes args changes) =^ change-cages ford-cache.args (checkout-changes args changes)
=/ sane-continuation (sane-changes changes change-cages) =/ sane-continuation (sane-changes changes change-cages)
=/ new-blobs=(map lobe blob) =/ new-pages=(map lobe page)
%- malt %- malt
%+ turn ~(tap by change-cages) %+ turn ~(tap by change-cages)
|= [=path =lobe =cage] |= [=path =lobe =cage]
[lobe %direct lobe [p q.q]:cage] [lobe [p q.q]:cage]
=/ data=(map path lobe) =/ data=(map path lobe)
%- ~(urn by new-data) %- ~(urn by new-data)
|= [=path value=(each page lobe)] |= [=path value=(each page lobe)]
@ -1638,7 +1571,7 @@
=: let.dom +(let.dom) =: let.dom +(let.dom)
hit.dom (~(put by hit.dom) +(let.dom) r.yaki) hit.dom (~(put by hit.dom) +(let.dom) r.yaki)
hut.ran (~(put by hut.ran) r.yaki yaki) hut.ran (~(put by hut.ran) r.yaki yaki)
lat.ran (uni-blobs lat.ran new-blobs) lat.ran (~(uni by new-pages) lat.ran)
== ==
=. file-store.args lat.ran =. file-store.args lat.ran
:: ::
@ -1690,14 +1623,9 @@
++ lobe-to-weft ++ lobe-to-weft
|= =lobe |= =lobe
^- weft ^- weft
=/ =blob (lobe-to-blob:ze lobe) =/ peg=(unit page) (~(get by lat.ran) lobe)
=/ =page ?~ peg ~|([%sys-kelvin-tombstoned syd] !!)
?- -.blob (page-to-weft u.peg)
%direct q.blob
%delta r.blob
%dead ~|([%sys-kelvin-tombstoned syd] !!)
==
(page-to-weft page)
:: ::
++ page-to-weft ++ page-to-weft
|= =page |= =page
@ -1817,21 +1745,10 @@
++ lobe-to-cord ++ lobe-to-cord
|= =lobe |= =lobe
^- @t ^- @t
=- ?:(?=(%& -<) p.- (of-wain:format p.-)) =/ peg=(unit page) (~(get by lat.ran) lobe)
|- ^- (each @t wain) ?~ peg
:: got:by ok since we must have a blob for anything at `let` ~|([%lobe-to-cord-tombstoned syd lobe] !!)
:: ;;(@t q.u.peg)
=/ =blob (lobe-to-blob:ze lobe)
?- -.blob
%dead ~|([%lobe-to-cord-tombstoned syd lobe] !!)
%direct [%& ;;(@t q.q.blob)]
%delta
:- %|
%+ lurk:differ
=- ?:(?=(%| -<) p.- (to-wain:format p.-))
$(lobe q.blob)
;;((urge cord) q.r.blob)
==
:: ::
:: Updated q.yaki :: Updated q.yaki
:: ::
@ -1964,20 +1881,15 @@
?~ tak ?~ tak
~ ~
=/ =yaki (~(got by hut.ran) u.tak) =/ =yaki (~(got by hut.ran) u.tak)
:: Assert all blobs hash to their lobe :: Assert all pages hash to their lobe
:: ::
=/ foo =/ foo
%- ~(urn by lat.ran) %- ~(urn by lat.ran)
|= [=lobe =blob] |= [=lobe =page]
?- -.blob =/ actual-lobe=^lobe `@uv`(page-to-lobe page)
%dead ~ ~| [%bad-lobe have=lobe need=actual-lobe]
%delta ~ ?> =(lobe actual-lobe)
%direct ~
=/ actual-lobe=^lobe `@uv`(page-to-lobe q.blob)
~| [lobe p.blob actual-lobe]
?> &(=(lobe p.blob) =(lobe actual-lobe))
~
==
:: Assert we calculated the same change-cages w/o cache :: Assert we calculated the same change-cages w/o cache
:: ::
:: ? remove deletes :: ? remove deletes
@ -2219,7 +2131,7 @@
next-yaki merged-yaki next-yaki merged-yaki
merges t.merges merges t.merges
hut.ran (~(put by hut.ran) r.merged-yaki merged-yaki) hut.ran (~(put by hut.ran) r.merged-yaki merged-yaki)
lat.rag (uni-blobs lat.rag lat.u.merge-result) lat.rag (~(uni by lat.u.merge-result) lat.rag)
parents [(~(got by hit.ali-dom) let.ali-dom) parents] parents [(~(got by hit.ali-dom) let.ali-dom) parents]
== ==
== ==
@ -2256,7 +2168,7 @@
(park | new.u.mr ~ lat.u.mr) (park | new.u.mr ~ lat.u.mr)
== ==
:: ::
+$ merge-result [conflicts=(set path) new=yoki lat=(map lobe blob)] +$ merge-result [conflicts=(set path) new=yoki lat=(map lobe page)]
:: ::
++ merge-helper ++ merge-helper
|= [=ali=ship =ali=desk =germ ali-dome=dome:clay next-yaki=(unit yaki)] |= [=ali=ship =ali=desk =germ ali-dome=dome:clay next-yaki=(unit yaki)]
@ -2560,9 +2472,7 @@
++ lobe-to-cage ++ lobe-to-cage
|= =lobe |= =lobe
^- (unit cage) ^- (unit cage)
=^ peg=(unit page) fod.dom =/ peg=(unit page) (~(get by lat.ran) lobe)
%- wrap:fusion
(lobe-to-unit-page:(ford:fusion static-ford-args) lobe)
?~ peg ?~ peg
~ ~
=^ =cage fod.dom =^ =cage fod.dom
@ -2688,14 +2598,15 @@
=< .(old q.bob) =< .(old q.bob)
|= [[pax=path ~] old=(map path lobe)] |= [[pax=path ~] old=(map path lobe)]
(~(del by old) pax) (~(del by old) pax)
=/ [hot=(map path lobe) lat=(map lobe blob)] :: new content =/ [hot=(map path lobe) lat=(map lobe page)] :: new content
%+ roll ~(tap by both-patched) %+ roll ~(tap by both-patched)
|= [[pax=path cay=cage] hat=(map path lobe) lat=(map lobe blob)] |= [[pax=path cay=cage] hat=(map path lobe) lat=(map lobe page)]
=/ =blob [%direct (page-to-lobe [p q.q]:cay) [p q.q]:cay] =/ =page [p q.q]:cay
:- (~(put by hat) pax p.blob) =/ =lobe (page-to-lobe page)
?: (~(has by lat) p.blob) :- (~(put by hat) pax lobe)
?: (~(has by lat) lobe)
lat lat
(uni-blobs lat (malt [p.blob blob] ~)) (~(uni by (malt [lobe page] ~)) lat)
=/ hat=(map path lobe) :: all the content =/ hat=(map path lobe) :: all the content
%- ~(uni by old) %- ~(uni by old)
%- ~(uni by new.dal) %- ~(uni by new.dal)
@ -3078,14 +2989,20 @@
:: ::
:: Respond to backfill request :: Respond to backfill request
:: ::
:: Maybe should verify the requester is allowed to access this blob? :: Maybe should verify the requester is allowed to access this lobe?
:: ::
++ give-backfill ++ give-backfill
|= =lobe |= [ver=?(%0 %1) =lobe]
^+ ..give-backfill ^+ ..give-backfill
(emit hen %give %boon (blob-to-blub (lobe-to-blob:ze lobe))) =/ peg=(unit page) (~(get by lat.ran) lobe)
=/ res
?- ver
%0 ?~(peg ~ [%direct lobe u.peg])
%1 [%1 peg]
==
(emit hen %give %boon res)
:: ::
:: Ingest foreign update, requesting missing blobs if necessary :: Ingest foreign update, requesting missing lobes if necessary
:: ::
++ foreign-update ++ foreign-update
|= inx=@ud |= inx=@ud
@ -3120,123 +3037,75 @@
work work
?> ?=(%nako p.r.u.rut) ?> ?=(%nako p.r.u.rut)
=/ nako ;;(nako q.r.u.rut) =/ nako ;;(nako q.r.u.rut)
=/ missing (missing-blobs nako) =/ missing (missing-lobes nako)
=. let.need.sat (welp let.need.sat ~(tap in let.missing)) =. need.sat (welp need.sat ~(tap in missing))
=. old.need.sat (welp old.need.sat ~(tap in old.missing))
=. lat.ran (uni-blobs lat.ran hav.missing)
=. nako.sat (~(put to nako.sat) ~ nako) =. nako.sat (~(put to nako.sat) ~ nako)
work work
:: ::
++ missing-blobs ++ missing-lobes
|= =nako |= =nako
^- [let=(set lobe) old=(set lobe) hav=(map lobe blob)] ^- (set lobe)
=/ let-tako (~(got by gar.nako) let.nako) =| missing=(set lobe)
=| let-lobes=(set lobe)
=| old-lobes=(set lobe)
=| hav-blobs=(map lobe blob)
=/ yakis ~(tap in lar.nako) =/ yakis ~(tap in lar.nako)
|- ^- [(set lobe) (set lobe) (map lobe blob)] |- ^- (set lobe)
=* yaki-loop $ =* yaki-loop $
?~ yakis ?~ yakis
[let-lobes old-lobes hav-blobs] missing
=/ =norm (~(gut by tom.dom) r.i.yakis nor.dom) =/ =norm (~(gut by tom.dom) r.i.yakis nor.dom)
=/ lobes=(list [=path =lobe]) ~(tap by q.i.yakis) =/ lobes=(list [=path =lobe]) ~(tap by q.i.yakis)
|- ^- [(set lobe) (set lobe) (map lobe blob)] |- ^- (set lobe)
=* blob-loop $ =* lobe-loop $
?~ lobes ?~ lobes
yaki-loop(yakis t.yakis) yaki-loop(yakis t.yakis)
?: (~(has by lat.ran) lobe.i.lobes)
lobe-loop(lobes t.lobes)
?: =([~ %|] (~(fit ^de norm) path.i.lobes)) ?: =([~ %|] (~(fit ^de norm) path.i.lobes))
=. hav-blobs (~(put by hav-blobs) lobe.i.lobes %dead lobe.i.lobes ~) lobe-loop(lobes t.lobes)
blob-loop(lobes t.lobes) =. missing (~(put in missing) lobe.i.lobes)
=/ lob (~(get by lat.ran) lobe.i.lobes) lobe-loop(lobes t.lobes)
?~ lob
=. old-lobes (~(put in old-lobes) lobe.i.lobes)
blob-loop(lobes t.lobes)
=? let-lobes &(=(let-tako r.i.yakis) ?=(%dead -.u.lob))
(~(put in let-lobes) lobe.i.lobes)
blob-loop(lobes t.lobes)
:: ::
:: Receive backfill response :: Receive backfill response
:: ::
++ take-backfill ++ take-backfill
|= =blub |= =fell
^+ ..abet ^+ ..abet
?: lost ..abet ?: lost ..abet
=/ =blob (blub-to-blob blub) =/ peg=(unit page) (fell-to-page fell)
?: ?& ?=(%dead -.blob) =? lat.ran ?=(^ peg)
(lien let.need.sat |=(=lobe =(p.blob lobe))) (~(uni by (malt [(page-to-lobe u.peg) u.peg] ~)) lat.ran)
==
%- (slog leaf+"clay: got dead backfill but need answer for {<p.blob>}" ~)
..abet
=? need.sat ?=(%delta -.blob)
=/ one (~(get by lat.ran) q.blob)
=/ two (~(get by have.sat) q.blob)
?: ?& ?| ?=(~ one)
?=(%dead -.u.one)
==
?| ?=(~ two)
?=(%dead -.u.two)
==
==
[[q.blob let.need.sat] old.need.sat]
need.sat
:: We can't put a blob in lat.ran if its parent isn't already
:: there.
::
=. ..abet
?: &(?=(%delta -.blob) !(~(has by lat.ran) q.blob))
..abet(have.sat (uni-blobs have.sat (malt [p.blob `^blob`blob] ~)))
..abet(lat.ran (uni-blobs lat.ran (malt [p.blob blob] ~)))
work(busy.sat |) work(busy.sat |)
:: ::
:: Fetch next blob :: Fetch next lobe
:: ::
++ work ++ work
^+ ..abet ^+ ..abet
?: busy.sat ?: busy.sat
..abet ..abet
|- ^+ ..abet |- ^+ ..abet
?: =([~ ~] need.sat) ?~ need.sat
:: NB: if you change to release nakos as we get enough blobs :: NB: if you change to release nakos as we get enough lobes
:: for them instead of all at the end, you *must* store the :: for them instead of all at the end, you *must* store the
:: `lim` that should be applied after the nako is complete and :: `lim` that should be applied after the nako is complete and
:: not use the one in the rave, since that will apply to the :: not use the one in the rave, since that will apply to the
:: end of subscription. :: end of subscription.
:: ::
=. lat.ran (uni-blobs lat.ran have.sat)
|- ^+ ..abet |- ^+ ..abet
?: =(~ nako.sat) ?: =(~ nako.sat)
..abet ..abet
=^ next=(unit nako) nako.sat ~(get to nako.sat) =^ next=(unit nako) nako.sat ~(get to nako.sat)
?~ next ?~ next
..abet(done &) ..abet(done &)
=. ..abet (apply-foreign-update u.next) =. ..abet =>((apply-foreign-update u.next) ?>(?=(~ need.sat) .))
=. ..foreign-update =<(?>(?=(^ ref) .) wake) =. ..foreign-update =<(?>(?=(^ ref) .) wake)
$ $
?~ let.need.sat :: This is what removes an item from `need`. This happens every
:: This is what removes an item from `need`. This happens every :: time we take a backfill response, but it could happen more than
:: time we take a backfill response, but it could happen more than :: once if we somehow got this data in the meantime (maybe from
:: once if we somehow got this data in the meantime (maybe from :: another desk updating concurrently, or a previous update on this
:: another desk updating concurrently, or a previous update on this :: same desk).
:: same desk). ?: (~(has by lat.ran) i.need.sat)
:: $(need.sat t.need.sat)
?> ?=(^ old.need.sat) (fetch i.need.sat)
?: ?| (~(has by lat.ran) i.old.need.sat)
(~(has by have.sat) i.old.need.sat)
==
$(old.need.sat t.old.need.sat)
(fetch i.old.need.sat)
=/ one (~(get by lat.ran) i.let.need.sat)
=/ two (~(get by have.sat) i.let.need.sat)
?: ?| ?& ?=(^ one)
!?=(%dead -.u.one)
==
?& ?=(^ two)
!?=(%dead -.u.two)
==
==
$(let.need.sat t.let.need.sat)
(fetch i.let.need.sat)
:: ::
++ fetch ++ fetch
|= =lobe |= =lobe
@ -3251,9 +3120,9 @@
:: ::
:: When we get a %w foreign update, store this in our state. :: When we get a %w foreign update, store this in our state.
:: ::
:: We get the commits and blobs from the nako and add them to our :: We get the commits from the nako and add them to our object
:: object store, then we update the map of aeons to commits and the :: store, then we update the map of aeons to commits and the latest
:: latest aeon. :: aeon.
:: ::
++ apply-foreign-update ++ apply-foreign-update
|= =nako |= =nako
@ -3261,18 +3130,14 @@
:: hit: updated commit-hashes by @ud case :: hit: updated commit-hashes by @ud case
:: nut: new commit-hash/commit pairs :: nut: new commit-hash/commit pairs
:: hut: updated commits by hash :: hut: updated commits by hash
:: nat: new blob-hash/blob pairs
:: lat: updated blobs by hash
:: ::
=/ hit (~(uni by hit.dom) gar.nako) =/ hit (~(uni by hit.dom) gar.nako)
=/ nut (turn ~(tap in lar.nako) |=(=yaki [r.yaki yaki])) =/ nut (turn ~(tap in lar.nako) |=(=yaki [r.yaki yaki]))
=/ hut (~(uni by (malt nut)) hut.ran) =/ hut (~(uni by (malt nut)) hut.ran)
=/ nat (turn ~(tap in bar.nako) |=(=blub [p.blub (blub-to-blob blub)]))
=/ lat (uni-blobs lat.ran (malt nat))
:: traverse updated state and sanity check :: traverse updated state and sanity check
:: ::
=+ ~| :* %bad-foreign-update =+ ~| :* %bad-foreign-update
[gar=gar.nako let=let.nako nut=(turn nut head) nat=(turn nat head)] [gar=gar.nako let=let.nako nut=(turn nut head)]
[hitdom=hit.dom letdom=let.dom] [hitdom=hit.dom letdom=let.dom]
== ==
?: =(0 let.nako) ?: =(0 let.nako)
@ -3283,12 +3148,6 @@
~| [%missing-aeon aeon] (~(got by hit) aeon) ~| [%missing-aeon aeon] (~(got by hit) aeon)
=/ =yaki =/ =yaki
~| [%missing-tako tako] (~(got by hut) tako) ~| [%missing-tako tako] (~(got by hut) tako)
=+ %+ turn
~(tap by q.yaki)
|= [=path =lobe]
~| [%missing-blob path lobe]
?> (~(has by lat) lobe)
~
?: =(let.nako aeon) ?: =(let.nako aeon)
~ ~
$(aeon +(aeon)) $(aeon +(aeon))
@ -3299,7 +3158,6 @@
=: let.dom (max let.nako let.dom) =: let.dom (max let.nako let.dom)
hit.dom hit hit.dom hit
hut.ran hut hut.ran hut
lat.ran lat
:: Is this correct? Seeems like it should only go to `to` if :: Is this correct? Seeems like it should only go to `to` if
:: we've gotten all the way to the end. Leaving this :: we've gotten all the way to the end. Leaving this
:: behavior unchanged for now, but I believe it's wrong. :: behavior unchanged for now, but I believe it's wrong.
@ -3677,22 +3535,12 @@
:: ::
++ ze ++ ze
|% |%
:: These convert between aeon (version number), tako (commit hash), yaki :: These convert between aeon (version number), tako (commit hash),
:: (commit data structure), lobe (content hash), and blob (content). :: and yaki (commit data structure)
:: ::
++ aeon-to-tako ~(got by hit.dom) ++ aeon-to-tako ~(got by hit.dom)
++ aeon-to-yaki |=(=aeon (tako-to-yaki (aeon-to-tako aeon))) ++ aeon-to-yaki |=(=aeon (tako-to-yaki (aeon-to-tako aeon)))
++ lobe-to-blob ~(got by lat.ran)
++ tako-to-yaki ~(got by hut.ran) ++ tako-to-yaki ~(got by hut.ran)
++ lobe-to-mark
|= a=lobe
^- (unit mark)
=> (lobe-to-blob a)
?- -
%delta `p.r
%direct `p.q
%dead ~
==
:: ::
:: Gets a map of the data at the given path and all children of it. :: Gets a map of the data at the given path and all children of it.
:: ::
@ -3935,7 +3783,7 @@
?~ x ~ ?~ x ~
?~ u.x [~ ~] ?~ u.x [~ ~]
``[p.u.u.x !>(q.u.u.x)] ``[p.u.u.x !>(q.u.u.x)]
:: +read-s: produce yaki or blob for given tako or lobe :: +read-s: produce miscellaneous
:: ::
++ read-s ++ read-s
|= [yon=aeon pax=path] |= [yon=aeon pax=path]
@ -3951,10 +3799,10 @@
``yaki+[-:!>(*yaki) u.yak] ``yaki+[-:!>(*yaki) u.yak]
:: ::
%blob %blob
=/ bol=(unit blob) (~(get by lat.ran) (slav %uv i.t.pax)) =/ peg=(unit page) (~(get by lat.ran) (slav %uv i.t.pax))
?~ bol ?~ peg
~ ~
``blob+[-:!>(*blob) u.bol] ``blob+[-:!>(*page) u.peg]
:: ::
%hash %hash
=/ yak=(unit yaki) (~(get by hut.ran) (slav %uv i.t.pax)) =/ yak=(unit yaki) (~(get by hut.ran) (slav %uv i.t.pax))
@ -3966,12 +3814,12 @@
:: should save ford cache :: should save ford cache
:: ::
=/ =lobe (slav %uv i.t.pax) =/ =lobe (slav %uv i.t.pax)
=^ =page fod.dom =/ peg=(unit page) (~(get by lat.ran) lobe)
%- wrap:fusion ?~ peg
(lobe-to-page:(ford:fusion static-ford-args) lobe) ~
=^ =cage fod.dom =^ =cage fod.dom
%- wrap:fusion %- wrap:fusion
(page-to-cage:(ford:fusion static-ford-args) page) (page-to-cage:(ford:fusion static-ford-args) u.peg)
``cage+[-:!>(*^cage) cage] ``cage+[-:!>(*^cage) cage]
:: ::
%open %open
@ -4087,9 +3935,8 @@
:: ::
:: If it's in our ankh (current state cache), we can just produce :: If it's in our ankh (current state cache), we can just produce
:: the result. Otherwise, we've got to look up the node at the :: the result. Otherwise, we've got to look up the node at the
:: aeon to get the content hash, use that to find the blob, and use :: aeon to get the content hash, use that to find the page. We
:: the blob to get the data. We also special-case the hoon mark :: also special-case the hoon mark for bootstrapping purposes.
:: for bootstrapping purposes.
:: ::
++ read-x ++ read-x
|= [yon=aeon pax=path] |= [yon=aeon pax=path]
@ -4108,40 +3955,16 @@
=+ lob=(~(get by q.yak) pax) =+ lob=(~(get by q.yak) pax)
?~ lob ?~ lob
[[~ ~] fod.dom] [[~ ~] fod.dom]
=+ mar=(lobe-to-mark u.lob) =/ pig=(unit page) (~(get by lat.ran) u.lob)
:: if tombstoned, nothing to return :: if tombstoned, nothing to return
:: ::
?~ mar ?~ pig
[~ fod.dom] [~ fod.dom]
:: should convert any lobe to cage :: should convert any lobe to cage
:: ::
?: ?=(%hoon u.mar) ?: ?=(%hoon p.u.pig)
=/ txt [``[%hoon [%atom %t ~] ;;(@t q.u.pig)] fod.dom]
|- ^- (unit @t) =/ peg=(unit page) (~(get by lat.ran) u.lob)
=+ bol=(lobe-to-blob u.lob)
?- -.bol
%dead ~
%direct `;;(@t q.q.bol)
%delta
=+ txt=$(u.lob q.bol)
?~ txt
~
?> ?=(%txt-diff p.r.bol)
=+ dif=;;((urge cord) q.r.bol)
=, format
=+ pac=(of-wain (lurk:differ (to-wain (cat 3 u.txt '\0a')) dif))
:- ~
?~ pac
''
(end [3 (dec (met 3 pac))] pac)
==
:_ fod.dom
?~ txt
~
[~ ~ u.mar [%atom %t ~] u.txt]
=^ peg=(unit page) fod.dom
%- wrap:fusion
(lobe-to-unit-page:(ford:fusion static-ford-args) u.lob)
?~ peg ?~ peg
[~ fod.dom] [~ fod.dom]
=^ =cage fod.dom =^ =cage fod.dom
@ -4491,7 +4314,7 @@
=+ ;;(=fill res) =+ ;;(=fill res)
=^ mos ruf =^ mos ruf
=/ den ((de now rof hen ruf) our desk.fill) =/ den ((de now rof hen ruf) our desk.fill)
abet:(give-backfill:den lobe.fill) abet:(give-backfill:den -.fill lobe.fill)
[[[hen %give %done ~] mos] ..^$] [[[hen %give %done ~] mos] ..^$]
?> ?=([%question *] pax) ?> ?=([%question *] pax)
=+ ryf=;;(riff-any res) =+ ryf=;;(riff-any res)
@ -4526,7 +4349,12 @@
== ==
+$ rang-11 +$ rang-11
$: hut=(map tako yaki) $: hut=(map tako yaki)
lat=(map lobe blub) lat=(map lobe blob-11)
==
+$ blob-11
$% [%delta p=lobe q=[p=mark q=lobe] r=page]
[%direct p=lobe q=page]
[%dead p=lobe ~]
== ==
+$ room-11 +$ room-11
$: hun=duct $: hun=duct
@ -4568,11 +4396,17 @@
+$ update-state-11 +$ update-state-11
$: =duct $: =duct
=rave =rave
have=(map lobe blub) have=(map lobe blob-11)
need=[let=(list lobe) old=(list lobe)] need=(list lobe)
nako=(qeu (unit nako)) nako=(qeu (unit nako-11))
busy=_| busy=_|
== ==
+$ nako-11
$: gar=(map aeon tako)
let=aeon
lar=(set yaki)
bar=(set blob-11)
==
+$ melt-11 +$ melt-11
[bas=beak con=(list [beak germ]) sto=(map beak (unit dome-clay-11))] [bas=beak con=(list [beak germ]) sto=(map beak (unit dome-clay-11))]
+$ dome-clay-11 +$ dome-clay-11
@ -4630,18 +4464,10 @@
== ==
+$ rind-10 +$ rind-10
$: nix=@ud $: nix=@ud
bom=(map @ud update-state-10) bom=(map @ud update-state-11)
fod=(map duct @ud) fod=(map duct @ud)
haw=(map mood (unit cage)) haw=(map mood (unit cage))
== ==
+$ update-state-10
$: =duct
=rave
have=(map lobe blub)
need=(list lobe)
nako=(qeu (unit nako))
busy=_|
==
+$ raft-9 +$ raft-9
$: rom=room-10 $: rom=room-10
hoy=(map ship rung-10) hoy=(map ship rung-10)
@ -4866,16 +4692,6 @@
^- rede-11 ^- rede-11
%= rede-10 %= rede-10
qyx (cult-10-to-cult qyx.rede-10) qyx (cult-10-to-cult qyx.rede-10)
ref
?~ ref.rede-10
~
%= ref.rede-10
bom.u
%- ~(run by bom.u.ref.rede-10)
|= =update-state-10
^- update-state-11
update-state-10(need [need.update-state-10 ~])
==
== ==
== ==
++ cult-10-to-cult ++ cult-10-to-cult
@ -4927,12 +4743,25 @@
:: ::
:: add tom and nor to dome :: add tom and nor to dome
:: remove parent-mark from delta blobs :: remove parent-mark from delta blobs
:: change blobs to pages
:: remove have from update-state
:: remove bar from nako
:: ::
++ raft-11-to-12 ++ raft-11-to-12
|= raf=raft-11 |= raf=raft-11
^- raft-12 ^- raft-12
%= raf %= raf
lat.ran (~(run by lat.ran.raf) blub-to-blob) lat.ran
%- ~(gas by *(map lobe page))
%+ murn ~(tap by lat.ran.raf)
|= [=lobe =blob-11]
^- (unit [^lobe page])
?- -.blob-11
%delta ((slog 'clay: tombstoning delta!' ~) ~)
%dead ~
%direct `[lobe q.blob-11]
==
::
dos.rom dos.rom
%- ~(run by dos.rom.raf) %- ~(run by dos.rom.raf)
|= =dojo-11 |= =dojo-11
@ -4955,7 +4784,17 @@
bom.u bom.u
%- ~(run by bom.u.ref.rede-11) %- ~(run by bom.u.ref.rede-11)
|= =update-state-11 |= =update-state-11
update-state-11(have (~(run by have.update-state-11) blub-to-blob)) %= update-state-11
|2
%= |3.update-state-11
nako
%- ~(gas to *(qeu (unit nako)))
%+ turn ~(tap to nako.update-state-11)
|= nak=(unit nako-11)
?~ nak ~
`u.nak(bar ~)
==
==
== ==
== ==
== ==
@ -4988,7 +4827,7 @@
::TODO if it ever gets filled properly, pass in the full fur. ::TODO if it ever gets filled properly, pass in the full fur.
:: ::
=/ for=(unit ship) ?~(lyc ~ ?~(u.lyc ~ `n.u.lyc)) =/ for=(unit ship) ?~(lyc ~ ?~(u.lyc ~ `n.u.lyc))
?: &(=(our his) =(%$ syd) =([%da now] u.luk)) ?: &(=(our his) =(%x ren) =(%$ syd) =([%da now] u.luk))
(read-buc u.run tyl) (read-buc u.run tyl)
=/ den ((de now rof [/scryduct ~] ruf) his syd) =/ den ((de now rof [/scryduct ~] ruf) his syd)
=/ result (mule |.(-:(aver:den for u.run u.luk tyl))) =/ result (mule |.(-:(aver:den for u.run u.luk tyl)))
@ -5142,7 +4981,7 @@
!! !!
:: ::
%boon %boon
=+ ;; =blub payload.hin =+ ;; =fell payload.hin
:: ::
=/ her=ship (slav %p i.t.tea) =/ her=ship (slav %p i.t.tea)
=/ =desk (slav %tas i.t.t.tea) =/ =desk (slav %tas i.t.t.tea)
@ -5150,7 +4989,7 @@
:: ::
=^ mos ruf =^ mos ruf
=/ den ((de now rof hen ruf) her desk) =/ den ((de now rof hen ruf) her desk)
abet:abet:(take-backfill:(foreign-update:den index) blub) abet:abet:(take-backfill:(foreign-update:den index) fell)
[mos ..^$] [mos ..^$]
== ==
:: ::
@ -5293,7 +5132,7 @@
foreign+&+hoy.ruf foreign+&+hoy.ruf
:+ %object-store %| :+ %object-store %|
:~ commits+&+hut.ran.ruf :~ commits+&+hut.ran.ruf
blobs+&+lat.ran.ruf pages+&+lat.ran.ruf
== ==
== ==
:: ::
@ -5307,7 +5146,7 @@
?- -.clue ?- -.clue
%lobe `(tomb-lobe lobe.clue &) %lobe `(tomb-lobe lobe.clue &)
%all %all
=/ lobes=(list [=lobe =blob]) ~(tap by lat.ran.ruf) =/ lobes=(list [=lobe =page]) ~(tap by lat.ran.ruf)
|- |-
?~ lobes ?~ lobes
`..^^$ `..^^$
@ -5332,10 +5171,8 @@
++ tomb-lobe ++ tomb-lobe
|= [lob=lobe veb=?] |= [lob=lobe veb=?]
^+ ..^$ ^+ ..^$
=/ bol (~(get by lat.ran.ruf) lob) =/ peg=(unit page) (~(get by lat.ran.ruf) lob)
?~ bol ?~ peg
(noop veb leaf+"clay: lobe doesn't exist" ~)
?: ?=(%dead -.u.bol)
(noop veb leaf+"clay: file already tombstoned" ~) (noop veb leaf+"clay: file already tombstoned" ~)
:: ::
=/ used=(unit beam) =/ used=(unit beam)
@ -5362,21 +5199,7 @@
?^ used ?^ used
(noop veb leaf+"clay: file used in {<(en-beam u.used)>}" ~) (noop veb leaf+"clay: file used in {<(en-beam u.used)>}" ~)
:: ::
=/ based=(unit lobe) =. lat.ran.ruf (~(del by lat.ran.ruf) lob)
=/ lobes=(list [=lobe =blob]) ~(tap by lat.ran.ruf)
|-
?~ lobes
~
?: ?& ?=(%delta -.blob.i.lobes)
=(lob q.blob.i.lobes)
==
`p.blob.i.lobes
$(lobes t.lobes)
::
?^ based
(noop veb leaf+"clay: file is base of delta for {<u.based>}" ~)
::
=. lat.ran.ruf (~(put by lat.ran.ruf) lob %dead lob ~)
(noop veb leaf+"clay: file successfully tombstoned" ~) (noop veb leaf+"clay: file successfully tombstoned" ~)
:: ::
++ noop ++ noop
@ -5439,7 +5262,7 @@
:: +pick: copying gc based on norms :: +pick: copying gc based on norms
:: ::
++ pick ++ pick
=| lat=(map lobe blob) =| lat=(map lobe page)
=| sen=(set [norm (map path lobe)]) =| sen=(set [norm (map path lobe)])
|^ |^
=. ..pick-raft pick-raft =. ..pick-raft pick-raft
@ -5454,7 +5277,7 @@
..pick-raft ..pick-raft
$(yakis t.yakis, ..pick-raft (pick-yaki i.yakis)) $(yakis t.yakis, ..pick-raft (pick-yaki i.yakis))
:: ::
:: NB: recurring in this way with the `sen` cache provides :: NB: recurring tree-wise with the `sen` cache provides
:: approximately a 100x speedup on a mainnet moon in 4/2022 :: approximately a 100x speedup on a mainnet moon in 4/2022
:: ::
++ pick-yaki ++ pick-yaki
@ -5466,26 +5289,11 @@
?: (~(has in sen) norm q.yaki) ?: (~(has in sen) norm q.yaki)
..pick-raft ..pick-raft
=. sen (~(put in sen) norm q.yaki) =. sen (~(put in sen) norm q.yaki)
=/ bob (~(get by lat) q.n.q.yaki) =/ peg=(unit page) (~(get by lat) q.n.q.yaki)
=? lat ?|(?=(~ bob) ?=([~ %dead *] bob)) =? lat &(?=(^ peg) !=([~ %|] +:(~(fit ^de norm) p.n.q.yaki)))
?: =([~ %|] +:(~(fit ^de norm) p.n.q.yaki)) (~(uni by `(map lobe page)`[[q.n.q.yaki u.peg] ~ ~]) lat)
(~(put by lat) q.n.q.yaki %dead q.n.q.yaki ~)
(uni-blobs lat (pick-lobe q.n.q.yaki))
=. ..pick-raft $(q.yaki l.q.yaki) =. ..pick-raft $(q.yaki l.q.yaki)
$(q.yaki r.q.yaki) $(q.yaki r.q.yaki)
::
++ pick-lobe
|= =lobe
^- (map ^lobe blob)
=/ bob=(unit blob) (~(get by lat.ran.ruf) lobe)
?~ bob
%- (slog leaf+"clay: picking strange lobe {<lobe>}")
~
?- -.u.bob
%dead [[lobe u.bob] ~ ~]
%direct [[lobe u.bob] ~ ~]
%delta (~(put by $(lobe q.u.bob)) lobe u.bob)
==
-- --
-- --
-- --

20
pkg/base-dev/sur/hood.hoon
View File

@ -130,14 +130,8 @@
=/ tak (scot %uv (~(got by hit.dom) let.dom)) =/ tak (scot %uv (~(got by hit.dom) let.dom))
=/ yak .^(yaki cs/~[her syd yon %yaki tak]) =/ yak .^(yaki cs/~[her syd yon %yaki tak])
=/ lob (scot %uv (~(got by q.yak) /sys/kelvin)) =/ lob (scot %uv (~(got by q.yak) /sys/kelvin))
=/ bob .^(blob cs/~[her syd yon %blob lob]) =/ peg .^(page cs/~[her syd yon %blob lob])
:: ;;(weft q.peg)
;; weft
?- -.bob
%direct q.q.bob
%delta q.r.bob
%dead ~|(%tombstoned-kelvin !!)
==
:: +read-kelvin-local: read /sys/kelvin from a local desk :: +read-kelvin-local: read /sys/kelvin from a local desk
:: ::
++ read-kelvin-local ++ read-kelvin-local
@ -165,14 +159,8 @@
=/ fil (~(get by q.yak) /desk/bill) =/ fil (~(get by q.yak) /desk/bill)
?~ fil ~ ?~ fil ~
=/ lob (scot %uv u.fil) =/ lob (scot %uv u.fil)
=/ bob .^(blob cs/~[her syd yon %blob lob]) =/ peg .^(page cs/~[her syd yon %blob lob])
:: ;;((list dude) q.peg)
;; (list dude)
?- -.bob
%direct q.q.bob
%delta q.r.bob
%dead ~|(%tombstoned-bill !!)
==
:: +read-bill: read contents of /desk/bill manifest :: +read-bill: read contents of /desk/bill manifest
:: ::
++ read-bill ++ read-bill