From 1b1b61372994c86ab7d04246444d2257e8a4f856 Mon Sep 17 00:00:00 2001 From: fang Date: Thu, 24 Dec 2020 13:03:56 +0100 Subject: [PATCH 001/103] clay: condense +pile-rule Rewrites the `+pile-rule` parsing rule for compactness and legibility. This is a purely stylistic change. --- pkg/arvo/sys/vane/clay.hoon | 102 ++++++++++++++---------------------- 1 file changed, 38 insertions(+), 64 deletions(-) diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index d665c5fb91..638827a180 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -807,75 +807,49 @@ ++ pile-rule |= pax=path %- full - %+ ifix [gay gay] - %+ cook |=(pile +<) - ;~ pfix + %+ ifix + :_ gay :: parse optional /? and ignore :: - ;~ pose - (cold ~ ;~(plug fas wut gap dem gap)) - (easy ~) + ;~(plug gay (punt ;~(plug fas wut gap dem gap))) + |^ + ;~ plug + %+ cook (bake zing (list (list taut))) + %+ rune hep + (most ;~(plug com gaw) taut-rule) + :: + %+ cook (bake zing (list (list taut))) + %+ rune lus + (most ;~(plug com gaw) taut-rule) + :: + %+ rune tis + ;~(plug sym ;~(pfix gap fas (more fas urs:ab))) + :: + %+ rune tar + ;~ (glue gap) + sym + ;~(pfix cen sym) + ;~(pfix fas (more fas urs:ab)) == :: - ;~ plug - ;~ pose - ;~ sfix - %+ cook |=((list (list taut)) (zing +<)) - %+ more gap - ;~ pfix ;~(plug fas hep gap) - (most ;~(plug com gaw) taut-rule) - == - gap - == - (easy ~) - == - :: - ;~ pose - ;~ sfix - %+ cook |=((list (list taut)) (zing +<)) - %+ more gap - ;~ pfix ;~(plug fas lus gap) - (most ;~(plug com gaw) taut-rule) - == - gap - == - (easy ~) - == - :: - ;~ pose - ;~ sfix - %+ cook |=((list [face=term =path]) +<) - %+ more gap - ;~ pfix ;~(plug fas tis gap) - %+ cook |=([term path] +<) - ;~(plug sym ;~(pfix ;~(plug gap fas) (more fas urs:ab))) - == - gap - == - (easy ~) - == - :: - ;~ pose - ;~ sfix - %+ cook |=((list [face=term =mark =path]) +<) - %+ more gap - ;~ pfix ;~(plug fas tar gap) - %+ cook |=([term mark path] +<) - ;~ plug - sym - ;~(pfix ;~(plug gap cen) sym) - ;~(pfix ;~(plug gap fas) (more fas urs:ab)) - == - == - gap - == - (easy ~) - == - :: - %+ cook |=(huz=(list hoon) `hoon`tssg+huz) - (most gap tall:(vang & pax)) - == + %+ stag %tssg + (most gap tall:(vang & pax)) == + :: + ++ pant + |* fel=^rule + ;~(pose fel (easy ~)) + :: + ++ mast + |* [bus=^rule fel=^rule] + ;~(sfix (more bus fel) bus) + :: + ++ rune + |* [bus=^rule fel=^rule] + %- pant + %+ mast gap + ;~(pfix fas bus gap fel) + -- :: ++ taut-rule %+ cook |=(taut +<) From ece8bbf0a281b27ab5900ebfbe3d29e4219ebae8 Mon Sep 17 00:00:00 2001 From: Ted Blackman Date: Tue, 12 Jan 2021 19:10:11 -0500 Subject: [PATCH 002/103] clay: WIP static mark and cast builds --- pkg/arvo/sys/lull.hoon | 15 ++- pkg/arvo/sys/vane/clay.hoon | 246 +++++++++++++++++++++--------------- 2 files changed, 157 insertions(+), 104 deletions(-) diff --git a/pkg/arvo/sys/lull.hoon b/pkg/arvo/sys/lull.hoon index 92f45dfa2a..236830c5eb 100644 --- a/pkg/arvo/sys/lull.hoon +++ b/pkg/arvo/sys/lull.hoon @@ -945,6 +945,20 @@ :: +$ mars [a=mark b=mark] +$ tube $-(vase vase) + +$ nave + |$ [typ dif] + $_ + |% + ++ bunt *typ + ++ diff |~([old=typ new=typ] *dif) + ++ form *mark + ++ join |~([a=dif b=dif] *(unit (unit dif))) + ++ mash + |~ [a=[ship desk dif] b=[ship desk dif]] + *(unit dif) + ++ pact |~([typ dif] *typ) + ++ vale typ + -- :: $dais: processed mark core :: +$ dais @@ -959,7 +973,6 @@ *(unit vase) ++ pact |~(diff=vase sam) ++ vale |~(noun sam) - ++ volt |~(noun sam) -- :: ++ get-fit diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index d665c5fb91..c45f0ed5a6 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -119,8 +119,10 @@ :: +$ ford-cache $: vases=(map path [res=vase dez=(set path)]) + naves=(map mark [res=vase dez=(set path)]) marks=(map mark [res=dais dez=(set path)]) - casts=(map mars [res=tube dez=(set path)]) + casts=(map mars [res=vase dez=(set path)]) + tubes=(map mars [res=tube dez=(set path)]) == :: $reef-cache: built system files :: @@ -462,7 +464,9 @@ +$ build $% [%file =path] [%mark =mark] + [%dais =mark] [%cast =mars] + [%tube =mars] [%vale =path] == +$ state @@ -519,13 +523,13 @@ ?< (~(has in deletes) path) ~| %file-not-found^path :_(nub (need (~(get an ankh) path))) - :: +get-mark: build a mark definition + :: +get-nave: build a statically typed mark core :: - ++ get-mark + ++ get-nave |= mak=mark - ^- [dais state] + ^- [nave state] ~| %error-building-mark^mak - ?^ got=(~(get by marks.cache.nub) mak) + ?^ got=(~(get by naves.cache.nub) mak) =? stack.nub ?=(^ stack.nub) stack.nub(i (~(uni in i.stack.nub) dez.u.got)) [res.u.got nub] @@ -533,99 +537,125 @@ ~|(cycle+mark+mak^stack.nub !!) =. cycle.nub (~(put in cycle.nub) mark+mak) =. stack.nub [~ stack.nub] + =; res=[=vase nub=state] + =. nub nub.res + =^ top stack.nub pop-stack + =. naves.cache.nub (~(put by naves.cache.nub) mak [vase.res top]) + [vase.res nub] + =^ cor=vase nub (build-fit %mar mak) + =/ gad=vase (slap cor %limb %grad) + ?@ q.gad + =+ !<(mok=mark gad) + =^ deg=vase nub $(mak mok) + =^ tub=vase nub (get-cast mak mok) + =^ but=vase nub (get-cast mok mak) + :_ nub + ^- vase :: vase of nave + %+ slap + (with-faces deg+deg tub+tub but+but cor+cor nave+!>(nave) ~) + !, *hoon + =/ typ=mold _+<.cor + =/ dif=mold vale:deg + ^- (nave typ dif) + |% + ++ bunt +<.cor + ++ diff + |= [old=typ new=typ] + ^- dif + (diff:deg (tub old) (tub new)) + ++ form form:deg + ++ join join:deg + ++ mash mash:deg + ++ pact + |= [v=typ d=dif] + ^- typ + (but (pact:deg (tub v) d)) + ++ vale noun:grab:cor + -- + :_ nub + =+ !<(fom=mark (slap gad %limb %form)) + ^- vase :: vase of nave + %+ slap + (with-faces cor+cor nave+!>(nave) ~) + !, *hoon + =/ typ=mold _+<.cor + =/ dif=mold diff:grad:cor + ^- (nave typ dif) + |% + ++ bunt +<.cor + ++ diff |=([old=typ new=typ] (diff:~(grad cor old) new)) + ++ form form:grad:cor + ++ join join:grad:cor + ++ mash + |= [a=[=ship =desk =dif] b=[=ship =desk =dif]] + ^- (unit dif) + ?: =(dif.a dif.b) + ~ + `(mash:grad:cor a b) + ++ pact |=([v=typ d=dif] (pact:~(grad cor v) d)) + ++ vale noun:grab:cor + -- + :: +get-mark: build a dynamically typed mark definition + :: + ++ get-mark + |= mak=mark + ^- [dais state] + ~| %error-building-dais^mak + ?^ got=(~(get by marks.cache.nub) mak) + =? stack.nub ?=(^ stack.nub) + stack.nub(i (~(uni in i.stack.nub) dez.u.got)) + [res.u.got nub] + ?: (~(has in cycle.nub) dais+mak) + ~|(cycle+dais+mak^stack.nub !!) + =. cycle.nub (~(put in cycle.nub) dais+mak) + =. stack.nub [~ stack.nub] =; res=[=dais nub=state] =. nub nub.res =^ top stack.nub pop-stack =. marks.cache.nub (~(put by marks.cache.nub) mak [dais.res top]) [dais.res nub] - =^ cor=vase nub (build-fit %mar mak) - =/ gad=vase (slap cor %limb %grad) - ?@ q.gad - =+ !<(mok=mark gad) - =^ deg=dais nub $(mak mok) - =^ tub=tube nub (get-cast mak mok) - =^ but=tube nub (get-cast mok mak) - :_ nub - ^- dais - |_ sam=vase - ++ bunt (slap cor $+6) - ++ diff - |= new=vase - ^- vase - (~(diff deg (tub sam)) (tub new)) - ++ form form:deg - ++ join join:deg - ++ mash mash:deg - ++ pact - |= diff=vase - ^+ sam - (but (~(pact deg (tub sam)) diff)) - ++ vale - |= =noun - ^+ sam - (slam (slap cor !,(*hoon noun:grab)) !>(noun)) - ++ volt - |= =noun - ^+ sam - [p:bunt noun] - -- + =^ nav=vase nub (get-nave mak) :_ nub - =+ !<(fom=mark (slap gad %limb %form)) ^- dais |_ sam=vase - ++ bunt (slap cor $+6) + ++ bunt (slap nav limb/%bunt) ++ diff |= new=vase - ^- vase - %+ slap - (with-faces cor+cor sam+sam new+new ~) - !, *hoon - (diff:~(grad cor sam) new) - ++ form fom + (slam (slap nav limb/%diff) (slop sam new)) + ++ form !<(mark (slap nav limb/%form)) ++ join |= [a=vase b=vase] ^- (unit (unit vase)) - ?: =(q.a q.b) - ~ - =; res `?~(q.res ~ `(slap res !,(*hoon ?~(. !! u)))) - (slam (slap cor !,(*hoon join:grad)) (slop a b)) + =/ res=vase (slam (slap nav limb/%join) (slop a b)) + ?~ q.res ~ + ?~ +.q.res [~ ~] + ``(slap res wing/u/u/~) ++ mash |= [a=[=ship =desk diff=vase] b=[=ship =desk diff=vase]] ^- (unit vase) - ?: =(q.diff.a q.diff.b) + =/ res=vase + %+ slam (slap nav limb/%mash) + %+ slop + :(slop !>(ship.a) !>(desk.a) diff.a) + :(slop !>(ship.b) !>(desk.b) diff.b) + ?~ q.res ~ - :- ~ - %+ slam (slap cor !,(*hoon mash:grad)) - %+ slop - :(slop !>(ship.a) !>(desk.a) diff.a) - :(slop !>(ship.b) !>(desk.b) diff.b) + `(slap res limb/%u) ++ pact |= diff=vase - ^+ sam - %+ slap - (with-faces cor+cor sam+sam diff+diff ~) - !, *hoon - (pact:~(grad cor sam) diff) + (slam (slap nav limb/%pact) (slop sam diff)) ++ vale |= =noun - ^+ sam - (slam (slap cor !,(*hoon noun:grab)) !>(noun)) - ++ volt - |= =noun - ^+ sam - [p:bunt noun] + (slam (slap nav limb/%vale) noun/noun) -- - :: +get-cast: produce a $tube mark conversion gate from .a to .b + :: +get-cast: produce gate to convert mark .a to, statically typed :: ++ get-cast |= [a=mark b=mark] - ^- [tube state] + ^- [vase state] ~| error-building-cast+[a b] ?: =([%mime %hoon] [a b]) - :_ nub - |= sam=vase - =+ !<(=mime sam) - !>(q.q.mime) + :_(nub !>(|=(m=mime q.q.m))) ?^ got=(~(get by casts.cache.nub) [a b]) =? stack.nub ?=(^ stack.nub) stack.nub(i (~(uni in i.stack.nub) dez.u.got)) @@ -633,11 +663,11 @@ ?: (~(has in cycle.nub) cast+[a b]) ~|(cycle+cast+[a b]^stack.nub !!) =. stack.nub [~ stack.nub] - =; res=[=tube nub=state] + =; res=[=vase nub=state] =. nub nub.res =^ top stack.nub pop-stack - =. casts.cache.nub (~(put by casts.cache.nub) [a b] [tube.res top]) - [tube.res nub] + =. casts.cache.nub (~(put by casts.cache.nub) [a b] [vase.res top]) + [vase.res nub] :: try +grow; is there a +grow core with a .b arm? :: =^ old=vase nub (build-fit %mar a) @@ -649,47 +679,55 @@ :: +grow core has .b arm; use that :: :_ nub - ^- tube - |= sam=vase - ^- vase - %+ slap - (with-faces old+old sam+sam ~) - :+ %sgzp !,(*hoon old=old) - :+ %sgzp !,(*hoon sam=sam) - :+ %tsgl [%limb b] - !, *hoon - ~(grow old sam) + %+ slap (with-faces cor+old ~) + ^- hoon + :+ %brts !,(*hoon $=(v bunt:cor)) + :+ %tsgl limb/b + !,(*hoon ~(grow cor v)) :: try direct +grab :: =^ new=vase nub (build-fit %mar b) - =/ rab - %- mule |. - %+ slap new - :+ %tsgl [%limb a] - [%limb %grab] + =/ rab (mule |.((slap new tsgl/[limb/a limb/%grab]))) ?: &(?=(%& -.rab) ?=(^ q.p.rab)) - :_(nub |=(sam=vase ~|([%grab a b] (slam p.rab sam)))) + :_(nub p.rab) :: try +jump :: - =/ jum - %- mule |. - %+ slap old - :+ %tsgl [%limb b] - [%limb %jump] + =/ jum (mule |.((slap old tsgl/[limb/b limb/%jump]))) ?: ?=(%& -.jum) (compose-casts a !<(mark p.jum) b) - :: try indirect +grab - :: ?: ?=(%& -.rab) (compose-casts a !<(mark p.rab) b) ~|(no-cast-from+[a b] !!) :: ++ compose-casts |= [x=mark y=mark z=mark] + ^- [vase state] + =^ uno=vase nub (get-cast x y) + =^ dos=vase nub (get-cast y z) + :_ nub + %+ slam + (with-faces uno+uno dos+dos cork+cork ~) + !,(*hoon (cork uno dos)) + :: +get-tube: produce a $tube mark conversion gate from .a to .b + :: + ++ get-tube + |= [a=mark b=mark] ^- [tube state] - =^ uno=tube nub (get-cast x y) - =^ dos=tube nub (get-cast y z) - :_(nub |=(sam=vase (dos (uno sam)))) + ~| error-building-tube+[a b] + ?^ got=(~(get by tubes.cache.nub) [a b]) + =? stack.nub ?=(^ stack.nub) + stack.nub(i (~(uni in i.stack.nub) dez.u.got)) + [res.u.got nub] + ?: (~(has in cycle.nub) tube+[a b]) + ~|(cycle+tube+[a b]^stack.nub !!) + =. stack.nub [~ stack.nub] + =; res=[=tube nub=state] + =. nub nub.res + =^ top stack.nub pop-stack + =. tubes.cache.nub (~(put by tubes.cache.nub) [a b] [tube.res top]) + [tube.res nub] + =^ gat=vase nub (get-cast a b) + :_(nub |=(v=vase (slap gat v))) :: ++ lobe-to-page |= =lobe @@ -1528,8 +1566,10 @@ |=(c=@tD `@tD`?:(=('/' c) '-' c)) :: convert '/' to '-' :: :* ((invalidate path vase) vases.ford-cache invalid) + ((invalidate mark vase) naves.ford-cache invalid) ((invalidate mark dais) marks.ford-cache invalid) - ((invalidate mars tube) casts.ford-cache invalid) + ((invalidate mars vase) casts.ford-cache invalid) + ((invalidate mars tube) tubes.ford-cache invalid) == :: ++ invalidate @@ -4164,7 +4204,7 @@ dos.rom %- ~(run by dos.rom.ruf) |= =dojo - dojo(fod.dom [~ ~ ~]) + dojo(fod.dom [~ ~ ~ ~ ~]) :: hoy %- ~(run by hoy.ruf) @@ -4173,7 +4213,7 @@ rus %- ~(run by rus.rung) |= =rede - rede(fod.dom [~ ~ ~]) + rede(fod.dom [~ ~ ~ ~ ~]) == == :: From f469dbf8bc83bb0e12b4ae7b242aa25b24290388 Mon Sep 17 00:00:00 2001 From: Ted Blackman Date: Wed, 13 Jan 2021 15:20:58 -0500 Subject: [PATCH 003/103] clay,lull: compiles with static marks --- bin/solid.pill | 4 ++-- pkg/arvo/sys/lull.hoon | 3 ++- pkg/arvo/sys/vane/clay.hoon | 20 ++++++++++---------- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/bin/solid.pill b/bin/solid.pill index 0d3ceab763..af2fc68fa4 100644 --- a/bin/solid.pill +++ b/bin/solid.pill @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c512d0c3da3ce7f0ac25babcbbceba04a9e62975e9d92aec56ab00e1c2fc6224 -size 8618963 +oid sha256:b38b016c4bcb23818328073e2b81fd09522699c9800e0744adbdbc15bb092795 +size 8749007 diff --git a/pkg/arvo/sys/lull.hoon b/pkg/arvo/sys/lull.hoon index 236830c5eb..98bebfa049 100644 --- a/pkg/arvo/sys/lull.hoon +++ b/pkg/arvo/sys/lull.hoon @@ -942,10 +942,11 @@ +$ taut [face=(unit term) pax=term] :: $mars: mark conversion request :: $tube: mark conversion gate + :: $nave: typed mark core :: +$ mars [a=mark b=mark] +$ tube $-(vase vase) - +$ nave + ++ nave |$ [typ dif] $_ |% diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index c45f0ed5a6..7cf3c4c62b 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -459,7 +459,7 @@ [+<- +<+>-] :: cache.state :: ++ ford - !. + ::!. :: TODO: reinstate => |% +$ build $% [%file =path] @@ -527,7 +527,7 @@ :: ++ get-nave |= mak=mark - ^- [nave state] + ^- [vase state] ~| %error-building-mark^mak ?^ got=(~(get by naves.cache.nub) mak) =? stack.nub ?=(^ stack.nub) @@ -681,7 +681,7 @@ :_ nub %+ slap (with-faces cor+old ~) ^- hoon - :+ %brts !,(*hoon $=(v bunt:cor)) + :+ %brts bcts/[%v like/[bunt/~ ~[cor/~]]] :+ %tsgl limb/b !,(*hoon ~(grow cor v)) :: try direct +grab @@ -705,8 +705,8 @@ =^ uno=vase nub (get-cast x y) =^ dos=vase nub (get-cast y z) :_ nub - %+ slam - (with-faces uno+uno dos+dos cork+cork ~) + %+ slap + (with-faces uno+uno dos+dos cork+!>(cork) ~) !,(*hoon (cork uno dos)) :: +get-tube: produce a $tube mark conversion gate from .a to .b :: @@ -727,7 +727,7 @@ =. tubes.cache.nub (~(put by tubes.cache.nub) [a b] [tube.res top]) [tube.res nub] =^ gat=vase nub (get-cast a b) - :_(nub |=(v=vase (slap gat v))) + :_(nub |=(v=vase (slam gat v))) :: ++ lobe-to-page |= =lobe @@ -751,7 +751,7 @@ ?: =(mak p.page) (page-to-cage page) =^ [mark vax=vase] nub (page-to-cage page) - =^ =tube nub (get-cast p.page mak) + =^ =tube nub (get-tube p.page mak) :_(nub [mak (tube vax)]) :: ++ page-to-cage @@ -772,7 +772,7 @@ =^ cag=cage nub (get-value path) ?: =(mok mak) [cag nub] - =^ =tube nub (get-cast mok mak) + =^ =tube nub (get-tube mok mak) ~| error-running-cast+[path mok mak] :_(nub [mak (tube q.cag)]) :: @@ -3498,12 +3498,12 @@ [~ fod.dom] ?. ?=([@ @ ~] path) [[~ ~] fod.dom] - =/ cached=(unit [=tube *]) (~(get by casts.fod.dom) [i i.t]:path) + =/ cached=(unit [=tube *]) (~(get by tubes.fod.dom) [i i.t]:path) ?^ cached :_(fod.dom [~ ~ %& %tube !>(tube.u.cached)]) =^ =tube fod.dom %- wrap:fusion - (get-cast:(ford:fusion static-ford-args) [i i.t]:path) + (get-tube:(ford:fusion static-ford-args) [i i.t]:path) :_(fod.dom [~ ~ %& %tube !>(tube)]) :: :: Gets the permissions that apply to a particular node. From 6be1ebc41ac70ee7a604a30e1591785bfe288510 Mon Sep 17 00:00:00 2001 From: Ted Blackman Date: Tue, 26 Jan 2021 15:59:36 -0500 Subject: [PATCH 004/103] clay,lull: +test-mar-mime passes --- pkg/arvo/mar/mime.hoon | 4 +-- pkg/arvo/sys/lull.hoon | 3 +- pkg/arvo/sys/vane/clay.hoon | 34 ++++++++++++--------- pkg/arvo/tests/sys/vane/clay.hoon | 51 ++++++++++++++++++++++++------- 4 files changed, 63 insertions(+), 29 deletions(-) diff --git a/pkg/arvo/mar/mime.hoon b/pkg/arvo/mar/mime.hoon index 87e920250b..72737153e5 100644 --- a/pkg/arvo/mar/mime.hoon +++ b/pkg/arvo/mar/mime.hoon @@ -3,7 +3,6 @@ :: /? 310 :: -^| |_ own=mime ++ grow ^? @@ -14,7 +13,7 @@ ++ grab :: convert from ^? |% - +$ noun mime :: clam from %noun + ++ noun mime :: clam from %noun ++ tape |=(a=_"" [/application/x-urb-unknown (as-octt:mimes:html a)]) -- @@ -25,5 +24,6 @@ ++ diff |=(mime +<) ++ pact |=(mime +<) ++ join |=([mime mime] `(unit mime)`~) + ++ mash |=([[ship desk mime] [ship desk mime]] ~|(%mime-mash !!)) -- -- diff --git a/pkg/arvo/sys/lull.hoon b/pkg/arvo/sys/lull.hoon index 98bebfa049..9796356dd2 100644 --- a/pkg/arvo/sys/lull.hoon +++ b/pkg/arvo/sys/lull.hoon @@ -949,6 +949,7 @@ ++ nave |$ [typ dif] $_ + ^? |% ++ bunt *typ ++ diff |~([old=typ new=typ] *dif) @@ -958,7 +959,7 @@ |~ [a=[ship desk dif] b=[ship desk dif]] *(unit dif) ++ pact |~([typ dif] *typ) - ++ vale typ + ++ vale |~(noun *typ) -- :: $dais: processed mark core :: diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index 7cf3c4c62b..2511940ba9 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -543,7 +543,7 @@ =. naves.cache.nub (~(put by naves.cache.nub) mak [vase.res top]) [vase.res nub] =^ cor=vase nub (build-fit %mar mak) - =/ gad=vase (slap cor %limb %grad) + =/ gad=vase (slap cor limb/%grad) ?@ q.gad =+ !<(mok=mark gad) =^ deg=vase nub $(mak mok) @@ -575,23 +575,27 @@ :_ nub =+ !<(fom=mark (slap gad %limb %form)) ^- vase :: vase of nave - %+ slap - (with-faces cor+cor nave+!>(nave) ~) + %+ slap (slop (with-face cor+cor) bud) !, *hoon - =/ typ=mold _+<.cor - =/ dif=mold diff:grad:cor - ^- (nave typ dif) + =/ typ _+<.cor + =/ dif _*diff:grad:cor + ^- (nave:clay typ dif) |% ++ bunt +<.cor ++ diff |=([old=typ new=typ] (diff:~(grad cor old) new)) ++ form form:grad:cor - ++ join join:grad:cor + ++ join + |= [a=dif b=dif] + ^- (unit (unit dif)) + ?: =(a b) + ~ + `(join:grad:cor a b) ++ mash |= [a=[=ship =desk =dif] b=[=ship =desk =dif]] ^- (unit dif) ?: =(dif.a dif.b) ~ - `(mash:grad:cor a b) + (mash:grad:cor a b) ++ pact |=([v=typ d=dif] (pact:~(grad cor v) d)) ++ vale noun:grab:cor -- @@ -629,7 +633,7 @@ =/ res=vase (slam (slap nav limb/%join) (slop a b)) ?~ q.res ~ ?~ +.q.res [~ ~] - ``(slap res wing/u/u/~) + ``(slap res !,(*hoon ?>(?=([~ ~ *] .) u.u))) ++ mash |= [a=[=ship =desk diff=vase] b=[=ship =desk diff=vase]] ^- (unit vase) @@ -640,7 +644,7 @@ :(slop !>(ship.b) !>(desk.b) diff.b) ?~ q.res ~ - `(slap res limb/%u) + `(slap res !,(*hoon ?>((^ .) u))) ++ pact |= diff=vase (slam (slap nav limb/%pact) (slop sam diff)) @@ -681,7 +685,7 @@ :_ nub %+ slap (with-faces cor+old ~) ^- hoon - :+ %brts bcts/[%v like/[bunt/~ ~[cor/~]]] + :+ %brcl tsgr/[limb/%cor limb/%bunt] :+ %tsgl limb/b !,(*hoon ~(grow cor v)) :: try direct +grab @@ -3453,7 +3457,7 @@ :: ford :: ++ read-a - !. + ::!. TODO reinstate |= [=aeon =path] ^- [(unit (unit (each cage lobe))) ford-cache] ?. =(aeon let.dom) @@ -3475,7 +3479,7 @@ :_(fod.dom [~ ~ %& %vase !>(vase)]) :: ++ read-b - !. + ::!. TODO reinstate |= [=aeon =path] ^- [(unit (unit (each cage lobe))) ford-cache] ?. =(aeon let.dom) @@ -3491,7 +3495,7 @@ :_(fod.dom [~ ~ %& %dais !>(dais)]) :: ++ read-c - !. + ::!. TODO reinstate |= [=aeon =path] ^- [(unit (unit (each cage lobe))) ford-cache] ?. =(aeon let.dom) @@ -3849,7 +3853,7 @@ [~ fod] :: virtualize to catch and produce deterministic failures :: - !. + ::!. TODO reinstate =- ?:(?=(%& -<) p.- ((slog p.-) [[~ ~] fod])) %- mule |. ?- care.mun diff --git a/pkg/arvo/tests/sys/vane/clay.hoon b/pkg/arvo/tests/sys/vane/clay.hoon index 376b7466e4..5b5b4a7514 100644 --- a/pkg/arvo/tests/sys/vane/clay.hoon +++ b/pkg/arvo/tests/sys/vane/clay.hoon @@ -1,9 +1,10 @@ /+ *test /= clay-raw /sys/vane/clay -/* hello-gen %hoon /gen/hello/hoon -/* strandio-lib %hoon /lib/strandio/hoon -/* strand-lib %hoon /lib/strand/hoon -/* spider-sur %hoon /sur/spider/hoon +/* gen-hello %hoon /gen/hello/hoon +/* lib-strandio %hoon /lib/strandio/hoon +/* lib-strand %hoon /lib/strand/hoon +/* sur-spider %hoon /sur/spider/hoon +/* mar-mime %hoon /mar/mime/hoon :: !: =, format @@ -89,7 +90,35 @@ %- expect-fail |. (parse-pile:(ford):fusion /sur/foo/hoon "[") :: -++ test-hello-gen ^- tang +++ test-mar-mime ^- tang + =/ =ankh:clay + :- fil=~ + %- ~(gas by *(map @tas ankh:clay)) + :~ :+ %mar fil=~ + %- ~(gas by *(map @tas ankh:clay)) + :~ :+ %hoon fil=`[*lobe:clay hoon+!>(mar-mime)] dir=~ + == == + =/ ford + %: ford:fusion + bud + ankh + deletes=~ + changes=(my [/mar/mime/hoon &+hoon+mar-mime]~) + file-store=~ + *ford-cache:fusion + == + =/ [res=vase nub=state:ford:fusion] (get-nave:ford %mime) + ;: weld + %+ expect-eq + !>(*mime) + (slap res limb/%bunt) + :: + %+ expect-eq + !> (~(gas in *(set path)) /mar/mime/hoon ~) + !> dez:(~(got by vases.cache.nub) /mar/mime/hoon) + == +:: +++ test-gen-hello ^- tang =/ =ankh:clay :- fil=~ %- ~(gas by *(map @tas ankh:clay)) @@ -97,14 +126,14 @@ %- ~(gas by *(map @tas ankh:clay)) :~ :+ %hello fil=~ %- ~(gas by *(map @tas ankh:clay)) - :~ :+ %hoon fil=`[*lobe:clay hoon+!>(hello-gen)] dir=~ + :~ :+ %hoon fil=`[*lobe:clay hoon+!>(gen-hello)] dir=~ == == == =/ ford %: ford:fusion bud ankh deletes=~ - changes=(my [/gen/hello/hoon &+hoon+hello-gen]~) + changes=(my [/gen/hello/hoon &+hoon+gen-hello]~) file-store=~ *ford-cache:fusion == @@ -119,7 +148,7 @@ !> dez:(~(got by vases.cache.nub) /gen/hello/hoon) == :: -++ test-strandio-lib ^- tang +++ test-lib-strandio ^- tang =/ =ankh:clay :- fil=~ %- ~(gas by *(map @tas ankh:clay)) @@ -127,19 +156,19 @@ %- ~(gas by *(map @tas ankh:clay)) :~ :+ %strandio fil=~ %- ~(gas by *(map @tas ankh:clay)) - :~ :+ %hoon fil=`[*lobe:clay hoon+!>(strandio-lib)] dir=~ + :~ :+ %hoon fil=`[*lobe:clay hoon+!>(lib-strandio)] dir=~ == :: :+ %strand fil=~ %- ~(gas by *(map @tas ankh:clay)) - :~ :+ %hoon fil=`[*lobe:clay hoon+!>(strand-lib)] dir=~ + :~ :+ %hoon fil=`[*lobe:clay hoon+!>(lib-strand)] dir=~ == == :: :+ %sur fil=~ %- ~(gas by *(map @tas ankh:clay)) :~ :+ %spider fil=~ %- ~(gas by *(map @tas ankh:clay)) - :~ :+ %hoon fil=`[*lobe:clay hoon+!>(spider-sur)] dir=~ + :~ :+ %hoon fil=`[*lobe:clay hoon+!>(sur-spider)] dir=~ == == == =/ ford %: ford:fusion From ce9f1eb3dac411d89ae24ebda2a261509258e610 Mon Sep 17 00:00:00 2001 From: Ted Blackman Date: Tue, 26 Jan 2021 20:43:23 -0500 Subject: [PATCH 005/103] clay: fix +grow in mark casting --- pkg/arvo/mar/html.hoon | 1 - pkg/arvo/sys/vane/clay.hoon | 2 +- pkg/arvo/tests/sys/vane/clay.hoon | 30 ++++++++++++++++++++++-------- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/pkg/arvo/mar/html.hoon b/pkg/arvo/mar/html.hoon index 4e161fae16..203d75f612 100644 --- a/pkg/arvo/mar/html.hoon +++ b/pkg/arvo/mar/html.hoon @@ -6,7 +6,6 @@ :::: compute :: =, html -^| |_ htm=@t ++ grow :: convert to ^? diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index 2511940ba9..bebcb7e0a9 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -685,7 +685,7 @@ :_ nub %+ slap (with-faces cor+old ~) ^- hoon - :+ %brcl tsgr/[limb/%cor limb/%bunt] + :+ %brcl !,(*hoon v=+<.cor) :+ %tsgl limb/b !,(*hoon ~(grow cor v)) :: try direct +grab diff --git a/pkg/arvo/tests/sys/vane/clay.hoon b/pkg/arvo/tests/sys/vane/clay.hoon index 5b5b4a7514..6d5289cec6 100644 --- a/pkg/arvo/tests/sys/vane/clay.hoon +++ b/pkg/arvo/tests/sys/vane/clay.hoon @@ -5,6 +5,7 @@ /* lib-strand %hoon /lib/strand/hoon /* sur-spider %hoon /sur/spider/hoon /* mar-mime %hoon /mar/mime/hoon +/* mar-html %hoon /mar/html/hoon :: !: =, format @@ -91,17 +92,10 @@ |. (parse-pile:(ford):fusion /sur/foo/hoon "[") :: ++ test-mar-mime ^- tang - =/ =ankh:clay - :- fil=~ - %- ~(gas by *(map @tas ankh:clay)) - :~ :+ %mar fil=~ - %- ~(gas by *(map @tas ankh:clay)) - :~ :+ %hoon fil=`[*lobe:clay hoon+!>(mar-mime)] dir=~ - == == =/ ford %: ford:fusion bud - ankh + *ankh:clay deletes=~ changes=(my [/mar/mime/hoon &+hoon+mar-mime]~) file-store=~ @@ -118,6 +112,26 @@ !> dez:(~(got by vases.cache.nub) /mar/mime/hoon) == :: +++ test-cast-html-mime ^- tang + =/ changes + %- my + :~ [/mar/mime/hoon &+hoon+mar-mime] + [/mar/html/hoon &+hoon+mar-html] + == + =/ ford + %: ford:fusion + bud + *ankh:clay + deletes=~ + changes + file-store=~ + *ford-cache:fusion + == + =/ [res=vase nub=state:ford:fusion] (get-cast:ford %html %mime) + %+ expect-eq + (slam res !>('')) + !> `mime`[/text/html 13 ''] +:: ++ test-gen-hello ^- tang =/ =ankh:clay :- fil=~ From ca1585c841080776b22228cfec75c53eea9b1ea6 Mon Sep 17 00:00:00 2001 From: Ted Blackman Date: Tue, 26 Jan 2021 22:51:52 -0500 Subject: [PATCH 006/103] clay: static clay boots --- bin/solid.pill | 4 ++-- pkg/arvo/mar/mime.hoon | 5 ++++- pkg/arvo/sys/vane/clay.hoon | 7 +++---- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/bin/solid.pill b/bin/solid.pill index af2fc68fa4..d86bb9efd4 100644 --- a/bin/solid.pill +++ b/bin/solid.pill @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b38b016c4bcb23818328073e2b81fd09522699c9800e0744adbdbc15bb092795 -size 8749007 +oid sha256:fd51d827ed2bd4823a87caa953ce9a508d6a390442246198b093b0c39fa4e5ed +size 8763109 diff --git a/pkg/arvo/mar/mime.hoon b/pkg/arvo/mar/mime.hoon index 72737153e5..83b4daeb5e 100644 --- a/pkg/arvo/mar/mime.hoon +++ b/pkg/arvo/mar/mime.hoon @@ -24,6 +24,9 @@ ++ diff |=(mime +<) ++ pact |=(mime +<) ++ join |=([mime mime] `(unit mime)`~) - ++ mash |=([[ship desk mime] [ship desk mime]] ~|(%mime-mash !!)) + ++ mash + |= [[ship desk mime] [ship desk mime]] + ^- mime + ~|(%mime-mash !!) -- -- diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index bebcb7e0a9..c69e6af08a 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -554,8 +554,8 @@ %+ slap (with-faces deg+deg tub+tub but+but cor+cor nave+!>(nave) ~) !, *hoon - =/ typ=mold _+<.cor - =/ dif=mold vale:deg + =/ typ _+<.cor + =/ dif vale:deg ^- (nave typ dif) |% ++ bunt +<.cor @@ -573,7 +573,6 @@ ++ vale noun:grab:cor -- :_ nub - =+ !<(fom=mark (slap gad %limb %form)) ^- vase :: vase of nave %+ slap (slop (with-face cor+cor) bud) !, *hoon @@ -595,7 +594,7 @@ ^- (unit dif) ?: =(dif.a dif.b) ~ - (mash:grad:cor a b) + `(mash:grad:cor a b) ++ pact |=([v=typ d=dif] (pact:~(grad cor v) d)) ++ vale noun:grab:cor -- From 353e0f7395049fe51844b5f1047850976db3238f Mon Sep 17 00:00:00 2001 From: fang Date: Thu, 24 Dec 2020 13:03:56 +0100 Subject: [PATCH 007/103] clay: condense +pile-rule Rewrites the `+pile-rule` parsing rule for compactness and legibility. This is a purely stylistic change. --- pkg/arvo/sys/vane/clay.hoon | 102 ++++++++++++++---------------------- 1 file changed, 38 insertions(+), 64 deletions(-) diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index c69e6af08a..5ec3065d9c 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -848,75 +848,49 @@ ++ pile-rule |= pax=path %- full - %+ ifix [gay gay] - %+ cook |=(pile +<) - ;~ pfix + %+ ifix + :_ gay :: parse optional /? and ignore :: - ;~ pose - (cold ~ ;~(plug fas wut gap dem gap)) - (easy ~) + ;~(plug gay (punt ;~(plug fas wut gap dem gap))) + |^ + ;~ plug + %+ cook (bake zing (list (list taut))) + %+ rune hep + (most ;~(plug com gaw) taut-rule) + :: + %+ cook (bake zing (list (list taut))) + %+ rune lus + (most ;~(plug com gaw) taut-rule) + :: + %+ rune tis + ;~(plug sym ;~(pfix gap fas (more fas urs:ab))) + :: + %+ rune tar + ;~ (glue gap) + sym + ;~(pfix cen sym) + ;~(pfix fas (more fas urs:ab)) == :: - ;~ plug - ;~ pose - ;~ sfix - %+ cook |=((list (list taut)) (zing +<)) - %+ more gap - ;~ pfix ;~(plug fas hep gap) - (most ;~(plug com gaw) taut-rule) - == - gap - == - (easy ~) - == - :: - ;~ pose - ;~ sfix - %+ cook |=((list (list taut)) (zing +<)) - %+ more gap - ;~ pfix ;~(plug fas lus gap) - (most ;~(plug com gaw) taut-rule) - == - gap - == - (easy ~) - == - :: - ;~ pose - ;~ sfix - %+ cook |=((list [face=term =path]) +<) - %+ more gap - ;~ pfix ;~(plug fas tis gap) - %+ cook |=([term path] +<) - ;~(plug sym ;~(pfix ;~(plug gap fas) (more fas urs:ab))) - == - gap - == - (easy ~) - == - :: - ;~ pose - ;~ sfix - %+ cook |=((list [face=term =mark =path]) +<) - %+ more gap - ;~ pfix ;~(plug fas tar gap) - %+ cook |=([term mark path] +<) - ;~ plug - sym - ;~(pfix ;~(plug gap cen) sym) - ;~(pfix ;~(plug gap fas) (more fas urs:ab)) - == - == - gap - == - (easy ~) - == - :: - %+ cook |=(huz=(list hoon) `hoon`tssg+huz) - (most gap tall:(vang & pax)) - == + %+ stag %tssg + (most gap tall:(vang & pax)) == + :: + ++ pant + |* fel=^rule + ;~(pose fel (easy ~)) + :: + ++ mast + |* [bus=^rule fel=^rule] + ;~(sfix (more bus fel) bus) + :: + ++ rune + |* [bus=^rule fel=^rule] + %- pant + %+ mast gap + ;~(pfix fas bus gap fel) + -- :: ++ taut-rule %+ cook |=(taut +<) From 3c8e73cbc81ee1ca09132269574f9f800c9eeb79 Mon Sep 17 00:00:00 2001 From: Liam Fitzgerald Date: Fri, 29 Jan 2021 10:09:52 +1000 Subject: [PATCH 008/103] Revert "language-server: parse commands permissively" This reverts commit 8f8ec8bcc5b413e7bed66a5f747ac4e76ba3ab44. --- pkg/arvo/lib/language-server/complete.hoon | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/arvo/lib/language-server/complete.hoon b/pkg/arvo/lib/language-server/complete.hoon index 6e352e8196..18f16c1687 100644 --- a/pkg/arvo/lib/language-server/complete.hoon +++ b/pkg/arvo/lib/language-server/complete.hoon @@ -247,15 +247,15 @@ :: ++ get-id |= [pos=@ud txt=tape] - ^- [forward=(unit @t) backward=(unit @t) id=(unit @t)] - =/ seek - ;~(sfix (punt (cook crip (star prn))) (star ;~(pose prn (just `@`10)))) - =/ forward=(unit @t) - (scan (slag pos txt) seek) - =/ backward=(unit @t) + ^- [forward=(unit term) backward=(unit term) id=(unit term)] + =/ forward=(unit term) + %+ scan `tape`(slag pos txt) + ;~(sfix (punt sym) (star ;~(pose prn (just `@`10)))) + =/ backward=(unit term) %- (lift |=(t=@tas (swp 3 t))) - (scan (flop (scag pos txt)) seek) - =/ id=(unit @t) + %+ scan `tape`(flop (scag pos txt)) + ;~(sfix (punt sym) (star ;~(pose prn (just `@`10)))) + =/ id=(unit term) ?~ forward ?~ backward ~ From 410fe58975754dbeb8cd6e3525f4aa31f62a0ddf Mon Sep 17 00:00:00 2001 From: Liam Fitzgerald Date: Fri, 29 Jan 2021 11:43:40 +1000 Subject: [PATCH 009/103] language-server: ignore /sys files --- pkg/arvo/app/language-server.hoon | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/arvo/app/language-server.hoon b/pkg/arvo/app/language-server.hoon index 88b249157f..6ba8704913 100644 --- a/pkg/arvo/app/language-server.hoon +++ b/pkg/arvo/app/language-server.hoon @@ -275,12 +275,14 @@ ++ handle-did-open |= item=text-document-item:lsp-sur ^- (quip card _state) + =/ =path + (uri-to-path:build uri.item) + ?: ?=(%sys -.path) + `state =/ buf=wall (to-wall (trip text.item)) =. bufs (~(put by bufs) uri.item buf) - =/ =path - (uri-to-path:build uri.item) :_ state %+ weld (give-rpc-notification (get-diagnostics uri.item)) From b4cbe7b5ebf86fc6a5ec408b39a8f01794a4f69a Mon Sep 17 00:00:00 2001 From: Liam Fitzgerald Date: Fri, 29 Jan 2021 11:44:13 +1000 Subject: [PATCH 010/103] language-server: prettier hover format --- pkg/arvo/app/language-server.hoon | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/arvo/app/language-server.hoon b/pkg/arvo/app/language-server.hoon index 6ba8704913..921cc20513 100644 --- a/pkg/arvo/app/language-server.hoon +++ b/pkg/arvo/app/language-server.hoon @@ -320,12 +320,12 @@ ?~ p.tab-list ~ ?~ u.p.tab-list ~ :- ~ - %- crip - ;: weld - "`" - ~(ram re ~(duck easy-print detail.i.u.p.tab-list)) - "`" - == + =- (crip :(weld "```hoon\0a" tape "\0a```")) + ^- =tape + %- zing + %+ join "\0a" + %+ scag 20 + (~(win re ~(duck easy-print detail.i.u.p.tab-list)) 2 80) :: ++ sync-buf |= [buf=wall changes=(list change:lsp-sur)] From 377b2cd10ca2db9cf0103bee9be9f85438315aa0 Mon Sep 17 00:00:00 2001 From: Liam Fitzgerald Date: Fri, 29 Jan 2021 11:44:37 +1000 Subject: [PATCH 011/103] language-server: prevent %play-open on %zpgl --- pkg/arvo/lib/language-server/complete.hoon | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/arvo/lib/language-server/complete.hoon b/pkg/arvo/lib/language-server/complete.hoon index 18f16c1687..1a2a6c1f1e 100644 --- a/pkg/arvo/lib/language-server/complete.hoon +++ b/pkg/arvo/lib/language-server/complete.hoon @@ -185,6 +185,7 @@ [%zpmc *] (both p.gen q.gen) [%zpts *] loop(gen p.gen) [%zppt *] (both q.gen r.gen) + [%zpgl *] (spec-and-hoon p.gen q.gen) [%zpzp *] ~ * =+ doz=~(open ap gen) From 7b46b8bfed104f0c019ec143eba73c12dd1904f5 Mon Sep 17 00:00:00 2001 From: Liam Fitzgerald Date: Fri, 29 Jan 2021 12:01:33 +1000 Subject: [PATCH 012/103] language-server: break up parser from shoe --- pkg/arvo/lib/language-server/complete.hoon | 32 ++++++++++++++-------- pkg/arvo/lib/shoe.hoon | 2 +- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/pkg/arvo/lib/language-server/complete.hoon b/pkg/arvo/lib/language-server/complete.hoon index 1a2a6c1f1e..e2beefcc0f 100644 --- a/pkg/arvo/lib/language-server/complete.hoon +++ b/pkg/arvo/lib/language-server/complete.hoon @@ -246,17 +246,27 @@ ^- (unit [term type]) ~ :: +++ get-id-sym + |= [pos=@ud =tape] + %^ get-id pos tape + ^- $-(nail (like (unit @t))) + ;~(sfix (punt sym) (star ;~(pose prn (just `@`10)))) +:: +++ get-id-cord + |= [pos=@ud =tape] + %^ get-id pos tape + ^- $-(nail (like (unit @t))) + ;~(sfix (punt (cook crip (star prn))) (star ;~(pose prn (just `@`10)))) +:: ++ get-id - |= [pos=@ud txt=tape] - ^- [forward=(unit term) backward=(unit term) id=(unit term)] - =/ forward=(unit term) - %+ scan `tape`(slag pos txt) - ;~(sfix (punt sym) (star ;~(pose prn (just `@`10)))) - =/ backward=(unit term) - %- (lift |=(t=@tas (swp 3 t))) - %+ scan `tape`(flop (scag pos txt)) - ;~(sfix (punt sym) (star ;~(pose prn (just `@`10)))) - =/ id=(unit term) + |= [pos=@ud txt=tape seek=$-(nail (like (unit @t)))] + ^- [forward=(unit @t) backward=(unit @t) id=(unit @t)] + =/ forward=(unit @t) + (scan (slag pos txt) seek) + =/ backward=(unit @t) + %- (lift |=(t=@t (swp 3 t))) + (scan (flop (scag pos txt)) seek) + =/ id=(unit @t) ?~ forward ?~ backward ~ @@ -273,7 +283,7 @@ ^- [back-pos=@ud fore-pos=@ud txt=tape] :: Find beg-pos by searching backward to where the current term :: begins - =+ (get-id pos txt) + =+ (get-id-sym pos txt) =/ back-pos ?~ backward pos diff --git a/pkg/arvo/lib/shoe.hoon b/pkg/arvo/lib/shoe.hoon index d3769ad00f..0348e10e52 100644 --- a/pkg/arvo/lib/shoe.hoon +++ b/pkg/arvo/lib/shoe.hoon @@ -291,7 +291,7 @@ ++ tab |= pos=@ud ^- (quip card _cli-state) - =+ (get-id:auto pos (tufa buf.cli-state)) + =+ (get-id-cord:auto pos (tufa buf.cli-state)) =/ needle=term (fall id %$) :: autocomplete empty command iff user at start of command From ac7aa12dd76b5e95a15ee7ec7563d65f311ab5dd Mon Sep 17 00:00:00 2001 From: Liam Fitzgerald Date: Sat, 30 Jan 2021 12:09:59 +1000 Subject: [PATCH 013/103] language-server: retain state on reload --- pkg/arvo/app/language-server.hoon | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/arvo/app/language-server.hoon b/pkg/arvo/app/language-server.hoon index 921cc20513..a93e57c7fc 100644 --- a/pkg/arvo/app/language-server.hoon +++ b/pkg/arvo/app/language-server.hoon @@ -5,7 +5,7 @@ easy-print=language-server-easy-print, rune-snippet=language-server-rune-snippet, build=language-server-build, - default-agent + default-agent, verb |% +$ card card:agent:gall +$ lsp-req @@ -44,6 +44,7 @@ == -- ^- agent:gall +%+ verb | =| state-zero =* state - =< @@ -69,7 +70,7 @@ |= old-state=vase ^- (quip card _this) ~& > %lsp-upgrade - [~ this(state *state-zero)] + [~ this(state !<(state-zero old-state))] :: ++ on-poke ^+ on-poke:*agent:gall @@ -324,8 +325,8 @@ ^- =tape %- zing %+ join "\0a" - %+ scag 20 - (~(win re ~(duck easy-print detail.i.u.p.tab-list)) 2 80) + %+ scag 40 + (~(win re ~(duck easy-print detail.i.u.p.tab-list)) 0 140) :: ++ sync-buf |= [buf=wall changes=(list change:lsp-sur)] From ad20ddb1d421244e5d1f2b469a81a93e6f157150 Mon Sep 17 00:00:00 2001 From: Ted Blackman Date: Wed, 17 Feb 2021 14:31:20 -0500 Subject: [PATCH 014/103] clay: added mark runes; can boot off solid pill --- bin/solid.pill | 4 +- pkg/arvo/lib/language-server/complete.hoon | 2 +- pkg/arvo/lib/language-server/parser.hoon | 110 +++++++++------------ pkg/arvo/sys/lull.hoon | 4 + pkg/arvo/sys/vane/clay.hoon | 28 ++++++ pkg/arvo/tests/sys/vane/clay.hoon | 4 +- 6 files changed, 84 insertions(+), 68 deletions(-) diff --git a/bin/solid.pill b/bin/solid.pill index d86bb9efd4..36e52f46dd 100644 --- a/bin/solid.pill +++ b/bin/solid.pill @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:fd51d827ed2bd4823a87caa953ce9a508d6a390442246198b093b0c39fa4e5ed -size 8763109 +oid sha256:789c5755516cf60993f01d6842437c9cf487b24fa6fca8d27f2ff52f9ec79364 +size 8824140 diff --git a/pkg/arvo/lib/language-server/complete.hoon b/pkg/arvo/lib/language-server/complete.hoon index 6e352e8196..5b0d0612f8 100644 --- a/pkg/arvo/lib/language-server/complete.hoon +++ b/pkg/arvo/lib/language-server/complete.hoon @@ -343,7 +343,7 @@ [%| p.res] :- %& ~? > debug %parsed-good - ((cury tab-list-hoon sut) hoon.p.res) + ((cury tab-list-hoon sut) hoon:`pile:clay`p.res) :: :: Generators ++ tab-generators diff --git a/pkg/arvo/lib/language-server/parser.hoon b/pkg/arvo/lib/language-server/parser.hoon index 339d319721..05667bee03 100644 --- a/pkg/arvo/lib/language-server/parser.hoon +++ b/pkg/arvo/lib/language-server/parser.hoon @@ -5,75 +5,59 @@ ++ pile-rule |= pax=path %- full - %+ ifix [gay gay] - %+ cook |=(pile +<) - ;~ pfix + %+ ifix + :_ gay :: parse optional /? and ignore :: - ;~ pose - (cold ~ ;~(plug fas wut gap dem gap)) - (easy ~) + ;~(plug gay (punt ;~(plug fas wut gap dem gap))) + |^ + ;~ plug + %+ cook (bake zing (list (list taut))) + %+ rune hep + (most ;~(plug com gaw) taut-rule) + :: + %+ cook (bake zing (list (list taut))) + %+ rune lus + (most ;~(plug com gaw) taut-rule) + :: + %+ rune tis + ;~(plug sym ;~(pfix gap fas (more fas urs:ab))) + :: + %+ rune cen + ;~(plug sym ;~(pfix cen sym)) + :: + %+ rune buc + ;~ (glue gap) + sym + ;~(pfix cen sym) + ;~(pfix cen sym) == :: - ;~ plug - ;~ pose - ;~ sfix - %+ cook |=((list (list taut)) (zing +<)) - %+ more gap - ;~ pfix ;~(plug fas hep gap) - (most ;~(plug com gaw) taut-rule) - == - gap - == - (easy ~) - == - :: - ;~ pose - ;~ sfix - %+ cook |=((list (list taut)) (zing +<)) - %+ more gap - ;~ pfix ;~(plug fas lus gap) - (most ;~(plug com gaw) taut-rule) - == - gap - == - (easy ~) - == - :: - ;~ pose - ;~ sfix - %+ cook |=((list [face=term =path]) +<) - %+ more gap - ;~ pfix ;~(plug fas tis gap) - %+ cook |=([term path] +<) - ;~(plug sym ;~(pfix ;~(plug gap fas) (more fas urs:ab))) - == - gap - == - (easy ~) - == - :: - ;~ pose - ;~ sfix - %+ cook |=((list [face=term =mark =path]) +<) - %+ more gap - ;~ pfix ;~(plug fas tar gap) - %+ cook |=([term mark path] +<) - ;~ plug - sym - ;~(pfix ;~(plug gap cen) sym) - ;~(pfix ;~(plug gap fas) (more fas urs:ab)) - == - == - gap - == - (easy ~) - == - :: - %+ cook |=(huz=(list hoon) `hoon`tssg+huz) - (most gap tall:(vang & pax)) + %+ rune tar + ;~ (glue gap) + sym + ;~(pfix cen sym) + ;~(pfix fas (more fas urs:ab)) == + :: + %+ stag %tssg + (most gap tall:(vang & pax)) == + :: + ++ pant + |* fel=^rule + ;~(pose fel (easy ~)) + :: + ++ mast + |* [bus=^rule fel=^rule] + ;~(sfix (more bus fel) bus) + :: + ++ rune + |* [bus=^rule fel=^rule] + %- pant + %+ mast gap + ;~(pfix fas bus gap fel) + -- :: ++ taut-rule %+ cook |=(taut +<) diff --git a/pkg/arvo/sys/lull.hoon b/pkg/arvo/sys/lull.hoon index 9796356dd2..48cdce15b4 100644 --- a/pkg/arvo/sys/lull.hoon +++ b/pkg/arvo/sys/lull.hoon @@ -928,12 +928,16 @@ :: /- sur-file :: surface imports from /sur :: /+ lib-file :: library imports from /lib :: /= face /path :: imports built hoon file at path + :: /% face %mark :: imports mark definition from /mar + :: /$ face %from %to :: imports mark converter from /mar :: /* face %mark /path :: unbuilt file imports, as mark :: +$ pile $: sur=(list taut) lib=(list taut) raw=(list [face=term =path]) + maz=(list [face=term =mark]) + caz=(list [face=term =mars]) bar=(list [face=term =mark =path]) =hoon == diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index 5ec3065d9c..b1057651f3 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -828,6 +828,8 @@ =^ sut=vase nub (run-tauts bud %sur sur.pile) =^ sut=vase nub (run-tauts sut %lib lib.pile) =^ sut=vase nub (run-raw sut raw.pile) + =^ sut=vase nub (run-maz sut maz.pile) + =^ sut=vase nub (run-caz sut caz.pile) =^ sut=vase nub (run-bar sut bar.pile) =/ res=vase (road |.((slap sut hoon.pile))) [res nub] @@ -865,6 +867,16 @@ :: %+ rune tis ;~(plug sym ;~(pfix gap fas (more fas urs:ab))) + :: + %+ rune cen + ;~(plug sym ;~(pfix cen sym)) + :: + %+ rune buc + ;~ (glue gap) + sym + ;~(pfix cen sym) + ;~(pfix cen sym) + == :: %+ rune tar ;~ (glue gap) @@ -916,6 +928,22 @@ =. p.pin [%face face.i.raw p.pin] $(sut (slop pin sut), raw t.raw) :: + ++ run-maz + |= [sut=vase maz=(list [face=term =mark])] + ^- [vase state] + ?~ maz [sut nub] + =^ pin=vase nub (get-nave mark.i.maz) + =. p.pin [%face face.i.maz p.pin] + $(sut (slop pin sut), maz t.maz) + :: + ++ run-caz + |= [sut=vase caz=(list [face=term =mars])] + ^- [vase state] + ?~ caz [sut nub] + =^ pin=vase nub (get-cast mars.i.caz) + =. p.pin [%face face.i.caz p.pin] + $(sut (slop pin sut), caz t.caz) + :: ++ run-bar |= [sut=vase bar=(list [face=term =mark =path])] ^- [vase state] diff --git a/pkg/arvo/tests/sys/vane/clay.hoon b/pkg/arvo/tests/sys/vane/clay.hoon index 6d5289cec6..755d5e43aa 100644 --- a/pkg/arvo/tests/sys/vane/clay.hoon +++ b/pkg/arvo/tests/sys/vane/clay.hoon @@ -18,7 +18,7 @@ ++ test-parse-pile ^- tang %+ expect-eq !> ^- pile:fusion - :* ~ ~ ~ ~ + :* ~ ~ ~ ~ ~ ~ tssg+[%dbug [/sur/foo/hoon [[1 1] [1 2]]] [%cnts ~[[%.y 1]] ~]]~ == !> (parse-pile:(ford):fusion /sur/foo/hoon ".") @@ -54,7 +54,7 @@ [`%hood-drum %hood-drum] [`%hood-write %hood-write] == - raw=~ bar=~ + raw=~ bar=~ maz=~ caz=~ hoon=tssg+[p:(need q:(tall:(vang & /app/hood/hoon) [17 1] "."))]~ == !> (parse-pile:(ford):fusion /app/hood/hoon src) From 22295f08fb7c77216a8f4be5305a3d8b1be1a47e Mon Sep 17 00:00:00 2001 From: Ted Blackman Date: Wed, 17 Feb 2021 17:57:44 -0500 Subject: [PATCH 015/103] gall: fix request queue desynchronization bug --- pkg/arvo/sys/vane/gall.hoon | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pkg/arvo/sys/vane/gall.hoon b/pkg/arvo/sys/vane/gall.hoon index 7bd5b3da23..ba6f37f81d 100644 --- a/pkg/arvo/sys/vane/gall.hoon +++ b/pkg/arvo/sys/vane/gall.hoon @@ -680,12 +680,9 @@ :: note this should only happen on reverse bones, so only facts :: and kicks :: - =/ sys-wire [%sys wire] :: TODO: %drip %kick so app crash can't kill the remote %pull :: - =/ =ames-request-all [%0 %u ~] - =. mo-core - (mo-pass sys-wire %a %plea ship %g /ge/[foreign-agent] ames-request-all) + =. mo-core (mo-send-foreign-request ship foreign-agent %leave ~) =. mo-core (mo-give %unto %kick ~) mo-core == From 473a520b27b321fb197f91ecb9f4421c5c6f7ae7 Mon Sep 17 00:00:00 2001 From: Ted Blackman Date: Thu, 18 Feb 2021 21:47:02 -0500 Subject: [PATCH 016/103] ames: more informative printing --- pkg/arvo/sys/vane/ames.hoon | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/pkg/arvo/sys/vane/ames.hoon b/pkg/arvo/sys/vane/ames.hoon index ad799da549..0495100249 100644 --- a/pkg/arvo/sys/vane/ames.hoon +++ b/pkg/arvo/sys/vane/ames.hoon @@ -1909,6 +1909,9 @@ =/ =bone bone.shut-packet :: ?: ?=(%& -.meat.shut-packet) + =+ ?~ dud ~ + %. ~ + (slog leaf+"ames: crashed on fragment" >mote.u.dud< tang.u.dud) (run-message-sink bone %hear lane shut-packet ?=(~ dud)) :: Just try again on error, printing trace :: @@ -1917,7 +1920,7 @@ :: =+ ?~ dud ~ %. ~ - (slog leaf+"ames: crashed on message ack" >mote.u.dud< tang.u.dud) + (slog leaf+"ames: crashed on ack" >mote.u.dud< tang.u.dud) (run-message-pump bone %hear [message-num +.meat]:shut-packet) :: +on-memo: handle request to send message :: @@ -2203,12 +2206,15 @@ ?. ?=([%hear * * ok=%.n] task) :: fresh boon; give message to client vane :: - %- (trace msg.veb |.("boon {}")) + %- %+ trace msg.veb + =/ dat [her.channel bone=bone message-num=message-num -.task] + |.("sink boon {}") peer-core :: we previously crashed on this message; notify client vane :: %- %+ trace msg.veb - |.("crashed on boon {}") + =/ dat [her.channel bone=bone message-num=message-num -.task] + |.("crashed on sink boon {}") boon-to-lost :: +boon-to-lost: convert all boons to losts :: @@ -2226,7 +2232,9 @@ ++ on-sink-nack-trace |= [=message-num message=*] ^+ peer-core - %- (trace msg.veb |.("nack trace {}")) + %- %+ trace msg.veb + =/ dat [her.channel bone=bone message-num=message-num] + |.("sink naxplanation {}") :: =+ ;; =naxplanation message :: ack nack-trace message (only applied if we don't later crash) @@ -2243,7 +2251,9 @@ ++ on-sink-plea |= [=message-num message=*] ^+ peer-core - %- (trace msg.veb |.("plea {}")) + %- %+ trace msg.veb + =/ dat [her.channel bone=bone message-num=message-num] + |.("sink plea {}") :: is this the first time we're trying to process this message? :: ?. ?=([%hear * * ok=%.n] task) From b702505ac8328cc220b025eec478f1e578263b98 Mon Sep 17 00:00:00 2001 From: Ted Blackman Date: Thu, 18 Feb 2021 21:50:59 -0500 Subject: [PATCH 017/103] clay: print if +read-at-aeon crashes --- pkg/arvo/sys/vane/clay.hoon | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index d665c5fb91..78f8544a3f 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -3810,7 +3810,8 @@ :: virtualize to catch and produce deterministic failures :: !. - =- ?:(?=(%& -<) p.- ((slog p.-) [[~ ~] fod])) + =- ?: ?=(%& -<) p.- + ((slog leaf+"gall: read-at-aeon fail {}" p.-) [[~ ~] fod]) %- mule |. ?- care.mun %d From d317a0847b88e209b5c0e29dfb791aabf33b3889 Mon Sep 17 00:00:00 2001 From: Ted Blackman Date: Thu, 18 Feb 2021 21:51:44 -0500 Subject: [PATCH 018/103] gall: crash properly on failed %boon from ames --- pkg/arvo/sys/vane/gall.hoon | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pkg/arvo/sys/vane/gall.hoon b/pkg/arvo/sys/vane/gall.hoon index ba6f37f81d..a481307874 100644 --- a/pkg/arvo/sys/vane/gall.hoon +++ b/pkg/arvo/sys/vane/gall.hoon @@ -939,15 +939,13 @@ =/ sky (rof ~ %cb [our %home case] /[mark.ames-response]) ?- sky ?(~ [~ ~]) - =/ ror "gall: ames mark fail {}" - (mo-give %done `vale+[leaf+ror]~) + (mean leaf+"gall: ames mark fail {}" ~) :: [~ ~ *] =+ !<(=dais:clay q.u.u.sky) =/ res (mule |.((vale:dais noun.ames-response))) ?: ?=(%| -.res) - =/ ror "gall: ames vale fail {}" - (mo-give %done `vale+[leaf+ror p.res]) + (mean leaf+"gall: ames vale fail {}" p.res) =. mo-core %+ mo-pass /nowhere [%c %warp our %home ~ %sing %b case /[mark.ames-response]] From 669169be0eb70a04f8ea8964df6cc8fb529a1a38 Mon Sep 17 00:00:00 2001 From: Ted Blackman Date: Thu, 18 Feb 2021 22:02:54 -0500 Subject: [PATCH 019/103] pill: update solid --- bin/solid.pill | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/solid.pill b/bin/solid.pill index 62daecc18a..bff0e9efa9 100644 --- a/bin/solid.pill +++ b/bin/solid.pill @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9433e0a7f1edbdcc6c8ac3e70c9516061d35218e5a1dc3192b2189dfb28cdc88 -size 9539470 +oid sha256:7ee4e9394edf0d6e16586f5c8a26d716aaa85435fc0cfba3cc4470315282dab4 +size 8723705 From eb56fbd3f4524bf6fce3e684208a496de6accfeb Mon Sep 17 00:00:00 2001 From: Ted Blackman Date: Fri, 19 Feb 2021 11:54:34 -0500 Subject: [PATCH 020/103] ames: shorten error printing; update pill --- bin/solid.pill | 4 ++-- pkg/arvo/sys/vane/ames.hoon | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/bin/solid.pill b/bin/solid.pill index bff0e9efa9..b218f38de1 100644 --- a/bin/solid.pill +++ b/bin/solid.pill @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:7ee4e9394edf0d6e16586f5c8a26d716aaa85435fc0cfba3cc4470315282dab4 -size 8723705 +oid sha256:c2a0dcf375880feafb317ad440bbf042ba2e4c7d15ad97e37b7b98197b7c7488 +size 8787841 diff --git a/pkg/arvo/sys/vane/ames.hoon b/pkg/arvo/sys/vane/ames.hoon index 0495100249..bc7272906b 100644 --- a/pkg/arvo/sys/vane/ames.hoon +++ b/pkg/arvo/sys/vane/ames.hoon @@ -1911,7 +1911,9 @@ ?: ?=(%& -.meat.shut-packet) =+ ?~ dud ~ %. ~ - (slog leaf+"ames: crashed on fragment" >mote.u.dud< tang.u.dud) + %+ slog + leaf+"ames: {} fragment crashed {}" + ?.(msg.veb ~ tang.u.dud) (run-message-sink bone %hear lane shut-packet ?=(~ dud)) :: Just try again on error, printing trace :: @@ -1920,7 +1922,8 @@ :: =+ ?~ dud ~ %. ~ - (slog leaf+"ames: crashed on ack" >mote.u.dud< tang.u.dud) + %+ slog leaf+"ames: {} ack crashed {}" + ?.(msg.veb ~ tang.u.dud) (run-message-pump bone %hear [message-num +.meat]:shut-packet) :: +on-memo: handle request to send message :: From 9bf6c6136ad589830252e4e3cdbc7a8dfef567b4 Mon Sep 17 00:00:00 2001 From: Ted Blackman Date: Mon, 22 Feb 2021 14:35:57 -0500 Subject: [PATCH 021/103] clay: fix mark +grad delegation bug --- pkg/arvo/sys/vane/clay.hoon | 57 +++++++++++++++++++++++++++++-- pkg/arvo/tests/sys/vane/clay.hoon | 33 +++++++++++++++++- 2 files changed, 86 insertions(+), 4 deletions(-) diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index b1057651f3..fa87236a7b 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -555,7 +555,7 @@ (with-faces deg+deg tub+tub but+but cor+cor nave+!>(nave) ~) !, *hoon =/ typ _+<.cor - =/ dif vale:deg + =/ dif diff:deg ^- (nave typ dif) |% ++ bunt +<.cor @@ -3916,7 +3916,7 @@ :: :::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: =| :: instrument state - $: ver=%6 :: vane version + $: ver=%7 :: vane version ruf=raft :: revision tree == :: |= [now=@da eny=@uvJ rof=roof] :: current invocation @@ -4158,8 +4158,59 @@ == :: ++ load - |= old=[%6 raft] + => |% + +$ raft-any + $% [%7 raft-7] + [%6 raft-6] + == + +$ raft-7 raft + +$ dojo-7 dojo + +$ ford-cache-7 ford-cache + +$ raft-6 + $: rom=room-6 :: domestic + hoy=(map ship rung) :: foreign + ran=rang :: hashes + mon=(map term beam) :: mount points + hez=(unit duct) :: sync duct + cez=(map @ta crew) :: permission groups + pud=(unit [=desk =yoki]) :: pending update + == :: + +$ room-6 [hun=duct dos=(map desk dojo-6)] + +$ dojo-6 + $: qyx=cult :: subscribers + dom=dome-6 :: desk state + per=regs :: read perms per path + pew=regs :: write perms per path + == + +$ dome-6 + $: ank=ankh :: state + let=aeon :: top id + hit=(map aeon tako) :: versions by id + lab=(map @tas aeon) :: labels + mim=(map path mime) :: mime cache + fod=ford-cache-6 :: ford cache + fer=(unit reef-cache) :: reef cache + == + +$ ford-cache-6 * :: discard old cache + -- + |= old=raft-any + |^ + =? old ?=(%6 -.old) 7+(raft-6-to-7 +.old) + ?> ?=(%7 -.old) ..^$(ruf +.old) + :: +raft-6-to-7: delete stale ford caches (they could all be invalid) + :: + ++ raft-6-to-7 + |= raf=raft-6 + ^- raft-7 + %= raf + dos.rom + %- ~(run by dos.rom.raf) + |= doj=dojo-6 + ^- dojo-7 + doj(fod.dom *ford-cache-7) + == + -- :: ++ scry :: inspect ^- roon diff --git a/pkg/arvo/tests/sys/vane/clay.hoon b/pkg/arvo/tests/sys/vane/clay.hoon index 755d5e43aa..0331073c83 100644 --- a/pkg/arvo/tests/sys/vane/clay.hoon +++ b/pkg/arvo/tests/sys/vane/clay.hoon @@ -1,11 +1,15 @@ /+ *test /= clay-raw /sys/vane/clay /* gen-hello %hoon /gen/hello/hoon +/* lib-cram %hoon /lib/cram/hoon /* lib-strandio %hoon /lib/strandio/hoon /* lib-strand %hoon /lib/strand/hoon /* sur-spider %hoon /sur/spider/hoon -/* mar-mime %hoon /mar/mime/hoon /* mar-html %hoon /mar/html/hoon +/* mar-mime %hoon /mar/mime/hoon +/* mar-udon %hoon /mar/udon/hoon +/* mar-txt %hoon /mar/txt/hoon +/* mar-txt-diff %hoon /mar/txt-diff/hoon :: !: =, format @@ -112,6 +116,33 @@ !> dez:(~(got by vases.cache.nub) /mar/mime/hoon) == :: +++ test-mar-udon ^- tang + =/ ford + %: ford:fusion + bud + *ankh:clay + deletes=~ + ^= changes + %- my + :~ [/mar/udon/hoon &+hoon+mar-udon] + [/lib/cram/hoon &+hoon+lib-cram] + [/mar/txt/hoon &+hoon+mar-txt] + [/mar/txt-diff/hoon &+hoon+mar-txt-diff] + == + file-store=~ + *ford-cache:fusion + == + =/ [res=vase nub=state:ford:fusion] (get-nave:ford %udon) + ;: weld + %+ expect-eq + !>(*@t) + (slap res limb/%bunt) + :: + %+ expect-eq + !> (~(gas in *(set path)) /mar/udon/hoon /lib/cram/hoon ~) + !> dez:(~(got by vases.cache.nub) /mar/udon/hoon) + == +:: ++ test-cast-html-mime ^- tang =/ changes %- my From dd87bd9e1c1eab11655a84f49729e0902f715d20 Mon Sep 17 00:00:00 2001 From: Ted Blackman Date: Mon, 22 Feb 2021 14:40:30 -0500 Subject: [PATCH 022/103] pill: update solid --- bin/solid.pill | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/solid.pill b/bin/solid.pill index 36e52f46dd..79b355a1d3 100644 --- a/bin/solid.pill +++ b/bin/solid.pill @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:789c5755516cf60993f01d6842437c9cf487b24fa6fca8d27f2ff52f9ec79364 -size 8824140 +oid sha256:be826725d4d7d3b7f6b88821b151baa3837c42874400016db838abd42e6b9c01 +size 8962205 From 9e69892631753c59aa2973cd2533c3b3fcd4f6e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C5=8Dshin?= Date: Sun, 21 Feb 2021 22:26:55 +0000 Subject: [PATCH 023/103] zuse: time conversion fixes State before: in chrono:userlib, there were second-resolution @da-to-unix and unix-to-@da functions. In en/dejs:format, there were millisecond-resolution @da-to-unix and unix-to-@da functions. The @da-to-unix path in time:enjs confusingly rounded to the nearest millisecond, meaning millisecond n was a label for [n-0.5, n+0.5) rather than [n, n+1). This adds a millisecond-resolution @da-to-unix and unix-to-@da to chrono:userlib, and a second-resolution conversion to en/dejs:format. It makes use of the chrono:userlib functions in en/dejs, and doesn't do any rounding. Backwards-incompatible changes: - made unt:chrono:userlib take a @da rather than @. --- pkg/arvo/sys/zuse.hoon | 32 +++++++++++++++++++++++------ pkg/arvo/tests/sys/zuse/format.hoon | 8 ++++++-- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/pkg/arvo/sys/zuse.hoon b/pkg/arvo/sys/zuse.hoon index c647366350..0ae410df48 100644 --- a/pkg/arvo/sys/zuse.hoon +++ b/pkg/arvo/sys/zuse.hoon @@ -3297,11 +3297,14 @@ %- flop |- ^- ^tape ?:(=(0 a) ~ [(add '0' (mod a 10)) $(a (div a 10))]) + :: :: ++sect:enjs:format + ++ sect :: s timestamp + |= a=^time + (numb (unt:chrono:userlib a)) :: :: ++time:enjs:format ++ time :: ms timestamp |= a=^time - =- (numb (div (mul - 1.000) ~s1)) - (add (div ~s1 2.000) (sub a ~1970.1.1)) + (numb (unm:chrono:userlib a)) :: :: ++path:enjs:format ++ path :: string from path |= a=^path @@ -3367,7 +3370,13 @@ ++ di :: millisecond date %+ cu |= a=@u ^- @da - (add ~1970.1.1 (div (mul ~s1 a) 1.000)) + (from-unix-ms:chrono:userlib a) + ni + :: :: ++du:dejs:format + ++ du :: second date + %+ cu + |= a=@u ^- @da + (from-unix:chrono:userlib a) ni :: :: ++mu:dejs:format ++ mu :: true unit @@ -3580,7 +3589,7 @@ ++ di :: millisecond date %+ cu |= a=@u ^- @da - (add ~1970.1.1 (div (mul ~s1 a) 1.000)) + (from-unix-ms:chrono:userlib a) ni :: ++ mu :: true unit @@ -5408,13 +5417,20 @@ :: :::: ++ chrono ^? |% - :: +from-unix: unix timestamp to @da + :: +from-unix: unix seconds to @da :: ++ from-unix |= timestamp=@ud ^- @da %+ add ~1970.1.1 (mul timestamp ~s1) + :: +from-unix-ms: unix milliseconds to @da + :: + ++ from-unix-ms + |= timestamp=@ud + ^- @da + %+ add ~1970.1.1 + (div (mul ~s1 timestamp) 1.000) :: :: ++dawn:chrono: ++ dawn :: Jan 1 weekday |= yer=@ud @@ -5533,9 +5549,13 @@ ++ dd :: two digits (bass 10 (stun 2^2 dit)) -- :: + :: :: ++unm:chrono:userlib + ++ unm :: Urbit to Unix ms + |= a=@da + (div (mul (sub a ~1970.1.1) 1.000) ~s1) :: :: ++unt:chrono:userlib ++ unt :: Urbit to Unix time - |= a=@ + |= a=@da (div (sub a ~1970.1.1) ~s1) :: :: ++yu:chrono:userlib ++ yu :: UTC format constants diff --git a/pkg/arvo/tests/sys/zuse/format.hoon b/pkg/arvo/tests/sys/zuse/format.hoon index 25d1918e99..8719ab241e 100644 --- a/pkg/arvo/tests/sys/zuse/format.hoon +++ b/pkg/arvo/tests/sys/zuse/format.hoon @@ -170,11 +170,15 @@ %+ expect-eq !> [%o (molt props)] !> (pairs props) + :: sect - stored as integer number of seconds since the unix epoch + %+ expect-eq + !> [%n '1'] + !> (sect ~1970.1.1..0.0.1) :: time - stored as integer number of milliseconds since the unix epoch :: %+ expect-eq - !> [%n '1000'] - !> (time ~1970.1.1..0.0.1) + !> [%n '1000'] + !> (time ~1970.1.1..0.0.1) :: ship - store ship identity as a string :: %+ expect-eq From 92a0b5eadf3628882bbd49ff43b21248ecf6db89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C5=8Dshin?= Date: Tue, 23 Feb 2021 00:02:56 +0000 Subject: [PATCH 024/103] zuse: simplify +di/du:dejs:format --- pkg/arvo/sys/zuse.hoon | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/pkg/arvo/sys/zuse.hoon b/pkg/arvo/sys/zuse.hoon index 0ae410df48..5c84367904 100644 --- a/pkg/arvo/sys/zuse.hoon +++ b/pkg/arvo/sys/zuse.hoon @@ -3368,16 +3368,10 @@ (poq (wit jon)) :: :: ++di:dejs:format ++ di :: millisecond date - %+ cu - |= a=@u ^- @da - (from-unix-ms:chrono:userlib a) - ni + (cu from-unix-ms:chrono:userlib ni) :: :: ++du:dejs:format ++ du :: second date - %+ cu - |= a=@u ^- @da - (from-unix:chrono:userlib a) - ni + (cu from-unix:chrono:userlib ni) :: :: ++mu:dejs:format ++ mu :: true unit |* wit=fist @@ -3587,10 +3581,7 @@ (bind (stud:chrono:userlib p.jon) |=(a=date (year a))) :: ++ di :: millisecond date - %+ cu - |= a=@u ^- @da - (from-unix-ms:chrono:userlib a) - ni + (cu from-unix-ms:chrono:userlib ni) :: ++ mu :: true unit |* wit=fist From 1a0d26f03b075c4096ae0101ab35664ba3a211f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C5=8Dshin?= Date: Tue, 23 Feb 2021 20:44:18 +0000 Subject: [PATCH 025/103] pill: solid --- bin/solid.pill | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/solid.pill b/bin/solid.pill index b218f38de1..0a1df699a9 100644 --- a/bin/solid.pill +++ b/bin/solid.pill @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c2a0dcf375880feafb317ad440bbf042ba2e4c7d15ad97e37b7b98197b7c7488 -size 8787841 +oid sha256:65ad594ab389394b97fcd8bb58f25b88ec283c4142983858c1a2c3b01e14215c +size 8841004 From 08791901b7da23bc30922ab87dc0916cdfb1d9c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C5=8Dshin?= Date: Tue, 16 Feb 2021 21:49:11 +0000 Subject: [PATCH 026/103] sh/poke-gcp-account-json: poke gcp storage values To support GCP storage, we want to poke entries from a service account JSON file into settings-store. This script does that. --- sh/poke-gcp-account-json | 53 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100755 sh/poke-gcp-account-json diff --git a/sh/poke-gcp-account-json b/sh/poke-gcp-account-json new file mode 100755 index 0000000000..3c5c24fa05 --- /dev/null +++ b/sh/poke-gcp-account-json @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 + +import json +import re +import subprocess +import sys + + +def herb_poke_gcp_setting(pier, key, val): + """ + Poke a value into settings-store under the %gcp-store bucket. + + This does not sanitize or check its inputs. Please make sure they are + correct before calling this function. + + :pier: Pier of the ship to poke. + :key: Key to poke. Must be a @tas (i.e. include the '%'). + :val: Value to poke. Must be a @t. (will be passed through crude_t.) + """ + print('herb_poke ' + key) + # XXX use +same because herb's cell parser is cursed. + poke_arg = "(same %put-entry %gcp-store {} %s {})".format( + key, crude_t(val)) + return subprocess.run(['herb', pier, '-p', 'settings-store', '-d', + poke_arg, '-m', 'settings-event'], + check=True) + +def crude_t(pin): + """ + Very crude, bad, dangerous, and evil @t transform. + + Puts single quotes around the string. Escapes instances of single quote and + backslash within the string, and turns newlines into \0a. + """ + replaces = [(r'\\', r'\\\\'), ("'", r"\\'"), ("\n", r'\\0a')] + for pattern, replace in replaces: + pin = re.sub(pattern, replace, pin, flags=re.MULTILINE) + return "'{}'".format(pin) + +def read_gcp_json(keyfile): + with open(keyfile, 'r') as f: + return json.loads(f.read()) + +def main(): + pier, keyfile = sys.argv[1:] + obj = read_gcp_json(keyfile) + herb_poke_gcp_setting(pier, '%token-uri', obj['token_uri']) + herb_poke_gcp_setting(pier, '%client-email', obj['client_email']) + herb_poke_gcp_setting(pier, '%private-key-id', obj['private_key_id']) + herb_poke_gcp_setting(pier, '%private-key', obj['private_key']) + +if __name__ == '__main__': + main() From ac40b5cf243dafdb99fec58404454c03fd3b85ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C5=8Dshin?= Date: Sat, 13 Feb 2021 02:47:29 +0000 Subject: [PATCH 027/103] -get-gcp-jwt: produce a JWT for use in GCP Storage --- pkg/arvo/ted/get-gcp-jwt.hoon | 134 ++++++++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 pkg/arvo/ted/get-gcp-jwt.hoon diff --git a/pkg/arvo/ted/get-gcp-jwt.hoon b/pkg/arvo/ted/get-gcp-jwt.hoon new file mode 100644 index 0000000000..82fd26accb --- /dev/null +++ b/pkg/arvo/ted/get-gcp-jwt.hoon @@ -0,0 +1,134 @@ +:: Thread that gets a JWT for use with Google Storage. +:: +:: This thread expects settings-store to contain relevant fields from a GCP +:: service account JSON file, e.g. as poked by urbit/sh/poke-gcp-account-json. +:: Specifically, it depends on the `token_uri`, `client_email`, +:: `private_key_id`, and `private_key` fields. If these fields are not in +:: settings-store at the time the thread is run, it will fail. +:: +:: The thread works by first constructing a self-signed JWT using the fields +:: in settings-store. Then, it sends this JWT to the specified token URI +:: (usually https://oauth2.googleapis.com/token), which gives us a JWT signed +:: by Google. This token can then be used as a bearer token for requests to +:: Google Storage. +:: +:: The returned token has an expiration time of 60 minutes after the time at +:: which the thread was called. +:: +:: +/- spider, settings +/+ jose, pkcs, strandio +=, strand=strand:spider +=, format +=, jose +=, pkcs +^- thread:spider +|^ +|= * +=/ m (strand ,vase) +^- form:m +;< =bowl:spider bind:m get-bowl:strandio +;< iss=@t bind:m (read-setting %client-email) +;< =key:rsa bind:m read-private-key +;< kid=@t bind:m (read-setting %private-key-id) +;< aud=@t bind:m (read-setting %token-uri) +=/ sot=@t + %: self-jwt + key kid iss + 'https://www.googleapis.com/auth/devstorage.editor' + aud now.bowl + == +;< jot=@t bind:m (sign-jwt sot aud) +(pure:m !>(jot)) +:: +++ read-setting + |= key=term + =/ m (strand @t) ^- form:m + ;< has=? bind:m + (scry:strandio ? /gx/settings-store/has-entry/gcp-store/[key]/noun) + ?. has + (strand-fail:strandio %no-setting key ~) + ;< =data:settings bind:m + %+ scry:strandio + data:settings + /gx/settings-store/entry/gcp-store/[key]/settings-data + ?> ?=([%entry %s @] data) + (pure:m p.val.data) +:: +++ read-private-key + =/ m (strand ,key:rsa) ^- form:m + ;< dat=@t bind:m (read-setting %private-key) + %- pure:m + %. dat + ;: cork + to-wain:format + ring:de:pem:pkcs8 + need + == +:: construct and return a self-signed JWT issued now, expiring in ~h1. +:: TODO: maybe move this into lib/jose/hoon +:: +++ self-jwt + |= [=key:rsa kid=@t iss=@t scope=@t aud=@t iat=@da] + ^- @t + =/ job=json + %^ sign:jws key + :: the JWT's "header" + %: pairs:enjs + alg+s+'RS256' + typ+s+'JWT' + kid+s+kid + ~ + == + :: the JWT's "payload" + %: pairs:enjs + iss+s+iss + sub+s+iss :: per g.co, use iss for sub + scope+s+scope + aud+s+aud + iat+(sect:enjs iat) + exp+(sect:enjs (add iat ~h1)) + ~ + == + ?> ?=([%o *] job) + =* mep p.job + =+ :~ pod=(sa:dejs (~(got by mep) 'protected')) + pad=(sa:dejs (~(got by mep) 'payload')) + sig=(sa:dejs (~(got by mep) 'signature')) + == + %- crip :: XX + :(weld pod "." pad "." sig) +:: RPC to get a signed JWT. Probably only works with Google. +:: Described at: +:: https://developers.google.com/identity/protocols/oauth2/service-account +:: +++ sign-jwt + |= [jot=@t url=@t] + =/ m (strand @t) ^- form:m + ;< ~ bind:m + %: send-request:strandio + method=%'POST' + url=url + header-list=['Content-Type'^'application/json' ~] + ^= body + %- some %- as-octt:mimes:html + %- en-json:html + %: pairs:enjs + ['grant_type' s+'urn:ietf:params:oauth:grant-type:jwt-bearer'] + assertion+s+jot + ~ + == + == + ;< rep=client-response:iris bind:m + take-client-response:strandio + ?> ?=(%finished -.rep) + ?~ full-file.rep + (strand-fail:strandio %no-response ~) + =/ body=@t q.data.u.full-file.rep + =/ jon=(unit json) (de-json:html body) + ?~ jon + (strand-fail:strandio %bad-body ~[body]) + ?. ?=([%o [[%'id_token' %s @] ~ ~]] +.jon) + (strand-fail:strandio %bad-json ~[body]) + (pure:m p.q.n.p.u.jon) +-- From 860343e3d4ab231aadbf6a45083b5b5b286025f8 Mon Sep 17 00:00:00 2001 From: J Date: Tue, 23 Feb 2021 19:41:18 +0000 Subject: [PATCH 028/103] -get-gcp-jwt: clean up references, use +ot Moves =, closer to call sites so it's clearer what's coming from where. Also uses +ot, allowing a less horrifying +sign-jwt. This also seems to not jump back and forth between tapes and cords as much, for what that's worth. --- pkg/arvo/ted/get-gcp-jwt.hoon | 41 ++++++++++++++++------------------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/pkg/arvo/ted/get-gcp-jwt.hoon b/pkg/arvo/ted/get-gcp-jwt.hoon index 82fd26accb..9fe3664da4 100644 --- a/pkg/arvo/ted/get-gcp-jwt.hoon +++ b/pkg/arvo/ted/get-gcp-jwt.hoon @@ -17,11 +17,9 @@ :: :: /- spider, settings -/+ jose, pkcs, strandio +/+ jose, pkcs, primitive-rsa, strandio =, strand=strand:spider -=, format -=, jose -=, pkcs +=, rsa=primitive-rsa ^- thread:spider |^ |= * @@ -62,7 +60,7 @@ %. dat ;: cork to-wain:format - ring:de:pem:pkcs8 + ring:de:pem:pkcs8:pkcs need == :: construct and return a self-signed JWT issued now, expiring in ~h1. @@ -72,32 +70,29 @@ |= [=key:rsa kid=@t iss=@t scope=@t aud=@t iat=@da] ^- @t =/ job=json - %^ sign:jws key + =, enjs:format + %^ sign:jws:jose key :: the JWT's "header" - %: pairs:enjs + %: pairs alg+s+'RS256' typ+s+'JWT' kid+s+kid ~ == :: the JWT's "payload" - %: pairs:enjs + %: pairs iss+s+iss sub+s+iss :: per g.co, use iss for sub scope+s+scope aud+s+aud - iat+(sect:enjs iat) - exp+(sect:enjs (add iat ~h1)) + iat+(sect iat) + exp+(sect (add iat ~h1)) ~ == - ?> ?=([%o *] job) - =* mep p.job - =+ :~ pod=(sa:dejs (~(got by mep) 'protected')) - pad=(sa:dejs (~(got by mep) 'payload')) - sig=(sa:dejs (~(got by mep) 'signature')) - == - %- crip :: XX - :(weld pod "." pad "." sig) + =/ [pod=@t pad=@t sig=@t] + =, dejs:format + ((ot 'protected'^so 'payload'^so 'signature'^so ~) job) + (rap 3 (join '.' `(list @t)`~[pod pad sig])) :: RPC to get a signed JWT. Probably only works with Google. :: Described at: :: https://developers.google.com/identity/protocols/oauth2/service-account @@ -113,7 +108,7 @@ ^= body %- some %- as-octt:mimes:html %- en-json:html - %: pairs:enjs + %: pairs:enjs:format ['grant_type' s+'urn:ietf:params:oauth:grant-type:jwt-bearer'] assertion+s+jot ~ @@ -128,7 +123,9 @@ =/ jon=(unit json) (de-json:html body) ?~ jon (strand-fail:strandio %bad-body ~[body]) - ?. ?=([%o [[%'id_token' %s @] ~ ~]] +.jon) - (strand-fail:strandio %bad-json ~[body]) - (pure:m p.q.n.p.u.jon) + =* job u.jon + %- pure:m + =, dejs:format + %- (ot 'id_token'^so ~) + job -- From d978e6a5510f991d08be1656084a650dfc324361 Mon Sep 17 00:00:00 2001 From: J Date: Tue, 23 Feb 2021 22:18:59 +0000 Subject: [PATCH 029/103] -get-gcp-jwt: produces an access token The correct scope to ask for is not devstorage.editor, but cloud-platform. This will, rather than returning a signed JWT from Google that looks like it works but doesn't, return a JSON object containing an access token and expiration time. --- pkg/arvo/ted/get-gcp-jwt.hoon | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/pkg/arvo/ted/get-gcp-jwt.hoon b/pkg/arvo/ted/get-gcp-jwt.hoon index 9fe3664da4..2cce5a50cd 100644 --- a/pkg/arvo/ted/get-gcp-jwt.hoon +++ b/pkg/arvo/ted/get-gcp-jwt.hoon @@ -33,11 +33,11 @@ =/ sot=@t %: self-jwt key kid iss - 'https://www.googleapis.com/auth/devstorage.editor' + 'https://www.googleapis.com/auth/cloud-platform' aud now.bowl == -;< jot=@t bind:m (sign-jwt sot aud) -(pure:m !>(jot)) +;< p=[tok=@t exp=@da] bind:m (sign-jwt sot aud) +(pure:m !>(p)) :: ++ read-setting |= key=term @@ -99,7 +99,7 @@ :: ++ sign-jwt |= [jot=@t url=@t] - =/ m (strand @t) ^- form:m + =/ m (strand ,[@t @da]) ^- form:m ;< ~ bind:m %: send-request:strandio method=%'POST' @@ -124,8 +124,11 @@ ?~ jon (strand-fail:strandio %bad-body ~[body]) =* job u.jon - %- pure:m + ~| job =, dejs:format - %- (ot 'id_token'^so ~) - job + =/ [typ=@t exp=@da tok=@t] + ((ot 'token_type'^so 'expires_in'^du 'access_token'^so ~) job) + ?> =('Bearer' typ) + %- pure:m + [tok exp] -- From b0bb659f121cb229fe10677048b03e88783e56ce Mon Sep 17 00:00:00 2001 From: J Date: Tue, 23 Feb 2021 22:34:48 +0000 Subject: [PATCH 030/103] -get-gcp-jwt: @dr expires_in, cleanup The expiry field we get from Google is not a timestamp, but a count of seconds after which the token is invalid (hence, 'expires_in'). We represent this as a @dr by multiplying the integer we get from Google by ~1s. (Perhaps it will wind up being easier on the Landscape side to consume the raw integer, but we may as well start off more type-correct.) Also cleans up some names now that I'm less confused about how the token API works, and makes some syntax more vertical. --- pkg/arvo/ted/get-gcp-jwt.hoon | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/pkg/arvo/ted/get-gcp-jwt.hoon b/pkg/arvo/ted/get-gcp-jwt.hoon index 2cce5a50cd..fb58a16253 100644 --- a/pkg/arvo/ted/get-gcp-jwt.hoon +++ b/pkg/arvo/ted/get-gcp-jwt.hoon @@ -31,12 +31,12 @@ ;< kid=@t bind:m (read-setting %private-key-id) ;< aud=@t bind:m (read-setting %token-uri) =/ sot=@t - %: self-jwt + %: make-jwt key kid iss 'https://www.googleapis.com/auth/cloud-platform' aud now.bowl == -;< p=[tok=@t exp=@da] bind:m (sign-jwt sot aud) +;< p=[tok=@t exp=@dr] bind:m (get-access-token sot aud) (pure:m !>(p)) :: ++ read-setting @@ -66,7 +66,7 @@ :: construct and return a self-signed JWT issued now, expiring in ~h1. :: TODO: maybe move this into lib/jose/hoon :: -++ self-jwt +++ make-jwt |= [=key:rsa kid=@t iss=@t scope=@t aud=@t iat=@da] ^- @t =/ job=json @@ -93,13 +93,13 @@ =, dejs:format ((ot 'protected'^so 'payload'^so 'signature'^so ~) job) (rap 3 (join '.' `(list @t)`~[pod pad sig])) -:: RPC to get a signed JWT. Probably only works with Google. +:: RPC to get an access token. Probably only works with Google. :: Described at: :: https://developers.google.com/identity/protocols/oauth2/service-account :: -++ sign-jwt +++ get-access-token |= [jot=@t url=@t] - =/ m (strand ,[@t @da]) ^- form:m + =/ m (strand ,[@t @dr]) ^- form:m ;< ~ bind:m %: send-request:strandio method=%'POST' @@ -109,7 +109,8 @@ %- some %- as-octt:mimes:html %- en-json:html %: pairs:enjs:format - ['grant_type' s+'urn:ietf:params:oauth:grant-type:jwt-bearer'] + :- 'grant_type' + s+'urn:ietf:params:oauth:grant-type:jwt-bearer' assertion+s+jot ~ == @@ -126,8 +127,14 @@ =* job u.jon ~| job =, dejs:format - =/ [typ=@t exp=@da tok=@t] - ((ot 'token_type'^so 'expires_in'^du 'access_token'^so ~) job) + =/ [typ=@t exp=@dr tok=@t] + %. job + %: ot + 'token_type'^so + 'expires_in'^(cu |=(a=@ (mul a ~s1)) ni) + 'access_token'^so + ~ + == ?> =('Bearer' typ) %- pure:m [tok exp] From 9f43fef85d1c9637f9d53543077c8cac0d5997a9 Mon Sep 17 00:00:00 2001 From: J Date: Tue, 23 Feb 2021 22:40:05 +0000 Subject: [PATCH 031/103] -get-gcp-jwt: returns time of expiry Couldn't bring myself to pass in a relative date without a base, so we instead pass now into get-access-token, and return the time of expiry. --- pkg/arvo/ted/get-gcp-jwt.hoon | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/arvo/ted/get-gcp-jwt.hoon b/pkg/arvo/ted/get-gcp-jwt.hoon index fb58a16253..600ed9a192 100644 --- a/pkg/arvo/ted/get-gcp-jwt.hoon +++ b/pkg/arvo/ted/get-gcp-jwt.hoon @@ -36,7 +36,8 @@ 'https://www.googleapis.com/auth/cloud-platform' aud now.bowl == -;< p=[tok=@t exp=@dr] bind:m (get-access-token sot aud) +;< p=[tok=@t exp=@da] bind:m + (get-access-token sot aud now.bowl) (pure:m !>(p)) :: ++ read-setting @@ -98,8 +99,8 @@ :: https://developers.google.com/identity/protocols/oauth2/service-account :: ++ get-access-token - |= [jot=@t url=@t] - =/ m (strand ,[@t @dr]) ^- form:m + |= [jot=@t url=@t now=@da] + =/ m (strand ,[@t @da]) ^- form:m ;< ~ bind:m %: send-request:strandio method=%'POST' @@ -137,5 +138,5 @@ == ?> =('Bearer' typ) %- pure:m - [tok exp] + [tok (add exp now)] -- From 61d8030ec97d6c1f01c17ed82a55bf84e519f694 Mon Sep 17 00:00:00 2001 From: J Date: Tue, 23 Feb 2021 22:53:42 +0000 Subject: [PATCH 032/103] -get-gcp-jwt: documentation, clearer return faces --- pkg/arvo/ted/get-gcp-jwt.hoon | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/pkg/arvo/ted/get-gcp-jwt.hoon b/pkg/arvo/ted/get-gcp-jwt.hoon index 600ed9a192..51453c062f 100644 --- a/pkg/arvo/ted/get-gcp-jwt.hoon +++ b/pkg/arvo/ted/get-gcp-jwt.hoon @@ -1,19 +1,20 @@ -:: Thread that gets a JWT for use with Google Storage. +:: Gets a Google Storage access token. :: -:: This thread expects settings-store to contain relevant fields from a GCP -:: service account JSON file, e.g. as poked by urbit/sh/poke-gcp-account-json. +:: This thread produces a pair of [access-token expires-at], where +:: access-token is a @t that can be used as a bearer token to talk to the +:: GCP Storage API on behalf of some service account, and expires-at is +:: a @da after which the token will stop working and need to be refreshed. +:: +:: It expects settings-store to contain relevant fields from a GCP service +:: account JSON file, generally as poked by sh/poke-gcp-account-json. :: Specifically, it depends on the `token_uri`, `client_email`, -:: `private_key_id`, and `private_key` fields. If these fields are not in -:: settings-store at the time the thread is run, it will fail. +:: `private_key_id`, and `private_key` fields. If these fields are not +:: in settings-store at the time the thread is run, it will fail. :: -:: The thread works by first constructing a self-signed JWT using the fields -:: in settings-store. Then, it sends this JWT to the specified token URI -:: (usually https://oauth2.googleapis.com/token), which gives us a JWT signed -:: by Google. This token can then be used as a bearer token for requests to -:: Google Storage. -:: -:: The returned token has an expiration time of 60 minutes after the time at -:: which the thread was called. +:: The thread works by first constructing a self-signed JWT using the +:: fields in settings-store. Then, it sends this JWT to the specified +:: token URI (usually https://oauth2.googleapis.com/token), which responds +:: with a bearer token and expiry. :: :: /- spider, settings @@ -36,7 +37,7 @@ 'https://www.googleapis.com/auth/cloud-platform' aud now.bowl == -;< p=[tok=@t exp=@da] bind:m +;< p=[access-token=@t expires-at=@da] bind:m (get-access-token sot aud now.bowl) (pure:m !>(p)) :: From aa93ac49c156a5ada0efe20617ca1706bc0e5f78 Mon Sep 17 00:00:00 2001 From: J Date: Tue, 23 Feb 2021 22:57:12 +0000 Subject: [PATCH 033/103] -get-gcp-jwt: sot -> jot, moar vertical Since we only have one JWT (the thing we produce is an "access token", not a JWT), we can just call it jot. --- pkg/arvo/ted/get-gcp-jwt.hoon | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/arvo/ted/get-gcp-jwt.hoon b/pkg/arvo/ted/get-gcp-jwt.hoon index 51453c062f..ac3bcc9bbd 100644 --- a/pkg/arvo/ted/get-gcp-jwt.hoon +++ b/pkg/arvo/ted/get-gcp-jwt.hoon @@ -31,21 +31,22 @@ ;< =key:rsa bind:m read-private-key ;< kid=@t bind:m (read-setting %private-key-id) ;< aud=@t bind:m (read-setting %token-uri) -=/ sot=@t +=/ jot=@t %: make-jwt key kid iss 'https://www.googleapis.com/auth/cloud-platform' aud now.bowl == ;< p=[access-token=@t expires-at=@da] bind:m - (get-access-token sot aud now.bowl) + (get-access-token jot aud now.bowl) (pure:m !>(p)) :: ++ read-setting |= key=term =/ m (strand @t) ^- form:m ;< has=? bind:m - (scry:strandio ? /gx/settings-store/has-entry/gcp-store/[key]/noun) + %+ scry:strandio ? + /gx/settings-store/has-entry/gcp-store/[key]/noun ?. has (strand-fail:strandio %no-setting key ~) ;< =data:settings bind:m From 6a07130f0df77b73bbe1181d041e9c8974085744 Mon Sep 17 00:00:00 2001 From: J Date: Tue, 23 Feb 2021 23:12:48 +0000 Subject: [PATCH 034/103] -get-gcp-jwt: correct scope, clean up call It turns out 'devstorage.read_write' also gives us an access token instead of a JWT, and is probably more the thing that we want. Took the opportunity to make scope a macro to clean up the make-jwt call site. --- pkg/arvo/ted/get-gcp-jwt.hoon | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/pkg/arvo/ted/get-gcp-jwt.hoon b/pkg/arvo/ted/get-gcp-jwt.hoon index ac3bcc9bbd..339f12d6fc 100644 --- a/pkg/arvo/ted/get-gcp-jwt.hoon +++ b/pkg/arvo/ted/get-gcp-jwt.hoon @@ -31,12 +31,9 @@ ;< =key:rsa bind:m read-private-key ;< kid=@t bind:m (read-setting %private-key-id) ;< aud=@t bind:m (read-setting %token-uri) +=* scope 'https://www.googleapis.com/auth/devstorage.read_write' =/ jot=@t - %: make-jwt - key kid iss - 'https://www.googleapis.com/auth/cloud-platform' - aud now.bowl - == + (make-jwt key kid iss scope aud now.bowl) ;< p=[access-token=@t expires-at=@da] bind:m (get-access-token jot aud now.bowl) (pure:m !>(p)) From a8113182a80b1c82366797d99bc932a002247a9d Mon Sep 17 00:00:00 2001 From: J Date: Tue, 23 Feb 2021 23:48:50 +0000 Subject: [PATCH 035/103] -get-gcp-jwt: vertical --- pkg/arvo/ted/get-gcp-jwt.hoon | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/arvo/ted/get-gcp-jwt.hoon b/pkg/arvo/ted/get-gcp-jwt.hoon index 339f12d6fc..17ed1db9b2 100644 --- a/pkg/arvo/ted/get-gcp-jwt.hoon +++ b/pkg/arvo/ted/get-gcp-jwt.hoon @@ -31,7 +31,8 @@ ;< =key:rsa bind:m read-private-key ;< kid=@t bind:m (read-setting %private-key-id) ;< aud=@t bind:m (read-setting %token-uri) -=* scope 'https://www.googleapis.com/auth/devstorage.read_write' +=* scope + 'https://www.googleapis.com/auth/devstorage.read_write' =/ jot=@t (make-jwt key kid iss scope aud now.bowl) ;< p=[access-token=@t expires-at=@da] bind:m From d94f35aa5c16d09580f37cc91e6d8260cd527638 Mon Sep 17 00:00:00 2001 From: J Date: Wed, 24 Feb 2021 00:45:33 +0000 Subject: [PATCH 036/103] gcp: add token type Also use it in -get-gcp-jwt. --- pkg/arvo/sur/gcp.hoon | 6 ++++++ pkg/arvo/ted/get-gcp-jwt.hoon | 8 ++++---- 2 files changed, 10 insertions(+), 4 deletions(-) create mode 100644 pkg/arvo/sur/gcp.hoon diff --git a/pkg/arvo/sur/gcp.hoon b/pkg/arvo/sur/gcp.hoon new file mode 100644 index 0000000000..9a63cf0b42 --- /dev/null +++ b/pkg/arvo/sur/gcp.hoon @@ -0,0 +1,6 @@ +|% ++$ token + $: access=@t + expiry=@da + == +-- diff --git a/pkg/arvo/ted/get-gcp-jwt.hoon b/pkg/arvo/ted/get-gcp-jwt.hoon index 17ed1db9b2..da83b33622 100644 --- a/pkg/arvo/ted/get-gcp-jwt.hoon +++ b/pkg/arvo/ted/get-gcp-jwt.hoon @@ -17,7 +17,7 @@ :: with a bearer token and expiry. :: :: -/- spider, settings +/- gcp, spider, settings /+ jose, pkcs, primitive-rsa, strandio =, strand=strand:spider =, rsa=primitive-rsa @@ -35,9 +35,9 @@ 'https://www.googleapis.com/auth/devstorage.read_write' =/ jot=@t (make-jwt key kid iss scope aud now.bowl) -;< p=[access-token=@t expires-at=@da] bind:m +;< =token:gcp bind:m (get-access-token jot aud now.bowl) -(pure:m !>(p)) +(pure:m !>(token)) :: ++ read-setting |= key=term @@ -100,7 +100,7 @@ :: ++ get-access-token |= [jot=@t url=@t now=@da] - =/ m (strand ,[@t @da]) ^- form:m + =/ m (strand ,token:gcp) ^- form:m ;< ~ bind:m %: send-request:strandio method=%'POST' From 328159da676261b766b1cb2b2086a968342a254a Mon Sep 17 00:00:00 2001 From: J Date: Wed, 24 Feb 2021 00:47:12 +0000 Subject: [PATCH 037/103] -get-gcp-token: renamed from -get-gcp-jwt --- pkg/arvo/ted/{get-gcp-jwt.hoon => get-gcp-token.hoon} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pkg/arvo/ted/{get-gcp-jwt.hoon => get-gcp-token.hoon} (100%) diff --git a/pkg/arvo/ted/get-gcp-jwt.hoon b/pkg/arvo/ted/get-gcp-token.hoon similarity index 100% rename from pkg/arvo/ted/get-gcp-jwt.hoon rename to pkg/arvo/ted/get-gcp-token.hoon From d447e70b094bf29d36f00824dd2c9e0668e0d5ba Mon Sep 17 00:00:00 2001 From: Ted Blackman Date: Wed, 24 Feb 2021 12:18:50 -0500 Subject: [PATCH 038/103] clay: fix /% parser; clean up unit tests --- pkg/arvo/sys/vane/clay.hoon | 2 +- pkg/arvo/tests/sys/vane/clay.hoon | 151 +++++++++++++++++------------- 2 files changed, 86 insertions(+), 67 deletions(-) diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index fa87236a7b..0e6b44bb41 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -869,7 +869,7 @@ ;~(plug sym ;~(pfix gap fas (more fas urs:ab))) :: %+ rune cen - ;~(plug sym ;~(pfix cen sym)) + ;~(plug sym ;~(pfix gap ;~(pfix cen sym))) :: %+ rune buc ;~ (glue gap) diff --git a/pkg/arvo/tests/sys/vane/clay.hoon b/pkg/arvo/tests/sys/vane/clay.hoon index 0331073c83..3579bb342c 100644 --- a/pkg/arvo/tests/sys/vane/clay.hoon +++ b/pkg/arvo/tests/sys/vane/clay.hoon @@ -20,32 +20,48 @@ :: |% ++ test-parse-pile ^- tang + =/ src "." %+ expect-eq !> ^- pile:fusion :* ~ ~ ~ ~ ~ ~ tssg+[%dbug [/sur/foo/hoon [[1 1] [1 2]]] [%cnts ~[[%.y 1]] ~]]~ == - !> (parse-pile:(ford):fusion /sur/foo/hoon ".") + !> (parse-pile:(ford):fusion /sur/foo/hoon src) +:: +++ test-parse-fascen ^- tang + =/ src "/% moo %mime\0a." + %+ expect-eq + !> ^- pile:fusion + :* sur=~ lib=~ raw=~ + maz=[face=%moo mark=%mime]~ + caz=~ bar=~ + tssg+[%dbug [/sur/foo/hoon [[2 1] [2 2]]] [%cnts ~[[%.y 1]] ~]]~ + == + !> (parse-pile:(ford):fusion /sur/foo/hoon src) +:: +++ test-parse-fasbuc ^- tang + =/ src "/$ goo %mime %txt\0a." + %+ expect-eq + !> ^- pile:fusion + :* sur=~ lib=~ raw=~ maz=~ + caz=[face=%goo from=%mime to=%txt]~ + bar=~ + tssg+[%dbug [/sur/foo/hoon [[2 1] [2 2]]] [%cnts ~[[%.y 1]] ~]]~ + == + !> (parse-pile:(ford):fusion /sur/foo/hoon src) :: ++ test-parse-multiline-faslus ^- tang =/ src """ - :: :: :: - :::: /hoon/hood/app :: :: - :: :: :: + :: /? 310 :: zuse version + :: /- *sole + :: /+ sole :: libraries - :: XX these should really be separate apps, as - :: none of them interact with each other in - :: any fashion; however, to reduce boot-time - :: complexity and work around the current - :: non-functionality of end-to-end acknowledgments, - :: they have been bundled into :hood - :: - :: |command handlers + :: /+ hood-helm, hood-kiln, hood-drum, hood-write - :: :: :: + :: . """ %+ expect-eq @@ -58,34 +74,21 @@ [`%hood-drum %hood-drum] [`%hood-write %hood-write] == - raw=~ bar=~ maz=~ caz=~ - hoon=tssg+[p:(need q:(tall:(vang & /app/hood/hoon) [17 1] "."))]~ + raw=~ maz=~ caz=~ bar=~ + tssg+[%dbug [/sur/foo/hoon [[10 1] [10 2]]] [%cnts ~[[%.y 1]] ~]]~ == - !> (parse-pile:(ford):fusion /app/hood/hoon src) + !> (parse-pile:(ford):fusion /sur/foo/hoon src) :: ++ test-cycle ^- tang - =/ source=@t - ''' - /+ self - . - ''' - =/ =ankh:clay - :- fil=~ - %- ~(gas by *(map @tas ankh:clay)) - :~ :+ %lib fil=~ - %- ~(gas by *(map @tas ankh:clay)) - :~ :+ %self fil=~ - %- ~(gas by *(map @tas ankh:clay)) - :~ :+ %hoon fil=`[*lobe:clay hoon+!>(source)] dir=~ - == == == + =/ source=@t '/+ self\0a.' %- expect-fail |. =/ ford %: ford:fusion bud - ankh + *ankh:clay deletes=~ - changes=~ + changes=(my [/lib/self/hoon &+hoon+source]~) file-store=~ *ford-cache:fusion == @@ -163,20 +166,52 @@ (slam res !>('')) !> `mime`[/text/html 13 ''] :: -++ test-gen-hello ^- tang - =/ =ankh:clay - :- fil=~ - %- ~(gas by *(map @tas ankh:clay)) - :~ :+ %gen fil=~ - %- ~(gas by *(map @tas ankh:clay)) - :~ :+ %hello fil=~ - %- ~(gas by *(map @tas ankh:clay)) - :~ :+ %hoon fil=`[*lobe:clay hoon+!>(gen-hello)] dir=~ - == == == +++ test-fascen ^- tang + =/ changes + %- my + :~ [/mar/mime/hoon &+hoon+mar-mime] + [/lib/foo/hoon &+hoon+'/% moo %mime\0abunt:moo'] + == =/ ford %: ford:fusion bud - ankh + *ankh:clay + deletes=~ + changes + file-store=~ + *ford-cache:fusion + == + =/ [res=vase nub=state:ford:fusion] (build-file:ford /lib/foo/hoon) + %+ expect-eq + res + !> *mime +:: +++ test-fasbuc ^- tang + =/ changes + %- my + :~ [/mar/mime/hoon &+hoon+mar-mime] + [/mar/html/hoon &+hoon+mar-html] + [/lib/foo/hoon &+hoon+'/$ foo %mime %html\0a*foo'] + == + =/ ford + %: ford:fusion + bud + *ankh:clay + deletes=~ + changes + file-store=~ + *ford-cache:fusion + == + =/ [res=vase nub=state:ford:fusion] (build-file:ford /lib/foo/hoon) + %+ expect-eq + res + !> '' +:: +++ test-gen-hello ^- tang + =/ ford + %: ford:fusion + bud + *ankh:clay deletes=~ changes=(my [/gen/hello/hoon &+hoon+gen-hello]~) file-store=~ @@ -194,33 +229,17 @@ == :: ++ test-lib-strandio ^- tang - =/ =ankh:clay - :- fil=~ - %- ~(gas by *(map @tas ankh:clay)) - :~ :+ %lib fil=~ - %- ~(gas by *(map @tas ankh:clay)) - :~ :+ %strandio fil=~ - %- ~(gas by *(map @tas ankh:clay)) - :~ :+ %hoon fil=`[*lobe:clay hoon+!>(lib-strandio)] dir=~ - == - :: - :+ %strand fil=~ - %- ~(gas by *(map @tas ankh:clay)) - :~ :+ %hoon fil=`[*lobe:clay hoon+!>(lib-strand)] dir=~ - == == - :: - :+ %sur fil=~ - %- ~(gas by *(map @tas ankh:clay)) - :~ :+ %spider fil=~ - %- ~(gas by *(map @tas ankh:clay)) - :~ :+ %hoon fil=`[*lobe:clay hoon+!>(sur-spider)] dir=~ - == == == =/ ford %: ford:fusion bud - ankh + *ankh:clay deletes=~ - changes=~ + ^= changes + %- my + :~ [/lib/strand/hoon &+hoon+lib-strand] + [/lib/strandio/hoon &+hoon+lib-strandio] + [/sur/spider/hoon &+hoon+sur-spider] + == file-store=~ *ford-cache:fusion == From 19bf2274286b850e2e158ea660293a7d807e8611 Mon Sep 17 00:00:00 2001 From: Ted Blackman Date: Wed, 24 Feb 2021 12:44:08 -0500 Subject: [PATCH 039/103] clay: remove debug artifacts --- pkg/arvo/sys/vane/clay.hoon | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index 0e6b44bb41..c09f1fa5f1 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -459,7 +459,7 @@ [+<- +<+>-] :: cache.state :: ++ ford - ::!. :: TODO: reinstate + !. => |% +$ build $% [%file =path] @@ -3458,7 +3458,7 @@ :: ford :: ++ read-a - ::!. TODO reinstate + !. |= [=aeon =path] ^- [(unit (unit (each cage lobe))) ford-cache] ?. =(aeon let.dom) @@ -3480,7 +3480,7 @@ :_(fod.dom [~ ~ %& %vase !>(vase)]) :: ++ read-b - ::!. TODO reinstate + !. |= [=aeon =path] ^- [(unit (unit (each cage lobe))) ford-cache] ?. =(aeon let.dom) @@ -3496,7 +3496,7 @@ :_(fod.dom [~ ~ %& %dais !>(dais)]) :: ++ read-c - ::!. TODO reinstate + !. |= [=aeon =path] ^- [(unit (unit (each cage lobe))) ford-cache] ?. =(aeon let.dom) @@ -3854,7 +3854,7 @@ [~ fod] :: virtualize to catch and produce deterministic failures :: - ::!. TODO reinstate + !. =- ?:(?=(%& -<) p.- ((slog p.-) [[~ ~] fod])) %- mule |. ?- care.mun From ce61d49ef9d9367d385085af24a6d299fab0a718 Mon Sep 17 00:00:00 2001 From: Ted Blackman Date: Wed, 24 Feb 2021 12:59:07 -0500 Subject: [PATCH 040/103] clay: fix |mass; some renaming --- pkg/arvo/sys/vane/clay.hoon | 73 +++++++++++++++++++------------------ 1 file changed, 38 insertions(+), 35 deletions(-) diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index c09f1fa5f1..69b3265431 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -118,7 +118,7 @@ :: Ford cache :: +$ ford-cache - $: vases=(map path [res=vase dez=(set path)]) + $: files=(map path [res=vase dez=(set path)]) naves=(map mark [res=vase dez=(set path)]) marks=(map mark [res=dais dez=(set path)]) casts=(map mars [res=vase dez=(set path)]) @@ -498,8 +498,9 @@ =? stack.nub ?=(^ stack.nub) stack.nub(i (~(uni in i.stack.nub) top)) [top stack.nub] + :: +read-file: retrieve marked, validated file contents at path :: - ++ get-value + ++ read-file |= =path ^- [cage state] ~| %error-validating^path @@ -523,9 +524,9 @@ ?< (~(has in deletes) path) ~| %file-not-found^path :_(nub (need (~(get an ankh) path))) - :: +get-nave: build a statically typed mark core + :: +build-nave: build a statically typed mark core :: - ++ get-nave + ++ build-nave |= mak=mark ^- [vase state] ~| %error-building-mark^mak @@ -547,8 +548,8 @@ ?@ q.gad =+ !<(mok=mark gad) =^ deg=vase nub $(mak mok) - =^ tub=vase nub (get-cast mak mok) - =^ but=vase nub (get-cast mok mak) + =^ tub=vase nub (build-cast mak mok) + =^ but=vase nub (build-cast mok mak) :_ nub ^- vase :: vase of nave %+ slap @@ -598,9 +599,9 @@ ++ pact |=([v=typ d=dif] (pact:~(grad cor v) d)) ++ vale noun:grab:cor -- - :: +get-mark: build a dynamically typed mark definition + :: +build-dais: build a dynamically typed mark definition :: - ++ get-mark + ++ build-dais |= mak=mark ^- [dais state] ~| %error-building-dais^mak @@ -617,7 +618,7 @@ =^ top stack.nub pop-stack =. marks.cache.nub (~(put by marks.cache.nub) mak [dais.res top]) [dais.res nub] - =^ nav=vase nub (get-nave mak) + =^ nav=vase nub (build-nave mak) :_ nub ^- dais |_ sam=vase @@ -651,9 +652,9 @@ |= =noun (slam (slap nav limb/%vale) noun/noun) -- - :: +get-cast: produce gate to convert mark .a to, statically typed + :: +build-cast: produce gate to convert mark .a to, statically typed :: - ++ get-cast + ++ build-cast |= [a=mark b=mark] ^- [vase state] ~| error-building-cast+[a b] @@ -705,15 +706,15 @@ ++ compose-casts |= [x=mark y=mark z=mark] ^- [vase state] - =^ uno=vase nub (get-cast x y) - =^ dos=vase nub (get-cast y z) + =^ uno=vase nub (build-cast x y) + =^ dos=vase nub (build-cast y z) :_ nub %+ slap (with-faces uno+uno dos+dos cork+!>(cork) ~) !,(*hoon (cork uno dos)) - :: +get-tube: produce a $tube mark conversion gate from .a to .b + :: +build-tube: produce a $tube mark conversion gate from .a to .b :: - ++ get-tube + ++ build-tube |= [a=mark b=mark] ^- [tube state] ~| error-building-tube+[a b] @@ -729,7 +730,7 @@ =^ top stack.nub pop-stack =. tubes.cache.nub (~(put by tubes.cache.nub) [a b] [tube.res top]) [tube.res nub] - =^ gat=vase nub (get-cast a b) + =^ gat=vase nub (build-cast a b) :_(nub |=(v=vase (slam gat v))) :: ++ lobe-to-page @@ -754,7 +755,7 @@ ?: =(mak p.page) (page-to-cage page) =^ [mark vax=vase] nub (page-to-cage page) - =^ =tube nub (get-tube p.page mak) + =^ =tube nub (build-tube p.page mak) :_(nub [mak (tube vax)]) :: ++ page-to-cage @@ -764,7 +765,7 @@ :_(nub [%hoon -:!>(*@t) q.page]) ?: =(%mime p.page) :_(nub [%mime !>(;;(mime q.page))]) - =^ =dais nub (get-mark p.page) + =^ =dais nub (build-dais p.page) :_(nub [p.page (vale:dais q.page)]) :: ++ cast-path @@ -772,10 +773,10 @@ ^- [cage state] =/ mok (head (flop path)) ~| error-casting-path+[path mok mak] - =^ cag=cage nub (get-value path) + =^ cag=cage nub (read-file path) ?: =(mok mak) [cag nub] - =^ =tube nub (get-tube mok mak) + =^ =tube nub (build-tube mok mak) ~| error-running-cast+[path mok mak] :_(nub [mak (tube q.cag)]) :: @@ -787,14 +788,14 @@ =+ ;;(dif=(urge cord) q.diff) =/ new=@t (of-wain:format (lurk:differ txt dif)) :_(nub [%hoon !>(new)]) - =^ dys=dais nub (get-mark p.old) - =^ syd=dais nub (get-mark p.diff) + =^ dys=dais nub (build-dais p.old) + =^ syd=dais nub (build-dais p.diff) :_(nub [p.old (~(pact dys (vale:dys q.old)) (vale:syd q.diff))]) :: ++ prelude |= =path ^- vase - =^ cag=cage nub (get-value path) + =^ cag=cage nub (read-file path) ?> =(%hoon p.cag) =/ tex=tape (trip !<(@t q.cag)) =/ =pile (parse-pile path tex) @@ -806,7 +807,7 @@ |= =path ^- [vase state] ~| %error-building^path - ?^ got=(~(get by vases.cache.nub) path) + ?^ got=(~(get by files.cache.nub) path) =? stack.nub ?=(^ stack.nub) stack.nub(i (~(uni in i.stack.nub) dez.u.got)) [res.u.got nub] @@ -814,13 +815,13 @@ ~|(cycle+file+path^stack.nub !!) =. cycle.nub (~(put in cycle.nub) file+path) =. stack.nub [(sy path ~) stack.nub] - =^ cag=cage nub (get-value path) + =^ cag=cage nub (read-file path) ?> =(%hoon p.cag) =/ tex=tape (trip !<(@t q.cag)) =/ =pile (parse-pile path tex) =^ res=vase nub (run-pile pile) =^ top stack.nub pop-stack - =. vases.cache.nub (~(put by vases.cache.nub) path [res top]) + =. files.cache.nub (~(put by files.cache.nub) path [res top]) [res nub] :: ++ run-pile @@ -932,7 +933,7 @@ |= [sut=vase maz=(list [face=term =mark])] ^- [vase state] ?~ maz [sut nub] - =^ pin=vase nub (get-nave mark.i.maz) + =^ pin=vase nub (build-nave mark.i.maz) =. p.pin [%face face.i.maz p.pin] $(sut (slop pin sut), maz t.maz) :: @@ -940,7 +941,7 @@ |= [sut=vase caz=(list [face=term =mars])] ^- [vase state] ?~ caz [sut nub] - =^ pin=vase nub (get-cast mars.i.caz) + =^ pin=vase nub (build-cast mars.i.caz) =. p.pin [%face face.i.caz p.pin] $(sut (slop pin sut), caz t.caz) :: @@ -1570,7 +1571,7 @@ %+ turn (tail (spud pux)) :: lose leading '/' |=(c=@tD `@tD`?:(=('/' c) '-' c)) :: convert '/' to '-' :: - :* ((invalidate path vase) vases.ford-cache invalid) + :* ((invalidate path vase) files.ford-cache invalid) ((invalidate mark vase) naves.ford-cache invalid) ((invalidate mark dais) marks.ford-cache invalid) ((invalidate mars vase) casts.ford-cache invalid) @@ -1692,7 +1693,7 @@ =^ cage ford-cache.ford-args :: ~> %slog.[0 leaf+"clay: validating {(spud path.i.cans)}"] %- wrap:fusion - (get-value:(ford:fusion ford-args) path.i.cans) + (read-file:(ford:fusion ford-args) path.i.cans) =/ =lobe ?- -.change.i.cans %| p.change.i.cans @@ -2280,7 +2281,7 @@ ^- dais =^ =dais fod.dom %- wrap:fusion - (get-mark:(ford:fusion static-ford-args) mark) + (build-dais:(ford:fusion static-ford-args) mark) dais :: :: Diff two files on bob-desk @@ -3463,7 +3464,7 @@ ^- [(unit (unit (each cage lobe))) ford-cache] ?. =(aeon let.dom) [~ fod.dom] - =/ cached=(unit [=vase *]) (~(get by vases.fod.dom) path) + =/ cached=(unit [=vase *]) (~(get by files.fod.dom) path) ?^ cached :_(fod.dom [~ ~ %& %vase !>(vase.u.cached)]) =/ x (read-x aeon path) @@ -3492,7 +3493,7 @@ :_(fod.dom [~ ~ %& %dais !>(dais.u.cached)]) =^ =dais fod.dom %- wrap:fusion - (get-mark:(ford:fusion static-ford-args) i.path) + (build-dais:(ford:fusion static-ford-args) i.path) :_(fod.dom [~ ~ %& %dais !>(dais)]) :: ++ read-c @@ -3508,7 +3509,7 @@ :_(fod.dom [~ ~ %& %tube !>(tube.u.cached)]) =^ =tube fod.dom %- wrap:fusion - (get-tube:(ford:fusion static-ford-args) [i i.t]:path) + (build-tube:(ford:fusion static-ford-args) [i i.t]:path) :_(fod.dom [~ ~ %& %tube !>(tube)]) :: :: Gets the permissions that apply to a particular node. @@ -4444,9 +4445,11 @@ :+ desk %| :~ ankh+&+ank.dom.dojo mime+&+mim.dom.dojo - ford-vases+&+vases.fod.dom.dojo + ford-files+&+files.fod.dom.dojo + ford-naves+&+naves.fod.dom.dojo ford-marks+&+marks.fod.dom.dojo ford-casts+&+casts.fod.dom.dojo + ford-tubes+&+tubes.fod.dom.dojo == :~ domestic+|+domestic foreign+&+hoy.ruf From 68e1d92fb4099d1d6513d027a4751876d718e173 Mon Sep 17 00:00:00 2001 From: Ted Blackman Date: Wed, 24 Feb 2021 13:07:27 -0500 Subject: [PATCH 041/103] clay: fix tests --- pkg/arvo/tests/sys/vane/clay.hoon | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/arvo/tests/sys/vane/clay.hoon b/pkg/arvo/tests/sys/vane/clay.hoon index 3579bb342c..636efd3a35 100644 --- a/pkg/arvo/tests/sys/vane/clay.hoon +++ b/pkg/arvo/tests/sys/vane/clay.hoon @@ -108,7 +108,7 @@ file-store=~ *ford-cache:fusion == - =/ [res=vase nub=state:ford:fusion] (get-nave:ford %mime) + =/ [res=vase nub=state:ford:fusion] (build-nave:ford %mime) ;: weld %+ expect-eq !>(*mime) @@ -116,7 +116,7 @@ :: %+ expect-eq !> (~(gas in *(set path)) /mar/mime/hoon ~) - !> dez:(~(got by vases.cache.nub) /mar/mime/hoon) + !> dez:(~(got by files.cache.nub) /mar/mime/hoon) == :: ++ test-mar-udon ^- tang @@ -135,7 +135,7 @@ file-store=~ *ford-cache:fusion == - =/ [res=vase nub=state:ford:fusion] (get-nave:ford %udon) + =/ [res=vase nub=state:ford:fusion] (build-nave:ford %udon) ;: weld %+ expect-eq !>(*@t) @@ -143,7 +143,7 @@ :: %+ expect-eq !> (~(gas in *(set path)) /mar/udon/hoon /lib/cram/hoon ~) - !> dez:(~(got by vases.cache.nub) /mar/udon/hoon) + !> dez:(~(got by files.cache.nub) /mar/udon/hoon) == :: ++ test-cast-html-mime ^- tang @@ -161,7 +161,7 @@ file-store=~ *ford-cache:fusion == - =/ [res=vase nub=state:ford:fusion] (get-cast:ford %html %mime) + =/ [res=vase nub=state:ford:fusion] (build-cast:ford %html %mime) %+ expect-eq (slam res !>('')) !> `mime`[/text/html 13 ''] @@ -225,7 +225,7 @@ :: %+ expect-eq !> (~(gas in *(set path)) /gen/hello/hoon ~) - !> dez:(~(got by vases.cache.nub) /gen/hello/hoon) + !> dez:(~(got by files.cache.nub) /gen/hello/hoon) == :: ++ test-lib-strandio ^- tang @@ -254,7 +254,7 @@ /lib/strand/hoon /sur/spider/hoon == - !> dez:(~(got by vases.cache.nub) /lib/strandio/hoon) + !> dez:(~(got by files.cache.nub) /lib/strandio/hoon) == :: :: |utilities: helper functions for testing From 42970829669f4e1e96e2fcf62007dbf8855be2e3 Mon Sep 17 00:00:00 2001 From: Ted Blackman Date: Wed, 24 Feb 2021 13:07:45 -0500 Subject: [PATCH 042/103] language-server: fix parser --- pkg/arvo/lib/language-server/parser.hoon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/arvo/lib/language-server/parser.hoon b/pkg/arvo/lib/language-server/parser.hoon index 05667bee03..d4f5002fb6 100644 --- a/pkg/arvo/lib/language-server/parser.hoon +++ b/pkg/arvo/lib/language-server/parser.hoon @@ -24,7 +24,7 @@ ;~(plug sym ;~(pfix gap fas (more fas urs:ab))) :: %+ rune cen - ;~(plug sym ;~(pfix cen sym)) + ;~(plug sym ;~(pfix gap ;~(pfix cen sym))) :: %+ rune buc ;~ (glue gap) From 9244ae9c5f053b2a2d809a260dcfa2bfd6875267 Mon Sep 17 00:00:00 2001 From: Ted Blackman Date: Wed, 24 Feb 2021 13:07:57 -0500 Subject: [PATCH 043/103] pill: update solid --- bin/solid.pill | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/solid.pill b/bin/solid.pill index 79b355a1d3..457591efd6 100644 --- a/bin/solid.pill +++ b/bin/solid.pill @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:be826725d4d7d3b7f6b88821b151baa3837c42874400016db838abd42e6b9c01 -size 8962205 +oid sha256:7811e54092e569c9e0f59ad32747fa9e0a615c0cd279793b5a20444dd6625a1c +size 8889102 From 713954a8bb449eb75d2ce06530c4ec86c5aebae5 Mon Sep 17 00:00:00 2001 From: J Date: Wed, 24 Feb 2021 19:30:49 +0000 Subject: [PATCH 044/103] gcp: mark for tokens, token-to-json --- pkg/arvo/lib/gcp.hoon | 12 ++++++++++++ pkg/arvo/mar/gcp-token.hoon | 13 +++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 pkg/arvo/lib/gcp.hoon create mode 100644 pkg/arvo/mar/gcp-token.hoon diff --git a/pkg/arvo/lib/gcp.hoon b/pkg/arvo/lib/gcp.hoon new file mode 100644 index 0000000000..a27ddfbe17 --- /dev/null +++ b/pkg/arvo/lib/gcp.hoon @@ -0,0 +1,12 @@ +/- *gcp +|% +++ token-to-json + |= =token + ^- json + =, enjs:format + %+ frond %gcp-token + %: pairs + access+s+access.token + expiry+(time expiry.token) + == +-- diff --git a/pkg/arvo/mar/gcp-token.hoon b/pkg/arvo/mar/gcp-token.hoon new file mode 100644 index 0000000000..3816324f23 --- /dev/null +++ b/pkg/arvo/mar/gcp-token.hoon @@ -0,0 +1,13 @@ +/+ *gcp +|_ tok=token +++ grad %noun +++ grow + |% + ++ noun tok + ++ json (token-to-json tok) + -- +++ grab + |% + ++ noun token + -- +-- From 9636b889aac5316107f01b587af487be6928ec49 Mon Sep 17 00:00:00 2001 From: Ted Blackman Date: Wed, 24 Feb 2021 14:43:35 -0500 Subject: [PATCH 045/103] clay: speed up +checkout-changes --- pkg/arvo/sys/vane/clay.hoon | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index 34f072bfc6..13af4a6908 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -1686,24 +1686,26 @@ :: ++ checkout-changes |= [=ford=args:ford:fusion changes=(map path (each page lobe))] - =/ cans=(list [=path change=(each page lobe)]) ~(tap by changes) - |- ^- [(map path [=lobe =cage]) ford-cache] - ?~ cans - [~ ford-cache.ford-args] + ^- [(map path [=lobe =cage]) ford-cache] + %+ roll `(list [path (each page lobe)])`~(tap by changes) + |= $: [=path change=(each page lobe)] + [built=(map path [lobe cage]) cache=_ford-cache.ford-args] + == + ^+ [built cache] + =. ford-cache.ford-args cache =^ cage ford-cache.ford-args - :: ~> %slog.[0 leaf+"clay: validating {(spud path.i.cans)}"] + :: ~> %slog.[0 leaf/"clay: validating {(spud path)}"] %- wrap:fusion - (read-file:(ford:fusion ford-args) path.i.cans) + (read-file:(ford:fusion ford-args) path) =/ =lobe - ?- -.change.i.cans - %| p.change.i.cans + ?- -.change + %| p.change :: Don't use p.change.i.cans because that's before casting to :: the correct mark. :: %& (page-to-lobe [p q.q]:cage) == - =^ so-far ford-cache.ford-args $(cans t.cans) - [(~(put by so-far) path.i.cans lobe cage) ford-cache.ford-args] + [(~(put by built) path [lobe cage]) ford-cache.ford-args] :: :: Update ankh :: From 02dec2d0b962b71a8b889e896ce35d17a12da27b Mon Sep 17 00:00:00 2001 From: Ted Blackman Date: Wed, 24 Feb 2021 14:44:02 -0500 Subject: [PATCH 046/103] pill: update solid --- bin/solid.pill | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/solid.pill b/bin/solid.pill index 457591efd6..0dc1c438b7 100644 --- a/bin/solid.pill +++ b/bin/solid.pill @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:7811e54092e569c9e0f59ad32747fa9e0a615c0cd279793b5a20444dd6625a1c -size 8889102 +oid sha256:8e29b2d6980412e234d09b12816d3e868344ca6df4b9fe3966c2b6b9bcea042b +size 8996949 From 03e99f52c915ca7887259dec8f7594398f8db2c4 Mon Sep 17 00:00:00 2001 From: J Date: Wed, 24 Feb 2021 20:33:41 +0000 Subject: [PATCH 047/103] gcp: make expiry relative This indeed makes it easier on the Landscape side. It also makes it easier to coerce from the JSON Google gives us, so a win all around. --- pkg/arvo/lib/gcp.hoon | 7 +++++-- pkg/arvo/sur/gcp.hoon | 4 ++-- pkg/arvo/ted/get-gcp-token.hoon | 11 +++++------ 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/pkg/arvo/lib/gcp.hoon b/pkg/arvo/lib/gcp.hoon index a27ddfbe17..41163cede1 100644 --- a/pkg/arvo/lib/gcp.hoon +++ b/pkg/arvo/lib/gcp.hoon @@ -6,7 +6,10 @@ =, enjs:format %+ frond %gcp-token %: pairs - access+s+access.token - expiry+(time expiry.token) + access-key+s+access-key.token + :- %expires-in + %- numb + (div (mul 1.000 expires-in.token) ~s1) + ~ == -- diff --git a/pkg/arvo/sur/gcp.hoon b/pkg/arvo/sur/gcp.hoon index 9a63cf0b42..268ba4d1e8 100644 --- a/pkg/arvo/sur/gcp.hoon +++ b/pkg/arvo/sur/gcp.hoon @@ -1,6 +1,6 @@ |% +$ token - $: access=@t - expiry=@da + $: access-key=@t + expires-in=@dr == -- diff --git a/pkg/arvo/ted/get-gcp-token.hoon b/pkg/arvo/ted/get-gcp-token.hoon index da83b33622..3093707f18 100644 --- a/pkg/arvo/ted/get-gcp-token.hoon +++ b/pkg/arvo/ted/get-gcp-token.hoon @@ -36,7 +36,7 @@ =/ jot=@t (make-jwt key kid iss scope aud now.bowl) ;< =token:gcp bind:m - (get-access-token jot aud now.bowl) + (get-access-token jot aud) (pure:m !>(token)) :: ++ read-setting @@ -99,7 +99,7 @@ :: https://developers.google.com/identity/protocols/oauth2/service-account :: ++ get-access-token - |= [jot=@t url=@t now=@da] + |= [jot=@t url=@t] =/ m (strand ,token:gcp) ^- form:m ;< ~ bind:m %: send-request:strandio @@ -128,15 +128,14 @@ =* job u.jon ~| job =, dejs:format - =/ [typ=@t exp=@dr tok=@t] + =/ [typ=@t =token:gcp] %. job %: ot 'token_type'^so - 'expires_in'^(cu |=(a=@ (mul a ~s1)) ni) 'access_token'^so + 'expires_in'^(cu |=(a=@ (mul a ~s1)) ni) ~ == ?> =('Bearer' typ) - %- pure:m - [tok (add exp now)] + (pure:m token) -- From 2a82da55ca419685dfb043c22ed78edf359759ee Mon Sep 17 00:00:00 2001 From: J Date: Wed, 24 Feb 2021 20:37:37 +0000 Subject: [PATCH 048/103] -get-gcp-token: error messages Don't dump extra console output on thread failure. Say 'gcp' in failures to aid debugging. --- pkg/arvo/ted/get-gcp-token.hoon | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/arvo/ted/get-gcp-token.hoon b/pkg/arvo/ted/get-gcp-token.hoon index 3093707f18..eb1d59c459 100644 --- a/pkg/arvo/ted/get-gcp-token.hoon +++ b/pkg/arvo/ted/get-gcp-token.hoon @@ -46,7 +46,7 @@ %+ scry:strandio ? /gx/settings-store/has-entry/gcp-store/[key]/noun ?. has - (strand-fail:strandio %no-setting key ~) + (strand-fail:strandio (rap 3 %gcp-missing- key ~) ~) ;< =data:settings bind:m %+ scry:strandio data:settings @@ -120,11 +120,12 @@ take-client-response:strandio ?> ?=(%finished -.rep) ?~ full-file.rep - (strand-fail:strandio %no-response ~) + (strand-fail:strandio %gcp-no-response ~) =/ body=@t q.data.u.full-file.rep =/ jon=(unit json) (de-json:html body) ?~ jon - (strand-fail:strandio %bad-body ~[body]) + ~| body + (strand-fail:strandio %gcp-bad-body ~) =* job u.jon ~| job =, dejs:format From 94556ce936e9ab36f0293096089c11a2f8b7b6c9 Mon Sep 17 00:00:00 2001 From: J Date: Wed, 24 Feb 2021 20:42:57 +0000 Subject: [PATCH 049/103] interface: GCP skeleton --- pkg/interface/src/logic/api/gcp.ts | 14 ++++++++++++++ pkg/interface/src/logic/api/global.ts | 2 ++ pkg/interface/src/logic/store/store.ts | 1 + pkg/interface/src/logic/store/type.ts | 2 ++ pkg/interface/src/types/gcp-state.ts | 3 +++ pkg/interface/src/views/App.js | 1 + 6 files changed, 23 insertions(+) create mode 100644 pkg/interface/src/logic/api/gcp.ts create mode 100644 pkg/interface/src/types/gcp-state.ts diff --git a/pkg/interface/src/logic/api/gcp.ts b/pkg/interface/src/logic/api/gcp.ts new file mode 100644 index 0000000000..6f85a322f5 --- /dev/null +++ b/pkg/interface/src/logic/api/gcp.ts @@ -0,0 +1,14 @@ +import BaseApi from './base'; +import {StoreState} from '../store/type'; + + +export default class GcpApi extends BaseApi { + startRefreshLoop() { + this.refreshToken(); + } + + private refreshToken() { + const res = this.spider('noun', 'gcp-token', 'get-gcp-token', {}); + // TODO: store the token, set a timer to refresh it + } +}; diff --git a/pkg/interface/src/logic/api/global.ts b/pkg/interface/src/logic/api/global.ts index 8ed02020b4..edf6f0cc5d 100644 --- a/pkg/interface/src/logic/api/global.ts +++ b/pkg/interface/src/logic/api/global.ts @@ -10,6 +10,7 @@ import GroupsApi from './groups'; import LaunchApi from './launch'; import GraphApi from './graph'; import S3Api from './s3'; +import GcpApi from './gcp'; import {HarkApi} from './hark'; import SettingsApi from './settings'; @@ -20,6 +21,7 @@ export default class GlobalApi extends BaseApi { contacts = new ContactsApi(this.ship, this.channel, this.store); groups = new GroupsApi(this.ship, this.channel, this.store); launch = new LaunchApi(this.ship, this.channel, this.store); + gcp = new GcpApi(this.ship, this.channel, this.store); s3 = new S3Api(this.ship, this.channel, this.store); graph = new GraphApi(this.ship, this.channel, this.store); hark = new HarkApi(this.ship, this.channel, this.store); diff --git a/pkg/interface/src/logic/store/store.ts b/pkg/interface/src/logic/store/store.ts index b2ab60c6c8..cd06ae91cb 100644 --- a/pkg/interface/src/logic/store/store.ts +++ b/pkg/interface/src/logic/store/store.ts @@ -71,6 +71,7 @@ export default class GlobalStore extends BaseStore { }, weather: {}, userLocation: null, + gcp: {}, s3: { configuration: { buckets: new Set(), diff --git a/pkg/interface/src/logic/store/type.ts b/pkg/interface/src/logic/store/type.ts index 96b68f8a2d..6cc9c23a83 100644 --- a/pkg/interface/src/logic/store/type.ts +++ b/pkg/interface/src/logic/store/type.ts @@ -3,6 +3,7 @@ import { Invites } from '~/types/invite-update'; import { Associations } from '~/types/metadata-update'; import { Rolodex } from '~/types/contact-update'; import { Groups } from '~/types/group-update'; +import { GcpState } from '~/types/gcp-state'; import { S3State } from '~/types/s3-update'; import { LaunchState, WeatherState } from '~/types/launch-update'; import { ConnectionStatus } from '~/types/connection'; @@ -32,6 +33,7 @@ export interface StoreState { groupKeys: Set; nackedContacts: Set s3: S3State; + gcp: GcpState; graphs: Graphs; graphKeys: Set; diff --git a/pkg/interface/src/types/gcp-state.ts b/pkg/interface/src/types/gcp-state.ts new file mode 100644 index 0000000000..3f83993060 --- /dev/null +++ b/pkg/interface/src/types/gcp-state.ts @@ -0,0 +1,3 @@ +export interface GcpState { + accessToken: string; +}; diff --git a/pkg/interface/src/views/App.js b/pkg/interface/src/views/App.js index 3e1650dc34..13fd7d9ef1 100644 --- a/pkg/interface/src/views/App.js +++ b/pkg/interface/src/views/App.js @@ -94,6 +94,7 @@ class App extends React.Component { // before the app has actually rendered, hence the timeout. this.updateTheme(this.themeWatcher); }, 500); + this.api.gcp.startRefreshLoop(); this.api.local.getBaseHash(); this.api.settings.getAll(); this.store.rehydrate(); From d60c3183f7de6777e8e18a9a7ec60169c42a4ef7 Mon Sep 17 00:00:00 2001 From: J Date: Wed, 24 Feb 2021 20:55:40 +0000 Subject: [PATCH 050/103] gcp: lib spacing --- pkg/arvo/lib/gcp.hoon | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/arvo/lib/gcp.hoon b/pkg/arvo/lib/gcp.hoon index 41163cede1..8329f4cb36 100644 --- a/pkg/arvo/lib/gcp.hoon +++ b/pkg/arvo/lib/gcp.hoon @@ -8,8 +8,8 @@ %: pairs access-key+s+access-key.token :- %expires-in - %- numb - (div (mul 1.000 expires-in.token) ~s1) + %- numb + (div (mul 1.000 expires-in.token) ~s1) ~ == -- From d0bb1cc849c8ff5d9958492cd0a6583b005cfb52 Mon Sep 17 00:00:00 2001 From: J Date: Wed, 24 Feb 2021 21:20:39 +0000 Subject: [PATCH 051/103] gcp: camel-case json key names Now we can do ({accessKey, expiresIn}) => {...} in JavaScript. --- pkg/arvo/lib/gcp.hoon | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/arvo/lib/gcp.hoon b/pkg/arvo/lib/gcp.hoon index 8329f4cb36..0b472ea975 100644 --- a/pkg/arvo/lib/gcp.hoon +++ b/pkg/arvo/lib/gcp.hoon @@ -6,8 +6,8 @@ =, enjs:format %+ frond %gcp-token %: pairs - access-key+s+access-key.token - :- %expires-in + [%'accessKey' s+access-key.token] + :- %'expiresIn' %- numb (div (mul 1.000 expires-in.token) ~s1) ~ From a80c3504a8f0be567af90f32fbc6582b8235317a Mon Sep 17 00:00:00 2001 From: J Date: Wed, 24 Feb 2021 21:22:28 +0000 Subject: [PATCH 052/103] interface: refresh loop Still not actually storing the token anywhere... --- pkg/interface/src/logic/api/gcp.ts | 20 ++++++++++++++++---- pkg/interface/src/types/gcp-state.ts | 9 +++++++-- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/pkg/interface/src/logic/api/gcp.ts b/pkg/interface/src/logic/api/gcp.ts index 6f85a322f5..4069ec0384 100644 --- a/pkg/interface/src/logic/api/gcp.ts +++ b/pkg/interface/src/logic/api/gcp.ts @@ -1,14 +1,26 @@ import BaseApi from './base'; import {StoreState} from '../store/type'; +import {GcpToken} from '../types/gcp-state'; export default class GcpApi extends BaseApi { startRefreshLoop() { - this.refreshToken(); + // TODO: don't allow more than one + this.refreshLoop(); } - private refreshToken() { - const res = this.spider('noun', 'gcp-token', 'get-gcp-token', {}); - // TODO: store the token, set a timer to refresh it + private refreshLoop() { + console.log("refreshing GCP token"); + this.refreshToken().then(({accessKey, expiresIn}: GcpToken) => { + console.log("new token: ", accessKey); + setTimeout(() => { + this.refreshLoop(); + }, expiresIn); + }); + } + + private async refreshToken() { + const token = await this.spider('noun', 'gcp-token', 'get-gcp-token', {}); + return token['gcp-token']; } }; diff --git a/pkg/interface/src/types/gcp-state.ts b/pkg/interface/src/types/gcp-state.ts index 3f83993060..8d24f12c17 100644 --- a/pkg/interface/src/types/gcp-state.ts +++ b/pkg/interface/src/types/gcp-state.ts @@ -1,3 +1,8 @@ -export interface GcpState { - accessToken: string; +export interface GcpToken { + accessKey: string; + expiresIn: number; +}; + +export interface GcpState { + token: GcpToken; }; From 0a23c0f4a4d611d8dc8da060c8cbb01a202aaad7 Mon Sep 17 00:00:00 2001 From: J Date: Wed, 24 Feb 2021 21:26:59 +0000 Subject: [PATCH 053/103] -get-gcp-token: docs, formatting Correct expiry from @da -> @dr, and set max cols to 72. --- pkg/arvo/ted/get-gcp-token.hoon | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/pkg/arvo/ted/get-gcp-token.hoon b/pkg/arvo/ted/get-gcp-token.hoon index eb1d59c459..3562aaf268 100644 --- a/pkg/arvo/ted/get-gcp-token.hoon +++ b/pkg/arvo/ted/get-gcp-token.hoon @@ -1,20 +1,22 @@ :: Gets a Google Storage access token. :: -:: This thread produces a pair of [access-token expires-at], where -:: access-token is a @t that can be used as a bearer token to talk to the -:: GCP Storage API on behalf of some service account, and expires-at is -:: a @da after which the token will stop working and need to be refreshed. +:: This thread produces a pair of [access-key expires-in], where +:: access-key is a @t that can be used as a bearer token to talk +:: to the GCP Storage API on behalf of some service account, and +:: expires-in is a @dr after which the token will stop working and +:: need to be refreshed. :: -:: It expects settings-store to contain relevant fields from a GCP service -:: account JSON file, generally as poked by sh/poke-gcp-account-json. -:: Specifically, it depends on the `token_uri`, `client_email`, -:: `private_key_id`, and `private_key` fields. If these fields are not -:: in settings-store at the time the thread is run, it will fail. +:: It expects settings-store to contain relevant fields from +:: a GCP service account JSON file, generally as poked by +:: sh/poke-gcp-account-json. Specifically, it depends on the +:: `token_uri`, `client_email`, `private_key_id`, and `private_key` +:: fields. If these fields are not in settings-store at the time +:: the thread is run, it will fail. :: -:: The thread works by first constructing a self-signed JWT using the -:: fields in settings-store. Then, it sends this JWT to the specified -:: token URI (usually https://oauth2.googleapis.com/token), which responds -:: with a bearer token and expiry. +:: The thread works by first constructing a self-signed JWT using +:: the fields in settings-store. Then, it sends this JWT to the +:: specified token URI (usually https://oauth2.googleapis.com/token), +:: which responds with a bearer token and expiry. :: :: /- gcp, spider, settings From 3c2ce636e731b36fd3d2f1ec3167d6417d47c5f9 Mon Sep 17 00:00:00 2001 From: J Date: Wed, 24 Feb 2021 22:32:20 +0000 Subject: [PATCH 054/103] interface: storing the gcp token --- pkg/interface/src/logic/api/gcp.ts | 27 +++++++++++++++++++++----- pkg/interface/src/logic/store/store.ts | 4 +++- pkg/interface/src/types/gcp-state.ts | 2 +- 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/pkg/interface/src/logic/api/gcp.ts b/pkg/interface/src/logic/api/gcp.ts index 4069ec0384..05791db22c 100644 --- a/pkg/interface/src/logic/api/gcp.ts +++ b/pkg/interface/src/logic/api/gcp.ts @@ -4,21 +4,38 @@ import {GcpToken} from '../types/gcp-state'; export default class GcpApi extends BaseApi { + #running = false; + startRefreshLoop() { - // TODO: don't allow more than one - this.refreshLoop(); + if (this.#running) { + console.error('GcpApi startRefreshLoop: already running'); + } else { + this.#running = true; + this.refreshLoop(); + } } private refreshLoop() { - console.log("refreshing GCP token"); + console.log("GcpApi refreshLoop"); this.refreshToken().then(({accessKey, expiresIn}: GcpToken) => { - console.log("new token: ", accessKey); + console.log("GcpApi new token"); + // XX maybe bad? + this.store.state.gcp.accessKey = accessKey; + const timeout = this.refreshInterval(expiresIn); + console.log("GcpApi refreshing in", timeout); setTimeout(() => { this.refreshLoop(); - }, expiresIn); + }, timeout); }); } + private refreshInterval(expiresIn: number) { + // Give ourselves 30 seconds for processing delays, but never refresh + // sooner than 10 minute from now. (The expiry window should be about an + // hour.) + return Math.max(600_000, expiresIn - 30_000); + } + private async refreshToken() { const token = await this.spider('noun', 'gcp-token', 'get-gcp-token', {}); return token['gcp-token']; diff --git a/pkg/interface/src/logic/store/store.ts b/pkg/interface/src/logic/store/store.ts index cd06ae91cb..17d5e497ef 100644 --- a/pkg/interface/src/logic/store/store.ts +++ b/pkg/interface/src/logic/store/store.ts @@ -71,7 +71,9 @@ export default class GlobalStore extends BaseStore { }, weather: {}, userLocation: null, - gcp: {}, + gcp: { + accessKey: null + }, s3: { configuration: { buckets: new Set(), diff --git a/pkg/interface/src/types/gcp-state.ts b/pkg/interface/src/types/gcp-state.ts index 8d24f12c17..41ebe12501 100644 --- a/pkg/interface/src/types/gcp-state.ts +++ b/pkg/interface/src/types/gcp-state.ts @@ -4,5 +4,5 @@ export interface GcpToken { }; export interface GcpState { - token: GcpToken; + accessKey: string; }; From 9fa086e0d3144d62384ae88482671a7acf769290 Mon Sep 17 00:00:00 2001 From: J Date: Thu, 25 Feb 2021 00:29:28 +0000 Subject: [PATCH 055/103] interface: extract GcpManager Now the API is just an API. (I did still have it produce an expiry time, since it makes the refresh loop easier to do.) Not yet storing the token now. --- pkg/interface/src/logic/api/gcp.ts | 48 ++++-------- pkg/interface/src/logic/lib/gcpManager.ts | 94 +++++++++++++++++++++++ pkg/interface/src/views/App.js | 4 +- 3 files changed, 111 insertions(+), 35 deletions(-) create mode 100644 pkg/interface/src/logic/lib/gcpManager.ts diff --git a/pkg/interface/src/logic/api/gcp.ts b/pkg/interface/src/logic/api/gcp.ts index 05791db22c..4ed9e8f655 100644 --- a/pkg/interface/src/logic/api/gcp.ts +++ b/pkg/interface/src/logic/api/gcp.ts @@ -4,40 +4,20 @@ import {GcpToken} from '../types/gcp-state'; export default class GcpApi extends BaseApi { - #running = false; + // Return value resolves to the token's expiry time if successful. + refreshToken() { + return this.spider('noun', 'gcp-token', 'get-gcp-token', {}) + .then((token) => { + this.store.handleEvent({ + data: token + }); - startRefreshLoop() { - if (this.#running) { - console.error('GcpApi startRefreshLoop: already running'); - } else { - this.#running = true; - this.refreshLoop(); - } - } - - private refreshLoop() { - console.log("GcpApi refreshLoop"); - this.refreshToken().then(({accessKey, expiresIn}: GcpToken) => { - console.log("GcpApi new token"); - // XX maybe bad? - this.store.state.gcp.accessKey = accessKey; - const timeout = this.refreshInterval(expiresIn); - console.log("GcpApi refreshing in", timeout); - setTimeout(() => { - this.refreshLoop(); - }, timeout); - }); - } - - private refreshInterval(expiresIn: number) { - // Give ourselves 30 seconds for processing delays, but never refresh - // sooner than 10 minute from now. (The expiry window should be about an - // hour.) - return Math.max(600_000, expiresIn - 30_000); - } - - private async refreshToken() { - const token = await this.spider('noun', 'gcp-token', 'get-gcp-token', {}); - return token['gcp-token']; + if (token['gcp-token'] !== undefined && + token['gcp-token']['expiresIn'] !== undefined && + typeof(token['gcp-token']['expiresIn']) === 'number') { + return Promise.resolve(token['gcp-token']['expiresIn']); + } + return Promise.reject({reason: 'invalid token'}); + }); } }; diff --git a/pkg/interface/src/logic/lib/gcpManager.ts b/pkg/interface/src/logic/lib/gcpManager.ts new file mode 100644 index 0000000000..c74c30b46e --- /dev/null +++ b/pkg/interface/src/logic/lib/gcpManager.ts @@ -0,0 +1,94 @@ +// Singleton that manages GCP token state. +// +// To use: +// +// 1. call setApi with a GlobalApi. +// 2. call start() to start the token refresh loop. +// +// +import GlobalApi from '../api/global'; + + +class GcpManager { + #api: GlobalApi | null = null; + + setApi(api: GlobalApi) { + this.#api = api; + } + + #running = false; + #timeoutId: number | null = null; + + start() { + if (this.#running) { + console.warn('GcpManager already running'); + return; + } + if (!this.#api) { + console.error('GcpManager must have api set'); + return; + } + this.#running = true; + this.refreshLoop(); + } + + stop() { + if (!this.#running) { + console.warn('GcpManager already stopped'); + console.assert(this.#timeoutId === null); + return; + } + this.#running = false; + if (this.#timeoutId !== null) { + clearTimeout(this.#timeoutId); + this.#timeoutId = null; + } + } + + restart() { + if (this.#running) + this.stop(); + this.start(); + } + + private refreshLoop() { + console.log('GcpManager refreshing token'); + this.#api.gcp.refreshToken() + .then( + (expiresIn: number) => { + const interval = this.refreshInterval(expiresIn); + this.refreshAfter(interval); + }) + .catch( + ({reason}) => { + console.error('GcpManager token refresh failed', reason); + this.refreshAfter(10_000); // XX backoff? + }); + } + + private refreshAfter(durationMs) { + if (!this.#running) + return; + if (this.#timeoutId !== null) { + console.warn('GcpManager already has a timeout set'); + return; + } + console.log('GcpManager refreshing after', durationMs, 'ms'); + this.#timeoutId = setTimeout(() => { + this.#timeoutId = null; + this.refreshLoop(); + }, durationMs); + } + + private refreshInterval(expiresIn: number) { + // Give ourselves 30 seconds for processing delays, but never refresh + // sooner than 30 minutes from now. (The expiry window should be about an + // hour.) + return Math.max(30 * 60_000, expiresIn - 30_000); + } +} + +const instance = new GcpManager(); +Object.freeze(instance); + +export default instance; diff --git a/pkg/interface/src/views/App.js b/pkg/interface/src/views/App.js index 13fd7d9ef1..67147819a1 100644 --- a/pkg/interface/src/views/App.js +++ b/pkg/interface/src/views/App.js @@ -27,6 +27,7 @@ import GlobalSubscription from '~/logic/subscription/global'; import GlobalApi from '~/logic/api/global'; import { uxToHex } from '~/logic/lib/util'; import { foregroundFromBackground } from '~/logic/lib/sigil'; +import gcpManager from '~/logic/lib/gcpManager'; import { withLocalState } from '~/logic/state/local'; @@ -78,6 +79,7 @@ class App extends React.Component { this.appChannel = new window.channel(); this.api = new GlobalApi(this.ship, this.appChannel, this.store); + gcpManager.setApi(this.api); this.subscription = new GlobalSubscription(this.store, this.api, this.appChannel); @@ -94,7 +96,7 @@ class App extends React.Component { // before the app has actually rendered, hence the timeout. this.updateTheme(this.themeWatcher); }, 500); - this.api.gcp.startRefreshLoop(); + gcpManager.start(); this.api.local.getBaseHash(); this.api.settings.getAll(); this.store.rehydrate(); From efea38f842e67f53526e7ae559d666a09d1694fe Mon Sep 17 00:00:00 2001 From: J Date: Thu, 25 Feb 2021 00:41:47 +0000 Subject: [PATCH 056/103] interface: GcpReducer Now we are correctly storing the GCP accessKey in state. --- pkg/interface/src/logic/reducers/gcp-token.ts | 21 +++++++++++++++++++ pkg/interface/src/logic/store/store.ts | 3 +++ 2 files changed, 24 insertions(+) create mode 100644 pkg/interface/src/logic/reducers/gcp-token.ts diff --git a/pkg/interface/src/logic/reducers/gcp-token.ts b/pkg/interface/src/logic/reducers/gcp-token.ts new file mode 100644 index 0000000000..98ef0b98fa --- /dev/null +++ b/pkg/interface/src/logic/reducers/gcp-token.ts @@ -0,0 +1,21 @@ +import _ from 'lodash'; +import {StoreState} from '../store/type'; +import {GcpToken} from '../../types/gcp-state'; + +type GcpState = Pick; + +export default class GcpReducer{ + reduce(json: Cage, state: S) { + let data = json['gcp-token']; + if (data) { + this.setAccessKey(data, state); + } + } + + setAccessKey(json: GcpToken, state: S) { + const data = _.get(json, 'accessKey'); + if (data) { + state.gcp.accessKey = data; + } + } +} diff --git a/pkg/interface/src/logic/store/store.ts b/pkg/interface/src/logic/store/store.ts index 17d5e497ef..85d1ef0e12 100644 --- a/pkg/interface/src/logic/store/store.ts +++ b/pkg/interface/src/logic/store/store.ts @@ -16,6 +16,7 @@ import GroupReducer from '../reducers/group-update'; import LaunchReducer from '../reducers/launch-update'; import ConnectionReducer from '../reducers/connection'; import SettingsReducer from '../reducers/settings-update'; +import GcpReducer from '../reducers/gcp-token'; import {OrderedMap} from '../lib/OrderedMap'; import { BigIntOrderedMap } from '../lib/BigIntOrderedMap'; import {GroupViewReducer} from '../reducers/group-view'; @@ -30,6 +31,7 @@ export default class GlobalStore extends BaseStore { launchReducer = new LaunchReducer(); connReducer = new ConnectionReducer(); settingsReducer = new SettingsReducer(); + gcpReducer = new GcpReducer(); pastActions: Record = {} @@ -118,6 +120,7 @@ export default class GlobalStore extends BaseStore { GraphReducer(data, this.state); HarkReducer(data, this.state); ContactReducer(data, this.state); + this.gcpReducer.reduce(data, this.state); this.settingsReducer.reduce(data, this.state); GroupViewReducer(data, this.state); } From c718894eb3fd4ba60e688e97af8872c4021f7652 Mon Sep 17 00:00:00 2001 From: J Date: Thu, 25 Feb 2021 22:50:50 +0000 Subject: [PATCH 057/103] interface: explicitly depend on querystring It will be used by GcpClient. --- pkg/interface/package-lock.json | 36 ++++++++++++++++++++++++++++++--- pkg/interface/package.json | 1 + 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/pkg/interface/package-lock.json b/pkg/interface/package-lock.json index d4a06d132a..a8f560f0ba 100644 --- a/pkg/interface/package-lock.json +++ b/pkg/interface/package-lock.json @@ -2149,6 +2149,13 @@ "url": "0.10.3", "uuid": "3.3.2", "xml2js": "0.4.19" + }, + "dependencies": { + "querystring": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", + "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=" + } } }, "babel-eslint": { @@ -6642,6 +6649,14 @@ "requires": { "punycode": "1.3.2", "querystring": "0.2.0" + }, + "dependencies": { + "querystring": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", + "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=", + "dev": true + } } } } @@ -7402,9 +7417,9 @@ "dev": true }, "querystring": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", - "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=" + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.1.tgz", + "integrity": "sha512-wkvS7mL/JMugcup3/rMitHmd9ecIGd2lhFhK9N3UUQ450h66d1r3Y9nvXzQAW1Lq+wyx61k/1pfKS5KuKiyEbg==" }, "querystring-es3": { "version": "0.2.1", @@ -9587,6 +9602,13 @@ "requires": { "punycode": "1.3.2", "querystring": "0.2.0" + }, + "dependencies": { + "querystring": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", + "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=" + } } }, "url-parse": { @@ -10345,6 +10367,14 @@ "requires": { "punycode": "1.3.2", "querystring": "0.2.0" + }, + "dependencies": { + "querystring": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", + "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=", + "dev": true + } } } } diff --git a/pkg/interface/package.json b/pkg/interface/package.json index a65578c438..52a3bd555f 100644 --- a/pkg/interface/package.json +++ b/pkg/interface/package.json @@ -28,6 +28,7 @@ "normalize-wheel": "1.0.1", "oembed-parser": "^1.4.5", "prop-types": "^15.7.2", + "querystring": "^0.2.0", "react": "^16.14.0", "react-codemirror2": "^6.0.1", "react-dom": "^16.14.0", From a6d4c3a431584f044eb21db8d7d25e9a33160025 Mon Sep 17 00:00:00 2001 From: J Date: Thu, 25 Feb 2021 22:52:18 +0000 Subject: [PATCH 058/103] interface: proof-of-concept GCP upload As of this commit, the Links app uploads files to GCP storage. (Note that it no longer uploads to S3. Still need to support both.) --- pkg/interface/src/logic/lib/GcpClient.ts | 46 +++++++++ pkg/interface/src/logic/lib/useGcp.ts | 99 +++++++++++++++++++ pkg/interface/src/types/index.ts | 1 + .../src/views/apps/links/LinkResource.tsx | 2 + .../src/views/apps/links/LinkWindow.tsx | 6 +- .../apps/links/components/LinkSubmit.tsx | 9 +- 6 files changed, 157 insertions(+), 6 deletions(-) create mode 100644 pkg/interface/src/logic/lib/GcpClient.ts create mode 100644 pkg/interface/src/logic/lib/useGcp.ts diff --git a/pkg/interface/src/logic/lib/GcpClient.ts b/pkg/interface/src/logic/lib/GcpClient.ts new file mode 100644 index 0000000000..dda539c847 --- /dev/null +++ b/pkg/interface/src/logic/lib/GcpClient.ts @@ -0,0 +1,46 @@ +// Very simple GCP Storage client. +// +import querystring from 'querystring'; + +export interface UploadParams { + bucket: string; // the bucket to upload the object to + name: string; // the desired location within the bucket + type: string; // the object's mime-type + body: File; // the object itself + predefinedAcl?: string; // optional acl, usually you want 'publicRead' +} + + +const ENDPOINT = 'storage.googleapis.com'; +const BASE_URL = 'https://storage.googleapis.com/upload/storage/v1/b'; + +export default class GcpClient { + #accessKey: string; + + constructor(accessKey: string) { + this.#accessKey = accessKey; + } + + async upload({bucket, name, type, body, predefinedAcl}: UploadParams) { + const urlParams = { + uploadType: 'media', + name, + predefinedAcl + }; + const url = `https://${ENDPOINT}/upload/storage/v1/b/${bucket}/o?` + + querystring.stringify(urlParams); + const headers = new Headers(); + headers.append('Authorization', `Bearer ${this.#accessKey}`); + headers.append('Content-Type', type); + const response = await fetch(url, { + method: 'POST', + mode: 'cors', + cache: 'default', + headers, + referrerPolicy: 'no-referrer', + body + }); + // TODO: response errors: 400 -> perhaps need fine-grained permissions on + return `https://${ENDPOINT}/${bucket}/${name}`; + } +} diff --git a/pkg/interface/src/logic/lib/useGcp.ts b/pkg/interface/src/logic/lib/useGcp.ts new file mode 100644 index 0000000000..a7af590cc2 --- /dev/null +++ b/pkg/interface/src/logic/lib/useGcp.ts @@ -0,0 +1,99 @@ +import {useCallback, useMemo, useEffect, useRef, useState} from 'react'; +import {S3State} from '../../types/s3-update'; +import {GcpState} from '../../types/gcp-state'; +import GcpClient from './GcpClient'; +import {dateToDa, deSig} from "./util"; + + +// TODO: unify with IuseS3 +export interface IuseGcp { + canUpload: boolean; + upload: (file: File, bucket: string) => Promise; + uploadDefault: (file: File) => Promise; + uploading: boolean; + promptUpload: () => Promise; +} + +// TODO: unify with useS3 / expose useStorage +const useGcp = (s3: S3State, gcp: GcpState, { accept = '*' } = { accept: '*' }): IuseGcp => { + const [uploading, setUploading] = useState(false); + + const client = useRef(null); + + useEffect(() => { + if (!gcp.accessKey) { + return; + } + client.current = new GcpClient(gcp.accessKey); + }, [gcp.accessKey]); + + const canUpload = useMemo( + () => + (client && gcp.accessKey && s3.configuration.currentBucket !== "") || false, + [gcp.accessKey, s3.configuration.currentBucket, client] + ); + + const upload = useCallback( + async (file: File, bucket: string) => { + if (!client.current) { + throw new Error("Gcp not ready"); + } + + const fileParts = file.name.split('.'); + const fileName = fileParts.slice(0, -1); + const fileExtension = fileParts.pop(); + const timestamp = deSig(dateToDa(new Date())); + + const params = { + bucket, + name: `${window.ship}/${timestamp}-${fileName}.${fileExtension}`, + body: file, + predefinedAcl: 'publicRead', + type: file.type, + }; + + setUploading(true); + + const location = await client.current.upload(params); + + setUploading(false); + + return location; + }, + [client, setUploading] + ); + + const uploadDefault = useCallback(async (file: File) => { + if (s3.configuration.currentBucket == '') { + throw new Error('current bucket not set'); + } + return upload(file, s3.configuration.currentBucket); + }, [s3]); + + const promptUpload = useCallback( + () => { + return new Promise((resolve, reject) => { + const fileSelector = document.createElement('input'); + fileSelector.setAttribute('type', 'file'); + fileSelector.setAttribute('accept', accept); + fileSelector.style.visibility = 'hidden'; + fileSelector.addEventListener('change', () => { + const files = fileSelector.files; + if (!files || files.length <= 0) { + reject(); + return; + } + uploadDefault(files[0]).then(resolve); + document.body.removeChild(fileSelector); + }) + document.body.appendChild(fileSelector); + fileSelector.click(); + }) + }, + [uploadDefault] + ); + + return {canUpload, upload, uploadDefault, uploading, promptUpload}; +}; + +export default useGcp; diff --git a/pkg/interface/src/types/index.ts b/pkg/interface/src/types/index.ts index 5c1d81b0b9..390a4fb950 100644 --- a/pkg/interface/src/types/index.ts +++ b/pkg/interface/src/types/index.ts @@ -11,6 +11,7 @@ export * from './launch-update'; export * from './local-update'; export * from './metadata-update'; export * from './noun'; +export * from './gcp-state'; export * from './s3-update'; export * from './workspace'; export * from './util'; diff --git a/pkg/interface/src/views/apps/links/LinkResource.tsx b/pkg/interface/src/views/apps/links/LinkResource.tsx index d85a2d7706..bb1dec2781 100644 --- a/pkg/interface/src/views/apps/links/LinkResource.tsx +++ b/pkg/interface/src/views/apps/links/LinkResource.tsx @@ -35,6 +35,7 @@ export function LinkResource(props: LinkResourceProps) { graphKeys, unreads, s3, + gcp, history } = props; @@ -70,6 +71,7 @@ export function LinkResource(props: LinkResourceProps) { render={(props) => { return ( { canWrite ? ( - + ) : ( There are no links here yet. You do not have permission to post to this collection. ) @@ -96,7 +98,7 @@ export function LinkWindow(props: LinkWindowProps) { return ( - + diff --git a/pkg/interface/src/views/apps/links/components/LinkSubmit.tsx b/pkg/interface/src/views/apps/links/components/LinkSubmit.tsx index 2eaba45559..2a2a74910f 100644 --- a/pkg/interface/src/views/apps/links/components/LinkSubmit.tsx +++ b/pkg/interface/src/views/apps/links/components/LinkSubmit.tsx @@ -2,21 +2,22 @@ import { BaseInput, Box, Button, LoadingSpinner, Text } from "@tlon/indigo-react import React, { useCallback, useState } from "react"; import GlobalApi from "~/logic/api/global"; import { useFileDrag } from "~/logic/lib/useDrag"; -import useS3 from "~/logic/lib/useS3"; -import { S3State } from "~/types"; +import useGcp from "~/logic/lib/useGcp"; +import { GcpState, S3State } from "~/types"; import SubmitDragger from "~/views/components/SubmitDragger"; import { createPost } from "~/logic/api/graph"; import { hasProvider } from "oembed-parser"; interface LinkSubmitProps { api: GlobalApi; + gcp: GcpState; s3: S3State; name: string; ship: string; }; const LinkSubmit = (props: LinkSubmitProps) => { - let { canUpload, uploadDefault, uploading, promptUpload } = useS3(props.s3); + let { canUpload, uploadDefault, uploading, promptUpload } = useGcp(props.s3, props.gcp); const [submitFocused, setSubmitFocused] = useState(false); const [urlFocused, setUrlFocused] = useState(false); @@ -223,4 +224,4 @@ const LinkSubmit = (props: LinkSubmitProps) => { ); }; -export default LinkSubmit; \ No newline at end of file +export default LinkSubmit; From 4814d61c48c0212cc9c9806d2d6bb94e31383f61 Mon Sep 17 00:00:00 2001 From: J Date: Thu, 25 Feb 2021 23:08:41 +0000 Subject: [PATCH 059/103] interface: unnecessary const --- pkg/interface/src/logic/lib/gcpManager.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/interface/src/logic/lib/gcpManager.ts b/pkg/interface/src/logic/lib/gcpManager.ts index c74c30b46e..97016a7efd 100644 --- a/pkg/interface/src/logic/lib/gcpManager.ts +++ b/pkg/interface/src/logic/lib/gcpManager.ts @@ -56,8 +56,7 @@ class GcpManager { this.#api.gcp.refreshToken() .then( (expiresIn: number) => { - const interval = this.refreshInterval(expiresIn); - this.refreshAfter(interval); + this.refreshAfter(this.refreshInterval(expiresIn)); }) .catch( ({reason}) => { From 11a58115b0b106d9e3a417b54d1ad57e1eee937d Mon Sep 17 00:00:00 2001 From: J Date: Fri, 26 Feb 2021 00:07:15 +0000 Subject: [PATCH 060/103] interface: implement useStorage and withStorage useS3 and withS3 are now gone. Storage should work with both GCP and S3, preferring GCP if available and falling back. --- pkg/interface/src/logic/lib/GcpClient.ts | 33 +++---- pkg/interface/src/logic/lib/StorageClient.ts | 27 +++++ pkg/interface/src/logic/lib/useS3.ts | 99 ------------------- .../logic/lib/{useGcp.ts => useStorage.ts} | 60 ++++++----- .../src/views/apps/chat/ChatResource.tsx | 2 +- .../views/apps/chat/components/ChatInput.tsx | 9 +- .../apps/links/components/LinkSubmit.tsx | 5 +- .../publish/components/MarkdownEditor.tsx | 7 +- .../src/views/components/ImageInput.tsx | 8 +- .../src/views/components/SubmitDragger.tsx | 4 +- pkg/interface/src/views/components/withS3.tsx | 12 --- .../src/views/components/withStorage.tsx | 12 +++ 12 files changed, 111 insertions(+), 167 deletions(-) create mode 100644 pkg/interface/src/logic/lib/StorageClient.ts delete mode 100644 pkg/interface/src/logic/lib/useS3.ts rename pkg/interface/src/logic/lib/{useGcp.ts => useStorage.ts} (58%) delete mode 100644 pkg/interface/src/views/components/withS3.tsx create mode 100644 pkg/interface/src/views/components/withStorage.tsx diff --git a/pkg/interface/src/logic/lib/GcpClient.ts b/pkg/interface/src/logic/lib/GcpClient.ts index dda539c847..a7f838b52e 100644 --- a/pkg/interface/src/logic/lib/GcpClient.ts +++ b/pkg/interface/src/logic/lib/GcpClient.ts @@ -1,46 +1,45 @@ // Very simple GCP Storage client. // import querystring from 'querystring'; - -export interface UploadParams { - bucket: string; // the bucket to upload the object to - name: string; // the desired location within the bucket - type: string; // the object's mime-type - body: File; // the object itself - predefinedAcl?: string; // optional acl, usually you want 'publicRead' -} +import { + StorageAcl, + StorageClient, + UploadParams, + UploadResult +} from './StorageClient'; const ENDPOINT = 'storage.googleapis.com'; const BASE_URL = 'https://storage.googleapis.com/upload/storage/v1/b'; -export default class GcpClient { +export default class GcpClient implements StorageClient { #accessKey: string; constructor(accessKey: string) { this.#accessKey = accessKey; } - async upload({bucket, name, type, body, predefinedAcl}: UploadParams) { + async upload({Bucket, Key, ContentType, Body}: UploadParams): UploadResult { const urlParams = { uploadType: 'media', - name, - predefinedAcl + name: Key, + predefinedAcl: 'publicRead' }; - const url = `https://${ENDPOINT}/upload/storage/v1/b/${bucket}/o?` + + const url = `https://${ENDPOINT}/upload/storage/v1/b/${Bucket}/o?` + querystring.stringify(urlParams); const headers = new Headers(); headers.append('Authorization', `Bearer ${this.#accessKey}`); - headers.append('Content-Type', type); + headers.append('Content-Type', ContentType); const response = await fetch(url, { method: 'POST', mode: 'cors', cache: 'default', headers, referrerPolicy: 'no-referrer', - body + body: Body }); - // TODO: response errors: 400 -> perhaps need fine-grained permissions on - return `https://${ENDPOINT}/${bucket}/${name}`; + // TODO: response errors. + // If we get a 400, perhaps we need fine-grained permissions on the bucket. + return {Location: `https://${ENDPOINT}/${Bucket}/${Key}`}; } } diff --git a/pkg/interface/src/logic/lib/StorageClient.ts b/pkg/interface/src/logic/lib/StorageClient.ts new file mode 100644 index 0000000000..16cc79087a --- /dev/null +++ b/pkg/interface/src/logic/lib/StorageClient.ts @@ -0,0 +1,27 @@ +// Defines a StorageClient interface interoperable between S3 and GCP Storage. +// + + +// XX kind of gross. S3 needs 'public-read', GCP needs 'publicRead'. +// Rather than write a wrapper around S3, we offer this field here, which +// should always be passed, and will be replaced by 'publicRead' in the +// GCP client. +export enum StorageAcl { + PublicRead = 'public-read' +}; + +export interface UploadParams { + Bucket: string; // the bucket to upload the object to + Key: string; // the desired location within the bucket + ContentType: string; // the object's mime-type + ACL: StorageAcl; // ACL, always 'public-read' + Body: File; // the object itself +}; + +export interface UploadResult { + Location: string; +}; + +export interface StorageClient { + upload(params: UploadParams): Promise; +}; diff --git a/pkg/interface/src/logic/lib/useS3.ts b/pkg/interface/src/logic/lib/useS3.ts deleted file mode 100644 index 0717d52f7f..0000000000 --- a/pkg/interface/src/logic/lib/useS3.ts +++ /dev/null @@ -1,99 +0,0 @@ -import { useCallback, useMemo, useEffect, useRef, useState } from "react"; -import { S3State } from "../../types/s3-update"; -import S3 from "aws-sdk/clients/s3"; -import { dateToDa, deSig } from "./util"; - -export interface IuseS3 { - canUpload: boolean; - upload: (file: File, bucket: string) => Promise; - uploadDefault: (file: File) => Promise; - uploading: boolean; - promptUpload: () => Promise; -} - -const useS3 = (s3: S3State, { accept = '*' } = { accept: '*' }): IuseS3 => { - const [uploading, setUploading] = useState(false); - - const client = useRef(null); - - useEffect(() => { - if (!s3.credentials) { - return; - } - client.current = new S3({ - credentials: s3.credentials, - endpoint: s3.credentials.endpoint - }); - }, [s3.credentials]); - - const canUpload = useMemo( - () => - (client && s3.credentials && s3.configuration.currentBucket !== "") || false, - [s3.credentials, s3.configuration.currentBucket, client] - ); - - const upload = useCallback( - async (file: File, bucket: string) => { - if (!client.current) { - throw new Error("S3 not ready"); - } - - const fileParts = file.name.split('.'); - const fileName = fileParts.slice(0, -1); - const fileExtension = fileParts.pop(); - const timestamp = deSig(dateToDa(new Date())); - - const params = { - Bucket: bucket, - Key: `${window.ship}/${timestamp}-${fileName}.${fileExtension}`, - Body: file, - ACL: "public-read", - ContentType: file.type, - }; - - setUploading(true); - - const { Location } = await client.current.upload(params).promise(); - - setUploading(false); - - return Location; - }, - [client, setUploading] - ); - - const uploadDefault = useCallback(async (file: File) => { - if (s3.configuration.currentBucket === "") { - throw new Error("current bucket not set"); - } - return upload(file, s3.configuration.currentBucket); - }, [s3]); - - const promptUpload = useCallback( - () => { - return new Promise((resolve, reject) => { - const fileSelector = document.createElement('input'); - fileSelector.setAttribute('type', 'file'); - fileSelector.setAttribute('accept', accept); - fileSelector.style.visibility = 'hidden'; - fileSelector.addEventListener('change', () => { - const files = fileSelector.files; - if (!files || files.length <= 0) { - reject(); - return; - } - uploadDefault(files[0]).then(resolve); - document.body.removeChild(fileSelector); - }) - document.body.appendChild(fileSelector); - fileSelector.click(); - }) - - }, - [uploadDefault] - ); - - return { canUpload, upload, uploadDefault, uploading, promptUpload }; -}; - -export default useS3; \ No newline at end of file diff --git a/pkg/interface/src/logic/lib/useGcp.ts b/pkg/interface/src/logic/lib/useStorage.ts similarity index 58% rename from pkg/interface/src/logic/lib/useGcp.ts rename to pkg/interface/src/logic/lib/useStorage.ts index a7af590cc2..a011c17275 100644 --- a/pkg/interface/src/logic/lib/useGcp.ts +++ b/pkg/interface/src/logic/lib/useStorage.ts @@ -1,12 +1,13 @@ import {useCallback, useMemo, useEffect, useRef, useState} from 'react'; import {S3State} from '../../types/s3-update'; import {GcpState} from '../../types/gcp-state'; +import S3 from "aws-sdk/clients/s3"; import GcpClient from './GcpClient'; +import {StorageClient, StorageAcl} from './StorageClient'; import {dateToDa, deSig} from "./util"; -// TODO: unify with IuseS3 -export interface IuseGcp { +export interface IuseStorage { canUpload: boolean; upload: (file: File, bucket: string) => Promise; uploadDefault: (file: File) => Promise; @@ -14,29 +15,42 @@ export interface IuseGcp { promptUpload: () => Promise; } -// TODO: unify with useS3 / expose useStorage -const useGcp = (s3: S3State, gcp: GcpState, { accept = '*' } = { accept: '*' }): IuseGcp => { +const useStorage = (s3: S3State, gcp: GcpState, + { accept = '*' } = { accept: '*' }): IuseStorage => { const [uploading, setUploading] = useState(false); - const client = useRef(null); + const gcpClient = useRef(null); + const s3Client = useRef(null); useEffect(() => { - if (!gcp.accessKey) { - return; + // prefer GCP if available, else use S3. + if (gcp.accessKey) { + gcpClient.current = new GcpClient(gcp.accessKey); + s3Client.current = null; + } else { + if (!s3.credentials) { + return; + } + s3Client.current = new S3({ + credentials: s3.credentials, + endpoint: s3.credentials.endpoint + }); + gcpClient.current = null; } - client.current = new GcpClient(gcp.accessKey); - }, [gcp.accessKey]); + }, [gcp.accessKey, s3.credentials]); const canUpload = useMemo( () => - (client && gcp.accessKey && s3.configuration.currentBucket !== "") || false, - [gcp.accessKey, s3.configuration.currentBucket, client] + ((gcpClient || s3Client) && s3.configuration.currentBucket !== "") || false, + [gcpClient, s3Client, s3.configuration.currentBucket] ); const upload = useCallback( async (file: File, bucket: string) => { - if (!client.current) { - throw new Error("Gcp not ready"); + const client: StorageClient | null = + gcpClient.current || s3Client.current; + if (!client) { + throw new Error("Storage not ready"); } const fileParts = file.name.split('.'); @@ -45,22 +59,22 @@ const useGcp = (s3: S3State, gcp: GcpState, { accept = '*' } = { accept: '*' }): const timestamp = deSig(dateToDa(new Date())); const params = { - bucket, - name: `${window.ship}/${timestamp}-${fileName}.${fileExtension}`, - body: file, - predefinedAcl: 'publicRead', - type: file.type, + Bucket: bucket, + Key: `${window.ship}/${timestamp}-${fileName}.${fileExtension}`, + Body: file, + ACL: StorageAcl.PublicRead, + ContentType: file.type, }; setUploading(true); - const location = await client.current.upload(params); + const { Location } = await client.upload(params); setUploading(false); - return location; + return Location; }, - [client, setUploading] + [gcpClient, s3Client, setUploading] ); const uploadDefault = useCallback(async (file: File) => { @@ -68,7 +82,7 @@ const useGcp = (s3: S3State, gcp: GcpState, { accept = '*' } = { accept: '*' }): throw new Error('current bucket not set'); } return upload(file, s3.configuration.currentBucket); - }, [s3]); + }, [s3.configuration, upload]); const promptUpload = useCallback( () => { @@ -96,4 +110,4 @@ const useGcp = (s3: S3State, gcp: GcpState, { accept = '*' } = { accept: '*' }): return {canUpload, upload, uploadDefault, uploading, promptUpload}; }; -export default useGcp; +export default useStorage; diff --git a/pkg/interface/src/views/apps/chat/ChatResource.tsx b/pkg/interface/src/views/apps/chat/ChatResource.tsx index 6b47a44091..3473e03147 100644 --- a/pkg/interface/src/views/apps/chat/ChatResource.tsx +++ b/pkg/interface/src/views/apps/chat/ChatResource.tsx @@ -13,7 +13,6 @@ import { ShareProfile } from '~/views/apps/chat/components/ShareProfile'; import SubmitDragger from '~/views/components/SubmitDragger'; import { useLocalStorageState } from '~/logic/lib/useLocalStorageState'; import { Loading } from '~/views/components/Loading'; -import useS3 from '~/logic/lib/useS3'; import { isWriter, resourceFromPath } from '~/logic/lib/group'; import './css/custom.css'; @@ -180,6 +179,7 @@ export function ChatResource(props: ChatResourceProps) { (!showBanner && hasLoadedAllowed) ? contacts : modifiedContacts } onUnmount={appendUnsent} + gcp={props.gcp} s3={props.s3} placeholder="Message..." message={unsent[station] || ''} diff --git a/pkg/interface/src/views/apps/chat/components/ChatInput.tsx b/pkg/interface/src/views/apps/chat/components/ChatInput.tsx index 664e4e9ac5..9a64e39099 100644 --- a/pkg/interface/src/views/apps/chat/components/ChatInput.tsx +++ b/pkg/interface/src/views/apps/chat/components/ChatInput.tsx @@ -1,6 +1,6 @@ import React, { Component } from 'react'; import ChatEditor from './chat-editor'; -import { IuseS3 } from '~/logic/lib/useS3'; +import { IuseStorage } from '~/logic/lib/useStorage'; import { uxToHex } from '~/logic/lib/util'; import { Sigil } from '~/logic/lib/sigil'; import { createPost } from '~/logic/api/graph'; @@ -9,10 +9,10 @@ import GlobalApi from '~/logic/api/global'; import { Envelope } from '~/types/chat-update'; import { Contacts, Content } from '~/types'; import { Row, BaseImage, Box, Icon, LoadingSpinner } from '@tlon/indigo-react'; -import withS3 from '~/views/components/withS3'; +import withStorage from '~/views/components/withStorage'; import { withLocalState } from '~/logic/state/local'; -type ChatInputProps = IuseS3 & { +type ChatInputProps = IuseStorage & { api: GlobalApi; numMsgs: number; station: any; @@ -20,6 +20,7 @@ type ChatInputProps = IuseS3 & { envelopes: Envelope[]; contacts: Contacts; onUnmount(msg: string): void; + gcp: any; s3: any; placeholder: string; message: string; @@ -200,4 +201,4 @@ class ChatInput extends Component { } } -export default withLocalState(withS3(ChatInput, {accept: 'image/*'}), ['hideAvatars']); +export default withLocalState(withStorage(ChatInput, {accept: 'image/*'}), ['hideAvatars']); diff --git a/pkg/interface/src/views/apps/links/components/LinkSubmit.tsx b/pkg/interface/src/views/apps/links/components/LinkSubmit.tsx index 2a2a74910f..09f6e4a699 100644 --- a/pkg/interface/src/views/apps/links/components/LinkSubmit.tsx +++ b/pkg/interface/src/views/apps/links/components/LinkSubmit.tsx @@ -2,7 +2,7 @@ import { BaseInput, Box, Button, LoadingSpinner, Text } from "@tlon/indigo-react import React, { useCallback, useState } from "react"; import GlobalApi from "~/logic/api/global"; import { useFileDrag } from "~/logic/lib/useDrag"; -import useGcp from "~/logic/lib/useGcp"; +import useStorage from "~/logic/lib/useStorage"; import { GcpState, S3State } from "~/types"; import SubmitDragger from "~/views/components/SubmitDragger"; import { createPost } from "~/logic/api/graph"; @@ -17,7 +17,8 @@ interface LinkSubmitProps { }; const LinkSubmit = (props: LinkSubmitProps) => { - let { canUpload, uploadDefault, uploading, promptUpload } = useGcp(props.s3, props.gcp); + let { canUpload, uploadDefault, uploading, promptUpload } = + useStorage(props.s3, props.gcp); const [submitFocused, setSubmitFocused] = useState(false); const [urlFocused, setUrlFocused] = useState(false); diff --git a/pkg/interface/src/views/apps/publish/components/MarkdownEditor.tsx b/pkg/interface/src/views/apps/publish/components/MarkdownEditor.tsx index 32f372f5d8..ada3c0e2ff 100644 --- a/pkg/interface/src/views/apps/publish/components/MarkdownEditor.tsx +++ b/pkg/interface/src/views/apps/publish/components/MarkdownEditor.tsx @@ -16,8 +16,8 @@ import "codemirror/lib/codemirror.css"; import { Box } from "@tlon/indigo-react"; import { useFileDrag } from "~/logic/lib/useDrag"; import SubmitDragger from "~/views/components/SubmitDragger"; -import useS3 from "~/logic/lib/useS3"; -import { S3State } from "~/types"; +import useStorage from "~/logic/lib/useStorage"; +import { GcpState, S3State } from "~/types"; const MARKDOWN_CONFIG = { name: "markdown", @@ -28,6 +28,7 @@ interface MarkdownEditorProps { value: string; onChange: (s: string) => void; onBlur?: (e: any) => void; + gcp: GcpState; s3: S3State; } @@ -74,7 +75,7 @@ export function MarkdownEditor( [onBlur] ); - const { uploadDefault, canUpload } = useS3(props.s3); + const { uploadDefault, canUpload } = useStorage(props.s3, props.gcp); const onFileDrag = useCallback( async (files: FileList | File[], e: DragEvent) => { diff --git a/pkg/interface/src/views/components/ImageInput.tsx b/pkg/interface/src/views/components/ImageInput.tsx index d234462b13..836d656993 100644 --- a/pkg/interface/src/views/components/ImageInput.tsx +++ b/pkg/interface/src/views/components/ImageInput.tsx @@ -11,19 +11,21 @@ import { } from "@tlon/indigo-react"; import { useField } from "formik"; import { S3State } from "~/types/s3-update"; -import useS3 from "~/logic/lib/useS3"; +import { GcpState } from "~/types/gcp-state"; +import useStorage from "~/logic/lib/useStorage"; type ImageInputProps = Parameters[0] & { id: string; label: string; s3: S3State; + gcp: GcpState; placeholder?: string; }; export function ImageInput(props: ImageInputProps) { - const { id, label, s3, caption, placeholder, ...rest } = props; + const { id, label, s3, gcp, caption, placeholder, ...rest } = props; - const { uploadDefault, canUpload, uploading } = useS3(s3); + const { uploadDefault, canUpload, uploading } = useStorage(s3, gcp); const [field, meta, { setValue, setError }] = useField(id); diff --git a/pkg/interface/src/views/components/SubmitDragger.tsx b/pkg/interface/src/views/components/SubmitDragger.tsx index 7bece2aafd..b8a98f7103 100644 --- a/pkg/interface/src/views/components/SubmitDragger.tsx +++ b/pkg/interface/src/views/components/SubmitDragger.tsx @@ -1,7 +1,5 @@ import { BaseInput, Box, Icon, LoadingSpinner, Text } from "@tlon/indigo-react"; import React, { useCallback } from "react"; -import useS3 from "~/logic/lib/useS3"; -import { S3State } from "~/types"; const SubmitDragger = () => ( ( ); -export default SubmitDragger; \ No newline at end of file +export default SubmitDragger; diff --git a/pkg/interface/src/views/components/withS3.tsx b/pkg/interface/src/views/components/withS3.tsx deleted file mode 100644 index 5bb9bae475..0000000000 --- a/pkg/interface/src/views/components/withS3.tsx +++ /dev/null @@ -1,12 +0,0 @@ -import React from "react"; -import useS3 from "~/logic/lib/useS3"; - -const withS3 = (Component, params = {}) => { - return React.forwardRef((props: any, ref) => { - const s3 = useS3(props.s3, params); - - return ; - }); -}; - -export default withS3; \ No newline at end of file diff --git a/pkg/interface/src/views/components/withStorage.tsx b/pkg/interface/src/views/components/withStorage.tsx new file mode 100644 index 0000000000..95b9d3cf48 --- /dev/null +++ b/pkg/interface/src/views/components/withStorage.tsx @@ -0,0 +1,12 @@ +import React from "react"; +import useStorage from "~/logic/lib/useStorage"; + +const withStorage = (Component, params = {}) => { + return React.forwardRef((props: any, ref) => { + const storage = useStorage(props.s3, props.gcp, params); + + return ; + }); +}; + +export default withStorage; From 1c0d8e524e1ce5de2c9ea184bc85fbf5e6d56da2 Mon Sep 17 00:00:00 2001 From: J Date: Fri, 26 Feb 2021 00:38:01 +0000 Subject: [PATCH 061/103] interface: thread gcp through props Basically just a grep for 's3', and I added gcp wherever it looked like I ought to. --- pkg/interface/src/logic/lib/s3.js | 41 ------------------- .../src/views/apps/links/LinkResource.tsx | 2 +- .../apps/profile/components/EditProfile.tsx | 6 +-- .../views/apps/profile/components/Profile.tsx | 1 + .../src/views/apps/profile/profile.tsx | 1 + .../views/apps/publish/PublishResource.tsx | 1 + .../apps/publish/components/EditPost.tsx | 6 ++- .../apps/publish/components/MarkdownField.tsx | 2 + .../apps/publish/components/NoteForm.tsx | 7 ++-- .../apps/publish/components/NoteRoutes.tsx | 11 ++++- .../publish/components/NotebookRoutes.tsx | 4 ++ .../apps/publish/components/new-post.tsx | 4 +- .../components/lib/BackgroundPicker.tsx | 5 ++- .../settings/components/lib/DisplayForm.tsx | 6 ++- .../apps/settings/components/settings.tsx | 2 + .../components/GroupSettings/Admin.tsx | 6 ++- .../GroupSettings/GroupSettings.tsx | 3 +- .../views/landscape/components/GroupsPane.tsx | 1 + .../landscape/components/PopoverRoutes.tsx | 9 +++- 19 files changed, 59 insertions(+), 59 deletions(-) delete mode 100644 pkg/interface/src/logic/lib/s3.js diff --git a/pkg/interface/src/logic/lib/s3.js b/pkg/interface/src/logic/lib/s3.js deleted file mode 100644 index 64bda11be7..0000000000 --- a/pkg/interface/src/logic/lib/s3.js +++ /dev/null @@ -1,41 +0,0 @@ -import S3 from 'aws-sdk/clients/s3'; - -export default class S3Client { - constructor() { - this.s3 = null; - - this.endpoint = ''; - this.accessKeyId = ''; - this.secretAccesskey = ''; - } - - setCredentials(endpoint, accessKeyId, secretAccessKey) { - this.endpoint = endpoint; - this.accessKeyId = accessKeyId; - this.secretAccessKey = secretAccessKey; - - this.s3 = new S3({ - endpoint: endpoint, - credentials: { - accessKeyId: this.accessKeyId, - secretAccessKey: this.secretAccessKey - } - }); - } - - async upload(bucket, filename, buffer) { - const params = { - Bucket: bucket, - Key: filename, - Body: buffer, - ACL: 'public-read', - ContentType: buffer.type - }; - - if(!this.s3) { - throw new Error('S3 not initialized'); - } - return this.s3.upload(params).promise(); - } -} - diff --git a/pkg/interface/src/views/apps/links/LinkResource.tsx b/pkg/interface/src/views/apps/links/LinkResource.tsx index bb1dec2781..5b4aeff556 100644 --- a/pkg/interface/src/views/apps/links/LinkResource.tsx +++ b/pkg/interface/src/views/apps/links/LinkResource.tsx @@ -34,8 +34,8 @@ export function LinkResource(props: LinkResourceProps) { associations, graphKeys, unreads, - s3, gcp, + s3, history } = props; diff --git a/pkg/interface/src/views/apps/profile/components/EditProfile.tsx b/pkg/interface/src/views/apps/profile/components/EditProfile.tsx index d43cfbaad2..1aca97d866 100644 --- a/pkg/interface/src/views/apps/profile/components/EditProfile.tsx +++ b/pkg/interface/src/views/apps/profile/components/EditProfile.tsx @@ -114,15 +114,15 @@ export function EditProfile(props: any) { Description - + - + - + diff --git a/pkg/interface/src/views/apps/profile/components/Profile.tsx b/pkg/interface/src/views/apps/profile/components/Profile.tsx index 0072b591bc..7252eec3c3 100644 --- a/pkg/interface/src/views/apps/profile/components/Profile.tsx +++ b/pkg/interface/src/views/apps/profile/components/Profile.tsx @@ -107,6 +107,7 @@ export function Profile(props: any) { diff --git a/pkg/interface/src/views/apps/publish/components/EditPost.tsx b/pkg/interface/src/views/apps/publish/components/EditPost.tsx index 0f6ed4dcc0..84ca37acf3 100644 --- a/pkg/interface/src/views/apps/publish/components/EditPost.tsx +++ b/pkg/interface/src/views/apps/publish/components/EditPost.tsx @@ -4,7 +4,7 @@ import { PostFormSchema, PostForm } from "./NoteForm"; import { FormikHelpers } from "formik"; import GlobalApi from "~/logic/api/global"; import { RouteComponentProps, useLocation } from "react-router-dom"; -import { GraphNode, TextContent, Association, S3State } from "~/types"; +import { GraphNode, TextContent, Association, GcpState, S3State } from "~/types"; import { getLatestRevision, editPost } from "~/logic/lib/publish"; import {useWaitForProps} from "~/logic/lib/useWaitForProps"; interface EditPostProps { @@ -13,11 +13,12 @@ interface EditPostProps { note: GraphNode; api: GlobalApi; book: string; + gcp: GcpState; s3: S3State; } export function EditPost(props: EditPostProps & RouteComponentProps) { - const { note, book, noteId, api, ship, history, s3 } = props; + const { note, book, noteId, api, ship, history, gcp, s3 } = props; const [revNum, title, body] = getLatestRevision(note); const location = useLocation(); @@ -54,6 +55,7 @@ export function EditPost(props: EditPostProps & RouteComponentProps) { cancel history={history} onSubmit={onSubmit} + gcp={gcp} s3={s3} submitLabel="Update" loadingText="Updating..." diff --git a/pkg/interface/src/views/apps/publish/components/MarkdownField.tsx b/pkg/interface/src/views/apps/publish/components/MarkdownField.tsx index 750a649784..c6adb0b6b8 100644 --- a/pkg/interface/src/views/apps/publish/components/MarkdownField.tsx +++ b/pkg/interface/src/views/apps/publish/components/MarkdownField.tsx @@ -6,6 +6,7 @@ import { MarkdownEditor } from "./MarkdownEditor"; export const MarkdownField = ({ id, + gcp, s3, ...rest }: { id: string } & Parameters[0]) => { @@ -35,6 +36,7 @@ export const MarkdownField = ({ onBlur={handleBlur} value={value} onChange={setValue} + gcp={gcp} s3={s3} /> diff --git a/pkg/interface/src/views/apps/publish/components/NoteForm.tsx b/pkg/interface/src/views/apps/publish/components/NoteForm.tsx index 6fbaaee20a..ff2001ff68 100644 --- a/pkg/interface/src/views/apps/publish/components/NoteForm.tsx +++ b/pkg/interface/src/views/apps/publish/components/NoteForm.tsx @@ -9,7 +9,7 @@ import { import { AsyncButton } from "../../../components/AsyncButton"; import { Formik, Form, FormikHelpers } from "formik"; import { MarkdownField } from "./MarkdownField"; -import { S3State } from "~/types"; +import { GcpState, S3State } from "~/types"; interface PostFormProps { initial: PostFormSchema; @@ -21,6 +21,7 @@ interface PostFormProps { ) => Promise; submitLabel: string; loadingText: string; + gcp: GcpState; s3: S3State; } @@ -35,7 +36,7 @@ export interface PostFormSchema { } export function PostForm(props: PostFormProps) { - const { initial, onSubmit, submitLabel, loadingText, s3, cancel, history } = props; + const { initial, onSubmit, submitLabel, loadingText, gcp, s3, cancel, history } = props; return ( @@ -66,7 +67,7 @@ export function PostForm(props: PostFormProps) { type="button">Cancel} - + diff --git a/pkg/interface/src/views/apps/publish/components/NoteRoutes.tsx b/pkg/interface/src/views/apps/publish/components/NoteRoutes.tsx index 916347038a..234ef941c6 100644 --- a/pkg/interface/src/views/apps/publish/components/NoteRoutes.tsx +++ b/pkg/interface/src/views/apps/publish/components/NoteRoutes.tsx @@ -6,7 +6,15 @@ import { RouteComponentProps } from "react-router-dom"; import Note from "./Note"; import { EditPost } from "./EditPost"; -import { GraphNode, Graph, Contacts, Association, S3State, Group } from "~/types"; +import { + GraphNode, + Graph, + Contacts, + Association, + GcpState, + S3State, + Group +} from "~/types"; interface NoteRoutesProps { ship: string; @@ -20,6 +28,7 @@ interface NoteRoutesProps { baseUrl?: string; rootUrl?: string; group: Group; + gcp: GcpState; s3: S3State; } diff --git a/pkg/interface/src/views/apps/publish/components/NotebookRoutes.tsx b/pkg/interface/src/views/apps/publish/components/NotebookRoutes.tsx index a3d1f5474f..a134613fe5 100644 --- a/pkg/interface/src/views/apps/publish/components/NotebookRoutes.tsx +++ b/pkg/interface/src/views/apps/publish/components/NotebookRoutes.tsx @@ -9,6 +9,7 @@ import { Contacts, Rolodex, Unreads, + GcpState, S3State } from "~/types"; import { Center, LoadingSpinner } from "@tlon/indigo-react"; @@ -33,6 +34,7 @@ interface NotebookRoutesProps { rootUrl: string; association: Association; associations: Associations; + gcp: GcpState; s3: S3State; } @@ -80,6 +82,7 @@ export function NotebookRoutes( association={props.association} graph={graph} baseUrl={baseUrl} + gcp={props.gcp} s3={props.s3} /> )} @@ -112,6 +115,7 @@ export function NotebookRoutes( contacts={notebookContacts} association={props.association} group={group} + gcp={props.gcp} s3={props.s3} {...routeProps} /> diff --git a/pkg/interface/src/views/apps/publish/components/new-post.tsx b/pkg/interface/src/views/apps/publish/components/new-post.tsx index 2b0dd366a2..c56410e2fd 100644 --- a/pkg/interface/src/views/apps/publish/components/new-post.tsx +++ b/pkg/interface/src/views/apps/publish/components/new-post.tsx @@ -6,7 +6,7 @@ import { RouteComponentProps } from "react-router-dom"; import { PostForm, PostFormSchema } from "./NoteForm"; import {createPost} from "~/logic/api/graph"; import {Graph} from "~/types/graph-update"; -import {Association, S3State} from "~/types"; +import {Association, GcpState, S3State} from "~/types"; import {newPost} from "~/logic/lib/publish"; interface NewPostProps { @@ -16,6 +16,7 @@ interface NewPostProps { graph: Graph; association: Association; baseUrl: string; + gcp: GcpState; s3: S3State; } @@ -53,6 +54,7 @@ export default function NewPost(props: NewPostProps & RouteComponentProps) { onSubmit={onSubmit} submitLabel="Publish" loadingText="Posting..." + gcp={props.gcp} s3={props.s3} /> ); diff --git a/pkg/interface/src/views/apps/settings/components/lib/BackgroundPicker.tsx b/pkg/interface/src/views/apps/settings/components/lib/BackgroundPicker.tsx index 53aed1b3e9..e90c757490 100644 --- a/pkg/interface/src/views/apps/settings/components/lib/BackgroundPicker.tsx +++ b/pkg/interface/src/views/apps/settings/components/lib/BackgroundPicker.tsx @@ -9,7 +9,7 @@ import { } from "@tlon/indigo-react"; import GlobalApi from "~/logic/api/global"; -import { S3State } from "~/types"; +import { GcpState, S3State } from "~/types"; import { ImageInput } from "~/views/components/ImageInput"; import {ColorInput} from "~/views/components/ColorInput"; @@ -19,11 +19,13 @@ export function BackgroundPicker({ bgType, bgUrl, api, + gcp, s3, }: { bgType: BgType; bgUrl?: string; api: GlobalApi; + gcp: GcpState; s3: S3State; }) { @@ -38,6 +40,7 @@ export function BackgroundPicker({ diff --git a/pkg/interface/src/views/landscape/components/GroupSettings/Admin.tsx b/pkg/interface/src/views/landscape/components/GroupSettings/Admin.tsx index 34fc6f9abf..a9ac19856b 100644 --- a/pkg/interface/src/views/landscape/components/GroupSettings/Admin.tsx +++ b/pkg/interface/src/views/landscape/components/GroupSettings/Admin.tsx @@ -22,7 +22,7 @@ import { ColorInput } from "~/views/components/ColorInput"; import { useHistory } from "react-router-dom"; import { uxToHex } from "~/logic/lib/util"; -import {S3State} from "~/types"; +import {GcpState, S3State} from "~/types"; import {ImageInput} from "~/views/components/ImageInput"; interface FormSchema { @@ -46,11 +46,12 @@ interface GroupAdminSettingsProps { group: Group; association: Association; api: GlobalApi; + gcp: GcpState; s3: S3State; } export function GroupAdminSettings(props: GroupAdminSettingsProps) { - const { group, association, s3 } = props; + const { group, association, gcp, s3 } = props; const { metadata } = association; const history = useHistory(); const currentPrivate = "invite" in props.group.policy; @@ -132,6 +133,7 @@ export function GroupAdminSettings(props: GroupAdminSettingsProps) { caption="A picture for your group" placeholder="Enter URL" disabled={disabled} + gcp={gcp} s3={s3} /> )} From 92dd46b176bc65da137017ff1507e27c2c83f2f3 Mon Sep 17 00:00:00 2001 From: J Date: Fri, 26 Feb 2021 00:49:57 +0000 Subject: [PATCH 062/103] GcpClient: remove unused BASE_URL --- pkg/interface/src/logic/lib/GcpClient.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/interface/src/logic/lib/GcpClient.ts b/pkg/interface/src/logic/lib/GcpClient.ts index a7f838b52e..d5060ccb88 100644 --- a/pkg/interface/src/logic/lib/GcpClient.ts +++ b/pkg/interface/src/logic/lib/GcpClient.ts @@ -10,7 +10,6 @@ import { const ENDPOINT = 'storage.googleapis.com'; -const BASE_URL = 'https://storage.googleapis.com/upload/storage/v1/b'; export default class GcpClient implements StorageClient { #accessKey: string; From 6ec574d32b8b3b6e184d853e0f686e8d0f5c26a7 Mon Sep 17 00:00:00 2001 From: J Date: Fri, 26 Feb 2021 20:18:07 +0000 Subject: [PATCH 063/103] interface: GcpManager fixes --- pkg/interface/src/logic/api/gcp.ts | 6 +++--- pkg/interface/src/logic/lib/gcpManager.ts | 5 ++--- pkg/interface/src/views/App.js | 2 +- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/pkg/interface/src/logic/api/gcp.ts b/pkg/interface/src/logic/api/gcp.ts index 4ed9e8f655..728b711782 100644 --- a/pkg/interface/src/logic/api/gcp.ts +++ b/pkg/interface/src/logic/api/gcp.ts @@ -12,12 +12,12 @@ export default class GcpApi extends BaseApi { data: token }); - if (token['gcp-token'] !== undefined && - token['gcp-token']['expiresIn'] !== undefined && + if (typeof(token) === 'object' && + typeof(token['gcp-token']) === 'object' && typeof(token['gcp-token']['expiresIn']) === 'number') { return Promise.resolve(token['gcp-token']['expiresIn']); } - return Promise.reject({reason: 'invalid token'}); + return Promise.reject(new Error("invalid token")); }); } }; diff --git a/pkg/interface/src/logic/lib/gcpManager.ts b/pkg/interface/src/logic/lib/gcpManager.ts index 97016a7efd..2ed03079ac 100644 --- a/pkg/interface/src/logic/lib/gcpManager.ts +++ b/pkg/interface/src/logic/lib/gcpManager.ts @@ -52,16 +52,15 @@ class GcpManager { } private refreshLoop() { - console.log('GcpManager refreshing token'); this.#api.gcp.refreshToken() .then( (expiresIn: number) => { this.refreshAfter(this.refreshInterval(expiresIn)); }) .catch( - ({reason}) => { + (reason) => { console.error('GcpManager token refresh failed', reason); - this.refreshAfter(10_000); // XX backoff? + this.refreshAfter(30_000); // XX backoff? }); } diff --git a/pkg/interface/src/views/App.js b/pkg/interface/src/views/App.js index 67147819a1..78a7cbea9f 100644 --- a/pkg/interface/src/views/App.js +++ b/pkg/interface/src/views/App.js @@ -96,10 +96,10 @@ class App extends React.Component { // before the app has actually rendered, hence the timeout. this.updateTheme(this.themeWatcher); }, 500); - gcpManager.start(); this.api.local.getBaseHash(); this.api.settings.getAll(); this.store.rehydrate(); + gcpManager.start(); Mousetrap.bindGlobal(['command+/', 'ctrl+/'], (e) => { e.preventDefault(); e.stopImmediatePropagation(); From 6b3397bd9fa9ceb52fd8f02e8630c036bd0d49f1 Mon Sep 17 00:00:00 2001 From: J Date: Fri, 26 Feb 2021 20:18:40 +0000 Subject: [PATCH 064/103] interface: supports both S3 and GCP Storage The S3 client has another layer of indirection we missed. To support it expediently, we just make the promise() method on GcpUpload do all the work in GcpClient. --- pkg/interface/src/logic/lib/GcpClient.ts | 27 +++++++++++++++++--- pkg/interface/src/logic/lib/StorageClient.ts | 9 +++++-- pkg/interface/src/logic/lib/useStorage.ts | 27 ++++++++------------ 3 files changed, 42 insertions(+), 21 deletions(-) diff --git a/pkg/interface/src/logic/lib/GcpClient.ts b/pkg/interface/src/logic/lib/GcpClient.ts index d5060ccb88..f071896979 100644 --- a/pkg/interface/src/logic/lib/GcpClient.ts +++ b/pkg/interface/src/logic/lib/GcpClient.ts @@ -1,9 +1,15 @@ // Very simple GCP Storage client. // +// It's designed to match a subset of the S3 client upload API. The upload +// function on S3 returns a ManagedUpload, which has a promise() method on +// it. We don't care about any of the other methods on ManagedUpload, so we +// just do the work in its promise() method. +// import querystring from 'querystring'; import { StorageAcl, StorageClient, + StorageUpload, UploadParams, UploadResult } from './StorageClient'; @@ -11,14 +17,17 @@ import { const ENDPOINT = 'storage.googleapis.com'; -export default class GcpClient implements StorageClient { +class GcpUpload implements StorageUpload { + #params: UploadParams; #accessKey: string; - constructor(accessKey: string) { + constructor(params: UploadParams, accessKey: string) { + this.#params = params; this.#accessKey = accessKey; } - async upload({Bucket, Key, ContentType, Body}: UploadParams): UploadResult { + async promise(): UploadResult { + const {Bucket, Key, ContentType, Body} = this.#params; const urlParams = { uploadType: 'media', name: Key, @@ -42,3 +51,15 @@ export default class GcpClient implements StorageClient { return {Location: `https://${ENDPOINT}/${Bucket}/${Key}`}; } } + +export default class GcpClient implements StorageClient { + #accessKey: string; + + constructor(accessKey: string) { + this.#accessKey = accessKey; + } + + upload(params: UploadParams): StorageUpload { + return new GcpUpload(params, this.#accessKey); + } +} diff --git a/pkg/interface/src/logic/lib/StorageClient.ts b/pkg/interface/src/logic/lib/StorageClient.ts index 16cc79087a..31e12f8233 100644 --- a/pkg/interface/src/logic/lib/StorageClient.ts +++ b/pkg/interface/src/logic/lib/StorageClient.ts @@ -22,6 +22,11 @@ export interface UploadResult { Location: string; }; -export interface StorageClient { - upload(params: UploadParams): Promise; +// Extra layer of indirection used by S3 client. +export interface StorageUpload { + promise(): Promise; +}; + +export interface StorageClient { + upload(params: UploadParams): StorageUpload; }; diff --git a/pkg/interface/src/logic/lib/useStorage.ts b/pkg/interface/src/logic/lib/useStorage.ts index a011c17275..2fd01932e2 100644 --- a/pkg/interface/src/logic/lib/useStorage.ts +++ b/pkg/interface/src/logic/lib/useStorage.ts @@ -19,37 +19,32 @@ const useStorage = (s3: S3State, gcp: GcpState, { accept = '*' } = { accept: '*' }): IuseStorage => { const [uploading, setUploading] = useState(false); - const gcpClient = useRef(null); - const s3Client = useRef(null); + const client = useRef(null); useEffect(() => { // prefer GCP if available, else use S3. if (gcp.accessKey) { - gcpClient.current = new GcpClient(gcp.accessKey); - s3Client.current = null; + client.current = new GcpClient(gcp.accessKey); } else { if (!s3.credentials) { return; } - s3Client.current = new S3({ + client.current = new S3({ credentials: s3.credentials, endpoint: s3.credentials.endpoint }); - gcpClient.current = null; } }, [gcp.accessKey, s3.credentials]); const canUpload = useMemo( () => - ((gcpClient || s3Client) && s3.configuration.currentBucket !== "") || false, - [gcpClient, s3Client, s3.configuration.currentBucket] + (client && s3.configuration.currentBucket !== "") || false, + [client, s3.configuration.currentBucket] ); const upload = useCallback( async (file: File, bucket: string) => { - const client: StorageClient | null = - gcpClient.current || s3Client.current; - if (!client) { + if (client.current === null) { throw new Error("Storage not ready"); } @@ -68,21 +63,21 @@ const useStorage = (s3: S3State, gcp: GcpState, setUploading(true); - const { Location } = await client.upload(params); + const { Location } = await client.current.upload(params).promise(); setUploading(false); return Location; }, - [gcpClient, s3Client, setUploading] + [client, setUploading] ); const uploadDefault = useCallback(async (file: File) => { - if (s3.configuration.currentBucket == '') { - throw new Error('current bucket not set'); + if (s3.configuration.currentBucket === '') { + throw new Error("current bucket not set"); } return upload(file, s3.configuration.currentBucket); - }, [s3.configuration, upload]); + }, [s3, upload]); const promptUpload = useCallback( () => { From 723a5a050e2535164e9dd890b2c37bc353ee7d20 Mon Sep 17 00:00:00 2001 From: J Date: Fri, 26 Feb 2021 20:29:05 +0000 Subject: [PATCH 065/103] interface: check accessKey !== undefined Also make it type-level optional rather than using explicit null. Perhaps one day we will want to use undefined to denote "the thread hasn't returned yet" and null for "GCP Storage is not configured." Perhaps. --- pkg/interface/src/logic/lib/useStorage.ts | 2 +- pkg/interface/src/logic/store/store.ts | 4 +--- pkg/interface/src/types/gcp-state.ts | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/pkg/interface/src/logic/lib/useStorage.ts b/pkg/interface/src/logic/lib/useStorage.ts index 2fd01932e2..a85549e006 100644 --- a/pkg/interface/src/logic/lib/useStorage.ts +++ b/pkg/interface/src/logic/lib/useStorage.ts @@ -23,7 +23,7 @@ const useStorage = (s3: S3State, gcp: GcpState, useEffect(() => { // prefer GCP if available, else use S3. - if (gcp.accessKey) { + if (gcp.accessKey !== undefined) { client.current = new GcpClient(gcp.accessKey); } else { if (!s3.credentials) { diff --git a/pkg/interface/src/logic/store/store.ts b/pkg/interface/src/logic/store/store.ts index 85d1ef0e12..af6708747f 100644 --- a/pkg/interface/src/logic/store/store.ts +++ b/pkg/interface/src/logic/store/store.ts @@ -73,9 +73,7 @@ export default class GlobalStore extends BaseStore { }, weather: {}, userLocation: null, - gcp: { - accessKey: null - }, + gcp: {}, s3: { configuration: { buckets: new Set(), diff --git a/pkg/interface/src/types/gcp-state.ts b/pkg/interface/src/types/gcp-state.ts index 41ebe12501..976214af42 100644 --- a/pkg/interface/src/types/gcp-state.ts +++ b/pkg/interface/src/types/gcp-state.ts @@ -4,5 +4,5 @@ export interface GcpToken { }; export interface GcpState { - accessKey: string; + accessKey?: string; }; From 347d51fde946f30552d50a58d42d22d8ab69bb3e Mon Sep 17 00:00:00 2001 From: J Date: Fri, 26 Feb 2021 22:10:15 +0000 Subject: [PATCH 066/103] interface: cleaner API, robust GcpManager retry GcpApi now acts like other APIs. Since GcpManager can no longer get at the token exipry by inspecting the raw update, it must depend on the global store instead. This also means it can check whether the user has configured S3, and not try to refresh the token in that case. In the case where no storage is configured, this will spam the console with request failures since the thread returns 500 if there is no token. Perhaps this is a good argument for making the thread return a unit. --- pkg/interface/src/logic/api/gcp.ts | 8 --- pkg/interface/src/logic/lib/gcpManager.ts | 72 ++++++++++++++----- pkg/interface/src/logic/reducers/gcp-token.ts | 14 ++-- pkg/interface/src/types/gcp-state.ts | 2 +- pkg/interface/src/views/App.js | 2 +- 5 files changed, 65 insertions(+), 33 deletions(-) diff --git a/pkg/interface/src/logic/api/gcp.ts b/pkg/interface/src/logic/api/gcp.ts index 728b711782..b86cbd8788 100644 --- a/pkg/interface/src/logic/api/gcp.ts +++ b/pkg/interface/src/logic/api/gcp.ts @@ -4,20 +4,12 @@ import {GcpToken} from '../types/gcp-state'; export default class GcpApi extends BaseApi { - // Return value resolves to the token's expiry time if successful. refreshToken() { return this.spider('noun', 'gcp-token', 'get-gcp-token', {}) .then((token) => { this.store.handleEvent({ data: token }); - - if (typeof(token) === 'object' && - typeof(token['gcp-token']) === 'object' && - typeof(token['gcp-token']['expiresIn']) === 'number') { - return Promise.resolve(token['gcp-token']['expiresIn']); - } - return Promise.reject(new Error("invalid token")); }); } }; diff --git a/pkg/interface/src/logic/lib/gcpManager.ts b/pkg/interface/src/logic/lib/gcpManager.ts index 2ed03079ac..813cf24c8b 100644 --- a/pkg/interface/src/logic/lib/gcpManager.ts +++ b/pkg/interface/src/logic/lib/gcpManager.ts @@ -2,18 +2,27 @@ // // To use: // -// 1. call setApi with a GlobalApi. +// 1. call configure with a GlobalApi and GlobalStore. // 2. call start() to start the token refresh loop. // +// If the ship has S3 credentials set, we don't try to get a token, but we keep +// checking at regular intervals to see if they get unset. Otherwise, we try to +// invoke the GCP token thread on the ship until it gives us an access token. +// Once we have a token, we refresh it every hour or so, since it has an +// intrinsic expiry. +// // import GlobalApi from '../api/global'; +import GlobalStore from '../store/store'; class GcpManager { #api: GlobalApi | null = null; + #store: GlobalStore | null = null; - setApi(api: GlobalApi) { + configure(api: GlobalApi, store: GlobalStore) { this.#api = api; + this.#store = store; } #running = false; @@ -24,8 +33,8 @@ class GcpManager { console.warn('GcpManager already running'); return; } - if (!this.#api) { - console.error('GcpManager must have api set'); + if (!this.#api || !this.#store) { + console.error('GcpManager must have api and store set'); return; } this.#running = true; @@ -51,17 +60,37 @@ class GcpManager { this.start(); } + #consecutiveFailures: number = 0; + private refreshLoop() { + const s3 = this.#store.state.s3; + // XX ships currently always have S3 credentials, but the fields are all + // set to '' if they are not configured. + if (s3 && + s3.credentials && + s3.credentials.accessKeyId && + s3.credentials.secretAccessKey) { + // do nothing, and check again in 5s. + this.refreshAfter(5_000); + return; + } this.#api.gcp.refreshToken() - .then( - (expiresIn: number) => { - this.refreshAfter(this.refreshInterval(expiresIn)); - }) - .catch( - (reason) => { - console.error('GcpManager token refresh failed', reason); - this.refreshAfter(30_000); // XX backoff? - }); + .then(() => { + const token = this.#store.state.gcp?.token; + if (token) { + this.#consecutiveFailures = 0; + const interval = this.refreshInterval(token.expiresIn); + console.log('GcpManager got token; refreshing after', interval); + this.refreshAfter(interval); + } else { + throw new Error('thread succeeded, but returned no token?'); + } + }) + .catch((reason) => { + this.#consecutiveFailures++; + console.warn('GcpManager refresh failed; retrying with backoff'); + this.refreshAfter(this.backoffInterval()); + }); } private refreshAfter(durationMs) { @@ -71,7 +100,6 @@ class GcpManager { console.warn('GcpManager already has a timeout set'); return; } - console.log('GcpManager refreshing after', durationMs, 'ms'); this.#timeoutId = setTimeout(() => { this.#timeoutId = null; this.refreshLoop(); @@ -79,10 +107,18 @@ class GcpManager { } private refreshInterval(expiresIn: number) { - // Give ourselves 30 seconds for processing delays, but never refresh - // sooner than 30 minutes from now. (The expiry window should be about an - // hour.) - return Math.max(30 * 60_000, expiresIn - 30_000); + // Give ourselves a minute for processing delays, but never refresh sooner + // than 30 minutes from now. (The expiry window should be about an hour.) + return Math.max(30 * 60_000, expiresIn - 60_000); + } + + private backoffInterval() { + // exponential backoff. + const slotMs = 5_000; + const maxSlot = 60; // 5 minutes + const backoffSlots = + Math.floor(Math.random() * Math.min(maxSlot, this.#consecutiveFailures)); + return slotMs * backoffSlots; } } diff --git a/pkg/interface/src/logic/reducers/gcp-token.ts b/pkg/interface/src/logic/reducers/gcp-token.ts index 98ef0b98fa..d9ecc3969c 100644 --- a/pkg/interface/src/logic/reducers/gcp-token.ts +++ b/pkg/interface/src/logic/reducers/gcp-token.ts @@ -8,14 +8,18 @@ export default class GcpReducer{ reduce(json: Cage, state: S) { let data = json['gcp-token']; if (data) { - this.setAccessKey(data, state); + this.setToken(data, state); } } - setAccessKey(json: GcpToken, state: S) { - const data = _.get(json, 'accessKey'); - if (data) { - state.gcp.accessKey = data; + setToken(data: any, state: S) { + if (this.isToken(data)) { + state.gcp.token = data; } } + + isToken(token: any): token is GcpToken { + return (typeof(token.accessKey) === 'string' && + typeof(token.expiresIn) === 'number'); + } } diff --git a/pkg/interface/src/types/gcp-state.ts b/pkg/interface/src/types/gcp-state.ts index 976214af42..e45a37011a 100644 --- a/pkg/interface/src/types/gcp-state.ts +++ b/pkg/interface/src/types/gcp-state.ts @@ -4,5 +4,5 @@ export interface GcpToken { }; export interface GcpState { - accessKey?: string; + token?: GcpToken }; diff --git a/pkg/interface/src/views/App.js b/pkg/interface/src/views/App.js index 78a7cbea9f..4089ec1eae 100644 --- a/pkg/interface/src/views/App.js +++ b/pkg/interface/src/views/App.js @@ -79,7 +79,7 @@ class App extends React.Component { this.appChannel = new window.channel(); this.api = new GlobalApi(this.ship, this.appChannel, this.store); - gcpManager.setApi(this.api); + gcpManager.configure(this.api, this.store); this.subscription = new GlobalSubscription(this.store, this.api, this.appChannel); From 040d8c06aed3d7ed078714ee741bee06832fd882 Mon Sep 17 00:00:00 2001 From: J Date: Fri, 26 Feb 2021 22:12:52 +0000 Subject: [PATCH 067/103] interface: fix S3 canUpload logic s3.credentials is always set, but the fields are all '' if s3 is not configured. So prior to this change, if the user had configured an active bucket, they would always be shown an upload dialog even if they had no storage configured. --- pkg/interface/src/logic/lib/useStorage.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/pkg/interface/src/logic/lib/useStorage.ts b/pkg/interface/src/logic/lib/useStorage.ts index a85549e006..ec71f96cbf 100644 --- a/pkg/interface/src/logic/lib/useStorage.ts +++ b/pkg/interface/src/logic/lib/useStorage.ts @@ -23,10 +23,14 @@ const useStorage = (s3: S3State, gcp: GcpState, useEffect(() => { // prefer GCP if available, else use S3. - if (gcp.accessKey !== undefined) { - client.current = new GcpClient(gcp.accessKey); + if (gcp.current !== undefined) { + client.current = new GcpClient(gcp.current.accessKey); } else { - if (!s3.credentials) { + // XX ships currently always have S3 credentials, but the fields are all + // set to '' if they are not configured. + if (!s3.credentials || + !s3.credentials.accessKeyId || + !s3.credentials.secretAccessKey) { return; } client.current = new S3({ @@ -38,8 +42,8 @@ const useStorage = (s3: S3State, gcp: GcpState, const canUpload = useMemo( () => - (client && s3.configuration.currentBucket !== "") || false, - [client, s3.configuration.currentBucket] + (client.current && s3.configuration.currentBucket !== "") || false, + [client.current, s3.configuration.currentBucket] ); const upload = useCallback( From c1b259af5baa3cda74c1af956be9f90b48facb27 Mon Sep 17 00:00:00 2001 From: J Date: Fri, 26 Feb 2021 23:42:57 +0000 Subject: [PATCH 068/103] interface, gcp: don't spam the console with 500s Options here were: make the get-token thread try to return a unit, or write a different thread to check whether GCP Storage seemed to be configured and poll on that thread, or move the whole thing into a Gall app. The compromise between time-to-implement and overall cleanliness seemed to be to write a different thread that just checks whether the settings fields have been poked. Unfortunately this means GcpManager is now a somewhat hefty JavaScript state machine. Took out the logic to check whether S3 was configured in GcpManager, since it was really only there to prevent spamming the console with 500s. If you have both S3 and GCP Storage configured for some reason, you will now use GCP per the logic in useStorage. --- .../get-token.hoon} | 0 pkg/arvo/ted/gcp/is-configured.hoon | 49 +++++++++++++++++++ pkg/interface/src/logic/api/gcp.ts | 13 ++++- pkg/interface/src/logic/lib/gcpManager.ts | 33 ++++++++----- .../reducers/{gcp-token.ts => gcp-reducer.ts} | 12 +++++ pkg/interface/src/logic/store/store.ts | 2 +- pkg/interface/src/types/gcp-state.ts | 1 + 7 files changed, 96 insertions(+), 14 deletions(-) rename pkg/arvo/ted/{get-gcp-token.hoon => gcp/get-token.hoon} (100%) create mode 100644 pkg/arvo/ted/gcp/is-configured.hoon rename pkg/interface/src/logic/reducers/{gcp-token.ts => gcp-reducer.ts} (69%) diff --git a/pkg/arvo/ted/get-gcp-token.hoon b/pkg/arvo/ted/gcp/get-token.hoon similarity index 100% rename from pkg/arvo/ted/get-gcp-token.hoon rename to pkg/arvo/ted/gcp/get-token.hoon diff --git a/pkg/arvo/ted/gcp/is-configured.hoon b/pkg/arvo/ted/gcp/is-configured.hoon new file mode 100644 index 0000000000..02b67e60b3 --- /dev/null +++ b/pkg/arvo/ted/gcp/is-configured.hoon @@ -0,0 +1,49 @@ +:: Tells whether GCP Storage appears to be configured. +:: +:: Thread since it needs to be called from Landscape. +:: +:: +/- gcp, spider, settings +/+ strandio +=, strand=strand:spider +=, enjs:format +^- thread:spider +|^ +|= * +=/ m (strand ,vase) +^- form:m +;< has=? bind:m + %: has-settings + %client-email + %private-key + %private-key-id + %token-uri + ~ + == +%- pure:m +!> +%+ frond %gcp-configured +b+has +:: +++ has-settings + |= set=(list @tas) + =/ m (strand ?) + ^- form:m + ?~ set + (pure:m %.y) + ;< has=? bind:m (has-setting i.set) + ?. has + (pure:m %.n) + ;< has=? bind:m (has-settings t.set) + (pure:m has) +:: +++ has-setting + |= key=@tas + =/ m (strand ?) + ^- form:m + ;< has=? bind:m + %+ scry:strandio ? + /gx/settings-store/has-entry/gcp-store/[key]/noun + (pure:m has) +:: +-- diff --git a/pkg/interface/src/logic/api/gcp.ts b/pkg/interface/src/logic/api/gcp.ts index b86cbd8788..c6b955b22d 100644 --- a/pkg/interface/src/logic/api/gcp.ts +++ b/pkg/interface/src/logic/api/gcp.ts @@ -4,8 +4,17 @@ import {GcpToken} from '../types/gcp-state'; export default class GcpApi extends BaseApi { - refreshToken() { - return this.spider('noun', 'gcp-token', 'get-gcp-token', {}) + isConfigured() { + return this.spider('noun', 'json', 'gcp-is-configured', {}) + .then((data) => { + this.store.handleEvent({ + data + }); + }); + } + + getToken() { + return this.spider('noun', 'gcp-token', 'gcp-get-token', {}) .then((token) => { this.store.handleEvent({ data: token diff --git a/pkg/interface/src/logic/lib/gcpManager.ts b/pkg/interface/src/logic/lib/gcpManager.ts index 813cf24c8b..4fba6c0cca 100644 --- a/pkg/interface/src/logic/lib/gcpManager.ts +++ b/pkg/interface/src/logic/lib/gcpManager.ts @@ -62,19 +62,30 @@ class GcpManager { #consecutiveFailures: number = 0; + private isConfigured() { + return this.#store.state.gcp.configured; + } + private refreshLoop() { - const s3 = this.#store.state.s3; - // XX ships currently always have S3 credentials, but the fields are all - // set to '' if they are not configured. - if (s3 && - s3.credentials && - s3.credentials.accessKeyId && - s3.credentials.secretAccessKey) { - // do nothing, and check again in 5s. - this.refreshAfter(5_000); + if (!this.isConfigured()) { + this.#api.gcp.isConfigured() + .then(() => { + if (this.isConfigured() === undefined) { + throw new Error("can't check whether GCP is configured?"); + } + if (this.isConfigured()) { + this.refreshLoop(); + } else { + this.refreshAfter(10_000); + } + }) + .catch((reason) => { + console.error('GcpManager failure; stopping.', reason); + this.stop(); + }); return; } - this.#api.gcp.refreshToken() + this.#api.gcp.getToken() .then(() => { const token = this.#store.state.gcp?.token; if (token) { @@ -88,7 +99,7 @@ class GcpManager { }) .catch((reason) => { this.#consecutiveFailures++; - console.warn('GcpManager refresh failed; retrying with backoff'); + console.warn('GcpManager token refresh failed; retrying with backoff'); this.refreshAfter(this.backoffInterval()); }); } diff --git a/pkg/interface/src/logic/reducers/gcp-token.ts b/pkg/interface/src/logic/reducers/gcp-reducer.ts similarity index 69% rename from pkg/interface/src/logic/reducers/gcp-token.ts rename to pkg/interface/src/logic/reducers/gcp-reducer.ts index d9ecc3969c..685dfbef22 100644 --- a/pkg/interface/src/logic/reducers/gcp-token.ts +++ b/pkg/interface/src/logic/reducers/gcp-reducer.ts @@ -6,6 +6,18 @@ type GcpState = Pick; export default class GcpReducer{ reduce(json: Cage, state: S) { + this.reduceConfigured(json, state); + this.reduceToken(json, state); + } + + reduceConfigured(json, state) { + let data = json['gcp-configured']; + if (data !== undefined) { + state.gcp.configured = data; + } + } + + reduceToken(json: Cage, state: S) { let data = json['gcp-token']; if (data) { this.setToken(data, state); diff --git a/pkg/interface/src/logic/store/store.ts b/pkg/interface/src/logic/store/store.ts index af6708747f..b877b68368 100644 --- a/pkg/interface/src/logic/store/store.ts +++ b/pkg/interface/src/logic/store/store.ts @@ -16,7 +16,7 @@ import GroupReducer from '../reducers/group-update'; import LaunchReducer from '../reducers/launch-update'; import ConnectionReducer from '../reducers/connection'; import SettingsReducer from '../reducers/settings-update'; -import GcpReducer from '../reducers/gcp-token'; +import GcpReducer from '../reducers/gcp-reducer'; import {OrderedMap} from '../lib/OrderedMap'; import { BigIntOrderedMap } from '../lib/BigIntOrderedMap'; import {GroupViewReducer} from '../reducers/group-view'; diff --git a/pkg/interface/src/types/gcp-state.ts b/pkg/interface/src/types/gcp-state.ts index e45a37011a..cdbcbc1c97 100644 --- a/pkg/interface/src/types/gcp-state.ts +++ b/pkg/interface/src/types/gcp-state.ts @@ -4,5 +4,6 @@ export interface GcpToken { }; export interface GcpState { + configured?: boolean; token?: GcpToken }; From d8a7ee56e137190418971c759226e9bb632b5e13 Mon Sep 17 00:00:00 2001 From: J Date: Sat, 27 Feb 2021 00:08:51 +0000 Subject: [PATCH 069/103] interface: fix field name that broke GCP support Tests would be great, wouldn't they... --- pkg/interface/src/logic/lib/useStorage.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/interface/src/logic/lib/useStorage.ts b/pkg/interface/src/logic/lib/useStorage.ts index ec71f96cbf..32442d8e1d 100644 --- a/pkg/interface/src/logic/lib/useStorage.ts +++ b/pkg/interface/src/logic/lib/useStorage.ts @@ -23,8 +23,8 @@ const useStorage = (s3: S3State, gcp: GcpState, useEffect(() => { // prefer GCP if available, else use S3. - if (gcp.current !== undefined) { - client.current = new GcpClient(gcp.current.accessKey); + if (gcp.token !== undefined) { + client.current = new GcpClient(gcp.token.accessKey); } else { // XX ships currently always have S3 credentials, but the fields are all // set to '' if they are not configured. From c66c1979361c27aebcd9cb350952be1708546845 Mon Sep 17 00:00:00 2001 From: J Date: Sat, 27 Feb 2021 00:25:17 +0000 Subject: [PATCH 070/103] interface: fix regression in canUpload --- pkg/interface/src/logic/lib/useStorage.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/interface/src/logic/lib/useStorage.ts b/pkg/interface/src/logic/lib/useStorage.ts index 32442d8e1d..2ea88b9a8c 100644 --- a/pkg/interface/src/logic/lib/useStorage.ts +++ b/pkg/interface/src/logic/lib/useStorage.ts @@ -42,8 +42,10 @@ const useStorage = (s3: S3State, gcp: GcpState, const canUpload = useMemo( () => - (client.current && s3.configuration.currentBucket !== "") || false, - [client.current, s3.configuration.currentBucket] + ((gcp.token || (s3.credentials && s3.credentials.accessKeyId && + s3.credentials.secretAccessKey)) && + s3.configuration.currentBucket !== "") || false, + [s3.credentials, gcp.token, s3.configuration.currentBucket] ); const upload = useCallback( From c7d86015aa0b525f921c669071b57c53a7864d00 Mon Sep 17 00:00:00 2001 From: fang Date: Sat, 27 Feb 2021 01:41:16 +0100 Subject: [PATCH 071/103] dbug: assume new lane encoding --- pkg/arvo/app/dbug.hoon | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/arvo/app/dbug.hoon b/pkg/arvo/app/dbug.hoon index f7132f05f9..e2c340ce29 100644 --- a/pkg/arvo/app/dbug.hoon +++ b/pkg/arvo/app/dbug.hoon @@ -593,10 +593,10 @@ %& (ship p.lane) :: %| - ?~ l=((soft ,[=@tas =@if =@ud]) (cue p.lane)) - s+(scot %x p.lane) - =, u.l - (tape "%{(trip tas)}, {(scow %if if)}, {(scow %ud ud)}") + %- tape + =/ ip=@if (end [0 32] p.lane) + =/ pt=@ud (cut 0 [32 16] p.lane) + "{(scow %if ip)}:{((d-co:co 1) pt)} ({(scow %ux p.lane)})" == == :: From 2442f9bb93052d0d4818642d43888280731959c3 Mon Sep 17 00:00:00 2001 From: Ted Blackman Date: Fri, 26 Feb 2021 21:42:22 -0500 Subject: [PATCH 072/103] zuse: fix ordered-map delete many --- pkg/arvo/sys/zuse.hoon | 36 ++++++++++++++++++------ pkg/arvo/tests/sys/zuse/ordered-map.hoon | 8 ++++++ 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/pkg/arvo/sys/zuse.hoon b/pkg/arvo/sys/zuse.hoon index 5c84367904..b57e02e99f 100644 --- a/pkg/arvo/sys/zuse.hoon +++ b/pkg/arvo/sys/zuse.hoon @@ -5207,6 +5207,20 @@ ?: (mor key.n.l.a key.n.r.a) l.a(r $(l.a r.l.a)) r.a(l $(r.a l.r.a)) + :: +nup: delete root, reporting which side was promoted + :: + ++ nup + |= a=(tree item) + ^- [pro=?(%l %r) res=(tree item)] + :: + ?> ?=(^ a) + :: delete .n.a; merge and balance .l.a and .r.a + :: + ?~ l.a r/r.a + ?~ r.a l/l.a + ?: (mor key.n.l.a key.n.r.a) + l/l.a(r (nip a(l r.l.a))) + r/r.a(l (nip a(r l.r.a))) :: +traverse: stateful partial inorder traversal :: :: Mutates .state on each run of .f. Starts at .start key, or if @@ -5255,18 +5269,22 @@ =^ res acc ?> ?=(^ a) (f state.acc n.a) - :: apply update to .a from .f's product + :: if we kept the node, replace its .val; order is unchanged :: - =. a - :: if .f requested node deletion, merge and balance .l.a and .r.a - :: - ?~ res (nip a) - :: we kept the node; replace its .val; order is unchanged - :: + ?^ res ?> ?=(^ a) - a(val.n u.res) + ..node(val.n.a u.res) + :: .f requested node deletion; delete root, then maybe recurse :: - ..node + :: If +nup promoted the left side, we're done; don't process + :: the left-hand side twice. If +nup promoted a non-null right + :: side, recurse on the new root. + :: + =^ pro a (nup a) + ?- pro + %l ..node + %r ?~(a ..node node) + == :: +left: recurse on left subtree, copying mutant back into .l.a :: ++ left diff --git a/pkg/arvo/tests/sys/zuse/ordered-map.hoon b/pkg/arvo/tests/sys/zuse/ordered-map.hoon index ad9f24a90f..d61ce8268a 100644 --- a/pkg/arvo/tests/sys/zuse/ordered-map.hoon +++ b/pkg/arvo/tests/sys/zuse/ordered-map.hoon @@ -129,6 +129,14 @@ !> -.b == :: +++ test-ordered-map-traverse-delete-all ^- tang + =/ q ((ordered-map ,@ ,~) lte) + =/ o (gas:q ~ ~[1/~ 2/~ 3/~]) + =/ b ((traverse:q ,~) o ~ |=([~ key=@ ~] [~ %| ~])) + %+ expect-eq + !> [~ ~] + !> b +:: ++ test-ordered-map-uni ^- tang :: =/ a=(tree [@ud @tas]) (gas:atom-map ~ (scag 4 test-items)) From fcbb70de9eae3499a290bd5012bd17dbd7392ed2 Mon Sep 17 00:00:00 2001 From: Ted Blackman Date: Fri, 26 Feb 2021 22:37:11 -0500 Subject: [PATCH 073/103] zuse: actually fix ordered-map +traverse this time --- pkg/arvo/sys/zuse.hoon | 53 ++++++++++-------------- pkg/arvo/tests/sys/zuse/ordered-map.hoon | 28 ++++++++++--- 2 files changed, 43 insertions(+), 38 deletions(-) diff --git a/pkg/arvo/sys/zuse.hoon b/pkg/arvo/sys/zuse.hoon index b57e02e99f..9d833a43b5 100644 --- a/pkg/arvo/sys/zuse.hoon +++ b/pkg/arvo/sys/zuse.hoon @@ -5244,59 +5244,48 @@ =/ acc [stop=`?`%.n state=state] =< abet =< main |% + ++ this . ++ abet [state.acc a] :: +main: main recursive loop; performs a partial inorder traversal :: ++ main - ^+ . + ^+ this :: stop if empty or we've been told to stop :: - ?~ a . - ?: stop.acc . + ?: =(~ a) this + ?: stop.acc this :: inorder traversal: left -> node -> right, until .f sets .stop :: - => left - ?: stop.acc . - => node - ?: stop.acc . - right + =. this left + ?: stop.acc this + =^ del this node + ?: stop.acc this + =. this right + =? a del (nip a) + this :: +node: run .f on .n.a, updating .a, .state, and .stop :: ++ node - ^+ . + ^+ [del=*? this] :: run .f on node, updating .stop.acc and .state.acc :: - =^ res acc - ?> ?=(^ a) - (f state.acc n.a) - :: if we kept the node, replace its .val; order is unchanged - :: - ?^ res - ?> ?=(^ a) - ..node(val.n.a u.res) - :: .f requested node deletion; delete root, then maybe recurse - :: - :: If +nup promoted the left side, we're done; don't process - :: the left-hand side twice. If +nup promoted a non-null right - :: side, recurse on the new root. - :: - =^ pro a (nup a) - ?- pro - %l ..node - %r ?~(a ..node node) - == + ?> ?=(^ a) + =^ res acc (f state.acc n.a) + ?~ res + [del=& ..node] + [del=| ..node(val.n.a u.res)] :: +left: recurse on left subtree, copying mutant back into .l.a :: ++ left - ^+ . - ?~ a . + ^+ this + ?~ a this =/ lef main(a l.a) lef(a a(l a.lef)) :: +right: recurse on right subtree, copying mutant back into .r.a :: ++ right - ^+ . - ?~ a . + ^+ this + ?~ a this =/ rig main(a r.a) rig(a a(r a.rig)) -- diff --git a/pkg/arvo/tests/sys/zuse/ordered-map.hoon b/pkg/arvo/tests/sys/zuse/ordered-map.hoon index d61ce8268a..261b8f0851 100644 --- a/pkg/arvo/tests/sys/zuse/ordered-map.hoon +++ b/pkg/arvo/tests/sys/zuse/ordered-map.hoon @@ -130,12 +130,28 @@ == :: ++ test-ordered-map-traverse-delete-all ^- tang - =/ q ((ordered-map ,@ ,~) lte) - =/ o (gas:q ~ ~[1/~ 2/~ 3/~]) - =/ b ((traverse:q ,~) o ~ |=([~ key=@ ~] [~ %| ~])) - %+ expect-eq - !> [~ ~] - !> b + ;: weld + =/ q ((ordered-map ,@ ,~) lte) + =/ o (gas:q ~ ~[1/~ 2/~ 3/~]) + =/ b ((traverse:q ,~) o ~ |=([~ key=@ ~] [~ %| ~])) + %+ expect-eq + !> [~ ~] + !> b + :: + =/ c + :~ [[2.127 1] ~] [[2.127 2] ~] [[2.127 3] ~] + [[2.127 7] ~] [[2.127 8] ~] [[2.127 9] ~] + == + =/ compare + |= [[aa=@ ab=@] [ba=@ bb=@]] + ?:((lth aa ba) %.y ?:((gth aa ba) %.n (lte ab bb))) + =/ q ((ordered-map ,[@ @] ,~) compare) + =/ o (gas:q ~ c) + =/ b ((traverse:q ,~) o ~ |=([~ key=[@ @] ~] [~ %| ~])) + %+ expect-eq + !> [~ ~] + !> b + == :: ++ test-ordered-map-uni ^- tang :: From ccc704b22e81ab3673567ed499054d787dbd0569 Mon Sep 17 00:00:00 2001 From: Ted Blackman Date: Fri, 26 Feb 2021 22:38:04 -0500 Subject: [PATCH 074/103] zuse: remove unused +nup:ordered-map --- pkg/arvo/sys/zuse.hoon | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/pkg/arvo/sys/zuse.hoon b/pkg/arvo/sys/zuse.hoon index 9d833a43b5..d8fffbe78c 100644 --- a/pkg/arvo/sys/zuse.hoon +++ b/pkg/arvo/sys/zuse.hoon @@ -5207,20 +5207,6 @@ ?: (mor key.n.l.a key.n.r.a) l.a(r $(l.a r.l.a)) r.a(l $(r.a l.r.a)) - :: +nup: delete root, reporting which side was promoted - :: - ++ nup - |= a=(tree item) - ^- [pro=?(%l %r) res=(tree item)] - :: - ?> ?=(^ a) - :: delete .n.a; merge and balance .l.a and .r.a - :: - ?~ l.a r/r.a - ?~ r.a l/l.a - ?: (mor key.n.l.a key.n.r.a) - l/l.a(r (nip a(l r.l.a))) - r/r.a(l (nip a(r l.r.a))) :: +traverse: stateful partial inorder traversal :: :: Mutates .state on each run of .f. Starts at .start key, or if From 4c190eb2e9031b386ffc456792dc9edd4bd7972e Mon Sep 17 00:00:00 2001 From: Ted Blackman Date: Fri, 26 Feb 2021 22:39:41 -0500 Subject: [PATCH 075/103] zuse: ordered-map minor cleanup --- pkg/arvo/sys/zuse.hoon | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/arvo/sys/zuse.hoon b/pkg/arvo/sys/zuse.hoon index d8fffbe78c..ad827c5915 100644 --- a/pkg/arvo/sys/zuse.hoon +++ b/pkg/arvo/sys/zuse.hoon @@ -5258,8 +5258,8 @@ ?> ?=(^ a) =^ res acc (f state.acc n.a) ?~ res - [del=& ..node] - [del=| ..node(val.n.a u.res)] + [del=& this] + [del=| this(val.n.a u.res)] :: +left: recurse on left subtree, copying mutant back into .l.a :: ++ left From 5ac70ec974537ef85aac284bba874ebb4a6d729d Mon Sep 17 00:00:00 2001 From: Philip Monk Date: Fri, 26 Feb 2021 20:53:27 -0800 Subject: [PATCH 076/103] ames: reset num-live to actual number of live packets --- pkg/arvo/sys/vane/ames.hoon | 51 ++++++++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/pkg/arvo/sys/vane/ames.hoon b/pkg/arvo/sys/vane/ames.hoon index bc7272906b..ec3ccad528 100644 --- a/pkg/arvo/sys/vane/ames.hoon +++ b/pkg/arvo/sys/vane/ames.hoon @@ -829,7 +829,7 @@ :: lifecycle arms; mostly pass-throughs to the contained adult ames :: ++ scry scry:adult-core - ++ stay [%4 %larva queued-events ames-state.adult-gate] + ++ stay [%5 %larva queued-events ames-state.adult-gate] ++ load |= $= old $% $: %4 @@ -839,6 +839,13 @@ == [%adult state=_ames-state.adult-gate] == == + $: %5 + $% $: %larva + events=(qeu queued-event) + state=_ames-state.adult-gate + == + [%adult state=_ames-state.adult-gate] + == == == ?- old [%4 %adult *] (load:adult-core %4 state.old) @@ -848,6 +855,14 @@ =. queued-events events.old =. adult-gate (load:adult-core %4 state.old) larval-gate + :: + [%5 %adult *] (load:adult-core %5 state.old) + :: + [%5 %larva *] + ~> %slog.1^leaf/"ames: larva: load" + =. queued-events events.old + =. adult-gate (load:adult-core %5 state.old) + larval-gate == -- :: adult ames, after metamorphosis from larva @@ -919,13 +934,39 @@ [moves ames-gate] :: +stay: extract state before reload :: -++ stay [%4 %adult ames-state] +++ stay [%5 %adult ames-state] :: +load: load in old state after reload :: ++ load - |= old-state=[%4 ^ames-state] + |= $= old-state + $% [%4 ^ames-state] + [%5 ^ames-state] + == + |^ ^+ ames-gate + =? old-state ?=(%4 -.old-state) %5^(state-4-to-5 +.old-state) + :: + ?> ?=(%5 -.old-state) ames-gate(ames-state +.old-state) + :: + ++ state-4-to-5 + |= =^ames-state + ^- ^^ames-state + =. peers.ames-state + %- ~(run by peers.ames-state) + |= =ship-state + ?. ?=(%known -.ship-state) + ship-state + =. snd.ship-state + %- ~(run by snd.ship-state) + |= =message-pump-state + =* p packet-pump-state.message-pump-state + =. num-live.metrics.packet-pump-state.message-pump-state + ~(wyt in live.packet-pump-state.message-pump-state) + message-pump-state + ship-state + ames-state + -- :: +scry: dereference namespace :: ++ scry @@ -1923,7 +1964,9 @@ =+ ?~ dud ~ %. ~ %+ slog leaf+"ames: {} ack crashed {}" - ?.(msg.veb ~ tang.u.dud) + ?. msg.veb ~ + :- >[bone=bone message-num=message-num meat=meat]:shut-packet< + tang.u.dud (run-message-pump bone %hear [message-num +.meat]:shut-packet) :: +on-memo: handle request to send message :: From 413413b70cb6f505dfd060c8f14a29406637440b Mon Sep 17 00:00:00 2001 From: Philip Monk Date: Fri, 26 Feb 2021 21:03:20 -0800 Subject: [PATCH 077/103] zuse: traverse always delete --- pkg/arvo/sys/vane/ames.hoon | 1 - pkg/arvo/sys/zuse.hoon | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/arvo/sys/vane/ames.hoon b/pkg/arvo/sys/vane/ames.hoon index ec3ccad528..f9418408c3 100644 --- a/pkg/arvo/sys/vane/ames.hoon +++ b/pkg/arvo/sys/vane/ames.hoon @@ -960,7 +960,6 @@ =. snd.ship-state %- ~(run by snd.ship-state) |= =message-pump-state - =* p packet-pump-state.message-pump-state =. num-live.metrics.packet-pump-state.message-pump-state ~(wyt in live.packet-pump-state.message-pump-state) message-pump-state diff --git a/pkg/arvo/sys/zuse.hoon b/pkg/arvo/sys/zuse.hoon index ad827c5915..eaf098937b 100644 --- a/pkg/arvo/sys/zuse.hoon +++ b/pkg/arvo/sys/zuse.hoon @@ -5245,8 +5245,7 @@ =. this left ?: stop.acc this =^ del this node - ?: stop.acc this - =. this right + =? this !stop.acc right =? a del (nip a) this :: +node: run .f on .n.a, updating .a, .state, and .stop From ce1c69e0d17c9bff15727e3f8326e392ff9d2e22 Mon Sep 17 00:00:00 2001 From: J Date: Mon, 1 Mar 2021 21:36:00 +0000 Subject: [PATCH 078/103] interface: braces around if statements --- pkg/interface/src/logic/lib/gcpManager.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/interface/src/logic/lib/gcpManager.ts b/pkg/interface/src/logic/lib/gcpManager.ts index 4fba6c0cca..5417dc1215 100644 --- a/pkg/interface/src/logic/lib/gcpManager.ts +++ b/pkg/interface/src/logic/lib/gcpManager.ts @@ -55,8 +55,9 @@ class GcpManager { } restart() { - if (this.#running) + if (this.#running) { this.stop(); + } this.start(); } @@ -105,8 +106,9 @@ class GcpManager { } private refreshAfter(durationMs) { - if (!this.#running) + if (!this.#running) { return; + } if (this.#timeoutId !== null) { console.warn('GcpManager already has a timeout set'); return; From f85ef9fcb49170ffbf8439dc8a54f25204d0a180 Mon Sep 17 00:00:00 2001 From: J Date: Mon, 1 Mar 2021 21:58:03 +0000 Subject: [PATCH 079/103] interface: storage prop unifying gcp and s3 N.B. this boldly assumes that store state will always contain a valid storage element that will contain gcp and s3 elements. This seems like a fair assumption. --- pkg/interface/src/logic/lib/gcpManager.ts | 4 +-- pkg/interface/src/logic/lib/useStorage.ts | 9 ++++--- .../src/logic/reducers/gcp-reducer.ts | 4 +-- pkg/interface/src/logic/reducers/s3-update.ts | 26 +++++++++---------- pkg/interface/src/logic/store/store.ts | 14 +++++----- pkg/interface/src/logic/store/type.ts | 6 ++--- pkg/interface/src/types/index.ts | 1 + pkg/interface/src/types/storage-state.ts | 8 ++++++ .../src/views/apps/chat/ChatResource.tsx | 3 +-- .../views/apps/chat/components/ChatInput.tsx | 5 ++-- .../src/views/apps/links/LinkResource.tsx | 6 ++--- .../src/views/apps/links/LinkWindow.tsx | 7 +++-- .../apps/links/components/LinkSubmit.tsx | 7 +++-- .../apps/profile/components/EditProfile.tsx | 6 ++--- .../views/apps/profile/components/Profile.tsx | 3 +-- .../src/views/apps/profile/profile.tsx | 3 +-- .../views/apps/publish/PublishResource.tsx | 3 +-- .../apps/publish/components/EditPost.tsx | 10 +++---- .../publish/components/MarkdownEditor.tsx | 7 +++-- .../apps/publish/components/MarkdownField.tsx | 6 ++--- .../apps/publish/components/NoteForm.tsx | 7 +++-- .../apps/publish/components/NoteRoutes.tsx | 6 ++--- .../publish/components/NotebookRoutes.tsx | 12 +++------ .../apps/publish/components/new-post.tsx | 6 ++--- .../components/lib/BackgroundPicker.tsx | 11 +++----- .../settings/components/lib/DisplayForm.tsx | 10 +++---- .../apps/settings/components/settings.tsx | 8 +++--- .../src/views/components/ImageInput.tsx | 10 +++---- .../src/views/components/withStorage.tsx | 2 +- .../components/GroupSettings/Admin.tsx | 10 +++---- .../GroupSettings/GroupSettings.tsx | 5 ++-- .../views/landscape/components/GroupsPane.tsx | 3 +-- .../landscape/components/PopoverRoutes.tsx | 13 +++------- 33 files changed, 105 insertions(+), 136 deletions(-) create mode 100644 pkg/interface/src/types/storage-state.ts diff --git a/pkg/interface/src/logic/lib/gcpManager.ts b/pkg/interface/src/logic/lib/gcpManager.ts index 5417dc1215..628bcfc29f 100644 --- a/pkg/interface/src/logic/lib/gcpManager.ts +++ b/pkg/interface/src/logic/lib/gcpManager.ts @@ -64,7 +64,7 @@ class GcpManager { #consecutiveFailures: number = 0; private isConfigured() { - return this.#store.state.gcp.configured; + return this.#store.state.storage.gcp.configured; } private refreshLoop() { @@ -88,7 +88,7 @@ class GcpManager { } this.#api.gcp.getToken() .then(() => { - const token = this.#store.state.gcp?.token; + const token = this.#store.state.storage.gcp?.token; if (token) { this.#consecutiveFailures = 0; const interval = this.refreshInterval(token.expiresIn); diff --git a/pkg/interface/src/logic/lib/useStorage.ts b/pkg/interface/src/logic/lib/useStorage.ts index 2ea88b9a8c..67cdb49d4f 100644 --- a/pkg/interface/src/logic/lib/useStorage.ts +++ b/pkg/interface/src/logic/lib/useStorage.ts @@ -1,6 +1,9 @@ import {useCallback, useMemo, useEffect, useRef, useState} from 'react'; -import {S3State} from '../../types/s3-update'; -import {GcpState} from '../../types/gcp-state'; +import { + GcpState, + S3State, + StorageState +} from '../../types'; import S3 from "aws-sdk/clients/s3"; import GcpClient from './GcpClient'; import {StorageClient, StorageAcl} from './StorageClient'; @@ -15,7 +18,7 @@ export interface IuseStorage { promptUpload: () => Promise; } -const useStorage = (s3: S3State, gcp: GcpState, +const useStorage = ({gcp, s3}: StorageState, { accept = '*' } = { accept: '*' }): IuseStorage => { const [uploading, setUploading] = useState(false); diff --git a/pkg/interface/src/logic/reducers/gcp-reducer.ts b/pkg/interface/src/logic/reducers/gcp-reducer.ts index 685dfbef22..2900df976d 100644 --- a/pkg/interface/src/logic/reducers/gcp-reducer.ts +++ b/pkg/interface/src/logic/reducers/gcp-reducer.ts @@ -13,7 +13,7 @@ export default class GcpReducer{ reduceConfigured(json, state) { let data = json['gcp-configured']; if (data !== undefined) { - state.gcp.configured = data; + state.storage.gcp.configured = data; } } @@ -26,7 +26,7 @@ export default class GcpReducer{ setToken(data: any, state: S) { if (this.isToken(data)) { - state.gcp.token = data; + state.storage.gcp.token = data; } } diff --git a/pkg/interface/src/logic/reducers/s3-update.ts b/pkg/interface/src/logic/reducers/s3-update.ts index d05f741b39..ba224cd9bf 100644 --- a/pkg/interface/src/logic/reducers/s3-update.ts +++ b/pkg/interface/src/logic/reducers/s3-update.ts @@ -23,14 +23,14 @@ export default class S3Reducer { credentials(json: S3Update, state: S) { const data = _.get(json, 'credentials', false); if (data) { - state.s3.credentials = data; + state.storage.s3.credentials = data; } } configuration(json: S3Update, state: S) { const data = _.get(json, 'configuration', false); if (data) { - state.s3.configuration = { + state.storage.s3.configuration = { buckets: new Set(data.buckets), currentBucket: data.currentBucket }; @@ -39,44 +39,44 @@ export default class S3Reducer { currentBucket(json: S3Update, state: S) { const data = _.get(json, 'setCurrentBucket', false); - if (data && state.s3) { - state.s3.configuration.currentBucket = data; + if (data && state.storage.s3) { + state.storage.s3.configuration.currentBucket = data; } } addBucket(json: S3Update, state: S) { const data = _.get(json, 'addBucket', false); if (data) { - state.s3.configuration.buckets = - state.s3.configuration.buckets.add(data); + state.storage.s3.configuration.buckets = + state.storage.s3.configuration.buckets.add(data); } } removeBucket(json: S3Update, state: S) { const data = _.get(json, 'removeBucket', false); if (data) { - state.s3.configuration.buckets.delete(data); + state.storage.s3.configuration.buckets.delete(data); } } endpoint(json: S3Update, state: S) { const data = _.get(json, 'setEndpoint', false); - if (data && state.s3.credentials) { - state.s3.credentials.endpoint = data; + if (data && state.storage.s3.credentials) { + state.storage.s3.credentials.endpoint = data; } } accessKeyId(json: S3Update , state: S) { const data = _.get(json, 'setAccessKeyId', false); - if (data && state.s3.credentials) { - state.s3.credentials.accessKeyId = data; + if (data && state.storage.s3.credentials) { + state.storage.s3.credentials.accessKeyId = data; } } secretAccessKey(json: S3Update, state: S) { const data = _.get(json, 'setSecretAccessKey', false); - if (data && state.s3.credentials) { - state.s3.credentials.secretAccessKey = data; + if (data && state.storage.s3.credentials) { + state.storage.s3.credentials.secretAccessKey = data; } } } diff --git a/pkg/interface/src/logic/store/store.ts b/pkg/interface/src/logic/store/store.ts index b877b68368..cf86c00d13 100644 --- a/pkg/interface/src/logic/store/store.ts +++ b/pkg/interface/src/logic/store/store.ts @@ -73,13 +73,15 @@ export default class GlobalStore extends BaseStore { }, weather: {}, userLocation: null, - gcp: {}, - s3: { - configuration: { - buckets: new Set(), - currentBucket: '' + storage: { + gcp: {}, + s3: { + configuration: { + buckets: new Set(), + currentBucket: '' + }, + credentials: null }, - credentials: null }, isContactPublic: false, contacts: {}, diff --git a/pkg/interface/src/logic/store/type.ts b/pkg/interface/src/logic/store/type.ts index 6cc9c23a83..662652ac8f 100644 --- a/pkg/interface/src/logic/store/type.ts +++ b/pkg/interface/src/logic/store/type.ts @@ -3,8 +3,7 @@ import { Invites } from '~/types/invite-update'; import { Associations } from '~/types/metadata-update'; import { Rolodex } from '~/types/contact-update'; import { Groups } from '~/types/group-update'; -import { GcpState } from '~/types/gcp-state'; -import { S3State } from '~/types/s3-update'; +import { StorageState } from '~/types/storage-state'; import { LaunchState, WeatherState } from '~/types/launch-update'; import { ConnectionStatus } from '~/types/connection'; import {Graphs} from '~/types/graph-update'; @@ -32,8 +31,7 @@ export interface StoreState { groups: Groups; groupKeys: Set; nackedContacts: Set - s3: S3State; - gcp: GcpState; + storage: StorageState; graphs: Graphs; graphKeys: Set; diff --git a/pkg/interface/src/types/index.ts b/pkg/interface/src/types/index.ts index 390a4fb950..39efb1cef6 100644 --- a/pkg/interface/src/types/index.ts +++ b/pkg/interface/src/types/index.ts @@ -11,6 +11,7 @@ export * from './launch-update'; export * from './local-update'; export * from './metadata-update'; export * from './noun'; +export * from './storage-state'; export * from './gcp-state'; export * from './s3-update'; export * from './workspace'; diff --git a/pkg/interface/src/types/storage-state.ts b/pkg/interface/src/types/storage-state.ts new file mode 100644 index 0000000000..61bf612ada --- /dev/null +++ b/pkg/interface/src/types/storage-state.ts @@ -0,0 +1,8 @@ +import {GcpState} from './gcp-state'; +import {S3State} from './s3-update'; + + +export interface StorageState { + gcp: GcpState; + s3: S3State; +}; diff --git a/pkg/interface/src/views/apps/chat/ChatResource.tsx b/pkg/interface/src/views/apps/chat/ChatResource.tsx index 3473e03147..6ba85b3272 100644 --- a/pkg/interface/src/views/apps/chat/ChatResource.tsx +++ b/pkg/interface/src/views/apps/chat/ChatResource.tsx @@ -179,8 +179,7 @@ export function ChatResource(props: ChatResourceProps) { (!showBanner && hasLoadedAllowed) ? contacts : modifiedContacts } onUnmount={appendUnsent} - gcp={props.gcp} - s3={props.s3} + storage={props.storage} placeholder="Message..." message={unsent[station] || ''} deleteMessage={clearUnsent} diff --git a/pkg/interface/src/views/apps/chat/components/ChatInput.tsx b/pkg/interface/src/views/apps/chat/components/ChatInput.tsx index 9a64e39099..9b6a04b363 100644 --- a/pkg/interface/src/views/apps/chat/components/ChatInput.tsx +++ b/pkg/interface/src/views/apps/chat/components/ChatInput.tsx @@ -7,7 +7,7 @@ import { createPost } from '~/logic/api/graph'; import tokenizeMessage, { isUrl } from '~/logic/lib/tokenizeMessage'; import GlobalApi from '~/logic/api/global'; import { Envelope } from '~/types/chat-update'; -import { Contacts, Content } from '~/types'; +import { StorageState, Contacts, Content } from '~/types'; import { Row, BaseImage, Box, Icon, LoadingSpinner } from '@tlon/indigo-react'; import withStorage from '~/views/components/withStorage'; import { withLocalState } from '~/logic/state/local'; @@ -20,8 +20,7 @@ type ChatInputProps = IuseStorage & { envelopes: Envelope[]; contacts: Contacts; onUnmount(msg: string): void; - gcp: any; - s3: any; + storage: StorageState; placeholder: string; message: string; deleteMessage(): void; diff --git a/pkg/interface/src/views/apps/links/LinkResource.tsx b/pkg/interface/src/views/apps/links/LinkResource.tsx index 5b4aeff556..ea45a2ed6d 100644 --- a/pkg/interface/src/views/apps/links/LinkResource.tsx +++ b/pkg/interface/src/views/apps/links/LinkResource.tsx @@ -34,8 +34,7 @@ export function LinkResource(props: LinkResourceProps) { associations, graphKeys, unreads, - gcp, - s3, + storage, history } = props; @@ -71,8 +70,7 @@ export function LinkResource(props: LinkResourceProps) { render={(props) => { return ( { canWrite ? ( - + ) : ( There are no links here yet. You do not have permission to post to this collection. ) @@ -98,7 +97,7 @@ export function LinkWindow(props: LinkWindowProps) { return ( - + diff --git a/pkg/interface/src/views/apps/links/components/LinkSubmit.tsx b/pkg/interface/src/views/apps/links/components/LinkSubmit.tsx index 09f6e4a699..67c5f90fe9 100644 --- a/pkg/interface/src/views/apps/links/components/LinkSubmit.tsx +++ b/pkg/interface/src/views/apps/links/components/LinkSubmit.tsx @@ -3,22 +3,21 @@ import React, { useCallback, useState } from "react"; import GlobalApi from "~/logic/api/global"; import { useFileDrag } from "~/logic/lib/useDrag"; import useStorage from "~/logic/lib/useStorage"; -import { GcpState, S3State } from "~/types"; +import { StorageState } from "~/types"; import SubmitDragger from "~/views/components/SubmitDragger"; import { createPost } from "~/logic/api/graph"; import { hasProvider } from "oembed-parser"; interface LinkSubmitProps { api: GlobalApi; - gcp: GcpState; - s3: S3State; + storage: StorageState; name: string; ship: string; }; const LinkSubmit = (props: LinkSubmitProps) => { let { canUpload, uploadDefault, uploading, promptUpload } = - useStorage(props.s3, props.gcp); + useStorage(props.storage); const [submitFocused, setSubmitFocused] = useState(false); const [urlFocused, setUrlFocused] = useState(false); diff --git a/pkg/interface/src/views/apps/profile/components/EditProfile.tsx b/pkg/interface/src/views/apps/profile/components/EditProfile.tsx index 1aca97d866..3d84ee4608 100644 --- a/pkg/interface/src/views/apps/profile/components/EditProfile.tsx +++ b/pkg/interface/src/views/apps/profile/components/EditProfile.tsx @@ -114,15 +114,15 @@ export function EditProfile(props: any) { Description - + - + - + diff --git a/pkg/interface/src/views/apps/profile/components/Profile.tsx b/pkg/interface/src/views/apps/profile/components/Profile.tsx index 7252eec3c3..b7cd9b24e5 100644 --- a/pkg/interface/src/views/apps/profile/components/Profile.tsx +++ b/pkg/interface/src/views/apps/profile/components/Profile.tsx @@ -107,8 +107,7 @@ export function Profile(props: any) { ); diff --git a/pkg/interface/src/views/apps/publish/components/EditPost.tsx b/pkg/interface/src/views/apps/publish/components/EditPost.tsx index 84ca37acf3..188a9833e0 100644 --- a/pkg/interface/src/views/apps/publish/components/EditPost.tsx +++ b/pkg/interface/src/views/apps/publish/components/EditPost.tsx @@ -4,7 +4,7 @@ import { PostFormSchema, PostForm } from "./NoteForm"; import { FormikHelpers } from "formik"; import GlobalApi from "~/logic/api/global"; import { RouteComponentProps, useLocation } from "react-router-dom"; -import { GraphNode, TextContent, Association, GcpState, S3State } from "~/types"; +import { GraphNode, TextContent, Association, StorageState } from "~/types"; import { getLatestRevision, editPost } from "~/logic/lib/publish"; import {useWaitForProps} from "~/logic/lib/useWaitForProps"; interface EditPostProps { @@ -13,12 +13,11 @@ interface EditPostProps { note: GraphNode; api: GlobalApi; book: string; - gcp: GcpState; - s3: S3State; + storage: StorageState; } export function EditPost(props: EditPostProps & RouteComponentProps) { - const { note, book, noteId, api, ship, history, gcp, s3 } = props; + const { note, book, noteId, api, ship, history, storage } = props; const [revNum, title, body] = getLatestRevision(note); const location = useLocation(); @@ -55,8 +54,7 @@ export function EditPost(props: EditPostProps & RouteComponentProps) { cancel history={history} onSubmit={onSubmit} - gcp={gcp} - s3={s3} + storage={storage} submitLabel="Update" loadingText="Updating..." /> diff --git a/pkg/interface/src/views/apps/publish/components/MarkdownEditor.tsx b/pkg/interface/src/views/apps/publish/components/MarkdownEditor.tsx index ada3c0e2ff..3bafea2b4c 100644 --- a/pkg/interface/src/views/apps/publish/components/MarkdownEditor.tsx +++ b/pkg/interface/src/views/apps/publish/components/MarkdownEditor.tsx @@ -17,7 +17,7 @@ import { Box } from "@tlon/indigo-react"; import { useFileDrag } from "~/logic/lib/useDrag"; import SubmitDragger from "~/views/components/SubmitDragger"; import useStorage from "~/logic/lib/useStorage"; -import { GcpState, S3State } from "~/types"; +import { StorageState } from "~/types"; const MARKDOWN_CONFIG = { name: "markdown", @@ -28,8 +28,7 @@ interface MarkdownEditorProps { value: string; onChange: (s: string) => void; onBlur?: (e: any) => void; - gcp: GcpState; - s3: S3State; + storage: StorageState; } const PromptIfDirty = () => { @@ -75,7 +74,7 @@ export function MarkdownEditor( [onBlur] ); - const { uploadDefault, canUpload } = useStorage(props.s3, props.gcp); + const { uploadDefault, canUpload } = useStorage(props.storage); const onFileDrag = useCallback( async (files: FileList | File[], e: DragEvent) => { diff --git a/pkg/interface/src/views/apps/publish/components/MarkdownField.tsx b/pkg/interface/src/views/apps/publish/components/MarkdownField.tsx index c6adb0b6b8..8712f70860 100644 --- a/pkg/interface/src/views/apps/publish/components/MarkdownField.tsx +++ b/pkg/interface/src/views/apps/publish/components/MarkdownField.tsx @@ -6,8 +6,7 @@ import { MarkdownEditor } from "./MarkdownEditor"; export const MarkdownField = ({ id, - gcp, - s3, + storage, ...rest }: { id: string } & Parameters[0]) => { const [{ value, onBlur }, { error, touched }, { setValue }] = useField(id); @@ -36,8 +35,7 @@ export const MarkdownField = ({ onBlur={handleBlur} value={value} onChange={setValue} - gcp={gcp} - s3={s3} + storage={storage} /> {error} diff --git a/pkg/interface/src/views/apps/publish/components/NoteForm.tsx b/pkg/interface/src/views/apps/publish/components/NoteForm.tsx index ff2001ff68..f56f3a8c01 100644 --- a/pkg/interface/src/views/apps/publish/components/NoteForm.tsx +++ b/pkg/interface/src/views/apps/publish/components/NoteForm.tsx @@ -21,8 +21,7 @@ interface PostFormProps { ) => Promise; submitLabel: string; loadingText: string; - gcp: GcpState; - s3: S3State; + storage: StorageState; } const formSchema = Yup.object({ @@ -36,7 +35,7 @@ export interface PostFormSchema { } export function PostForm(props: PostFormProps) { - const { initial, onSubmit, submitLabel, loadingText, gcp, s3, cancel, history } = props; + const { initial, onSubmit, submitLabel, loadingText, storage, cancel, history } = props; return ( @@ -67,7 +66,7 @@ export function PostForm(props: PostFormProps) { type="button">Cancel} - + diff --git a/pkg/interface/src/views/apps/publish/components/NoteRoutes.tsx b/pkg/interface/src/views/apps/publish/components/NoteRoutes.tsx index 234ef941c6..7152fe9fc5 100644 --- a/pkg/interface/src/views/apps/publish/components/NoteRoutes.tsx +++ b/pkg/interface/src/views/apps/publish/components/NoteRoutes.tsx @@ -11,8 +11,7 @@ import { Graph, Contacts, Association, - GcpState, - S3State, + StorageState, Group } from "~/types"; @@ -28,8 +27,7 @@ interface NoteRoutesProps { baseUrl?: string; rootUrl?: string; group: Group; - gcp: GcpState; - s3: S3State; + storage: StorageState; } export function NoteRoutes(props: NoteRoutesProps & RouteComponentProps) { diff --git a/pkg/interface/src/views/apps/publish/components/NotebookRoutes.tsx b/pkg/interface/src/views/apps/publish/components/NotebookRoutes.tsx index a134613fe5..18d6c10bfd 100644 --- a/pkg/interface/src/views/apps/publish/components/NotebookRoutes.tsx +++ b/pkg/interface/src/views/apps/publish/components/NotebookRoutes.tsx @@ -9,8 +9,7 @@ import { Contacts, Rolodex, Unreads, - GcpState, - S3State + StorageState } from "~/types"; import { Center, LoadingSpinner } from "@tlon/indigo-react"; import bigInt from 'big-integer'; @@ -34,8 +33,7 @@ interface NotebookRoutesProps { rootUrl: string; association: Association; associations: Associations; - gcp: GcpState; - s3: S3State; + storage: StorageState; } export function NotebookRoutes( @@ -82,8 +80,7 @@ export function NotebookRoutes( association={props.association} graph={graph} baseUrl={baseUrl} - gcp={props.gcp} - s3={props.s3} + storage={props.storage} /> )} /> @@ -115,8 +112,7 @@ export function NotebookRoutes( contacts={notebookContacts} association={props.association} group={group} - gcp={props.gcp} - s3={props.s3} + storage={props.storage} {...routeProps} /> ); diff --git a/pkg/interface/src/views/apps/publish/components/new-post.tsx b/pkg/interface/src/views/apps/publish/components/new-post.tsx index c56410e2fd..d039ca38a2 100644 --- a/pkg/interface/src/views/apps/publish/components/new-post.tsx +++ b/pkg/interface/src/views/apps/publish/components/new-post.tsx @@ -16,8 +16,7 @@ interface NewPostProps { graph: Graph; association: Association; baseUrl: string; - gcp: GcpState; - s3: S3State; + storage: StorageState; } export default function NewPost(props: NewPostProps & RouteComponentProps) { @@ -54,8 +53,7 @@ export default function NewPost(props: NewPostProps & RouteComponentProps) { onSubmit={onSubmit} submitLabel="Publish" loadingText="Posting..." - gcp={props.gcp} - s3={props.s3} + storage={props.storage} /> ); } diff --git a/pkg/interface/src/views/apps/settings/components/lib/BackgroundPicker.tsx b/pkg/interface/src/views/apps/settings/components/lib/BackgroundPicker.tsx index e90c757490..f1ec8b299a 100644 --- a/pkg/interface/src/views/apps/settings/components/lib/BackgroundPicker.tsx +++ b/pkg/interface/src/views/apps/settings/components/lib/BackgroundPicker.tsx @@ -9,7 +9,7 @@ import { } from "@tlon/indigo-react"; import GlobalApi from "~/logic/api/global"; -import { GcpState, S3State } from "~/types"; +import { StorageState, } from "~/types"; import { ImageInput } from "~/views/components/ImageInput"; import {ColorInput} from "~/views/components/ColorInput"; @@ -19,14 +19,12 @@ export function BackgroundPicker({ bgType, bgUrl, api, - gcp, - s3, + storage, }: { bgType: BgType; bgUrl?: string; api: GlobalApi; - gcp: GcpState; - s3: S3State; + storage: StorageState; }) { const rowSpace = { my: 0, alignItems: 'center' }; @@ -40,8 +38,7 @@ export function BackgroundPicker({ - + ); diff --git a/pkg/interface/src/views/components/ImageInput.tsx b/pkg/interface/src/views/components/ImageInput.tsx index 836d656993..868228f7e5 100644 --- a/pkg/interface/src/views/components/ImageInput.tsx +++ b/pkg/interface/src/views/components/ImageInput.tsx @@ -10,22 +10,20 @@ import { BaseInput } from "@tlon/indigo-react"; import { useField } from "formik"; -import { S3State } from "~/types/s3-update"; -import { GcpState } from "~/types/gcp-state"; +import { StorageState } from "~/types/storage-state"; import useStorage from "~/logic/lib/useStorage"; type ImageInputProps = Parameters[0] & { id: string; label: string; - s3: S3State; - gcp: GcpState; + storage: StorageState; placeholder?: string; }; export function ImageInput(props: ImageInputProps) { - const { id, label, s3, gcp, caption, placeholder, ...rest } = props; + const { id, label, storage, caption, placeholder, ...rest } = props; - const { uploadDefault, canUpload, uploading } = useStorage(s3, gcp); + const { uploadDefault, canUpload, uploading } = useStorage(storage); const [field, meta, { setValue, setError }] = useField(id); diff --git a/pkg/interface/src/views/components/withStorage.tsx b/pkg/interface/src/views/components/withStorage.tsx index 95b9d3cf48..a0898d7455 100644 --- a/pkg/interface/src/views/components/withStorage.tsx +++ b/pkg/interface/src/views/components/withStorage.tsx @@ -3,7 +3,7 @@ import useStorage from "~/logic/lib/useStorage"; const withStorage = (Component, params = {}) => { return React.forwardRef((props: any, ref) => { - const storage = useStorage(props.s3, props.gcp, params); + const storage = useStorage(props.storage, params); return ; }); diff --git a/pkg/interface/src/views/landscape/components/GroupSettings/Admin.tsx b/pkg/interface/src/views/landscape/components/GroupSettings/Admin.tsx index a9ac19856b..d97be994b7 100644 --- a/pkg/interface/src/views/landscape/components/GroupSettings/Admin.tsx +++ b/pkg/interface/src/views/landscape/components/GroupSettings/Admin.tsx @@ -22,7 +22,7 @@ import { ColorInput } from "~/views/components/ColorInput"; import { useHistory } from "react-router-dom"; import { uxToHex } from "~/logic/lib/util"; -import {GcpState, S3State} from "~/types"; +import {StorageState} from "~/types"; import {ImageInput} from "~/views/components/ImageInput"; interface FormSchema { @@ -46,12 +46,11 @@ interface GroupAdminSettingsProps { group: Group; association: Association; api: GlobalApi; - gcp: GcpState; - s3: S3State; + storage: StorageState; } export function GroupAdminSettings(props: GroupAdminSettingsProps) { - const { group, association, gcp, s3 } = props; + const { group, association, storage } = props; const { metadata } = association; const history = useHistory(); const currentPrivate = "invite" in props.group.policy; @@ -133,8 +132,7 @@ export function GroupAdminSettings(props: GroupAdminSettingsProps) { caption="A picture for your group" placeholder="Enter URL" disabled={disabled} - gcp={gcp} - s3={s3} + storage={storage} /> )} {view === "participants" && ( From af9e829464ad49c3b1318730babf9ea303607d5b Mon Sep 17 00:00:00 2001 From: J Date: Mon, 1 Mar 2021 22:14:41 +0000 Subject: [PATCH 080/103] interface: correct useEffect memo --- pkg/interface/src/logic/lib/useStorage.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/interface/src/logic/lib/useStorage.ts b/pkg/interface/src/logic/lib/useStorage.ts index 67cdb49d4f..c322f159ff 100644 --- a/pkg/interface/src/logic/lib/useStorage.ts +++ b/pkg/interface/src/logic/lib/useStorage.ts @@ -41,7 +41,7 @@ const useStorage = ({gcp, s3}: StorageState, endpoint: s3.credentials.endpoint }); } - }, [gcp.accessKey, s3.credentials]); + }, [gcp.token, s3.credentials]); const canUpload = useMemo( () => From 35089f6656202baf26dfe5f9f400a11d93f77406 Mon Sep 17 00:00:00 2001 From: J Date: Mon, 1 Mar 2021 22:28:29 +0000 Subject: [PATCH 081/103] interface: basic error handling on upload failures --- pkg/interface/src/logic/lib/GcpClient.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/interface/src/logic/lib/GcpClient.ts b/pkg/interface/src/logic/lib/GcpClient.ts index f071896979..b03c7fd0a6 100644 --- a/pkg/interface/src/logic/lib/GcpClient.ts +++ b/pkg/interface/src/logic/lib/GcpClient.ts @@ -46,8 +46,10 @@ class GcpUpload implements StorageUpload { referrerPolicy: 'no-referrer', body: Body }); - // TODO: response errors. - // If we get a 400, perhaps we need fine-grained permissions on the bucket. + if (!response.ok) { + console.error('GcpClient server error', await response.json()); + throw new Error(`GcpClient: response ${response.status}`); + } return {Location: `https://${ENDPOINT}/${Bucket}/${Key}`}; } } From 359896a23dfa902c4b0ce69af76faed35f97b8ce Mon Sep 17 00:00:00 2001 From: J Date: Mon, 1 Mar 2021 22:38:20 +0000 Subject: [PATCH 082/103] interface: fix a few type imports Missed these in the initial pass, found them by looking at "Files Changed" in the Github UI. --- pkg/interface/src/views/apps/publish/components/NoteForm.tsx | 2 +- pkg/interface/src/views/apps/publish/components/new-post.tsx | 2 +- .../src/views/apps/settings/components/lib/BackgroundPicker.tsx | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/interface/src/views/apps/publish/components/NoteForm.tsx b/pkg/interface/src/views/apps/publish/components/NoteForm.tsx index f56f3a8c01..06b1cc10d0 100644 --- a/pkg/interface/src/views/apps/publish/components/NoteForm.tsx +++ b/pkg/interface/src/views/apps/publish/components/NoteForm.tsx @@ -9,7 +9,7 @@ import { import { AsyncButton } from "../../../components/AsyncButton"; import { Formik, Form, FormikHelpers } from "formik"; import { MarkdownField } from "./MarkdownField"; -import { GcpState, S3State } from "~/types"; +import { StorageState } from "~/types"; interface PostFormProps { initial: PostFormSchema; diff --git a/pkg/interface/src/views/apps/publish/components/new-post.tsx b/pkg/interface/src/views/apps/publish/components/new-post.tsx index d039ca38a2..6f5cdbee0e 100644 --- a/pkg/interface/src/views/apps/publish/components/new-post.tsx +++ b/pkg/interface/src/views/apps/publish/components/new-post.tsx @@ -6,7 +6,7 @@ import { RouteComponentProps } from "react-router-dom"; import { PostForm, PostFormSchema } from "./NoteForm"; import {createPost} from "~/logic/api/graph"; import {Graph} from "~/types/graph-update"; -import {Association, GcpState, S3State} from "~/types"; +import {Association, StorageState} from "~/types"; import {newPost} from "~/logic/lib/publish"; interface NewPostProps { diff --git a/pkg/interface/src/views/apps/settings/components/lib/BackgroundPicker.tsx b/pkg/interface/src/views/apps/settings/components/lib/BackgroundPicker.tsx index f1ec8b299a..f1a76dae9e 100644 --- a/pkg/interface/src/views/apps/settings/components/lib/BackgroundPicker.tsx +++ b/pkg/interface/src/views/apps/settings/components/lib/BackgroundPicker.tsx @@ -9,7 +9,7 @@ import { } from "@tlon/indigo-react"; import GlobalApi from "~/logic/api/global"; -import { StorageState, } from "~/types"; +import { StorageState } from "~/types"; import { ImageInput } from "~/views/components/ImageInput"; import {ColorInput} from "~/views/components/ColorInput"; From 2823b2729b8165c342773ed8122033337660fade Mon Sep 17 00:00:00 2001 From: "Raymond E. Pasco" Date: Tue, 2 Mar 2021 14:00:12 -0500 Subject: [PATCH 083/103] Change PNG aura from @t to @ No PNG is (sane %t) - the PNG header alone isn't. This should also be octs, not an atom, as a binary filetype - it works for PNG purely by coincidence (no valid PNG can end with a sequence of 00 bytes, because they have to end with 00 00 00 00 49 45 4e 44 ae 42 60 82). --- pkg/arvo/mar/png.hoon | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/arvo/mar/png.hoon b/pkg/arvo/mar/png.hoon index e68e502cd1..6a60a6a27b 100644 --- a/pkg/arvo/mar/png.hoon +++ b/pkg/arvo/mar/png.hoon @@ -1,4 +1,4 @@ -|_ dat=@t +|_ dat=@ ++ grow |% ++ mime [/image/png (as-octs:mimes:html dat)] @@ -6,7 +6,7 @@ ++ grab |% ++ mime |=([p=mite q=octs] q.q) - ++ noun @t + ++ noun @ -- ++ grad %mime -- From 883a56382ffc280b4b611fc0ba14365b91c4ac78 Mon Sep 17 00:00:00 2001 From: fang Date: Wed, 3 Mar 2021 17:14:07 +0100 Subject: [PATCH 084/103] tally: be safer wrt mangled metadata Previously, if metadata-store said a graph existed, we'd unconditionally scry for it. Now, we make sure the graph actually exists, to avoid risking a crash. --- pkg/arvo/gen/tally.hoon | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pkg/arvo/gen/tally.hoon b/pkg/arvo/gen/tally.hoon index 75e3a53cc0..fbaf04e0a9 100644 --- a/pkg/arvo/gen/tally.hoon +++ b/pkg/arvo/gen/tally.hoon @@ -61,14 +61,24 @@ ::NOTE we only count graphs for now ?. &(=(%graph app-name.m) =(our creator.metadatum)) ~ `[module.metadatum resource.m] +:: for sanity checks +:: +=/ real=(set resource:re) + =/ upd=update:ga + %+ scry update:ga + [%x %graph-store /keys/graph-update] + ?> ?=(%keys -.q.upd) + resources.q.upd :: count activity per channel :: =/ activity=(list [resource:re members=@ud (list [resource:re mod=term week=@ud authors=@ud])]) %+ turn crowds |= [g=resource:re m=@ud] :+ g m - %+ turn (~(got by channels) g) + %+ murn (~(got by channels) g) |= [m=term r=resource:re] + ?. (~(has in real) r) ~ + %- some :+ r m ::NOTE graph-store doesn't use the full resource-style path here! =/ upd=update:ga From 4a066c8d4d42ad0b2874cf38301d8060dde579d6 Mon Sep 17 00:00:00 2001 From: Philip Monk Date: Wed, 3 Mar 2021 20:02:29 -0800 Subject: [PATCH 085/103] clay: fix upgrade --- pkg/arvo/sys/vane/clay.hoon | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index 13af4a6908..b4798db1b2 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -4172,7 +4172,7 @@ +$ ford-cache-7 ford-cache +$ raft-6 $: rom=room-6 :: domestic - hoy=(map ship rung) :: foreign + hoy=(map ship rung-6) :: foreign ran=rang :: hashes mon=(map term beam) :: mount points hez=(unit duct) :: sync duct @@ -4195,13 +4195,24 @@ fod=ford-cache-6 :: ford cache fer=(unit reef-cache) :: reef cache == + +$ rung-6 + $: rus=(map desk rede-6) + == + +$ rede-6 + $: lim=@da + ref=(unit rind) + qyx=cult + dom=dome-6 + per=regs + pew=regs + == +$ ford-cache-6 * :: discard old cache -- |= old=raft-any |^ =? old ?=(%6 -.old) 7+(raft-6-to-7 +.old) ?> ?=(%7 -.old) - ..^$(ruf +.old) + ..^^$(ruf +.old) :: +raft-6-to-7: delete stale ford caches (they could all be invalid) :: ++ raft-6-to-7 @@ -4213,6 +4224,13 @@ |= doj=dojo-6 ^- dojo-7 doj(fod.dom *ford-cache-7) + :: + hoy + %- ~(run by hoy.raf) + |= =rung-6 + %- ~(run by rus.rung-6) + |= =rede-6 + rede-6(dom dom.rede-6(fod *ford-cache-7)) == -- :: From 7e6672d356e9c0edc725c90b7cb0b36b97568a95 Mon Sep 17 00:00:00 2001 From: janeway Date: Thu, 4 Mar 2021 14:28:54 -0800 Subject: [PATCH 086/103] glob: update to 0v1.5ahkf.kj160.veq26.sbnii.p6h5t --- bin/solid.pill | 4 ++-- pkg/arvo/app/glob.hoon | 2 +- pkg/arvo/app/landscape/index.html | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/solid.pill b/bin/solid.pill index 0f64ec0753..176c70e1ec 100644 --- a/bin/solid.pill +++ b/bin/solid.pill @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e9d09dc80599aa7239d6190506d64b69dbcaa099694fc8eb6f705eaea9d32a64 -size 10967317 +oid sha256:c3c9a3d510c1c545e4094a8ce699177bd22590092cf86fbbd4f0f283f05d40b2 +size 9019287 diff --git a/pkg/arvo/app/glob.hoon b/pkg/arvo/app/glob.hoon index 5aada8ac2d..a365b58f82 100644 --- a/pkg/arvo/app/glob.hoon +++ b/pkg/arvo/app/glob.hoon @@ -5,7 +5,7 @@ /- glob /+ default-agent, verb, dbug |% -++ hash 0v5.ip41o.9jcdb.4jb1f.sd508.fdssj +++ hash 0v1.5ahkf.kj160.veq26.sbnii.p6h5t +$ state-0 [%0 hash=@uv glob=(unit (each glob:glob tid=@ta))] +$ all-states $% state-0 diff --git a/pkg/arvo/app/landscape/index.html b/pkg/arvo/app/landscape/index.html index 1f359a06b6..a60e2f490c 100644 --- a/pkg/arvo/app/landscape/index.html +++ b/pkg/arvo/app/landscape/index.html @@ -24,6 +24,6 @@
- + From 3c99ad06abf4b934da4be8ce0b6b9e9e863632ca Mon Sep 17 00:00:00 2001 From: Philip Monk Date: Thu, 4 Mar 2021 14:41:38 -0800 Subject: [PATCH 087/103] noun: add +mash --- pkg/arvo/mar/noun.hoon | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/arvo/mar/noun.hoon b/pkg/arvo/mar/noun.hoon index fb07674981..5c798d3c2a 100644 --- a/pkg/arvo/mar/noun.hoon +++ b/pkg/arvo/mar/noun.hoon @@ -14,5 +14,6 @@ ++ diff |=(* +<) ++ pact |=(* +<) ++ join |=([* *] *(unit *)) + ++ mash |=([[ship desk *] [ship desk *]] `*`~|(%noun-mash !!)) -- -- From aa07cc4cba5f7e7f81704b7c0a1d30a6adb1c0a4 Mon Sep 17 00:00:00 2001 From: janeway Date: Thu, 4 Mar 2021 14:47:10 -0800 Subject: [PATCH 088/103] glob: update to 0vah4o5.ark7i.iqunl.or5d7.3rm2e --- bin/solid.pill | 4 ++-- pkg/arvo/app/glob.hoon | 2 +- pkg/arvo/app/landscape/index.html | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/solid.pill b/bin/solid.pill index 176c70e1ec..a76163bdb0 100644 --- a/bin/solid.pill +++ b/bin/solid.pill @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c3c9a3d510c1c545e4094a8ce699177bd22590092cf86fbbd4f0f283f05d40b2 -size 9019287 +oid sha256:a14132a9e059f582b03d940e491701d0086d86d3882a4696e160159472aa3fcb +size 9084082 diff --git a/pkg/arvo/app/glob.hoon b/pkg/arvo/app/glob.hoon index a365b58f82..7bf30ff3d6 100644 --- a/pkg/arvo/app/glob.hoon +++ b/pkg/arvo/app/glob.hoon @@ -5,7 +5,7 @@ /- glob /+ default-agent, verb, dbug |% -++ hash 0v1.5ahkf.kj160.veq26.sbnii.p6h5t +++ hash 0vah4o5.ark7i.iqunl.or5d7.3rm2e +$ state-0 [%0 hash=@uv glob=(unit (each glob:glob tid=@ta))] +$ all-states $% state-0 diff --git a/pkg/arvo/app/landscape/index.html b/pkg/arvo/app/landscape/index.html index a60e2f490c..e86ab509fd 100644 --- a/pkg/arvo/app/landscape/index.html +++ b/pkg/arvo/app/landscape/index.html @@ -24,6 +24,6 @@
- + From 008f8ac54de372f1935053cd646dc51cc314165f Mon Sep 17 00:00:00 2001 From: Philip Monk Date: Thu, 4 Mar 2021 15:20:43 -0800 Subject: [PATCH 089/103] tests: update for removing +volt --- pkg/arvo/tests/sys/vane/eyre.hoon | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/arvo/tests/sys/vane/eyre.hoon b/pkg/arvo/tests/sys/vane/eyre.hoon index 0bc7c271ed..0b83d3a843 100644 --- a/pkg/arvo/tests/sys/vane/eyre.hoon +++ b/pkg/arvo/tests/sys/vane/eyre.hoon @@ -2360,7 +2360,6 @@ ++ mash !! ++ pact !! ++ vale |=(=noun !>(;;(json noun))) - ++ volt !! -- :: ?> =(%j view) From f3f875304f9937ecb5e044c842f8783a8595a21d Mon Sep 17 00:00:00 2001 From: Philip Monk Date: Thu, 4 Mar 2021 16:26:05 -0800 Subject: [PATCH 090/103] drum-put: clam --- pkg/arvo/mar/drum-put.hoon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/arvo/mar/drum-put.hoon b/pkg/arvo/mar/drum-put.hoon index 29b6a10c34..e6094158ed 100644 --- a/pkg/arvo/mar/drum-put.hoon +++ b/pkg/arvo/mar/drum-put.hoon @@ -11,6 +11,6 @@ -- ++ grab :: convert from |% - ++ noun [path @] :: clam from %noun + +$ noun [path @] :: clam from %noun -- -- From 69fa1296beb47cefe21b83dec7e6e6fb4aba676b Mon Sep 17 00:00:00 2001 From: Logan Allen Date: Thu, 4 Mar 2021 19:09:41 -0600 Subject: [PATCH 091/103] graph-threads: make %groupify and %restore compile --- pkg/arvo/ted/graph/groupify.hoon | 28 +++++++++++++++------------- pkg/arvo/ted/graph/restore.hoon | 19 ++++++++++++------- 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/pkg/arvo/ted/graph/groupify.hoon b/pkg/arvo/ted/graph/groupify.hoon index 7117f01e64..b3681e0171 100644 --- a/pkg/arvo/ted/graph/groupify.hoon +++ b/pkg/arvo/ted/graph/groupify.hoon @@ -1,4 +1,5 @@ -/- spider, graph-view, graph=graph-store, *metadata-store, *group, *metadata-store +/- spider, graph-view, graph=graph-store, + met=metadata-store, *group, *metadata-store /+ strandio, resource => |% @@ -28,11 +29,11 @@ :: ++ scry-metadatum |= rid=resource - =/ m (strand ,metadata) + =/ m (strand ,metadatum:met) ^- form:m =/ enc-path=@t (scot %t (spat (en-path:resource rid))) - ;< umeta=(unit metadata) bind:m - %+ scry:strandio (unit metadata) + ;< umeta=(unit metadatum:met) bind:m + %+ scry:strandio (unit metadatum:met) %+ weld /gx/metadata-store/metadata /[enc-path]/graph/[enc-path]/noun ?> ?=(^ umeta) @@ -48,24 +49,25 @@ ;< =group bind:m (scry-group rid.action) ?. hidden.group (strand-fail:strandio %bad-request ~) -;< =metadata bind:m (scry-metadatum rid.action) +;< =metadatum:met bind:m (scry-metadatum rid.action) ?~ to.action ;< ~ bind:m %+ poke-our %contact-view :- %contact-view-action - !>([%groupify rid.action title.metadata description.metadata]) + !>([%groupify rid.action title.metadatum description.metadatum]) (pure:m !>(~)) ;< new=^group bind:m (scry-group u.to.action) ?< hidden.new -=/ new-path (en-path:resource u.to.action) -=/ app-path (en-path:resource rid.action) -=/ add-md=metadata-action - [%add new-path graph+app-path metadata] -;< ~ bind:m - (poke-our %metadata-store metadata-action+!>(add-md)) ;< ~ bind:m %+ poke-our %metadata-store - metadata-action+!>([%remove app-path graph+app-path]) + :- %metadata-action + !> ^- action:met + [%add u.to.action [%graph rid.action] metadatum] +;< ~ bind:m + %+ poke-our %metadata-store + :- %metadata-action + !> ^- action:met + [%remove rid.action [%graph rid.action]] ;< ~ bind:m (poke-our %group-store %group-update !>([%remove-group rid.action ~])) (pure:m !>(~)) diff --git a/pkg/arvo/ted/graph/restore.hoon b/pkg/arvo/ted/graph/restore.hoon index ae03aa15e3..e399deb4da 100644 --- a/pkg/arvo/ted/graph/restore.hoon +++ b/pkg/arvo/ted/graph/restore.hoon @@ -1,4 +1,4 @@ -/- spider, graph=graph-store, *metadata-store, *group, group-store +/- spider, graph=graph-store, met=metadata-store, *group, group-store, push-hook /+ strandio, resource, graph-view => |% @@ -23,17 +23,22 @@ :: :: Setup metadata :: -=/ =metadata - %* . *metadata +=/ =metadatum:met + %* . *metadatum:met title title description description date-created now.bowl creator our.bowl module module == -=/ act=metadata-action - [%add (en-path:resource group) graph+(en-path:resource rid) metadata] -;< ~ bind:m (poke-our %metadata-hook %metadata-action !>(act)) ;< ~ bind:m - (poke-our %metadata-hook %metadata-hook-action !>([%add-owned (en-path:resource group)])) + %+ poke-our %metadata-push-hook + :- %metadata-action + !> ^- action:met + [%add group [%graph rid] metadatum] +;< ~ bind:m + %+ poke-our %metadata-push-hook + :- %push-hook-action + !> ^- action:push-hook + [%add group] (pure:m !>(~)) From a0146f4a4445b442d4095bb1e40fd2fb816783d5 Mon Sep 17 00:00:00 2001 From: Philip Monk Date: Thu, 4 Mar 2021 18:28:16 -0800 Subject: [PATCH 092/103] clay: add %e, %f requests For static naves, casts respectively --- pkg/arvo/lib/strandio.hoon | 33 ++++++++++++++++++++++++++++++--- pkg/arvo/sys/lull.hoon | 2 +- pkg/arvo/sys/vane/clay.hoon | 36 ++++++++++++++++++++++++++++++++++++ pkg/arvo/ted/build-cast.hoon | 4 ++-- pkg/arvo/ted/build-nave.hoon | 13 +++++++++++++ pkg/arvo/ted/build-tube.hoon | 13 +++++++++++++ 6 files changed, 95 insertions(+), 6 deletions(-) create mode 100644 pkg/arvo/ted/build-nave.hoon create mode 100644 pkg/arvo/ted/build-tube.hoon diff --git a/pkg/arvo/lib/strandio.hoon b/pkg/arvo/lib/strandio.hoon index 039b493df9..a8b52ede1f 100644 --- a/pkg/arvo/lib/strandio.hoon +++ b/pkg/arvo/lib/strandio.hoon @@ -468,9 +468,9 @@ (strand-fail %build-mark >arg< ~) ?> =(%dais p.r.u.riot) (pure:m !<(dais:clay q.r.u.riot)) -:: +build-cast: build a mark conversion gate ($tube) +:: +build-tube: build a mark conversion gate ($tube) :: -++ build-cast +++ build-tube |= [[=ship =desk =case] =mars:clay] =* arg +< =/ m (strand ,tube:clay) @@ -478,10 +478,37 @@ ;< =riot:clay bind:m (warp ship desk ~ %sing %c case /[a.mars]/[b.mars]) ?~ riot - (strand-fail %build-cast >arg< ~) + (strand-fail %build-tube >arg< ~) ?> =(%tube p.r.u.riot) (pure:m !<(tube:clay q.r.u.riot)) :: +:: +build-nave: build a mark definition to a $nave +:: +++ build-nave + |= [[=ship =desk =case] mak=mark] + =* arg +< + =/ m (strand ,vase) + ^- form:m + ;< =riot:clay bind:m + (warp ship desk ~ %sing %b case /[mak]) + ?~ riot + (strand-fail %build-nave >arg< ~) + ?> =(%nave p.r.u.riot) + (pure:m q.r.u.riot) +:: +build-cast: build a mark conversion gate (static) +:: +++ build-cast + |= [[=ship =desk =case] =mars:clay] + =* arg +< + =/ m (strand ,vase) + ^- form:m + ;< =riot:clay bind:m + (warp ship desk ~ %sing %f case /[a.mars]/[b.mars]) + ?~ riot + (strand-fail %build-cast >arg< ~) + ?> =(%cast p.r.u.riot) + (pure:m q.r.u.riot) +:: :: Read from Clay :: ++ warp diff --git a/pkg/arvo/sys/lull.hoon b/pkg/arvo/sys/lull.hoon index 48cdce15b4..23d08ae5bf 100644 --- a/pkg/arvo/sys/lull.hoon +++ b/pkg/arvo/sys/lull.hoon @@ -796,7 +796,7 @@ $: face=(unit term) file-path=term == - +$ care ?(%a %b %c %d %p %r %s %t %u %v %w %x %y %z) :: clay submode + +$ care ?(%a %b %c %d %e %f %p %r %s %t %u %v %w %x %y %z) :: clay submode +$ case :: ship desk case spur $% [%da p=@da] :: date [%tas p=@tas] :: label diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index b4798db1b2..a72612b563 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -2713,6 +2713,8 @@ %b ~| %i-guess-you-ought-to-build-your-own-marks !! %c ~| %casts-should-be-compiled-on-your-own-ship !! %d ~| %totally-temporary-error-please-replace-me !! + %e ~| %yes-naves-also-shouldnt-cross-the-network !! + %f ~| %even-static-casts-should-be-built-locally !! %p ~| %requesting-foreign-permissions-is-invalid !! %r ~| %no-cages-please-they-are-just-way-too-big !! %s ~| %please-dont-get-your-takos-over-a-network !! @@ -3514,6 +3516,38 @@ (build-tube:(ford:fusion static-ford-args) [i i.t]:path) :_(fod.dom [~ ~ %& %tube !>(tube)]) :: + ++ read-e + !. + |= [=aeon =path] + ^- [(unit (unit (each cage lobe))) ford-cache] + ?. =(aeon let.dom) + [~ fod.dom] + ?. ?=([@ ~] path) + [[~ ~] fod.dom] + =/ cached=(unit [=vase *]) (~(get by naves.fod.dom) i.path) + ?^ cached + :_(fod.dom [~ ~ %& %nave !>(vase.u.cached)]) + =^ =vase fod.dom + %- wrap:fusion + (build-nave:(ford:fusion static-ford-args) i.path) + :_(fod.dom [~ ~ %& %nave !>(vase)]) + :: + ++ read-f + !. + |= [=aeon =path] + ^- [(unit (unit (each cage lobe))) ford-cache] + ?. =(aeon let.dom) + [~ fod.dom] + ?. ?=([@ @ ~] path) + [[~ ~] fod.dom] + =/ cached=(unit [=vase *]) (~(get by casts.fod.dom) [i i.t]:path) + ?^ cached + :_(fod.dom [~ ~ %& %cast vase.u.cached]) + =^ =vase fod.dom + %- wrap:fusion + (build-cast:(ford:fusion static-ford-args) [i i.t]:path) + :_(fod.dom [~ ~ %& %cast vase]) + :: :: Gets the permissions that apply to a particular node. :: :: If the node has no permissions of its own, we use its parent's. @@ -3875,6 +3909,8 @@ %a (read-a yon path.mun) %b (read-b yon path.mun) %c (read-c yon path.mun) + %e (read-e yon path.mun) + %f (read-f yon path.mun) %p :_(fod (read-p path.mun)) %r :_(fod (bind (read-r yon path.mun) (lift |=(a=cage [%& a])))) %s :_(fod (bind (read-s yon path.mun) (lift |=(a=cage [%& a])))) diff --git a/pkg/arvo/ted/build-cast.hoon b/pkg/arvo/ted/build-cast.hoon index 017a2ee82d..6122babca2 100644 --- a/pkg/arvo/ted/build-cast.hoon +++ b/pkg/arvo/ted/build-cast.hoon @@ -9,5 +9,5 @@ ?~ bem=(de-beam pax) (strand-fail:strand %path-not-beam >pax< ~) =/ =mars:clay [i i.t]:?>(?=([@ @ ~] s.u.bem) s.u.bem) -;< =tube:clay bind:m (build-cast:strandio -.u.bem mars) -(pure:m !>(tube)) +;< =vase bind:m (build-cast:strandio -.u.bem mars) +(pure:m vase) diff --git a/pkg/arvo/ted/build-nave.hoon b/pkg/arvo/ted/build-nave.hoon new file mode 100644 index 0000000000..f85cbc520e --- /dev/null +++ b/pkg/arvo/ted/build-nave.hoon @@ -0,0 +1,13 @@ +/- spider +/+ strandio +=, strand=strand:spider +^- thread:spider +|= arg=vase +=/ m (strand ,vase) +^- form:m +=+ !<([~ pax=path] arg) +?~ bem=(de-beam pax) + (strand-fail:strand %path-not-beam >pax< ~) +=/ =mark (rear s.u.bem) +;< =vase bind:m (build-nave:strandio -.u.bem mark) +(pure:m vase) diff --git a/pkg/arvo/ted/build-tube.hoon b/pkg/arvo/ted/build-tube.hoon new file mode 100644 index 0000000000..da075e172b --- /dev/null +++ b/pkg/arvo/ted/build-tube.hoon @@ -0,0 +1,13 @@ +/- spider +/+ strandio +=, strand=strand:spider +^- thread:spider +|= arg=vase +=/ m (strand ,vase) +^- form:m +=+ !<([~ pax=path] arg) +?~ bem=(de-beam pax) + (strand-fail:strand %path-not-beam >pax< ~) +=/ =mars:clay [i i.t]:?>(?=([@ @ ~] s.u.bem) s.u.bem) +;< =tube:clay bind:m (build-tube:strandio -.u.bem mars) +(pure:m !>(tube)) From f5ce79da9de5c67ead6bbff9519a09c1bff3456b Mon Sep 17 00:00:00 2001 From: Philip Monk Date: Thu, 4 Mar 2021 19:21:34 -0800 Subject: [PATCH 093/103] marks: fixup Many of the marks were broken, and staticizing the marks revealed this. This makes them build. --- pkg/arvo/mar/elem.hoon | 2 +- pkg/arvo/mar/helm-hi.hoon | 2 +- pkg/arvo/mar/hymn.hoon | 2 +- pkg/arvo/mar/language-server/rpc/notification.hoon | 2 +- pkg/arvo/mar/language-server/rpc/request.hoon | 2 +- pkg/arvo/mar/pill.hoon | 4 ++-- pkg/arvo/mar/snip.hoon | 3 ++- pkg/arvo/mar/urb.hoon | 2 +- pkg/arvo/sys/vane/clay.hoon | 2 ++ pkg/arvo/ted/aqua/behn.hoon | 10 ++++++---- 10 files changed, 18 insertions(+), 13 deletions(-) diff --git a/pkg/arvo/mar/elem.hoon b/pkg/arvo/mar/elem.hoon index c7533ad908..6ce6f6f7fe 100644 --- a/pkg/arvo/mar/elem.hoon +++ b/pkg/arvo/mar/elem.hoon @@ -6,7 +6,7 @@ =, html |_ own=manx :: -++ grad %mime +++ grad %noun ++ grow :: convert to |% ++ hymn ;html:(head body:"+{own}") :: convert to %hymn diff --git a/pkg/arvo/mar/helm-hi.hoon b/pkg/arvo/mar/helm-hi.hoon index 85c617f615..e1d586411b 100644 --- a/pkg/arvo/mar/helm-hi.hoon +++ b/pkg/arvo/mar/helm-hi.hoon @@ -6,7 +6,7 @@ =, format |_ txt=cord :: -++ grad %mime +++ grad %noun ++ grab :: convert from |% ++ noun @t :: clam from %noun diff --git a/pkg/arvo/mar/hymn.hoon b/pkg/arvo/mar/hymn.hoon index d8ad099697..cf01508eae 100644 --- a/pkg/arvo/mar/hymn.hoon +++ b/pkg/arvo/mar/hymn.hoon @@ -6,7 +6,7 @@ =, html |_ own=manx :: -++ grad %mime +++ grad %noun ++ grow :: convert to |% ++ html (crip (en-xml own)) :: convert to %html diff --git a/pkg/arvo/mar/language-server/rpc/notification.hoon b/pkg/arvo/mar/language-server/rpc/notification.hoon index 88fdaa8c1b..d464da489a 100644 --- a/pkg/arvo/mar/language-server/rpc/notification.hoon +++ b/pkg/arvo/mar/language-server/rpc/notification.hoon @@ -4,7 +4,7 @@ ++ grad %noun ++ grab |% - ++ noun not + ++ noun all:notification ++ json |= jon=^json (notification:dejs:lsp-json jon) diff --git a/pkg/arvo/mar/language-server/rpc/request.hoon b/pkg/arvo/mar/language-server/rpc/request.hoon index 48db6b7eae..989a6f41c4 100644 --- a/pkg/arvo/mar/language-server/rpc/request.hoon +++ b/pkg/arvo/mar/language-server/rpc/request.hoon @@ -8,7 +8,7 @@ -- ++ grab |% - ++ noun req + ++ noun all:request ++ json |= jon=^json (request:dejs:lsp-json jon) diff --git a/pkg/arvo/mar/pill.hoon b/pkg/arvo/mar/pill.hoon index 6d69bd167f..f1e1ffa739 100644 --- a/pkg/arvo/mar/pill.hoon +++ b/pkg/arvo/mar/pill.hoon @@ -15,7 +15,7 @@ ++ mime |= (pair mite octs) =+ o=(pair ,* ,*) :: ,*) - =+ (,[boot-ova=* kernel-ova=(list o) userspace-ova=(list o)] (cue q.q)) + =+ (,[%pill nam=term boot-ova=(list) kernel-ova=(list o) userspace-ova=(list o)] (cue q.q)) =/ convert |= ova=(list o) ^- (list unix-event) @@ -30,7 +30,7 @@ :: =/ boot-ova (convert boot-ova) =/ kernel-ova (convert kernel-ova) =/ userspace-ova (convert userspace-ova) - [boot-ova kernel-ova userspace-ova] + [%pill nam boot-ova kernel-ova userspace-ova] -- ++ grad %mime -- diff --git a/pkg/arvo/mar/snip.hoon b/pkg/arvo/mar/snip.hoon index 205348f495..5760a7bb5e 100644 --- a/pkg/arvo/mar/snip.hoon +++ b/pkg/arvo/mar/snip.hoon @@ -43,7 +43,7 @@ :: =, mimes:html |_ [hed=marl tal=marl] -++ grad %mime +++ grad %noun :: ++ grow :: convert to |% @@ -55,6 +55,7 @@ ++ html (crip (en-xml hymn)) :: convert to %html ++ mime [/text/html (as-octs html)] :: convert to %mime -- + ++ noun [hed tal] -- ++ grab |% :: convert from ++ noun ,[marl marl] :: clam from %noun diff --git a/pkg/arvo/mar/urb.hoon b/pkg/arvo/mar/urb.hoon index 85686ad6ef..49545e73f5 100644 --- a/pkg/arvo/mar/urb.hoon +++ b/pkg/arvo/mar/urb.hoon @@ -6,7 +6,7 @@ =, html |_ own=manx :: -++ grad %mime +++ grad %noun ++ grow :: convert to |% ++ hymn ;html:(head body:"+{own}") :: convert to %hymn diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index a72612b563..af2f653037 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -701,6 +701,8 @@ (compose-casts a !<(mark p.jum) b) ?: ?=(%& -.rab) (compose-casts a !<(mark p.rab) b) + ?: ?=(%noun b) + :_(nub !>(|=(* +<))) ~|(no-cast-from+[a b] !!) :: ++ compose-casts diff --git a/pkg/arvo/ted/aqua/behn.hoon b/pkg/arvo/ted/aqua/behn.hoon index 17a56e0841..1c6c8d8baf 100644 --- a/pkg/arvo/ted/aqua/behn.hoon +++ b/pkg/arvo/ted/aqua/behn.hoon @@ -74,13 +74,15 @@ =. next-timer ~ =. this %- emit-aqua-events + ?^ error + :: Should pass through errors to aqua, but doesn't + :: + %- (slog leaf+'aqua-behn: timer failed' u.error) + ~ :_ ~ ^- aqua-event :+ %event who - :- //behn/0v1n.2m9vh - ?~ error - [%wake ~] - [%crud %fail u.error] + [//behn/0v1n.2m9vh [%wake ~]] ..abet-pe -- -- From 538a5eaf1503585f81a18352e5e31b26ce51ed9e Mon Sep 17 00:00:00 2001 From: Philip Monk Date: Thu, 4 Mar 2021 19:35:31 -0800 Subject: [PATCH 094/103] threads: make compile mostly by deleting them --- pkg/arvo/ted/aqua/behn.hoon | 2 +- pkg/arvo/ted/migrate-channels.hoon | 84 ------------------- pkg/arvo/ted/ph/migrate/make-groups.hoon | 46 ---------- .../ted/ph/migrate/post-import-groups.hoon | 24 ------ .../post-import-metadata-contacts.hoon | 61 -------------- 5 files changed, 1 insertion(+), 216 deletions(-) delete mode 100644 pkg/arvo/ted/migrate-channels.hoon delete mode 100644 pkg/arvo/ted/ph/migrate/make-groups.hoon delete mode 100644 pkg/arvo/ted/ph/migrate/post-import-groups.hoon delete mode 100644 pkg/arvo/ted/ph/migrate/post-import-metadata-contacts.hoon diff --git a/pkg/arvo/ted/aqua/behn.hoon b/pkg/arvo/ted/aqua/behn.hoon index 1c6c8d8baf..517d2f3c72 100644 --- a/pkg/arvo/ted/aqua/behn.hoon +++ b/pkg/arvo/ted/aqua/behn.hoon @@ -77,7 +77,7 @@ ?^ error :: Should pass through errors to aqua, but doesn't :: - %- (slog leaf+'aqua-behn: timer failed' u.error) + %- (slog leaf+"aqua-behn: timer failed" u.error) ~ :_ ~ ^- aqua-event diff --git a/pkg/arvo/ted/migrate-channels.hoon b/pkg/arvo/ted/migrate-channels.hoon deleted file mode 100644 index c6d9937042..0000000000 --- a/pkg/arvo/ted/migrate-channels.hoon +++ /dev/null @@ -1,84 +0,0 @@ -/- spider, *metadata-store -/+ strandio -=, strand=strand:spider -^- thread:spider -|= arg=vase -=/ m (strand ,vase) -^- form:m -=/ [~ og-path=path ng-path=path] !<([~ path path] arg) -;< bol=bowl:spider bind:m get-bowl:strandio -|^ -:: -=/ og=(unit (set ship)) (scry-for (unit (set ship)) %group-store og-path) -?~ og - (pure:m !>("no such group: {}")) -=/ ng=(unit (set ship)) (scry-for (unit (set ship)) %group-store ng-path) -?~ ng - (pure:m !>("no such group: {}")) -:: -=/ assoc=associations (scry-for associations %metadata-store [%group og-path]) -=/ assoc-list=(list [[group-path md-resource] metadata]) ~(tap by assoc) -:: -|- -=* loop $ -?~ assoc-list - ;< ~ bind:m - (poke-our:strandio %group-store %group-action !>([%unbundle og-path])) - (pure:m !>("done")) -=/ [[g-path=group-path res=md-resource] meta=metadata] i.assoc-list -?. =(our.bol creator.meta) - loop(assoc-list t.assoc-list) -?> =(g-path og-path) -=/ output=(list card:agent:gall) - ?+ app-name.res ~ - :: - ?(%chat %link) - %- (slog leaf+"migrating {} : {}" ~) - :~ :* %pass /poke %agent - [our.bol %metadata-hook] - %poke %metadata-action - !>([%add ng-path res meta]) - == - :* %pass /poke %agent - [our.bol %metadata-hook] - %poke %metadata-action - !>([%remove g-path res]) - == - == - %publish - %- (slog leaf+"migrating {} : {}" ~) - =/ book (scry-for notebook %publish [%book app-path.res]) - ?> ?=([@tas @tas ~] app-path.res) - :~ :* %pass /poke %agent - [our.bol %publish] - %poke %publish-action - !> - :* %edit-book - i.t.app-path.res - title.book - description.book - comments.book - `[ng-path ~ %.y %.n] - == - == - :* %pass /poke %agent - [our.bol %metadata-hook] - %poke %metadata-action - !>([%remove g-path res]) - == - == - == -:: -;< ~ bind:m (send-raw-cards:strandio output) -loop(assoc-list t.assoc-list) -:: -++ scry-for - |* [mol=mold app=term pax=path] - .^ mol - %gx - (scot %p our.bol) - app - (scot %da now.bol) - (snoc `path`pax %noun) - == --- diff --git a/pkg/arvo/ted/ph/migrate/make-groups.hoon b/pkg/arvo/ted/ph/migrate/make-groups.hoon deleted file mode 100644 index 62a09dffa6..0000000000 --- a/pkg/arvo/ted/ph/migrate/make-groups.hoon +++ /dev/null @@ -1,46 +0,0 @@ -/- spider, - contact-view, - *resource -/+ *ph-io, strandio -=, strand=strand:spider -:: -^- thread:spider -|= vase -=/ m (strand ,vase) -;< ~ bind:m start-simple -;< bol=bowl:spider bind:m get-bowl:strandio -:: -:: group setup -:: - ~zod creates an open group -:: - ~zod creates and invite-only group, and invites ~bus and ~web -:: - ~bus and ~web join the first, but not the second group, to keep -:: invite-store populated -:: -=/ group-1=contact-view-action:contact-view - :* %create - %group-1 - [%open ~ ~] - 'Group 1' - 'this is group 1' - == -=/ group-2=contact-view-action:contact-view - :* %create - %group-2 - [%invite (sy ~bus ~web ~)] - 'Group 2' - 'this is group 2' - == -=/ join=contact-view-action:contact-view [%join ~zod %group-1] -;< ~ bind:m (poke-app ~zod %contact-view %contact-view-action group-1) -;< ~ bind:m (wait-for-output ~zod ">=") -;< ~ bind:m (poke-app ~zod %contact-view %contact-view-action group-2) -;< ~ bind:m (wait-for-output ~zod ">=") -;< ~ bind:m (sleep ~s10) -;< ~ bind:m (poke-app ~bus %contact-view %contact-view-action join) -;< ~ bind:m (wait-for-output ~bus ">=") -;< ~ bind:m (poke-app ~web %contact-view %contact-view-action join) -;< ~ bind:m (wait-for-output ~web ">=") -;< ~ bind:m (send-hi ~bus ~zod) -;< ~ bind:m (send-hi ~web ~zod) -;< ~ bind:m (sleep ~s2) -(pure:m *vase) diff --git a/pkg/arvo/ted/ph/migrate/post-import-groups.hoon b/pkg/arvo/ted/ph/migrate/post-import-groups.hoon deleted file mode 100644 index 8681c14527..0000000000 --- a/pkg/arvo/ted/ph/migrate/post-import-groups.hoon +++ /dev/null @@ -1,24 +0,0 @@ -/- spider, - contact-view, - *resource, - group-store -/+ *ph-io, strandio -=, strand=strand:spider -:: -^- thread:spider -|= vase -=/ m (strand ,vase) -;< ~ bind:m start-simple -;< bol=bowl:spider bind:m get-bowl:strandio -:: -=/ join-2=contact-view-action:contact-view [%join ~zod %group-2] -=/ add-members-1=action:group-store - [%add-members [~zod %group-1] (sy ~def ~ten ~)] -=/ add-members-2=action:group-store - [%add-members [~zod %group-2] (sy ~def ~ten ~)] -;< ~ bind:m (poke-app ~bus %contact-view %contact-view-action join-2) -;< ~ bind:m (poke-app ~web %contact-view %contact-view-action join-2) -;< ~ bind:m (poke-app ~zod %group-store %group-action add-members-1) -;< ~ bind:m (poke-app ~zod %group-store %group-action add-members-2) -:: -(pure:m *vase) diff --git a/pkg/arvo/ted/ph/migrate/post-import-metadata-contacts.hoon b/pkg/arvo/ted/ph/migrate/post-import-metadata-contacts.hoon deleted file mode 100644 index 81a9825937..0000000000 --- a/pkg/arvo/ted/ph/migrate/post-import-metadata-contacts.hoon +++ /dev/null @@ -1,61 +0,0 @@ -/- spider, - contact-view, - contact-store, - group-store, - metadata-store, - post, - graph-store, - *resource -/+ *ph-io, strandio -=, strand=strand:spider -:: -:: -^- thread:spider -|= vase -=/ m (strand ,vase) -;< ~ bind:m start-simple -;< bol=bowl:spider bind:m get-bowl:strandio -:: -:: test metadata import -:: -=/ change-group-1=metadata-action:metadata-store - :* %add - /ship/~zod/group-1 - [%contacts /ship/~zod/group-1] - 'New Group 1 Title' - 'new description' - 0x0 - now.bol - ~zod - 'fake' - == -=/ change-web-book=metadata-action:metadata-store - :* %add - /ship/~web/graph-3 - [%graph /ship/~web/graph-3] - 'New Graph 3 Title' - 'new description' - 0x0 - now.bol - ~web - 'fake' - == -;< ~ bind:m (poke-app ~zod %metadata-hook %metadata-action change-group-1) -;< ~ bind:m (sleep ~s5) -;< ~ bind:m (poke-app ~web %metadata-hook %metadata-action change-web-book) -;< ~ bind:m (sleep ~s5) -:: -:: test contacts import -:: -=/ add-zod=contact-action:contact-store - :* %add /ship/~zod/group-1 ~zod - 'ZOD' '' '' '' '' 0x0 ~ - == -=/ add-bus=contact-action:contact-store - :* %add /ship/~zod/group-2 ~bus - 'BUS' '' '' '' '' 0x0 ~ - == -;< ~ bind:m (poke-app ~zod %contact-hook %contact-action add-zod) -;< ~ bind:m (sleep ~s5) -;< ~ bind:m (poke-app ~bus %contact-hook %contact-action add-bus) -(pure:m *vase) From 9c9446d77f0969846b1cebd12f6290d513375ad4 Mon Sep 17 00:00:00 2001 From: janeway Date: Thu, 4 Mar 2021 19:45:20 -0800 Subject: [PATCH 095/103] glob: update to 0v8buag.lcoib.dupqj.iprqk.spvqf --- bin/solid.pill | 4 ++-- pkg/arvo/app/glob.hoon | 2 +- pkg/arvo/app/landscape/index.html | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/solid.pill b/bin/solid.pill index a76163bdb0..65f89637de 100644 --- a/bin/solid.pill +++ b/bin/solid.pill @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a14132a9e059f582b03d940e491701d0086d86d3882a4696e160159472aa3fcb -size 9084082 +oid sha256:ec80a42446a1d80974f32bf87283435547441cb7ea3fcd340711df2ce6cbec81 +size 9146390 diff --git a/pkg/arvo/app/glob.hoon b/pkg/arvo/app/glob.hoon index 7bf30ff3d6..fc1b73771d 100644 --- a/pkg/arvo/app/glob.hoon +++ b/pkg/arvo/app/glob.hoon @@ -5,7 +5,7 @@ /- glob /+ default-agent, verb, dbug |% -++ hash 0vah4o5.ark7i.iqunl.or5d7.3rm2e +++ hash 0v8buag.lcoib.dupqj.iprqk.spvqf +$ state-0 [%0 hash=@uv glob=(unit (each glob:glob tid=@ta))] +$ all-states $% state-0 diff --git a/pkg/arvo/app/landscape/index.html b/pkg/arvo/app/landscape/index.html index e86ab509fd..16b761ea2f 100644 --- a/pkg/arvo/app/landscape/index.html +++ b/pkg/arvo/app/landscape/index.html @@ -24,6 +24,6 @@
- + From ac096d85ae847fcfe8786b51039c92c69abc006e Mon Sep 17 00:00:00 2001 From: Philip Monk Date: Thu, 4 Mar 2021 21:17:50 -0800 Subject: [PATCH 096/103] merge: cast to wain to avoid ford bug /* is giving us something with an extra `txt` face wrapped around it, which interacts poorly with wet gates. An explicit cast fixes it. --- pkg/arvo/gen/hood/merge.hoon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/arvo/gen/hood/merge.hoon b/pkg/arvo/gen/hood/merge.hoon index 35991526f3..f929620ca4 100644 --- a/pkg/arvo/gen/hood/merge.hoon +++ b/pkg/arvo/gen/hood/merge.hoon @@ -21,7 +21,7 @@ |^ :- %kiln-merge ^- $@(~ [syd=desk her=ship sud=desk cas=case gem=?(germ %auto)]) ?- arg - ~ ((slog (turn help-text |=(=@t leaf+(trip t)))) ~) + ~ ((slog (turn `wain`help-text |=(=@t leaf+(trip t)))) ~) [@ @ ~] =+(arg [sud ?.(=(our her) her (sein:title p.bek now her)) sud (opt-case da+now) gem]) :: From a1a581c510a7d82286b6fdc6c72e869663c5318e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C5=8Dshin?= Date: Thu, 4 Mar 2021 21:22:25 -0800 Subject: [PATCH 097/103] interface: correct gcgManager comment --- pkg/interface/src/logic/lib/gcpManager.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/interface/src/logic/lib/gcpManager.ts b/pkg/interface/src/logic/lib/gcpManager.ts index 628bcfc29f..69f157c2c5 100644 --- a/pkg/interface/src/logic/lib/gcpManager.ts +++ b/pkg/interface/src/logic/lib/gcpManager.ts @@ -5,10 +5,11 @@ // 1. call configure with a GlobalApi and GlobalStore. // 2. call start() to start the token refresh loop. // -// If the ship has S3 credentials set, we don't try to get a token, but we keep -// checking at regular intervals to see if they get unset. Otherwise, we try to -// invoke the GCP token thread on the ship until it gives us an access token. -// Once we have a token, we refresh it every hour or so, since it has an +// If the ship does not have GCP storage configured, we don't try to get +// a token, but we keep checking at regular intervals to see if it gets +// configured. If GCP storage is configured, we try to invoke the GCP +// get-token thread on the ship until it gives us an access token. Once +// we have a token, we refresh it every hour or so according to its // intrinsic expiry. // // From 803cdba7549bc8863bd71b4afecfb7b4ae9d9e1e Mon Sep 17 00:00:00 2001 From: Matilde Park Date: Fri, 5 Mar 2021 17:59:08 -0500 Subject: [PATCH 098/103] publish: fix link color Fixes urbit/landscape#521 --- .../src/views/apps/publish/components/Note.tsx | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/pkg/interface/src/views/apps/publish/components/Note.tsx b/pkg/interface/src/views/apps/publish/components/Note.tsx index 2bb8e77e01..ba1c8dc538 100644 --- a/pkg/interface/src/views/apps/publish/components/Note.tsx +++ b/pkg/interface/src/views/apps/publish/components/Note.tsx @@ -1,5 +1,5 @@ import React, { useState, useEffect } from 'react'; -import { Box, Text, Col } from '@tlon/indigo-react'; +import { Box, Text, Col, Anchor } from '@tlon/indigo-react'; import ReactMarkdown from 'react-markdown'; import bigInt from 'big-integer'; @@ -32,6 +32,14 @@ export function Note(props: NoteProps & RouteComponentProps) { const { notebook, note, contacts, ship, book, api, rootUrl, baseUrl, group } = props; const editCommentId = props.match.params.commentId; + const renderers = { + link: ({ href, children }) => { + return ( + {children} + ) + } + }; + const deletePost = async () => { setDeleting(true); const indices = [note.post.index]; @@ -107,7 +115,7 @@ export function Note(props: NoteProps & RouteComponentProps) { - + Date: Fri, 5 Mar 2021 18:00:26 -0500 Subject: [PATCH 099/103] chat, publish: styling on light/dark hybrid client If your OS was in light mode but you chose dark mode, sometimes you'd see the wrong colour. We ensure we inherit from indigo's theme in those places. --- pkg/interface/src/views/apps/chat/css/custom.css | 7 +++---- .../src/views/apps/publish/components/MarkdownField.tsx | 1 + pkg/interface/src/views/apps/publish/css/custom.css | 2 ++ 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/pkg/interface/src/views/apps/chat/css/custom.css b/pkg/interface/src/views/apps/chat/css/custom.css index 46c6d68f8e..58b4bb02b9 100644 --- a/pkg/interface/src/views/apps/chat/css/custom.css +++ b/pkg/interface/src/views/apps/chat/css/custom.css @@ -156,7 +156,8 @@ h2 { blockquote { padding: 0 0 0 16px; margin: 0; - border-left: 1px solid black; + color: inherit; + border-left: 1px solid; } :root { @@ -173,6 +174,7 @@ blockquote { height: 100% !important; width: 100% !important; cursor: text; + color: inherit; background: transparent; } @@ -308,9 +310,6 @@ pre.CodeMirror-placeholder.CodeMirror-line-like { /* dark */ @media (prefers-color-scheme: dark) { - blockquote { - border-left: 1px solid inherit; - } /* codemirror */ .chat .cm-s-tlon.CodeMirror { diff --git a/pkg/interface/src/views/apps/publish/components/MarkdownField.tsx b/pkg/interface/src/views/apps/publish/components/MarkdownField.tsx index 611505c733..0a1a1125a5 100644 --- a/pkg/interface/src/views/apps/publish/components/MarkdownField.tsx +++ b/pkg/interface/src/views/apps/publish/components/MarkdownField.tsx @@ -28,6 +28,7 @@ export const MarkdownField = ({ width="100%" display="flex" flexDirection="column" + color="black" {...rest} > Date: Fri, 5 Mar 2021 23:42:25 -0500 Subject: [PATCH 100/103] landscape: fix favicons for flattened contacts --- pkg/interface/src/views/App.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/interface/src/views/App.js b/pkg/interface/src/views/App.js index b23ade2941..77b89f0428 100644 --- a/pkg/interface/src/views/App.js +++ b/pkg/interface/src/views/App.js @@ -119,8 +119,8 @@ class App extends React.Component { faviconString() { let background = '#ffffff'; - if (this.state.contacts.hasOwnProperty('/~/default')) { - background = `#${uxToHex(this.state.contacts['/~/default'][window.ship].color)}`; + if (this.state.contacts.hasOwnProperty(`~${window.ship}`)) { + background = `#${uxToHex(this.state.contacts[`~${window.ship}`].color)}`; } const foreground = foregroundFromBackground(background); const svg = sigiljs({ From 772750b180986b8578084a81f26bcd2b30dabfaf Mon Sep 17 00:00:00 2001 From: Matilde Park Date: Sat, 6 Mar 2021 12:20:17 -0500 Subject: [PATCH 101/103] term: flex history container width --- pkg/interface/src/views/apps/term/app.js | 1 + pkg/interface/src/views/apps/term/components/history.js | 1 + 2 files changed, 2 insertions(+) diff --git a/pkg/interface/src/views/apps/term/app.js b/pkg/interface/src/views/apps/term/app.js index 6413a4f6de..b63a950028 100644 --- a/pkg/interface/src/views/apps/term/app.js +++ b/pkg/interface/src/views/apps/term/app.js @@ -67,6 +67,7 @@ export default class TermApp extends Component { backgroundColor='white' width='100%' minHeight='0' + minWidth='0' color='washedGray' borderRadius='2' mx={['0','3']} diff --git a/pkg/interface/src/views/apps/term/components/history.js b/pkg/interface/src/views/apps/term/components/history.js index 93eb0a0110..f8e4acede6 100644 --- a/pkg/interface/src/views/apps/term/components/history.js +++ b/pkg/interface/src/views/apps/term/components/history.js @@ -13,6 +13,7 @@ export class History extends Component { Date: Mon, 8 Mar 2021 10:27:48 +1000 Subject: [PATCH 102/103] VirtualScroller: document props, remove redundant --- .../src/views/components/VirtualScroller.tsx | 49 +++++++++++++++---- 1 file changed, 39 insertions(+), 10 deletions(-) diff --git a/pkg/interface/src/views/components/VirtualScroller.tsx b/pkg/interface/src/views/components/VirtualScroller.tsx index 369f1914d5..1e929d5a81 100644 --- a/pkg/interface/src/views/components/VirtualScroller.tsx +++ b/pkg/interface/src/views/components/VirtualScroller.tsx @@ -24,19 +24,48 @@ interface RendererProps { } interface VirtualScrollerProps { + /** + * Start scroll from + */ origin: 'top' | 'bottom'; + /** + * Load more of the graph + * + * @returns boolean whether or not the graph is now fully loaded + */ loadRows(newer: boolean): Promise; + /** + * The data to iterate over + */ data: BigIntOrderedMap; - id: string; + /** + * The component to render the items + * + * @remarks + * + * This component must be referentially stable, so either use `useCallback` or + * a instance method. It must also forward the DOM ref from its root DOM node + */ renderer: (props: RendererProps) => JSX.Element | null; onStartReached?(): void; onEndReached?(): void; size: number; - totalSize: number; + /** + * Average height of a single rendered item + * + * @remarks + * This is used primarily to calculate how many items should be onscreen. If + * size is variable, err on the lower side. + */ averageHeight: number; + /** + * The offset to begin rendering at, on load. + * + * @remarks + * This is only looked up once, on component creation. Subsequent changes to + * this prop will have no effect + */ offset: number; - onCalculateVisibleItems?(visibleItems: BigIntOrderedMap): void; - onScroll?({ scrollTop, scrollHeight, windowHeight }): void; style?: any; } @@ -61,6 +90,12 @@ const ZONE_SIZE = IS_IOS ? 10 : 40; // nb: in this file, an index refers to a BigInteger and an offset refers to a // number used to index a listified BigIntOrderedMap +/** + * A virtualscroller for a `BigIntOrderedMap`. + * + * VirtualScroller does not clean up or reset itself, so please use `key` + * to ensure a new instance is created for each BigIntOrderedMap + */ export default class VirtualScroller extends Component, VirtualScrollerState> { /** * A reference to our scroll container @@ -87,8 +122,6 @@ export default class VirtualScroller extends Component extends Component extends Component( @@ -191,14 +222,12 @@ export default class VirtualScroller extends Component { requestAnimationFrame(() => { this.restore(); requestAnimationFrame(() => { - this.isUpdating = false; }); }); From d11d3ad9194ca1aa29819831fe5ac2a61f5849c8 Mon Sep 17 00:00:00 2001 From: Liam Fitzgerald Date: Mon, 8 Mar 2021 10:28:07 +1000 Subject: [PATCH 103/103] LinkWindow: referentially stable renderItem --- .../src/views/apps/links/LinkWindow.tsx | 192 +++++++++++------- 1 file changed, 113 insertions(+), 79 deletions(-) diff --git a/pkg/interface/src/views/apps/links/LinkWindow.tsx b/pkg/interface/src/views/apps/links/LinkWindow.tsx index 28aa704c4f..968d804fdd 100644 --- a/pkg/interface/src/views/apps/links/LinkWindow.tsx +++ b/pkg/interface/src/views/apps/links/LinkWindow.tsx @@ -1,21 +1,21 @@ -import React, { useRef, useCallback, useEffect, useMemo } from 'react'; +import React, { + useRef, + useCallback, + useEffect, + useMemo, + Component, +} from "react"; -import { Col, Text } from '@tlon/indigo-react'; -import bigInt from 'big-integer'; -import { - Association, - Graph, - Unreads, - Group, - Rolodex, -} from '@urbit/api'; +import { Col, Text } from "@tlon/indigo-react"; +import bigInt from "big-integer"; +import { Association, Graph, Unreads, Group, Rolodex } from "@urbit/api"; -import GlobalApi from '~/logic/api/global'; -import VirtualScroller from '~/views/components/VirtualScroller'; -import { LinkItem } from './components/LinkItem'; -import LinkSubmit from './components/LinkSubmit'; -import { isWriter } from '~/logic/lib/group'; -import { StorageState } from '~/types'; +import GlobalApi from "~/logic/api/global"; +import VirtualScroller from "~/views/components/VirtualScroller"; +import { LinkItem } from "./components/LinkItem"; +import LinkSubmit from "./components/LinkSubmit"; +import { isWriter } from "~/logic/lib/group"; +import { StorageState } from "~/types"; interface LinkWindowProps { association: Association; @@ -31,74 +31,108 @@ interface LinkWindowProps { api: GlobalApi; storage: StorageState; } -export function LinkWindow(props: LinkWindowProps) { - const { graph, api, association } = props; - const fetchLinks = useCallback( - async (newer: boolean) => { - return true; - /* stubbed, should we generalize the display of graphs in virtualscroller? */ - }, [] - ); - const first = graph.peekLargest()?.[0]; - const [,,ship, name] = association.resource.split('/'); - const canWrite = isWriter(props.group, association.resource); +const style = { + height: "100%", + width: "100%", + display: "flex", + flexDirection: "column", + alignItems: "center", +}; - const style = useMemo(() => - ({ - height: '100%', - width: '100%', - display: 'flex', - flexDirection: 'column', - alignItems: 'center' - }), []); +export class LinkWindow extends Component { + fetchLinks = async () => true; - if (!first) { - return ( - - { canWrite ? ( - + canWrite() { + const { group, association } = this.props; + return isWriter(group, association.resource); + } + + renderItem = ({ index, scrollWindow }) => { + const { props } = this; + const { association, graph, api } = props; + const [, , ship, name] = association.resource.split("/"); + const node = graph.get(index); + const first = graph.peekLargest()?.[0]; + const post = node?.post; + if (!node || !post) { + return null; + } + const linkProps = { + ...props, + node, + }; + if (this.canWrite() && index.eq(first ?? bigInt.zero)) { + return ( + + + + + + + ); + } + return ; + }; + + render() { + const { graph, api, association, storage } = this.props; + const first = graph.peekLargest()?.[0]; + const [, , ship, name] = association.resource.split("/"); + if (!first) { + return ( + + {this.canWrite() ? ( + ) : ( - There are no links here yet. You do not have permission to post to this collection. - ) - } + + There are no links here yet. You do not have permission to post to + this collection. + + )} + + ); + } + + return ( + + ); } - - return ( - - {}} - onScroll={() => {}} - data={graph} - averageHeight={100} - size={graph.size} - renderer={({ index, scrollWindow }) => { - const node = graph.get(index); - const post = node?.post; - if (!node || !post) -return null; - const linkProps = { - ...props, - node, - }; - if(canWrite && index.eq(first ?? bigInt.zero)) { - return ( - - - - - - - ); - } - return ; - }} - loadRows={fetchLinks} - /> - - ); }