From c630b560ee96de21152b7173d92d7981105d5e25 Mon Sep 17 00:00:00 2001 From: ryjm Date: Fri, 24 Sep 2021 16:24:27 -0400 Subject: [PATCH 01/14] gall: don't verify spider marks --- pkg/arvo/sys/vane/clay.hoon | 2 +- pkg/arvo/sys/vane/gall.hoon | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index 28576e926..ceddb498e 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -1590,7 +1590,7 @@ =. fod.dom ford-cache.args =. ..park (emil (print q.old-yaki data)) :: - =? ..park &(updated !dist-upgraded.ruf) migrate-dist + :: =? ..park &(updated !dist-upgraded.ruf) migrate-dist :: wake:(ergo mim) :: diff --git a/pkg/arvo/sys/vane/gall.hoon b/pkg/arvo/sys/vane/gall.hoon index 41a94bc1a..98cdfd688 100644 --- a/pkg/arvo/sys/vane/gall.hoon +++ b/pkg/arvo/sys/vane/gall.hoon @@ -1408,6 +1408,12 @@ ?. ?=(%raw-fact -.unto) [unto ap-core] =/ =case:clay da+now + ~& > take+beak.yoke + ~& > take+mark.unto + ~& > take+[agent-name agent-routes agent-duct] + ?: ?=(%spider agent-name) + :- [%fact mark.unto !>(noun.unto)] + ap-core =/ sky (rof ~ %cb [our q.beak.yoke case] /[mark.unto]) ?. ?=([~ ~ *] sky) (mean leaf+"gall: ames mark fail {}" ~) From a08c38b83cb1adecc84e4cf5444943bdb7419be8 Mon Sep 17 00:00:00 2001 From: ryjm Date: Sat, 25 Sep 2021 02:50:21 -0400 Subject: [PATCH 02/14] spider: track desks and validate marks (wip) --- pkg/arvo/app/spider.hoon | 45 ++++++++++++++++++++++-------------- pkg/base-dev/lib/strand.hoon | 23 +++++++++++++++++- 2 files changed, 50 insertions(+), 18 deletions(-) diff --git a/pkg/arvo/app/spider.hoon b/pkg/arvo/app/spider.hoon index 0bc957b20..f95faf1c9 100644 --- a/pkg/arvo/app/spider.hoon +++ b/pkg/arvo/app/spider.hoon @@ -18,7 +18,7 @@ $: starting=(map yarn [=trying =vase]) running=trie tid=(map tid yarn) - serving=(map tid [@ta =mark =desk]) + serving=(map tid [(unit @ta) =mark =desk]) == :: +$ clean-slate-any @@ -318,14 +318,10 @@ =* output-mark i.t.t.t.t.site.url =/ =tid (new-thread-id thread) =. serving.state - (~(put by serving.state) tid [eyre-id output-mark desk]) + (~(put by serving.state) tid [`eyre-id output-mark desk]) :: TODO: speed this up somehow. we spend about 15ms in this arm alone :: - =+ .^ - =tube:clay - %cc - /(scot %p our.bowl)/[desk]/(scot %da now.bowl)/json/[input-mark] - == + =/ tube (convert-tube %json input-mark desk bowl) ?> ?=(^ body.request.inbound-request) =/ body=json (need (de-json:html q.u.body.request.inbound-request)) =/ input=vase (slop !>(~) (tube !>(body))) @@ -543,15 +539,11 @@ =- (fall - `state) %+ bind (~(get by serving.state) tid) - |= [eyre-id=@ta output=mark =desk] - =+ .^ - =tube:clay - %cc - /(scot %p our.bowl)/[desk]/(scot %da now.bowl)/[output]/json - == - :_ state(serving (~(del by serving.state) tid)) - %+ give-simple-payload:app:server eyre-id - (json-response:gen:server !<(json (tube vase))) + |= [eyre-id=(unit @ta) output=mark =desk] + =/ tube (convert-tube output %json desk bowl) + :_ state(serving (~(del by serving.state) tid) + %+ give-simple-payload:app:server (need eyre-id) + (json-response:gen:server !<(%json (tube vase)) :: ++ thread-done |= [=yarn =vase] @@ -604,7 +596,7 @@ sup.bowl eny.bowl now.bowl - byk.bowl + (yarn-to-byk yarn bowl) == :: ++ yarn-to-tid @@ -623,7 +615,26 @@ ~ `i.t.nary :: +++ yarn-to-byk + |= [=yarn =bowl] + ^- beak + =/ tid (yarn-to-tid yarn) + %+ bind + (~(get by serving.state) tid) + |= [* * =desk] + =/ boc bec + boc(q desk) +:: ++ clean-state !> ^- clean-slate 3+state(running (turn (tap-yarn running.state) head)) +:: +++ convert-tube + |= [from=mark to=mark =desk =bowl] + .^ + tube:clay + %cc + /(scot %p our.bowl)/[desk]/(scot %da now.bowl)/[from]/[to] + == + -- diff --git a/pkg/base-dev/lib/strand.hoon b/pkg/base-dev/lib/strand.hoon index b618f2d4e..ab54f15fb 100644 --- a/pkg/base-dev/lib/strand.hoon +++ b/pkg/base-dev/lib/strand.hoon @@ -128,6 +128,19 @@ [%done value=a] == :: + ++ validate-mark + |= [in=* =mark =desk =bowl] + =+ .^ + sky=* + %cb + /(scot %p our.bowl)/[desk]/(scot %da now.bowl)/[mark] + == + =+ !<(=dais:clay q.u.u.sky) + =/ res (mule |.((vale:dais in))) + ?: ?=(%| -.res) + (mean leaf+"gall: ames vale fail {}" p.res) + p.res + :: :: Take a new sign and run the strand against it :: ++ take @@ -137,9 +150,16 @@ |= [=eval-form =strand-input] ^- [[(list card) =eval-result] _eval-form] =* take-loop $ + =/ validated-input=^strand-input + =. in.strand-input + %+ bind + in.strand-input + |= in=input:strand + ?. ?=(%sign -.in) in + `(validate-mark q.cage.sign.in p.cage.sign.in q.byk.bowl.strand-input) :: run the strand callback :: - =/ =output (form.eval-form strand-input) + =/ =output (form.eval-form validated-input) :: add cards to cards :: =. cards @@ -165,3 +185,4 @@ -- -- -- +:: From ddf9e769ae518cba8c9d04c12d80eb38cd765d3a Mon Sep 17 00:00:00 2001 From: ryjm Date: Sat, 25 Sep 2021 19:45:05 -0400 Subject: [PATCH 03/14] spider, docket: move back to threads --- pkg/arvo/app/spider.hoon | 78 +++++++++++++++++++++++++------------- pkg/garden/app/docket.hoon | 53 +++----------------------- 2 files changed, 58 insertions(+), 73 deletions(-) diff --git a/pkg/arvo/app/spider.hoon b/pkg/arvo/app/spider.hoon index f95faf1c9..119ae929a 100644 --- a/pkg/arvo/app/spider.hoon +++ b/pkg/arvo/app/spider.hoon @@ -1,5 +1,5 @@ /- spider -/+ libstrand=strand, default-agent, verb, server +/+ libstrand=strand, default-agent, verb, server, dbug =, strand=strand:libstrand ~% %spider-top ..part ~ |% @@ -26,10 +26,19 @@ $% clean-slate-sig clean-slate-1 clean-slate-2 + clean-slate-3 clean-slate == :: +$ clean-slate + $: %4 + starting=(map yarn [=trying =vase]) + running=(list yarn) + tid=(map tid yarn) + serving=(map tid [(unit @ta) =mark =desk]) + == +:: ++$ clean-slate-3 $: %3 starting=(map yarn [=trying =vase]) running=(list yarn) @@ -144,6 +153,7 @@ (welp next-1 next-2) -- :: +%- agent:dbug ^- agent:gall =| =state =< @@ -156,7 +166,7 @@ def ~(. (default-agent this %|) bowl) bec byk.bowl(r da+now.bowl) :: - ++ on-init + ++ on-init ^- (quip card _this) :_ this ~[bind-eyre:sc] @@ -167,15 +177,17 @@ =+ !<(any=clean-slate-any old-state) =? any ?=(^ -.any) (old-to-1 any) =? any ?=(~ -.any) (old-to-1 any) - =^ upgrade-cards any + =^ upgrade-cards any (old-to-2 any) =. any (old-to-3 any) - ?> ?=(%3 -.any) + =. any (old-to-4 any) + ?> ?=(%4 -.any) :: =. tid.state tid.any =/ yarns=(list yarn) %+ welp running.any ~(tap in ~(key by starting.any)) + |- ^- (quip card _this) ?~ yarns [~[bind-eyre:sc] this] @@ -193,8 +205,8 @@ ++ old-to-2 |= old=clean-slate-any ^- (quip card clean-slate-any) - ?> ?=(?(%1 %2 %3) -.old) - ?: ?=(?(%2 %3) -.old) + ?> ?=(?(%1 %2 %3 %4) -.old) + ?: ?=(?(%2 %3 %4) -.old) `old :- ~[bind-eyre:sc] :* %2 @@ -206,9 +218,9 @@ :: ++ old-to-3 |= old=clean-slate-any - ^- clean-slate - ?> ?=(?(%2 %3) -.old) - ?: ?=(%3 -.old) + ^- clean-slate-any + ?> ?=(?(%2 %3 %4) -.old) + ?: ?=(?(%3 %4) -.old) old :* %3 starting.old @@ -216,6 +228,18 @@ tid.old (~(run by serving.old) |=([id=@ta =mark] [id mark q.byk.bowl])) == + ++ old-to-4 + |= old=clean-slate-any + ^- clean-slate + ?> ?=(?(%3 %4) -.old) + ?: ?=(%4 -.old) + old + :* %4 + starting.old + running.old + tid.old + (~(run by serving.old) |=([id=@ta =mark =desk] [`id mark q.byk.bowl])) + == -- :: ++ on-poke @@ -230,7 +254,7 @@ %spider-start (handle-start-thread:sc !<(start-args vase)) %spider-stop (handle-stop-thread:sc !<([tid ?] vase)) :: - %handle-http-request + %handle-http-request (handle-http-request:sc !<([@ta =inbound-request:eyre] vase)) == [cards this] @@ -309,7 +333,7 @@ |= [eyre-id=@ta =inbound-request:eyre] ^- (quip card _state) ::?> authenticated.inbound-request - =/ url + =/ url (parse-request-line:server url.request.inbound-request) ?> ?=([%spider @t @t @t @t ~] site.url) =* desk i.t.site.url @@ -507,11 +531,11 @@ |= [=tid =term =tang] ^- (quip card ^state) =- (fall - `state) - %+ bind + %+ bind (~(get by serving.state) tid) - |= [eyre-id=@ta output=mark =desk] + |= [eyre-id=(unit @ta) output=mark =desk] :_ state(serving (~(del by serving.state) tid)) - %+ give-simple-payload:app:server eyre-id + %+ give-simple-payload:app:server (need eyre-id) ^- simple-payload:http :_ ~ :_ ~ ?. ?=(http-error:spider term) @@ -537,13 +561,13 @@ |= [=tid =vase] ^- (quip card ^state) =- (fall - `state) - %+ bind + %+ bind (~(get by serving.state) tid) |= [eyre-id=(unit @ta) output=mark =desk] =/ tube (convert-tube output %json desk bowl) - :_ state(serving (~(del by serving.state) tid) + :_ state(serving (~(del by serving.state) tid)) %+ give-simple-payload:app:server (need eyre-id) - (json-response:gen:server !<(%json (tube vase)) + (json-response:gen:server !<(json (tube vase))) :: ++ thread-done |= [=yarn =vase] @@ -589,7 +613,7 @@ |= [=yarn =bowl:gall] ^- bowl:spider :* our.bowl - src.bowl + src.bowl (yarn-to-tid yarn) (yarn-to-parent yarn) wex.bowl @@ -616,21 +640,23 @@ `i.t.nary :: ++ yarn-to-byk - |= [=yarn =bowl] + |= [=yarn =bowl:gall] ^- beak =/ tid (yarn-to-tid yarn) - %+ bind - (~(get by serving.state) tid) - |= [* * =desk] - =/ boc bec - boc(q desk) + =/ byk + %+ bind + (~(get by serving.state) tid) + |= [* * =desk] + =/ boc bec + boc(q desk) + ?~ byk byk.bowl u.byk :: ++ clean-state !> ^- clean-slate - 3+state(running (turn (tap-yarn running.state) head)) + 4+state(running (turn (tap-yarn running.state) head)) :: ++ convert-tube - |= [from=mark to=mark =desk =bowl] + |= [from=mark to=mark =desk =bowl:gall] .^ tube:clay %cc diff --git a/pkg/garden/app/docket.hoon b/pkg/garden/app/docket.hoon index d7f293bbc..7f3457fde 100644 --- a/pkg/garden/app/docket.hoon +++ b/pkg/garden/app/docket.hoon @@ -274,53 +274,17 @@ ?~ p.sign `state ((slog leaf+"Failed to uninstall %{(trip desk)}" u.p.sign) `state) :: - [%glob @ %ames @ ~] - ?- -.sign - %kick `state - %poke-ack ~&([dap.bowl %unexpected-poke-ack] `state) - :: - %watch-ack - ?~ p.sign `state - %- %- slog - leaf+"docket: failed to fetch glob over ames for {(trip desk)}" - =. charges (new-chad:cha hung+'failed to fetch glob via ames') - [~[add-fact:cha] state] - :: - %fact - ?. =(%glob p.cage.sign) - ~& [dap.bowl %unexpected-fact from=src.bowl p.cage.sign] - `state - =+ !<(=glob q.cage.sign) - =/ =docket docket:(~(got by charges) desk) - ?. ?=([%glob @ @ %ames *] href.docket) - `state - =* want=@uv hash.glob-reference.href.docket - =/ plea=@uv (slav %uv i.t.wire) - ?. =(want plea) - :: we requested this at some point but no longer want it - :: - `state - =/ have=@uv (hash-glob glob) - ?. =(want have) - %. `state - %- slog - :~ leaf+"docket: glob hash mismatch on {} from {}" - leaf+"expected: {}" - leaf+"received: {}" - == - =. charges (new-chad:cha glob+glob) - =. by-base (~(put by by-base) base.href.docket desk) - [~[add-fact:cha] state] - == - :: - [%glob @ %http @ ~] + ?([%glob @ %http @ ~] [%glob @ %ames @ ~]) ?- -.sign %kick `state :: ?(%poke-ack %watch-ack) ?~ p.sign `state + =/ err =/ act=tape ?:(?=(%poke-ack -.sign) "start" "listen") - =. charges (new-chad:cha hung+'glob-failed') + =. charges %- new-chad:cha + ?: ?=(i.t.t.wire %http) hung+'glob-failed' + hung+'failed to fetch glob via ames' :- ~[add-fact:cha] ((slog leaf+"docket: couldn't {act} thread; will retry" u.p.sign) state) :: @@ -671,12 +635,7 @@ ?: =(our.bowl ship.location.ref) ~> %slog.0^leaf/"docket: awaiting manual glob for {} desk" ~ - ~> %slog.0^leaf/"docket: fetching ames glob for {} desk" - :_ ~ - %+ watch:(pass (glob-wire ref)) - [ship.location.ref %docket] - /glob/[base.href.docket.charge]/(scot %uv hash.ref) - ~> %slog.0^leaf/"docket: fetching http glob for {} desk" + ~> %slog.0^leaf/"docket: fetching {<-.location.ref>} glob for {} desk" =/ =cage spider-start+!>([~ `tid byk.bowl(r da+now.bowl) %glob !>(`[ref desk])]) :~ (watch-our:(pass (glob-wire ref)) %spider /thread-result/[tid]) (poke-our:(pass (glob-wire ref)) %spider cage) From 571649c6313e0f9d906b092b9c532b543e9510e5 Mon Sep 17 00:00:00 2001 From: ryjm Date: Sat, 25 Sep 2021 20:55:26 -0400 Subject: [PATCH 04/14] spider: strand fixes --- pkg/base-dev/lib/strand.hoon | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/pkg/base-dev/lib/strand.hoon b/pkg/base-dev/lib/strand.hoon index ab54f15fb..1ff5256cb 100644 --- a/pkg/base-dev/lib/strand.hoon +++ b/pkg/base-dev/lib/strand.hoon @@ -135,7 +135,8 @@ %cb /(scot %p our.bowl)/[desk]/(scot %da now.bowl)/[mark] == - =+ !<(=dais:clay q.u.u.sky) + :: TODO less shit + =+ !<(=dais:clay +.-.-.sky) =/ res (mule |.((vale:dais in))) ?: ?=(%| -.res) (mean leaf+"gall: ames vale fail {}" p.res) @@ -152,12 +153,15 @@ =* take-loop $ =/ validated-input=^strand-input =. in.strand-input - %+ bind - in.strand-input - |= in=input:strand - ?. ?=(%sign -.in) in - `(validate-mark q.cage.sign.in p.cage.sign.in q.byk.bowl.strand-input) - :: run the strand callback + %+ ^bind + in.strand-input + |= in=input + ?. ?=(%sign -.in) in + ?. ?=(%fact sign-arvo.in) in + :: TODO less shit + (validate-mark q.cage.sign-arvo.in p.cage.sign-arvo.in q.byk.bowl.strand-input) + strand-input + :: run the stranad callback :: =/ =output (form.eval-form validated-input) :: add cards to cards From 3fd6722f907549a430e6ab2c94bb529622c90136 Mon Sep 17 00:00:00 2001 From: ryjm Date: Sat, 25 Sep 2021 21:36:55 -0400 Subject: [PATCH 05/14] garden: update glob thread for ames --- pkg/garden/ted/glob.hoon | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pkg/garden/ted/glob.hoon b/pkg/garden/ted/glob.hoon index 15d9301c8..18377af59 100644 --- a/pkg/garden/ted/glob.hoon +++ b/pkg/garden/ted/glob.hoon @@ -9,7 +9,7 @@ |^ ?- -.location.ref %http (fetch-http [url.location hash]:ref) - %ames !! ::NOTE done within docket itself + %ames (fetch-ames [ship.location hash]:ref base) == :: ++ fetch-http @@ -25,4 +25,14 @@ =- ?>(=(hash (sham -)) -) ;;(=glob:docket (cue cord)) (pure:m !>(glob)) +:: +:: download from ship's docket state +++ fetch-ames + |= [[=ship hash=@uvH] base=term] + ^- form:m + ;< =bowl:spider bind:m get-bowl:strandio + ;< =cage bind:m + (watch-one:strandio /glob/(scot %da now.bowl) [ship %docket] /glob/[base]/(scot %uv hash)) + ?> ?=(%glob p.cage) + (pure:m q.cage) -- From c8cf0bb98dc5bdeab73b26278cb4aed5f702cde2 Mon Sep 17 00:00:00 2001 From: ryjm Date: Mon, 27 Sep 2021 10:34:41 -0400 Subject: [PATCH 06/14] garden spider: (testing) correctly validate just need to properly store the desk in `serving:spider` now. --- pkg/base-dev/lib/strand.hoon | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pkg/base-dev/lib/strand.hoon b/pkg/base-dev/lib/strand.hoon index 1ff5256cb..37aae40c7 100644 --- a/pkg/base-dev/lib/strand.hoon +++ b/pkg/base-dev/lib/strand.hoon @@ -152,14 +152,16 @@ ^- [[(list card) =eval-result] _eval-form] =* take-loop $ =/ validated-input=^strand-input - =. in.strand-input + =/ pox %+ ^bind in.strand-input |= in=input - ?. ?=(%sign -.in) in - ?. ?=(%fact sign-arvo.in) in + ~& input+-.in + ?. ?=(%agent -.in) ~& sign+-.in in + ?. ?=(%fact -.sign.in) ~& sign+-.sign.in in :: TODO less shit - (validate-mark q.cage.sign-arvo.in p.cage.sign-arvo.in q.byk.bowl.strand-input) + (validate-mark q.cage.sign.in p.cage.sign.in q.byk.bowl.strand-input bowl.strand-input) + ~& > pox+~ strand-input :: run the stranad callback :: From 121a7c94683d655b08e928ce026bc8a557ba0d96 Mon Sep 17 00:00:00 2001 From: ryjm Date: Sun, 26 Sep 2021 18:30:12 -0400 Subject: [PATCH 07/14] garden spider: add desk tracking --- pkg/arvo/app/spider.hoon | 6 +++++- pkg/base-dev/lib/strand.hoon | 16 ++++++++-------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/pkg/arvo/app/spider.hoon b/pkg/arvo/app/spider.hoon index 119ae929a..041341531 100644 --- a/pkg/arvo/app/spider.hoon +++ b/pkg/arvo/app/spider.hoon @@ -404,7 +404,10 @@ :: =: starting.state (~(put by starting.state) yarn [%build vase]) tid.state (~(put by tid.state) new-tid yarn) - == + serving.state + ?. (~(has by serving.state) yrn) serving.state + (~(put by serving.state) yarn) + == =/ pax=path ~| no-file-for-thread+file (need (get-fit:clay beak %ted file)) @@ -597,6 +600,7 @@ =/ =tid (yarn-to-tid yarn) =: running.state (del-yarn running.state yarn) tid.state (~(del by tid.state) tid) + serving.state (~(del by serving.state) yarn) == :_ state %+ murn ~(tap by wex.bowl) diff --git a/pkg/base-dev/lib/strand.hoon b/pkg/base-dev/lib/strand.hoon index 37aae40c7..49265a2f6 100644 --- a/pkg/base-dev/lib/strand.hoon +++ b/pkg/base-dev/lib/strand.hoon @@ -130,17 +130,17 @@ :: ++ validate-mark |= [in=* =mark =desk =bowl] - =+ .^ - sky=* + + ~& validate+[mark desk] + =+ .^ + =dais %cb /(scot %p our.bowl)/[desk]/(scot %da now.bowl)/[mark] == - :: TODO less shit - =+ !<(=dais:clay +.-.-.sky) - =/ res (mule |.((vale:dais in))) - ?: ?=(%| -.res) - (mean leaf+"gall: ames vale fail {}" p.res) - p.res + =/ res (mule |.((vale:dais in))) + ?: ?=(%| -.res) + ~|(%spider-mark-fail (mean leaf+"spider: ames vale fail {}" p.res)) + p.res :: :: Take a new sign and run the strand against it :: From 035b29f0c022c2c55f8318ea080bcf89a1f1672e Mon Sep 17 00:00:00 2001 From: ryjm Date: Sun, 26 Sep 2021 19:30:59 -0400 Subject: [PATCH 08/14] spider: strand fixes --- pkg/base-dev/lib/strand.hoon | 41 ++++++++++++++++-------------------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/pkg/base-dev/lib/strand.hoon b/pkg/base-dev/lib/strand.hoon index 49265a2f6..8e2b1fc03 100644 --- a/pkg/base-dev/lib/strand.hoon +++ b/pkg/base-dev/lib/strand.hoon @@ -129,15 +129,13 @@ == :: ++ validate-mark - |= [in=* =mark =desk =bowl] - - ~& validate+[mark desk] - =+ .^ - =dais - %cb - /(scot %p our.bowl)/[desk]/(scot %da now.bowl)/[mark] - == - =/ res (mule |.((vale:dais in))) + |= [in=* =mark =bowl] + ^- cage + ~& validate+[mark q.byk.bowl] + =+ .^ =dais:clay %cb + /(scot %p our.bowl)/[q.byk.bowl]/(scot %da now.bowl)/[mark] + == + =/ res (mule |.((vale.dais in))) ?: ?=(%| -.res) ~|(%spider-mark-fail (mean leaf+"spider: ames vale fail {}" p.res)) p.res @@ -151,21 +149,18 @@ |= [=eval-form =strand-input] ^- [[(list card) =eval-result] _eval-form] =* take-loop $ - =/ validated-input=^strand-input - =/ pox - %+ ^bind - in.strand-input - |= in=input - ~& input+-.in - ?. ?=(%agent -.in) ~& sign+-.in in - ?. ?=(%fact -.sign.in) ~& sign+-.sign.in in - :: TODO less shit - (validate-mark q.cage.sign.in p.cage.sign.in q.byk.bowl.strand-input bowl.strand-input) - ~& > pox+~ - strand-input - :: run the stranad callback + =. in.strand-input + ?~ in.strand-input ~ + =/ in u.in.strand-input + ?. ?=(%agent -.in) `in + ?. ?=(%fact -.sign.in) `in + :: + :- ~ + :+ %agent wire.in + [%fact (validate-mark q.cage.sign.in p.cage.sign.in bowl.strand-input)] + :: run the strand callback :: - =/ =output (form.eval-form validated-input) + =/ =output (form.eval-form strand-input) :: add cards to cards :: =. cards From 098587bf8bc1b9c38a6d60462aa3e7ed36019b71 Mon Sep 17 00:00:00 2001 From: ryjm Date: Mon, 27 Sep 2021 02:43:34 -0400 Subject: [PATCH 09/14] spider: fix mark validation glob is succesfully downloaded and inserted into the docket state, but as an octet stream. need to define the correct marks. --- pkg/arvo/app/spider.hoon | 6 +++--- pkg/base-dev/lib/strand.hoon | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/arvo/app/spider.hoon b/pkg/arvo/app/spider.hoon index 041341531..b8659d046 100644 --- a/pkg/arvo/app/spider.hoon +++ b/pkg/arvo/app/spider.hoon @@ -405,8 +405,8 @@ =: starting.state (~(put by starting.state) yarn [%build vase]) tid.state (~(put by tid.state) new-tid yarn) serving.state - ?. (~(has by serving.state) yrn) serving.state - (~(put by serving.state) yarn) + ?. (~(has by serving.state) new-tid) serving.state + (~(put by serving.state) new-tid [~ %noun q.beak]) == =/ pax=path ~| no-file-for-thread+file @@ -476,7 +476,7 @@ ^- (quip card ^state) =/ m (strand ,vase) ?. (has-yarn running.state yarn) - %- (slog leaf+"spider got input for non-existent {} 2" ~) + %- (slog leaf+"spider got input for non-existent {}" ~) `state =/ =eval-form:eval:m thread-form:(need (get-yarn running.state yarn)) diff --git a/pkg/base-dev/lib/strand.hoon b/pkg/base-dev/lib/strand.hoon index 8e2b1fc03..13713640d 100644 --- a/pkg/base-dev/lib/strand.hoon +++ b/pkg/base-dev/lib/strand.hoon @@ -138,7 +138,7 @@ =/ res (mule |.((vale.dais in))) ?: ?=(%| -.res) ~|(%spider-mark-fail (mean leaf+"spider: ames vale fail {}" p.res)) - p.res + [mark p.res] :: :: Take a new sign and run the strand against it :: From 47a6aa27537222f0326c75978904df3f3793972f Mon Sep 17 00:00:00 2001 From: ryjm Date: Mon, 27 Sep 2021 15:02:41 -0400 Subject: [PATCH 10/14] spider garden: remove sigpams and move glob mark --- pkg/arvo/sys/vane/gall.hoon | 3 --- pkg/base-dev/lib/strand.hoon | 1 - pkg/base-dev/mar/glob.hoon | 11 ----------- pkg/garden/mar/glob.hoon | 12 +++++++++++- 4 files changed, 11 insertions(+), 16 deletions(-) delete mode 100644 pkg/base-dev/mar/glob.hoon mode change 120000 => 100644 pkg/garden/mar/glob.hoon diff --git a/pkg/arvo/sys/vane/gall.hoon b/pkg/arvo/sys/vane/gall.hoon index 98cdfd688..25086045d 100644 --- a/pkg/arvo/sys/vane/gall.hoon +++ b/pkg/arvo/sys/vane/gall.hoon @@ -1408,9 +1408,6 @@ ?. ?=(%raw-fact -.unto) [unto ap-core] =/ =case:clay da+now - ~& > take+beak.yoke - ~& > take+mark.unto - ~& > take+[agent-name agent-routes agent-duct] ?: ?=(%spider agent-name) :- [%fact mark.unto !>(noun.unto)] ap-core diff --git a/pkg/base-dev/lib/strand.hoon b/pkg/base-dev/lib/strand.hoon index 13713640d..a955daec8 100644 --- a/pkg/base-dev/lib/strand.hoon +++ b/pkg/base-dev/lib/strand.hoon @@ -131,7 +131,6 @@ ++ validate-mark |= [in=* =mark =bowl] ^- cage - ~& validate+[mark q.byk.bowl] =+ .^ =dais:clay %cb /(scot %p our.bowl)/[q.byk.bowl]/(scot %da now.bowl)/[mark] == diff --git a/pkg/base-dev/mar/glob.hoon b/pkg/base-dev/mar/glob.hoon deleted file mode 100644 index 5c23a80cf..000000000 --- a/pkg/base-dev/mar/glob.hoon +++ /dev/null @@ -1,11 +0,0 @@ -|_ gob=(map path mime) -++ grow - |% - ++ noun gob - -- -++ grab - |% - ++ noun (map path mime) - -- -++ grad ~&(%grid-mar %noun) --- diff --git a/pkg/garden/mar/glob.hoon b/pkg/garden/mar/glob.hoon deleted file mode 120000 index 15b77e5a3..000000000 --- a/pkg/garden/mar/glob.hoon +++ /dev/null @@ -1 +0,0 @@ -../../base-dev/mar/glob.hoon \ No newline at end of file diff --git a/pkg/garden/mar/glob.hoon b/pkg/garden/mar/glob.hoon new file mode 100644 index 000000000..5c23a80cf --- /dev/null +++ b/pkg/garden/mar/glob.hoon @@ -0,0 +1,11 @@ +|_ gob=(map path mime) +++ grow + |% + ++ noun gob + -- +++ grab + |% + ++ noun (map path mime) + -- +++ grad ~&(%grid-mar %noun) +-- From 472f16fd2edada8b20c1492e6947579f8559711d Mon Sep 17 00:00:00 2001 From: ryjm Date: Mon, 27 Sep 2021 16:51:33 -0400 Subject: [PATCH 11/14] garden spider: address comments --- pkg/arvo/app/spider.hoon | 11 ++++++++--- pkg/arvo/sys/vane/clay.hoon | 4 +--- pkg/garden/app/docket.hoon | 10 +++++----- pkg/garden/ted/glob.hoon | 2 -- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/pkg/arvo/app/spider.hoon b/pkg/arvo/app/spider.hoon index b8659d046..e070d60d7 100644 --- a/pkg/arvo/app/spider.hoon +++ b/pkg/arvo/app/spider.hoon @@ -187,7 +187,6 @@ =/ yarns=(list yarn) %+ welp running.any ~(tap in ~(key by starting.any)) - |- ^- (quip card _this) ?~ yarns [~[bind-eyre:sc] this] @@ -538,7 +537,10 @@ (~(get by serving.state) tid) |= [eyre-id=(unit @ta) output=mark =desk] :_ state(serving (~(del by serving.state) tid)) - %+ give-simple-payload:app:server (need eyre-id) + ?~ eyre-id + %- (slog leaf+"spider got input for non-existent eyre-id in thread {}" ~) + `state + %+ give-simple-payload:app:server u.eyre-id ^- simple-payload:http :_ ~ :_ ~ ?. ?=(http-error:spider term) @@ -567,9 +569,12 @@ %+ bind (~(get by serving.state) tid) |= [eyre-id=(unit @ta) output=mark =desk] + ?~ eyre-id + %- (slog leaf+"spider got input for non-existent eyre-id in thread {}" ~) + `state =/ tube (convert-tube output %json desk bowl) :_ state(serving (~(del by serving.state) tid)) - %+ give-simple-payload:app:server (need eyre-id) + %+ give-simple-payload:app:server u.eyre-id (json-response:gen:server !<(json (tube vase))) :: ++ thread-done diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index ceddb498e..9e040c482 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -1589,9 +1589,7 @@ =. mim.dom (apply-changes-to-mim mim.dom mim) =. fod.dom ford-cache.args =. ..park (emil (print q.old-yaki data)) - :: - :: =? ..park &(updated !dist-upgraded.ruf) migrate-dist - :: + =? ..park &(updated !dist-upgraded.ruf) migrate-dist wake:(ergo mim) :: ++ migrate-dist diff --git a/pkg/garden/app/docket.hoon b/pkg/garden/app/docket.hoon index 7f3457fde..53a02ceff 100644 --- a/pkg/garden/app/docket.hoon +++ b/pkg/garden/app/docket.hoon @@ -280,12 +280,12 @@ :: ?(%poke-ack %watch-ack) ?~ p.sign `state - =/ err =/ act=tape ?:(?=(%poke-ack -.sign) "start" "listen") - =. charges %- new-chad:cha - ?: ?=(i.t.t.wire %http) hung+'glob-failed' - hung+'failed to fetch glob via ames' - :- ~[add-fact:cha] + =. charges + %- new-chad:cha + ?: ?=(i.t.t.wire %http) + hung+'failed to fetch glob via http' + hung+'failed to fetch glob via ames' ((slog leaf+"docket: couldn't {act} thread; will retry" u.p.sign) state) :: %fact diff --git a/pkg/garden/ted/glob.hoon b/pkg/garden/ted/glob.hoon index 18377af59..3f5f4e1e0 100644 --- a/pkg/garden/ted/glob.hoon +++ b/pkg/garden/ted/glob.hoon @@ -21,8 +21,6 @@ ;< =cord bind:n (fetch-cord:strandio (trip url)) %- pure:n %- mole - |. - =- ?>(=(hash (sham -)) -) ;;(=glob:docket (cue cord)) (pure:m !>(glob)) :: From 7fd9e6fcaad85bea3c53f37e5df105ead64c5b56 Mon Sep 17 00:00:00 2001 From: ryjm Date: Mon, 27 Sep 2021 17:12:56 -0400 Subject: [PATCH 12/14] spider garden: restore |. --- pkg/garden/ted/glob.hoon | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/garden/ted/glob.hoon b/pkg/garden/ted/glob.hoon index 3f5f4e1e0..7774704a5 100644 --- a/pkg/garden/ted/glob.hoon +++ b/pkg/garden/ted/glob.hoon @@ -21,6 +21,7 @@ ;< =cord bind:n (fetch-cord:strandio (trip url)) %- pure:n %- mole + |. ;;(=glob:docket (cue cord)) (pure:m !>(glob)) :: From bf534ffe71f71b2d853de3f8d35cfb7a71c59741 Mon Sep 17 00:00:00 2001 From: ryjm Date: Mon, 27 Sep 2021 22:09:35 -0400 Subject: [PATCH 13/14] spider: clean serving threads --- pkg/arvo/app/spider.hoon | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/pkg/arvo/app/spider.hoon b/pkg/arvo/app/spider.hoon index e070d60d7..0ef8a33ee 100644 --- a/pkg/arvo/app/spider.hoon +++ b/pkg/arvo/app/spider.hoon @@ -403,9 +403,7 @@ :: =: starting.state (~(put by starting.state) yarn [%build vase]) tid.state (~(put by tid.state) new-tid yarn) - serving.state - ?. (~(has by serving.state) new-tid) serving.state - (~(put by serving.state) new-tid [~ %noun q.beak]) + serving.state (~(put by serving.state) new-tid [~ %noun q.beak]) == =/ pax=path ~| no-file-for-thread+file @@ -539,7 +537,7 @@ :_ state(serving (~(del by serving.state) tid)) ?~ eyre-id %- (slog leaf+"spider got input for non-existent eyre-id in thread {}" ~) - `state + ~ %+ give-simple-payload:app:server u.eyre-id ^- simple-payload:http :_ ~ :_ ~ @@ -605,7 +603,7 @@ =/ =tid (yarn-to-tid yarn) =: running.state (del-yarn running.state yarn) tid.state (~(del by tid.state) tid) - serving.state (~(del by serving.state) yarn) + serving.state (~(del by serving.state) (yarn-to-tid yarn)) == :_ state %+ murn ~(tap by wex.bowl) @@ -650,15 +648,12 @@ :: ++ yarn-to-byk |= [=yarn =bowl:gall] - ^- beak - =/ tid (yarn-to-tid yarn) - =/ byk - %+ bind - (~(get by serving.state) tid) - |= [* * =desk] - =/ boc bec - boc(q desk) - ?~ byk byk.bowl u.byk + + =/ [* * =desk] + ~| "no desk associated with {}" + %- ~(got by serving.state) (yarn-to-tid yarn) + =/ boc bec + boc(q desk) :: ++ clean-state !> ^- clean-slate From ed5e7e84ce290c3a7e29d2f0752c382cd0c56355 Mon Sep 17 00:00:00 2001 From: ryjm Date: Mon, 27 Sep 2021 22:17:43 -0400 Subject: [PATCH 14/14] spider: remove noisy logs we weren't logging these anyway --- pkg/arvo/app/spider.hoon | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/arvo/app/spider.hoon b/pkg/arvo/app/spider.hoon index 0ef8a33ee..b2be418b4 100644 --- a/pkg/arvo/app/spider.hoon +++ b/pkg/arvo/app/spider.hoon @@ -536,7 +536,6 @@ |= [eyre-id=(unit @ta) output=mark =desk] :_ state(serving (~(del by serving.state) tid)) ?~ eyre-id - %- (slog leaf+"spider got input for non-existent eyre-id in thread {}" ~) ~ %+ give-simple-payload:app:server u.eyre-id ^- simple-payload:http @@ -568,7 +567,6 @@ (~(get by serving.state) tid) |= [eyre-id=(unit @ta) output=mark =desk] ?~ eyre-id - %- (slog leaf+"spider got input for non-existent eyre-id in thread {}" ~) `state =/ tube (convert-tube output %json desk bowl) :_ state(serving (~(del by serving.state) tid))