From 016719ca6b914bde4f1da55c637ea4555be0999c Mon Sep 17 00:00:00 2001 From: Philip Monk Date: Wed, 15 Jul 2020 12:26:55 -0700 Subject: [PATCH] glob: added app to distribute js outside clay --- pkg/arvo/app/eth-watcher.hoon | 8 +- pkg/arvo/app/file-server.hoon | 83 +++++++++++------ pkg/arvo/app/glob.hoon | 163 ++++++++++++++++++++++++++++++++++ pkg/arvo/lib/strandio.hoon | 12 ++- pkg/arvo/sur/file-server.hoon | 2 + pkg/arvo/sur/glob.hoon | 3 + pkg/arvo/ted/glob.hoon | 12 +++ 7 files changed, 249 insertions(+), 34 deletions(-) create mode 100644 pkg/arvo/app/glob.hoon create mode 100644 pkg/arvo/sur/glob.hoon create mode 100644 pkg/arvo/ted/glob.hoon diff --git a/pkg/arvo/app/eth-watcher.hoon b/pkg/arvo/app/eth-watcher.hoon index 426523586..0a4355a47 100644 --- a/pkg/arvo/app/eth-watcher.hoon +++ b/pkg/arvo/app/eth-watcher.hoon @@ -318,7 +318,11 @@ %watch-ack ?~ p.sign [~ this] - %- (slog leaf+"eth-watcher couldn't start listen to thread" u.p.sign) + %- (slog leaf+"eth-watcher couldn't start listening to thread" u.p.sign) + :: TODO: kill thread that may have started, although it may not + :: have started yet since we get this response before the + :: %start-spider poke is processed + :: [~ (clear-running t.wire)] :: %kick [~ (clear-running t.wire)] @@ -413,7 +417,7 @@ ++ on-arvo |= [=wire =sign-arvo] ^- (quip card agent:gall) - ?+ +<.sign-arvo ~|([%strange-sign-arvo -.sign-arvo] !!) + ?+ +<.sign-arvo ~|([%strange-sign-arvo -.sign-arvo] !!) %wake ?. ?=([%timer *] wire) ~& weird-wire=wire [~ this] =* path t.wire diff --git a/pkg/arvo/app/file-server.hoon b/pkg/arvo/app/file-server.hoon index adc4e7125..d4b741bac 100644 --- a/pkg/arvo/app/file-server.hoon +++ b/pkg/arvo/app/file-server.hoon @@ -1,4 +1,4 @@ -/- srv=file-server +/- srv=file-server, glob /+ *server, default-agent, verb, dbug |% +$ card card:agent:gall @@ -6,7 +6,11 @@ $% state-zero == :: -+$ serving (map url-base=path [clay-base=path public=?]) ++$ serving (map url-base=path [=content public=?]) ++$ content + $% [%clay =path] + [%glob =glob:glob] + == +$ state-zero $: %0 =configuration:srv @@ -33,7 +37,7 @@ %+ turn ^- (list path) [/ /'~landscape' ~] - |=(pax=path [pax [/app/landscape %.n]]) + |=(pax=path [pax [clay+/app/landscape %.n]]) == :~ (connect /) (connect /'~landscape') @@ -75,7 +79,14 @@ ?: (~(has by serving) url-base) ~|("url already bound to {<(~(got by serving) url-base.act)>}" !!) :- [%pass url-base %arvo %e %connect [~ url-base] %file-server]~ - this(serving (~(put by serving) url-base [clay-base.act public.act])) + this(serving (~(put by serving) url-base clay+clay-base.act public.act)) + :: + %serve-glob + =* url-base url-base.act + ?: (~(has by serving) url-base) + ~|("url already bound to {<(~(got by serving) url-base.act)>}" !!) + :- [%pass url-base %arvo %e %connect [~ url-base] %file-server]~ + this(serving (~(put by serving) url-base glob+glob.act public.act)) :: %unserve-dir :- [%pass url-base.act %arvo %e %disconnect [~ url-base.act]]~ @@ -84,9 +95,9 @@ %toggle-permission ?. (~(has by serving) url-base.act) ~|("url is not bound" !!) - =/ [clay-base=path public=?] (~(got by serving) url-base.act) + =/ [=content public=?] (~(got by serving) url-base.act) :- ~ - this(serving (~(put by serving) url-base.act [clay-base !public])) + this(serving (~(put by serving) url-base.act [content !public])) :: %set-landscape-homepage-prefix =. landscape-homepage-prefix.configuration prefix.act @@ -133,22 +144,36 @@ |= req-line=request-line ^- [simple-payload:http ?] =/ pax=path (snoc site.req-line (need ext.req-line)) - =/ clay-path=(unit [path ?]) (get-clay-path pax) - ?~ clay-path [not-found:gen %.n] - =/ scry-path - :* (scot %p our.bowl) - q.byk.bowl - (scot %da now.bowl) - (lowercase -.u.clay-path) + =/ content=(unit [=content suffix=path public=?]) (get-content pax) + ?~ content [not-found:gen %.n] + ?- -.content.u.content + %clay + =/ scry-path + :* (scot %p our.bowl) + q.byk.bowl + (scot %da now.bowl) + (lowercase (weld path.content.u.content suffix.u.content)) + == + ?. .^(? %cu scry-path) [not-found:gen %.n] + =/ file (as-octs:mimes:html .^(@ %cx scry-path)) + :_ public.u.content + ?+ ext.req-line not-found:gen + [~ %html] (html-response:gen file) + [~ %js] (js-response:gen file) + [~ %css] (css-response:gen file) + [~ %png] (png-response:gen file) == - ?. .^(? %cu scry-path) [not-found:gen %.n] - =/ file (as-octs:mimes:html .^(@ %cx scry-path)) - :_ +.u.clay-path - ?+ ext.req-line not-found:gen - [~ %html] (html-response:gen file) - [~ %js] (js-response:gen file) - [~ %css] (css-response:gen file) - [~ %png] (png-response:gen file) + :: + %glob + =/ data=(unit mime) + (~(get by glob.content.u.content) suffix.u.content) + ?~ data + [not-found:gen %.n] + :_ public.u.content + =/ mime-type=@t (rsh 3 1 (crip )) + :: Should maybe inspect to see how long cache should hold + :: + [[200 ['content-type' mime-type] max-1-da:gen ~] `q.u.data] == :: ++ lowercase @@ -162,24 +187,24 @@ char (add char ^~((sub 'a' 'A'))) :: - ++ get-clay-path + ++ get-content |= pax=path - ^- (unit [path ?]) - =/ first-try (match-clay-path pax (~(del by serving) /)) + ^- (unit [content path ?]) + =/ first-try (match-content-path pax (~(del by serving) /)) ?^ first-try first-try =/ root (~(get by serving) /) ?~ root ~ - (match-clay-path pax (~(gas by *^serving) [[/ u.root] ~])) + (match-content-path pax (~(gas by *^serving) [[/ u.root] ~])) :: - ++ match-clay-path + ++ match-content-path |= [pax=path =^serving] - ^- (unit [path ?]) + ^- (unit [content path ?]) %- ~(rep by serving) - |= [[url-base=path clay-base=path public=?] out=(unit [path ?])] + |= [[url-base=path =content public=?] out=(unit [content path ?])] ?^ out out =/ suf (get-suffix url-base pax) ?~ suf ~ - `[(weld clay-base u.suf) public] + `[content u.suf public] :: ++ get-suffix |= [a=path b=path] diff --git a/pkg/arvo/app/glob.hoon b/pkg/arvo/app/glob.hoon new file mode 100644 index 000000000..9625e4fa3 --- /dev/null +++ b/pkg/arvo/app/glob.hoon @@ -0,0 +1,163 @@ +/- glob +/+ default-agent, verb, dbug +|% +++ hash 0v4.jhi39.ns412.vhlmi.i1b1h.56kvu ++$ state-0 [%0 hash=@uv glob=(unit (each glob:glob tid=@ta))] ++$ all-states + $% state-0 + == ++$ card card:agent:gall +-- +|% +++ wait + |= [=path now=@da time=@dr] + ^- card + [%pass [%timer path] %arvo %b %wait (add now time)] +:: +++ poke-file-server + |= [our=@p =cage] + ^- card + [%pass /serving/(scot %uv hash) %agent [our %file-server] %poke cage] +:: +++ poke-spider + |= [=path our=@p =cage] + ^- card + [%pass [%running path] %agent [our %spider] %poke cage] +:: +++ watch-spider + |= [=path our=@p =sub=path] + ^- card + [%pass [%running path] %agent [our %spider] %watch sub-path] +:: +++ leave-spider + |= [=path our=@p] + ^- card + [%pass [%running path] %agent [our %spider] %leave ~] +-- +=| state=state-0 +=. hash.state hash +^- agent:gall +%+ verb | +%- agent:dbug +^- agent:gall +|_ =bowl:gall ++* this . + def ~(. (default-agent this %|) bowl) +++ on-init + ^- (quip card _this) + =/ new-tid=@ta (cat 3 'glob--' (scot %uv eny.bowl)) + =/ args [~ `new-tid %glob !>([hash.state ~])] + :_ this(glob.state `[%| new-tid]) + =/ action !>([%unserve-dir /'~glob']) + :~ (poke-file-server our.bowl %file-server-action action) + (wait /[new-tid] now.bowl ~s15) + (watch-spider /(scot %uv hash.state) our.bowl /thread-result/[new-tid]) + (poke-spider /(scot %uv hash.state) our.bowl %spider-start !>(args)) + == +:: +++ on-save !>(state) +++ on-load + |= old-state=vase + ^- (quip card _this) + ~& > %initting + =+ !<(old=all-states old-state) + ?> ?=(%0 -.old) + ?~ glob.old + on-init + ?: ?=(%& -.u.glob.old) + ?: =(hash.old hash.state) + `this(state old) + on-init + =/ cancel-cards + =/ args [tid.p.u.glob.old &] + :~ (leave-spider /(scot %uv hash.old) our.bowl) + (poke-spider /(scot %uv hash.old) our.bowl %spider-stop !>(args)) + == + =^ init-cards this on-init + [(weld cancel-cards init-cards) this] +:: +++ on-poke + |= [=mark =vase] + ^- (quip card _this) + ?: =(%kick q.vase) + (on-load !>(state(hash *@uv))) + (on-poke:def mark vase) +:: +++ on-watch on-watch:def +++ on-leave on-leave:def +++ on-peek on-peek:def +++ on-agent + |= [=wire =sign:agent:gall] + ^- (quip card _this) + ?: ?=([%serving @ ~] wire) + (on-agent:def wire sign) + ?. ?=([%running @ ~] wire) + %- (slog leaf+"glob: strange on-agent! {}" ~) + (on-agent:def wire sign) + ?- -.sign + %poke-ack + ?~ p.sign + [~ this] + %- (slog leaf+"glob: couldn't start thread; will retry" u.p.sign) + :_ this(glob.state ~) :_ ~ + (leave-spider t.wire our.bowl) + :: + %watch-ack + ?~ p.sign + [~ this] + %- (slog leaf+"glob: couldn't listen to thread; will retry" u.p.sign) + [~ this(glob.state ~)] + :: + %kick + =? glob.state ?=([~ %| *] glob.state) + ~ + `this + :: + %fact + =/ produced-hash (slav %uv i.t.wire) + ?. =(hash.state produced-hash) + [~ this] + ?+ p.cage.sign (on-agent:def wire sign) + %thread-fail + =+ !<([=term =tang] q.cage.sign) + %- (slog leaf+"glob: thread failed; will retry" leaf+ tang) + [~ this(glob.state ~)] + :: + %thread-done + =+ !<(=glob:glob q.cage.sign) + ?. =(hash.state (sham glob)) + %: mean + leaf+"glob: hash doesn't match!" + >expected=hash.state< + >got=(sham glob)< + ~ + == + :_ this(glob.state `[%& glob]) :_ ~ + %+ poke-file-server our.bowl + [%file-server-action !>([%serve-glob /'~glob' glob %|])] + == + == +:: +++ on-arvo + |= [=wire =sign-arvo] + ^- (quip card _this) + ?. ?=([%timer @ ~] wire) + %- (slog leaf+"glob: strange on-arvo wire: {}" ~) + `this + ?. ?=(%wake +<.sign-arvo) + %- (slog leaf+"glob: strange on-arvo sign: {}" ~) + `this + ?: ?=([~ %& *] glob.state) + `this + ?. ?| ?=(~ glob.state) + =(i.t.wire tid.p.u.glob.state) + == + `this + ?^ error.sign-arvo + %- (slog leaf+"glob: timer handling failed; will retry" ~) + [[(wait t.wire now.bowl ~s15)]~ this] + %- (slog leaf+"glob: timed out; retrying" ~) + (on-load !>(state(hash *@uv))) +:: +++ on-fail on-fail:def +-- diff --git a/pkg/arvo/lib/strandio.hoon b/pkg/arvo/lib/strandio.hoon index db9db877e..71299cb8b 100644 --- a/pkg/arvo/lib/strandio.hoon +++ b/pkg/arvo/lib/strandio.hoon @@ -366,14 +366,20 @@ ?> ?=(^ full-file.client-response) (pure:m q.data.u.full-file.client-response) :: -++ fetch-json +++ fetch-cord |= url=tape - =/ m (strand ,json) + =/ m (strand ,cord) ^- form:m =/ =request:http [%'GET' (crip url) ~ ~] ;< ~ bind:m (send-request request) ;< =client-response:iris bind:m take-client-response - ;< =cord bind:m (extract-body client-response) + (extract-body client-response) +:: +++ fetch-json + |= url=tape + =/ m (strand ,json) + ^- form:m + ;< =cord bind:m (fetch-cord url) =/ json=(unit json) (de-json:html cord) ?~ json (strand-fail %json-parse-error ~) diff --git a/pkg/arvo/sur/file-server.hoon b/pkg/arvo/sur/file-server.hoon index 7bd02922f..b29379c91 100644 --- a/pkg/arvo/sur/file-server.hoon +++ b/pkg/arvo/sur/file-server.hoon @@ -1,6 +1,8 @@ +/- glob |% +$ action $% [%serve-dir url-base=path clay-base=path public=?] + [%serve-glob url-base=path =glob:glob public=?] [%unserve-dir url-base=path] [%toggle-permission url-base=path] [%set-landscape-homepage-prefix prefix=(unit term)] diff --git a/pkg/arvo/sur/glob.hoon b/pkg/arvo/sur/glob.hoon new file mode 100644 index 000000000..4c2950310 --- /dev/null +++ b/pkg/arvo/sur/glob.hoon @@ -0,0 +1,3 @@ +|% ++$ glob (map path mime) +-- diff --git a/pkg/arvo/ted/glob.hoon b/pkg/arvo/ted/glob.hoon new file mode 100644 index 000000000..d50e7a114 --- /dev/null +++ b/pkg/arvo/ted/glob.hoon @@ -0,0 +1,12 @@ +/- spider, glob +/+ strandio +=, strand=strand:spider +^- thread:spider +|= arg=vase +=/ m (strand ,vase) +^- form:m +=+ !<([hash=@uv ~] arg) +=/ url "https://bootstrap.urbit.org/glob-{(scow %uv hash)}.glob" +;< =cord bind:m (fetch-cord:strandio url) +=+ ;;(=glob:glob (cue cord)) +(pure:m !>(glob))