glob: added app to distribute js outside clay

pkg/arvo/app/eth-watcher.hoon

@@ -318,7 +318,11 @@
       %watch-ack
     ?~  p.sign
       [~ this]
-    %-  (slog leaf+"eth-watcher couldn't start listen to thread" u.p.sign)
+    %-  (slog leaf+"eth-watcher couldn't start listening to thread" u.p.sign)
+    ::  TODO: kill thread that may have started, although it may not
+    ::  have started yet since we get this response before the
+    ::  %start-spider poke is processed
+    ::
     [~ (clear-running t.wire)]
   ::
       %kick  [~ (clear-running t.wire)]
@@ -413,7 +417,7 @@
 ++  on-arvo
   |=  [=wire =sign-arvo]
   ^-  (quip card agent:gall)
-  ?+  +<.sign-arvo  ~|([%strange-sign-arvo -.sign-arvo] !!)
+  ?+    +<.sign-arvo  ~|([%strange-sign-arvo -.sign-arvo] !!)
       %wake
     ?.  ?=([%timer *] wire)  ~&  weird-wire=wire  [~ this]
     =*  path  t.wire

pkg/arvo/app/file-server.hoon

@@ -1,4 +1,4 @@
-/-  srv=file-server
+/-  srv=file-server, glob
 /+  *server, default-agent, verb, dbug
 |%
 +$  card  card:agent:gall
@@ -6,7 +6,11 @@
   $%  state-zero
   ==
 ::
-+$  serving  (map url-base=path [clay-base=path public=?])
++$  serving  (map url-base=path [=content public=?])
++$  content
+  $%  [%clay =path]
+      [%glob =glob:glob]
+  ==
 +$  state-zero
   $:  %0
       =configuration:srv
@@ -33,7 +37,7 @@
         %+  turn
           ^-  (list path)
           [/ /'~landscape' ~]
-        |=(pax=path [pax [/app/landscape %.n]])
+        |=(pax=path [pax [clay+/app/landscape %.n]])
       ==
   :~  (connect /)
       (connect /'~landscape')
@@ -75,7 +79,14 @@
       ?:  (~(has by serving) url-base)
         ~|("url already bound to {<(~(got by serving) url-base.act)>}" !!)
       :-  [%pass url-base %arvo %e %connect [~ url-base] %file-server]~
-      this(serving (~(put by serving) url-base [clay-base.act public.act]))
+      this(serving (~(put by serving) url-base clay+clay-base.act public.act))
+    ::
+        %serve-glob
+      =*  url-base  url-base.act
+      ?:  (~(has by serving) url-base)
+        ~|("url already bound to {<(~(got by serving) url-base.act)>}" !!)
+      :-  [%pass url-base %arvo %e %connect [~ url-base] %file-server]~
+      this(serving (~(put by serving) url-base glob+glob.act public.act))
     ::
         %unserve-dir
       :-  [%pass url-base.act %arvo %e %disconnect [~ url-base.act]]~
@@ -84,9 +95,9 @@
         %toggle-permission
       ?.  (~(has by serving) url-base.act)
         ~|("url is not bound" !!)
-      =/  [clay-base=path public=?]  (~(got by serving) url-base.act)
+      =/  [=content public=?]  (~(got by serving) url-base.act)
       :-  ~
-      this(serving (~(put by serving) url-base.act [clay-base !public]))
+      this(serving (~(put by serving) url-base.act [content !public]))
     ::
         %set-landscape-homepage-prefix
       =.  landscape-homepage-prefix.configuration  prefix.act
@@ -133,22 +144,36 @@
       |=  req-line=request-line
       ^-  [simple-payload:http ?]
       =/  pax=path  (snoc site.req-line (need ext.req-line))
-      =/  clay-path=(unit [path ?])  (get-clay-path pax)
-      ?~  clay-path  [not-found:gen %.n]
-      =/  scry-path
-        :*  (scot %p our.bowl)
-            q.byk.bowl
-            (scot %da now.bowl)
-            (lowercase -.u.clay-path)
+      =/  content=(unit [=content suffix=path public=?])  (get-content pax)
+      ?~  content  [not-found:gen %.n]
+      ?-  -.content.u.content
+          %clay
+        =/  scry-path
+          :*  (scot %p our.bowl)
+              q.byk.bowl
+              (scot %da now.bowl)
+              (lowercase (weld path.content.u.content suffix.u.content))
+          ==
+        ?.  .^(? %cu scry-path)  [not-found:gen %.n]
+        =/  file  (as-octs:mimes:html .^(@ %cx scry-path))
+        :_  public.u.content
+        ?+  ext.req-line  not-found:gen
+            [~ %html]  (html-response:gen file)
+            [~ %js]    (js-response:gen file)
+            [~ %css]   (css-response:gen file)
+            [~ %png]   (png-response:gen file)
         ==
-      ?.  .^(? %cu scry-path)  [not-found:gen %.n]
-      =/  file  (as-octs:mimes:html .^(@ %cx scry-path))
-      :_  +.u.clay-path
-      ?+  ext.req-line  not-found:gen
-          [~ %html]  (html-response:gen file)
-          [~ %js]    (js-response:gen file)
-          [~ %css]   (css-response:gen file)
-          [~ %png]   (png-response:gen file)
+      ::
+          %glob
+        =/  data=(unit mime)
+          (~(get by glob.content.u.content) suffix.u.content)
+        ?~  data
+          [not-found:gen %.n]
+        :_  public.u.content
+        =/  mime-type=@t  (rsh 3 1 (crip <p.u.data>))
+        ::  Should maybe inspect to see how long cache should hold
+        ::
+        [[200 ['content-type' mime-type] max-1-da:gen ~] `q.u.data]
       ==
     ::
     ++  lowercase
@@ -162,24 +187,24 @@
         char
       (add char ^~((sub 'a' 'A')))
     ::
-    ++  get-clay-path
+    ++  get-content
       |=  pax=path
-      ^-  (unit [path ?])
-      =/  first-try  (match-clay-path pax (~(del by serving) /))
+      ^-  (unit [content path ?])
+      =/  first-try  (match-content-path pax (~(del by serving) /))
       ?^  first-try  first-try
       =/  root  (~(get by serving) /)
       ?~  root  ~
-      (match-clay-path pax (~(gas by *^serving) [[/ u.root] ~]))
+      (match-content-path pax (~(gas by *^serving) [[/ u.root] ~]))
     ::
-    ++  match-clay-path
+    ++  match-content-path
       |=  [pax=path =^serving]
-      ^-  (unit [path ?])
+      ^-  (unit [content path ?])
       %-  ~(rep by serving)
-      |=  [[url-base=path clay-base=path public=?] out=(unit [path ?])]
+      |=  [[url-base=path =content public=?] out=(unit [content path ?])]
       ?^  out  out
       =/  suf  (get-suffix url-base pax)
       ?~  suf  ~
-      `[(weld clay-base u.suf) public]
+      `[content u.suf public]
     ::
     ++  get-suffix
       |=  [a=path b=path]

pkg/arvo/app/glob.hoon

@@ -0,0 +1,163 @@
+/-  glob
+/+  default-agent, verb, dbug
+|%
+++  hash   0v4.jhi39.ns412.vhlmi.i1b1h.56kvu
++$  state-0  [%0 hash=@uv glob=(unit (each glob:glob tid=@ta))]
++$  all-states
+  $%  state-0
+  ==
++$  card  card:agent:gall
+--
+|%
+++  wait
+  |=  [=path now=@da time=@dr]
+  ^-  card
+  [%pass [%timer path] %arvo %b %wait (add now time)]
+::
+++  poke-file-server
+  |=  [our=@p =cage]
+  ^-  card
+  [%pass /serving/(scot %uv hash) %agent [our %file-server] %poke cage]
+::
+++  poke-spider
+  |=  [=path our=@p =cage]
+  ^-  card
+  [%pass [%running path] %agent [our %spider] %poke cage]
+::
+++  watch-spider
+  |=  [=path our=@p =sub=path]
+  ^-  card
+  [%pass [%running path] %agent [our %spider] %watch sub-path]
+::
+++  leave-spider
+  |=  [=path our=@p]
+  ^-  card
+  [%pass [%running path] %agent [our %spider] %leave ~]
+--
+=|  state=state-0
+=.  hash.state  hash
+^-  agent:gall
+%+  verb  |
+%-  agent:dbug
+^-  agent:gall
+|_  =bowl:gall
++*  this  .
+    def  ~(. (default-agent this %|) bowl)
+++  on-init
+  ^-  (quip card _this)
+  =/  new-tid=@ta  (cat 3 'glob--' (scot %uv eny.bowl))
+  =/  args  [~ `new-tid %glob !>([hash.state ~])]
+  :_  this(glob.state `[%| new-tid])
+  =/  action  !>([%unserve-dir /'~glob'])
+  :~  (poke-file-server our.bowl %file-server-action action)
+      (wait /[new-tid] now.bowl ~s15)
+      (watch-spider /(scot %uv hash.state) our.bowl /thread-result/[new-tid])
+      (poke-spider /(scot %uv hash.state) our.bowl %spider-start !>(args))
+  ==
+::
+++  on-save   !>(state)
+++  on-load
+  |=  old-state=vase
+  ^-  (quip card _this)
+  ~&  >  %initting
+  =+  !<(old=all-states old-state)
+  ?>  ?=(%0 -.old)
+  ?~  glob.old
+    on-init
+  ?:  ?=(%& -.u.glob.old)
+    ?:  =(hash.old hash.state)
+      `this(state old)
+    on-init
+  =/  cancel-cards
+    =/  args  [tid.p.u.glob.old &]
+    :~  (leave-spider /(scot %uv hash.old) our.bowl)
+        (poke-spider /(scot %uv hash.old) our.bowl %spider-stop !>(args))
+    ==
+  =^  init-cards  this  on-init
+  [(weld cancel-cards init-cards) this]
+::
+++  on-poke
+  |=  [=mark =vase]
+  ^-  (quip card _this)
+  ?:  =(%kick q.vase)
+    (on-load !>(state(hash *@uv)))
+  (on-poke:def mark vase)
+::
+++  on-watch  on-watch:def
+++  on-leave  on-leave:def
+++  on-peek   on-peek:def
+++  on-agent
+  |=  [=wire =sign:agent:gall]
+  ^-  (quip card _this)
+  ?:  ?=([%serving @ ~] wire)
+    (on-agent:def wire sign)
+  ?.  ?=([%running @ ~] wire)
+    %-  (slog leaf+"glob: strange on-agent! {<wire -.sign>}" ~)
+    (on-agent:def wire sign)
+  ?-    -.sign
+      %poke-ack
+    ?~  p.sign
+      [~ this]
+    %-  (slog leaf+"glob: couldn't start thread; will retry" u.p.sign)
+    :_  this(glob.state ~)  :_  ~
+    (leave-spider t.wire our.bowl)
+  ::
+      %watch-ack
+    ?~  p.sign
+      [~ this]
+    %-  (slog leaf+"glob: couldn't listen to thread; will retry" u.p.sign)
+    [~ this(glob.state ~)]
+  ::
+      %kick
+    =?  glob.state  ?=([~ %| *] glob.state)
+      ~
+    `this
+  ::
+      %fact
+    =/  produced-hash  (slav %uv i.t.wire)
+    ?.  =(hash.state produced-hash)
+      [~ this]
+    ?+    p.cage.sign  (on-agent:def wire sign)
+        %thread-fail
+      =+  !<([=term =tang] q.cage.sign)
+      %-  (slog leaf+"glob: thread failed; will retry" leaf+<term> tang)
+      [~ this(glob.state ~)]
+    ::
+        %thread-done
+      =+  !<(=glob:glob q.cage.sign)
+      ?.  =(hash.state (sham glob))
+        %:  mean
+          leaf+"glob: hash doesn't match!"
+          >expected=hash.state<
+          >got=(sham glob)<
+          ~
+        ==
+      :_  this(glob.state `[%& glob])  :_  ~
+      %+  poke-file-server  our.bowl
+      [%file-server-action !>([%serve-glob /'~glob' glob %|])]
+    ==
+  ==
+::
+++  on-arvo
+  |=  [=wire =sign-arvo]
+  ^-  (quip card _this)
+  ?.  ?=([%timer @ ~] wire)
+    %-  (slog leaf+"glob: strange on-arvo wire: {<wire [- +<]:sign-arvo>}" ~)
+    `this
+  ?.  ?=(%wake +<.sign-arvo)
+    %-  (slog leaf+"glob: strange on-arvo sign: {<wire [- +<]:sign-arvo>}" ~)
+    `this
+  ?:  ?=([~ %& *] glob.state)
+    `this
+  ?.  ?|  ?=(~ glob.state)
+          =(i.t.wire tid.p.u.glob.state)
+      ==
+    `this
+  ?^  error.sign-arvo
+    %-  (slog leaf+"glob: timer handling failed; will retry" ~)
+    [[(wait t.wire now.bowl ~s15)]~ this]
+  %-  (slog leaf+"glob: timed out; retrying" ~)
+  (on-load !>(state(hash *@uv)))
+::
+++  on-fail   on-fail:def
+--

pkg/arvo/lib/strandio.hoon

@@ -366,14 +366,20 @@
   ?>  ?=(^ full-file.client-response)
   (pure:m q.data.u.full-file.client-response)
 ::
-++  fetch-json
+++  fetch-cord
   |=  url=tape
-  =/  m  (strand ,json)
+  =/  m  (strand ,cord)
   ^-  form:m
   =/  =request:http  [%'GET' (crip url) ~ ~]
   ;<  ~                      bind:m  (send-request request)
   ;<  =client-response:iris  bind:m  take-client-response
-  ;<  =cord                  bind:m  (extract-body client-response)
+  (extract-body client-response)
+::
+++  fetch-json
+  |=  url=tape
+  =/  m  (strand ,json)
+  ^-  form:m
+  ;<  =cord  bind:m  (fetch-cord url)
   =/  json=(unit json)  (de-json:html cord)
   ?~  json
     (strand-fail %json-parse-error ~)

pkg/arvo/sur/file-server.hoon

@@ -1,6 +1,8 @@
+/-  glob
 |%
 +$  action
   $%  [%serve-dir url-base=path clay-base=path public=?]
+      [%serve-glob url-base=path =glob:glob public=?]
       [%unserve-dir url-base=path]
       [%toggle-permission url-base=path]
       [%set-landscape-homepage-prefix prefix=(unit term)]

pkg/arvo/sur/glob.hoon

@@ -0,0 +1,3 @@
+|%
++$  glob  (map path mime)
+--

pkg/arvo/ted/glob.hoon

@@ -0,0 +1,12 @@
+/-  spider, glob
+/+  strandio
+=,  strand=strand:spider
+^-  thread:spider
+|=  arg=vase
+=/  m  (strand ,vase)
+^-  form:m
+=+  !<([hash=@uv ~] arg)
+=/  url  "https://bootstrap.urbit.org/glob-{(scow %uv hash)}.glob"
+;<  =cord  bind:m  (fetch-cord:strandio url)
+=+  ;;(=glob:glob (cue cord))
+(pure:m !>(glob))