@@ -1,4 +1,5 @@
1−/+ extract-text, extract-links, words, dbug
1+/- *seax
2+/+ dbug, engines
23
34|%
45 +$ url @t
@@ -8,16 +9,10 @@
89 +$ rwi (map word number-of-hits-mop)
910 +$ word-index (map url (map word @u))
1011 +$ search-index [rwi=rwi wi=word-index]
11− +$ state-0
12− $:
13− rwi=search-index
14− to-crawl=(list [url=url depth=@u])
15− ==
12+ +$ state-0 ~
1613 +$ poke
1714 $%
18− [%start-crawl url=url]
19− [%crawl-next ~]
20− [%search @t]
15+ [%search query=tape]
2116 ==
2217 ++ urls-per-hits
2318 ((on number-of-hits (set url)) gth)
@@ -32,34 +27,6 @@
3227 ++ second
3328 |* [a=* b=*]
3429 b
35− ++ add-to-index
36− |= [index=search-index new-words=(map word number-of-hits) =url]
37− ^- search-index
38− :- (add-to-rwi rwi.index new-words url)
39− %+ ~(put by wi.index) url new-words
40− ++ add-to-rwi
41− |= [index=rwi new-words=(map word number-of-hits) =url]
42− %- second
43− ^- [(list ~) rwi]
44− %^ spin ~(tap by new-words) index
45− |= [[=word =number-of-hits] index=rwi]
46− :- ~
47− ^- rwi
48− =/ existing-number-of-hits (~(get by index) word)
49− ?~ existing-number-of-hits
50− %+ ~(put by index) word
51− (gas:urls-per-hits *number-of-hits-mop ~[[number-of-hits (silt ~[url])]])
52− =/ existing-number-of-hits
53− ^- number-of-hits-mop
54− +.existing-number-of-hits
55− =/ urls (get:urls-per-hits existing-number-of-hits number-of-hits)
56− ?~ urls
57− %+ ~(put by index) word
58− (put:urls-per-hits existing-number-of-hits number-of-hits (silt ~[url]))
59− =/ urls ^- (set ^url) +.urls
60− %+ ~(put by index) word
61− %^ put:urls-per-hits existing-number-of-hits
62− number-of-hits (~(put in urls) url)
6330--
6431
6532=/ state *state-0
@@ -68,8 +35,6 @@
6835%- agent:dbug
6936|_ =bowl:gall
7037+* this .
71− next-crawl-card
72− [%pass / %agent [our.bowl %seax] %poke [%noun !>((poke [%crawl-next ~]))]]
7338++ on-init `..on-init
7439++ on-save !>(state)
7540++ on-load
@@ -79,48 +44,21 @@
7944++ on-poke
8045 |= =cage
8146 ^- (quip card:agent:gall _this)
47+ ~& cage
8248 ?+ -.cage !!
8349 %noun
8450 =/ poke !<(poke +.cage)
8551 ?- -.poke
86− %start-crawl
87− =/ url url.poke
52+ %search
53+ =/ url
54+ %- crip (url:engines-alexandria query.poke)
55+ ~& url
8856 =/ req
8957 ^- task:iris
9058 [%request [%'GET' url ~ ~] *outbound-config:iris]
9159 :_ this
92− ~[[%pass ~[%crawl-page (scot %ud initial-crawl-depth) url] %arvo %i req]]
93− ::
94− %crawl-next
95− ?~ to-crawl.state `this
96− =/ url url.i.to-crawl.state
97− ?: (~(has by wi.rwi.state) url) [~[next-crawl-card] this(to-crawl.state t.to-crawl.state)]
98− =/ req
99− ^- task:iris
100− [%request [%'GET' url ~ ~] *outbound-config:iris]
101− :_ this(to-crawl.state t.to-crawl.state)
102− ~[[%pass ~[%crawl-page (scot %ud depth.i.to-crawl.state) url.i.to-crawl.state] %arvo %i req]]
103− ::
104− %search
105− =/ query
106− ^- (list tape)
107− (turn (words (trip +.poke)) normalize-word)
108− =/ top-results
109− ^- (list cord)
110− %- zing
111− ^- (list (list cord))
112− %- zing
113− ^- (list (list (list cord)))
114− %+ turn query
115− |= word=tape
116− =/ results (~(get by rwi.rwi.state) (crip word))
117− ?~ results ~
118− %+ turn (tap:urls-per-hits +.results)
119− |= [* set=(set cord)]
120− ~(tap in set)
60+ ~[[%pass ~[%yep] %arvo %i req]]
12161
122− ~& top-results
123− `this
12462 ==
12563
12664 ==
@@ -133,46 +71,14 @@
13371++ on-arvo
13472 |= [=wire =sign-arvo]
13573 ^- (quip card:agent:gall _this)
74+ ~& wire
13675 ?+ wire !!
137− [%crawl-page @t @t ~]
138− ~& "crawling {<+>-.wire>}"
76+ [%yep ~]
13977 ?> ?=([%iris %http-response %finished *] sign-arvo)
140− =/ url=@t +>-.wire
141− =/ depth=@u (slav %ud +<.wire)
142− =/ purl (need (de-purl:html url))
143− =/ response-body
144− %- trip q.data:(need full-file:client-response.sign-arvo)
145− =/ words
146− ^- (unit (list tape))
147− %+ bind
148− (extract-text response-body)
149− |= text=tape
150− (turn (words text) normalize-word)
151− =/ word-index-map
152− ^- (unit (list (map cord @u)))
153− %+ bind words
154− |= words=(list tape)
155− %+ turn words
156− |= word=tape
157− %- malt ~[[(crip word) 1]]
158− =/ word-index
159− ^- (map word number-of-hits)
160− ?~ word-index-map *(map word number-of-hits)
161− %+ roll `(list (map cord @u))`+.word-index-map
162− |= [lhs=(map cord @u) rhs=(map cord @u)]
163− %- %- ~(uno by lhs) rhs
164− |= [* a=@u b=@u] (add a b)
165− =/ index (add-to-index rwi.state word-index url)
166− =/ links=(list tape) (extract-links p.purl response-body)
167− =/ next-depth (dec depth)
168− =/ to-crawl
169− ?~ next-depth to-crawl.state
170− %+ weld
171− to-crawl.state
172− %+ turn links
173− |= link=tape [(crip link) next-depth]
174− :_ this(to-crawl.state to-crawl, rwi.state index)
175− ~[next-crawl-card]
78+ =/ results
79+ %- results:engines-alexandria (need full-file.client-response.sign-arvo)
80+ ~& results
81+ `this
17682 ==
17783++ on-fail |=([term tang] `..on-init)
17884--