git.iko.soy/seax
on master help sign in
sink doesn't work
commitb8fd2ca14b8043aa551351f3423da2c7d2f89982 authoriko <mail@iko.soy> dateMon 31 Jul 2023, 09:27 (3 years ago) parentc325951
10 files changed+454−74
compare with master
unifiedsplit
zod/seax/app/seax.hoon+102−49 zod/seax/lib/engines.hoon+8−4 zod/seax/lib/engines/alexandria.hoon+4−4 zod/seax/lib/engines/crowdview.hoon+4−4 zod/seax/lib/engines/wiby.hoon+4−4 zod/seax/lib/engines/yep.hoon+6−7 zod/seax/lib/noun-diff.hoon+227 zod/seax/lib/rank.hoon+71 zod/seax/lib/sink.hoon+26 zod/seax/sur/seax.hoon+2−2
zod/seax/app/seax.hoon+102−49
@@ -1,35 +1,57 @@
11/- *seax
2−/+ dbug, engines
2+/+ dbug, engines, rank, sink
33
44|%
5− +$ url @t
6− +$ word @t
7− +$ number-of-hits @u
8− +$ number-of-hits-mop ((mop number-of-hits (set url)) gth)
9− +$ rwi (map word number-of-hits-mop)
10− +$ word-index (map url (map word @u))
11− +$ search-index [rwi=rwi wi=word-index]
12− +$ state-0 ~
13− +$ poke
14− $%
15− [%search query=tape]
16− ==
17− ++ urls-per-hits
18− ((on number-of-hits (set url)) gth)
19− ++ normalize-word
20− |= word=tape
21− ^- tape
22− (cass word)
23− ++ initial-crawl-depth 2
24− ++ swap
25− |* [a=* b=*]
26− [b a]
27− ++ second
28− |* [a=* b=*]
29− b
5++$ query cord
6++$ search-state
7+$:
8+ number-of-subscriptions=@u
9+ search-results=(map term (unit (list search-result)))
10+==
11++$ state-0
12+$:
13+ search-subscriptions=(map query search-state)
14+==
15++$ poke
16+ $%
17+ [%search query=tape]
18+ ==
19+++ rank-results
20+ |= search-results=(map term (unit (list search-result)))
21+ %- rank
22+ %- cat-units
23+ %+ turn ~(tap by search-results)
24+ |= [engine=term results=(unit (list search-result))]
25+ %+ bind results
26+ |= results=(list search-result)
27+ [engine results]
28+++ cat-units
29+ |* x=(list (unit *))
30+ ?~ x ~
31+ ?~ i.x $(x t.x)
32+ [+.i.x $(x t.x)]
33+++ search-cards
34+ |= =query
35+ ^- (list card:agent:gall)
36+ %+ turn ~(tap by engines)
37+ |= [engine-name=term engine=engine *]
38+ =/ req
39+ ^- task:iris
40+ [%request [%'GET' (crip (url.engine (trip query))) ~ ~] *outbound-config:iris]
41+ [%pass ~[%search-result query engine-name] %arvo %i req]
3042--
3143
3244=/ state *state-0
45+=* search-results
46+ |= =query
47+ %- rank-results
48+ =/ search-state=search-state
49+ (~(got by search-subscriptions.state) query)
50+ search-results.search-state
51+=* sink
52+ |= =query
53+ ((sink:^sink ~[%search query]) (search-results query))
54+:: =/ sinks *(map query _(sink ''))
3355
3456^- agent:gall
3557%- agent:dbug
@@ -40,30 +62,52 @@
4062++ on-load
4163 |= =vase
4264 =/ state !<(state-0 vase)
43− `this(state state)
65+ :- ~
66+ %= this
67+ state state
68+ ==
4469++ on-poke
4570 |= =cage
4671 ^- (quip card:agent:gall _this)
47− ~& cage
48− ?+ -.cage !!
49− %noun
50− =/ poke !<(poke +.cage)
51− ?- -.poke
52− %search
53− =/ url
54− %- crip (url:engines-alexandria query.poke)
55− ~& url
56− =/ req
57− ^- task:iris
58− [%request [%'GET' url ~ ~] *outbound-config:iris]
59− :_ this
60− ~[[%pass ~[%yep] %arvo %i req]]
72+ `this
6173
62− ==
63−
74+++ on-watch
75+ |= =path
76+ ?+ path !!
77+ [%search query ~]
78+ =/ query +<.path
79+ =/ subscription=(unit search-state)
80+ (~(get by search-subscriptions.state) query)
81+ ?~ subscription
82+ =. search-subscriptions.state
83+ (~(put by search-subscriptions.state) query [1 ~])
84+ =/ cards (search-cards query)
85+ [cards this]
86+ =/ subscription=search-state +.subscription
87+ =. subscription
88+ [+(number-of-subscriptions.subscription) search-results.subscription]
89+ =. search-subscriptions.state
90+ (~(put by search-subscriptions.state) query subscription)
91+ [~[flush:(sink query)] this]
6492 ==
65−++ on-watch |=(path !!)
66−++ on-leave |=(path `..on-init)
93+
94+:: This was not tested because it requires waiting 12 hours.
95+++ on-leave
96+ |= =path
97+ ~& path
98+ ?+ path !!
99+ [%search query ~]
100+ =/ query +<.path
101+ ~& path
102+ =/ search-state (~(got by search-subscriptions.state) query)
103+ =/ number-of-subscriptions (dec number-of-subscriptions.search-state)
104+ =. search-subscriptions.state
105+ ?~ number-of-subscriptions
106+ (~(del by search-subscriptions.state) query)
107+ (~(put by search-subscriptions.state) query search-state(number-of-subscriptions number-of-subscriptions))
108+ `this
109+ ==
110+
67111++ on-peek
68112 |= path
69113 ~
@@ -72,12 +116,21 @@
72116 |= [=wire =sign-arvo]
73117 ^- (quip card:agent:gall _this)
74118 ~& wire
75− ?+ wire !!
76− [%yep ~]
119+ ?+ wire `this
120+ [%search-result query=@t engine=term ~]
77121 ?> ?=([%iris %http-response %finished *] sign-arvo)
78− =/ results
79− %- results:engines-alexandria (need full-file.client-response.sign-arvo)
122+ =/ query=@t +<.wire
123+ =/ engine-name=term +>-.wire
124+ =/ [=engine *] (~(got by engines) engine-name)
125+ =/ search-subscription (~(got by search-subscriptions.state) query)
126+ =/ results=(unit (list search-result))
127+ ;< data=mime-data:iris _biff full-file.client-response.sign-arvo
128+ (results.engine data)
80129 ~& results
130+ =. search-results.search-subscription
131+ (~(put by search-results.search-subscription) engine-name results)
132+ =. search-subscriptions.state
133+ (~(put by search-subscriptions.state) query search-subscription)
81134 `this
82135 ==
83136++ on-fail |=([term tang] `..on-init)
zod/seax/lib/engines.hoon+8−4
@@ -1,10 +1,14 @@
11/- *seax
22/+ engines-yep, engines-crowdview, engines-wiby, engines-alexandria
33
4+^- engines
5+
6+%- malt
7+%- limo
48:~
5− [engine=engines-yep weight=.1]
6− [engine=engines-crowdview weight=.2]
7− [engine=engines-wiby weight=.2]
8− [engine=engines-alexandria weight=.2]
9+ [name=%yep engine=engines-yep weight=.1]
10+ [name=%crowdview engine=engines-crowdview weight=.2]
11+ [name=%wiby engine=engines-wiby weight=.2]
12+ [name=%alexandria engine=engines-alexandria weight=.2]
913==
1014
zod/seax/lib/engines/alexandria.hoon+4−4
@@ -11,19 +11,19 @@
1111 (en-urlt:html query)
1212++ results
1313 |= response=mime-data:iris
14− ^- (list search-result)
14+ ^- (unit (list search-result))
1515 =/ body q.data.response
16− =/ jon (need (de:json:html body))
16+ ;< jon=json _biff (de:json:html body)
1717 (apex jon)
1818--
1919::
2020|%
2121++ apex
22− =, dejs:format
22+ =, dejs-soft:format
2323 %- ot
2424 ~[results+(ar parse-search-result)]
2525++ parse-search-result
26− =, dejs:format
26+ =, dejs-soft:format
2727 %- ot
2828 ~[title+so url+so]
2929--
zod/seax/lib/engines/crowdview.hoon+4−4
@@ -11,19 +11,19 @@
1111 (en-urlt:html query)
1212++ results
1313 |= response=mime-data:iris
14− ^- (list search-result)
14+ ^- (unit (list search-result))
1515 =/ body q.data.response
16− =/ jon (need (de:json:html body))
16+ ;< jon=json _biff (de:json:html body)
1717 (apex jon)
1818--
1919::
2020|%
2121++ apex
22− =, dejs:format
22+ =, dejs-soft:format
2323 %- ot
2424 ~[results+(ar parse-search-result)]
2525++ parse-search-result
26− =, dejs:format
26+ =, dejs-soft:format
2727 %- ot
2828 ~[title+so link+so]
2929--
zod/seax/lib/engines/wiby.hoon+4−4
@@ -11,18 +11,18 @@
1111 (en-urlt:html query)
1212++ results
1313 |= response=mime-data:iris
14− ^- (list search-result)
14+ ^- (unit (list search-result))
1515 =/ body q.data.response
16− =/ jon (need (de:json:html body))
16+ ;< jon=json _biff (de:json:html body)
1717 (apex jon)
1818--
1919::
2020|%
2121++ apex
22− =, dejs:format
22+ =, dejs-soft:format
2323 (ar parse-search-result)
2424++ parse-search-result
25− =, dejs:format
25+ =, dejs-soft:format
2626 %- ot
2727 ~[[%'Title' so] [%'URL' so]]
2828--
zod/seax/lib/engines/yep.hoon+6−7
@@ -11,21 +11,20 @@
1111 (en-urlt:html query)
1212++ results
1313 |= response=mime-data:iris
14− ^- (list search-result)
1514 =/ body q.data.response
16− =/ jon (need (de:json:html body))
15+ ;< jon=json _biff (de:json:html body)
1716 (apex jon)
1817--
1918::
2019|%
2120++ apex
2221 |= jon=json
23− =, dejs:format
24− =/ [* results=(list search-result)]
25− %- (at ~[so (ot ~[results+(ar parse-search-result)])]) jon
26− results
22+ ^- (unit (list search-result))
23+ =, dejs-soft:format
24+ ;< [* results=(list search-result)] _biff %- (at ~[so (ot ~[results+(ar parse-search-result)])]) jon
25+ `results
2726++ parse-search-result
28− =, dejs:format
27+ =, dejs-soft:format
2928 %- ot
3029 ~[title+so url+so]
3130--
zod/seax/lib/noun-diff.hoonnew file+227
@@ -0,0 +1,227 @@
1+=<
2+|%
3+ ++ diff
4+ |= [old=* new=*]
5+ ^- patch
6+ =/ del (extract-del (oracle old new) old)
7+ =/ ins (extract-ins (oracle old new) new)
8+ =/ allowed-holes (~(int in (find-del-holes del)) (find-ins-holes ins))
9+ =. del (filter-del-holes allowed-holes del)
10+ =/ ins (filter-ins-holes allowed-holes ins)
11+ =/ closed-patch (closure (gcp [del ins]))
12+ ?> =(& +.closed-patch)
13+ -.closed-patch
14+ ++ apply
15+ |= [patch=_id noun=*]
16+ ?- -.patch
17+ %diff
18+ =/ var-map (del del.patch noun)
19+ (ins ins.patch var-map)
20+ %cell
21+ ?> ?=(^ noun)
22+ [$(patch lhs.patch, noun -.noun) $(patch rhs.patch, noun +.noun)]
23+ ==
24+ ++ id
25+ ^- patch
26+ [%diff [%hole ~] [%hole ~]]
27+ +$ patch
28+ $%
29+ [%cell lhs=patch rhs=patch]
30+ [%diff ^diff]
31+ ==
32+--
33+::
34+|%
35+ +$ del-diff
36+ $%
37+ [%hole @]
38+ [%cell lhs=del-diff rhs=del-diff]
39+ [%ignore ~]
40+ ==
41+ +$ ins-diff
42+ $%
43+ [%hole @ original=*]
44+ [%cell lhs=ins-diff rhs=ins-diff]
45+ [%atom @]
46+ ==
47+ +$ final-ins-diff
48+ $%
49+ [%hole @]
50+ [%cell lhs=final-ins-diff rhs=final-ins-diff]
51+ [%atom @]
52+ ==
53+ +$ diff [del=del-diff ins=final-ins-diff]
54+ +$ patch
55+ $%
56+ [%cell lhs=patch rhs=patch]
57+ [%diff diff]
58+ ==
59+ ++ insify-noun
60+ |= noun=*
61+ ^- final-ins-diff
62+ ?- noun
63+ ^ [%cell $(noun -.noun) $(noun +.noun)]
64+ @ [%atom noun]
65+ ==
66+ ++ empty-set (silt `(list @)`~)
67+ ++ find-del-holes
68+ |= diff=del-diff
69+ ~+
70+ ^- (set @)
71+ ?- -.diff
72+ %hole (silt ~[+.diff])
73+ %cell
74+ (~(uni in $(diff lhs.diff)) $(diff rhs.diff))
75+ %ignore empty-set
76+ ==
77+ ++ find-final-ins-holes
78+ |= diff=final-ins-diff
79+ ~+
80+ ^- (set @)
81+ ?- -.diff
82+ %hole (silt ~[+.diff])
83+ %cell
84+ (~(uni in $(diff lhs.diff)) $(diff rhs.diff))
85+ %atom empty-set
86+ ==
87+ ++ find-ins-holes
88+ |= diff=ins-diff
89+ ~+
90+ ^- (set @)
91+ ?- -.diff
92+ %hole (silt ~[+<.diff])
93+ %cell
94+ (~(uni in $(diff lhs.diff)) $(diff rhs.diff))
95+ %atom empty-set
96+ ==
97+ ++ filter-del-holes
98+ |= [allowed-holes=(set @) diff=del-diff]
99+ ^- del-diff
100+ ?: ?=(%ignore -.diff) diff
101+ ?- -.diff
102+ %hole
103+ ?: (~(has in allowed-holes) +.diff) diff
104+ [%ignore ~]
105+ %cell [%cell $(diff +<.diff) $(diff +>.diff)]
106+ ==
107+ ++ filter-ins-holes
108+ |= [allowed-holes=(set @) diff=ins-diff]
109+ ^- final-ins-diff
110+ ?- -.diff
111+ %hole
112+ ?: (~(has in allowed-holes) +<.diff) [%hole +<.diff]
113+ (insify-noun original:diff)
114+ %cell [%cell $(diff +<.diff) $(diff +>.diff)]
115+ %atom diff
116+ ==
117+ ++ gcp
118+ |= diff=diff
119+ ^- patch
120+ ?- -.ins.diff
121+ %atom
122+ [%diff diff]
123+ %cell
124+ ?: ?=(%cell -.del.diff)
125+ [%cell $(diff [+<.del.diff +<.ins.diff]) $(diff [+>.del.diff +>.ins.diff])]
126+ [%diff diff]
127+ %hole
128+ [%diff diff]
129+ ==
130+ ++ closure
131+ |= =patch
132+ ^- [^patch ?]
133+ ?- -.patch
134+ %diff
135+ =/ del-holes (find-del-holes del:patch)
136+ =/ ins-holes (find-final-ins-holes ins:patch)
137+ =/ difference (~(dif in ins-holes) del-holes)
138+ [patch =(difference empty-set)]
139+ %cell
140+ =/ lhs $(patch lhs:patch)
141+ =/ rhs $(patch rhs:patch)
142+ ?: ?&(+.lhs +.rhs) [[%cell -.lhs -.rhs] &]
143+ $(patch (pull-diff [%cell -.lhs -.rhs]))
144+ ==
145+ ++ pull-diff
146+ |= =patch
147+ ^- [%diff del=del-diff ins=final-ins-diff]
148+ ?- -.patch
149+ %diff patch
150+ %cell
151+ =/ pulled-lhs $(patch lhs:patch)
152+ =/ pulled-rhs $(patch rhs:patch)
153+ :+
154+ %diff
155+ [%cell del:pulled-lhs del:pulled-rhs]
156+ [%cell ins:pulled-lhs ins:pulled-rhs]
157+ ==
158+ ++ is-subtree
159+ |= [tree=* subtree=*]
160+ ~+
161+ ^- ?
162+ ?: =(tree subtree) &
163+ ?@ tree |
164+ ?|
165+ (is-subtree -.tree subtree)
166+ (is-subtree +.tree subtree)
167+ ==
168+ ++ oracle
169+ |= [a=* b=*]
170+ |= subtree=*
171+ ^- (unit @)
172+ ?:
173+ ?&
174+ (is-subtree a subtree)
175+ (is-subtree b subtree)
176+ ==
177+ `(mug subtree)
178+ ~
179+ ++ extract-del
180+ |= [oracle=$-(* (unit @)) subtree=*]
181+ ~+
182+ ^- del-diff
183+ =/ hash (oracle subtree)
184+ ?^ hash [%hole +.hash]
185+ ?@ subtree [%ignore ~]
186+ [%cell (extract-del oracle -.subtree) (extract-del oracle +.subtree)]
187+ ++ extract-ins
188+ |= [oracle=$-(* (unit @)) subtree=*]
189+ ~+
190+ ^- ins-diff
191+ =/ hash (oracle subtree)
192+ ?^ hash [%hole +.hash subtree]
193+ ?@ subtree [%atom subtree]
194+ [%cell (extract-ins oracle -.subtree) (extract-ins oracle +.subtree)]
195+ ++ ins
196+ |= [diff=final-ins-diff var-map=(map @ *)]
197+ ^- *
198+ ?- -.diff
199+ %atom
200+ +.diff
201+ %cell
202+ [$(diff +<.diff) $(diff +>.diff)]
203+ %hole
204+ (~(got by var-map) +.diff)
205+ ==
206+ ++ del
207+ |= [diff=del-diff noun=*]
208+ ^- (map @ *)
209+ |^ (go diff noun ((map @ *) ~))
210+ ++ go
211+ |= [diff=del-diff noun=* var-map=(map @ *)]
212+ ^- (map @ *)
213+ ?- -.diff
214+ %ignore var-map
215+ %hole
216+ =/ subtree (~(get by var-map) +.diff)
217+ ?~ subtree (~(put by var-map) +.diff noun)
218+ ?> =(+.subtree noun)
219+ var-map
220+ %cell
221+ ?> ?=(^ noun)
222+ =/ lhs-var-map $(diff +<.diff, noun -.noun)
223+ =/ rhs-var-map $(diff +>.diff, noun +.noun, var-map lhs-var-map)
224+ rhs-var-map
225+ ==
226+ --
227+--
zod/seax/lib/rank.hoonnew file+71
@@ -0,0 +1,71 @@
1+/- *seax
2+/+ engines
3+
4+=<
5+|= results=(list [engine=term results=(list search-result)])
6+^- (list search-result)
7+=/ url-to-engine-map
8+ ^- (list (map @t [engines=(list term) ords=(list @u) result=search-result]))
9+ %- zing
10+ %+ turn results
11+ |= [engine=term results=(list search-result)]
12+ %+ turn (zip (gulf 1 (lent results)) results)
13+ |= [ord=@u result=search-result]
14+ %- malt ~[[url.result [~[engine] ~[ord] result]]]
15+=/ url-to-engines-map
16+ ^- (map @t [engines=(list term) ords=(list @u) result=search-result])
17+ %+ roll url-to-engine-map
18+ |=
19+ $:
20+ a=(map @t [engines=(list term) ords=(list @u) result=search-result])
21+ b=(map @t [engines=(list term) ords=(list @u) result=search-result])
22+ ==
23+ %- (~(uno by a) b)
24+ |=
25+ $:
26+ *
27+ [engines-lhs=(list term) ords-lhs=(list @u) result=search-result]
28+ [engines-rhs=(list term) ords-rhs=(list @u) *]
29+ ==
30+ [(weld engines-lhs engines-rhs) (weld ords-lhs ords-rhs) result]
31+=/ ranking
32+ ^- (list [ord=@rs result=search-result])
33+ %+ turn ~(val by url-to-engines-map)
34+ |= [engines=(list term) ords=(list @u) result=search-result]
35+ =/ engine-weights
36+ ^- (list @rs)
37+ %+ turn engines
38+ |= engine=term
39+ =/ engine (~(got by ^engines) engine)
40+ weight.engine
41+ =/ engine-weights
42+ ^- @rs
43+ %+ roll engine-weights
44+ |= [a=_.1 b=_.1]
45+ (mul.rs a b)
46+ =/ position-weights
47+ ^- @rs
48+ %+ mul.rs (lent ords)
49+ %+ roll
50+ (turn ords |=(ord=@u (div.rs 1 (sun:rs ord))))
51+ add
52+ =/ rank
53+ ^- @rs
54+ (mul.rs engine-weights position-weights)
55+ [ord=rank result=result]
56+=/ sorted-results
57+ %+ sort ranking
58+ |= [[lhs=@rs *] [rhs=@rs *]]
59+ %+ gth lhs rhs
60+~& sorted-results
61+%+ turn sorted-results
62+|= [* result=search-result]
63+result
64+
65+|%
66+++ zip
67+ |* [a=(list) b=(list)]
68+ ?~ a ~
69+ ?~ b ~
70+ [[i.a i.b] $(a t.a, b t.b)]
71+--
zod/seax/lib/sink.hoonnew file+26
@@ -0,0 +1,26 @@
1+/+ noun-diff
2+|%
3+++ clog
4+ |$ [stut]
5+ $%
6+ [%flush stut]
7+ [%drain patch:noun-diff]
8+ ==
9+++ sink
10+ |* pats=(list path)
11+ |* stat=*
12+ |@
13+ ++ sync
14+ |= [stat=_stat]
15+ ^- [card:agent:gall _..sync]
16+ =/ dif
17+ %+ diff:noun-diff ^stat stat
18+ :-
19+ [%give %fact pats %noun !>(^-((clog) [%drain dif]))]
20+ ..sync(stat stat)
21+ ++ paths pats
22+ ++ flush
23+ ^- card:agent:gall
24+ [%give %fact pats %noun !>(^-((clog) [%flush stat]))]
25+ --
26+--
zod/seax/sur/seax.hoon+2−2
@@ -8,7 +8,7 @@
88 $_ ^|
99 |%
1010 ++ url |~ tape *tape
11− ++ results |~ mime-data:iris *(list search-result)
11+ ++ results |~ mime-data:iris *(unit (list search-result))
1212 --
13−+$ engines (list [=engine weight=@rs])
13++$ engines (map name=term [=engine weight=@rs])
1414--