git.iko.soy/sphinx
on master help sign in
improving ranking
54fe925Hunter Miller
desk/app/sphinx.hoon+33−102
compare with master
unifiedsplit
@@ -1,7 +1,6 @@
1/- s=sphinx
1/- s=sphinx
2/+ gossip, default-agent, verb, dbug
2/+ gossip, default-agent, verb, dbug
3/+ *sphinx
3/+ *sphinx
4−/+ *validate-url
5/+ sift
4/+ sift
6/+ m=metaphone
5/+ m=metaphone
7/$ grab-listing %noun %directory-listing
6/$ grab-listing %noun %directory-listing
@@ -12,29 +11,12 @@
12 |%
11 |%
13 +$ card card:agent:gall
12 +$ card card:agent:gall
14 +$ edition
13 +$ edition
15− $% state-0
14+ $% state-0:s
16− state-1
17− ==
18− +$ state-1
19− $: %1
20− =lookup:s
21− =trigrams:s
22− =phonetics:s
23− =trail:s
24− =directory:s
25− published=directory:s
26− ==
27− +$ state-0
15+ state-1:s
28− $: %0
16+ state:s
29− =lookup:s
30− =trigrams:s
31− =phonetics:s
32− =trail:s
33− =directory:directory-state-zero:s
34− published=directory:directory-state-zero:s
35 ==
17 ==
36 --
18 --
37−=| state-1
19+=| state:s
38=* state -
20=* state -
39=<
21=<
40 %+ verb &
22 %+ verb &
@@ -62,13 +44,9 @@
62 =/ old !<(edition vase)
44 =/ old !<(edition vase)
63 |^
45 |^
64 ?- -.old
46 ?- -.old
65− %1 `this(state old(published (purge published.old)))
47+ %2 `this(state (get-state-2 directory.old))
66− %0
48+ %1 `this(state (get-state-2 directory.old))
67− %_ $
68− -.old %1
69− published.old (purge (directory-0-to-1 published.old))
70− directory.old (directory-0-to-1 directory.old)
49+ %0 `this(state (get-state-2 (directory-0-to-1 directory.old)))
71− ==
72 ==
50 ==
73 ++ directory-0-to-1
51 ++ directory-0-to-1
74 |= =directory:directory-state-zero:s
52 |= =directory:directory-state-zero:s
@@ -79,6 +57,19 @@
79 =/ post post.listing(+63 [+63.post.listing ''])
57 =/ post post.listing(+63 [+63.post.listing ''])
80 listing(+2 post)
58 listing(+2 post)
81 ::
59 ::
60+ ++ get-state-2
61+ |= =directory:s
62+ :* %2
63+ (reindex directory)
64+ directory
65+ (purge directory)
66+ ==
67+ ++ reindex
68+ |= =directory:s
69+ %+ roll
70+ ~(tap by directory)
71+ |= [[=hash:s l=listing:s] i=index:s]
72+ (~(catalog delver i) hash l)
82 ++ purge
73 ++ purge
83 |= =directory:s
74 |= =directory:s
84 ^- directory:s
75 ^- directory:s
@@ -193,10 +184,12 @@
193 %directory
184 %directory
194 =+ !<(d=directory:s vase)
185 =+ !<(d=directory:s vase)
195 =. directory (~(uni by directory) d)
186 =. directory (~(uni by directory) d)
187+ =. index
196− %+ roll
188+ %+ roll
197− ~(tap by d)
189+ ~(tap by d)
198− |= [[=hash:s l=listing:s] c=_cor]
190+ |= [[=hash:s l=listing:s] i=_index]
199− (index:c hash l)
191+ (~(catalog delver i) hash l)
192+ cor
200 ==
193 ==
201 ==
194 ==
202 ==
195 ==
@@ -234,7 +227,7 @@
234 ~(val by directory)
227 ~(val by directory)
235 |= =listing:s
228 |= =listing:s
236 |(=(filter %all) =(filter type.post.listing))
229 |(=(filter %all) =(filter type.post.listing))
237− =/ entries (~(get-entries delver [lookup trigrams phonetics]) term)
230+ =/ entries (~(get-entries delver index) term)
238 :: ~& %+ turn
231 :: ~& %+ turn
239 :: entries
232 :: entries
240 :: |= [=hash:s =rank:s]
233 :: |= [=hash:s =rank:s]
@@ -272,11 +265,11 @@
272 di-core(hash h, listing (~(gut by directory) h *listing:s))
265 di-core(hash h, listing (~(gut by directory) h *listing:s))
273 ++ di-remove
266 ++ di-remove
274 =. gone &
267 =. gone &
275− =/ keys (~(got by trail) hash)
268+ =/ keys (~(got by trail.index) hash)
276− =. trail (~(del by trail) hash)
269+ =. trail.index (~(del by trail.index) hash)
277− =. lookup
270+ =. lookup.index
278 ^- lookup:s
271 ^- lookup:s
279− %- ~(uni by lookup)
272+ %- ~(uni by lookup.index)
280 %- malt
273 %- malt
281 %+ turn
274 %+ turn
282 keys
275 keys
@@ -284,7 +277,7 @@
284 :- key
277 :- key
285 ^- (list entry:s)
278 ^- (list entry:s)
286 %+ skip
279 %+ skip
287− (~(got by lookup) key)
280+ (~(got by lookup.index) key)
288 |=([h=hash:s =rank:s] =(h hash))
281 |=([h=hash:s =rank:s] =(h hash))
289 =? published (~(has by published) hash)
282 =? published (~(has by published) hash)
290 (~(del by published) hash)
283 (~(del by published) hash)
@@ -292,73 +285,11 @@
292 ++ di-publish
285 ++ di-publish
293 |= l=listing:s
286 |= l=listing:s
294 =. listing l
287 =. listing l
295− =. cor (index hash l)
288+ =. index (~(catalog delver index) hash l)
296 =. published
289 =. published
297 ?. =(src our):bowl published
290 ?. =(src our):bowl published
298 (~(put by directory) hash.listing listing)
291 (~(put by directory) hash.listing listing)
299 =. cor (emit (invent:gossip %directory-listing !>(listing)))
292 =. cor (emit (invent:gossip %directory-listing !>(listing)))
300 di-core
293 di-core
301 --
294 --
302−++ index
303− |= [=hash:s l=listing:s]
304− ?> (lte (lent (trip title.post.l)) 77)
305− ?> (lte (lent (trip description.post.l)) 256)
306− ?> (lte (lent tags.post.l) 8)
307− ?> (lte (lent (trip link.post.l)) 1.024)
308− ?> (lte (lent (trip image.post.l)) 1.024)
309− =. cor
310− ?: =(image.post.l '') cor
311− =/ image (validate image.post.l)
312− cor
313− =/ link (validate link.post.l)
314− =/ entries=lookup:s
315− %- malt
316− %- zing
317− :~ ~[[(crip (cass (trip title.post.l))) ~[[hash rank=0]]]]
318− %+ turn
319− (sift:sift title.post.l)
320− |= word=@t
321− [word ~[[hash rank=1]]]
322− %+ turn
323− tags.post.l
324− |= tag=@t
325− [(norm:sift tag) ~[[hash rank=2]]]
326− %+ turn
327− (sift:sift description.post.l)
328− |= word=@t
329− [word ~[[hash rank=3]]]
330− ~[[type.post.l ~[[hash rank=4]]]]
331− ==
332− =/ keys ~(tap in ~(key by entries))
333− =. trail (~(put by trail) hash keys)
334− =. lookup
335− %- ~(uni by entries)
336− %- ~(rut by lookup)
337− |= [=key:s value=(list entry:s)]
338− ?. (~(has by entries) key) value
339− (weld value (~(got by entries) key))
340− =. trigrams
341− %+ roll
342− %+ snoc
343− %+ turn keys
344− |= =key:s
345− %- malt
346− %+ turn (iching key)
347− |= =trigram:s
348− [trigram (silt ~[key])]
349− trigrams
350− |= [next=trigrams:s t=trigrams:s]
351− %- (~(uno by t) next)
352− |= [k=key:s v=(set key:s) w=(set key:s)]
353− (~(uni in v) w)
354− =. phonetics
355− %-
356− %- ~(uno by phonetics)
357− %- malt
358− %+ turn keys
359− |= =key:s
360− [(utter:m key) (silt ~[key])]
361− |= [k=key:s v=(set key:s) w=(set key:s)]
362− (~(uni in v) w)
363− cor
364--
295--
desk/gen/get.hoonnew file+9
@@ -0,0 +1,9 @@
1+/- s=sphinx
2+:- %say
3+|= $: [now=@da eny=@uv =beak]
4+ [[=hash:s ~] ~]
5+ ==
6+=/ our (scot %p p.beak)
7+=/ wen (scot %da now)
8+=/ state=state:s .^(state:s /gx/[our]/sphinx/[wen]/state/noun)
9+noun+(~(got by directory.state) hash)
desk/gen/ranks.hoonnew file+37
@@ -0,0 +1,37 @@
1+/- s=sphinx
2+/+ *sphinx, *sift
3+:- %say
4+|= $: [now=@da eny=@uv =beak]
5+ [[word=@t algo=?(%all %exact %phonetic %ngram) ~] ~]
6+ ==
7+=/ our (scot %p p.beak)
8+=/ wen (scot %da now)
9+=/ state=state:s .^(state:s /gx/[our]/sphinx/[wen]/state/noun)
10+=/ title (norm:sift word) :: full query
11+=/ parts (sift:sift word) :: split query
12+=/ index ~(. delver index.state)
13+=/ lookup
14+ ?- algo
15+ %all get-entries.index
16+ %exact get-exact.index
17+ %ngram get-trigrams.index
18+ %phonetic get-phonetics.index
19+ ==
20+=/ title-entries=(list entry:s)
21+ %+ turn
22+ (lookup title)
23+ (uprank 10)
24+=/ entries=(list entry:s)
25+ %- sum-ranks.index
26+ %- zing
27+ %+ turn
28+ parts
29+ |= word=@t
30+ (lookup word)
31+:- %noun
32+%+ turn
33+ ?: =(algo %all) (lookup word)
34+ (welp entries title-entries)
35+|= [=hash:s =rank:s]
36+=/ listing (~(got by directory.state) hash)
37+[rank title.post.listing hash]
desk/lib/sphinx.hoon+132−35
@@ -1,5 +1,6 @@
1/- *sphinx
1/- *sphinx
2/+ sift
2/+ sift
3+/+ *validate-url
3/+ m=metaphone
4/+ m=metaphone
4|%
5|%
5++ iching
6++ iching
@@ -31,51 +32,150 @@
31 description.post
32 description.post
32 (crip tags.post)
33 (crip tags.post)
33 ==
34 ==
34−++ derank
35+++ uprank
35− |= [=entry offset=@ud]
36+ |= offset=@ud
37+ |= =entry
38+ ^- ^entry
36− [hash.entry (add rank.entry offset)] :: phonetically similar should be lower
39+ [hash.entry (add rank.entry offset)]
37++ sort-entries
40++ sort-entries
38 |= entries=(list entry)
41 |= entries=(list entry)
39 %+ sort entries
42 %+ sort entries
40 |= [a=entry b=entry]
43 |= [a=entry b=entry]
41− (lth rank.a rank.b)
44+ (gth rank.a rank.b)
42++ get-hashes
45++ get-hashes
43 |= entries=(list entry)
46 |= entries=(list entry)
44 %+ turn
47 %+ turn
45 entries
48 entries
46 |=(=entry hash.entry)
49 |=(=entry hash.entry)
47++ delver
50++ delver
48− |_ [=lookup =trigrams =phonetics]
51+ |_ [=lookup =trigrams =phonetics =trail]
49− ++ get-entries
52+ ++ get
50− |= =key
53+ ^- index
51− ^- (list entry)
54+ [lookup trigrams phonetics trail]
52− =/ title (norm:sift key) :: full query
55+ ++ catalog
53− =/ parts (sift:sift key) :: split query
56+ |= [=hash l=listing]
57+ ?.
58+ ?& (lte (lent (trip title.post.l)) 77)
59+ (lte (lent (trip description.post.l)) 256)
60+ (lte (lent tags.post.l) 8)
61+ (lte (lent (trip link.post.l)) 1.024)
62+ (lte (lent (trip image.post.l)) 1.024)
63+ ==
64+ get
65+ =/ image
66+ ?: =(image.post.l '') 'X'
67+ (validate image.post.l)
68+ =/ link (validate link.post.l)
69+ ?: |(=(image ~) =(link ~))
70+ get
54− =/ title-entries
71+ =/ entries=^lookup
55 %- malt
72 %- malt
56− ;: weld
57− (get-phonetics title)
58− (get-trigram-entries title)
59− (~(gut by lookup) title *(list entry))
73+ ^- (list [@t (list entry)])
74+ %- zing
75+ :~ ~[[type.post.l ~[[hash rank=4]]]]
76+ %+ turn
77+ (sift:sift description.post.l)
78+ |= word=@t
79+ [word ~[[hash rank=8]]]
80+ %+ turn
81+ tags.post.l
82+ |= tag=@t
83+ [(norm:sift tag) ~[[hash rank=12]]]
84+ %+ turn
85+ (sift:sift title.post.l)
86+ |= word=@t
87+ [word ~[[hash rank=16]]]
88+ ~[[(crip (cass (trip title.post.l))) ~[[hash rank=20]]]]
60 ==
89 ==
90+ =/ keys ~(tap in ~(key by entries))
91+ =. trail (~(put by trail) hash keys)
92+ =. lookup
93+ %- ~(uni by entries)
94+ %- ~(rut by lookup)
95+ |= [=key value=(list entry)]
96+ ?. (~(has by entries) key) value
97+ (weld value (~(got by entries) key))
98+ =. trigrams
99+ %+ roll
100+ %+ snoc
101+ %+ turn keys
102+ |= =key
103+ %- malt
104+ %+ turn (iching key)
105+ |= =trigram
106+ [trigram (silt ~[key])]
107+ trigrams
108+ |= [next=^trigrams t=^trigrams]
109+ %- (~(uno by t) next)
110+ |= [k=key v=(set key) w=(set key)]
111+ (~(uni in v) w)
112+ =. phonetics
113+ %-
114+ %- ~(uno by phonetics)
115+ %- malt
116+ %+ turn keys
117+ |= =key
118+ [(utter:m key) (silt ~[key])]
119+ |= [k=key v=(set key) w=(set key)]
120+ (~(uni in v) w)
121+ get
122+ ++ get-entries
123+ |= query=@t
124+ ^- (list entry)
125+ ?~ (find " " (trip query))
126+ =/ t (get-title-entries query)
127+ :: ~& t
128+ ~(tap by (malt t))
61 %~ tap by
129 %~ tap by
62 %-
130 %-
63 %~ uni by :: union and prefer title entries (better rank)
131 %~ uni by :: union and prefer title entries (better rank)
64− %- malt
132+ (malt (get-part-entries query))
133+ (malt (get-title-entries query))
134+ ++ get-part-entries
135+ |= query=@t
136+ ^- (list entry)
137+ =/ parts (sift:sift query) :: split query
138+ %- sum-ranks
65− %- zing
139+ %- zing
66− %+ turn
140+ %+ turn
67− parts
141+ parts
68− |= word=@t
142+ |= word=@t
69− %+ turn
70− ;: weld
143+ ;: weld
71− (get-phonetics word)
144+ (get-phonetics word)
145+ (turn (get-trigrams word) (uprank 2))
146+ (turn (get-exact word) (uprank 6))
147+ ==
72− (get-trigram-entries word)
148+ ++ get-title-entries
149+ |= query=@t
150+ ^- (list entry)
151+ =/ title (norm:sift query) :: full query
152+ =/ exact (turn (get-exact title) (uprank 8))
153+ =/ trigrams (turn (get-trigrams title) (uprank 4))
154+ =/ phonetics (get-phonetics title)
155+ ~& [%exact exact]
156+ ~& [%trigrams trigrams]
157+ ~& [%phonetics phonetics]
158+ ;: weld
159+ phonetics
160+ trigrams
161+ exact
162+ ==
163+ ++ sum-ranks
164+ |= entries=(list entry)
165+ ^- (list entry)
166+ %+ turn
167+ %~ tap by
168+ %+ roll
169+ entries
170+ |= [=entry gourd=(jar hash rank)]
171+ (~(add ja gourd) entry)
172+ |= [=hash es=(list rank)]
173+ ^- entry
174+ [hash (roll es add)]
175+ ++ get-exact
176+ |= term=@t
73− (~(gut by lookup) word *(list entry))
177+ (~(gut by lookup) term *(list entry))
74− ==
178+ ++ get-trigrams
75− |= =entry
76− (derank entry 10)
77− title-entries
78− ++ get-trigram-entries
79 |= term=@t
179 |= term=@t
80 ^- (list entry)
180 ^- (list entry)
81 %- zing
181 %- zing
@@ -90,17 +190,14 @@
90 %+ skim
190 %+ skim
91 %~ tap in
191 %~ tap in
92 (~(gut by phonetics) (utter:m term) *(set key))
192 (~(gut by phonetics) (utter:m term) *(set key))
93− |= [word=key]
193+ |= word=key
94 !=(term word)
194 !=(term word)
95 %+ roll
195 %+ roll
96 keys
196 keys
97 |= [word=key entries=(list entry)]
197 |= [word=key entries=(list entry)]
98 %+ weld
198 %+ weld
99 entries
199 entries
100− %+ turn
101− (~(gut by lookup) word *(list entry))
200+ (~(gut by lookup) word *(list entry))
102− |= =entry
103− (derank entry 10)
104 ++ get-keys
201 ++ get-keys
105 |= term=@t
202 |= term=@t
106 ^- (list @t)
203 ^- (list @t)
desk/lib/validate-url.hoon+2−2
@@ -1,8 +1,8 @@
1|%
1|%
2++ validate
2++ validate
3 |= url=@t
3 |= url=@t
4− ^- @t
4+ ^- (unit @t)
5− %+ rash url
5+ %+ rush url
6 %+ cold url
6 %+ cold url
7 ;~ plug
7 ;~ plug
8 ;~ pose
8 ;~ pose
desk/sur/sphinx.hoon+23
@@ -82,7 +82,21 @@
82 total=@ud
82 total=@ud
83 ==
83 ==
84::
84::
85++$ index
86+ $: =lookup
87+ =trigrams
88+ =phonetics
89+ =trail
90+ ==
91+::
85+$ state
92+$ state
93+ $: %2
94+ =index
95+ =directory
96+ published=directory
97+ ==
98+::
99++$ state-1
86 $: %1
100 $: %1
87 =lookup
101 =lookup
88 =trigrams
102 =trigrams
@@ -92,6 +106,15 @@
92 published=directory
106 published=directory
93 ==
107 ==
94::
108::
109++$ state-0
110+ $: %0
111+ =lookup
112+ =trigrams
113+ =phonetics
114+ =trail
115+ =directory:directory-state-zero
116+ published=directory:directory-state-zero
117+ ==
95++ directory-state-zero
118++ directory-state-zero
96 |%
119 |%
97 +$ post
120 +$ post