git.iko.soy/sphinx
on master help sign in
adding fuzzy trigram matching
f2a901aHunter Miller
desk/app/sphinx.hoon+16−57
compare with master
unifiedsplit
@@ -12,6 +12,7 @@
12 +$ state-0
12 +$ state-0
13 $: %0
13 $: %0
14 =lookup:s
14 =lookup:s
15+ =trigrams:s
15 =phonetics:s
16 =phonetics:s
16 =trail:s
17 =trail:s
17 =directory:s
18 =directory:s
@@ -183,7 +184,7 @@
183 %- get-listings
184 %- get-listings
184 %- get-hashes
185 %- get-hashes
185 %- sort-entries
186 %- sort-entries
186− (get-entries term)
187+ (~(get-entries delver [lookup trigrams phonetics]) term)
187 |= =listing:s
188 |= =listing:s
188 |(=(filter %all) =(filter type.post.listing))
189 |(=(filter %all) =(filter type.post.listing))
189 =/ listings (swag [start limit] all)
190 =/ listings (swag [start limit] all)
@@ -194,62 +195,6 @@
194 (lent all)
195 (lent all)
195 ==
196 ==
196 ==
197 ==
197−++ get-entries
198− |= =key:s
199− ^- (list entry:s)
200− =/ title (norm:sift key) :: full query
201− =/ parts (sift:sift key) :: split query
202− =/ title-entries
203− %- malt
204− %+ weld
205− (get-phonetics title)
206− (~(gut by lookup) title *(list entry:s))
207− %~ tap by
208− %-
209− %~ uni by :: union and prefer title entries (better rank)
210− %- malt
211− %- zing
212− %+ turn
213− parts
214− |= word=@t
215− %+ turn
216− %+ weld
217− (get-phonetics word)
218− (~(gut by lookup) word *(list entry:s))
219− |= =entry:s
220− (derank entry 10)
221− title-entries
222−++ get-phonetics
223− |= =key:s
224− ^- (list entry:s)
225− =/ keys
226− %+ skim
227− %~ tap in
228− (~(gut by phonetics) (utter:m key) *(set key:s))
229− |= [word=key:s]
230− !=(key word)
231− %+ roll
232− keys
233− |= [word=key:s entries=(list entry:s)]
234− %+ weld
235− entries
236− %+ turn
237− (~(gut by lookup) word *(list entry:s))
238− |= =entry:s
239− (derank entry 10)
240−++ derank
241− |= [=entry:s offset=@ud]
242− [hash.entry (add rank.entry offset)] :: phonetically similar should be lower
243−++ sort-entries
244− |= entries=(list entry:s)
245− %+ sort entries
246− |= [a=entry:s b=entry:s]
247− (lth rank.a rank.b)
248−++ get-hashes
249− |= entries=(list entry:s)
250− %+ turn
251− entries
252− |=(=entry:s hash.entry)
253++ get-listings
198++ get-listings
254 |= l=(list hash)
199 |= l=(list hash)
255 %+ turn l
200 %+ turn l
@@ -293,6 +238,20 @@
293 |= [=key:s value=(list entry:s)]
238 |= [=key:s value=(list entry:s)]
294 ?. (~(has by entries) key) value
239 ?. (~(has by entries) key) value
295 (weld value (~(got by entries) key))
240 (weld value (~(got by entries) key))
241+ =. trigrams
242+ %+ roll
243+ %+ snoc
244+ %+ turn keys
245+ |= =key:s
246+ %- malt
247+ %+ turn (iching key)
248+ |= =trigram:s
249+ [trigram (silt ~[key])]
250+ trigrams
251+ |= [next=trigrams:s t=trigrams:s]
252+ %- (~(uno by t) next)
253+ |= [k=key:s v=(set key:s) w=(set key:s)]
254+ (~(uni in v) w)
296 =. phonetics
255 =. phonetics
297 %-
256 %-
298 %- ~(uno by phonetics)
257 %- ~(uno by phonetics)
desk/desk.docket-0+1−1
@@ -1,6 +1,6 @@
1:~ title+'sphinx'
1:~ title+'sphinx'
2 info+'An Urbit app that does great things'
2 info+'An Urbit app that does great things'
3− color+0x2e.4347
3+ color+0xb2.5068
4 glob-http+['https://bootstrap.urbit.org/glob-0v5.hurm4.ejod5.ngg9h.iub9i.n1j7o.glob' 0v5.hurm4.ejod5.ngg9h.iub9i.n1j7o]
4 glob-http+['https://bootstrap.urbit.org/glob-0v5.hurm4.ejod5.ngg9h.iub9i.n1j7o.glob' 0v5.hurm4.ejod5.ngg9h.iub9i.n1j7o]
5 base+'sphinx'
5 base+'sphinx'
6 version+[0 0 1]
6 version+[0 0 1]
desk/lib/sphinx.hoon+114−1
@@ -1,4 +1,6 @@
1/- *sphinx
1/- *sphinx
2+/+ sift
3+/+ m=metaphone
2|%
4|%
3++ iching
5++ iching
4 |= word=@t
6 |= word=@t
@@ -10,7 +12,13 @@
10 %+ turn
12 %+ turn
11 (gulf 0 (sub (lent w) 3))
13 (gulf 0 (sub (lent w) 3))
12 |= index=@ud
14 |= index=@ud
15+ %- crip
16+ ^- tape
17+ ^- (list @)
18+ %+ sort
13− (crip (swag [index 3] w))
19+ `(list @ud)``(list @)`(swag [index 3] w)
20+ |= [p=@ud q=@ud]
21+ (gth q p)
14++ digest
22++ digest
15 |= =post
23 |= =post
16 ^- @uv
24 ^- @uv
@@ -23,4 +31,109 @@
23 description.post
31 description.post
24 (crip tags.post)
32 (crip tags.post)
25 ==
33 ==
34+++ derank
35+ |= [=entry offset=@ud]
36+ [hash.entry (add rank.entry offset)] :: phonetically similar should be lower
37+++ sort-entries
38+ |= entries=(list entry)
39+ %+ sort entries
40+ |= [a=entry b=entry]
41+ (lth rank.a rank.b)
42+++ get-hashes
43+ |= entries=(list entry)
44+ %+ turn
45+ entries
46+ |=(=entry hash.entry)
47+++ delver
48+ |_ [=lookup =trigrams =phonetics]
49+ ++ get-entries
50+ |= =key
51+ ^- (list entry)
52+ =/ title (norm:sift key) :: full query
53+ =/ parts (sift:sift key) :: split query
54+ =/ title-entries
55+ %- malt
56+ ;: weld
57+ (get-phonetics title)
58+ (get-trigram-entries title)
59+ (~(gut by lookup) title *(list entry))
60+ ==
61+ %~ tap by
62+ %-
63+ %~ uni by :: union and prefer title entries (better rank)
64+ %- malt
65+ %- zing
66+ %+ turn
67+ parts
68+ |= word=@t
69+ %+ turn
70+ ;: weld
71+ (get-phonetics word)
72+ (get-trigram-entries word)
73+ (~(gut by lookup) word *(list entry))
74+ ==
75+ |= =entry
76+ (derank entry 10)
77+ title-entries
78+ ++ get-trigram-entries
79+ |= term=@t
80+ ^- (list entry)
81+ %- zing
82+ %+ turn
83+ (get-keys term)
84+ |= item=@t
85+ (~(gut by lookup) item *(list entry))
86+ ++ get-phonetics
87+ |= term=key
88+ ^- (list entry)
89+ =/ keys
90+ %+ skim
91+ %~ tap in
92+ (~(gut by phonetics) (utter:m term) *(set key))
93+ |= [word=key]
94+ !=(term word)
95+ %+ roll
96+ keys
97+ |= [word=key entries=(list entry)]
98+ %+ weld
99+ entries
100+ %+ turn
101+ (~(gut by lookup) word *(list entry))
102+ |= =entry
103+ (derank entry 10)
104+ ++ get-keys
105+ |= term=@t
106+ ^- (list @t)
107+ =/ tally (get-term-counts term)
108+ ~& tally
109+ =/ tallys ~(tap by tally)
110+ =/ count (lent (iching term))
111+ %+ turn
112+ %+ sort
113+ %+ skim
114+ tallys
115+ |= [k=@t v=@ud]
116+ (gth:rs (div:rs (sun:rs v) (sun:rs count)) .0.6)
117+ |= [[k=@t v=@ud] [l=@t w=@ud]]
118+ (lth v w)
119+ |= [k=@t v=@ud]
120+ k
121+ ++ get-term-counts
122+ |= term=@t
123+ ^- (map @t @ud)
124+ %+ roll
125+ (iching term)
126+ |= [ngram=@t tally=(map @t @ud)]
127+ ^- (map @t @ud)
128+ =/ words=(set @t) (~(gut by trigrams) ngram *(set @t))
129+ %-
130+ %- ~(uno by tally)
131+ %- malt
132+ %+ turn
133+ ~(tap in words)
134+ |= word=@t
135+ [word 1]
136+ |= [k=@t v=@ud w=@ud]
137+ (add v w)
138+ --
26--
139--
desk/sur/sphinx.hoon+6
@@ -3,6 +3,7 @@
3+$ hash @uv :: @uv version of shax result
3+$ hash @uv :: @uv version of shax result
4+$ rank @ud :: @ud representing priority of listing
4+$ rank @ud :: @ud representing priority of listing
5+$ phonetic @t :: @t consisting of phonetic characters from metaphone
5+$ phonetic @t :: @t consisting of phonetic characters from metaphone
6++$ trigram @t :: @t consisting of three characters taken from a word
6::
7::
7:: an entry points to a listing hash and provides
8:: an entry points to a listing hash and provides
8:: a semblance of priority for the listing
9:: a semblance of priority for the listing
@@ -57,6 +58,11 @@
57:: use that to add other words to the results.
58:: use that to add other words to the results.
58+$ phonetics (map phonetic (set key))
59+$ phonetics (map phonetic (set key))
59::
60::
61+:: trigrams act similarly to phonetics. we keep a map of trigram to set
62+:: of lookup keys. we take the query's trigrams and count how many
63+:: times a particular key comes up.
64++$ trigrams (map trigram (set key))
65+::
60:: trail is a way for us to know all the keys required to update
66:: trail is a way for us to know all the keys required to update
61:: a listing with the new hash of it's contents
67:: a listing with the new hash of it's contents
62::
68::