54fe925 · sphinx · git.iko.soy

git.iko.soy/sphinx

improving ranking

commit54fe92552153a54e150fc206bf93ae93138cf002 authorHunter Miller <hunter@hmiller.dev> dateTue 26 Jul 2022, 23:57 (4 years ago) parent37926b8

6 files changed+236−139

compare with master

unifiedsplit

desk/app/sphinx.hoon+33−102 desk/gen/get.hoon+9 desk/gen/ranks.hoon+37 desk/lib/sphinx.hoon+132−35 desk/lib/validate-url.hoon+2−2 desk/sur/sphinx.hoon+23

desk/app/sphinx.hoon+33−102

@@ -1,7 +1,6 @@

11/- s=sphinx

22/+ gossip, default-agent, verb, dbug

33/+ *sphinx

4−/+ *validate-url

54/+ sift

65/+ m=metaphone

76/$ grab-listing %noun %directory-listing

@@ -12,29 +11,12 @@

1211 |%

1312 +$ card card:agent:gall

1413 +$ edition

15− $% state-0

16− state-1

17− ==

18− +$ state-1

19− $: %1

20− =lookup:s

21− =trigrams:s

22− =phonetics:s

23− =trail:s

24− =directory:s

25− published=directory:s

26− ==

27− +$ state-0

28− $: %0

29− =lookup:s

30− =trigrams:s

31− =phonetics:s

32− =trail:s

33− =directory:directory-state-zero:s

34− published=directory:directory-state-zero:s

14+ $% state-0:s

15+ state-1:s

16+ state:s

3517 ==

3618 --

37−=| state-1

19+=| state:s

3820=* state -

3921=<

4022 %+ verb &

@@ -62,13 +44,9 @@

6244 =/ old !<(edition vase)

6345 |^

6446 ?- -.old

65− %1 `this(state old(published (purge published.old)))

66− %0

67− %_ $

68− -.old %1

69− published.old (purge (directory-0-to-1 published.old))

70− directory.old (directory-0-to-1 directory.old)

71− ==

47+ %2 `this(state (get-state-2 directory.old))

48+ %1 `this(state (get-state-2 directory.old))

49+ %0 `this(state (get-state-2 (directory-0-to-1 directory.old)))

7250 ==

7351 ++ directory-0-to-1

7452 |= =directory:directory-state-zero:s

@@ -79,6 +57,19 @@

7957 =/ post post.listing(+63 [+63.post.listing ''])

8058 listing(+2 post)

8159 ::

60+ ++ get-state-2

61+ |= =directory:s

62+ :* %2

63+ (reindex directory)

64+ directory

65+ (purge directory)

66+ ==

67+ ++ reindex

68+ |= =directory:s

69+ %+ roll

70+ ~(tap by directory)

71+ |= [[=hash:s l=listing:s] i=index:s]

72+ (~(catalog delver i) hash l)

8273 ++ purge

8374 |= =directory:s

8475 ^- directory:s

@@ -193,10 +184,12 @@

193184 %directory

194185 =+ !<(d=directory:s vase)

195186 =. directory (~(uni by directory) d)

196− %+ roll

197− ~(tap by d)

198− |= [[=hash:s l=listing:s] c=_cor]

199− (index:c hash l)

187+ =. index

188+ %+ roll

189+ ~(tap by d)

190+ |= [[=hash:s l=listing:s] i=_index]

191+ (~(catalog delver i) hash l)

192+ cor

200193 ==

201194 ==

202195 ==

@@ -234,7 +227,7 @@

234227 ~(val by directory)

235228 |= =listing:s

236229 |(=(filter %all) =(filter type.post.listing))

237− =/ entries (~(get-entries delver [lookup trigrams phonetics]) term)

230+ =/ entries (~(get-entries delver index) term)

238231 :: ~& %+ turn

239232 :: entries

240233 :: |= [=hash:s =rank:s]

@@ -272,11 +265,11 @@

272265 di-core(hash h, listing (~(gut by directory) h *listing:s))

273266 ++ di-remove

274267 =. gone &

275− =/ keys (~(got by trail) hash)

276− =. trail (~(del by trail) hash)

277− =. lookup

268+ =/ keys (~(got by trail.index) hash)

269+ =. trail.index (~(del by trail.index) hash)

270+ =. lookup.index

278271 ^- lookup:s

279− %- ~(uni by lookup)

272+ %- ~(uni by lookup.index)

280273 %- malt

281274 %+ turn

282275 keys

@@ -284,7 +277,7 @@

284277 :- key

285278 ^- (list entry:s)

286279 %+ skip

287− (~(got by lookup) key)

280+ (~(got by lookup.index) key)

288281 |=([h=hash:s =rank:s] =(h hash))

289282 =? published (~(has by published) hash)

290283 (~(del by published) hash)

@@ -292,73 +285,11 @@

292285 ++ di-publish

293286 |= l=listing:s

294287 =. listing l

295− =. cor (index hash l)

288+ =. index (~(catalog delver index) hash l)

296289 =. published

297290 ?. =(src our):bowl published

298291 (~(put by directory) hash.listing listing)

299292 =. cor (emit (invent:gossip %directory-listing !>(listing)))

300293 di-core

301294 --

302−++ index

303− |= [=hash:s l=listing:s]

304− ?> (lte (lent (trip title.post.l)) 77)

305− ?> (lte (lent (trip description.post.l)) 256)

306− ?> (lte (lent tags.post.l) 8)

307− ?> (lte (lent (trip link.post.l)) 1.024)

308− ?> (lte (lent (trip image.post.l)) 1.024)

309− =. cor

310− ?: =(image.post.l '') cor

311− =/ image (validate image.post.l)

312− cor

313− =/ link (validate link.post.l)

314− =/ entries=lookup:s

315− %- malt

316− %- zing

317− :~ ~[[(crip (cass (trip title.post.l))) ~[[hash rank=0]]]]

318− %+ turn

319− (sift:sift title.post.l)

320− |= word=@t

321− [word ~[[hash rank=1]]]

322− %+ turn

323− tags.post.l

324− |= tag=@t

325− [(norm:sift tag) ~[[hash rank=2]]]

326− %+ turn

327− (sift:sift description.post.l)

328− |= word=@t

329− [word ~[[hash rank=3]]]

330− ~[[type.post.l ~[[hash rank=4]]]]

331− ==

332− =/ keys ~(tap in ~(key by entries))

333− =. trail (~(put by trail) hash keys)

334− =. lookup

335− %- ~(uni by entries)

336− %- ~(rut by lookup)

337− |= [=key:s value=(list entry:s)]

338− ?. (~(has by entries) key) value

339− (weld value (~(got by entries) key))

340− =. trigrams

341− %+ roll

342− %+ snoc

343− %+ turn keys

344− |= =key:s

345− %- malt

346− %+ turn (iching key)

347− |= =trigram:s

348− [trigram (silt ~[key])]

349− trigrams

350− |= [next=trigrams:s t=trigrams:s]

351− %- (~(uno by t) next)

352− |= [k=key:s v=(set key:s) w=(set key:s)]

353− (~(uni in v) w)

354− =. phonetics

355− %-

356− %- ~(uno by phonetics)

357− %- malt

358− %+ turn keys

359− |= =key:s

360− [(utter:m key) (silt ~[key])]

361− |= [k=key:s v=(set key:s) w=(set key:s)]

362− (~(uni in v) w)

363− cor

364295--

desk/gen/get.hoonnew file+9

@@ -0,0 +1,9 @@

1+/- s=sphinx

2+:- %say

3+|= $: [now=@da eny=@uv =beak]

4+ [[=hash:s ~] ~]

5+ ==

6+=/ our (scot %p p.beak)

7+=/ wen (scot %da now)

8+=/ state=state:s .^(state:s /gx/[our]/sphinx/[wen]/state/noun)

9+noun+(~(got by directory.state) hash)

desk/gen/ranks.hoonnew file+37

@@ -0,0 +1,37 @@

1+/- s=sphinx

2+/+ *sphinx, *sift

3+:- %say

4+|= $: [now=@da eny=@uv =beak]

5+ [[word=@t algo=?(%all %exact %phonetic %ngram) ~] ~]

6+ ==

7+=/ our (scot %p p.beak)

8+=/ wen (scot %da now)

9+=/ state=state:s .^(state:s /gx/[our]/sphinx/[wen]/state/noun)

10+=/ title (norm:sift word) :: full query

11+=/ parts (sift:sift word) :: split query

12+=/ index ~(. delver index.state)

13+=/ lookup

14+ ?- algo

15+ %all get-entries.index

16+ %exact get-exact.index

17+ %ngram get-trigrams.index

18+ %phonetic get-phonetics.index

19+ ==

20+=/ title-entries=(list entry:s)

21+ %+ turn

22+ (lookup title)

23+ (uprank 10)

24+=/ entries=(list entry:s)

25+ %- sum-ranks.index

26+ %- zing

27+ %+ turn

28+ parts

29+ |= word=@t

30+ (lookup word)

31+:- %noun

32+%+ turn

33+ ?: =(algo %all) (lookup word)

34+ (welp entries title-entries)

35+|= [=hash:s =rank:s]

36+=/ listing (~(got by directory.state) hash)

37+[rank title.post.listing hash]

desk/lib/sphinx.hoon+132−35

@@ -1,5 +1,6 @@

11/- *sphinx

22/+ sift

3+/+ *validate-url

34/+ m=metaphone

45|%

56++ iching

@@ -31,51 +32,150 @@

3132 description.post

3233 (crip tags.post)

3334 ==

34−++ derank

35− |= [=entry offset=@ud]

36− [hash.entry (add rank.entry offset)] :: phonetically similar should be lower

35+++ uprank

36+ |= offset=@ud

37+ |= =entry

38+ ^- ^entry

39+ [hash.entry (add rank.entry offset)]

3740++ sort-entries

3841 |= entries=(list entry)

3942 %+ sort entries

4043 |= [a=entry b=entry]

41− (lth rank.a rank.b)

44+ (gth rank.a rank.b)

4245++ get-hashes

4346 |= entries=(list entry)

4447 %+ turn

4548 entries

4649 |=(=entry hash.entry)

4750++ delver

48− |_ [=lookup =trigrams =phonetics]

49− ++ get-entries

50− |= =key

51− ^- (list entry)

52− =/ title (norm:sift key) :: full query

53− =/ parts (sift:sift key) :: split query

54− =/ title-entries

51+ |_ [=lookup =trigrams =phonetics =trail]

52+ ++ get

53+ ^- index

54+ [lookup trigrams phonetics trail]

55+ ++ catalog

56+ |= [=hash l=listing]

57+ ?.

58+ ?& (lte (lent (trip title.post.l)) 77)

59+ (lte (lent (trip description.post.l)) 256)

60+ (lte (lent tags.post.l) 8)

61+ (lte (lent (trip link.post.l)) 1.024)

62+ (lte (lent (trip image.post.l)) 1.024)

63+ ==

64+ get

65+ =/ image

66+ ?: =(image.post.l '') 'X'

67+ (validate image.post.l)

68+ =/ link (validate link.post.l)

69+ ?: |(=(image ~) =(link ~))

70+ get

71+ =/ entries=^lookup

5572 %- malt

56− ;: weld

57− (get-phonetics title)

58− (get-trigram-entries title)

59− (~(gut by lookup) title *(list entry))

73+ ^- (list [@t (list entry)])

74+ %- zing

75+ :~ ~[[type.post.l ~[[hash rank=4]]]]

76+ %+ turn

77+ (sift:sift description.post.l)

78+ |= word=@t

79+ [word ~[[hash rank=8]]]

80+ %+ turn

81+ tags.post.l

82+ |= tag=@t

83+ [(norm:sift tag) ~[[hash rank=12]]]

84+ %+ turn

85+ (sift:sift title.post.l)

86+ |= word=@t

87+ [word ~[[hash rank=16]]]

88+ ~[[(crip (cass (trip title.post.l))) ~[[hash rank=20]]]]

6089 ==

90+ =/ keys ~(tap in ~(key by entries))

91+ =. trail (~(put by trail) hash keys)

92+ =. lookup

93+ %- ~(uni by entries)

94+ %- ~(rut by lookup)

95+ |= [=key value=(list entry)]

96+ ?. (~(has by entries) key) value

97+ (weld value (~(got by entries) key))

98+ =. trigrams

99+ %+ roll

100+ %+ snoc

101+ %+ turn keys

102+ |= =key

103+ %- malt

104+ %+ turn (iching key)

105+ |= =trigram

106+ [trigram (silt ~[key])]

107+ trigrams

108+ |= [next=^trigrams t=^trigrams]

109+ %- (~(uno by t) next)

110+ |= [k=key v=(set key) w=(set key)]

111+ (~(uni in v) w)

112+ =. phonetics

113+ %-

114+ %- ~(uno by phonetics)

115+ %- malt

116+ %+ turn keys

117+ |= =key

118+ [(utter:m key) (silt ~[key])]

119+ |= [k=key v=(set key) w=(set key)]

120+ (~(uni in v) w)

121+ get

122+ ++ get-entries

123+ |= query=@t

124+ ^- (list entry)

125+ ?~ (find " " (trip query))

126+ =/ t (get-title-entries query)

127+ :: ~& t

128+ ~(tap by (malt t))

61129 %~ tap by

62130 %-

63131 %~ uni by :: union and prefer title entries (better rank)

64− %- malt

65− %- zing

66− %+ turn

67− parts

68− |= word=@t

69− %+ turn

70− ;: weld

71− (get-phonetics word)

72− (get-trigram-entries word)

73− (~(gut by lookup) word *(list entry))

74− ==

75− |= =entry

76− (derank entry 10)

77− title-entries

78− ++ get-trigram-entries

132+ (malt (get-part-entries query))

133+ (malt (get-title-entries query))

134+ ++ get-part-entries

135+ |= query=@t

136+ ^- (list entry)

137+ =/ parts (sift:sift query) :: split query

138+ %- sum-ranks

139+ %- zing

140+ %+ turn

141+ parts

142+ |= word=@t

143+ ;: weld

144+ (get-phonetics word)

145+ (turn (get-trigrams word) (uprank 2))

146+ (turn (get-exact word) (uprank 6))

147+ ==

148+ ++ get-title-entries

149+ |= query=@t

150+ ^- (list entry)

151+ =/ title (norm:sift query) :: full query

152+ =/ exact (turn (get-exact title) (uprank 8))

153+ =/ trigrams (turn (get-trigrams title) (uprank 4))

154+ =/ phonetics (get-phonetics title)

155+ ~& [%exact exact]

156+ ~& [%trigrams trigrams]

157+ ~& [%phonetics phonetics]

158+ ;: weld

159+ phonetics

160+ trigrams

161+ exact

162+ ==

163+ ++ sum-ranks

164+ |= entries=(list entry)

165+ ^- (list entry)

166+ %+ turn

167+ %~ tap by

168+ %+ roll

169+ entries

170+ |= [=entry gourd=(jar hash rank)]

171+ (~(add ja gourd) entry)

172+ |= [=hash es=(list rank)]

173+ ^- entry

174+ [hash (roll es add)]

175+ ++ get-exact

176+ |= term=@t

177+ (~(gut by lookup) term *(list entry))

178+ ++ get-trigrams

79179 |= term=@t

80180 ^- (list entry)

81181 %- zing

@@ -90,17 +190,14 @@

90190 %+ skim

91191 %~ tap in

92192 (~(gut by phonetics) (utter:m term) *(set key))

93− |= [word=key]

193+ |= word=key

94194 !=(term word)

95195 %+ roll

96196 keys

97197 |= [word=key entries=(list entry)]

98198 %+ weld

99199 entries

100− %+ turn

101− (~(gut by lookup) word *(list entry))

102− |= =entry

103− (derank entry 10)

200+ (~(gut by lookup) word *(list entry))

104201 ++ get-keys

105202 |= term=@t

106203 ^- (list @t)

desk/lib/validate-url.hoon+2−2

@@ -1,8 +1,8 @@

11|%

22++ validate

33 |= url=@t

4− ^- @t

5− %+ rash url

4+ ^- (unit @t)

5+ %+ rush url

66 %+ cold url

77 ;~ plug

88 ;~ pose

desk/sur/sphinx.hoon+23

@@ -82,7 +82,21 @@

8282 total=@ud

8383 ==

8484::

85++$ index

86+ $: =lookup

87+ =trigrams

88+ =phonetics

89+ =trail

90+ ==

91+::

8592+$ state

93+ $: %2

94+ =index

95+ =directory

96+ published=directory

97+ ==

98+::

99++$ state-1

86100 $: %1

87101 =lookup

88102 =trigrams

@@ -92,6 +106,15 @@

92106 published=directory

93107 ==

94108::

109++$ state-0

110+ $: %0

111+ =lookup

112+ =trigrams

113+ =phonetics

114+ =trail

115+ =directory:directory-state-zero

116+ published=directory:directory-state-zero

117+ ==

95118++ directory-state-zero

96119 |%

97120 +$ post