on master help sign in
resetting from ngram experiment, trying to get more data in
commit52699e4f51dba0f8b34b9a3bd2a4660cc3e33652 authorHunter Miller <hunter@hmiller.dev> dateSat 9 Jul 2022, 18:16 (4 years ago) parent89cffd6
8 files changed+126−93
desk/app/seek.hoon+23−81 desk/gen/publish-all.hoon+29 desk/gen/publish.hoon+4−4 desk/lib/seek-json.hoon+18−1 desk/lib/sift.hoon+20−5 desk/mar/declarations.hoon+14 desk/mar/declare.hoon+14 desk/sur/seek.hoon+4−2
desk/app/seek.hoon+23−81
@@ -98,22 +98,31 @@
9898 |= [=mark =vase]
9999 |^ ^+ cor
100100 ?+ mark ~|(bad-poke/mark !!)
101− %directory-listing
102− =+ !<(=listing:s vase)
103− (publish listing)
101+ %declare
102+ =+ !<(=declare:s vase)
103+ di-abet:(publish declare di-core)
104+ %declarations
105+ =+ !<(declared=(list declare:s) vase)
106+ =. di-core (roll declared publish)
107+ cor
104108 ==
105109 ++ publish
106− |= =listing:s
107− ^+ cor
110+ |= [=declare:s core=_di-core]
111+ ^+ di-core
108112 ::
109− ?> =(src.bowl source.listing)
110113 ?> from-self
111− ?> =(hash.listing (digest post.listing))
114+ =/ listing
115+ :* post=q.declare
116+ hash=(digest q.declare)
117+ reach=p.declare
118+ source=our.bowl
119+ time=now.bowl
120+ ==
112121 ?: (~(has by directory) hash.listing)
113122 ~& 'Listing already exists.'
114123 cor
115124 =. published (~(put by directory) hash.listing listing)
116− di-abet:(di-publish:(di-abed:di-core hash.listing) listing)
125+ (di-publish:(di-abed:core hash.listing) listing)
117126 --
118127::
119128++ watch
@@ -165,16 +174,7 @@
165174 =/ limit (slav %ud i.t.t.t.t.path)
166175 :: expects encoded @t values)
167176 =/ term `@t`(slav %t i.t.t.path)
168− =/ term-counts (get-term-counts term)
169− ~& term-counts
170− =/ terms (get-terms term-counts)
171− ~& terms
172− =/ all=(list listing:s)
173− %- zing
174− %+ turn
175− terms
176− |= [=key:s v=@ud]
177− (get-listings (get-hashes key))
177+ =/ all (get-listings (get-hashes term))
178178 =/ listings (swag [start limit] all)
179179 :* listings
180180 start
@@ -183,69 +183,15 @@
183183 (lent all)
184184 ==
185185 ==
186−::
187−++ get-terms
188− |= tally=(map @t @ud)
189− ^- (list [@t @ud])
190− =/ sorted
191− %+ sort
192− ~(tap by tally)
193− |= [[k=@t v=@ud] [l=@t w=@ud]]
194− (lth v w)
195− =/ median-index (div (lent sorted) 2)
196− =/ q1 (median (scag +(median-index) sorted))
197− =/ q3 (median (slag median-index sorted))
198− =/ iqr (sub q3 q1)
199− %+ skim
200− sorted
201− |= [k=@t v=@ud]
202− (gth:rs (sun:rs v) (mul:rs (sun:rs q3) .1.5))
203−++ median
204− |= terms=(list [k=@t v=@ud])
205− ^- @ud
206− =/ length (lent terms)
207− =/ middle (div length 2)
208− ?. =((mod length 2) 0) (tail (snag middle terms))
209− %+ div
210− %+ add
211− (tail (snag middle terms))
212− (tail (snag +(middle) terms))
213− 2
214−++ get-term-counts
215− |= term=@t
216− ^- (map @t @ud)
217− =/ parts (split:sift term)
218− =/ ngrams=(list @t)
219− %- zing
220− %+ turn
221− parts
222− |= part=@t
223− (iching part)
224− %+ roll
225− ngrams
226− |= [ngram=@t tally=(map @t @ud)]
227− ^- (map @t @ud)
228− =/ words=(set @t) (~(gut by trigrams) ngram *(set @t))
229− %-
230− %- ~(uno by tally)
231− %- malt
232− %+ turn
233− ~(tap in words)
234− |= word=@t
235− [word 1]
236− |= [k=@t v=@ud w=@ud]
237− (add v w)
238186++ get-hashes
239187 |= =key:s
240− ^- (list hash:s)
241188 %+ turn
242189 %+ sort (~(gut by lookup) key *(list entry:s))
243190 |= [a=entry:s b=entry:s]
244191 (gte rank.a rank.b)
245192 |=(=entry:s hash.entry)
246193++ get-listings
247− |= l=(list hash:s)
248− ^- (list listing:s)
194+ |= l=(list hash)
249195 %+ turn l
250196 |=(=hash:s (~(got by directory) hash))
251197++ from-self =(our src):bowl
@@ -264,22 +210,18 @@
264210 =/ entries=lookup:s
265211 %- malt
266212 %- zing
267− :~ %+ turn
268− (sift:sift title.post.l)
269− |= word=@t
270− [word ~[[hash rank=0]]]
271− ~[[type.post.l ~[[hash rank=1]]]]
213+ :~ ~[[title.post.l ~[[hash rank=0]]]]
272214 %+ turn
273215 tags.post.l
274216 |= tag=@t
275− [tag ~[[hash rank=2]]]
217+ [tag ~[[hash rank=1]]]
276218 %+ turn
277219 (sift:sift description.post.l)
278220 |= word=@t
279− [word ~[[hash rank=3]]]
221+ [word ~[[hash rank=2]]]
222+ ~[[type.post.l ~[[hash rank=3]]]]
280223 ==
281224 =/ keys ~(tap in ~(key by entries))
282− ~& keys
283225 =. trail (~(put by trail) hash keys)
284226 =. lookup
285227 %- ~(uni by entries)
desk/gen/publish-all.hoonnew file+29
@@ -0,0 +1,29 @@
1+/- s=seek
2+/+ *seek
3+:- %say
4+|= $: [now=@da eny=@uv =beak]
5+ [ ~]
6+ ==
7+:- %declarations
8+^- (list declare:s)
9+:* :- %friends
10+ ['networked subject' %group 'web+urbitgraph://group/~matwet/networked-subject' 'https://subject.network | networked subject' 'https://urbits3.ams3.digitaloceanspaces.com/sitful-hatred/2022.7.07..17.15.52-ns.png' ~['hosting' 'networking' 'urbit' 'ops']]
11+ :- %friends
12+ ['Hooniverse' %group 'web+urbitgraph://group/~hiddev-dannut/new-hooniverse' 'Community based Hoon learning for all levels. For discussion of Hoon specific to Uqbar, join ~hiddev-dannut/uhoon' 'https://i.imgur.com/ghThlz7.png' ~['hoon' 'dev' 'urbit' 'programming' 'education']]
13+ :- %friends
14+ ['Hollow Mars Theory' %group 'web+urbitgraph://group/~rabsef-bicrym/hollow-mars-theory' 'Definitely NOT a Conspiracy Chat' '' ~['conspiracies' 'ufos' 'steel beams']]
15+ :- %friends
16+ ['celestial systems' %group 'web+urbitgraph://group/~nocsyx-lassul/celestial-systems' 'A place for pilots who are building hosting providers' '' ~['hosting' 'ops' 'urbit']]
17+ :- %friends
18+ ['Structure' %group 'web+urbitgraph://group/~fabled-faster/structure' 'Urbit Structural Design and Engineering Group. Always Thinking About Mechanics.' 'https://fabled-faster.nyc3.digitaloceanspaces.com/fabled-faster/2022.1.27..17.59.43-image.png' ~['design' 'ux' 'hci' 'urbit']]
19+ :- %friends
20+ ['The Marketplace' %group 'web+urbitgraph://group/~tirrel/the-marketplace' 'Welcome to The Marketplace, featuring The Pit, Urbit\'s first open-outcry market!' 'https://snipboard.io/U0IYyi.jpg' ~['urbit' 'market' 'sell' 'buy']]
21+ :- %friends
22+ ['The Forge' %group 'web+urbitgraph://group/~middev/the-forge' 'pale fire computing' 'https://nyc3.digitaloceanspaces.com/archiv/littel-wolfur/2021.5.06..21.01.58-the%20forge.png' ~['dev' 'programming' 'hoon' 'urbit']]
23+ :- %friends
24+ ['UFORIA' %group 'web+urbitgraph://group/~tiplec-lacnyx/ufora' 'UFO Research, Investigations, and Analysis' 'https://tiplec-lacnyx.nyc3.digitaloceanspaces.com/tiplec-lacnyx/2021.12.27..06.09.47-change1.jpg' ~['UFO' 'conspiracies' 'research']]
25+ :- %friends
26+ ['Urbit Community' %group 'web+urbitgraph://group/~bitbet-bolbel/urbit-community' 'World hub, help desk, meet and greet, etc.' 'https://fabled-faster.nyc3.digitaloceanspaces.com/fabled-faster/2021.4.02..21.52.41-UC.png' ~['general' 'urbit' 'community' 'help']]
27+ :- %friends
28+ ['Urbytes' %group 'web+urbitgraph://group/~nartes-fasrum/urbytes' 'We feed and water Mars.' '' ~['cooking' 'food' 'beverages' 'peanut butter eggs']]
29+==
desk/gen/publish.hoon+4−4
@@ -2,8 +2,8 @@
22/+ *seek
33:- %say
44|= $: [now=@da eny=@uv =beak]
5− [[=post:s source=@p ~] ~]
5+ [[=post:s ~] ~]
66 ==
7−:- %directory-listing
8−^- listing:s
9−[post (digest post) %friends source now]
7+:- %declare
8+^- declare:s
9+[%friends post]
desk/lib/seek-json.hoon+18−1
@@ -34,6 +34,23 @@
3434++ dejs
3535 =, dejs:format
3636 |%
37−
37+ ++ declarations (ar declare)
38+ ++ declare
39+ ^- $-(json declare:s)
40+ %- ot
41+ :~ reach/(su (perk [%friends %public %private ~]))
42+ post/post
43+ ==
44+ ++ post
45+ ^- $-(json post:s)
46+ %- ot
47+ :~ title/so
48+ type/type
49+ link/so
50+ description/so
51+ tags/(ar so)
52+ image/so
53+ ==
54+ ++ type (su (perk [%app %group %content %other ~]))
3855 --
3956--
desk/lib/sift.hoon+20−5
@@ -6,16 +6,31 @@
66 (split corpus)
77 |= word=@t
88 (~(has in words) word)
9+++ norm
10+ |= corpus=@t
11+ (crip (cass (trimall (crip (expunge corpus)))))
912++ split
1013 |= corpus=@t
1114 ^- (list @t)
12− =/ lines (crip (trimall corpus))
13− %+ rash lines
14− (more ace (cook crip (plus ;~(pose aln hep))))
15+ ~& (norm corpus)
16+ %+ rash (norm corpus)
17+ (more ace (cook crip (plus allowed)))
18+++ allowed ;~(pose aln hep ace)
19+++ banned ;~(less allowed prn)
20+++ expunge
21+ |= corpus=@t
22+ ^- (list @t)
23+ %- zing
24+ %+ rash corpus
25+ %- star
26+ ;~ pose
27+ ;~(sfix allowed (plus banned))
28+ (plus allowed)
29+ ==
1530++ trimall
16− |= value=@t
31+ |= corpus=@t
1732 |^ ^- tape
18− %+ rash value
33+ %+ rash corpus
1934 %+ ifix [(star ws) (star ws)]
2035 %- star
2136 ;~ less
desk/mar/declarations.hoonnew file+14
@@ -0,0 +1,14 @@
1+/- s=seek
2+/+ j=seek-json
3+|_ d=(list declare:s)
4+++ grad %noun
5+++ grow
6+ |%
7+ ++ noun d
8+ --
9+++ grab
10+ |%
11+ ++ noun (list declare:s)
12+ ++ json declarations:dejs:j
13+ --
14+--
desk/mar/declare.hoonnew file+14
@@ -0,0 +1,14 @@
1+/- s=seek
2+/+ j=seek-json
3+|_ =declare:s
4+++ grad %noun
5+++ grow
6+ |%
7+ ++ noun declare
8+ --
9+++ grab
10+ |%
11+ ++ noun declare:s
12+ ++ json declare:dejs:j
13+ --
14+--
desk/sur/seek.hoon+4−2
@@ -28,14 +28,16 @@
2828 image=@t
2929 ==
3030::
31++$ reach ?(%private %friends %public)
32+::
33++$ declare (pair reach post)
34+::
3135:: a listing includes the content of the post as well as information
3236:: that would affect the hash so are left out, namely time, source and
3337:: hash. each post has a source and time so that they can be updated
3438:: via the source requesting to replace the listing and hash.
3539::
3640::
37−+$ reach ?(%private %friends %public)
38−::
3941+$ listing
4042 $: =post
4143 =hash