on master help sign in
Merge pull request #1850 from urbit/mp/redo-glossary

build: generate glossary on build

commitd7fa942559ffc3ffd14b2cb1f5483e954a26b3b2 authormatildepark <matilde.park@urbit.org> dateWed 19 Oct 2022, 13:23 (4 years ago) parentec5c661aa9cd0e
6 files changed+288−241
components/Search.js+216−228 lib/buildGlossary.js+59 package.json+2−1 pages/api/dev-search.js+1−1 pages/api/glossary.js−10 pages/api/search.js+10−1
components/Search.js+216−228
@@ -14,26 +14,21 @@ class Search extends Component {
1414 this.searchEndpoint = this.searchEndpoint.bind(this);
1515 this.onInputValueChange = this.onInputValueChange.bind(this);
1616 this.onSelect = this.onSelect.bind(this);
17− this.glossarySearch = this.glossarySearch.bind(this);
1817 this.patpSearch = this.patpSearch.bind(this);
1918 this.devSearch = this.devSearch.bind(this);
2019 this.opsSearch = this.opsSearch.bind(this);
2120 }
2221
2322 searchEndpoint(query) {
24− return `/api/search?q=${query}`;
23+ return `/api/search?q=${encodeURIComponent(query)}`;
2524 }
2625
2726 devSearch(query) {
28− return `/api/dev-search?q=${query}`;
29− }
30−
31− glossarySearch(query) {
32− return `/api/glossary?q=${encodeURIComponent(query)}`;
27+ return `/api/dev-search?q=${encodeURIComponent(query)}`;
3328 }
3429
3530 opsSearch(query) {
36− return `/api/ops-search?q=${query}`;
31+ return `/api/ops-search?q=${encodeURIComponent(query)}`;
3732 }
3833
3934 patpSearch(query) {
@@ -45,16 +40,19 @@ class Search extends Component {
4540 }
4641
4742 onSelect(item) {
48− if (item.slug) {
49− this.props.router.push(item.slug);
50− }
51−
5243 this.setState({
5344 query: "",
5445 results: [],
5546 });
5647
57− this.props.closeSearch();
48+ if (item.url) {
49+ this.props.router.push(`${item.url}${item?.slug ? item.slug : ""}`);
50+ return this.props.closeSearch();
51+ }
52+ if (item.slug) {
53+ this.props.router.push(item.slug);
54+ return this.props.closeSearch();
55+ }
5856 }
5957
6058 onInputValueChange = debounce(async (query) => {
@@ -62,10 +60,13 @@ class Search extends Component {
6260 const search = fetch(this.searchEndpoint(query))
6361 .then((res) => res.json())
6462 .then(async (res) => {
65− return res.results.map((item) => ({
63+ return [...res.glossary.map((item) => ({
64+ type: "GLOSSARY_RESULT",
65+ content: item
66+ })), ...res.results.map((item) => ({
6667 type: "RESULT",
6768 content: item,
68− }));
69+ }))];
6970 });
7071
7172 const patp = this.patpSearch(query)
@@ -76,53 +77,54 @@ class Search extends Component {
7677
7778 const patpResult = this.patpSearch(query)
7879 ? [
79− {
80− type: "PATP",
81− content: {
82− patp: patp,
83− slug: `/ids/${patp}`,
84− },
80+ {
81+ type: "PATP",
82+ content: {
83+ patp: patp,
84+ slug: `/ids/${patp}`,
8585 },
86− ]
86+ },
87+ ]
8788 : [];
8889
89− const glossarySearch = fetch(this.glossarySearch(query))
90− .then((res) => res.json())
91− .then((res) => {
92− return res.results.map((item) => ({
93− type: "GLOSSARY_RESULT",
94− content: item,
95− }));
96− });
97−
9890 const devSearch = fetch(this.devSearch(query))
9991 .then((res) => res.json())
10092 .then((res) => {
101− return res.results.map((item) => ({
93+ return [...res.glossary.map((item) => ({
94+ type: "GLOSSARY_RESULT",
95+ content: item
96+ })), ...res.results.map((item) => ({
10297 type: "DEV_RESULT",
10398 content: item,
104− }));
99+ }))];
105100 });
106101
107102 const opsSearch = fetch(this.opsSearch(query))
108103 .then((res) => res.json())
109104 .then((res) => {
110− return res.results.map((item) => ({
105+ return [...res.glossary.map((item) => ({
106+ type: "GLOSSARY_RESULT",
107+ content: item
108+ })), ...res.results.map((item) => ({
111109 type: "OPS_RESULT",
112110 content: item,
113− }));
111+ }))];
114112 });
115113
116− const [results, glossaryResults, devResults, opsResults] =
117− await Promise.all([search, glossarySearch, devSearch, opsSearch]);
114+ const [results, devResults, opsResults] =
115+ await Promise.all([search, devSearch, opsSearch]);
118116
119117 const list = [
120− ...glossaryResults,
121118 ...patpResult,
122119 ...results,
123120 ...devResults,
124121 ...opsResults,
125− ];
122+ ]
123+ .sort((a, b) => {
124+ const aNum = a?.type === "GLOSSARY_RESULT" ? 1 : -1;
125+ const bNum = b?.type === "GLOSSARY_RESULT" ? 1 : -1;
126+ return bNum - aNum;
127+ });;
126128
127129 this.setState({ results: list });
128130 } else {
@@ -182,209 +184,195 @@ class Search extends Component {
182184 <ul {...getMenuProps()} className="overflow-y-scroll">
183185 {isOpen
184186 ? state.results.map((item, index) => {
185− const selected = highlightedIndex === index;
186− if (item.type === "PATP") {
187− return (
188− <li
189− className={`cursor-pointer p-2 flex space-x-2 items-center text-left w-full ${
190− selected ? "bg-green-400" : ""
187+ const selected = highlightedIndex === index;
188+ if (item.type === "PATP") {
189+ return (
190+ <li
191+ className={`cursor-pointer p-2 flex space-x-2 items-center text-left w-full ${selected ? "bg-green-400" : ""
191192 }`}
192− {...getItemProps({
193− key: item.content.slug + "-" + index,
194− index,
195− item: item.content,
196− selected: highlightedIndex === index,
197− })}
198− >
199− <div className="rounded-md overflow-hidden">
200− <Sigil
201− patp={item.content.patp}
202− size={25}
203− icon
204− />
205− </div>
206− <p className="font-mono">{item.content.patp}</p>
207− </li>
208− );
209− }
210− if (item.type === "GLOSSARY_RESULT") {
211− return (
212− <li
213− className={`cursor-pointer flex text-left w-full ${
214− selected ? "bg-green-400" : ""
193+ {...getItemProps({
194+ key: item.content.slug + "-" + index,
195+ index,
196+ item: item.content,
197+ selected: highlightedIndex === index,
198+ })}
199+ >
200+ <div className="rounded-md overflow-hidden">
201+ <Sigil
202+ patp={item.content.patp}
203+ size={25}
204+ icon
205+ />
206+ </div>
207+ <p className="font-mono">{item.content.patp}</p>
208+ </li>
209+ );
210+ }
211+ if (item.type === "GLOSSARY_RESULT") {
212+ return (
213+ <li
214+ className={`cursor-pointer flex text-left w-full ${selected ? "bg-green-400" : ""
215215 }`}
216− {...getItemProps({
217− key: item.content.slug + "-" + index,
218− index,
219− item: item.content,
220− selected: highlightedIndex === index,
221− })}
222− >
223− <div className="font-semibold p-3">
224− <p
225− className={`text-base ${
226− selected ? "text-white" : "text-wall-600"
216+ {...getItemProps({
217+ key: item.content.slug + "-" + index,
218+ index,
219+ item: item.content,
220+ selected: highlightedIndex === index,
221+ })}
222+ >
223+ <div className="font-semibold p-3">
224+ <p
225+ className={`text-base ${selected ? "text-white" : "text-wall-600"
227226 }`}
228− >
229− {item.content.symbol.length > 0 && (
230− <code
231− className={`mr-1 rounded px-1 py-0.5 ${
232− selected
233− ? "bg-washedWhite"
234− : "bg-wall-100"
227+ >
228+ {item.content.symbol.length > 0 && (
229+ <code
230+ className={`mr-1 rounded px-1 py-0.5 ${selected
231+ ? "bg-washedWhite"
232+ : "bg-wall-100"
235233 }`}
236− >
237− {item.content.symbol}
238− </code>
239− )}
240− {item.content.name}
241− </p>
242− <p
243− className={`font-normal text-base mt-1 ${
244− selected ? "text-white" : "text-wall-600"
234+ >
235+ {item.content.symbol}
236+ </code>
237+ )}
238+ {item.content.name}
239+ </p>
240+ <p
241+ className={`font-normal text-base mt-1 ${selected ? "text-white" : "text-wall-600"
245242 }`}
246− dangerouslySetInnerHTML={{
247− __html: item.content.desc,
248− }}
249− ></p>
250− </div>
251− </li>
252− );
253− }
254− if (item.type === "RESULT") {
255− return (
256− <li
257− className={`cursor-pointer flex text-left w-full ${
258− selected ? "bg-green-400" : ""
243+ dangerouslySetInnerHTML={{
244+ __html: item.content.desc,
245+ }}
246+ ></p>
247+ </div>
248+ </li>
249+ );
250+ }
251+ if (item.type === "RESULT") {
252+ return (
253+ <li
254+ className={`cursor-pointer flex text-left w-full ${selected ? "bg-green-400" : ""
259255 }`}
260− {...getItemProps({
261− key: item.content.link + "-" + index,
262− index,
263− item: item.content,
264− selected,
265− })}
266− >
267− <div className="p-3">
268− <p
269− className={`font-medium text-base ${
270− selected ? "text-white" : "text-wall-600"
256+ {...getItemProps({
257+ key: item.content.link + "-" + index,
258+ index,
259+ item: item.content,
260+ selected,
261+ })}
262+ >
263+ <div className="p-3">
264+ <p
265+ className={`font-medium text-base ${selected ? "text-white" : "text-wall-600"
271266 }`}
272− >
273− {item.content.parent !== "Content"
274− ? `${item.content.parent} /`
275− : ""}{" "}
276− {item.content.title}
277− </p>
278− <p
279− className={`text-base font-regular text-small ${
280− selected ? "text-midWhite" : "text-wall-500"
267+ >
268+ {item.content.parent !== "Content"
269+ ? `${item.content.parent} /`
270+ : ""}{" "}
271+ {item.content.title}
272+ </p>
273+ <p
274+ className={`text-base font-regular text-small ${selected ? "text-midWhite" : "text-wall-500"
281275 }`}
282− >
283− {item.content.content}
284− {item?.content?.foundOnPage && (
285− <span className="italic block">
286− Found in page content
287− </span>
288− )}
289− </p>
290− </div>
291− </li>
292− );
293− }
294− if (item.type === "DEV_RESULT") {
295− const devItem = Object.assign({}, item.content);
296− devItem[
297− "slug"
298− ] = `https://developers.urbit.org${item.content.slug}`;
299− return (
300− <li
301− className={`cursor-pointer flex text-left w-full ${
302− selected ? "bg-green-400" : ""
276+ >
277+ {item.content.content}
278+ {item?.content?.foundOnPage && (
279+ <span className="italic block">
280+ Found in page content
281+ </span>
282+ )}
283+ </p>
284+ </div>
285+ </li>
286+ );
287+ }
288+ if (item.type === "DEV_RESULT") {
289+ const devItem = Object.assign({}, item.content);
290+ devItem[
291+ "slug"
292+ ] = `https://developers.urbit.org${item.content.slug}`;
293+ return (
294+ <li
295+ className={`cursor-pointer flex text-left w-full ${selected ? "bg-green-400" : ""
303296 }`}
304− {...getItemProps({
305− key: item.content.link + "-" + index,
306− index,
307− item: devItem,
308− selected,
309− })}
310− >
311− <div className="p-3">
312− <p
313− className={`font-medium text-base ${
314− selected ? "text-white" : "text-wall-600"
297+ {...getItemProps({
298+ key: item.content.link + "-" + index,
299+ index,
300+ item: devItem,
301+ selected,
302+ })}
303+ >
304+ <div className="p-3">
305+ <p
306+ className={`font-medium text-base ${selected ? "text-white" : "text-wall-600"
315307 }`}
316− >
317− <span className="text-wall-400">
318− {item.content.parent !== "Content"
319− ? `developers.urbit.org / ${item.content.parent} /`
320− : "developers.urbit.org /"}{" "}
321− </span>
322− {item.content.title}
323− </p>
324− <p
325− className={`text-base font-regular text-small ${
326− selected ? "text-midWhite" : "text-wall-500"
308+ >
309+ <span className="text-wall-400">
310+ {item.content.parent !== "Content"
311+ ? `developers.urbit.org / ${item.content.parent} /`
312+ : "developers.urbit.org /"}{" "}
313+ </span>
314+ {item.content.title}
315+ </p>
316+ <p
317+ className={`text-base font-regular text-small ${selected ? "text-midWhite" : "text-wall-500"
327318 }`}
328− >
329− {item.content.content}
330− {item?.content?.foundOnPage && (
331− <span className="italic block">
332− Found in page content
333− </span>
334− )}
335− </p>
336− </div>
337− </li>
338− );
339− }
340− if (item.type === "OPS_RESULT") {
341− const opsItem = Object.assign({}, item.content);
342− opsItem[
343− "slug"
344− ] = `https://operators.urbit.org${item.content.slug}`;
345− return (
346− <li
347− className={`cursor-pointer flex text-left w-full ${
348− selected ? "bg-green-400" : ""
319+ >
320+ {item.content.content}
321+ {item?.content?.foundOnPage && (
322+ <span className="italic block">
323+ Found in page content
324+ </span>
325+ )}
326+ </p>
327+ </div>
328+ </li>
329+ );
330+ }
331+ if (item.type === "OPS_RESULT") {
332+ const opsItem = Object.assign({}, item.content);
333+ opsItem[
334+ "slug"
335+ ] = `https://operators.urbit.org${item.content.slug}`;
336+ return (
337+ <li
338+ className={`cursor-pointer flex text-left w-full ${selected ? "bg-green-400" : ""
349339 }`}
350− {...getItemProps({
351− key: item.content.link + "-" + index,
352− index,
353− item: opsItem,
354− selected,
355− })}
356− >
357− <div className="p-3">
358− <p
359− className={`font-medium text-base ${
360− selected ? "text-white" : "text-wall-600"
340+ {...getItemProps({
341+ key: item.content.link + "-" + index,
342+ index,
343+ item: opsItem,
344+ selected,
345+ })}
346+ >
347+ <div className="p-3">
348+ <p
349+ className={`font-medium text-base ${selected ? "text-white" : "text-wall-600"
361350 }`}
362− >
363− <span className="text-wall-400">
364− {item.content.parent !== "Content"
365− ? `operators.urbit.org / ${item.content.parent} /`
366− : "operators.urbit.org /"}{" "}
367− </span>
368− {item.content.title}
369− </p>
370− <p
371− className={`text-base font-regular text-small ${
372− selected ? "text-midWhite" : "text-wall-500"
351+ >
352+ <span className="text-wall-400">
353+ {item.content.parent !== "Content"
354+ ? `operators.urbit.org / ${item.content.parent} /`
355+ : "operators.urbit.org /"}{" "}
356+ </span>
357+ {item.content.title}
358+ </p>
359+ <p
360+ className={`text-base font-regular text-small ${selected ? "text-midWhite" : "text-wall-500"
373361 }`}
374− >
375− {item.content.content}
376− {item?.content?.foundOnPage && (
377− <span className="italic block">
378− Found in page content
379− </span>
380− )}
381− </p>
382− </div>
383− </li>
384− );
385− }
386− return null;
387− })
362+ >
363+ {item.content.content}
364+ {item?.content?.foundOnPage && (
365+ <span className="italic block">
366+ Found in page content
367+ </span>
368+ )}
369+ </p>
370+ </div>
371+ </li>
372+ );
373+ }
374+ return null;
375+ })
388376 : null}
389377 </ul>
390378 </div>
lib/buildGlossary.jsnew file+59
@@ -0,0 +1,59 @@
1+const fs = require("fs");
2+const path = require("path");
3+const matter = require("gray-matter");
4+const toml = require("@iarna/toml");
5+
6+const options = {
7+ engines: {
8+ toml: toml.parse.bind(toml),
9+ },
10+ language: "toml",
11+ delimiters: "+++",
12+};
13+
14+const glossary = [];
15+
16+function buildGlossary(dir) {
17+ const children = fs.readdirSync(dir, { withFileTypes: true });
18+ const pages = children.filter((f) => f.isFile() && f.name !== "_index.md");
19+ const subdirs = children.filter(
20+ (f) => f.isDirectory()
21+ );
22+ pages.map((page) => {
23+ const { data } = matter(fs.readFileSync(path.join(dir, page.name)), options);
24+ const glossaryEntry = data.glossaryEntry
25+ ? Object.values(data.glossaryEntry)
26+ .map((e) => ({
27+ ...e, ...{
28+ url: "https://urbit.org" + path.join(
29+ dir.substr(dir.indexOf("content") + 7),
30+ "/",
31+ page.name.replace(/.md$/, "")
32+ )
33+ }
34+ }))
35+ : [];
36+ glossaryEntry.forEach((entry) => glossary.push(entry));
37+ });
38+
39+ Object.fromEntries(
40+ subdirs.map((subdir) => [
41+ subdir.name,
42+ buildGlossary(path.join(dir, subdir.name)),
43+ ])
44+ );
45+}
46+
47+buildGlossary(path.join(process.cwd(), "content"));
48+const fileContents = `export const glossary = ${JSON.stringify(glossary)}`;
49+
50+try {
51+ fs.readdirSync("cache");
52+} catch (err) {
53+ fs.mkdirSync("cache");
54+}
55+
56+fs.writeFile("cache/glossary.js", fileContents, function (err) {
57+ if (err) return console.error(err);
58+ console.log("Site glossary created.");
59+});
package.json+2−1
@@ -7,11 +7,12 @@
77 "build": "next build",
88 "export": "npm run prebuild && next build && next export",
99 "start": "next start",
10− "prebuild": "npm run cache-posts && npm run rss && npm run community-tree && npm run getting-started-tree && npm run overview-tree",
10+ "prebuild": "npm run cache-posts && npm run rss && npm run glossary && npm run community-tree && npm run getting-started-tree && npm run overview-tree",
1111 "community-tree": "node lib/buildPageTree community weight",
1212 "getting-started-tree": "node lib/buildPageTree getting-started weight",
1313 "overview-tree": "node lib/buildPageTree overview weight",
1414 "cache-posts": "node lib/cache.js",
15+ "glossary": "node lib/buildGlossary.js",
1516 "rss": "node lib/rss.js",
1617 "vercel-build": "yum install libuuid-devel libmount-devel && cp /lib64/{libuuid,libmount,libblkid}.so.1 node_modules/canvas/build/Release/ && npm run build"
1718 },
pages/api/dev-search.js+1−1
@@ -1,6 +1,6 @@
11export default async (req, res) => {
22 const results = await fetch(
3− `https://developers.urbit.org/api/search?q=${req.query.q}`
3+ `https://developers.urbit.org/api/search?q=${encodeURIComponent(req.query.q)}`
44 ).then((res) => res.json());
55 res.statusCode = 200;
66 res.setHeader("Content-Type", "application/json");
pages/api/glossary.jsdeleted−10
@@ -1,10 +0,0 @@
1−export default async (req, res) => {
2− const results = await fetch(
3− `https://developers.urbit.org/api/glossary?q=${encodeURIComponent(
4− req.query.q
5− )}`
6− ).then((res) => res.json());
7− res.statusCode = 200;
8− res.setHeader("Content-Type", "application/json");
9− res.end(JSON.stringify(results));
10−};
pages/api/search.js+10−1
@@ -1,4 +1,5 @@
11import { index } from "../../cache/data";
2+import { glossary } from "../../cache/glossary";
23const levenSort = require("leven-sort");
34
45export default (req, res) => {
@@ -9,6 +10,13 @@ export default (req, res) => {
910 e?.parent.toLowerCase().includes(req.query.q.toLowerCase())
1011 );
1112
13+ const entries = glossary.filter((entry) => {
14+ return (
15+ entry.name.includes(req.query.q.toLowerCase()) ||
16+ entry.symbol.includes(req.query.q)
17+ );
18+ });
19+
1220 results.push(
1321 ...index
1422 .filter((e) => !results.includes(e))
@@ -30,7 +38,8 @@ export default (req, res) => {
3038 "parent",
3139 "headings",
3240 ]);
41+ const sortedEntries = levenSort(entries, req.query.q, ["symbol"]);
3342 res.statusCode = 200;
3443 res.setHeader("Content-Type", "application/json");
35− res.end(JSON.stringify({ results: sorted }));
44+ res.end(JSON.stringify({ results: sorted, glossary: sortedEntries }));
3645};