add fuzzy search (#32)

* add fuzzy search
* fix debug logger
* clean up

website/components/NixFunctions/nixFunctions.tsx

@@ -1,11 +1,12 @@
 import { Box } from "@mui/system";
-import { useMemo } from "react";
+import { useEffect, useMemo } from "react";
 import { PageState } from "../../models/internals";
-import { byQuery, byType, pipe } from "../../queries";
+import { byType, pipe } from "../../queries";
 import { DocItem } from "../../models/nix";
 import { BasicList, BasicListItem } from "../basicList";
 import FunctionItem from "../functionItem/functionItem";
 import { SetPageStateVariable } from "../pageContext";
+import { useMiniSearch } from "react-minisearch";
 
 interface FunctionsProps {
   pageState: PageState;
@@ -18,10 +19,36 @@ export function NixFunctions(props: FunctionsProps) {
 
   const setSelected = setPageStateVariable<string | null>("selected");
 
-  const filteredData = useMemo(
+  const { search, searchResults, rawResults } = useMiniSearch<DocItem>(data, {
-    () => pipe(byType(filter), byQuery(term))(data),
+    fields: ["id", "name", "category", "description", "example", "fn_type"],
-    [filter, term, data]
+    searchOptions: {
-  );
+      // allow 25% levenshtein distance (e.g. 2.5 of 10 characters don't match)
+      fuzzy: 0.25,
+      // prefer to show builtins first
+      boostDocument: (id, term) => {
+        let boost = 1;
+        boost += id.includes("builtins") ? 1 : 0;
+        boost += id.includes(term) ? 10 : 0;
+        return boost;
+      },
+    },
+    tokenize: (text: string): string[] => {
+      const tokens = text.split(/\W|(?=[A-Z])/);
+      return tokens;
+    },
+  });
+
+  //initial site-load is safe to call
+  useEffect(() => {
+    search(term);
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, []);
+
+  const filteredData = useMemo(() => {
+    const dataForFilter = term ? searchResults || [] : data;
+    console.debug({ rawResults });
+    return pipe(byType(filter))(dataForFilter);
+  }, [filter, term, rawResults, searchResults, data]);
 
   const preRenderedItems: BasicListItem[] = filteredData.map(
     (docItem: DocItem) => {
@@ -50,7 +77,7 @@ export function NixFunctions(props: FunctionsProps) {
 
   return (
     <Box sx={{ ml: { xs: 0, md: 2 } }}>
-      <BasicList items={preRenderedItems} />
+      <BasicList items={preRenderedItems} search={search} />
     </Box>
   );
 }

website/components/basicList/basicList.tsx

@@ -8,6 +8,7 @@ import { usePageContext } from "../pageContext";
 import { useMobile } from "../layout/layout";
 import { EmptyRecordsPlaceholder } from "../emptyRecordsPlaceholder";
 import { FunctionOfTheDay } from "../functionOfTheDay";
+import { Query, SearchOptions } from "minisearch";
 
 export type BasicListItem = {
   item: React.ReactNode;
@@ -15,12 +16,13 @@ export type BasicListItem = {
 };
 export type BasicListProps = BasicDataViewProps & {
   selected?: string | null;
+  search: (query: Query, options?: SearchOptions) => void;
 };
 
 type ViewMode = "explore" | "browse";
 
 export function BasicList(props: BasicListProps) {
-  const { items } = props;
+  const { items, search } = props;
   const { pageState, setPageStateVariable, resetQueries } = usePageContext();
   const isMobile = useMobile();
   const { page, itemsPerPage, filter, term, FOTD, data } = pageState;
@@ -60,6 +62,16 @@ export function BasicList(props: BasicListProps) {
 
   const handleSearch = (term: string) => {
     setTerm(term);
+    search(term, {
+      boost: {
+        id: 10,
+        name: 8,
+        category: 6,
+        example: 0.5,
+        fn_type: 3,
+        description: 1,
+      },
+    });
     setPage(1);
   };
 

website/package-lock.json

@@ -1,11 +1,11 @@
 {
-  "name": "nextapp",
+  "name": "noogle",
   "version": "0.1.0",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
-      "name": "nextapp",
+      "name": "noogle",
       "version": "0.1.0",
       "dependencies": {
         "@emotion/react": "^11.10.5",
@@ -19,12 +19,14 @@
         "eslint": "8.27.0",
         "eslint-config-next": "13.0.2",
         "highlight.js": "^11.7.0",
+        "minisearch": "^6.0.1",
         "next": "13.0.2",
         "notistack": "^2.0.8",
         "react": "18.2.0",
         "react-dom": "18.2.0",
         "react-highlight": "^0.15.0",
         "react-markdown": "^8.0.4",
+        "react-minisearch": "^6.0.2",
         "rehype-highlight": "^6.0.0",
         "seedrandom": "^3.0.5",
         "typescript": "4.8.4",
@@ -3832,6 +3834,11 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/minisearch": {
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/minisearch/-/minisearch-6.0.1.tgz",
+      "integrity": "sha512-Ly1w0nHKnlhAAh6/BF/+9NgzXfoJxaJ8nhopFhQ3NcvFJrFIL+iCg9gw9e9UMBD+XIsp/RyznJ/o5UIe5Kw+kg=="
+    },
     "node_modules/mri": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/mri/-/mri-1.2.0.tgz",
@@ -4334,6 +4341,15 @@
       "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.2.0.tgz",
       "integrity": "sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w=="
     },
+    "node_modules/react-minisearch": {
+      "version": "6.0.2",
+      "resolved": "https://registry.npmjs.org/react-minisearch/-/react-minisearch-6.0.2.tgz",
+      "integrity": "sha512-oBhLz2XEdQeneYCYaFCwtoZJbc5KqBQkWXp9O6l3O/woRxSWGcefYctzB9+LtQTiVNEMgo3FEvCnRFPFeVKXfg==",
+      "peerDependencies": {
+        "minisearch": "^6.0.0",
+        "react": "^16.8.0 || ^17.0.0 || ^18.0.0"
+      }
+    },
     "node_modules/react-transition-group": {
       "version": "4.4.5",
       "resolved": "https://registry.npmjs.org/react-transition-group/-/react-transition-group-4.4.5.tgz",
@@ -7693,6 +7709,11 @@
       "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.7.tgz",
       "integrity": "sha512-bzfL1YUZsP41gmu/qjrEk0Q6i2ix/cVeAhbCbqH9u3zYutS1cLg00qhrD0M2MVdCcx4Sc0UpP2eBWo9rotpq6g=="
     },
+    "minisearch": {
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/minisearch/-/minisearch-6.0.1.tgz",
+      "integrity": "sha512-Ly1w0nHKnlhAAh6/BF/+9NgzXfoJxaJ8nhopFhQ3NcvFJrFIL+iCg9gw9e9UMBD+XIsp/RyznJ/o5UIe5Kw+kg=="
+    },
     "mri": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/mri/-/mri-1.2.0.tgz",
@@ -8016,6 +8037,12 @@
         }
       }
     },
+    "react-minisearch": {
+      "version": "6.0.2",
+      "resolved": "https://registry.npmjs.org/react-minisearch/-/react-minisearch-6.0.2.tgz",
+      "integrity": "sha512-oBhLz2XEdQeneYCYaFCwtoZJbc5KqBQkWXp9O6l3O/woRxSWGcefYctzB9+LtQTiVNEMgo3FEvCnRFPFeVKXfg==",
+      "requires": {}
+    },
     "react-transition-group": {
       "version": "4.4.5",
       "resolved": "https://registry.npmjs.org/react-transition-group/-/react-transition-group-4.4.5.tgz",

website/package.json

@@ -21,12 +21,14 @@
     "eslint": "8.27.0",
     "eslint-config-next": "13.0.2",
     "highlight.js": "^11.7.0",
+    "minisearch": "^6.0.1",
     "next": "13.0.2",
     "notistack": "^2.0.8",
     "react": "18.2.0",
     "react-dom": "18.2.0",
     "react-highlight": "^0.15.0",
     "react-markdown": "^8.0.4",
+    "react-minisearch": "^6.0.2",
     "rehype-highlight": "^6.0.0",
     "seedrandom": "^3.0.5",
     "typescript": "4.8.4",

website/pages/index.tsx

@@ -49,7 +49,7 @@ export default function FunctionsPage() {
     if (router.isReady && initialProps === null) {
       getInitialProps(router).then((r) => {
         const { props } = r;
-        console.info("Url Query changed\n\nUpdating pageState with delta:", {
+        console.debug("Url Query changed\n\nUpdating pageState with delta:", {
           props,
         });
         setInitialProps((curr) => ({ ...curr, ...props }));

website/queries/byQuery.ts

@@ -1,4 +1,9 @@
 import { DocItem, MetaData } from "../models/nix";
+// import MiniSearch from 'minisearch'
+
+// export const byMinisearch = (term: string, miniSearch: MiniSearch<DocItem> ) => (data: MetaData): MetaData => {
+//    return miniSearch.search(term);
+// }
 
 export const byQuery =
 (rawTerm: string) =>