add fuzzy search (#32)

* add fuzzy search * fix debug logger * clean up
2024-10-26 22:11:56 +03:00 · 2023-04-27 17:11:31 +02:00 · 2023-04-27 17:11:31 +02:00 · c707bd74a7
commit c707bd74a7
parent db000540b1
6 changed files with 84 additions and 11 deletions
--- a/website/components/NixFunctions/nixFunctions.tsx
+++ b/website/components/NixFunctions/nixFunctions.tsx
@ -1,11 +1,12 @@
 import { Box } from "@mui/system";
-import { useMemo } from "react";
+import { useEffect, useMemo } from "react";
 import { PageState } from "../../models/internals";
-import { byQuery, byType, pipe } from "../../queries";
+import { byType, pipe } from "../../queries";
 import { DocItem } from "../../models/nix";
 import { BasicList, BasicListItem } from "../basicList";
 import FunctionItem from "../functionItem/functionItem";
 import { SetPageStateVariable } from "../pageContext";
+import { useMiniSearch } from "react-minisearch";

 interface FunctionsProps {
  pageState: PageState;
@ -18,10 +19,36 @@ export function NixFunctions(props: FunctionsProps) {

  const setSelected = setPageStateVariable<string | null>("selected");

-  const filteredData = useMemo(
-    () => pipe(byType(filter), byQuery(term))(data),
-    [filter, term, data]
-  );
+  const { search, searchResults, rawResults } = useMiniSearch<DocItem>(data, {
+    fields: ["id", "name", "category", "description", "example", "fn_type"],
+    searchOptions: {
+      // allow 25% levenshtein distance (e.g. 2.5 of 10 characters don't match)
+      fuzzy: 0.25,
+      // prefer to show builtins first
+      boostDocument: (id, term) => {
+        let boost = 1;
+        boost += id.includes("builtins") ? 1 : 0;
+        boost += id.includes(term) ? 10 : 0;
+        return boost;
+      },
+    },
+    tokenize: (text: string): string[] => {
+      const tokens = text.split(/\W|(?=[A-Z])/);
+      return tokens;
+    },
+  });
+
+  //initial site-load is safe to call
+  useEffect(() => {
+    search(term);
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, []);
+
+  const filteredData = useMemo(() => {
+    const dataForFilter = term ? searchResults || [] : data;
+    console.debug({ rawResults });
+    return pipe(byType(filter))(dataForFilter);
+  }, [filter, term, rawResults, searchResults, data]);

  const preRenderedItems: BasicListItem[] = filteredData.map(
    (docItem: DocItem) => {
@ -50,7 +77,7 @@ export function NixFunctions(props: FunctionsProps) {

  return (
    <Box sx={{ ml: { xs: 0, md: 2 } }}>
-      <BasicList items={preRenderedItems} />
+      <BasicList items={preRenderedItems} search={search} />
    </Box>
  );
 }
--- a/website/components/basicList/basicList.tsx
+++ b/website/components/basicList/basicList.tsx
@ -8,6 +8,7 @@ import { usePageContext } from "../pageContext";
 import { useMobile } from "../layout/layout";
 import { EmptyRecordsPlaceholder } from "../emptyRecordsPlaceholder";
 import { FunctionOfTheDay } from "../functionOfTheDay";
+import { Query, SearchOptions } from "minisearch";

 export type BasicListItem = {
  item: React.ReactNode;
@ -15,12 +16,13 @@ export type BasicListItem = {
 };
 export type BasicListProps = BasicDataViewProps & {
  selected?: string | null;
+  search: (query: Query, options?: SearchOptions) => void;
 };

 type ViewMode = "explore" | "browse";

 export function BasicList(props: BasicListProps) {
-  const { items } = props;
+  const { items, search } = props;
  const { pageState, setPageStateVariable, resetQueries } = usePageContext();
  const isMobile = useMobile();
  const { page, itemsPerPage, filter, term, FOTD, data } = pageState;
@ -60,6 +62,16 @@ export function BasicList(props: BasicListProps) {

  const handleSearch = (term: string) => {
    setTerm(term);
+    search(term, {
+      boost: {
+        id: 10,
+        name: 8,
+        category: 6,
+        example: 0.5,
+        fn_type: 3,
+        description: 1,
+      },
+    });
    setPage(1);
  };

--- a/website/package-lock.json
+++ b/website/package-lock.json
@ -1,11 +1,11 @@
 {
-  "name": "nextapp",
+  "name": "noogle",
  "version": "0.1.0",
  "lockfileVersion": 2,
  "requires": true,
  "packages": {
    "": {
-      "name": "nextapp",
+      "name": "noogle",
      "version": "0.1.0",
      "dependencies": {
        "@emotion/react": "^11.10.5",
@ -19,12 +19,14 @@
        "eslint": "8.27.0",
        "eslint-config-next": "13.0.2",
        "highlight.js": "^11.7.0",
+        "minisearch": "^6.0.1",
        "next": "13.0.2",
        "notistack": "^2.0.8",
        "react": "18.2.0",
        "react-dom": "18.2.0",
        "react-highlight": "^0.15.0",
        "react-markdown": "^8.0.4",
+        "react-minisearch": "^6.0.2",
        "rehype-highlight": "^6.0.0",
        "seedrandom": "^3.0.5",
        "typescript": "4.8.4",
@ -3832,6 +3834,11 @@
        "url": "https://github.com/sponsors/ljharb"
      }
    },
+    "node_modules/minisearch": {
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/minisearch/-/minisearch-6.0.1.tgz",
+      "integrity": "sha512-Ly1w0nHKnlhAAh6/BF/+9NgzXfoJxaJ8nhopFhQ3NcvFJrFIL+iCg9gw9e9UMBD+XIsp/RyznJ/o5UIe5Kw+kg=="
+    },
    "node_modules/mri": {
      "version": "1.2.0",
      "resolved": "https://registry.npmjs.org/mri/-/mri-1.2.0.tgz",
@ -4334,6 +4341,15 @@
      "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.2.0.tgz",
      "integrity": "sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w=="
    },
+    "node_modules/react-minisearch": {
+      "version": "6.0.2",
+      "resolved": "https://registry.npmjs.org/react-minisearch/-/react-minisearch-6.0.2.tgz",
+      "integrity": "sha512-oBhLz2XEdQeneYCYaFCwtoZJbc5KqBQkWXp9O6l3O/woRxSWGcefYctzB9+LtQTiVNEMgo3FEvCnRFPFeVKXfg==",
+      "peerDependencies": {
+        "minisearch": "^6.0.0",
+        "react": "^16.8.0 || ^17.0.0 || ^18.0.0"
+      }
+    },
    "node_modules/react-transition-group": {
      "version": "4.4.5",
      "resolved": "https://registry.npmjs.org/react-transition-group/-/react-transition-group-4.4.5.tgz",
@ -7693,6 +7709,11 @@
      "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.7.tgz",
      "integrity": "sha512-bzfL1YUZsP41gmu/qjrEk0Q6i2ix/cVeAhbCbqH9u3zYutS1cLg00qhrD0M2MVdCcx4Sc0UpP2eBWo9rotpq6g=="
    },
+    "minisearch": {
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/minisearch/-/minisearch-6.0.1.tgz",
+      "integrity": "sha512-Ly1w0nHKnlhAAh6/BF/+9NgzXfoJxaJ8nhopFhQ3NcvFJrFIL+iCg9gw9e9UMBD+XIsp/RyznJ/o5UIe5Kw+kg=="
+    },
    "mri": {
      "version": "1.2.0",
      "resolved": "https://registry.npmjs.org/mri/-/mri-1.2.0.tgz",
@ -8016,6 +8037,12 @@
        }
      }
    },
+    "react-minisearch": {
+      "version": "6.0.2",
+      "resolved": "https://registry.npmjs.org/react-minisearch/-/react-minisearch-6.0.2.tgz",
+      "integrity": "sha512-oBhLz2XEdQeneYCYaFCwtoZJbc5KqBQkWXp9O6l3O/woRxSWGcefYctzB9+LtQTiVNEMgo3FEvCnRFPFeVKXfg==",
+      "requires": {}
+    },
    "react-transition-group": {
      "version": "4.4.5",
      "resolved": "https://registry.npmjs.org/react-transition-group/-/react-transition-group-4.4.5.tgz",
--- a/website/package.json
+++ b/website/package.json
@ -21,12 +21,14 @@
    "eslint": "8.27.0",
    "eslint-config-next": "13.0.2",
    "highlight.js": "^11.7.0",
+    "minisearch": "^6.0.1",
    "next": "13.0.2",
    "notistack": "^2.0.8",
    "react": "18.2.0",
    "react-dom": "18.2.0",
    "react-highlight": "^0.15.0",
    "react-markdown": "^8.0.4",
+    "react-minisearch": "^6.0.2",
    "rehype-highlight": "^6.0.0",
    "seedrandom": "^3.0.5",
    "typescript": "4.8.4",
--- a/website/pages/index.tsx
+++ b/website/pages/index.tsx
@ -49,7 +49,7 @@ export default function FunctionsPage() {
    if (router.isReady && initialProps === null) {
      getInitialProps(router).then((r) => {
        const { props } = r;
-        console.info("Url Query changed\n\nUpdating pageState with delta:", {
+        console.debug("Url Query changed\n\nUpdating pageState with delta:", {
          props,
        });
        setInitialProps((curr) => ({ ...curr, ...props }));
--- a/website/queries/byQuery.ts
+++ b/website/queries/byQuery.ts
@ -1,4 +1,9 @@
 import { DocItem, MetaData } from "../models/nix";
+// import MiniSearch from 'minisearch'
+
+// export const byMinisearch = (term: string, miniSearch: MiniSearch<DocItem> ) => (data: MetaData): MetaData => {
+//    return miniSearch.search(term);
+// }

 export const byQuery =
 (rawTerm: string) =>