Swapped elasticlunr with flexsearch

refs https://github.com/TryGhost/Team/issues/1665

- elasticlunr in an abandoned package with quite a lot of security vulnerabilities. it also has worse performance memory/processing wise comparing to flexsearch (benchmark: https://nextapps-de.github.io/flexsearch/bench/)
- fusejs was another option that was consideres. it was not chosed due to it's poor performance.

ghost/sodo-search/package.json

@@ -21,8 +21,8 @@
     "@testing-library/jest-dom": "5.16.2",
     "@testing-library/react": "12.1.2",
     "@testing-library/user-event": "14.0.0",
-    "@tryghost/content-api": "^1.11.0",
+    "@tryghost/content-api": "1.11.0",
-    "elasticlunr": "0.9.5",
+    "flexsearch": "0.7.21",
     "react": "17.0.2",
     "react-dom": "17.0.2",
     "react-scripts": "4.0.3"

ghost/sodo-search/src/search-index.js

@@ -1,4 +1,4 @@
-import elasticlunr from 'elasticlunr';
+import {Document} from 'flexsearch';
 import GhostContentAPI from '@tryghost/content-api';
 
 export default class SearchIndex {
@@ -9,9 +9,27 @@ export default class SearchIndex {
             version: 'v5.0'
         });
 
-        this.postsIndex = null;
+        this.postsIndex = new Document({
-        this.authorsIndex = null;
+            document: {
-        this.tagsIndex = null;
+                id: 'id',
+                index: ['title', 'excerpt'],
+                store: true
+            }
+        });
+        this.authorsIndex = new Document({
+            document: {
+                id: 'id',
+                index: ['name'],
+                store: true
+            }
+        });
+        this.tagsIndex = new Document({
+            document: {
+                id: 'id',
+                index: ['name'],
+                store: true
+            }
+        });
 
         this.init = this.init.bind(this);
         this.search = this.search.bind(this);
@@ -19,41 +37,23 @@ export default class SearchIndex {
 
     #updatePostIndex(posts) {
         posts.forEach((post) => {
-            this.postsIndex.addDoc({
+            this.postsIndex.add(post);
-                id: post.id,
-                title: post.title,
-                excerpt: post.excerpt,
-                slug: post.slug,
-                url: post.url
-            });
         });
     }
 
     #updateAuthorsIndex(authors) {
         authors.forEach((author) => {
-            this.authorsIndex.addDoc({
+            this.authorsIndex.add(author);
-                id: author.id,
-                name: author.name,
-                url: author.url,
-                profile_image: author.profile_image
-            });
         });
     }
 
     #updateTagsIndex(tags) {
         tags.forEach((tag) => {
-            this.tagsIndex.addDoc({
+            this.tagsIndex.add(tag);
-                id: tag.id,
-                name: tag.name,
-                url: tag.url
-            });
         });
     }
 
     async init() {
-        // remove default stop words to search of *any* word
-        elasticlunr.clearStopWords();
-
         let posts = await this.api.posts.browse({
             limit: 'all',
             fields: 'id,slug,title,excerpt,url,updated_at,visibility',
@@ -61,12 +61,6 @@ export default class SearchIndex {
             formats: 'plaintext'
         });
 
-        this.postsIndex = elasticlunr();
-        this.postsIndex.addField('title');
-        this.postsIndex.addField('url');
-        this.postsIndex.addField('excerpt');
-        this.postsIndex.setRef('id');
-
         if (posts || posts.length > 0) {
             if (!posts.length) {
                 posts = [posts];
@@ -79,12 +73,6 @@ export default class SearchIndex {
             fields: 'id,slug,name,url,profile_image'
         });
 
-        this.authorsIndex = elasticlunr();
-        this.authorsIndex.addField('name');
-        this.authorsIndex.addField('url');
-        this.authorsIndex.addField('profile_image');
-        this.authorsIndex.setRef('id');
-
         if (authors || authors.length > 0) {
             if (!authors.length) {
                 authors = [authors];
@@ -97,10 +85,6 @@ export default class SearchIndex {
             limit: 'all',
             fields: 'id,slug,name,url'
         });
-        this.tagsIndex = elasticlunr();
-        this.tagsIndex.addField('name');
-        this.tagsIndex.addField('url');
-        this.tagsIndex.setRef('id');
 
         if (tags || tags.length > 0) {
             if (!tags.length) {
@@ -111,37 +95,37 @@ export default class SearchIndex {
         }
     }
 
+    #normalizeSearchResult(result) {
+        const normalized = [];
+        const usedIds = {};
+
+        result.forEach((resultItem) => {
+            resultItem.result.forEach((doc) => {
+                if (!usedIds[doc.id]) {
+                    normalized.push(doc.doc);
+                    usedIds[doc.id] = true;
+                }
+            });
+        });
+
+        return normalized;
+    }
+
     search(value) {
         const posts = this.postsIndex.search(value, {
-            fields: {
+            enrich: true
-                title: {boost: 1},
-                excerpt: {boost: 1}
-            },
-            expand: true
         });
         const authors = this.authorsIndex.search(value, {
-            fields: {
+            enrich: true
-                name: {boost: 1}
-            },
-            expand: true
         });
         const tags = this.tagsIndex.search(value, {
-            fields: {
+            enrich: true
-                name: {boost: 1}
-            },
-            expand: true
         });
 
         return {
-            posts: posts.map((doc) => {
+            posts: this.#normalizeSearchResult(posts),
-                return this.postsIndex.documentStore.docs[doc.ref];
+            authors: this.#normalizeSearchResult(authors),
-            }),
+            tags: this.#normalizeSearchResult(tags)
-            authors: authors.map((doc) => {
-                return this.authorsIndex.documentStore.docs[doc.ref];
-            }),
-            tags: tags.map((doc) => {
-                return this.tagsIndex.documentStore.docs[doc.ref];
-            })
         };
     }
 }

ghost/sodo-search/src/search-index.test.js

@@ -43,7 +43,7 @@ describe('search index', function () {
                 posts: [{
                     id: 'sounique',
                     title: 'Awesome Barcelona Life',
-                    excerpt: 'We are sitting by the pool and smashing out search features',
+                    excerpt: 'We are sitting by the pool and smashing out search features. Barcelona life is great!',
                     url: 'http://localhost/ghost/awesome-barcelona-life/'
                 }]
             })

ghost/sodo-search/yarn.lock

@@ -1680,7 +1680,7 @@
   resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-1.1.2.tgz#ccb91445360179a04e7fe6aff78c00ffc1eeaf82"
   integrity sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==
 
-"@tryghost/content-api@^1.11.0":
+"@tryghost/content-api@1.11.0":
   version "1.11.0"
   resolved "https://registry.yarnpkg.com/@tryghost/content-api/-/content-api-1.11.0.tgz#25ab0e2c5618f4ae190d5ad739611a07a551b245"
   integrity sha512-0JTlp5Ln4BfCJzCYuT2X3MC9ZupIkRtzZaHpf9KZw7O8uOsRnO9RwjItN+lwvkoLIesMzfgrZd/tBJ4BAzeBrg==
@@ -4455,11 +4455,6 @@ ejs@^3.1.5:
   dependencies:
     jake "^10.8.5"
 
-elasticlunr@0.9.5:
-  version "0.9.5"
-  resolved "https://registry.yarnpkg.com/elasticlunr/-/elasticlunr-0.9.5.tgz#65541bb309dddd0cf94f2d1c8861b2be651bb0d5"
-  integrity sha512-5YM9LFQgVYfuLNEoqMqVWIBuF2UNCA+xu/jz1TyryLN/wmBcQSb+GNAwvLKvEpGESwgGN8XA1nbLAt6rKlyHYQ==
-
 electron-to-chromium@^1.3.564, electron-to-chromium@^1.4.172:
   version "1.4.177"
   resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.177.tgz#b6a4436eb788ca732556cd69f384b8a3c82118c5"
@@ -5389,6 +5384,11 @@ flatten@^1.0.2:
   resolved "https://registry.yarnpkg.com/flatten/-/flatten-1.0.3.tgz#c1283ac9f27b368abc1e36d1ff7b04501a30356b"
   integrity sha512-dVsPA/UwQ8+2uoFe5GHtiBMu48dWLTdsuEd7CKGlZlD78r1TTWBvDuFaFGKCo/ZfEr95Uk56vZoX86OsHkUeIg==
 
+flexsearch@0.7.21:
+  version "0.7.21"
+  resolved "https://registry.yarnpkg.com/flexsearch/-/flexsearch-0.7.21.tgz#0f5ede3f2aae67ddc351efbe3b24b69d29e9d48b"
+  integrity sha512-W7cHV7Hrwjid6lWmy0IhsWDFQboWSng25U3VVywpHOTJnnAZNPScog67G+cVpeX9f7yDD21ih0WDrMMT+JoaYg==
+
 flush-write-stream@^1.0.0:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/flush-write-stream/-/flush-write-stream-1.1.1.tgz#8dd7d873a1babc207d94ead0c2e0e44276ebf2e8"