From df3d6cee748d762ca7e787217fd137eb8e87e7ff Mon Sep 17 00:00:00 2001 From: Naz Date: Thu, 7 Jul 2022 12:52:01 +0200 Subject: [PATCH] Swapped elasticlunr with flexsearch refs https://github.com/TryGhost/Team/issues/1665 - elasticlunr in an abandoned package with quite a lot of security vulnerabilities. it also has worse performance memory/processing wise comparing to flexsearch (benchmark: https://nextapps-de.github.io/flexsearch/bench/) - fusejs was another option that was consideres. it was not chosed due to it's poor performance. --- ghost/sodo-search/package.json | 4 +- ghost/sodo-search/src/search-index.js | 110 +++++++++------------ ghost/sodo-search/src/search-index.test.js | 2 +- ghost/sodo-search/yarn.lock | 12 +-- 4 files changed, 56 insertions(+), 72 deletions(-) diff --git a/ghost/sodo-search/package.json b/ghost/sodo-search/package.json index bf5d9ba852..02dc9706ab 100644 --- a/ghost/sodo-search/package.json +++ b/ghost/sodo-search/package.json @@ -21,8 +21,8 @@ "@testing-library/jest-dom": "5.16.2", "@testing-library/react": "12.1.2", "@testing-library/user-event": "14.0.0", - "@tryghost/content-api": "^1.11.0", - "elasticlunr": "0.9.5", + "@tryghost/content-api": "1.11.0", + "flexsearch": "0.7.21", "react": "17.0.2", "react-dom": "17.0.2", "react-scripts": "4.0.3" diff --git a/ghost/sodo-search/src/search-index.js b/ghost/sodo-search/src/search-index.js index b376cef836..c0a930567e 100644 --- a/ghost/sodo-search/src/search-index.js +++ b/ghost/sodo-search/src/search-index.js @@ -1,4 +1,4 @@ -import elasticlunr from 'elasticlunr'; +import {Document} from 'flexsearch'; import GhostContentAPI from '@tryghost/content-api'; export default class SearchIndex { @@ -9,9 +9,27 @@ export default class SearchIndex { version: 'v5.0' }); - this.postsIndex = null; - this.authorsIndex = null; - this.tagsIndex = null; + this.postsIndex = new Document({ + document: { + id: 'id', + index: ['title', 'excerpt'], + store: true + } + }); + this.authorsIndex = new Document({ + document: { + id: 'id', + index: ['name'], + store: true + } + }); + this.tagsIndex = new Document({ + document: { + id: 'id', + index: ['name'], + store: true + } + }); this.init = this.init.bind(this); this.search = this.search.bind(this); @@ -19,41 +37,23 @@ export default class SearchIndex { #updatePostIndex(posts) { posts.forEach((post) => { - this.postsIndex.addDoc({ - id: post.id, - title: post.title, - excerpt: post.excerpt, - slug: post.slug, - url: post.url - }); + this.postsIndex.add(post); }); } #updateAuthorsIndex(authors) { authors.forEach((author) => { - this.authorsIndex.addDoc({ - id: author.id, - name: author.name, - url: author.url, - profile_image: author.profile_image - }); + this.authorsIndex.add(author); }); } #updateTagsIndex(tags) { tags.forEach((tag) => { - this.tagsIndex.addDoc({ - id: tag.id, - name: tag.name, - url: tag.url - }); + this.tagsIndex.add(tag); }); } async init() { - // remove default stop words to search of *any* word - elasticlunr.clearStopWords(); - let posts = await this.api.posts.browse({ limit: 'all', fields: 'id,slug,title,excerpt,url,updated_at,visibility', @@ -61,12 +61,6 @@ export default class SearchIndex { formats: 'plaintext' }); - this.postsIndex = elasticlunr(); - this.postsIndex.addField('title'); - this.postsIndex.addField('url'); - this.postsIndex.addField('excerpt'); - this.postsIndex.setRef('id'); - if (posts || posts.length > 0) { if (!posts.length) { posts = [posts]; @@ -79,12 +73,6 @@ export default class SearchIndex { fields: 'id,slug,name,url,profile_image' }); - this.authorsIndex = elasticlunr(); - this.authorsIndex.addField('name'); - this.authorsIndex.addField('url'); - this.authorsIndex.addField('profile_image'); - this.authorsIndex.setRef('id'); - if (authors || authors.length > 0) { if (!authors.length) { authors = [authors]; @@ -97,10 +85,6 @@ export default class SearchIndex { limit: 'all', fields: 'id,slug,name,url' }); - this.tagsIndex = elasticlunr(); - this.tagsIndex.addField('name'); - this.tagsIndex.addField('url'); - this.tagsIndex.setRef('id'); if (tags || tags.length > 0) { if (!tags.length) { @@ -111,37 +95,37 @@ export default class SearchIndex { } } + #normalizeSearchResult(result) { + const normalized = []; + const usedIds = {}; + + result.forEach((resultItem) => { + resultItem.result.forEach((doc) => { + if (!usedIds[doc.id]) { + normalized.push(doc.doc); + usedIds[doc.id] = true; + } + }); + }); + + return normalized; + } + search(value) { const posts = this.postsIndex.search(value, { - fields: { - title: {boost: 1}, - excerpt: {boost: 1} - }, - expand: true + enrich: true }); const authors = this.authorsIndex.search(value, { - fields: { - name: {boost: 1} - }, - expand: true + enrich: true }); const tags = this.tagsIndex.search(value, { - fields: { - name: {boost: 1} - }, - expand: true + enrich: true }); return { - posts: posts.map((doc) => { - return this.postsIndex.documentStore.docs[doc.ref]; - }), - authors: authors.map((doc) => { - return this.authorsIndex.documentStore.docs[doc.ref]; - }), - tags: tags.map((doc) => { - return this.tagsIndex.documentStore.docs[doc.ref]; - }) + posts: this.#normalizeSearchResult(posts), + authors: this.#normalizeSearchResult(authors), + tags: this.#normalizeSearchResult(tags) }; } } diff --git a/ghost/sodo-search/src/search-index.test.js b/ghost/sodo-search/src/search-index.test.js index c835d66201..ccac14458d 100644 --- a/ghost/sodo-search/src/search-index.test.js +++ b/ghost/sodo-search/src/search-index.test.js @@ -43,7 +43,7 @@ describe('search index', function () { posts: [{ id: 'sounique', title: 'Awesome Barcelona Life', - excerpt: 'We are sitting by the pool and smashing out search features', + excerpt: 'We are sitting by the pool and smashing out search features. Barcelona life is great!', url: 'http://localhost/ghost/awesome-barcelona-life/' }] }) diff --git a/ghost/sodo-search/yarn.lock b/ghost/sodo-search/yarn.lock index 09a6920000..0b82a5c616 100644 --- a/ghost/sodo-search/yarn.lock +++ b/ghost/sodo-search/yarn.lock @@ -1680,7 +1680,7 @@ resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-1.1.2.tgz#ccb91445360179a04e7fe6aff78c00ffc1eeaf82" integrity sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw== -"@tryghost/content-api@^1.11.0": +"@tryghost/content-api@1.11.0": version "1.11.0" resolved "https://registry.yarnpkg.com/@tryghost/content-api/-/content-api-1.11.0.tgz#25ab0e2c5618f4ae190d5ad739611a07a551b245" integrity sha512-0JTlp5Ln4BfCJzCYuT2X3MC9ZupIkRtzZaHpf9KZw7O8uOsRnO9RwjItN+lwvkoLIesMzfgrZd/tBJ4BAzeBrg== @@ -4455,11 +4455,6 @@ ejs@^3.1.5: dependencies: jake "^10.8.5" -elasticlunr@0.9.5: - version "0.9.5" - resolved "https://registry.yarnpkg.com/elasticlunr/-/elasticlunr-0.9.5.tgz#65541bb309dddd0cf94f2d1c8861b2be651bb0d5" - integrity sha512-5YM9LFQgVYfuLNEoqMqVWIBuF2UNCA+xu/jz1TyryLN/wmBcQSb+GNAwvLKvEpGESwgGN8XA1nbLAt6rKlyHYQ== - electron-to-chromium@^1.3.564, electron-to-chromium@^1.4.172: version "1.4.177" resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.177.tgz#b6a4436eb788ca732556cd69f384b8a3c82118c5" @@ -5389,6 +5384,11 @@ flatten@^1.0.2: resolved "https://registry.yarnpkg.com/flatten/-/flatten-1.0.3.tgz#c1283ac9f27b368abc1e36d1ff7b04501a30356b" integrity sha512-dVsPA/UwQ8+2uoFe5GHtiBMu48dWLTdsuEd7CKGlZlD78r1TTWBvDuFaFGKCo/ZfEr95Uk56vZoX86OsHkUeIg== +flexsearch@0.7.21: + version "0.7.21" + resolved "https://registry.yarnpkg.com/flexsearch/-/flexsearch-0.7.21.tgz#0f5ede3f2aae67ddc351efbe3b24b69d29e9d48b" + integrity sha512-W7cHV7Hrwjid6lWmy0IhsWDFQboWSng25U3VVywpHOTJnnAZNPScog67G+cVpeX9f7yDD21ih0WDrMMT+JoaYg== + flush-write-stream@^1.0.0: version "1.1.1" resolved "https://registry.yarnpkg.com/flush-write-stream/-/flush-write-stream-1.1.1.tgz#8dd7d873a1babc207d94ead0c2e0e44276ebf2e8"