diff --git a/packages/frontend/core/src/modules/docs-search/entities/docs-indexer.ts b/packages/frontend/core/src/modules/docs-search/entities/docs-indexer.ts index e506ec2e63..a54e304f31 100644 --- a/packages/frontend/core/src/modules/docs-search/entities/docs-indexer.ts +++ b/packages/frontend/core/src/modules/docs-search/entities/docs-indexer.ts @@ -1,5 +1,10 @@ import { DebugLogger } from '@affine/debug'; -import type { Job, JobQueue, WorkspaceService } from '@toeverything/infra'; +import type { + Job, + JobQueue, + WorkspaceLocalState, + WorkspaceService, +} from '@toeverything/infra'; import { Entity, IndexedDBIndexStorage, @@ -21,12 +26,18 @@ export function isEmptyUpdate(binary: Uint8Array) { } const logger = new DebugLogger('crawler'); +const WORKSPACE_DOCS_INDEXER_VERSION_KEY = 'docs-indexer-version'; interface IndexerJobPayload { storageDocId: string; } export class DocsIndexer extends Entity { + /** + * increase this number to re-index all docs + */ + static INDEXER_VERSION = 1; + private readonly jobQueue: JobQueue = new IndexedDBJobQueue( 'jq:' + this.workspaceService.workspace.id @@ -66,7 +77,10 @@ export class DocsIndexer extends Entity { {} ); - constructor(private readonly workspaceService: WorkspaceService) { + constructor( + private readonly workspaceService: WorkspaceService, + private readonly workspaceLocalState: WorkspaceLocalState + ) { super(); } @@ -96,6 +110,16 @@ export class DocsIndexer extends Entity { return; } + const dbVersion = this.getVersion(); + + if (dbVersion > DocsIndexer.INDEXER_VERSION) { + // stop if db version is higher then self + this.runner.stop(); + throw new Error('Indexer is outdated'); + } + + const isUpgrade = dbVersion < DocsIndexer.INDEXER_VERSION; + // jobs should have the same storage docId, so we just pick the first one const storageDocId = jobs[0].payload.storageDocId; @@ -111,7 +135,6 @@ export class DocsIndexer extends Entity { await this.workspaceEngine.doc.storage.loadDocFromLocal( this.workspaceId ); - if (!rootDocBuffer) { return; } @@ -134,6 +157,7 @@ export class DocsIndexer extends Entity { type: 'rootDoc', allIndexedDocs, rootDocBuffer, + reindexAll: isUpgrade, }); } else { const rootDocBuffer = @@ -226,6 +250,10 @@ export class DocsIndexer extends Entity { ); } + if (isUpgrade) { + this.setVersion(); + } + const duration = performance.now() - startTime; logger.debug( 'Finish crawling job for storageDocId:' + @@ -238,6 +266,7 @@ export class DocsIndexer extends Entity { startCrawling() { this.runner.start(); + this.jobQueue .enqueue([ { @@ -257,6 +286,27 @@ export class DocsIndexer extends Entity { return this.worker; } + getVersion() { + const version = this.workspaceLocalState.get( + WORKSPACE_DOCS_INDEXER_VERSION_KEY + ); + if (typeof version !== 'number') { + return -1; + } else { + return version; + } + } + + setVersion(version = DocsIndexer.INDEXER_VERSION) { + if (this.getVersion() >= version) { + return; + } + return this.workspaceLocalState.set( + WORKSPACE_DOCS_INDEXER_VERSION_KEY, + version + ); + } + override dispose(): void { this.runner.stop(); } diff --git a/packages/frontend/core/src/modules/docs-search/index.ts b/packages/frontend/core/src/modules/docs-search/index.ts index 19df647ba9..03fd0e157a 100644 --- a/packages/frontend/core/src/modules/docs-search/index.ts +++ b/packages/frontend/core/src/modules/docs-search/index.ts @@ -2,6 +2,7 @@ export { DocsSearchService } from './services/docs-search'; import { type Framework, + WorkspaceLocalState, WorkspaceScope, WorkspaceService, } from '@toeverything/infra'; @@ -13,5 +14,5 @@ export function configureDocsSearchModule(framework: Framework) { framework .scope(WorkspaceScope) .service(DocsSearchService, [WorkspaceService]) - .entity(DocsIndexer, [WorkspaceService]); + .entity(DocsIndexer, [WorkspaceService, WorkspaceLocalState]); } diff --git a/packages/frontend/core/src/modules/docs-search/worker/in-worker.ts b/packages/frontend/core/src/modules/docs-search/worker/in-worker.ts index 40a91304db..640d142fc7 100644 --- a/packages/frontend/core/src/modules/docs-search/worker/in-worker.ts +++ b/packages/frontend/core/src/modules/docs-search/worker/in-worker.ts @@ -285,6 +285,7 @@ async function crawlingDocData({ function crawlingRootDocData({ allIndexedDocs, rootDocBuffer, + reindexAll, }: WorkerInput & { type: 'rootDoc'; }): WorkerOutput { @@ -317,7 +318,9 @@ function crawlingRootDocData({ } const needDelete = difference(allIndexedDocs, availableDocs); - const needAdd = difference(availableDocs, allIndexedDocs); + const needAdd = reindexAll + ? availableDocs + : difference(availableDocs, allIndexedDocs); return { reindexDoc: [...needAdd, ...needDelete].map(docId => ({ diff --git a/packages/frontend/core/src/modules/docs-search/worker/types.ts b/packages/frontend/core/src/modules/docs-search/worker/types.ts index 9240ac63cc..b3d33a8ec7 100644 --- a/packages/frontend/core/src/modules/docs-search/worker/types.ts +++ b/packages/frontend/core/src/modules/docs-search/worker/types.ts @@ -31,6 +31,7 @@ export type WorkerInput = type: 'rootDoc'; rootDocBuffer: Uint8Array; allIndexedDocs: string[]; + reindexAll?: boolean; } | { type: 'doc';