From fbe4bca634e8e03c9455843e1a1f89706d1557d2 Mon Sep 17 00:00:00 2001 From: Michael Kret <88898367+michael-radency@users.noreply.github.com> Date: Thu, 4 Jul 2024 15:16:35 +0300 Subject: [PATCH] feat: Qdrant Vector Store search filter (#9900) Signed-off-by: Oleg Ivaniv Co-authored-by: Oleg Ivaniv --- .../VectorStoreQdrant.node.ts | 62 +++++++++++++++++-- packages/@n8n/nodes-langchain/package.json | 3 +- .../@n8n/nodes-langchain/utils/helpers.ts | 24 +++++-- pnpm-lock.yaml | 45 +++++++++++--- 4 files changed, 114 insertions(+), 20 deletions(-) diff --git a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreQdrant/VectorStoreQdrant.node.ts b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreQdrant/VectorStoreQdrant.node.ts index 7714b800f5..5568243418 100644 --- a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreQdrant/VectorStoreQdrant.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreQdrant/VectorStoreQdrant.node.ts @@ -1,10 +1,35 @@ -import { type INodeProperties } from 'n8n-workflow'; -import type { QdrantLibArgs } from '@langchain/community/vectorstores/qdrant'; -import { QdrantVectorStore } from '@langchain/community/vectorstores/qdrant'; +import type { IDataObject, INodeProperties } from 'n8n-workflow'; +import type { QdrantLibArgs } from '@langchain/qdrant'; +import { QdrantVectorStore } from '@langchain/qdrant'; import type { Schemas as QdrantSchemas } from '@qdrant/js-client-rest'; import { createVectorStoreNode } from '../shared/createVectorStoreNode'; import { qdrantCollectionRLC } from '../shared/descriptions'; import { qdrantCollectionsSearch } from '../shared/methods/listSearch'; +import type { Embeddings } from '@langchain/core/embeddings'; +import type { Callbacks } from '@langchain/core/callbacks/manager'; + +class ExtendedQdrantVectorStore extends QdrantVectorStore { + private static defaultFilter: IDataObject = {}; + + static async fromExistingCollection( + embeddings: Embeddings, + args: QdrantLibArgs, + defaultFilter: IDataObject = {}, + ): Promise { + ExtendedQdrantVectorStore.defaultFilter = defaultFilter; + return await super.fromExistingCollection(embeddings, args); + } + + async similaritySearch( + query: string, + k: number, + filter?: IDataObject, + callbacks?: Callbacks | undefined, + ) { + const mergedFilter = { ...ExtendedQdrantVectorStore.defaultFilter, ...filter }; + return await super.similaritySearch(query, k, mergedFilter, callbacks); + } +} const sharedFields: INodeProperties[] = [qdrantCollectionRLC]; @@ -28,6 +53,31 @@ const insertFields: INodeProperties[] = [ }, ]; +const retrieveFields: INodeProperties[] = [ + { + displayName: 'Options', + name: 'options', + type: 'collection', + placeholder: 'Add Option', + default: {}, + options: [ + { + displayName: 'Search Filter', + name: 'searchFilterJson', + type: 'json', + typeOptions: { + rows: 5, + }, + default: + '{\n "should": [\n {\n "key": "metadata.batch",\n "match": {\n "value": 12345\n }\n }\n ]\n}', + validateType: 'object', + description: + 'Filter pageContent or metadata using this filtering syntax', + }, + ], + }, +]; + export const VectorStoreQdrant = createVectorStoreNode({ meta: { displayName: 'Qdrant Vector Store', @@ -44,9 +94,11 @@ export const VectorStoreQdrant = createVectorStoreNode({ ], }, methods: { listSearch: { qdrantCollectionsSearch } }, + loadFields: retrieveFields, insertFields, sharedFields, - async getVectorStoreClient(context, _, embeddings, itemIndex) { + retrieveFields, + async getVectorStoreClient(context, filter, embeddings, itemIndex) { const collection = context.getNodeParameter('qdrantCollection', itemIndex, '', { extractValue: true, }) as string; @@ -59,7 +111,7 @@ export const VectorStoreQdrant = createVectorStoreNode({ collectionName: collection, }; - return await QdrantVectorStore.fromExistingCollection(embeddings, config); + return await ExtendedQdrantVectorStore.fromExistingCollection(embeddings, config, filter); }, async populateVectorStore(context, embeddings, documents, itemIndex) { const collectionName = context.getNodeParameter('qdrantCollection', itemIndex, '', { diff --git a/packages/@n8n/nodes-langchain/package.json b/packages/@n8n/nodes-langchain/package.json index 4923c56d5f..f9f241bd72 100644 --- a/packages/@n8n/nodes-langchain/package.json +++ b/packages/@n8n/nodes-langchain/package.json @@ -134,8 +134,8 @@ "dependencies": { "@aws-sdk/client-bedrock-runtime": "3.535.0", "@aws-sdk/credential-provider-node": "3.535.0", - "@getzep/zep-js": "0.9.0", "@getzep/zep-cloud": "1.0.6", + "@getzep/zep-js": "0.9.0", "@google-ai/generativelanguage": "2.5.0", "@google/generative-ai": "0.11.4", "@huggingface/inference": "2.7.0", @@ -148,6 +148,7 @@ "@langchain/mistralai": "0.0.22", "@langchain/openai": "0.0.33", "@langchain/pinecone": "0.0.6", + "@langchain/qdrant": "^0.0.5", "@langchain/redis": "0.0.5", "@langchain/textsplitters": "0.0.2", "@mozilla/readability": "^0.5.0", diff --git a/packages/@n8n/nodes-langchain/utils/helpers.ts b/packages/@n8n/nodes-langchain/utils/helpers.ts index b67fa7bdcd..cae16a37ee 100644 --- a/packages/@n8n/nodes-langchain/utils/helpers.ts +++ b/packages/@n8n/nodes-langchain/utils/helpers.ts @@ -10,12 +10,24 @@ export function getMetadataFiltersValues( ctx: IExecuteFunctions, itemIndex: number, ): Record | undefined { - const metadata = ctx.getNodeParameter('options.metadata.metadataValues', itemIndex, []) as Array<{ - name: string; - value: string; - }>; - if (metadata.length > 0) { - return metadata.reduce((acc, { name, value }) => ({ ...acc, [name]: value }), {}); + const options = ctx.getNodeParameter('options', itemIndex); + + if (options.metadata) { + const { metadataValues: metadata } = options.metadata as { + metadataValues: Array<{ + name: string; + value: string; + }>; + }; + if (metadata.length > 0) { + return metadata.reduce((acc, { name, value }) => ({ ...acc, [name]: value }), {}); + } + } + + if (options.searchFilterJson) { + return ctx.getNodeParameter('options.searchFilterJson', itemIndex, '', { + ensureType: 'object', + }) as Record; } return undefined; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b6b3a56995..cf589b40d7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -298,6 +298,9 @@ importers: '@langchain/pinecone': specifier: 0.0.6 version: 0.0.6(langchain@0.2.2(@aws-sdk/client-s3@3.478.0)(@aws-sdk/credential-provider-node@3.535.0)(@google-ai/generativelanguage@2.5.0(encoding@0.1.13))(@pinecone-database/pinecone@2.2.1)(@supabase/supabase-js@2.43.4)(@xata.io/client@0.28.4(typescript@5.5.2))(axios@1.6.7)(cheerio@1.0.0-rc.12)(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2(ts-toolbelt@9.6.0))(fast-xml-parser@4.3.5)(handlebars@4.7.8)(html-to-text@9.0.5)(ignore@5.2.4)(ioredis@5.3.2)(jsdom@23.0.1)(mammoth@1.7.2)(openai@4.47.1(encoding@0.1.13))(pdf-parse@1.1.1)(redis@4.6.12)(ws@8.17.1))(openai@4.47.1(encoding@0.1.13)) + '@langchain/qdrant': + specifier: ^0.0.5 + version: 0.0.5(langchain@0.2.2(@aws-sdk/client-s3@3.478.0)(@aws-sdk/credential-provider-node@3.535.0)(@google-ai/generativelanguage@2.5.0(encoding@0.1.13))(@pinecone-database/pinecone@2.2.1)(@supabase/supabase-js@2.43.4)(@xata.io/client@0.28.4(typescript@5.5.2))(axios@1.6.7)(cheerio@1.0.0-rc.12)(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2(ts-toolbelt@9.6.0))(fast-xml-parser@4.3.5)(handlebars@4.7.8)(html-to-text@9.0.5)(ignore@5.2.4)(ioredis@5.3.2)(jsdom@23.0.1)(mammoth@1.7.2)(openai@4.47.1(encoding@0.1.13))(pdf-parse@1.1.1)(redis@4.6.12)(ws@8.17.1))(openai@4.47.1(encoding@0.1.13))(typescript@5.5.2) '@langchain/redis': specifier: 0.0.5 version: 0.0.5(langchain@0.2.2(@aws-sdk/client-s3@3.478.0)(@aws-sdk/credential-provider-node@3.535.0)(@google-ai/generativelanguage@2.5.0(encoding@0.1.13))(@pinecone-database/pinecone@2.2.1)(@supabase/supabase-js@2.43.4)(@xata.io/client@0.28.4(typescript@5.5.2))(axios@1.6.7)(cheerio@1.0.0-rc.12)(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2(ts-toolbelt@9.6.0))(fast-xml-parser@4.3.5)(handlebars@4.7.8)(html-to-text@9.0.5)(ignore@5.2.4)(ioredis@5.3.2)(jsdom@23.0.1)(mammoth@1.7.2)(openai@4.47.1(encoding@0.1.13))(pdf-parse@1.1.1)(redis@4.6.12)(ws@8.17.1))(openai@4.47.1(encoding@0.1.13)) @@ -3949,6 +3952,10 @@ packages: resolution: {integrity: sha512-iSi+3DZwK/ocwRVNDAAzjC9oa3E4yRlcgx53GSZ/w22QdDfb5PsxQoOXeqviEwUejLsGawbbex2ip1J4LKJ4/g==} engines: {node: '>=18'} + '@langchain/qdrant@0.0.5': + resolution: {integrity: sha512-zhBHE3rSBUBzIqBnR4RQB48DwXLPp5LGbCPfFCDrum4ZXDUXHQNq6Jrq8OAm6UFfx9IzMQ07Dlr7/VO6w3BlaQ==} + engines: {node: '>=18'} + '@langchain/redis@0.0.5': resolution: {integrity: sha512-Jleqg5nbZ4795r8ctdwawafjJs4/1gdQnpcz1VnQEBrGqPne+4jwuwBDzeMhByMBLRBB/I6jaiVs9t5wi5H7Tg==} engines: {node: '>=18'} @@ -17159,6 +17166,28 @@ snapshots: - langchain - openai + + '@langchain/pinecone@0.0.6(langchain@0.2.2(@aws-sdk/client-s3@3.478.0)(@aws-sdk/credential-provider-node@3.535.0)(@pinecone-database/pinecone@2.1.0)(@supabase/supabase-js@2.43.4)(@xata.io/client@0.28.4(typescript@5.5.2))(axios@1.6.7)(cheerio@1.0.0-rc.12)(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2)(fast-xml-parser@4.3.5)(handlebars@4.7.8)(html-to-text@9.0.5)(ignore@5.2.4)(ioredis@5.3.2)(jsdom@23.0.1)(mammoth@1.7.2)(openai@4.52.1(encoding@0.1.13))(pdf-parse@1.1.1)(redis@4.6.14)(ws@8.17.1))(openai@4.52.1(encoding@0.1.13))': + dependencies: + '@langchain/core': 0.2.9(langchain@0.2.2(@aws-sdk/client-s3@3.478.0)(@aws-sdk/credential-provider-node@3.535.0)(@pinecone-database/pinecone@2.1.0)(@supabase/supabase-js@2.43.4)(@xata.io/client@0.28.4(typescript@5.5.2))(axios@1.6.7)(cheerio@1.0.0-rc.12)(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2)(fast-xml-parser@4.3.5)(handlebars@4.7.8)(html-to-text@9.0.5)(ignore@5.2.4)(ioredis@5.3.2)(jsdom@23.0.1)(mammoth@1.7.2)(openai@4.52.1(encoding@0.1.13))(pdf-parse@1.1.1)(redis@4.6.14)(ws@8.17.1))(openai@4.52.1(encoding@0.1.13)) + '@pinecone-database/pinecone': 2.2.1 + flat: 5.0.2 + uuid: 9.0.1 + transitivePeerDependencies: + - langchain + - openai + + '@langchain/qdrant@0.0.5(langchain@0.2.2(@aws-sdk/client-s3@3.478.0)(@aws-sdk/credential-provider-node@3.535.0)(@google-ai/generativelanguage@2.5.0(encoding@0.1.13))(@pinecone-database/pinecone@2.2.1)(@supabase/supabase-js@2.43.4)(@xata.io/client@0.28.4(typescript@5.5.2))(axios@1.6.7)(cheerio@1.0.0-rc.12)(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2(ts-toolbelt@9.6.0))(fast-xml-parser@4.3.5)(handlebars@4.7.8)(html-to-text@9.0.5)(ignore@5.2.4)(ioredis@5.3.2)(jsdom@23.0.1)(mammoth@1.7.2)(openai@4.47.1(encoding@0.1.13))(pdf-parse@1.1.1)(redis@4.6.12)(ws@8.17.1))(openai@4.47.1(encoding@0.1.13))(typescript@5.5.2)': + dependencies: + '@langchain/core': 0.2.9(langchain@0.2.2(@aws-sdk/client-s3@3.478.0)(@aws-sdk/credential-provider-node@3.535.0)(@google-ai/generativelanguage@2.5.0(encoding@0.1.13))(@pinecone-database/pinecone@2.2.1)(@supabase/supabase-js@2.43.4)(@xata.io/client@0.28.4(typescript@5.5.2))(axios@1.6.7)(cheerio@1.0.0-rc.12)(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2(ts-toolbelt@9.6.0))(fast-xml-parser@4.3.5)(handlebars@4.7.8)(html-to-text@9.0.5)(ignore@5.2.4)(ioredis@5.3.2)(jsdom@23.0.1)(mammoth@1.7.2)(openai@4.47.1(encoding@0.1.13))(pdf-parse@1.1.1)(redis@4.6.12)(ws@8.17.1))(openai@4.47.1(encoding@0.1.13)) + '@qdrant/js-client-rest': 1.9.0(typescript@5.5.2) + uuid: 9.0.1 + transitivePeerDependencies: + - langchain + - openai + - typescript + + '@langchain/redis@0.0.5(langchain@0.2.2(@aws-sdk/client-s3@3.478.0)(@aws-sdk/credential-provider-node@3.535.0)(@google-ai/generativelanguage@2.5.0(encoding@0.1.13))(@pinecone-database/pinecone@2.2.1)(@supabase/supabase-js@2.43.4)(@xata.io/client@0.28.4(typescript@5.5.2))(axios@1.6.7)(cheerio@1.0.0-rc.12)(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2(ts-toolbelt@9.6.0))(fast-xml-parser@4.3.5)(handlebars@4.7.8)(html-to-text@9.0.5)(ignore@5.2.4)(ioredis@5.3.2)(jsdom@23.0.1)(mammoth@1.7.2)(openai@4.47.1(encoding@0.1.13))(pdf-parse@1.1.1)(redis@4.6.12)(ws@8.17.1))(openai@4.47.1(encoding@0.1.13))': dependencies: '@langchain/core': 0.2.9(langchain@0.2.2(@aws-sdk/client-s3@3.478.0)(@aws-sdk/credential-provider-node@3.535.0)(@google-ai/generativelanguage@2.5.0(encoding@0.1.13))(@pinecone-database/pinecone@2.2.1)(@supabase/supabase-js@2.43.4)(@xata.io/client@0.28.4(typescript@5.5.2))(axios@1.6.7)(cheerio@1.0.0-rc.12)(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2(ts-toolbelt@9.6.0))(fast-xml-parser@4.3.5)(handlebars@4.7.8)(html-to-text@9.0.5)(ignore@5.2.4)(ioredis@5.3.2)(jsdom@23.0.1)(mammoth@1.7.2)(openai@4.47.1(encoding@0.1.13))(pdf-parse@1.1.1)(redis@4.6.12)(ws@8.17.1))(openai@4.47.1(encoding@0.1.13)) @@ -22479,7 +22508,7 @@ snapshots: eslint-import-resolver-node@0.3.9: dependencies: - debug: 3.2.7(supports-color@5.5.0) + debug: 3.2.7(supports-color@8.1.1) is-core-module: 2.13.1 resolve: 1.22.8 transitivePeerDependencies: @@ -22504,7 +22533,7 @@ snapshots: eslint-module-utils@2.8.0(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.5.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.5.2))(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0): dependencies: - debug: 3.2.7(supports-color@5.5.0) + debug: 3.2.7(supports-color@8.1.1) optionalDependencies: '@typescript-eslint/parser': 7.2.0(eslint@8.57.0)(typescript@5.5.2) eslint: 8.57.0 @@ -22524,7 +22553,7 @@ snapshots: array.prototype.findlastindex: 1.2.3 array.prototype.flat: 1.3.2 array.prototype.flatmap: 1.3.2 - debug: 3.2.7(supports-color@5.5.0) + debug: 3.2.7(supports-color@8.1.1) doctrine: 2.1.0 eslint: 8.57.0 eslint-import-resolver-node: 0.3.9 @@ -23059,7 +23088,7 @@ snapshots: follow-redirects@1.15.6(debug@3.2.7): optionalDependencies: - debug: 3.2.7(supports-color@5.5.0) + debug: 3.2.7(supports-color@8.1.1) follow-redirects@1.15.6(debug@4.3.4): optionalDependencies: @@ -23408,7 +23437,7 @@ snapshots: array-parallel: 0.1.3 array-series: 0.1.5 cross-spawn: 4.0.2 - debug: 3.2.7(supports-color@5.5.0) + debug: 3.2.7(supports-color@8.1.1) transitivePeerDependencies: - supports-color @@ -26214,7 +26243,7 @@ snapshots: pdf-parse@1.1.1: dependencies: - debug: 3.2.7(supports-color@5.5.0) + debug: 3.2.7(supports-color@8.1.1) node-ensure: 0.0.0 transitivePeerDependencies: - supports-color @@ -27245,7 +27274,7 @@ snapshots: rhea@1.0.24: dependencies: - debug: 3.2.7(supports-color@5.5.0) + debug: 3.2.7(supports-color@8.1.1) transitivePeerDependencies: - supports-color @@ -27648,7 +27677,7 @@ snapshots: binascii: 0.0.2 bn.js: 5.2.1 browser-request: 0.3.3 - debug: 3.2.7(supports-color@5.5.0) + debug: 3.2.7(supports-color@8.1.1) expand-tilde: 2.0.2 extend: 3.0.2 fast-xml-parser: 4.2.7