mirror of
https://github.com/n8n-io/n8n.git
synced 2024-08-16 16:40:30 +03:00
feat: Qdrant Vector Store search filter (#9900)
Signed-off-by: Oleg Ivaniv <me@olegivaniv.com> Co-authored-by: Oleg Ivaniv <me@olegivaniv.com>
This commit is contained in:
parent
058fa32ca3
commit
fbe4bca634
@ -1,10 +1,35 @@
|
||||
import { type INodeProperties } from 'n8n-workflow';
|
||||
import type { QdrantLibArgs } from '@langchain/community/vectorstores/qdrant';
|
||||
import { QdrantVectorStore } from '@langchain/community/vectorstores/qdrant';
|
||||
import type { IDataObject, INodeProperties } from 'n8n-workflow';
|
||||
import type { QdrantLibArgs } from '@langchain/qdrant';
|
||||
import { QdrantVectorStore } from '@langchain/qdrant';
|
||||
import type { Schemas as QdrantSchemas } from '@qdrant/js-client-rest';
|
||||
import { createVectorStoreNode } from '../shared/createVectorStoreNode';
|
||||
import { qdrantCollectionRLC } from '../shared/descriptions';
|
||||
import { qdrantCollectionsSearch } from '../shared/methods/listSearch';
|
||||
import type { Embeddings } from '@langchain/core/embeddings';
|
||||
import type { Callbacks } from '@langchain/core/callbacks/manager';
|
||||
|
||||
class ExtendedQdrantVectorStore extends QdrantVectorStore {
|
||||
private static defaultFilter: IDataObject = {};
|
||||
|
||||
static async fromExistingCollection(
|
||||
embeddings: Embeddings,
|
||||
args: QdrantLibArgs,
|
||||
defaultFilter: IDataObject = {},
|
||||
): Promise<QdrantVectorStore> {
|
||||
ExtendedQdrantVectorStore.defaultFilter = defaultFilter;
|
||||
return await super.fromExistingCollection(embeddings, args);
|
||||
}
|
||||
|
||||
async similaritySearch(
|
||||
query: string,
|
||||
k: number,
|
||||
filter?: IDataObject,
|
||||
callbacks?: Callbacks | undefined,
|
||||
) {
|
||||
const mergedFilter = { ...ExtendedQdrantVectorStore.defaultFilter, ...filter };
|
||||
return await super.similaritySearch(query, k, mergedFilter, callbacks);
|
||||
}
|
||||
}
|
||||
|
||||
const sharedFields: INodeProperties[] = [qdrantCollectionRLC];
|
||||
|
||||
@ -28,6 +53,31 @@ const insertFields: INodeProperties[] = [
|
||||
},
|
||||
];
|
||||
|
||||
const retrieveFields: INodeProperties[] = [
|
||||
{
|
||||
displayName: 'Options',
|
||||
name: 'options',
|
||||
type: 'collection',
|
||||
placeholder: 'Add Option',
|
||||
default: {},
|
||||
options: [
|
||||
{
|
||||
displayName: 'Search Filter',
|
||||
name: 'searchFilterJson',
|
||||
type: 'json',
|
||||
typeOptions: {
|
||||
rows: 5,
|
||||
},
|
||||
default:
|
||||
'{\n "should": [\n {\n "key": "metadata.batch",\n "match": {\n "value": 12345\n }\n }\n ]\n}',
|
||||
validateType: 'object',
|
||||
description:
|
||||
'Filter pageContent or metadata using this <a href="https://qdrant.tech/documentation/concepts/filtering/" target="_blank">filtering syntax</a>',
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
export const VectorStoreQdrant = createVectorStoreNode({
|
||||
meta: {
|
||||
displayName: 'Qdrant Vector Store',
|
||||
@ -44,9 +94,11 @@ export const VectorStoreQdrant = createVectorStoreNode({
|
||||
],
|
||||
},
|
||||
methods: { listSearch: { qdrantCollectionsSearch } },
|
||||
loadFields: retrieveFields,
|
||||
insertFields,
|
||||
sharedFields,
|
||||
async getVectorStoreClient(context, _, embeddings, itemIndex) {
|
||||
retrieveFields,
|
||||
async getVectorStoreClient(context, filter, embeddings, itemIndex) {
|
||||
const collection = context.getNodeParameter('qdrantCollection', itemIndex, '', {
|
||||
extractValue: true,
|
||||
}) as string;
|
||||
@ -59,7 +111,7 @@ export const VectorStoreQdrant = createVectorStoreNode({
|
||||
collectionName: collection,
|
||||
};
|
||||
|
||||
return await QdrantVectorStore.fromExistingCollection(embeddings, config);
|
||||
return await ExtendedQdrantVectorStore.fromExistingCollection(embeddings, config, filter);
|
||||
},
|
||||
async populateVectorStore(context, embeddings, documents, itemIndex) {
|
||||
const collectionName = context.getNodeParameter('qdrantCollection', itemIndex, '', {
|
||||
|
@ -134,8 +134,8 @@
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-bedrock-runtime": "3.535.0",
|
||||
"@aws-sdk/credential-provider-node": "3.535.0",
|
||||
"@getzep/zep-js": "0.9.0",
|
||||
"@getzep/zep-cloud": "1.0.6",
|
||||
"@getzep/zep-js": "0.9.0",
|
||||
"@google-ai/generativelanguage": "2.5.0",
|
||||
"@google/generative-ai": "0.11.4",
|
||||
"@huggingface/inference": "2.7.0",
|
||||
@ -148,6 +148,7 @@
|
||||
"@langchain/mistralai": "0.0.22",
|
||||
"@langchain/openai": "0.0.33",
|
||||
"@langchain/pinecone": "0.0.6",
|
||||
"@langchain/qdrant": "^0.0.5",
|
||||
"@langchain/redis": "0.0.5",
|
||||
"@langchain/textsplitters": "0.0.2",
|
||||
"@mozilla/readability": "^0.5.0",
|
||||
|
@ -10,12 +10,24 @@ export function getMetadataFiltersValues(
|
||||
ctx: IExecuteFunctions,
|
||||
itemIndex: number,
|
||||
): Record<string, never> | undefined {
|
||||
const metadata = ctx.getNodeParameter('options.metadata.metadataValues', itemIndex, []) as Array<{
|
||||
name: string;
|
||||
value: string;
|
||||
}>;
|
||||
if (metadata.length > 0) {
|
||||
return metadata.reduce((acc, { name, value }) => ({ ...acc, [name]: value }), {});
|
||||
const options = ctx.getNodeParameter('options', itemIndex);
|
||||
|
||||
if (options.metadata) {
|
||||
const { metadataValues: metadata } = options.metadata as {
|
||||
metadataValues: Array<{
|
||||
name: string;
|
||||
value: string;
|
||||
}>;
|
||||
};
|
||||
if (metadata.length > 0) {
|
||||
return metadata.reduce((acc, { name, value }) => ({ ...acc, [name]: value }), {});
|
||||
}
|
||||
}
|
||||
|
||||
if (options.searchFilterJson) {
|
||||
return ctx.getNodeParameter('options.searchFilterJson', itemIndex, '', {
|
||||
ensureType: 'object',
|
||||
}) as Record<string, never>;
|
||||
}
|
||||
|
||||
return undefined;
|
||||
|
@ -298,6 +298,9 @@ importers:
|
||||
'@langchain/pinecone':
|
||||
specifier: 0.0.6
|
||||
version: 0.0.6(langchain@0.2.2(@aws-sdk/client-s3@3.478.0)(@aws-sdk/credential-provider-node@3.535.0)(@google-ai/generativelanguage@2.5.0(encoding@0.1.13))(@pinecone-database/pinecone@2.2.1)(@supabase/supabase-js@2.43.4)(@xata.io/client@0.28.4(typescript@5.5.2))(axios@1.6.7)(cheerio@1.0.0-rc.12)(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2(ts-toolbelt@9.6.0))(fast-xml-parser@4.3.5)(handlebars@4.7.8)(html-to-text@9.0.5)(ignore@5.2.4)(ioredis@5.3.2)(jsdom@23.0.1)(mammoth@1.7.2)(openai@4.47.1(encoding@0.1.13))(pdf-parse@1.1.1)(redis@4.6.12)(ws@8.17.1))(openai@4.47.1(encoding@0.1.13))
|
||||
'@langchain/qdrant':
|
||||
specifier: ^0.0.5
|
||||
version: 0.0.5(langchain@0.2.2(@aws-sdk/client-s3@3.478.0)(@aws-sdk/credential-provider-node@3.535.0)(@google-ai/generativelanguage@2.5.0(encoding@0.1.13))(@pinecone-database/pinecone@2.2.1)(@supabase/supabase-js@2.43.4)(@xata.io/client@0.28.4(typescript@5.5.2))(axios@1.6.7)(cheerio@1.0.0-rc.12)(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2(ts-toolbelt@9.6.0))(fast-xml-parser@4.3.5)(handlebars@4.7.8)(html-to-text@9.0.5)(ignore@5.2.4)(ioredis@5.3.2)(jsdom@23.0.1)(mammoth@1.7.2)(openai@4.47.1(encoding@0.1.13))(pdf-parse@1.1.1)(redis@4.6.12)(ws@8.17.1))(openai@4.47.1(encoding@0.1.13))(typescript@5.5.2)
|
||||
'@langchain/redis':
|
||||
specifier: 0.0.5
|
||||
version: 0.0.5(langchain@0.2.2(@aws-sdk/client-s3@3.478.0)(@aws-sdk/credential-provider-node@3.535.0)(@google-ai/generativelanguage@2.5.0(encoding@0.1.13))(@pinecone-database/pinecone@2.2.1)(@supabase/supabase-js@2.43.4)(@xata.io/client@0.28.4(typescript@5.5.2))(axios@1.6.7)(cheerio@1.0.0-rc.12)(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2(ts-toolbelt@9.6.0))(fast-xml-parser@4.3.5)(handlebars@4.7.8)(html-to-text@9.0.5)(ignore@5.2.4)(ioredis@5.3.2)(jsdom@23.0.1)(mammoth@1.7.2)(openai@4.47.1(encoding@0.1.13))(pdf-parse@1.1.1)(redis@4.6.12)(ws@8.17.1))(openai@4.47.1(encoding@0.1.13))
|
||||
@ -3949,6 +3952,10 @@ packages:
|
||||
resolution: {integrity: sha512-iSi+3DZwK/ocwRVNDAAzjC9oa3E4yRlcgx53GSZ/w22QdDfb5PsxQoOXeqviEwUejLsGawbbex2ip1J4LKJ4/g==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
'@langchain/qdrant@0.0.5':
|
||||
resolution: {integrity: sha512-zhBHE3rSBUBzIqBnR4RQB48DwXLPp5LGbCPfFCDrum4ZXDUXHQNq6Jrq8OAm6UFfx9IzMQ07Dlr7/VO6w3BlaQ==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
'@langchain/redis@0.0.5':
|
||||
resolution: {integrity: sha512-Jleqg5nbZ4795r8ctdwawafjJs4/1gdQnpcz1VnQEBrGqPne+4jwuwBDzeMhByMBLRBB/I6jaiVs9t5wi5H7Tg==}
|
||||
engines: {node: '>=18'}
|
||||
@ -17159,6 +17166,28 @@ snapshots:
|
||||
- langchain
|
||||
- openai
|
||||
|
||||
|
||||
'@langchain/pinecone@0.0.6(langchain@0.2.2(@aws-sdk/client-s3@3.478.0)(@aws-sdk/credential-provider-node@3.535.0)(@pinecone-database/pinecone@2.1.0)(@supabase/supabase-js@2.43.4)(@xata.io/client@0.28.4(typescript@5.5.2))(axios@1.6.7)(cheerio@1.0.0-rc.12)(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2)(fast-xml-parser@4.3.5)(handlebars@4.7.8)(html-to-text@9.0.5)(ignore@5.2.4)(ioredis@5.3.2)(jsdom@23.0.1)(mammoth@1.7.2)(openai@4.52.1(encoding@0.1.13))(pdf-parse@1.1.1)(redis@4.6.14)(ws@8.17.1))(openai@4.52.1(encoding@0.1.13))':
|
||||
dependencies:
|
||||
'@langchain/core': 0.2.9(langchain@0.2.2(@aws-sdk/client-s3@3.478.0)(@aws-sdk/credential-provider-node@3.535.0)(@pinecone-database/pinecone@2.1.0)(@supabase/supabase-js@2.43.4)(@xata.io/client@0.28.4(typescript@5.5.2))(axios@1.6.7)(cheerio@1.0.0-rc.12)(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2)(fast-xml-parser@4.3.5)(handlebars@4.7.8)(html-to-text@9.0.5)(ignore@5.2.4)(ioredis@5.3.2)(jsdom@23.0.1)(mammoth@1.7.2)(openai@4.52.1(encoding@0.1.13))(pdf-parse@1.1.1)(redis@4.6.14)(ws@8.17.1))(openai@4.52.1(encoding@0.1.13))
|
||||
'@pinecone-database/pinecone': 2.2.1
|
||||
flat: 5.0.2
|
||||
uuid: 9.0.1
|
||||
transitivePeerDependencies:
|
||||
- langchain
|
||||
- openai
|
||||
|
||||
'@langchain/qdrant@0.0.5(langchain@0.2.2(@aws-sdk/client-s3@3.478.0)(@aws-sdk/credential-provider-node@3.535.0)(@google-ai/generativelanguage@2.5.0(encoding@0.1.13))(@pinecone-database/pinecone@2.2.1)(@supabase/supabase-js@2.43.4)(@xata.io/client@0.28.4(typescript@5.5.2))(axios@1.6.7)(cheerio@1.0.0-rc.12)(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2(ts-toolbelt@9.6.0))(fast-xml-parser@4.3.5)(handlebars@4.7.8)(html-to-text@9.0.5)(ignore@5.2.4)(ioredis@5.3.2)(jsdom@23.0.1)(mammoth@1.7.2)(openai@4.47.1(encoding@0.1.13))(pdf-parse@1.1.1)(redis@4.6.12)(ws@8.17.1))(openai@4.47.1(encoding@0.1.13))(typescript@5.5.2)':
|
||||
dependencies:
|
||||
'@langchain/core': 0.2.9(langchain@0.2.2(@aws-sdk/client-s3@3.478.0)(@aws-sdk/credential-provider-node@3.535.0)(@google-ai/generativelanguage@2.5.0(encoding@0.1.13))(@pinecone-database/pinecone@2.2.1)(@supabase/supabase-js@2.43.4)(@xata.io/client@0.28.4(typescript@5.5.2))(axios@1.6.7)(cheerio@1.0.0-rc.12)(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2(ts-toolbelt@9.6.0))(fast-xml-parser@4.3.5)(handlebars@4.7.8)(html-to-text@9.0.5)(ignore@5.2.4)(ioredis@5.3.2)(jsdom@23.0.1)(mammoth@1.7.2)(openai@4.47.1(encoding@0.1.13))(pdf-parse@1.1.1)(redis@4.6.12)(ws@8.17.1))(openai@4.47.1(encoding@0.1.13))
|
||||
'@qdrant/js-client-rest': 1.9.0(typescript@5.5.2)
|
||||
uuid: 9.0.1
|
||||
transitivePeerDependencies:
|
||||
- langchain
|
||||
- openai
|
||||
- typescript
|
||||
|
||||
|
||||
'@langchain/redis@0.0.5(langchain@0.2.2(@aws-sdk/client-s3@3.478.0)(@aws-sdk/credential-provider-node@3.535.0)(@google-ai/generativelanguage@2.5.0(encoding@0.1.13))(@pinecone-database/pinecone@2.2.1)(@supabase/supabase-js@2.43.4)(@xata.io/client@0.28.4(typescript@5.5.2))(axios@1.6.7)(cheerio@1.0.0-rc.12)(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2(ts-toolbelt@9.6.0))(fast-xml-parser@4.3.5)(handlebars@4.7.8)(html-to-text@9.0.5)(ignore@5.2.4)(ioredis@5.3.2)(jsdom@23.0.1)(mammoth@1.7.2)(openai@4.47.1(encoding@0.1.13))(pdf-parse@1.1.1)(redis@4.6.12)(ws@8.17.1))(openai@4.47.1(encoding@0.1.13))':
|
||||
dependencies:
|
||||
'@langchain/core': 0.2.9(langchain@0.2.2(@aws-sdk/client-s3@3.478.0)(@aws-sdk/credential-provider-node@3.535.0)(@google-ai/generativelanguage@2.5.0(encoding@0.1.13))(@pinecone-database/pinecone@2.2.1)(@supabase/supabase-js@2.43.4)(@xata.io/client@0.28.4(typescript@5.5.2))(axios@1.6.7)(cheerio@1.0.0-rc.12)(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2(ts-toolbelt@9.6.0))(fast-xml-parser@4.3.5)(handlebars@4.7.8)(html-to-text@9.0.5)(ignore@5.2.4)(ioredis@5.3.2)(jsdom@23.0.1)(mammoth@1.7.2)(openai@4.47.1(encoding@0.1.13))(pdf-parse@1.1.1)(redis@4.6.12)(ws@8.17.1))(openai@4.47.1(encoding@0.1.13))
|
||||
@ -22479,7 +22508,7 @@ snapshots:
|
||||
|
||||
eslint-import-resolver-node@0.3.9:
|
||||
dependencies:
|
||||
debug: 3.2.7(supports-color@5.5.0)
|
||||
debug: 3.2.7(supports-color@8.1.1)
|
||||
is-core-module: 2.13.1
|
||||
resolve: 1.22.8
|
||||
transitivePeerDependencies:
|
||||
@ -22504,7 +22533,7 @@ snapshots:
|
||||
|
||||
eslint-module-utils@2.8.0(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.5.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.5.2))(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0):
|
||||
dependencies:
|
||||
debug: 3.2.7(supports-color@5.5.0)
|
||||
debug: 3.2.7(supports-color@8.1.1)
|
||||
optionalDependencies:
|
||||
'@typescript-eslint/parser': 7.2.0(eslint@8.57.0)(typescript@5.5.2)
|
||||
eslint: 8.57.0
|
||||
@ -22524,7 +22553,7 @@ snapshots:
|
||||
array.prototype.findlastindex: 1.2.3
|
||||
array.prototype.flat: 1.3.2
|
||||
array.prototype.flatmap: 1.3.2
|
||||
debug: 3.2.7(supports-color@5.5.0)
|
||||
debug: 3.2.7(supports-color@8.1.1)
|
||||
doctrine: 2.1.0
|
||||
eslint: 8.57.0
|
||||
eslint-import-resolver-node: 0.3.9
|
||||
@ -23059,7 +23088,7 @@ snapshots:
|
||||
|
||||
follow-redirects@1.15.6(debug@3.2.7):
|
||||
optionalDependencies:
|
||||
debug: 3.2.7(supports-color@5.5.0)
|
||||
debug: 3.2.7(supports-color@8.1.1)
|
||||
|
||||
follow-redirects@1.15.6(debug@4.3.4):
|
||||
optionalDependencies:
|
||||
@ -23408,7 +23437,7 @@ snapshots:
|
||||
array-parallel: 0.1.3
|
||||
array-series: 0.1.5
|
||||
cross-spawn: 4.0.2
|
||||
debug: 3.2.7(supports-color@5.5.0)
|
||||
debug: 3.2.7(supports-color@8.1.1)
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
|
||||
@ -26214,7 +26243,7 @@ snapshots:
|
||||
|
||||
pdf-parse@1.1.1:
|
||||
dependencies:
|
||||
debug: 3.2.7(supports-color@5.5.0)
|
||||
debug: 3.2.7(supports-color@8.1.1)
|
||||
node-ensure: 0.0.0
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
@ -27245,7 +27274,7 @@ snapshots:
|
||||
|
||||
rhea@1.0.24:
|
||||
dependencies:
|
||||
debug: 3.2.7(supports-color@5.5.0)
|
||||
debug: 3.2.7(supports-color@8.1.1)
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
|
||||
@ -27648,7 +27677,7 @@ snapshots:
|
||||
binascii: 0.0.2
|
||||
bn.js: 5.2.1
|
||||
browser-request: 0.3.3
|
||||
debug: 3.2.7(supports-color@5.5.0)
|
||||
debug: 3.2.7(supports-color@8.1.1)
|
||||
expand-tilde: 2.0.2
|
||||
extend: 3.0.2
|
||||
fast-xml-parser: 4.2.7
|
||||
|
Loading…
Reference in New Issue
Block a user