mudhorn/n8n
1
0
Fork 0
mirror of https://github.com/n8n-io/n8n.git synced 2024-11-09 22:24:05 -08:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: Qdrant Vector Store search filter (#9900)

Browse source 
Signed-off-by: Oleg Ivaniv <me@olegivaniv.com>
Co-authored-by: Oleg Ivaniv <me@olegivaniv.com>
This commit is contained in:
Michael Kret 2024-07-04 15:16:35 +03:00 committed by GitHub
parent 058fa32ca3
commit fbe4bca634
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 114 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

62
packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreQdrant/VectorStoreQdrant.node.ts
View file

@ -1,10 +1,35 @@
import { type INodeProperties } from 'n8n-workflow';
import type { QdrantLibArgs } from '@langchain/community/vectorstores/qdrant';
import { QdrantVectorStore } from '@langchain/community/vectorstores/qdrant';
import type { IDataObject, INodeProperties } from 'n8n-workflow';
import type { QdrantLibArgs } from '@langchain/qdrant';
import { QdrantVectorStore } from '@langchain/qdrant';
import type { Schemas as QdrantSchemas } from '@qdrant/js-client-rest';
import { createVectorStoreNode } from '../shared/createVectorStoreNode';
import { qdrantCollectionRLC } from '../shared/descriptions';
import { qdrantCollectionsSearch } from '../shared/methods/listSearch';
import type { Embeddings } from '@langchain/core/embeddings';
import type { Callbacks } from '@langchain/core/callbacks/manager';
class ExtendedQdrantVectorStore extends QdrantVectorStore {
private static defaultFilter: IDataObject = {};
static async fromExistingCollection(
embeddings: Embeddings,
args: QdrantLibArgs,
defaultFilter: IDataObject = {},
): Promise<QdrantVectorStore> {
ExtendedQdrantVectorStore.defaultFilter = defaultFilter;
return await super.fromExistingCollection(embeddings, args);
}
async similaritySearch(
query: string,
k: number,
filter?: IDataObject,
callbacks?: Callbacks | undefined,
) {
const mergedFilter = { ...ExtendedQdrantVectorStore.defaultFilter, ...filter };
return await super.similaritySearch(query, k, mergedFilter, callbacks);
}
}
const sharedFields: INodeProperties[] = [qdrantCollectionRLC];
@ -28,6 +53,31 @@ const insertFields: INodeProperties[] = [
},
];
const retrieveFields: INodeProperties[] = [
{
displayName: 'Options',
name: 'options',
type: 'collection',
placeholder: 'Add Option',
default: {},
options: [
{
displayName: 'Search Filter',
name: 'searchFilterJson',
type: 'json',
typeOptions: {
rows: 5,
},
default:
'{\n "should": [\n {\n "key": "metadata.batch",\n "match": {\n "value": 12345\n }\n }\n ]\n}',
validateType: 'object',
description:
'Filter pageContent or metadata using this <a href="https://qdrant.tech/documentation/concepts/filtering/" target="_blank">filtering syntax</a>',
},
],
},
];
export const VectorStoreQdrant = createVectorStoreNode({
meta: {
displayName: 'Qdrant Vector Store',
@ -44,9 +94,11 @@ export const VectorStoreQdrant = createVectorStoreNode({
],
},
methods: { listSearch: { qdrantCollectionsSearch } },
loadFields: retrieveFields,
insertFields,
sharedFields,
async getVectorStoreClient(context, _, embeddings, itemIndex) {
retrieveFields,
async getVectorStoreClient(context, filter, embeddings, itemIndex) {
const collection = context.getNodeParameter('qdrantCollection', itemIndex, '', {
extractValue: true,
}) as string;
@ -59,7 +111,7 @@ export const VectorStoreQdrant = createVectorStoreNode({
collectionName: collection,
};
return await QdrantVectorStore.fromExistingCollection(embeddings, config);
return await ExtendedQdrantVectorStore.fromExistingCollection(embeddings, config, filter);
},
async populateVectorStore(context, embeddings, documents, itemIndex) {
const collectionName = context.getNodeParameter('qdrantCollection', itemIndex, '', {

3
packages/@n8n/nodes-langchain/package.json
View file

@ -134,8 +134,8 @@
"dependencies": {
"@aws-sdk/client-bedrock-runtime": "3.535.0",
"@aws-sdk/credential-provider-node": "3.535.0",
"@getzep/zep-js": "0.9.0",
"@getzep/zep-cloud": "1.0.6",
"@getzep/zep-js": "0.9.0",
"@google-ai/generativelanguage": "2.5.0",
"@google/generative-ai": "0.11.4",
"@huggingface/inference": "2.7.0",
@ -148,6 +148,7 @@
"@langchain/mistralai": "0.0.22",
"@langchain/openai": "0.0.33",
"@langchain/pinecone": "0.0.6",
"@langchain/qdrant": "^0.0.5",
"@langchain/redis": "0.0.5",
"@langchain/textsplitters": "0.0.2",
"@mozilla/readability": "^0.5.0",

14
packages/@n8n/nodes-langchain/utils/helpers.ts
View file

@ -10,13 +10,25 @@ export function getMetadataFiltersValues(
ctx: IExecuteFunctions,
itemIndex: number,
): Record<string, never> | undefined {
const metadata = ctx.getNodeParameter('options.metadata.metadataValues', itemIndex, []) as Array<{
const options = ctx.getNodeParameter('options', itemIndex);
if (options.metadata) {
const { metadataValues: metadata } = options.metadata as {
metadataValues: Array<{
name: string;
value: string;
}>;
};
if (metadata.length > 0) {
return metadata.reduce((acc, { name, value }) => ({ ...acc, [name]: value }), {});
}
}
if (options.searchFilterJson) {
return ctx.getNodeParameter('options.searchFilterJson', itemIndex, '', {
ensureType: 'object',
}) as Record<string, never>;
}
return undefined;
}

45
pnpm-lock.yaml
View file

@ -298,6 +298,9 @@ importers:
'@langchain/pinecone':
specifier: 0.0.6
version: 0.0.6(langchain@0.2.2(@aws-sdk/client-s3@3.478.0)(@aws-sdk/credential-provider-node@3.535.0)(@google-ai/generativelanguage@2.5.0(encoding@0.1.13))(@pinecone-database/pinecone@2.2.1)(@supabase/supabase-js@2.43.4)(@xata.io/client@0.28.4(typescript@5.5.2))(axios@1.6.7)(cheerio@1.0.0-rc.12)(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2(ts-toolbelt@9.6.0))(fast-xml-parser@4.3.5)(handlebars@4.7.8)(html-to-text@9.0.5)(ignore@5.2.4)(ioredis@5.3.2)(jsdom@23.0.1)(mammoth@1.7.2)(openai@4.47.1(encoding@0.1.13))(pdf-parse@1.1.1)(redis@4.6.12)(ws@8.17.1))(openai@4.47.1(encoding@0.1.13))
'@langchain/qdrant':
specifier: ^0.0.5
version: 0.0.5(langchain@0.2.2(@aws-sdk/client-s3@3.478.0)(@aws-sdk/credential-provider-node@3.535.0)(@google-ai/generativelanguage@2.5.0(encoding@0.1.13))(@pinecone-database/pinecone@2.2.1)(@supabase/supabase-js@2.43.4)(@xata.io/client@0.28.4(typescript@5.5.2))(axios@1.6.7)(cheerio@1.0.0-rc.12)(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2(ts-toolbelt@9.6.0))(fast-xml-parser@4.3.5)(handlebars@4.7.8)(html-to-text@9.0.5)(ignore@5.2.4)(ioredis@5.3.2)(jsdom@23.0.1)(mammoth@1.7.2)(openai@4.47.1(encoding@0.1.13))(pdf-parse@1.1.1)(redis@4.6.12)(ws@8.17.1))(openai@4.47.1(encoding@0.1.13))(typescript@5.5.2)
'@langchain/redis':
specifier: 0.0.5
version: 0.0.5(langchain@0.2.2(@aws-sdk/client-s3@3.478.0)(@aws-sdk/credential-provider-node@3.535.0)(@google-ai/generativelanguage@2.5.0(encoding@0.1.13))(@pinecone-database/pinecone@2.2.1)(@supabase/supabase-js@2.43.4)(@xata.io/client@0.28.4(typescript@5.5.2))(axios@1.6.7)(cheerio@1.0.0-rc.12)(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2(ts-toolbelt@9.6.0))(fast-xml-parser@4.3.5)(handlebars@4.7.8)(html-to-text@9.0.5)(ignore@5.2.4)(ioredis@5.3.2)(jsdom@23.0.1)(mammoth@1.7.2)(openai@4.47.1(encoding@0.1.13))(pdf-parse@1.1.1)(redis@4.6.12)(ws@8.17.1))(openai@4.47.1(encoding@0.1.13))
@ -3949,6 +3952,10 @@ packages:
resolution: {integrity: sha512-iSi+3DZwK/ocwRVNDAAzjC9oa3E4yRlcgx53GSZ/w22QdDfb5PsxQoOXeqviEwUejLsGawbbex2ip1J4LKJ4/g==}
engines: {node: '>=18'}
'@langchain/qdrant@0.0.5':
resolution: {integrity: sha512-zhBHE3rSBUBzIqBnR4RQB48DwXLPp5LGbCPfFCDrum4ZXDUXHQNq6Jrq8OAm6UFfx9IzMQ07Dlr7/VO6w3BlaQ==}
engines: {node: '>=18'}
'@langchain/redis@0.0.5':
resolution: {integrity: sha512-Jleqg5nbZ4795r8ctdwawafjJs4/1gdQnpcz1VnQEBrGqPne+4jwuwBDzeMhByMBLRBB/I6jaiVs9t5wi5H7Tg==}
engines: {node: '>=18'}
@ -17159,6 +17166,28 @@ snapshots:
- langchain
- openai
'@langchain/pinecone@0.0.6(langchain@0.2.2(@aws-sdk/client-s3@3.478.0)(@aws-sdk/credential-provider-node@3.535.0)(@pinecone-database/pinecone@2.1.0)(@supabase/supabase-js@2.43.4)(@xata.io/client@0.28.4(typescript@5.5.2))(axios@1.6.7)(cheerio@1.0.0-rc.12)(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2)(fast-xml-parser@4.3.5)(handlebars@4.7.8)(html-to-text@9.0.5)(ignore@5.2.4)(ioredis@5.3.2)(jsdom@23.0.1)(mammoth@1.7.2)(openai@4.52.1(encoding@0.1.13))(pdf-parse@1.1.1)(redis@4.6.14)(ws@8.17.1))(openai@4.52.1(encoding@0.1.13))':
dependencies:
'@langchain/core': 0.2.9(langchain@0.2.2(@aws-sdk/client-s3@3.478.0)(@aws-sdk/credential-provider-node@3.535.0)(@pinecone-database/pinecone@2.1.0)(@supabase/supabase-js@2.43.4)(@xata.io/client@0.28.4(typescript@5.5.2))(axios@1.6.7)(cheerio@1.0.0-rc.12)(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2)(fast-xml-parser@4.3.5)(handlebars@4.7.8)(html-to-text@9.0.5)(ignore@5.2.4)(ioredis@5.3.2)(jsdom@23.0.1)(mammoth@1.7.2)(openai@4.52.1(encoding@0.1.13))(pdf-parse@1.1.1)(redis@4.6.14)(ws@8.17.1))(openai@4.52.1(encoding@0.1.13))
'@pinecone-database/pinecone': 2.2.1
flat: 5.0.2
uuid: 9.0.1
transitivePeerDependencies:
- langchain
- openai
'@langchain/qdrant@0.0.5(langchain@0.2.2(@aws-sdk/client-s3@3.478.0)(@aws-sdk/credential-provider-node@3.535.0)(@google-ai/generativelanguage@2.5.0(encoding@0.1.13))(@pinecone-database/pinecone@2.2.1)(@supabase/supabase-js@2.43.4)(@xata.io/client@0.28.4(typescript@5.5.2))(axios@1.6.7)(cheerio@1.0.0-rc.12)(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2(ts-toolbelt@9.6.0))(fast-xml-parser@4.3.5)(handlebars@4.7.8)(html-to-text@9.0.5)(ignore@5.2.4)(ioredis@5.3.2)(jsdom@23.0.1)(mammoth@1.7.2)(openai@4.47.1(encoding@0.1.13))(pdf-parse@1.1.1)(redis@4.6.12)(ws@8.17.1))(openai@4.47.1(encoding@0.1.13))(typescript@5.5.2)':
dependencies:
'@langchain/core': 0.2.9(langchain@0.2.2(@aws-sdk/client-s3@3.478.0)(@aws-sdk/credential-provider-node@3.535.0)(@google-ai/generativelanguage@2.5.0(encoding@0.1.13))(@pinecone-database/pinecone@2.2.1)(@supabase/supabase-js@2.43.4)(@xata.io/client@0.28.4(typescript@5.5.2))(axios@1.6.7)(cheerio@1.0.0-rc.12)(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2(ts-toolbelt@9.6.0))(fast-xml-parser@4.3.5)(handlebars@4.7.8)(html-to-text@9.0.5)(ignore@5.2.4)(ioredis@5.3.2)(jsdom@23.0.1)(mammoth@1.7.2)(openai@4.47.1(encoding@0.1.13))(pdf-parse@1.1.1)(redis@4.6.12)(ws@8.17.1))(openai@4.47.1(encoding@0.1.13))
'@qdrant/js-client-rest': 1.9.0(typescript@5.5.2)
uuid: 9.0.1
transitivePeerDependencies:
- langchain
- openai
- typescript
'@langchain/redis@0.0.5(langchain@0.2.2(@aws-sdk/client-s3@3.478.0)(@aws-sdk/credential-provider-node@3.535.0)(@google-ai/generativelanguage@2.5.0(encoding@0.1.13))(@pinecone-database/pinecone@2.2.1)(@supabase/supabase-js@2.43.4)(@xata.io/client@0.28.4(typescript@5.5.2))(axios@1.6.7)(cheerio@1.0.0-rc.12)(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2(ts-toolbelt@9.6.0))(fast-xml-parser@4.3.5)(handlebars@4.7.8)(html-to-text@9.0.5)(ignore@5.2.4)(ioredis@5.3.2)(jsdom@23.0.1)(mammoth@1.7.2)(openai@4.47.1(encoding@0.1.13))(pdf-parse@1.1.1)(redis@4.6.12)(ws@8.17.1))(openai@4.47.1(encoding@0.1.13))':
dependencies:
'@langchain/core': 0.2.9(langchain@0.2.2(@aws-sdk/client-s3@3.478.0)(@aws-sdk/credential-provider-node@3.535.0)(@google-ai/generativelanguage@2.5.0(encoding@0.1.13))(@pinecone-database/pinecone@2.2.1)(@supabase/supabase-js@2.43.4)(@xata.io/client@0.28.4(typescript@5.5.2))(axios@1.6.7)(cheerio@1.0.0-rc.12)(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2(ts-toolbelt@9.6.0))(fast-xml-parser@4.3.5)(handlebars@4.7.8)(html-to-text@9.0.5)(ignore@5.2.4)(ioredis@5.3.2)(jsdom@23.0.1)(mammoth@1.7.2)(openai@4.47.1(encoding@0.1.13))(pdf-parse@1.1.1)(redis@4.6.12)(ws@8.17.1))(openai@4.47.1(encoding@0.1.13))
@ -22479,7 +22508,7 @@ snapshots:
eslint-import-resolver-node@0.3.9:
dependencies:
debug: 3.2.7(supports-color@5.5.0)
debug: 3.2.7(supports-color@8.1.1)
is-core-module: 2.13.1
resolve: 1.22.8
transitivePeerDependencies:
@ -22504,7 +22533,7 @@ snapshots:
eslint-module-utils@2.8.0(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.5.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.5.2))(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0):
dependencies:
debug: 3.2.7(supports-color@5.5.0)
debug: 3.2.7(supports-color@8.1.1)
optionalDependencies:
'@typescript-eslint/parser': 7.2.0(eslint@8.57.0)(typescript@5.5.2)
eslint: 8.57.0
@ -22524,7 +22553,7 @@ snapshots:
array.prototype.findlastindex: 1.2.3
array.prototype.flat: 1.3.2
array.prototype.flatmap: 1.3.2
debug: 3.2.7(supports-color@5.5.0)
debug: 3.2.7(supports-color@8.1.1)
doctrine: 2.1.0
eslint: 8.57.0
eslint-import-resolver-node: 0.3.9
@ -23059,7 +23088,7 @@ snapshots:
follow-redirects@1.15.6(debug@3.2.7):
optionalDependencies:
debug: 3.2.7(supports-color@5.5.0)
debug: 3.2.7(supports-color@8.1.1)
follow-redirects@1.15.6(debug@4.3.4):
optionalDependencies:
@ -23408,7 +23437,7 @@ snapshots:
array-parallel: 0.1.3
array-series: 0.1.5
cross-spawn: 4.0.2
debug: 3.2.7(supports-color@5.5.0)
debug: 3.2.7(supports-color@8.1.1)
transitivePeerDependencies:
- supports-color
@ -26214,7 +26243,7 @@ snapshots:
pdf-parse@1.1.1:
dependencies:
debug: 3.2.7(supports-color@5.5.0)
debug: 3.2.7(supports-color@8.1.1)
node-ensure: 0.0.0
transitivePeerDependencies:
- supports-color
@ -27245,7 +27274,7 @@ snapshots:
rhea@1.0.24:
dependencies:
debug: 3.2.7(supports-color@5.5.0)
debug: 3.2.7(supports-color@8.1.1)
transitivePeerDependencies:
- supports-color
@ -27648,7 +27677,7 @@ snapshots:
binascii: 0.0.2
bn.js: 5.2.1
browser-request: 0.3.3
debug: 3.2.7(supports-color@5.5.0)
debug: 3.2.7(supports-color@8.1.1)
expand-tilde: 2.0.2
extend: 3.0.2
fast-xml-parser: 4.2.7