mudhorn/n8n
1
0
Fork 0
mirror of https://github.com/n8n-io/n8n.git synced 2025-03-05 20:50:17 -08:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix(Default Data Loader Node): Fix binary data loader in s3 mode (#8626)

Browse source
This commit is contained in:
oleg 2024-02-22 09:20:07 +01:00 committed by GitHub
parent 7c1cf33616
commit a5e6f5928a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 18 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
packages/@n8n/nodes-langchain/utils/N8nBinaryLoader.ts
View file

@ -1,3 +1,5 @@
import { pipeline } from 'stream/promises';
import { createWriteStream } from 'fs';
import type { IExecuteFunctions, INodeExecutionData } from 'n8n-workflow'; import type { IExecuteFunctions, INodeExecutionData } from 'n8n-workflow';
import { NodeOperationError, BINARY_ENCODING } from 'n8n-workflow'; import { NodeOperationError, BINARY_ENCODING } from 'n8n-workflow';
@ -10,8 +12,6 @@ import { PDFLoader } from 'langchain/document_loaders/fs/pdf';
import { TextLoader } from 'langchain/document_loaders/fs/text'; import { TextLoader } from 'langchain/document_loaders/fs/text';
import { EPubLoader } from 'langchain/document_loaders/fs/epub'; import { EPubLoader } from 'langchain/document_loaders/fs/epub';
import { file as tmpFile, type DirectoryResult } from 'tmp-promise'; import { file as tmpFile, type DirectoryResult } from 'tmp-promise';
import { pipeline } from 'stream/promises';
import { createWriteStream } from 'fs';
import { getMetadataFiltersValues } from './helpers'; import { getMetadataFiltersValues } from './helpers';
@ -34,7 +34,12 @@ export class N8nBinaryLoader {
private textSplitter?: TextSplitter; private textSplitter?: TextSplitter;
constructor(context: IExecuteFunctions, optionsPrefix = '', binaryDataKey = '', textSplitter?: TextSplitter) { constructor(
context: IExecuteFunctions,
optionsPrefix = '',
binaryDataKey = '',
textSplitter?: TextSplitter,
) {
this.context = context; this.context = context;
this.textSplitter = textSplitter; this.textSplitter = textSplitter;
this.optionsPrefix = optionsPrefix; this.optionsPrefix = optionsPrefix;
@ -67,7 +72,7 @@ export class N8nBinaryLoader {
if (!item) return []; if (!item) return [];
const binaryData = this.context.helpers.assertBinaryData(itemIndex, this.binaryDataKey) const binaryData = this.context.helpers.assertBinaryData(itemIndex, this.binaryDataKey);
const { mimeType } = binaryData; const { mimeType } = binaryData;
// Check if loader matches the mime-type of the data // Check if loader matches the mime-type of the data
@ -98,7 +103,12 @@ export class N8nBinaryLoader {
let filePathOrBlob: string | Blob; let filePathOrBlob: string | Blob;
if (binaryData.id) { if (binaryData.id) {
filePathOrBlob = this.context.helpers.getBinaryPath(binaryData.id); const binaryBuffer = await this.context.helpers.binaryToBuffer(
await this.context.helpers.getBinaryStream(binaryData.id),
);
filePathOrBlob = new Blob([binaryBuffer], {
type: mimeType,
});
} else { } else {
filePathOrBlob = new Blob([Buffer.from(binaryData.data, BINARY_ENCODING)], { filePathOrBlob = new Blob([Buffer.from(binaryData.data, BINARY_ENCODING)], {
type: mimeType, type: mimeType,
@ -175,8 +185,9 @@ export class N8nBinaryLoader {
loader = new TextLoader(filePathOrBlob); loader = new TextLoader(filePathOrBlob);
} }
const loadedDoc = this.textSplitter
const loadedDoc = this.textSplitter ? await loader.loadAndSplit(this.textSplitter) : await loader.load(); ? await loader.loadAndSplit(this.textSplitter)
: await loader.load();
docs.push(...loadedDoc); docs.push(...loadedDoc);