From 178235e148bb539bc4e1d6ecbc2d1d5784ca115a Mon Sep 17 00:00:00 2001 From: Ahsan Virani Date: Fri, 20 Aug 2021 18:08:40 +0200 Subject: [PATCH] :sparkles: Initial changes for binary data management (#2105) * introduce binary data management * cr * add binary data changes to awsS3 node * add binary data changes to Box node * add binary data changes to CiscoWebex node * add binary data changes to HumaniticAi node * add binary data changes to Jira node * add binary data changes to Line node * add binary data changes to MicrosoftOneDrive node * add binary data changes to MicrosoftOutlook node * add binary data changes to Mindee node * add binary data changes to NocoDB node * add binary data changes to Pushbullet node * add binary data changes to Pushover node * add binary data changes to Raindrop node * add binary data changes to S3 node * add binary data changes to Salesforce node * add binary data changes to Ssh node * add binary data changes to TheHive node * add binary data changes to Twist node * add binary data changes to Twitter node * remove changes not needed right now * :bug: Fix issue with multiple runs * :bug: Revert fix and add support for multiple inputs Co-authored-by: Jan Oberhauser --- packages/core/src/Interfaces.ts | 7 +++--- packages/core/src/NodeExecuteFunctions.ts | 18 +++++++++++++ packages/core/test/WorkflowExecute.test.ts | 2 +- .../nodes-base/nodes/Aws/S3/AwsS3.node.ts | 4 +-- packages/nodes-base/nodes/Box/Box.node.ts | 4 +-- .../nodes/Cisco/Webex/CiscoWebex.node.ts | 4 +-- packages/nodes-base/nodes/Compression.node.ts | 15 +++++------ .../nodes/HumanticAI/HumanticAi.node.ts | 7 +++--- packages/nodes-base/nodes/Jira/Jira.node.ts | 4 +-- packages/nodes-base/nodes/Line/Line.node.ts | 4 +-- .../OneDrive/MicrosoftOneDrive.node.ts | 3 +-- .../Outlook/MicrosoftOutlook.node.ts | 2 +- .../nodes-base/nodes/Mindee/Mindee.node.ts | 7 +++--- .../nodes-base/nodes/NocoDB/NocoDB.node.ts | 7 +++--- .../nodes/Pushbullet/Pushbullet.node.ts | 4 +-- .../nodes/Pushover/Pushover.node.ts | 5 ++-- .../nodes/Raindrop/Raindrop.node.ts | 4 +-- packages/nodes-base/nodes/S3/S3.node.ts | 4 +-- .../nodes/Salesforce/Salesforce.node.ts | 5 ++-- packages/nodes-base/nodes/Ssh/Ssh.node.ts | 5 ++-- .../nodes-base/nodes/TheHive/TheHive.node.ts | 7 +++--- packages/nodes-base/nodes/Twist/Twist.node.ts | 25 +++++++++++++------ .../nodes/Twitter/GenericFunctions.ts | 10 +++++--- .../nodes-base/nodes/WriteBinaryFile.node.ts | 12 ++++----- 24 files changed, 103 insertions(+), 66 deletions(-) diff --git a/packages/core/src/Interfaces.ts b/packages/core/src/Interfaces.ts index 763afdf4b8..52bd80f700 100644 --- a/packages/core/src/Interfaces.ts +++ b/packages/core/src/Interfaces.ts @@ -34,9 +34,10 @@ export interface IProcessMessage { export interface IExecuteFunctions extends IExecuteFunctionsBase { helpers: { prepareBinaryData(binaryData: Buffer, filePath?: string, mimeType?: string): Promise; - request: requestPromise.RequestPromiseAPI, - requestOAuth2(this: IAllExecuteFunctions, credentialsType: string, requestOptions: OptionsWithUri | requestPromise.RequestPromiseOptions, oAuth2Options?: IOAuth2Options): Promise, // tslint:disable-line:no-any - requestOAuth1(this: IAllExecuteFunctions, credentialsType: string, requestOptions: OptionsWithUrl | requestPromise.RequestPromiseOptions): Promise, // tslint:disable-line:no-any + getBinaryDataBuffer(itemIndex: number, propertyName: string): Promise; + request: requestPromise.RequestPromiseAPI; + requestOAuth2(this: IAllExecuteFunctions, credentialsType: string, requestOptions: OptionsWithUri | requestPromise.RequestPromiseOptions, oAuth2Options?: IOAuth2Options): Promise; // tslint:disable-line:no-any + requestOAuth1(this: IAllExecuteFunctions, credentialsType: string, requestOptions: OptionsWithUrl | requestPromise.RequestPromiseOptions): Promise; // tslint:disable-line:no-any returnJsonArray(jsonData: IDataObject | IDataObject[]): INodeExecutionData[]; }; } diff --git a/packages/core/src/NodeExecuteFunctions.ts b/packages/core/src/NodeExecuteFunctions.ts index 218eb097d4..1042257e65 100644 --- a/packages/core/src/NodeExecuteFunctions.ts +++ b/packages/core/src/NodeExecuteFunctions.ts @@ -59,6 +59,21 @@ const requestPromiseWithDefaults = requestPromise.defaults({ timeout: 300000, // 5 minutes }); +/** + * Returns binary data buffer for given item index and property name. + * + * @export + * @param {ITaskDataConnections} inputData + * @param {number} itemIndex + * @param {string} propertyName + * @param {number} inputIndex + * @returns {Promise} + */ +export async function getBinaryDataBuffer(inputData: ITaskDataConnections, itemIndex: number, propertyName: string, inputIndex: number): Promise { + const binaryData = inputData['main']![inputIndex]![itemIndex]!.binary![propertyName]!; + return Buffer.from(binaryData.data, BINARY_ENCODING); +} + /** * Takes a buffer and converts it into the format n8n uses. It encodes the binary data as * base64 and adds metadata. @@ -763,6 +778,9 @@ export function getExecuteFunctions(workflow: Workflow, runExecutionData: IRunEx }, helpers: { prepareBinaryData, + getBinaryDataBuffer(itemIndex: number, propertyName: string, inputIndex = 0): Promise { + return getBinaryDataBuffer.call(this, inputData, itemIndex, propertyName, inputIndex); + }, request: requestPromiseWithDefaults, requestOAuth2(this: IAllExecuteFunctions, credentialsType: string, requestOptions: OptionsWithUri | requestPromise.RequestPromiseOptions, oAuth2Options?: IOAuth2Options): Promise { // tslint:disable-line:no-any return requestOAuth2.call(this, credentialsType, requestOptions, node, additionalData, oAuth2Options); diff --git a/packages/core/test/WorkflowExecute.test.ts b/packages/core/test/WorkflowExecute.test.ts index 454dd521b5..2bbdb83be8 100644 --- a/packages/core/test/WorkflowExecute.test.ts +++ b/packages/core/test/WorkflowExecute.test.ts @@ -1348,7 +1348,7 @@ describe('WorkflowExecute', () => { const workflowExecute = new WorkflowExecute(additionalData, executionMode); - const executionData = await workflowExecute.run(workflowInstance, undefined); + const executionData = await workflowExecute.run(workflowInstance); const result = await waitPromise.promise(); diff --git a/packages/nodes-base/nodes/Aws/S3/AwsS3.node.ts b/packages/nodes-base/nodes/Aws/S3/AwsS3.node.ts index 65cc1f3c1c..a58e122f12 100644 --- a/packages/nodes-base/nodes/Aws/S3/AwsS3.node.ts +++ b/packages/nodes-base/nodes/Aws/S3/AwsS3.node.ts @@ -13,7 +13,6 @@ import { } from 'xml2js'; import { - BINARY_ENCODING, IExecuteFunctions, } from 'n8n-core'; @@ -607,8 +606,9 @@ export class AwsS3 implements INodeType { } const binaryData = (items[i].binary as IBinaryKeyData)[binaryPropertyName]; + const binaryDataBuffer = await this.helpers.getBinaryDataBuffer(i, binaryPropertyName); - body = Buffer.from(binaryData.data, BINARY_ENCODING) as Buffer; + body = binaryDataBuffer; headers['Content-Type'] = binaryData.mimeType; diff --git a/packages/nodes-base/nodes/Box/Box.node.ts b/packages/nodes-base/nodes/Box/Box.node.ts index 00d19f596e..b5a6ab529f 100644 --- a/packages/nodes-base/nodes/Box/Box.node.ts +++ b/packages/nodes-base/nodes/Box/Box.node.ts @@ -1,5 +1,4 @@ import { - BINARY_ENCODING, IExecuteFunctions, } from 'n8n-core'; @@ -277,6 +276,7 @@ export class Box implements INodeType { } const binaryData = (items[i].binary as IBinaryKeyData)[binaryPropertyName]; + const binaryDataBuffer = await this.helpers.getBinaryDataBuffer(i, binaryPropertyName); const body: IDataObject = {}; @@ -285,7 +285,7 @@ export class Box implements INodeType { body['attributes'] = JSON.stringify(attributes); body['file'] = { - value: Buffer.from(binaryData.data, BINARY_ENCODING), + value: binaryDataBuffer, options: { filename: binaryData.fileName, contentType: binaryData.mimeType, diff --git a/packages/nodes-base/nodes/Cisco/Webex/CiscoWebex.node.ts b/packages/nodes-base/nodes/Cisco/Webex/CiscoWebex.node.ts index 430f69cfd6..80cb3c71f2 100644 --- a/packages/nodes-base/nodes/Cisco/Webex/CiscoWebex.node.ts +++ b/packages/nodes-base/nodes/Cisco/Webex/CiscoWebex.node.ts @@ -1,5 +1,4 @@ import { - BINARY_ENCODING, IExecuteFunctions, } from 'n8n-core'; @@ -174,10 +173,11 @@ export class CiscoWebex implements INodeType { const binaryPropertyName = file.binaryPropertyName as string; const binaryData = items[i].binary![binaryPropertyName] as IBinaryData; + const binaryDataBuffer = await this.helpers.getBinaryDataBuffer(i, binaryPropertyName); const formData = { files: { - value: Buffer.from(binaryData.data, BINARY_ENCODING), + value: binaryDataBuffer, options: { filename: binaryData.fileName, contentType: binaryData.mimeType, diff --git a/packages/nodes-base/nodes/Compression.node.ts b/packages/nodes-base/nodes/Compression.node.ts index 14cbdd5008..bb8d6ab45c 100644 --- a/packages/nodes-base/nodes/Compression.node.ts +++ b/packages/nodes-base/nodes/Compression.node.ts @@ -1,5 +1,4 @@ import { - BINARY_ENCODING, IExecuteFunctions, } from 'n8n-core'; @@ -226,9 +225,10 @@ export class Compression implements INodeType { } const binaryData = (items[i].binary as IBinaryKeyData)[binaryPropertyName]; + const binaryDataBuffer = await this.helpers.getBinaryDataBuffer(i, binaryPropertyName); if (binaryData.fileExtension === 'zip') { - const files = await unzip(Buffer.from(binaryData.data as string, BINARY_ENCODING)); + const files = await unzip(binaryDataBuffer); for (const key of Object.keys(files)) { // when files are compresed using MACOSX for some reason they are duplicated under __MACOSX @@ -241,7 +241,7 @@ export class Compression implements INodeType { binaryObject[`${outputPrefix}${zipIndex++}`] = data; } } else if (binaryData.fileExtension === 'gz') { - const file = await gunzip(Buffer.from(binaryData.data as string, BINARY_ENCODING)); + const file = await gunzip(binaryDataBuffer); const fileName = binaryData.fileName?.split('.')[0]; @@ -280,10 +280,11 @@ export class Compression implements INodeType { } const binaryData = (items[i].binary as IBinaryKeyData)[binaryPropertyName]; + const binaryDataBuffer = await this.helpers.getBinaryDataBuffer(i, binaryPropertyName); if (outputFormat === 'zip') { zipData[binaryData.fileName as string] = [ - Buffer.from(binaryData.data, BINARY_ENCODING), { + binaryDataBuffer, { level: ALREADY_COMPRESSED.includes(binaryData.fileExtension as string) ? 0 : 6, }, ]; @@ -291,7 +292,7 @@ export class Compression implements INodeType { } else if (outputFormat === 'gzip') { const outputPrefix = this.getNodeParameter('outputPrefix', 0) as string; - const data = await gzip(Buffer.from(binaryData.data, BINARY_ENCODING)) as Uint8Array; + const data = await gzip(binaryDataBuffer) as Uint8Array; const fileName = binaryData.fileName?.split('.')[0]; @@ -323,10 +324,10 @@ export class Compression implements INodeType { }); } } - + } catch (error) { if (this.continueOnFail()) { - returnData.push({json:{ error: error.message }}); + returnData.push({ json: { error: error.message } }); continue; } throw error; diff --git a/packages/nodes-base/nodes/HumanticAI/HumanticAi.node.ts b/packages/nodes-base/nodes/HumanticAI/HumanticAi.node.ts index d72598be51..9fd8d8a3a9 100644 --- a/packages/nodes-base/nodes/HumanticAI/HumanticAi.node.ts +++ b/packages/nodes-base/nodes/HumanticAI/HumanticAi.node.ts @@ -1,5 +1,4 @@ import { - BINARY_ENCODING, IExecuteFunctions, } from 'n8n-core'; @@ -88,6 +87,7 @@ export class HumanticAi implements INodeType { const item = items[i].binary as IBinaryKeyData; const binaryData = item[binaryPropertyName] as IBinaryData; + const binaryDataBuffer = await this.helpers.getBinaryDataBuffer(i, binaryPropertyName); if (binaryData === undefined) { throw new NodeOperationError(this.getNode(), `No binary data property "${binaryPropertyName}" does not exists on item!`); @@ -102,7 +102,7 @@ export class HumanticAi implements INodeType { { formData: { resume: { - value: Buffer.from(binaryData.data, BINARY_ENCODING), + value: binaryDataBuffer, options: { filename: binaryData.fileName, }, @@ -148,6 +148,7 @@ export class HumanticAi implements INodeType { const item = items[i].binary as IBinaryKeyData; const binaryData = item[binaryPropertyName] as IBinaryData; + const binaryDataBuffer = await this.helpers.getBinaryDataBuffer(i, binaryPropertyName); if (binaryData === undefined) { throw new NodeOperationError(this.getNode(), `No binary data property "${binaryPropertyName}" does not exists on item!`); @@ -162,7 +163,7 @@ export class HumanticAi implements INodeType { { formData: { resume: { - value: Buffer.from(binaryData.data, BINARY_ENCODING), + value: binaryDataBuffer, options: { filename: binaryData.fileName, }, diff --git a/packages/nodes-base/nodes/Jira/Jira.node.ts b/packages/nodes-base/nodes/Jira/Jira.node.ts index 1f26a3e5ff..0f844630a8 100644 --- a/packages/nodes-base/nodes/Jira/Jira.node.ts +++ b/packages/nodes-base/nodes/Jira/Jira.node.ts @@ -1,5 +1,4 @@ import { - BINARY_ENCODING, IExecuteFunctions, } from 'n8n-core'; @@ -754,6 +753,7 @@ export class Jira implements INodeType { const item = items[i].binary as IBinaryKeyData; const binaryData = item[binaryPropertyName] as IBinaryData; + const binaryDataBuffer = await this.helpers.getBinaryDataBuffer(i, binaryPropertyName); if (binaryData === undefined) { throw new NodeOperationError(this.getNode(), `No binary data property "${binaryPropertyName}" does not exists on item!`); @@ -769,7 +769,7 @@ export class Jira implements INodeType { { formData: { file: { - value: Buffer.from(binaryData.data, BINARY_ENCODING), + value: binaryDataBuffer, options: { filename: binaryData.fileName, }, diff --git a/packages/nodes-base/nodes/Line/Line.node.ts b/packages/nodes-base/nodes/Line/Line.node.ts index 2b2deac3ab..f7e84c2daa 100644 --- a/packages/nodes-base/nodes/Line/Line.node.ts +++ b/packages/nodes-base/nodes/Line/Line.node.ts @@ -1,5 +1,4 @@ import { - BINARY_ENCODING, IExecuteFunctions, } from 'n8n-core'; @@ -118,9 +117,10 @@ export class Line implements INodeType { } const binaryData = (items[i].binary as IBinaryKeyData)[image.binaryProperty as string]; + const binaryDataBuffer = await this.helpers.getBinaryDataBuffer(i, image.binaryProperty as string); body.imageFile = { - value: Buffer.from(binaryData.data, BINARY_ENCODING), + value: binaryDataBuffer, options: { filename: binaryData.fileName, }, diff --git a/packages/nodes-base/nodes/Microsoft/OneDrive/MicrosoftOneDrive.node.ts b/packages/nodes-base/nodes/Microsoft/OneDrive/MicrosoftOneDrive.node.ts index 052450eae9..da16900725 100644 --- a/packages/nodes-base/nodes/Microsoft/OneDrive/MicrosoftOneDrive.node.ts +++ b/packages/nodes-base/nodes/Microsoft/OneDrive/MicrosoftOneDrive.node.ts @@ -1,5 +1,4 @@ import { - BINARY_ENCODING, IExecuteFunctions, } from 'n8n-core'; @@ -192,8 +191,8 @@ export class MicrosoftOneDrive implements INodeType { } const binaryData = (items[i].binary as IBinaryKeyData)[binaryPropertyName]; + const body = await this.helpers.getBinaryDataBuffer(i, binaryPropertyName); - const body = Buffer.from(binaryData.data, BINARY_ENCODING); responseData = await microsoftApiRequest.call(this, 'PUT', `/drive/items/${parentId}:/${fileName || binaryData.fileName}:/content`, body, {}, undefined, { 'Content-Type': binaryData.mimeType, 'Content-length': body.length }, {} ); returnData.push(JSON.parse(responseData) as IDataObject); diff --git a/packages/nodes-base/nodes/Microsoft/Outlook/MicrosoftOutlook.node.ts b/packages/nodes-base/nodes/Microsoft/Outlook/MicrosoftOutlook.node.ts index 86cf504b15..4a47128c88 100644 --- a/packages/nodes-base/nodes/Microsoft/Outlook/MicrosoftOutlook.node.ts +++ b/packages/nodes-base/nodes/Microsoft/Outlook/MicrosoftOutlook.node.ts @@ -648,7 +648,7 @@ export class MicrosoftOutlook implements INodeType { } const binaryData = (items[i].binary as IBinaryKeyData)[binaryPropertyName]; - const dataBuffer = Buffer.from(binaryData.data, 'base64'); + const dataBuffer = await this.helpers.getBinaryDataBuffer(i, binaryPropertyName); const fileName = additionalFields.fileName === undefined ? binaryData.fileName : additionalFields.fileName; diff --git a/packages/nodes-base/nodes/Mindee/Mindee.node.ts b/packages/nodes-base/nodes/Mindee/Mindee.node.ts index 2b23ee27ff..d6e437a364 100644 --- a/packages/nodes-base/nodes/Mindee/Mindee.node.ts +++ b/packages/nodes-base/nodes/Mindee/Mindee.node.ts @@ -1,5 +1,4 @@ import { - BINARY_ENCODING, IExecuteFunctions, } from 'n8n-core'; @@ -140,6 +139,7 @@ export class Mindee implements INodeType { const item = items[i].binary as IBinaryKeyData; const binaryData = item[binaryPropertyName] as IBinaryData; + const dataBuffer = await this.helpers.getBinaryDataBuffer(i, binaryPropertyName); if (binaryData === undefined) { throw new NodeOperationError(this.getNode(), `No binary data property "${binaryPropertyName}" does not exists on item!`); @@ -154,7 +154,7 @@ export class Mindee implements INodeType { { formData: { file: { - value: Buffer.from(binaryData.data, BINARY_ENCODING), + value: dataBuffer, options: { filename: binaryData.fileName, }, @@ -182,6 +182,7 @@ export class Mindee implements INodeType { const item = items[i].binary as IBinaryKeyData; const binaryData = item[binaryPropertyName] as IBinaryData; + const dataBuffer = await this.helpers.getBinaryDataBuffer(i, binaryPropertyName); if (binaryData === undefined) { throw new NodeOperationError(this.getNode(), `No binary data property "${binaryPropertyName}" does not exists on item!`); @@ -196,7 +197,7 @@ export class Mindee implements INodeType { { formData: { file: { - value: Buffer.from(binaryData.data, BINARY_ENCODING), + value: dataBuffer, options: { filename: binaryData.fileName, }, diff --git a/packages/nodes-base/nodes/NocoDB/NocoDB.node.ts b/packages/nodes-base/nodes/NocoDB/NocoDB.node.ts index a32eb12db8..f42ab38ea0 100644 --- a/packages/nodes-base/nodes/NocoDB/NocoDB.node.ts +++ b/packages/nodes-base/nodes/NocoDB/NocoDB.node.ts @@ -1,5 +1,4 @@ import { - BINARY_ENCODING, IExecuteFunctions, } from 'n8n-core'; @@ -160,10 +159,11 @@ export class NocoDB implements INodeType { throw new NodeOperationError(this.getNode(), `Binary property ${binaryPropertyName} does not exist on item!`); } const binaryData = items[i].binary![binaryPropertyName] as IBinaryData; + const dataBuffer = await this.helpers.getBinaryDataBuffer(i, binaryPropertyName); const formData = { file: { - value: Buffer.from(binaryData.data, BINARY_ENCODING), + value: dataBuffer, options: { filename: binaryData.fileName, contentType: binaryData.mimeType, @@ -338,10 +338,11 @@ export class NocoDB implements INodeType { throw new NodeOperationError(this.getNode(), `Binary property ${binaryPropertyName} does not exist on item!`); } const binaryData = items[i].binary![binaryPropertyName] as IBinaryData; + const dataBuffer = await this.helpers.getBinaryDataBuffer(i, binaryPropertyName); const formData = { file: { - value: Buffer.from(binaryData.data, BINARY_ENCODING), + value: dataBuffer, options: { filename: binaryData.fileName, contentType: binaryData.mimeType, diff --git a/packages/nodes-base/nodes/Pushbullet/Pushbullet.node.ts b/packages/nodes-base/nodes/Pushbullet/Pushbullet.node.ts index 718a5f8a5b..1e57ab093c 100644 --- a/packages/nodes-base/nodes/Pushbullet/Pushbullet.node.ts +++ b/packages/nodes-base/nodes/Pushbullet/Pushbullet.node.ts @@ -1,5 +1,4 @@ import { - BINARY_ENCODING, IExecuteFunctions, } from 'n8n-core'; @@ -495,6 +494,7 @@ export class Pushbullet implements INodeType { } const binaryData = (items[i].binary as IBinaryKeyData)[binaryPropertyName]; + const dataBuffer = await this.helpers.getBinaryDataBuffer(i, binaryPropertyName); //create upload url const { @@ -523,7 +523,7 @@ export class Pushbullet implements INodeType { { formData: { file: { - value: Buffer.from(binaryData.data, BINARY_ENCODING), + value: dataBuffer, options: { filename: binaryData.fileName, }, diff --git a/packages/nodes-base/nodes/Pushover/Pushover.node.ts b/packages/nodes-base/nodes/Pushover/Pushover.node.ts index b6b56f1459..deea02da99 100644 --- a/packages/nodes-base/nodes/Pushover/Pushover.node.ts +++ b/packages/nodes-base/nodes/Pushover/Pushover.node.ts @@ -1,5 +1,4 @@ import { - BINARY_ENCODING, IExecuteFunctions, } from 'n8n-core'; @@ -366,8 +365,10 @@ export class Pushover implements INodeType { throw new NodeOperationError(this.getNode(), `No binary data property "${binaryPropertyName}" does not exists on item!`); } + const dataBuffer = await this.helpers.getBinaryDataBuffer(i, binaryPropertyName); + body.attachment = { - value: Buffer.from(binaryData.data, BINARY_ENCODING), + value: dataBuffer, options: { filename: binaryData.fileName, }, diff --git a/packages/nodes-base/nodes/Raindrop/Raindrop.node.ts b/packages/nodes-base/nodes/Raindrop/Raindrop.node.ts index 036d3e8649..d208ef0093 100644 --- a/packages/nodes-base/nodes/Raindrop/Raindrop.node.ts +++ b/packages/nodes-base/nodes/Raindrop/Raindrop.node.ts @@ -1,5 +1,4 @@ import { - BINARY_ENCODING, IExecuteFunctions, } from 'n8n-core'; @@ -345,10 +344,11 @@ export class Raindrop implements INodeType { const binaryPropertyName = updateFields.cover as string; const binaryData = items[i].binary![binaryPropertyName] as IBinaryData; + const dataBuffer = await this.helpers.getBinaryDataBuffer(i, binaryPropertyName); const formData = { cover: { - value: Buffer.from(binaryData.data, BINARY_ENCODING), + value: dataBuffer, options: { filename: binaryData.fileName, contentType: binaryData.mimeType, diff --git a/packages/nodes-base/nodes/S3/S3.node.ts b/packages/nodes-base/nodes/S3/S3.node.ts index 4b44ff810e..f9cdb0e2a8 100644 --- a/packages/nodes-base/nodes/S3/S3.node.ts +++ b/packages/nodes-base/nodes/S3/S3.node.ts @@ -1,4 +1,3 @@ - import { paramCase, snakeCase, @@ -608,8 +607,7 @@ export class S3 implements INodeType { } const binaryData = (items[i].binary as IBinaryKeyData)[binaryPropertyName]; - - body = Buffer.from(binaryData.data, BINARY_ENCODING) as Buffer; + body = await this.helpers.getBinaryDataBuffer(i, binaryPropertyName); headers['Content-Type'] = binaryData.mimeType; diff --git a/packages/nodes-base/nodes/Salesforce/Salesforce.node.ts b/packages/nodes-base/nodes/Salesforce/Salesforce.node.ts index b8a61e32f6..4c7f45db7e 100644 --- a/packages/nodes-base/nodes/Salesforce/Salesforce.node.ts +++ b/packages/nodes-base/nodes/Salesforce/Salesforce.node.ts @@ -1,5 +1,4 @@ import { - BINARY_ENCODING, IExecuteFunctions, } from 'n8n-core'; @@ -1686,6 +1685,8 @@ export class Salesforce implements INodeType { } if (items[i].binary && items[i].binary![binaryPropertyName]) { const binaryData = items[i].binary![binaryPropertyName]; + const dataBuffer = await this.helpers.getBinaryDataBuffer(i, binaryPropertyName); + body.entity_content['PathOnClient'] = `${title}.${binaryData.fileExtension}`; data = { entity_content: { @@ -1695,7 +1696,7 @@ export class Salesforce implements INodeType { }, }, VersionData: { - value: Buffer.from(binaryData.data, BINARY_ENCODING), + value: dataBuffer, options: { filename: binaryData.fileName, contentType: binaryData.mimeType, diff --git a/packages/nodes-base/nodes/Ssh/Ssh.node.ts b/packages/nodes-base/nodes/Ssh/Ssh.node.ts index 3a3ed5fb30..b136752527 100644 --- a/packages/nodes-base/nodes/Ssh/Ssh.node.ts +++ b/packages/nodes-base/nodes/Ssh/Ssh.node.ts @@ -1,5 +1,4 @@ import { - BINARY_ENCODING, IExecuteFunctions, } from 'n8n-core'; @@ -389,9 +388,11 @@ export class Ssh implements INodeType { throw new Error(`No binary data property "${propertyNameUpload}" does not exists on item!`); } + const dataBuffer = await this.helpers.getBinaryDataBuffer(i, propertyNameUpload); + const { path } = await file({ prefix: 'n8n-ssh-' }); temporaryFiles.push(path); - await writeFile(path, Buffer.from(binaryData.data, BINARY_ENCODING)); + await writeFile(path, dataBuffer); await ssh.putFile(path, `${parameterPath}${(parameterPath.charAt(parameterPath.length - 1) === '/') ? '' : '/'}${fileName || binaryData.fileName}`); diff --git a/packages/nodes-base/nodes/TheHive/TheHive.node.ts b/packages/nodes-base/nodes/TheHive/TheHive.node.ts index 554b22dd20..62894dae66 100644 --- a/packages/nodes-base/nodes/TheHive/TheHive.node.ts +++ b/packages/nodes-base/nodes/TheHive/TheHive.node.ts @@ -1,5 +1,4 @@ import { - BINARY_ENCODING, IExecuteFunctions } from 'n8n-core'; @@ -929,11 +928,12 @@ export class TheHive implements INodeType { } const binaryData = item.binary[binaryPropertyName] as IBinaryData; + const dataBuffer = await this.helpers.getBinaryDataBuffer(i, binaryPropertyName); options = { formData: { attachment: { - value: Buffer.from(binaryData.data, BINARY_ENCODING), + value: dataBuffer, options: { contentType: binaryData.mimeType, filename: binaryData.fileName, @@ -1885,11 +1885,12 @@ export class TheHive implements INodeType { } const binaryData = item.binary[binaryPropertyName] as IBinaryData; + const dataBuffer = await this.helpers.getBinaryDataBuffer(i, binaryPropertyName); options = { formData: { attachment: { - value: Buffer.from(binaryData.data, BINARY_ENCODING), + value: dataBuffer, options: { contentType: binaryData.mimeType, filename: binaryData.fileName, diff --git a/packages/nodes-base/nodes/Twist/Twist.node.ts b/packages/nodes-base/nodes/Twist/Twist.node.ts index c749e2b577..647dd54812 100644 --- a/packages/nodes-base/nodes/Twist/Twist.node.ts +++ b/packages/nodes-base/nodes/Twist/Twist.node.ts @@ -1,5 +1,4 @@ import { - BINARY_ENCODING, IExecuteFunctions, } from 'n8n-core'; @@ -278,6 +277,8 @@ export class Twist implements INodeType { throw new Error(`No binary data property "${binaryProperty}" does not exists on item!`); } + const dataBuffer = await this.helpers.getBinaryDataBuffer(i, binaryProperty); + attachments.push(await twistApiRequest.call( this, 'POST', @@ -287,7 +288,7 @@ export class Twist implements INodeType { { formData: { file_name: { - value: Buffer.from(binaryData.data, BINARY_ENCODING), + value: dataBuffer, options: { filename: binaryData.fileName, }, @@ -380,6 +381,8 @@ export class Twist implements INodeType { throw new Error(`No binary data property "${binaryProperty}" does not exists on item!`); } + const dataBuffer = await this.helpers.getBinaryDataBuffer(i, binaryProperty); + attachments.push(await twistApiRequest.call( this, 'POST', @@ -389,7 +392,7 @@ export class Twist implements INodeType { { formData: { file_name: { - value: Buffer.from(binaryData.data, BINARY_ENCODING), + value: dataBuffer, options: { filename: binaryData.fileName, }, @@ -452,6 +455,8 @@ export class Twist implements INodeType { throw new NodeOperationError(this.getNode(), `No binary data property "${binaryProperty}" does not exists on item!`); } + const dataBuffer = await this.helpers.getBinaryDataBuffer(i, binaryProperty); + attachments.push(await twistApiRequest.call( this, 'POST', @@ -461,7 +466,7 @@ export class Twist implements INodeType { { formData: { file_name: { - value: Buffer.from(binaryData.data, BINARY_ENCODING), + value: dataBuffer, options: { filename: binaryData.fileName, }, @@ -547,6 +552,8 @@ export class Twist implements INodeType { throw new Error(`No binary data property "${binaryProperty}" does not exists on item!`); } + const dataBuffer = await this.helpers.getBinaryDataBuffer(i, binaryProperty); + attachments.push(await twistApiRequest.call( this, 'POST', @@ -556,7 +563,7 @@ export class Twist implements INodeType { { formData: { file_name: { - value: Buffer.from(binaryData.data, BINARY_ENCODING), + value: dataBuffer, options: { filename: binaryData.fileName, }, @@ -619,6 +626,8 @@ export class Twist implements INodeType { throw new Error(`No binary data property "${binaryProperty}" does not exists on item!`); } + const dataBuffer = await this.helpers.getBinaryDataBuffer(i, binaryProperty); + attachments.push(await twistApiRequest.call( this, 'POST', @@ -628,7 +637,7 @@ export class Twist implements INodeType { { formData: { file_name: { - value: Buffer.from(binaryData.data, BINARY_ENCODING), + value: dataBuffer, options: { filename: binaryData.fileName, }, @@ -720,6 +729,8 @@ export class Twist implements INodeType { throw new Error(`No binary data property "${binaryProperty}" does not exists on item!`); } + const dataBuffer = await this.helpers.getBinaryDataBuffer(i, binaryProperty); + attachments.push(await twistApiRequest.call( this, 'POST', @@ -729,7 +740,7 @@ export class Twist implements INodeType { { formData: { file_name: { - value: Buffer.from(binaryData.data, BINARY_ENCODING), + value: dataBuffer, options: { filename: binaryData.fileName, }, diff --git a/packages/nodes-base/nodes/Twitter/GenericFunctions.ts b/packages/nodes-base/nodes/Twitter/GenericFunctions.ts index 260f2ac3ca..9df93ced2b 100644 --- a/packages/nodes-base/nodes/Twitter/GenericFunctions.ts +++ b/packages/nodes-base/nodes/Twitter/GenericFunctions.ts @@ -3,7 +3,6 @@ import { } from 'request'; import { - BINARY_ENCODING, IExecuteFunctions, IExecuteSingleFunctions, IHookFunctions, @@ -96,7 +95,8 @@ export async function uploadAttachments(this: IExecuteFunctions, binaryPropertie let attachmentBody = {}; let response: IDataObject = {}; - const isAnimatedWebp = (Buffer.from(binaryData[binaryPropertyName].data, 'base64').toString().indexOf('ANMF') !== -1); + const dataBuffer = await this.helpers.getBinaryDataBuffer(i, binaryPropertyName); + const isAnimatedWebp = (dataBuffer.toString().indexOf('ANMF') !== -1); const isImage = binaryData[binaryPropertyName].mimeType.includes('image'); @@ -118,9 +118,11 @@ export async function uploadAttachments(this: IExecuteFunctions, binaryPropertie // https://developer.twitter.com/en/docs/media/upload-media/api-reference/post-media-upload-init + const dataBuffer = await this.helpers.getBinaryDataBuffer(i, binaryPropertyName); + attachmentBody = { command: 'INIT', - total_bytes: Buffer.from(binaryData[binaryPropertyName].data, BINARY_ENCODING).byteLength, + total_bytes: dataBuffer.byteLength, media_type: binaryData[binaryPropertyName].mimeType, }; @@ -130,7 +132,7 @@ export async function uploadAttachments(this: IExecuteFunctions, binaryPropertie // break the data on 5mb chunks (max size that can be uploaded at once) - const binaryParts = chunks(Buffer.from(binaryData[binaryPropertyName].data, BINARY_ENCODING), 5242880); + const binaryParts = chunks(dataBuffer, 5242880); let index = 0; diff --git a/packages/nodes-base/nodes/WriteBinaryFile.node.ts b/packages/nodes-base/nodes/WriteBinaryFile.node.ts index db1b6371ed..a9e52696c6 100644 --- a/packages/nodes-base/nodes/WriteBinaryFile.node.ts +++ b/packages/nodes-base/nodes/WriteBinaryFile.node.ts @@ -1,7 +1,5 @@ import { - BINARY_ENCODING, IExecuteFunctions, - IExecuteSingleFunctions } from 'n8n-core'; import { IDataObject, @@ -76,14 +74,16 @@ export class WriteBinaryFile implements INodeType { throw new NodeOperationError(this.getNode(), `The binary property "${dataPropertyName}" does not exist. So no file can be written!`); } - // Write the file to disk - await fsWriteFile(fileName, Buffer.from(item.binary[dataPropertyName].data, BINARY_ENCODING), 'binary'); - const newItem: INodeExecutionData = { json: {}, }; Object.assign(newItem.json, item.json); + const binaryDataBuffer = await this.helpers.getBinaryDataBuffer(itemIndex, dataPropertyName); + + // Write the file to disk + await fsWriteFile(fileName, binaryDataBuffer, 'binary'); + if (item.binary !== undefined) { // Create a shallow copy of the binary data so that the old // data references which do not get changed still stay behind @@ -100,7 +100,7 @@ export class WriteBinaryFile implements INodeType { } catch (error) { if (this.continueOnFail()) { - returnData.push({json:{ error: error.message }}); + returnData.push({ json: { error: error.message } }); continue; } throw error;