🐛 Binary data handling fixes (#2629)

* Update node airtable

* Update nodenextcloud

* Update node spreadsheet

* Update node cortex, dropbox, editImage nodes

* Update node emailSend

* Update node ftp

* Update node googleDrive

* Update node googleDrive fix

* Update node youtube

* Update node htmlExtract

* Update node linkedIn

* Update node mailgun

* Update node matrix

* Update node pipedrive

* Update node readPdf

* Update node facebookGraphApi

* Update node httpRequest

* Update node nocoDB

* Update node httpRequest, respondToWebhook

* Update node signi4

* Update node slack

* Update node zulip

* cleanup

* fix generic funcs

* 🐛 Fix EditImage Node

Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
This commit is contained in:
Ahsan Virani 2022-01-03 22:42:42 +01:00 committed by GitHub
parent aff93480d4
commit 224ef736de
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
27 changed files with 66 additions and 117 deletions

View file

@ -126,11 +126,7 @@ export async function downloadRecordAttachments(this: IExecuteFunctions | IPollF
if (record.fields[fieldName] !== undefined) {
for (const [index, attachment] of (record.fields[fieldName] as IAttachment[]).entries()) {
const file = await apiRequest.call(this, 'GET', '', {}, {}, attachment.url, { json: false, encoding: null });
element.binary![`${fieldName}_${index}`] = {
data: Buffer.from(file).toString('base64'),
fileName: attachment.filename,
mimeType: attachment.type,
};
element.binary![`${fieldName}_${index}`] = await this.helpers.prepareBinaryData(Buffer.from(file), attachment.filename, attachment.type);
}
}
}

View file

@ -1,5 +1,4 @@
import {
BINARY_ENCODING,
IExecuteFunctions,
} from 'n8n-core';
@ -242,7 +241,7 @@ export class Cortex implements INodeType {
throw new NodeOperationError(this.getNode(), `No binary data property "${binaryPropertyName}" does not exists on item!`);
}
const fileBufferData = Buffer.from(item.binary[binaryPropertyName].data, BINARY_ENCODING);
const fileBufferData = await this.helpers.getBinaryDataBuffer(i, binaryPropertyName);
const options = {
formData: {
@ -425,7 +424,7 @@ export class Cortex implements INodeType {
throw new NodeOperationError(this.getNode(), `No binary data property "${binaryPropertyName}" does not exists on item!`);
}
const fileBufferData = Buffer.from(item.binary[binaryPropertyName].data, BINARY_ENCODING);
const fileBufferData = await this.helpers.getBinaryDataBuffer(i, binaryPropertyName);
const sha256 = createHash('sha256').update(fileBufferData).digest('hex');
(body.data as IDataObject).attachment = {

View file

@ -1,7 +1,4 @@
import {
BINARY_ENCODING,
IExecuteFunctions,
} from 'n8n-core';
import { IExecuteFunctions } from 'n8n-core';
import {
IDataObject,
@ -859,7 +856,7 @@ export class Dropbox implements INodeType {
throw new NodeOperationError(this.getNode(), `No binary data property "${propertyNameUpload}" does not exists on item!`);
}
body = Buffer.from(item.binary[propertyNameUpload].data, BINARY_ENCODING);
body = await this.helpers.getBinaryDataBuffer(i, propertyNameUpload);
} else {
// Is text file
body = Buffer.from(this.getNodeParameter('fileContent', i) as string, 'utf8');

View file

@ -1179,7 +1179,8 @@ export class EditImage implements INodeType {
throw new NodeOperationError(this.getNode(), `Item does not contain any binary data with the name "${dataPropertyName}".`);
}
gmInstance = gm(Buffer.from(item.binary![dataPropertyName as string].data, BINARY_ENCODING));
const binaryDataBuffer = await this.helpers.getBinaryDataBuffer(itemIndex, dataPropertyName);
gmInstance = gm(binaryDataBuffer);
gmInstance = gmInstance.background('transparent');
}
@ -1218,7 +1219,8 @@ export class EditImage implements INodeType {
const { fd, path, cleanup } = await file();
cleanupFunctions.push(cleanup);
await fsWriteFileAsync(fd, Buffer.from(item.binary![operationData.dataPropertyNameComposite as string].data, BINARY_ENCODING));
const binaryDataBuffer = await this.helpers.getBinaryDataBuffer(itemIndex, operationData.dataPropertyNameComposite as string);
await fsWriteFileAsync(fd, binaryDataBuffer);
if (operations[0].operation === 'create') {
// It seems like if the image gets created newly we have to create a new gm instance
@ -1349,14 +1351,14 @@ export class EditImage implements INodeType {
returnData.push(await (new Promise<INodeExecutionData>((resolve, reject) => {
gmInstance
.toBuffer((error: Error | null, buffer: Buffer) => {
.toBuffer(async (error: Error | null, buffer: Buffer) => {
cleanupFunctions.forEach(async cleanup => await cleanup());
if (error) {
return reject(error);
}
newItem.binary![dataPropertyName as string].data = buffer.toString(BINARY_ENCODING);
newItem.binary![dataPropertyName as string].data = (await this.helpers.prepareBinaryData(Buffer.from(buffer))).data;
return resolve(newItem);
});

View file

@ -1,7 +1,4 @@
import {
BINARY_ENCODING,
IExecuteFunctions
} from 'n8n-core';
import { IExecuteFunctions } from 'n8n-core';
import {
IDataObject,
INodeExecutionData,
@ -198,7 +195,7 @@ export class EmailSend implements INodeType {
}
attachments.push({
filename: item.binary[propertyName].fileName || 'unknown',
content: Buffer.from(item.binary[propertyName].data, BINARY_ENCODING),
content: await this.helpers.getBinaryDataBuffer(itemIndex, propertyName),
});
}

View file

@ -1,7 +1,4 @@
import {
BINARY_ENCODING,
IExecuteFunctions,
} from 'n8n-core';
import { IExecuteFunctions } from 'n8n-core';
import {
IBinaryData,
IDataObject,
@ -391,9 +388,10 @@ export class FacebookGraphApi implements INodeType {
const binaryProperty = item.binary[binaryPropertyName] as IBinaryData;
const binaryDataBuffer = await this.helpers.getBinaryDataBuffer(itemIndex, binaryPropertyName);
requestOptions.formData = {
[propertyName]: {
value: Buffer.from(binaryProperty.data, BINARY_ENCODING),
value: binaryDataBuffer,
options: {
filename: binaryProperty.fileName,
contentType: binaryProperty.mimeType,

View file

@ -1,7 +1,4 @@
import {
BINARY_ENCODING,
IExecuteFunctions
} from 'n8n-core';
import { IExecuteFunctions } from 'n8n-core';
import {
ICredentialDataDecryptedObject,
IDataObject,
@ -503,7 +500,7 @@ export class Ftp implements INodeType {
throw new NodeOperationError(this.getNode(), `No binary data property "${propertyNameUpload}" does not exists on item!`);
}
const buffer = Buffer.from(item.binary[propertyNameUpload].data, BINARY_ENCODING) as Buffer;
const buffer = await this.helpers.getBinaryDataBuffer(i, propertyNameUpload);
await sftp!.put(buffer, remotePath);
} else {
// Is text file
@ -597,7 +594,7 @@ export class Ftp implements INodeType {
throw new NodeOperationError(this.getNode(), `No binary data property "${propertyNameUpload}" does not exists on item!`);
}
const buffer = Buffer.from(item.binary[propertyNameUpload].data, BINARY_ENCODING) as Buffer;
const buffer = await this.helpers.getBinaryDataBuffer(i, propertyNameUpload);
try {
await ftp!.put(buffer, remotePath);

View file

@ -1,7 +1,4 @@
import {
BINARY_ENCODING,
IExecuteFunctions,
} from 'n8n-core';
import { IExecuteFunctions } from 'n8n-core';
import {
IDataObject,
@ -2423,7 +2420,7 @@ export class GoogleDrive implements INodeType {
originalFilename = item.binary[propertyNameUpload].fileName;
}
body = Buffer.from(item.binary[propertyNameUpload].data, BINARY_ENCODING);
body = await this.helpers.getBinaryDataBuffer(i, propertyNameUpload);
} else {
// Is text file
body = Buffer.from(this.getNodeParameter('fileContent', i) as string, 'utf8');

View file

@ -1,7 +1,4 @@
import {
BINARY_ENCODING,
IExecuteFunctions,
} from 'n8n-core';
import { IExecuteFunctions } from 'n8n-core';
import {
IDataObject,
@ -426,7 +423,7 @@ export class YouTube implements INodeType {
mimeType = item.binary[binaryProperty].mimeType;
}
const body = Buffer.from(item.binary[binaryProperty].data, BINARY_ENCODING);
const body = await this.helpers.getBinaryDataBuffer(i, binaryProperty);
const requestOptions = {
headers: {
@ -913,7 +910,7 @@ export class YouTube implements INodeType {
mimeType = item.binary[binaryProperty].mimeType;
}
const body = Buffer.from(item.binary[binaryProperty].data, BINARY_ENCODING);
const body = await this.helpers.getBinaryDataBuffer(i, binaryProperty);
const requestOptions = {
headers: {

View file

@ -240,7 +240,9 @@ export class HtmlExtract implements INodeType {
if (item.binary[dataPropertyName] === undefined) {
throw new NodeOperationError(this.getNode(), `No property named "${dataPropertyName}" exists!`);
}
htmlArray = Buffer.from(item.binary[dataPropertyName].data, 'base64').toString('utf8');
const binaryDataBuffer = await this.helpers.getBinaryDataBuffer(itemIndex, dataPropertyName);
htmlArray = binaryDataBuffer.toString('utf-8');
}
// Convert it always to array that it works with a string or an array of strings

View file

@ -1,5 +1,4 @@
import {
BINARY_ENCODING,
IExecuteFunctions,
} from 'n8n-core';
import {
@ -771,8 +770,9 @@ export class HttpRequest implements INodeType {
if (item.binary[binaryPropertyName] === undefined) {
throw new NodeOperationError(this.getNode(), `No binary data property "${binaryPropertyName}" does not exists on item!`);
}
const binaryProperty = item.binary[binaryPropertyName] as IBinaryData;
requestOptions.body = Buffer.from(binaryProperty.data, BINARY_ENCODING);
const binaryDataBuffer = await this.helpers.getBinaryDataBuffer(itemIndex, binaryPropertyName);
requestOptions.body = binaryDataBuffer;
} else if (options.bodyContentType === 'multipart-form-data') {
requestOptions.body = {};
const binaryPropertyNameFull = this.getNodeParameter('binaryPropertyName', itemIndex) as string;
@ -794,9 +794,10 @@ export class HttpRequest implements INodeType {
}
const binaryProperty = item.binary[binaryPropertyName] as IBinaryData;
const binaryDataBuffer = await this.helpers.getBinaryDataBuffer(itemIndex, binaryPropertyName);
requestOptions.body[propertyName] = {
value: Buffer.from(binaryProperty.data, BINARY_ENCODING),
value: binaryDataBuffer,
options: {
filename: binaryProperty.fileName,
contentType: binaryProperty.mimeType,

View file

@ -1,7 +1,4 @@
import {
BINARY_ENCODING,
IExecuteFunctions,
} from 'n8n-core';
import { IExecuteFunctions } from 'n8n-core';
import {
IDataObject,
ILoadOptionsFunctions,
@ -149,7 +146,7 @@ export class LinkedIn implements INodeType {
}
// Buffer binary data
const buffer = Buffer.from(item.binary[propertyNameUpload].data, BINARY_ENCODING) as Buffer;
const buffer = await this.helpers.getBinaryDataBuffer(i, propertyNameUpload);
// Upload image
await linkedInApiRequest.call(this, 'POST', uploadUrl, buffer, true);

View file

@ -1,7 +1,4 @@
import {
BINARY_ENCODING,
IExecuteFunctions,
} from 'n8n-core';
import { IExecuteFunctions } from 'n8n-core';
import {
IDataObject,
INodeExecutionData,
@ -158,8 +155,9 @@ export class Mailgun implements INodeType {
if (!item.binary.hasOwnProperty(propertyName)) {
continue;
}
const binaryDataBuffer = await this.helpers.getBinaryDataBuffer(itemIndex, propertyName);
attachments.push({
value: Buffer.from(item.binary[propertyName].data, BINARY_ENCODING),
value: binaryDataBuffer,
options: {
filename: item.binary[propertyName].fileName || 'unknown',

View file

@ -5,7 +5,6 @@ import {
import { IDataObject, NodeApiError, NodeOperationError, } from 'n8n-workflow';
import {
BINARY_ENCODING,
IExecuteFunctions,
IExecuteSingleFunctions,
ILoadOptionsFunctions,
@ -69,7 +68,7 @@ export async function matrixApiRequest(this: IExecuteFunctions | IExecuteSingleF
}
}
export async function handleMatrixCall(this: IExecuteFunctions | IExecuteSingleFunctions | ILoadOptionsFunctions, item: IDataObject, index: number, resource: string, operation: string): Promise<any> { // tslint:disable-line:no-any
export async function handleMatrixCall(this: IExecuteFunctions, item: IDataObject, index: number, resource: string, operation: string): Promise<any> { // tslint:disable-line:no-any
if (resource === 'account') {
if (operation === 'me') {
@ -193,13 +192,12 @@ export async function handleMatrixCall(this: IExecuteFunctions | IExecuteSingleF
throw new NodeOperationError(this.getNode(), `No binary data property "${binaryPropertyName}" does not exists on item!`);
}
//@ts-ignore
// @ts-ignore
qs.filename = item.binary[binaryPropertyName].fileName;
//@ts-ignore
filename = item.binary[binaryPropertyName].fileName;
//@ts-ignore
body = Buffer.from(item.binary[binaryPropertyName].data, BINARY_ENCODING);
body = await this.helpers.getBinaryDataBuffer(index, binaryPropertyName);
//@ts-ignore
headers['Content-Type'] = item.binary[binaryPropertyName].mimeType;
headers['accept'] = 'application/json,text/*;q=0.99';

View file

@ -1,7 +1,4 @@
import {
BINARY_ENCODING,
IExecuteFunctions,
} from 'n8n-core';
import { IExecuteFunctions } from 'n8n-core';
import {
IDataObject,
@ -1066,7 +1063,7 @@ export class NextCloud implements INodeType {
throw new NodeOperationError(this.getNode(), `No binary data property "${propertyNameUpload}" does not exists on item!`);
}
body = Buffer.from(item.binary[propertyNameUpload].data, BINARY_ENCODING);
body = await this.helpers.getBinaryDataBuffer(i, propertyNameUpload);
} else {
// Is text file
body = this.getNodeParameter('fileContent', i) as string;

View file

@ -118,11 +118,7 @@ export async function downloadRecordAttachments(this: IExecuteFunctions | IPollF
if (record[fieldName]) {
for (const [index, attachment] of (JSON.parse(record[fieldName] as string) as IAttachment[]).entries()) {
const file = await apiRequest.call(this, 'GET', '', {}, {}, attachment.url, { json: false, encoding: null });
element.binary![`${fieldName}_${index}`] = {
data: Buffer.from(file).toString('base64'),
fileName: attachment.title,
mimeType: attachment.mimetype,
};
element.binary![`${fieldName}_${index}`] = await this.helpers.prepareBinaryData(Buffer.from(file), attachment.title, attachment.mimetype);
}
}
}

View file

@ -1,5 +1,4 @@
import {
BINARY_ENCODING,
IExecuteFunctions,
ILoadOptionsFunctions,
} from 'n8n-core';
@ -4500,7 +4499,7 @@ export class Pipedrive implements INodeType {
throw new NodeOperationError(this.getNode(), `No binary data property "${binaryPropertyName}" does not exists on item!`);
}
const fileBufferData = Buffer.from(item.binary[binaryPropertyName].data, BINARY_ENCODING);
const fileBufferData = await this.helpers.getBinaryDataBuffer(i, binaryPropertyName);
formData.file = {
value: fileBufferData,

View file

@ -1,7 +1,4 @@
import {
BINARY_ENCODING,
IExecuteFunctions,
} from 'n8n-core';
import { IExecuteFunctions } from 'n8n-core';
import {
INodeExecutionData,
@ -55,7 +52,7 @@ export class ReadPdf implements INodeType {
item.binary = {};
}
const binaryData = Buffer.from(item.binary[binaryPropertyName].data, BINARY_ENCODING);
const binaryData = await this.helpers.getBinaryDataBuffer(itemIndex, binaryPropertyName);
returnData.push({
binary: item.binary,
json: await pdf(binaryData),

View file

@ -1,10 +1,9 @@
import {
BINARY_ENCODING,
IExecuteFunctions,
} from 'n8n-core';
import {
IDataObject,
IExecuteFunctions,
IN8nHttpFullResponse,
IN8nHttpResponse,
INodeExecutionData,
@ -202,7 +201,7 @@ export class RespondToWebhook implements INodeType {
],
};
execute(this: IExecuteFunctions): Promise<INodeExecutionData[][]> {
async execute(this: IExecuteFunctions): Promise<INodeExecutionData[][]> {
const items = this.getInputData();
const respondWith = this.getNodeParameter('respondWith', 0) as string;
@ -250,6 +249,7 @@ export class RespondToWebhook implements INodeType {
}
const binaryData = item.binary[responseBinaryPropertyName];
const binaryDataBuffer = await this.helpers.getBinaryDataBuffer(0, responseBinaryPropertyName);
if (binaryData === undefined) {
throw new NodeOperationError(this.getNode(), `No binary data property "${responseBinaryPropertyName}" does not exists on item!`);
@ -258,7 +258,7 @@ export class RespondToWebhook implements INodeType {
if (headers['content-type']) {
headers['content-type'] = binaryData.mimeType;
}
responseBody = Buffer.from(binaryData.data, BINARY_ENCODING);
responseBody = binaryDataBuffer;
} else if (respondWith !== 'noData') {
throw new NodeOperationError(this.getNode(), `The Response Data option "${respondWith}" is not supported!`);
}

View file

@ -3,18 +3,11 @@ import {
snakeCase,
} from 'change-case';
import {
createHash,
} from 'crypto';
import { createHash } from 'crypto';
import {
Builder,
} from 'xml2js';
import { Builder } from 'xml2js';
import {
BINARY_ENCODING,
IExecuteFunctions,
} from 'n8n-core';
import { IExecuteFunctions } from 'n8n-core';
import {
IBinaryKeyData,

View file

@ -1,7 +1,4 @@
import {
BINARY_ENCODING,
IExecuteFunctions,
} from 'n8n-core';
import { IExecuteFunctions } from 'n8n-core';
import {
IDataObject,

View file

@ -1,7 +1,4 @@
import {
BINARY_ENCODING,
IExecuteFunctions,
} from 'n8n-core';
import { IExecuteFunctions } from 'n8n-core';
import {
IBinaryKeyData,
@ -312,8 +309,9 @@ export class Signl4 implements INodeType {
throw new NodeOperationError(this.getNode(), `Invalid extension, just ${supportedFileExtension.join(',')} are supported}`);
}
const binaryDataBuffer = await this.helpers.getBinaryDataBuffer(i, propertyName);
data.attachment = {
value: Buffer.from(binaryProperty.data, BINARY_ENCODING),
value: binaryDataBuffer,
options: {
filename: binaryProperty.fileName,
contentType: binaryProperty.mimeType,

View file

@ -1,7 +1,4 @@
import {
BINARY_ENCODING,
IExecuteFunctions,
} from 'n8n-core';
import { IExecuteFunctions } from 'n8n-core';
import {
ICredentialsDecrypted,
@ -977,9 +974,10 @@ export class Slack implements INodeType {
|| items[i].binary[binaryPropertyName] === undefined) {
throw new NodeOperationError(this.getNode(), `No binary data property "${binaryPropertyName}" does not exists on item!`);
}
const binaryDataBuffer = await this.helpers.getBinaryDataBuffer(i, binaryPropertyName);
body.file = {
//@ts-ignore
value: Buffer.from(items[i].binary[binaryPropertyName].data, BINARY_ENCODING),
value: binaryDataBuffer,
options: {
//@ts-ignore
filename: items[i].binary[binaryPropertyName].fileName,

View file

@ -1,7 +1,4 @@
import {
BINARY_ENCODING,
IExecuteFunctions,
} from 'n8n-core';
import { IExecuteFunctions } from 'n8n-core';
import {
IDataObject,
@ -346,7 +343,7 @@ export class SpreadsheetFile implements INodeType {
}
// Read the binary spreadsheet data
const binaryData = Buffer.from(item.binary[binaryPropertyName].data, BINARY_ENCODING);
const binaryData = await this.helpers.getBinaryDataBuffer(i, binaryPropertyName);
let workbook;
if (options.readAsString === true) {
workbook = xlsxRead(binaryData.toString(), { type: 'string', raw: options.rawData as boolean });

View file

@ -1,5 +1,4 @@
import {
BINARY_ENCODING,
IExecuteFunctions,
IHookFunctions,
ILoadOptionsFunctions,

View file

@ -1,4 +1,5 @@
import {
BINARY_ENCODING,
IExecuteFunctions,
WAIT_TIME_UNLIMITED,
} from 'n8n-core';

View file

@ -1,5 +1,4 @@
import {
BINARY_ENCODING,
IExecuteFunctions,
} from 'n8n-core';
import {
@ -217,10 +216,12 @@ export class Zulip implements INodeType {
if (items[i].binary[binaryProperty] === undefined) {
throw new NodeOperationError(this.getNode(), `No binary data property "${binaryProperty}" does not exists on item!`);
}
const binaryDataBuffer = await this.helpers.getBinaryDataBuffer(i, binaryProperty);
const formData = {
file: {
//@ts-ignore
value: Buffer.from(items[i].binary[binaryProperty].data, BINARY_ENCODING),
value: binaryDataBuffer,
options: {
//@ts-ignore
filename: items[i].binary[binaryProperty].fileName,