n8n/packages/nodes-base/nodes/S3/S3.node.ts

643 lines
23 KiB
TypeScript
Raw Normal View History

2020-08-18 05:56:53 -07:00
import {
paramCase,
snakeCase,
2020-08-18 05:56:53 -07:00
} from 'change-case';
import {
createHash,
} from 'crypto';
import {
Builder,
} from 'xml2js';
import {
BINARY_ENCODING,
IExecuteFunctions,
} from 'n8n-core';
import {
IBinaryKeyData,
IDataObject,
INodeExecutionData,
INodeType,
INodeTypeDescription,
:sparkles: Improve node error handling (#1309) * Add path mapping and response error interfaces * Add error handling and throwing functionality * Refactor error handling into a single function * Re-implement error handling in Hacker News node * Fix linting details * Re-implement error handling in Spotify node * Re-implement error handling in G Suite Admin node * :construction: create basic setup NodeError * :construction: add httpCodes * :construction: add path priolist * :construction: handle statusCode in error, adjust interfaces * :construction: fixing type issues w/Ivan * :construction: add error exploration * 👔 fix linter issues * :wrench: improve object check * :construction: remove path passing from NodeApiError * :construction: add multi error + refactor findProperty method * 👔 allow any * :wrench: handle multi error message callback * :zap: change return type of callback * :zap: add customCallback to MultiError * :construction: refactor to use INode * :hammer: handle arrays, continue search after first null property found * 🚫 refactor method access * :construction: setup NodeErrorView * :zap: change timestamp to Date.now * :books: Add documentation for methods and constants * :construction: change message setting * 🚚 move NodeErrors to workflow * :sparkles: add new ErrorView for Nodes * :art: improve error notification * :art: refactor interfaces * :zap: add WorkflowOperationError, refactor error throwing * 👕 fix linter issues * :art: rename param * :bug: fix handling normal errors * :zap: add usage of NodeApiError * :art: fix throw new error instead of constructor * :art: remove unnecessary code/comments * :art: adjusted spacing + updated status messages * :art: fix tab indentation * ✨ Replace current errors with custom errors (#1576) * :zap: Introduce NodeApiError in catch blocks * :zap: Introduce NodeOperationError in nodes * :zap: Add missing errors and remove incompatible * :zap: Fix NodeOperationError in incompatible nodes * :wrench: Adjust error handling in missed nodes PayPal, FileMaker, Reddit, Taiga and Facebook Graph API nodes * :hammer: Adjust Strava Trigger node error handling * :hammer: Adjust AWS nodes error handling * :hammer: Remove duplicate instantiation of NodeApiError * :bug: fix strava trigger node error handling * Add XML parsing to NodeApiError constructor (#1633) * :bug: Remove type annotation from catch variable * :sparkles: Add XML parsing to NodeApiError * :zap: Simplify error handling in Rekognition node * :zap: Pass in XML flag in generic functions * :fire: Remove try/catch wrappers at call sites * :hammer: Refactor setting description from XML * :hammer: Refactor let to const in resource loaders * :zap: Find property in parsed XML * :zap: Change let to const * :fire: Remove unneeded try/catch block * :shirt: Fix linting issues * :bug: Fix errors from merge conflict resolution * :zap: Add custom errors to latest contributions * :shirt: Fix linting issues * :zap: Refactor MongoDB helpers for custom errors * :bug: Correct custom error type * :zap: Apply feedback to A nodes * :zap: Apply feedback to missed A node * :zap: Apply feedback to B-D nodes * :zap: Apply feedback to E-F nodes * :zap: Apply feedback to G nodes * :zap: Apply feedback to H-L nodes * :zap: Apply feedback to M nodes * :zap: Apply feedback to P nodes * :zap: Apply feedback to R nodes * :zap: Apply feedback to S nodes * :zap: Apply feedback to T nodes * :zap: Apply feedback to V-Z nodes * :zap: Add HTTP code to iterable node error * :hammer: Standardize e as error * :hammer: Standardize err as error * :zap: Fix error handling for non-standard nodes Co-authored-by: Ben Hesseldieck <b.hesseldieck@gmail.com> Co-authored-by: Ben Hesseldieck <b.hesseldieck@gmail.com> Co-authored-by: Ben Hesseldieck <1849459+BHesseldieck@users.noreply.github.com>
2021-04-16 09:33:36 -07:00
NodeApiError,
NodeOperationError,
2020-08-18 05:56:53 -07:00
} from 'n8n-workflow';
import {
bucketFields,
bucketOperations,
2020-09-01 08:51:45 -07:00
} from '../Aws/S3/BucketDescription';
2020-08-18 05:56:53 -07:00
import {
folderFields,
folderOperations,
2020-09-01 08:51:45 -07:00
} from '../Aws/S3/FolderDescription';
2020-08-18 05:56:53 -07:00
import {
fileFields,
fileOperations,
2020-09-01 08:51:45 -07:00
} from '../Aws/S3/FileDescription';
2020-08-18 05:56:53 -07:00
import {
s3ApiRequestREST,
s3ApiRequestSOAP,
s3ApiRequestSOAPAllItems,
} from './GenericFunctions';
export class S3 implements INodeType {
description: INodeTypeDescription = {
displayName: 'S3',
2020-09-01 23:30:01 -07:00
name: 's3',
icon: 'file:s3.png',
2020-08-18 05:56:53 -07:00
group: ['output'],
version: 1,
subtitle: '={{$parameter["operation"] + ": " + $parameter["resource"]}}',
2020-09-01 23:30:01 -07:00
description: 'Sends data to any S3-compatible service',
2020-08-18 05:56:53 -07:00
defaults: {
name: 'S3',
color: '#d05b4b',
},
inputs: ['main'],
outputs: ['main'],
credentials: [
{
2020-09-01 23:30:01 -07:00
name: 's3',
2020-08-18 05:56:53 -07:00
required: true,
},
],
properties: [
{
displayName: 'Resource',
name: 'resource',
type: 'options',
options: [
{
name: 'Bucket',
value: 'bucket',
},
{
name: 'File',
value: 'file',
},
{
name: 'Folder',
value: 'folder',
},
],
default: 'file',
description: 'The operation to perform.',
},
// BUCKET
...bucketOperations,
...bucketFields,
// FOLDER
...folderOperations,
...folderFields,
// UPLOAD
...fileOperations,
...fileFields,
],
};
async execute(this: IExecuteFunctions): Promise<INodeExecutionData[][]> {
const items = this.getInputData();
const returnData: IDataObject[] = [];
const qs: IDataObject = {};
let responseData;
const resource = this.getNodeParameter('resource', 0) as string;
const operation = this.getNodeParameter('operation', 0) as string;
for (let i = 0; i < items.length; i++) {
2020-12-28 13:59:35 -08:00
const headers: IDataObject = {};
2020-08-18 05:56:53 -07:00
if (resource === 'bucket') {
//https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateBucket.html
if (operation === 'create') {
let credentials;
try {
2020-09-01 23:30:01 -07:00
credentials = this.getCredentials('s3');
2020-08-18 05:56:53 -07:00
} catch (error) {
:sparkles: Improve node error handling (#1309) * Add path mapping and response error interfaces * Add error handling and throwing functionality * Refactor error handling into a single function * Re-implement error handling in Hacker News node * Fix linting details * Re-implement error handling in Spotify node * Re-implement error handling in G Suite Admin node * :construction: create basic setup NodeError * :construction: add httpCodes * :construction: add path priolist * :construction: handle statusCode in error, adjust interfaces * :construction: fixing type issues w/Ivan * :construction: add error exploration * 👔 fix linter issues * :wrench: improve object check * :construction: remove path passing from NodeApiError * :construction: add multi error + refactor findProperty method * 👔 allow any * :wrench: handle multi error message callback * :zap: change return type of callback * :zap: add customCallback to MultiError * :construction: refactor to use INode * :hammer: handle arrays, continue search after first null property found * 🚫 refactor method access * :construction: setup NodeErrorView * :zap: change timestamp to Date.now * :books: Add documentation for methods and constants * :construction: change message setting * 🚚 move NodeErrors to workflow * :sparkles: add new ErrorView for Nodes * :art: improve error notification * :art: refactor interfaces * :zap: add WorkflowOperationError, refactor error throwing * 👕 fix linter issues * :art: rename param * :bug: fix handling normal errors * :zap: add usage of NodeApiError * :art: fix throw new error instead of constructor * :art: remove unnecessary code/comments * :art: adjusted spacing + updated status messages * :art: fix tab indentation * ✨ Replace current errors with custom errors (#1576) * :zap: Introduce NodeApiError in catch blocks * :zap: Introduce NodeOperationError in nodes * :zap: Add missing errors and remove incompatible * :zap: Fix NodeOperationError in incompatible nodes * :wrench: Adjust error handling in missed nodes PayPal, FileMaker, Reddit, Taiga and Facebook Graph API nodes * :hammer: Adjust Strava Trigger node error handling * :hammer: Adjust AWS nodes error handling * :hammer: Remove duplicate instantiation of NodeApiError * :bug: fix strava trigger node error handling * Add XML parsing to NodeApiError constructor (#1633) * :bug: Remove type annotation from catch variable * :sparkles: Add XML parsing to NodeApiError * :zap: Simplify error handling in Rekognition node * :zap: Pass in XML flag in generic functions * :fire: Remove try/catch wrappers at call sites * :hammer: Refactor setting description from XML * :hammer: Refactor let to const in resource loaders * :zap: Find property in parsed XML * :zap: Change let to const * :fire: Remove unneeded try/catch block * :shirt: Fix linting issues * :bug: Fix errors from merge conflict resolution * :zap: Add custom errors to latest contributions * :shirt: Fix linting issues * :zap: Refactor MongoDB helpers for custom errors * :bug: Correct custom error type * :zap: Apply feedback to A nodes * :zap: Apply feedback to missed A node * :zap: Apply feedback to B-D nodes * :zap: Apply feedback to E-F nodes * :zap: Apply feedback to G nodes * :zap: Apply feedback to H-L nodes * :zap: Apply feedback to M nodes * :zap: Apply feedback to P nodes * :zap: Apply feedback to R nodes * :zap: Apply feedback to S nodes * :zap: Apply feedback to T nodes * :zap: Apply feedback to V-Z nodes * :zap: Add HTTP code to iterable node error * :hammer: Standardize e as error * :hammer: Standardize err as error * :zap: Fix error handling for non-standard nodes Co-authored-by: Ben Hesseldieck <b.hesseldieck@gmail.com> Co-authored-by: Ben Hesseldieck <b.hesseldieck@gmail.com> Co-authored-by: Ben Hesseldieck <1849459+BHesseldieck@users.noreply.github.com>
2021-04-16 09:33:36 -07:00
throw new NodeApiError(this.getNode(), error);
2020-08-18 05:56:53 -07:00
}
const name = this.getNodeParameter('name', i) as string;
const additionalFields = this.getNodeParameter('additionalFields', i) as IDataObject;
if (additionalFields.acl) {
headers['x-amz-acl'] = paramCase(additionalFields.acl as string);
}
if (additionalFields.bucketObjectLockEnabled) {
headers['x-amz-bucket-object-lock-enabled'] = additionalFields.bucketObjectLockEnabled as boolean;
}
if (additionalFields.grantFullControl) {
headers['x-amz-grant-full-control'] = '';
}
if (additionalFields.grantRead) {
headers['x-amz-grant-read'] = '';
}
if (additionalFields.grantReadAcp) {
headers['x-amz-grant-read-acp'] = '';
}
if (additionalFields.grantWrite) {
headers['x-amz-grant-write'] = '';
}
if (additionalFields.grantWriteAcp) {
headers['x-amz-grant-write-acp'] = '';
}
let region = credentials!.region as string;
if (additionalFields.region) {
region = additionalFields.region as string;
}
const body: IDataObject = {
CreateBucketConfiguration: {
'$': {
xmlns: 'http://s3.amazonaws.com/doc/2006-03-01/',
},
2020-10-22 06:46:03 -07:00
},
2020-08-18 05:56:53 -07:00
};
let data = '';
// if credentials has the S3 defaul region (us-east-1) the body (XML) does not have to be sent.
if (region !== 'us-east-1') {
// @ts-ignore
body.CreateBucketConfiguration.LocationConstraint = [region];
const builder = new Builder();
data = builder.buildObject(body);
}
responseData = await s3ApiRequestSOAP.call(this, `${name}`, 'PUT', '', data, qs, headers);
returnData.push({ success: true });
}
//https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListBuckets.html
if (operation === 'getAll') {
const returnAll = this.getNodeParameter('returnAll', 0) as boolean;
if (returnAll) {
responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListAllMyBucketsResult.Buckets.Bucket', '', 'GET', '');
} else {
qs.limit = this.getNodeParameter('limit', 0) as number;
responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListAllMyBucketsResult.Buckets.Bucket', '', 'GET', '', '', qs);
responseData = responseData.slice(0, qs.limit);
}
returnData.push.apply(returnData, responseData);
}
//https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html
if (operation === 'search') {
const bucketName = this.getNodeParameter('bucketName', i) as string;
const returnAll = this.getNodeParameter('returnAll', 0) as boolean;
const additionalFields = this.getNodeParameter('additionalFields', 0) as IDataObject;
if (additionalFields.prefix) {
qs['prefix'] = additionalFields.prefix as string;
}
if (additionalFields.encodingType) {
qs['encoding-type'] = additionalFields.encodingType as string;
}
if (additionalFields.delmiter) {
qs['delimiter'] = additionalFields.delmiter as string;
}
if (additionalFields.fetchOwner) {
qs['fetch-owner'] = additionalFields.fetchOwner as string;
}
if (additionalFields.startAfter) {
qs['start-after'] = additionalFields.startAfter as string;
}
if (additionalFields.requesterPays) {
qs['x-amz-request-payer'] = 'requester';
}
qs['list-type'] = 2;
responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' });
const region = responseData.LocationConstraint._ as string;
if (returnAll) {
responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '', '', qs, {}, {}, region);
} else {
qs['max-keys'] = this.getNodeParameter('limit', 0) as number;
responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', qs, {}, {}, region);
responseData = responseData.ListBucketResult.Contents;
}
if (Array.isArray(responseData)) {
returnData.push.apply(returnData, responseData);
} else {
returnData.push(responseData);
}
}
}
if (resource === 'folder') {
//https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html
if (operation === 'create') {
const bucketName = this.getNodeParameter('bucketName', i) as string;
const folderName = this.getNodeParameter('folderName', i) as string;
const additionalFields = this.getNodeParameter('additionalFields', i) as IDataObject;
let path = `/${folderName}/`;
if (additionalFields.requesterPays) {
headers['x-amz-request-payer'] = 'requester';
}
if (additionalFields.parentFolderKey) {
path = `/${additionalFields.parentFolderKey}${folderName}/`;
}
if (additionalFields.storageClass) {
headers['x-amz-storage-class'] = (snakeCase(additionalFields.storageClass as string)).toUpperCase();
}
responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' });
const region = responseData.LocationConstraint._;
responseData = await s3ApiRequestSOAP.call(this, bucketName, 'PUT', path, '', qs, headers, {}, region);
returnData.push({ success: true });
}
//https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html
if (operation === 'delete') {
const bucketName = this.getNodeParameter('bucketName', i) as string;
const folderKey = this.getNodeParameter('folderKey', i) as string;
responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' });
const region = responseData.LocationConstraint._;
responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '/', '', { 'list-type': 2, prefix: folderKey }, {}, {}, region);
// folder empty then just delete it
if (responseData.length === 0) {
responseData = await s3ApiRequestSOAP.call(this, bucketName, 'DELETE', `/${folderKey}`, '', qs, {}, {}, region);
2020-09-01 23:30:01 -07:00
responseData = { deleted: [{ 'Key': folderKey }] };
2020-08-18 05:56:53 -07:00
} else {
2020-09-01 23:30:01 -07:00
// delete everything inside the folder
2020-08-18 05:56:53 -07:00
const body: IDataObject = {
Delete: {
'$': {
xmlns: 'http://s3.amazonaws.com/doc/2006-03-01/',
},
Object: [],
},
};
for (const childObject of responseData) {
//@ts-ignore
(body.Delete.Object as IDataObject[]).push({
2020-10-22 06:46:03 -07:00
Key: childObject.Key as string,
2020-08-18 05:56:53 -07:00
});
}
const builder = new Builder();
const data = builder.buildObject(body);
headers['Content-MD5'] = createHash('md5').update(data).digest('base64');
headers['Content-Type'] = 'application/xml';
2020-09-01 23:30:01 -07:00
responseData = await s3ApiRequestSOAP.call(this, bucketName, 'POST', '/', data, { delete: '' }, headers, {}, region);
2020-08-18 05:56:53 -07:00
responseData = { deleted: responseData.DeleteResult.Deleted };
}
returnData.push(responseData);
}
//https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html
if (operation === 'getAll') {
const bucketName = this.getNodeParameter('bucketName', i) as string;
const returnAll = this.getNodeParameter('returnAll', 0) as boolean;
const options = this.getNodeParameter('options', 0) as IDataObject;
if (options.folderKey) {
qs['prefix'] = options.folderKey as string;
}
if (options.fetchOwner) {
qs['fetch-owner'] = options.fetchOwner as string;
}
qs['list-type'] = 2;
responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' });
const region = responseData.LocationConstraint._;
if (returnAll) {
responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '', '', qs, {}, {}, region);
} else {
qs.limit = this.getNodeParameter('limit', 0) as number;
responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '', '', qs, {}, {}, region);
}
if (Array.isArray(responseData)) {
responseData = responseData.filter((e: IDataObject) => (e.Key as string).endsWith('/') && e.Size === '0' && e.Key !== options.folderKey);
if (qs.limit) {
responseData = responseData.splice(0, qs.limit as number);
}
returnData.push.apply(returnData, responseData);
}
}
}
if (resource === 'file') {
//https://docs.aws.amazon.com/AmazonS3/latest/API/API_CopyObject.html
if (operation === 'copy') {
const sourcePath = this.getNodeParameter('sourcePath', i) as string;
const destinationPath = this.getNodeParameter('destinationPath', i) as string;
const additionalFields = this.getNodeParameter('additionalFields', i) as IDataObject;
headers['x-amz-copy-source'] = sourcePath;
if (additionalFields.requesterPays) {
headers['x-amz-request-payer'] = 'requester';
}
if (additionalFields.storageClass) {
headers['x-amz-storage-class'] = (snakeCase(additionalFields.storageClass as string)).toUpperCase();
}
if (additionalFields.acl) {
headers['x-amz-acl'] = paramCase(additionalFields.acl as string);
}
if (additionalFields.grantFullControl) {
headers['x-amz-grant-full-control'] = '';
}
if (additionalFields.grantRead) {
headers['x-amz-grant-read'] = '';
}
if (additionalFields.grantReadAcp) {
headers['x-amz-grant-read-acp'] = '';
}
if (additionalFields.grantWriteAcp) {
headers['x-amz-grant-write-acp'] = '';
}
if (additionalFields.lockLegalHold) {
headers['x-amz-object-lock-legal-hold'] = (additionalFields.lockLegalHold as boolean) ? 'ON' : 'OFF';
}
if (additionalFields.lockMode) {
headers['x-amz-object-lock-mode'] = (additionalFields.lockMode as string).toUpperCase();
}
if (additionalFields.lockRetainUntilDate) {
headers['x-amz-object-lock-retain-until-date'] = additionalFields.lockRetainUntilDate as string;
}
if (additionalFields.serverSideEncryption) {
headers['x-amz-server-side-encryption'] = additionalFields.serverSideEncryption as string;
}
if (additionalFields.encryptionAwsKmsKeyId) {
headers['x-amz-server-side-encryption-aws-kms-key-id'] = additionalFields.encryptionAwsKmsKeyId as string;
}
if (additionalFields.serverSideEncryptionContext) {
headers['x-amz-server-side-encryption-context'] = additionalFields.serverSideEncryptionContext as string;
}
if (additionalFields.serversideEncryptionCustomerAlgorithm) {
headers['x-amz-server-side-encryption-customer-algorithm'] = additionalFields.serversideEncryptionCustomerAlgorithm as string;
}
if (additionalFields.serversideEncryptionCustomerKey) {
headers['x-amz-server-side-encryption-customer-key'] = additionalFields.serversideEncryptionCustomerKey as string;
}
if (additionalFields.serversideEncryptionCustomerKeyMD5) {
headers['x-amz-server-side-encryption-customer-key-MD5'] = additionalFields.serversideEncryptionCustomerKeyMD5 as string;
}
if (additionalFields.taggingDirective) {
headers['x-amz-tagging-directive'] = (additionalFields.taggingDirective as string).toUpperCase();
}
if (additionalFields.metadataDirective) {
headers['x-amz-metadata-directive'] = (additionalFields.metadataDirective as string).toUpperCase();
}
const destinationParts = destinationPath.split('/');
const bucketName = destinationParts[1];
const destination = `/${destinationParts.slice(2, destinationParts.length).join('/')}`;
responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' });
const region = responseData.LocationConstraint._;
responseData = await s3ApiRequestSOAP.call(this, bucketName, 'PUT', destination, '', qs, headers, {}, region);
returnData.push(responseData.CopyObjectResult);
}
//https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html
if (operation === 'download') {
const bucketName = this.getNodeParameter('bucketName', i) as string;
const fileKey = this.getNodeParameter('fileKey', i) as string;
const fileName = fileKey.split('/')[fileKey.split('/').length - 1];
if (fileKey.substring(fileKey.length - 1) === '/') {
:sparkles: Improve node error handling (#1309) * Add path mapping and response error interfaces * Add error handling and throwing functionality * Refactor error handling into a single function * Re-implement error handling in Hacker News node * Fix linting details * Re-implement error handling in Spotify node * Re-implement error handling in G Suite Admin node * :construction: create basic setup NodeError * :construction: add httpCodes * :construction: add path priolist * :construction: handle statusCode in error, adjust interfaces * :construction: fixing type issues w/Ivan * :construction: add error exploration * 👔 fix linter issues * :wrench: improve object check * :construction: remove path passing from NodeApiError * :construction: add multi error + refactor findProperty method * 👔 allow any * :wrench: handle multi error message callback * :zap: change return type of callback * :zap: add customCallback to MultiError * :construction: refactor to use INode * :hammer: handle arrays, continue search after first null property found * 🚫 refactor method access * :construction: setup NodeErrorView * :zap: change timestamp to Date.now * :books: Add documentation for methods and constants * :construction: change message setting * 🚚 move NodeErrors to workflow * :sparkles: add new ErrorView for Nodes * :art: improve error notification * :art: refactor interfaces * :zap: add WorkflowOperationError, refactor error throwing * 👕 fix linter issues * :art: rename param * :bug: fix handling normal errors * :zap: add usage of NodeApiError * :art: fix throw new error instead of constructor * :art: remove unnecessary code/comments * :art: adjusted spacing + updated status messages * :art: fix tab indentation * ✨ Replace current errors with custom errors (#1576) * :zap: Introduce NodeApiError in catch blocks * :zap: Introduce NodeOperationError in nodes * :zap: Add missing errors and remove incompatible * :zap: Fix NodeOperationError in incompatible nodes * :wrench: Adjust error handling in missed nodes PayPal, FileMaker, Reddit, Taiga and Facebook Graph API nodes * :hammer: Adjust Strava Trigger node error handling * :hammer: Adjust AWS nodes error handling * :hammer: Remove duplicate instantiation of NodeApiError * :bug: fix strava trigger node error handling * Add XML parsing to NodeApiError constructor (#1633) * :bug: Remove type annotation from catch variable * :sparkles: Add XML parsing to NodeApiError * :zap: Simplify error handling in Rekognition node * :zap: Pass in XML flag in generic functions * :fire: Remove try/catch wrappers at call sites * :hammer: Refactor setting description from XML * :hammer: Refactor let to const in resource loaders * :zap: Find property in parsed XML * :zap: Change let to const * :fire: Remove unneeded try/catch block * :shirt: Fix linting issues * :bug: Fix errors from merge conflict resolution * :zap: Add custom errors to latest contributions * :shirt: Fix linting issues * :zap: Refactor MongoDB helpers for custom errors * :bug: Correct custom error type * :zap: Apply feedback to A nodes * :zap: Apply feedback to missed A node * :zap: Apply feedback to B-D nodes * :zap: Apply feedback to E-F nodes * :zap: Apply feedback to G nodes * :zap: Apply feedback to H-L nodes * :zap: Apply feedback to M nodes * :zap: Apply feedback to P nodes * :zap: Apply feedback to R nodes * :zap: Apply feedback to S nodes * :zap: Apply feedback to T nodes * :zap: Apply feedback to V-Z nodes * :zap: Add HTTP code to iterable node error * :hammer: Standardize e as error * :hammer: Standardize err as error * :zap: Fix error handling for non-standard nodes Co-authored-by: Ben Hesseldieck <b.hesseldieck@gmail.com> Co-authored-by: Ben Hesseldieck <b.hesseldieck@gmail.com> Co-authored-by: Ben Hesseldieck <1849459+BHesseldieck@users.noreply.github.com>
2021-04-16 09:33:36 -07:00
throw new NodeOperationError(this.getNode(), 'Downloding a whole directory is not yet supported, please provide a file key');
2020-08-18 05:56:53 -07:00
}
let region = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' });
region = region.LocationConstraint._;
2020-09-01 23:30:01 -07:00
const response = await s3ApiRequestREST.call(this, bucketName, 'GET', `/${fileKey}`, '', qs, {}, { encoding: null, resolveWithFullResponse: true }, region);
2020-08-18 05:56:53 -07:00
let mimeType: string | undefined;
if (response.headers['content-type']) {
mimeType = response.headers['content-type'];
}
const newItem: INodeExecutionData = {
json: items[i].json,
binary: {},
};
if (items[i].binary !== undefined) {
// Create a shallow copy of the binary data so that the old
// data references which do not get changed still stay behind
// but the incoming data does not get changed.
Object.assign(newItem.binary, items[i].binary);
}
items[i] = newItem;
const dataPropertyNameDownload = this.getNodeParameter('binaryPropertyName', i) as string;
const data = Buffer.from(response.body as string, 'utf8');
2020-09-01 23:30:01 -07:00
items[i].binary![dataPropertyNameDownload] = await this.helpers.prepareBinaryData(data as unknown as Buffer, fileName, mimeType);
2020-08-18 05:56:53 -07:00
}
//https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObject.html
if (operation === 'delete') {
const bucketName = this.getNodeParameter('bucketName', i) as string;
const fileKey = this.getNodeParameter('fileKey', i) as string;
const options = this.getNodeParameter('options', i) as IDataObject;
if (options.versionId) {
qs.versionId = options.versionId as string;
}
responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' });
const region = responseData.LocationConstraint._;
responseData = await s3ApiRequestSOAP.call(this, bucketName, 'DELETE', `/${fileKey}`, '', qs, {}, {}, region);
returnData.push({ success: true });
}
//https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html
if (operation === 'getAll') {
const bucketName = this.getNodeParameter('bucketName', i) as string;
const returnAll = this.getNodeParameter('returnAll', 0) as boolean;
const options = this.getNodeParameter('options', 0) as IDataObject;
if (options.folderKey) {
qs['prefix'] = options.folderKey as string;
}
if (options.fetchOwner) {
qs['fetch-owner'] = options.fetchOwner as string;
}
qs['delimiter'] = '/';
qs['list-type'] = 2;
responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' });
const region = responseData.LocationConstraint._;
if (returnAll) {
responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '', '', qs, {}, {}, region);
} else {
qs.limit = this.getNodeParameter('limit', 0) as number;
responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '', '', qs, {}, {}, region);
responseData = responseData.splice(0, qs.limit);
}
if (Array.isArray(responseData)) {
responseData = responseData.filter((e: IDataObject) => !(e.Key as string).endsWith('/') && e.Size !== '0');
if (qs.limit) {
responseData = responseData.splice(0, qs.limit as number);
}
returnData.push.apply(returnData, responseData);
}
}
//https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html
if (operation === 'upload') {
const bucketName = this.getNodeParameter('bucketName', i) as string;
const fileName = this.getNodeParameter('fileName', i) as string;
const isBinaryData = this.getNodeParameter('binaryData', i) as boolean;
const additionalFields = this.getNodeParameter('additionalFields', i) as IDataObject;
const tagsValues = (this.getNodeParameter('tagsUi', i) as IDataObject).tagsValues as IDataObject[];
let path = '/';
let body;
if (additionalFields.requesterPays) {
headers['x-amz-request-payer'] = 'requester';
}
if (additionalFields.parentFolderKey) {
path = `/${additionalFields.parentFolderKey}/`;
}
if (additionalFields.storageClass) {
headers['x-amz-storage-class'] = (snakeCase(additionalFields.storageClass as string)).toUpperCase();
}
if (additionalFields.acl) {
headers['x-amz-acl'] = paramCase(additionalFields.acl as string);
}
if (additionalFields.grantFullControl) {
headers['x-amz-grant-full-control'] = '';
}
if (additionalFields.grantRead) {
headers['x-amz-grant-read'] = '';
}
if (additionalFields.grantReadAcp) {
headers['x-amz-grant-read-acp'] = '';
}
if (additionalFields.grantWriteAcp) {
headers['x-amz-grant-write-acp'] = '';
}
if (additionalFields.lockLegalHold) {
headers['x-amz-object-lock-legal-hold'] = (additionalFields.lockLegalHold as boolean) ? 'ON' : 'OFF';
}
if (additionalFields.lockMode) {
headers['x-amz-object-lock-mode'] = (additionalFields.lockMode as string).toUpperCase();
}
if (additionalFields.lockRetainUntilDate) {
headers['x-amz-object-lock-retain-until-date'] = additionalFields.lockRetainUntilDate as string;
}
if (additionalFields.serverSideEncryption) {
headers['x-amz-server-side-encryption'] = additionalFields.serverSideEncryption as string;
}
if (additionalFields.encryptionAwsKmsKeyId) {
headers['x-amz-server-side-encryption-aws-kms-key-id'] = additionalFields.encryptionAwsKmsKeyId as string;
}
if (additionalFields.serverSideEncryptionContext) {
headers['x-amz-server-side-encryption-context'] = additionalFields.serverSideEncryptionContext as string;
}
if (additionalFields.serversideEncryptionCustomerAlgorithm) {
headers['x-amz-server-side-encryption-customer-algorithm'] = additionalFields.serversideEncryptionCustomerAlgorithm as string;
}
if (additionalFields.serversideEncryptionCustomerKey) {
headers['x-amz-server-side-encryption-customer-key'] = additionalFields.serversideEncryptionCustomerKey as string;
}
if (additionalFields.serversideEncryptionCustomerKeyMD5) {
headers['x-amz-server-side-encryption-customer-key-MD5'] = additionalFields.serversideEncryptionCustomerKeyMD5 as string;
}
if (tagsValues) {
const tags: string[] = [];
tagsValues.forEach((o: IDataObject) => { tags.push(`${o.key}=${o.value}`); });
headers['x-amz-tagging'] = tags.join('&');
}
responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' });
const region = responseData.LocationConstraint._;
if (isBinaryData) {
const binaryPropertyName = this.getNodeParameter('binaryPropertyName', 0) as string;
if (items[i].binary === undefined) {
:sparkles: Improve node error handling (#1309) * Add path mapping and response error interfaces * Add error handling and throwing functionality * Refactor error handling into a single function * Re-implement error handling in Hacker News node * Fix linting details * Re-implement error handling in Spotify node * Re-implement error handling in G Suite Admin node * :construction: create basic setup NodeError * :construction: add httpCodes * :construction: add path priolist * :construction: handle statusCode in error, adjust interfaces * :construction: fixing type issues w/Ivan * :construction: add error exploration * 👔 fix linter issues * :wrench: improve object check * :construction: remove path passing from NodeApiError * :construction: add multi error + refactor findProperty method * 👔 allow any * :wrench: handle multi error message callback * :zap: change return type of callback * :zap: add customCallback to MultiError * :construction: refactor to use INode * :hammer: handle arrays, continue search after first null property found * 🚫 refactor method access * :construction: setup NodeErrorView * :zap: change timestamp to Date.now * :books: Add documentation for methods and constants * :construction: change message setting * 🚚 move NodeErrors to workflow * :sparkles: add new ErrorView for Nodes * :art: improve error notification * :art: refactor interfaces * :zap: add WorkflowOperationError, refactor error throwing * 👕 fix linter issues * :art: rename param * :bug: fix handling normal errors * :zap: add usage of NodeApiError * :art: fix throw new error instead of constructor * :art: remove unnecessary code/comments * :art: adjusted spacing + updated status messages * :art: fix tab indentation * ✨ Replace current errors with custom errors (#1576) * :zap: Introduce NodeApiError in catch blocks * :zap: Introduce NodeOperationError in nodes * :zap: Add missing errors and remove incompatible * :zap: Fix NodeOperationError in incompatible nodes * :wrench: Adjust error handling in missed nodes PayPal, FileMaker, Reddit, Taiga and Facebook Graph API nodes * :hammer: Adjust Strava Trigger node error handling * :hammer: Adjust AWS nodes error handling * :hammer: Remove duplicate instantiation of NodeApiError * :bug: fix strava trigger node error handling * Add XML parsing to NodeApiError constructor (#1633) * :bug: Remove type annotation from catch variable * :sparkles: Add XML parsing to NodeApiError * :zap: Simplify error handling in Rekognition node * :zap: Pass in XML flag in generic functions * :fire: Remove try/catch wrappers at call sites * :hammer: Refactor setting description from XML * :hammer: Refactor let to const in resource loaders * :zap: Find property in parsed XML * :zap: Change let to const * :fire: Remove unneeded try/catch block * :shirt: Fix linting issues * :bug: Fix errors from merge conflict resolution * :zap: Add custom errors to latest contributions * :shirt: Fix linting issues * :zap: Refactor MongoDB helpers for custom errors * :bug: Correct custom error type * :zap: Apply feedback to A nodes * :zap: Apply feedback to missed A node * :zap: Apply feedback to B-D nodes * :zap: Apply feedback to E-F nodes * :zap: Apply feedback to G nodes * :zap: Apply feedback to H-L nodes * :zap: Apply feedback to M nodes * :zap: Apply feedback to P nodes * :zap: Apply feedback to R nodes * :zap: Apply feedback to S nodes * :zap: Apply feedback to T nodes * :zap: Apply feedback to V-Z nodes * :zap: Add HTTP code to iterable node error * :hammer: Standardize e as error * :hammer: Standardize err as error * :zap: Fix error handling for non-standard nodes Co-authored-by: Ben Hesseldieck <b.hesseldieck@gmail.com> Co-authored-by: Ben Hesseldieck <b.hesseldieck@gmail.com> Co-authored-by: Ben Hesseldieck <1849459+BHesseldieck@users.noreply.github.com>
2021-04-16 09:33:36 -07:00
throw new NodeOperationError(this.getNode(), 'No binary data exists on item!');
2020-08-18 05:56:53 -07:00
}
if ((items[i].binary as IBinaryKeyData)[binaryPropertyName] === undefined) {
:sparkles: Improve node error handling (#1309) * Add path mapping and response error interfaces * Add error handling and throwing functionality * Refactor error handling into a single function * Re-implement error handling in Hacker News node * Fix linting details * Re-implement error handling in Spotify node * Re-implement error handling in G Suite Admin node * :construction: create basic setup NodeError * :construction: add httpCodes * :construction: add path priolist * :construction: handle statusCode in error, adjust interfaces * :construction: fixing type issues w/Ivan * :construction: add error exploration * 👔 fix linter issues * :wrench: improve object check * :construction: remove path passing from NodeApiError * :construction: add multi error + refactor findProperty method * 👔 allow any * :wrench: handle multi error message callback * :zap: change return type of callback * :zap: add customCallback to MultiError * :construction: refactor to use INode * :hammer: handle arrays, continue search after first null property found * 🚫 refactor method access * :construction: setup NodeErrorView * :zap: change timestamp to Date.now * :books: Add documentation for methods and constants * :construction: change message setting * 🚚 move NodeErrors to workflow * :sparkles: add new ErrorView for Nodes * :art: improve error notification * :art: refactor interfaces * :zap: add WorkflowOperationError, refactor error throwing * 👕 fix linter issues * :art: rename param * :bug: fix handling normal errors * :zap: add usage of NodeApiError * :art: fix throw new error instead of constructor * :art: remove unnecessary code/comments * :art: adjusted spacing + updated status messages * :art: fix tab indentation * ✨ Replace current errors with custom errors (#1576) * :zap: Introduce NodeApiError in catch blocks * :zap: Introduce NodeOperationError in nodes * :zap: Add missing errors and remove incompatible * :zap: Fix NodeOperationError in incompatible nodes * :wrench: Adjust error handling in missed nodes PayPal, FileMaker, Reddit, Taiga and Facebook Graph API nodes * :hammer: Adjust Strava Trigger node error handling * :hammer: Adjust AWS nodes error handling * :hammer: Remove duplicate instantiation of NodeApiError * :bug: fix strava trigger node error handling * Add XML parsing to NodeApiError constructor (#1633) * :bug: Remove type annotation from catch variable * :sparkles: Add XML parsing to NodeApiError * :zap: Simplify error handling in Rekognition node * :zap: Pass in XML flag in generic functions * :fire: Remove try/catch wrappers at call sites * :hammer: Refactor setting description from XML * :hammer: Refactor let to const in resource loaders * :zap: Find property in parsed XML * :zap: Change let to const * :fire: Remove unneeded try/catch block * :shirt: Fix linting issues * :bug: Fix errors from merge conflict resolution * :zap: Add custom errors to latest contributions * :shirt: Fix linting issues * :zap: Refactor MongoDB helpers for custom errors * :bug: Correct custom error type * :zap: Apply feedback to A nodes * :zap: Apply feedback to missed A node * :zap: Apply feedback to B-D nodes * :zap: Apply feedback to E-F nodes * :zap: Apply feedback to G nodes * :zap: Apply feedback to H-L nodes * :zap: Apply feedback to M nodes * :zap: Apply feedback to P nodes * :zap: Apply feedback to R nodes * :zap: Apply feedback to S nodes * :zap: Apply feedback to T nodes * :zap: Apply feedback to V-Z nodes * :zap: Add HTTP code to iterable node error * :hammer: Standardize e as error * :hammer: Standardize err as error * :zap: Fix error handling for non-standard nodes Co-authored-by: Ben Hesseldieck <b.hesseldieck@gmail.com> Co-authored-by: Ben Hesseldieck <b.hesseldieck@gmail.com> Co-authored-by: Ben Hesseldieck <1849459+BHesseldieck@users.noreply.github.com>
2021-04-16 09:33:36 -07:00
throw new NodeOperationError(this.getNode(), `No binary data property "${binaryPropertyName}" does not exists on item!`);
2020-08-18 05:56:53 -07:00
}
const binaryData = (items[i].binary as IBinaryKeyData)[binaryPropertyName];
body = Buffer.from(binaryData.data, BINARY_ENCODING) as Buffer;
headers['Content-Type'] = binaryData.mimeType;
headers['Content-MD5'] = createHash('md5').update(body).digest('base64');
responseData = await s3ApiRequestSOAP.call(this, bucketName, 'PUT', `${path}${fileName || binaryData.fileName}`, body, qs, headers, {}, region);
} else {
const fileContent = this.getNodeParameter('fileContent', i) as string;
body = Buffer.from(fileContent, 'utf8');
headers['Content-Type'] = 'text/html';
headers['Content-MD5'] = createHash('md5').update(fileContent).digest('base64');
responseData = await s3ApiRequestSOAP.call(this, bucketName, 'PUT', `${path}${fileName}`, body, qs, headers, {}, region);
}
returnData.push({ success: true });
}
}
}
if (resource === 'file' && operation === 'download') {
// For file downloads the files get attached to the existing items
return this.prepareOutputData(items);
} else {
return [this.helpers.returnJsonArray(returnData)];
}
}
}