n8n/packages/nodes-base/nodes/S3/S3.node.ts

import {
	paramCase,
	snakeCase,
} from 'change-case';

import {
	createHash,
} from 'crypto';

import {
	Builder,
} from 'xml2js';

import {
	BINARY_ENCODING,
	IExecuteFunctions,
} from 'n8n-core';

import {
	IBinaryKeyData,
	IDataObject,
	INodeExecutionData,
	INodeType,
	INodeTypeDescription,
	NodeApiError,
	NodeOperationError,
} from 'n8n-workflow';

import {
	bucketFields,
	bucketOperations,
} from '../Aws/S3/BucketDescription';

import {
	folderFields,
	folderOperations,
} from '../Aws/S3/FolderDescription';

import {
	fileFields,
	fileOperations,
} from '../Aws/S3/FileDescription';

import {
	s3ApiRequestREST,
	s3ApiRequestSOAP,
	s3ApiRequestSOAPAllItems,
} from './GenericFunctions';

export class S3 implements INodeType {
	description: INodeTypeDescription = {
		displayName: 'S3',
		name: 's3',
		icon: 'file:s3.png',
		group: ['output'],
		version: 1,
		subtitle: '={{$parameter["operation"] + ": " + $parameter["resource"]}}',
		description: 'Sends data to any S3-compatible service',
		defaults: {
			name: 'S3',
			color: '#d05b4b',
		},
		inputs: ['main'],
		outputs: ['main'],
		credentials: [
			{
				name: 's3',
				required: true,
			},
		],
		properties: [
			{
				displayName: 'Resource',
				name: 'resource',
				type: 'options',
				options: [
					{
						name: 'Bucket',
						value: 'bucket',
					},
					{
						name: 'File',
						value: 'file',
					},
					{
						name: 'Folder',
						value: 'folder',
					},
				],
				default: 'file',
				description: 'The operation to perform.',
			},
			// BUCKET
			...bucketOperations,
			...bucketFields,
			// FOLDER
			...folderOperations,
			...folderFields,
			// UPLOAD
			...fileOperations,
			...fileFields,
		],
	};

	async execute(this: IExecuteFunctions): Promise<INodeExecutionData[][]> {
		const items = this.getInputData();
		const returnData: IDataObject[] = [];
		const qs: IDataObject = {};
		let responseData;
		const resource = this.getNodeParameter('resource', 0) as string;
		const operation = this.getNodeParameter('operation', 0) as string;
		for (let i = 0; i < items.length; i++) {
			try {
				const headers: IDataObject = {};
				if (resource === 'bucket') {
					//https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateBucket.html
					if (operation === 'create') {

						let credentials;

						try {
							credentials = this.getCredentials('s3');
						} catch (error) {
							throw new NodeApiError(this.getNode(), error);
						}

						const name = this.getNodeParameter('name', i) as string;
						const additionalFields = this.getNodeParameter('additionalFields', i) as IDataObject;
						if (additionalFields.acl) {
							headers['x-amz-acl'] = paramCase(additionalFields.acl as string);
						}
						if (additionalFields.bucketObjectLockEnabled) {
							headers['x-amz-bucket-object-lock-enabled'] = additionalFields.bucketObjectLockEnabled as boolean;
						}
						if (additionalFields.grantFullControl) {
							headers['x-amz-grant-full-control'] = '';
						}
						if (additionalFields.grantRead) {
							headers['x-amz-grant-read'] = '';
						}
						if (additionalFields.grantReadAcp) {
							headers['x-amz-grant-read-acp'] = '';
						}
						if (additionalFields.grantWrite) {
							headers['x-amz-grant-write'] = '';
						}
						if (additionalFields.grantWriteAcp) {
							headers['x-amz-grant-write-acp'] = '';
						}
						let region = credentials!.region as string;

						if (additionalFields.region) {
							region = additionalFields.region as string;
						}

						const body: IDataObject = {
							CreateBucketConfiguration: {
								'$': {
									xmlns: 'http://s3.amazonaws.com/doc/2006-03-01/',
								},
							},
						};
						let data = '';
						// if credentials has the S3 defaul region (us-east-1) the body (XML) does not have to be sent.
						if (region !== 'us-east-1') {
							// @ts-ignore
							body.CreateBucketConfiguration.LocationConstraint = [region];
							const builder = new Builder();
							data = builder.buildObject(body);
						}
						responseData = await s3ApiRequestSOAP.call(this, `${name}`, 'PUT', '', data, qs, headers);

						returnData.push({ success: true });
					}
					//https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListBuckets.html
					if (operation === 'getAll') {
						const returnAll = this.getNodeParameter('returnAll', 0) as boolean;
						if (returnAll) {
							responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListAllMyBucketsResult.Buckets.Bucket', '', 'GET', '');
						} else {
							qs.limit = this.getNodeParameter('limit', 0) as number;
							responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListAllMyBucketsResult.Buckets.Bucket', '', 'GET', '', '', qs);
							responseData = responseData.slice(0, qs.limit);
						}
						returnData.push.apply(returnData, responseData);
					}

					//https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html
					if (operation === 'search') {
						const bucketName = this.getNodeParameter('bucketName', i) as string;
						const returnAll = this.getNodeParameter('returnAll', 0) as boolean;
						const additionalFields = this.getNodeParameter('additionalFields', 0) as IDataObject;

						if (additionalFields.prefix) {
							qs['prefix'] = additionalFields.prefix as string;
						}

						if (additionalFields.encodingType) {
							qs['encoding-type'] = additionalFields.encodingType as string;
						}

						if (additionalFields.delmiter) {
							qs['delimiter'] = additionalFields.delmiter as string;
						}

						if (additionalFields.fetchOwner) {
							qs['fetch-owner'] = additionalFields.fetchOwner as string;
						}

						if (additionalFields.startAfter) {
							qs['start-after'] = additionalFields.startAfter as string;
						}

						if (additionalFields.requesterPays) {
							qs['x-amz-request-payer'] = 'requester';
						}

						qs['list-type'] = 2;


						responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' });

						const region = responseData.LocationConstraint._ as string;

						if (returnAll) {
							responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '', '', qs, {}, {}, region);
						} else {
							qs['max-keys'] = this.getNodeParameter('limit', 0) as number;
							responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', qs, {}, {}, region);
							responseData = responseData.ListBucketResult.Contents;
						}
						if (Array.isArray(responseData)) {
							returnData.push.apply(returnData, responseData);
						} else {
							returnData.push(responseData);
						}
					}
				}
				if (resource === 'folder') {
					//https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html
					if (operation === 'create') {
						const bucketName = this.getNodeParameter('bucketName', i) as string;
						const folderName = this.getNodeParameter('folderName', i) as string;
						const additionalFields = this.getNodeParameter('additionalFields', i) as IDataObject;
						let path = `/${folderName}/`;

						if (additionalFields.requesterPays) {
							headers['x-amz-request-payer'] = 'requester';
						}
						if (additionalFields.parentFolderKey) {
							path = `/${additionalFields.parentFolderKey}${folderName}/`;
						}
						if (additionalFields.storageClass) {
							headers['x-amz-storage-class'] = (snakeCase(additionalFields.storageClass as string)).toUpperCase();
						}
						responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' });

						const region = responseData.LocationConstraint._;

						responseData = await s3ApiRequestSOAP.call(this, bucketName, 'PUT', path, '', qs, headers, {}, region);
						returnData.push({ success: true });
					}
					//https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html
					if (operation === 'delete') {
						const bucketName = this.getNodeParameter('bucketName', i) as string;
						const folderKey = this.getNodeParameter('folderKey', i) as string;

						responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' });

						const region = responseData.LocationConstraint._;

						responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '/', '', { 'list-type': 2, prefix: folderKey }, {}, {}, region);

						// folder empty then just delete it
						if (responseData.length === 0) {

							responseData = await s3ApiRequestSOAP.call(this, bucketName, 'DELETE', `/${folderKey}`, '', qs, {}, {}, region);

							responseData = { deleted: [{ 'Key': folderKey }] };

						} else {
							// delete everything inside the folder
							const body: IDataObject = {
								Delete: {
									'$': {
										xmlns: 'http://s3.amazonaws.com/doc/2006-03-01/',
									},
									Object: [],
								},
							};

							for (const childObject of responseData) {
								//@ts-ignore
								(body.Delete.Object as IDataObject[]).push({
									Key: childObject.Key as string,
								});
							}

							const builder = new Builder();
							const data = builder.buildObject(body);

							headers['Content-MD5'] = createHash('md5').update(data).digest('base64');

							headers['Content-Type'] = 'application/xml';

							responseData = await s3ApiRequestSOAP.call(this, bucketName, 'POST', '/', data, { delete: '' }, headers, {}, region);

							responseData = { deleted: responseData.DeleteResult.Deleted };
						}
						returnData.push(responseData);
					}
					//https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html
					if (operation === 'getAll') {
						const bucketName = this.getNodeParameter('bucketName', i) as string;
						const returnAll = this.getNodeParameter('returnAll', 0) as boolean;
						const options = this.getNodeParameter('options', 0) as IDataObject;

						if (options.folderKey) {
							qs['prefix'] = options.folderKey as string;
						}

						if (options.fetchOwner) {
							qs['fetch-owner'] = options.fetchOwner as string;
						}

						qs['list-type'] = 2;

						responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' });

						const region = responseData.LocationConstraint._;

						if (returnAll) {
							responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '', '', qs, {}, {}, region);
						} else {
							qs.limit = this.getNodeParameter('limit', 0) as number;
							responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '', '', qs, {}, {}, region);
						}
						if (Array.isArray(responseData)) {
							responseData = responseData.filter((e: IDataObject) => (e.Key as string).endsWith('/') && e.Size === '0' && e.Key !== options.folderKey);
							if (qs.limit) {
								responseData = responseData.splice(0, qs.limit as number);
							}
							returnData.push.apply(returnData, responseData);
						}
					}
				}
				if (resource === 'file') {
					//https://docs.aws.amazon.com/AmazonS3/latest/API/API_CopyObject.html
					if (operation === 'copy') {
						const sourcePath = this.getNodeParameter('sourcePath', i) as string;
						const destinationPath = this.getNodeParameter('destinationPath', i) as string;
						const additionalFields = this.getNodeParameter('additionalFields', i) as IDataObject;

						headers['x-amz-copy-source'] = sourcePath;

						if (additionalFields.requesterPays) {
							headers['x-amz-request-payer'] = 'requester';
						}
						if (additionalFields.storageClass) {
							headers['x-amz-storage-class'] = (snakeCase(additionalFields.storageClass as string)).toUpperCase();
						}
						if (additionalFields.acl) {
							headers['x-amz-acl'] = paramCase(additionalFields.acl as string);
						}
						if (additionalFields.grantFullControl) {
							headers['x-amz-grant-full-control'] = '';
						}
						if (additionalFields.grantRead) {
							headers['x-amz-grant-read'] = '';
						}
						if (additionalFields.grantReadAcp) {
							headers['x-amz-grant-read-acp'] = '';
						}
						if (additionalFields.grantWriteAcp) {
							headers['x-amz-grant-write-acp'] = '';
						}
						if (additionalFields.lockLegalHold) {
							headers['x-amz-object-lock-legal-hold'] = (additionalFields.lockLegalHold as boolean) ? 'ON' : 'OFF';
						}
						if (additionalFields.lockMode) {
							headers['x-amz-object-lock-mode'] = (additionalFields.lockMode as string).toUpperCase();
						}
						if (additionalFields.lockRetainUntilDate) {
							headers['x-amz-object-lock-retain-until-date'] = additionalFields.lockRetainUntilDate as string;
						}
						if (additionalFields.serverSideEncryption) {
							headers['x-amz-server-side-encryption'] = additionalFields.serverSideEncryption as string;
						}
						if (additionalFields.encryptionAwsKmsKeyId) {
							headers['x-amz-server-side-encryption-aws-kms-key-id'] = additionalFields.encryptionAwsKmsKeyId as string;
						}
						if (additionalFields.serverSideEncryptionContext) {
							headers['x-amz-server-side-encryption-context'] = additionalFields.serverSideEncryptionContext as string;
						}
						if (additionalFields.serversideEncryptionCustomerAlgorithm) {
							headers['x-amz-server-side-encryption-customer-algorithm'] = additionalFields.serversideEncryptionCustomerAlgorithm as string;
						}
						if (additionalFields.serversideEncryptionCustomerKey) {
							headers['x-amz-server-side-encryption-customer-key'] = additionalFields.serversideEncryptionCustomerKey as string;
						}
						if (additionalFields.serversideEncryptionCustomerKeyMD5) {
							headers['x-amz-server-side-encryption-customer-key-MD5'] = additionalFields.serversideEncryptionCustomerKeyMD5 as string;
						}
						if (additionalFields.taggingDirective) {
							headers['x-amz-tagging-directive'] = (additionalFields.taggingDirective as string).toUpperCase();
						}
						if (additionalFields.metadataDirective) {
							headers['x-amz-metadata-directive'] = (additionalFields.metadataDirective as string).toUpperCase();
						}

						const destinationParts = destinationPath.split('/');

						const bucketName = destinationParts[1];

						const destination = `/${destinationParts.slice(2, destinationParts.length).join('/')}`;

						responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' });

						const region = responseData.LocationConstraint._;

						responseData = await s3ApiRequestSOAP.call(this, bucketName, 'PUT', destination, '', qs, headers, {}, region);
						returnData.push(responseData.CopyObjectResult);

					}
					//https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html
					if (operation === 'download') {

						const bucketName = this.getNodeParameter('bucketName', i) as string;

						const fileKey = this.getNodeParameter('fileKey', i) as string;

						const fileName = fileKey.split('/')[fileKey.split('/').length - 1];

						if (fileKey.substring(fileKey.length - 1) === '/') {
							throw new NodeOperationError(this.getNode(), 'Downloding a whole directory is not yet supported, please provide a file key');
						}

						let region = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' });

						region = region.LocationConstraint._;

						const response = await s3ApiRequestREST.call(this, bucketName, 'GET', `/${fileKey}`, '', qs, {}, { encoding: null, resolveWithFullResponse: true }, region);

						let mimeType: string | undefined;
						if (response.headers['content-type']) {
							mimeType = response.headers['content-type'];
						}

						const newItem: INodeExecutionData = {
							json: items[i].json,
							binary: {},
						};

						if (items[i].binary !== undefined) {
							// Create a shallow copy of the binary data so that the old
							// data references which do not get changed still stay behind
							// but the incoming data does not get changed.
							Object.assign(newItem.binary, items[i].binary);
						}

						items[i] = newItem;

						const dataPropertyNameDownload = this.getNodeParameter('binaryPropertyName', i) as string;

						const data = Buffer.from(response.body as string, 'utf8');

						items[i].binary![dataPropertyNameDownload] = await this.helpers.prepareBinaryData(data as unknown as Buffer, fileName, mimeType);
					}
					//https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObject.html
					if (operation === 'delete') {
						const bucketName = this.getNodeParameter('bucketName', i) as string;

						const fileKey = this.getNodeParameter('fileKey', i) as string;

						const options = this.getNodeParameter('options', i) as IDataObject;

						if (options.versionId) {
							qs.versionId = options.versionId as string;
						}

						responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' });

						const region = responseData.LocationConstraint._;

						responseData = await s3ApiRequestSOAP.call(this, bucketName, 'DELETE', `/${fileKey}`, '', qs, {}, {}, region);

						returnData.push({ success: true });
					}
					//https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html
					if (operation === 'getAll') {
						const bucketName = this.getNodeParameter('bucketName', i) as string;
						const returnAll = this.getNodeParameter('returnAll', 0) as boolean;
						const options = this.getNodeParameter('options', 0) as IDataObject;

						if (options.folderKey) {
							qs['prefix'] = options.folderKey as string;
						}

						if (options.fetchOwner) {
							qs['fetch-owner'] = options.fetchOwner as string;
						}

						qs['delimiter'] = '/';

						qs['list-type'] = 2;

						responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' });

						const region = responseData.LocationConstraint._;

						if (returnAll) {
							responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '', '', qs, {}, {}, region);
						} else {
							qs.limit = this.getNodeParameter('limit', 0) as number;
							responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '', '', qs, {}, {}, region);
							responseData = responseData.splice(0, qs.limit);
						}
						if (Array.isArray(responseData)) {
							responseData = responseData.filter((e: IDataObject) => !(e.Key as string).endsWith('/') && e.Size !== '0');
							if (qs.limit) {
								responseData = responseData.splice(0, qs.limit as number);
							}
							returnData.push.apply(returnData, responseData);
						}
					}
					//https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html
					if (operation === 'upload') {
						const bucketName = this.getNodeParameter('bucketName', i) as string;
						const fileName = this.getNodeParameter('fileName', i) as string;
						const isBinaryData = this.getNodeParameter('binaryData', i) as boolean;
						const additionalFields = this.getNodeParameter('additionalFields', i) as IDataObject;
						const tagsValues = (this.getNodeParameter('tagsUi', i) as IDataObject).tagsValues as IDataObject[];
						let path = '/';
						let body;

						if (additionalFields.requesterPays) {
							headers['x-amz-request-payer'] = 'requester';
						}

						if (additionalFields.parentFolderKey) {
							path = `/${additionalFields.parentFolderKey}/`;
						}
						if (additionalFields.storageClass) {
							headers['x-amz-storage-class'] = (snakeCase(additionalFields.storageClass as string)).toUpperCase();
						}
						if (additionalFields.acl) {
							headers['x-amz-acl'] = paramCase(additionalFields.acl as string);
						}
						if (additionalFields.grantFullControl) {
							headers['x-amz-grant-full-control'] = '';
						}
						if (additionalFields.grantRead) {
							headers['x-amz-grant-read'] = '';
						}
						if (additionalFields.grantReadAcp) {
							headers['x-amz-grant-read-acp'] = '';
						}
						if (additionalFields.grantWriteAcp) {
							headers['x-amz-grant-write-acp'] = '';
						}
						if (additionalFields.lockLegalHold) {
							headers['x-amz-object-lock-legal-hold'] = (additionalFields.lockLegalHold as boolean) ? 'ON' : 'OFF';
						}
						if (additionalFields.lockMode) {
							headers['x-amz-object-lock-mode'] = (additionalFields.lockMode as string).toUpperCase();
						}
						if (additionalFields.lockRetainUntilDate) {
							headers['x-amz-object-lock-retain-until-date'] = additionalFields.lockRetainUntilDate as string;
						}
						if (additionalFields.serverSideEncryption) {
							headers['x-amz-server-side-encryption'] = additionalFields.serverSideEncryption as string;
						}
						if (additionalFields.encryptionAwsKmsKeyId) {
							headers['x-amz-server-side-encryption-aws-kms-key-id'] = additionalFields.encryptionAwsKmsKeyId as string;
						}
						if (additionalFields.serverSideEncryptionContext) {
							headers['x-amz-server-side-encryption-context'] = additionalFields.serverSideEncryptionContext as string;
						}
						if (additionalFields.serversideEncryptionCustomerAlgorithm) {
							headers['x-amz-server-side-encryption-customer-algorithm'] = additionalFields.serversideEncryptionCustomerAlgorithm as string;
						}
						if (additionalFields.serversideEncryptionCustomerKey) {
							headers['x-amz-server-side-encryption-customer-key'] = additionalFields.serversideEncryptionCustomerKey as string;
						}
						if (additionalFields.serversideEncryptionCustomerKeyMD5) {
							headers['x-amz-server-side-encryption-customer-key-MD5'] = additionalFields.serversideEncryptionCustomerKeyMD5 as string;
						}
						if (tagsValues) {
							const tags: string[] = [];
							tagsValues.forEach((o: IDataObject) => { tags.push(`${o.key}=${o.value}`); });
							headers['x-amz-tagging'] = tags.join('&');
						}

						responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' });

						const region = responseData.LocationConstraint._;

						if (isBinaryData) {
							const binaryPropertyName = this.getNodeParameter('binaryPropertyName', 0) as string;

							if (items[i].binary === undefined) {
								throw new NodeOperationError(this.getNode(), 'No binary data exists on item!');
							}

							if ((items[i].binary as IBinaryKeyData)[binaryPropertyName] === undefined) {
								throw new NodeOperationError(this.getNode(), `No binary data property "${binaryPropertyName}" does not exists on item!`);
							}

							const binaryData = (items[i].binary as IBinaryKeyData)[binaryPropertyName];

							body = Buffer.from(binaryData.data, BINARY_ENCODING) as Buffer;

							headers['Content-Type'] = binaryData.mimeType;

							headers['Content-MD5'] = createHash('md5').update(body).digest('base64');

							responseData = await s3ApiRequestSOAP.call(this, bucketName, 'PUT', `${path}${fileName || binaryData.fileName}`, body, qs, headers, {}, region);

						} else {

							const fileContent = this.getNodeParameter('fileContent', i) as string;

							body = Buffer.from(fileContent, 'utf8');

							headers['Content-Type'] = 'text/html';

							headers['Content-MD5'] = createHash('md5').update(fileContent).digest('base64');

							responseData = await s3ApiRequestSOAP.call(this, bucketName, 'PUT', `${path}${fileName}`, body, qs, headers, {}, region);
						}
						returnData.push({ success: true });
					}
				}
			} catch (error) {
				if (this.continueOnFail()) {
					if (resource === 'file' && operation === 'download') {
						items[i].json = { error: error.message };
					}else {
						returnData.push({ error: error.message });
					}
					continue;
				}
				throw error;
			}
		}
		if (resource === 'file' && operation === 'download') {
			// For file downloads the files get attached to the existing items
			return this.prepareOutputData(items);
		} else {
			return [this.helpers.returnJsonArray(returnData)];
		}
	}
}