diff --git a/packages/nodes-base/credentials/CustomS3Endpoint.credentials.ts b/packages/nodes-base/credentials/CustomS3Endpoint.credentials.ts new file mode 100644 index 0000000000..266a38fddf --- /dev/null +++ b/packages/nodes-base/credentials/CustomS3Endpoint.credentials.ts @@ -0,0 +1,45 @@ +import { + ICredentialType, + NodePropertyTypes, +} from 'n8n-workflow'; + + +export class CustomS3Endpoint implements ICredentialType { + name = 'customS3Endpoint'; + displayName = 'Custom S3'; + properties = [ + { + displayName: 'S3 endpoint', + name: 'endpoint', + type: 'string' as NodePropertyTypes, + default: '' + }, + { + displayName: 'Region', + name: 'region', + type: 'string' as NodePropertyTypes, + default: 'us-east-1', + }, + { + displayName: 'Access Key Id', + name: 'accessKeyId', + type: 'string' as NodePropertyTypes, + default: '', + }, + { + displayName: 'Secret Access Key', + name: 'secretAccessKey', + type: 'string' as NodePropertyTypes, + default: '', + typeOptions: { + password: true, + }, + }, + { + displayName: 'Force path style', + name: 'forcePathStyle', + type: 'boolean' as NodePropertyTypes, + default: false + }, + ]; +} diff --git a/packages/nodes-base/nodes/Aws/GenericFunctions.ts b/packages/nodes-base/nodes/Aws/GenericFunctions.ts index ef334b1d67..880c0a02bb 100644 --- a/packages/nodes-base/nodes/Aws/GenericFunctions.ts +++ b/packages/nodes-base/nodes/Aws/GenericFunctions.ts @@ -20,7 +20,7 @@ export async function awsApiRequest(this: IHookFunctions | IExecuteFunctions | I // Sign AWS API request with the user credentials const signOpts = { headers: headers || {}, host: endpoint, method, path, body }; - sign(signOpts, { accessKeyId: `${credentials.accessKeyId}`, secretAccessKey: `${credentials.secretAccessKey}` }); + sign(signOpts, { accessKeyId: `${credentials.accessKeyId}`.trim(), secretAccessKey: `${credentials.secretAccessKey}`.trim() }); const options: OptionsWithUri = { headers: signOpts.headers, diff --git a/packages/nodes-base/nodes/Aws/S3/GenericFunctions.ts b/packages/nodes-base/nodes/Aws/S3/GenericFunctions.ts index 92d620d102..fd7fe838f6 100644 --- a/packages/nodes-base/nodes/Aws/S3/GenericFunctions.ts +++ b/packages/nodes-base/nodes/Aws/S3/GenericFunctions.ts @@ -36,7 +36,7 @@ export async function awsApiRequest(this: IHookFunctions | IExecuteFunctions | I // Sign AWS API request with the user credentials const signOpts = {headers: headers || {}, host: endpoint, method, path: `${path}?${queryToString(query).replace(/\+/g, '%2B')}`, body}; - sign(signOpts, { accessKeyId: `${credentials.accessKeyId}`, secretAccessKey: `${credentials.secretAccessKey}`}); + sign(signOpts, { accessKeyId: `${credentials.accessKeyId}`.trim(), secretAccessKey: `${credentials.secretAccessKey}`.trim()}); const options: OptionsWithUri = { headers: signOpts.headers, diff --git a/packages/nodes-base/nodes/S3/GenericFunctions.ts b/packages/nodes-base/nodes/S3/GenericFunctions.ts new file mode 100644 index 0000000000..093b7cfb71 --- /dev/null +++ b/packages/nodes-base/nodes/S3/GenericFunctions.ts @@ -0,0 +1,155 @@ +import { + sign, +} from 'aws4'; + +import { + get, +} from 'lodash'; + +import { + OptionsWithUri, +} from 'request'; + +import { + parseString, +} from 'xml2js'; + +import { + IExecuteFunctions, + IHookFunctions, + ILoadOptionsFunctions, + IWebhookFunctions, +} from 'n8n-core'; + +import { + IDataObject, + } from 'n8n-workflow'; + +import { URL } from 'url'; + +export async function s3ApiRequest(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions | IWebhookFunctions, bucket: string, method: string, path: string, body?: string | Buffer, query: IDataObject = {}, headers?: object, option: IDataObject = {}, region?: string): Promise { // tslint:disable-line:no-any + + let credentials; + + credentials = this.getCredentials('customS3Endpoint'); + + if (credentials === undefined) { + throw new Error('No credentials got returned!'); + } + + if (!(credentials.endpoint as string).startsWith('http')) { + throw new Error('HTTP(S) Scheme is required in endpoint definition'); + } + + const endpoint = new URL(credentials.endpoint as string); + + if (bucket) { + if (credentials.forcePathStyle) { + path = `/${bucket}${path}`; + } else { + endpoint.host = `${bucket}.${endpoint.host}`; + } + } + + endpoint.pathname = path; + + // Sign AWS API request with the user credentials + const signOpts = { + headers: headers || {}, + region: region || credentials.region, + host: endpoint.host, + method, + path: `${path}?${queryToString(query).replace(/\+/g, '%2B')}`, + service: 's3', + body + }; + + sign(signOpts, { accessKeyId: `${credentials.accessKeyId}`.trim(), secretAccessKey: `${credentials.secretAccessKey}`.trim()}); + + const options: OptionsWithUri = { + headers: signOpts.headers, + method, + qs: query, + uri: endpoint, + body: signOpts.body, + }; + + if (Object.keys(option).length !== 0) { + Object.assign(options, option); + } + try { + return await this.helpers.request!(options); + } catch (error) { + const errorMessage = error.response?.body.message || error.response?.body.Message || error.message; + + if (error.statusCode === 403) { + if (errorMessage === 'The security token included in the request is invalid.') { + throw new Error('The S3 credentials are not valid!'); + } else if (errorMessage.startsWith('The request signature we calculated does not match the signature you provided')) { + throw new Error('The S3 credentials are not valid!'); + } + } + + throw new Error(`S3 error response [${error.statusCode}]: ${errorMessage}`); + } +} + +export async function s3ApiRequestREST(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions, bucket: string, method: string, path: string, body?: string, query: IDataObject = {}, headers?: object, options: IDataObject = {}, region?: string): Promise { // tslint:disable-line:no-any + const response = await s3ApiRequest.call(this, bucket, method, path, body, query, headers, options, region); + try { + return JSON.parse(response); + } catch (e) { + return response; + } +} + +export async function s3ApiRequestSOAP(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions | IWebhookFunctions, bucket: string, method: string, path: string, body?: string | Buffer, query: IDataObject = {}, headers?: object, option: IDataObject = {}, region?: string): Promise { // tslint:disable-line:no-any + const response = await s3ApiRequest.call(this, bucket, method, path, body, query, headers, option, region); + try { + return await new Promise((resolve, reject) => { + parseString(response, { explicitArray: false }, (err, data) => { + if (err) { + return reject(err); + } + resolve(data); + }); + }); + } catch (e) { + return e; + } +} + +export async function s3ApiRequestSOAPAllItems(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions | IWebhookFunctions, propertyName: string, service: string, method: string, path: string, body?: string, query: IDataObject = {}, headers: IDataObject = {}, option: IDataObject = {}, region?: string): Promise { // tslint:disable-line:no-any + + const returnData: IDataObject[] = []; + + let responseData; + + do { + responseData = await s3ApiRequestSOAP.call(this, service, method, path, body, query, headers, option, region); + + //https://forums.aws.amazon.com/thread.jspa?threadID=55746 + if (get(responseData, `${propertyName.split('.')[0]}.NextContinuationToken`)) { + query['continuation-token'] = get(responseData, `${propertyName.split('.')[0]}.NextContinuationToken`); + } + if (get(responseData, propertyName)) { + if (Array.isArray(get(responseData, propertyName))) { + returnData.push.apply(returnData, get(responseData, propertyName)); + } else { + returnData.push(get(responseData, propertyName)); + } + } + if (query.limit && query.limit <= returnData.length) { + return returnData; + } + } while ( + get(responseData, `${propertyName.split('.')[0]}.IsTruncated`) !== undefined && + get(responseData, `${propertyName.split('.')[0]}.IsTruncated`) !== 'false' + ); + + return returnData; +} + +function queryToString(params: IDataObject) { + return Object.keys(params).map(key => key + '=' + params[key]).join('&'); +} diff --git a/packages/nodes-base/nodes/S3/S3.node.ts b/packages/nodes-base/nodes/S3/S3.node.ts new file mode 100644 index 0000000000..4272adb5b2 --- /dev/null +++ b/packages/nodes-base/nodes/S3/S3.node.ts @@ -0,0 +1,640 @@ + +import { + snakeCase, + paramCase, +} from 'change-case'; + +import { + createHash, +} from 'crypto'; + +import { + Builder, +} from 'xml2js'; + +import { + BINARY_ENCODING, + IExecuteFunctions, +} from 'n8n-core'; + +import { + IBinaryKeyData, + IDataObject, + INodeExecutionData, + INodeType, + INodeTypeDescription, +} from 'n8n-workflow'; + +import { + bucketFields, + bucketOperations, +} from '../Aws/S3/BucketDescription'; + +import { + folderFields, + folderOperations, +} from '../Aws/S3/FolderDescription'; + +import { + fileFields, + fileOperations, +} from '../Aws/S3/FileDescription'; + +import { + s3ApiRequestREST, + s3ApiRequestSOAP, + s3ApiRequestSOAPAllItems, +} from './GenericFunctions'; + +export class S3 implements INodeType { + description: INodeTypeDescription = { + displayName: 'S3', + name: 'S3', + icon: 'file:generic-s3.png', + group: ['output'], + version: 1, + subtitle: '={{$parameter["operation"] + ": " + $parameter["resource"]}}', + description: 'Sends data to any S3-compatible services', + defaults: { + name: 'S3', + color: '#d05b4b', + }, + inputs: ['main'], + outputs: ['main'], + credentials: [ + { + name: 'customS3Endpoint', + required: true, + }, + ], + properties: [ + { + displayName: 'Resource', + name: 'resource', + type: 'options', + options: [ + { + name: 'Bucket', + value: 'bucket', + }, + { + name: 'File', + value: 'file', + }, + { + name: 'Folder', + value: 'folder', + }, + ], + default: 'file', + description: 'The operation to perform.', + }, + // BUCKET + ...bucketOperations, + ...bucketFields, + // FOLDER + ...folderOperations, + ...folderFields, + // UPLOAD + ...fileOperations, + ...fileFields, + ], + }; + + async execute(this: IExecuteFunctions): Promise { + const items = this.getInputData(); + const returnData: IDataObject[] = []; + const qs: IDataObject = {}; + const headers: IDataObject = {}; + let responseData; + const resource = this.getNodeParameter('resource', 0) as string; + const operation = this.getNodeParameter('operation', 0) as string; + for (let i = 0; i < items.length; i++) { + if (resource === 'bucket') { + //https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateBucket.html + if (operation === 'create') { + + let credentials; + + try { + credentials = this.getCredentials('customS3Endpoint'); + } catch (error) { + throw new Error(error); + } + + const name = this.getNodeParameter('name', i) as string; + const additionalFields = this.getNodeParameter('additionalFields', i) as IDataObject; + if (additionalFields.acl) { + headers['x-amz-acl'] = paramCase(additionalFields.acl as string); + } + if (additionalFields.bucketObjectLockEnabled) { + headers['x-amz-bucket-object-lock-enabled'] = additionalFields.bucketObjectLockEnabled as boolean; + } + if (additionalFields.grantFullControl) { + headers['x-amz-grant-full-control'] = ''; + } + if (additionalFields.grantRead) { + headers['x-amz-grant-read'] = ''; + } + if (additionalFields.grantReadAcp) { + headers['x-amz-grant-read-acp'] = ''; + } + if (additionalFields.grantWrite) { + headers['x-amz-grant-write'] = ''; + } + if (additionalFields.grantWriteAcp) { + headers['x-amz-grant-write-acp'] = ''; + } + let region = credentials!.region as string; + + if (additionalFields.region) { + region = additionalFields.region as string; + } + + const body: IDataObject = { + CreateBucketConfiguration: { + '$': { + xmlns: 'http://s3.amazonaws.com/doc/2006-03-01/', + }, + } + }; + let data = ''; + // if credentials has the S3 defaul region (us-east-1) the body (XML) does not have to be sent. + if (region !== 'us-east-1') { + // @ts-ignore + body.CreateBucketConfiguration.LocationConstraint = [region]; + const builder = new Builder(); + data = builder.buildObject(body); + } + responseData = await s3ApiRequestSOAP.call(this, `${name}`, 'PUT', '', data, qs, headers); + + returnData.push({ success: true }); + } + //https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListBuckets.html + if (operation === 'getAll') { + const returnAll = this.getNodeParameter('returnAll', 0) as boolean; + if (returnAll) { + responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListAllMyBucketsResult.Buckets.Bucket', '', 'GET', ''); + } else { + qs.limit = this.getNodeParameter('limit', 0) as number; + responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListAllMyBucketsResult.Buckets.Bucket', '', 'GET', '', '', qs); + responseData = responseData.slice(0, qs.limit); + } + returnData.push.apply(returnData, responseData); + } + + //https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html + if (operation === 'search') { + const bucketName = this.getNodeParameter('bucketName', i) as string; + const returnAll = this.getNodeParameter('returnAll', 0) as boolean; + const additionalFields = this.getNodeParameter('additionalFields', 0) as IDataObject; + + if (additionalFields.prefix) { + qs['prefix'] = additionalFields.prefix as string; + } + + if (additionalFields.encodingType) { + qs['encoding-type'] = additionalFields.encodingType as string; + } + + if (additionalFields.delmiter) { + qs['delimiter'] = additionalFields.delmiter as string; + } + + if (additionalFields.fetchOwner) { + qs['fetch-owner'] = additionalFields.fetchOwner as string; + } + + if (additionalFields.startAfter) { + qs['start-after'] = additionalFields.startAfter as string; + } + + if (additionalFields.requesterPays) { + qs['x-amz-request-payer'] = 'requester'; + } + + qs['list-type'] = 2; + + + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' }); + + const region = responseData.LocationConstraint._ as string; + + if (returnAll) { + responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '', '', qs, {}, {}, region); + } else { + qs['max-keys'] = this.getNodeParameter('limit', 0) as number; + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', qs, {}, {}, region); + responseData = responseData.ListBucketResult.Contents; + } + if (Array.isArray(responseData)) { + returnData.push.apply(returnData, responseData); + } else { + returnData.push(responseData); + } + } + } + if (resource === 'folder') { + //https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html + if (operation === 'create') { + const bucketName = this.getNodeParameter('bucketName', i) as string; + const folderName = this.getNodeParameter('folderName', i) as string; + const additionalFields = this.getNodeParameter('additionalFields', i) as IDataObject; + let path = `/${folderName}/`; + + if (additionalFields.requesterPays) { + headers['x-amz-request-payer'] = 'requester'; + } + if (additionalFields.parentFolderKey) { + path = `/${additionalFields.parentFolderKey}${folderName}/`; + } + if (additionalFields.storageClass) { + headers['x-amz-storage-class'] = (snakeCase(additionalFields.storageClass as string)).toUpperCase(); + } + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' }); + + const region = responseData.LocationConstraint._; + + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'PUT', path, '', qs, headers, {}, region); + returnData.push({ success: true }); + } + //https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html + if (operation === 'delete') { + const bucketName = this.getNodeParameter('bucketName', i) as string; + const folderKey = this.getNodeParameter('folderKey', i) as string; + + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' }); + + const region = responseData.LocationConstraint._; + + responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '/', '', { 'list-type': 2, prefix: folderKey }, {}, {}, region); + + // folder empty then just delete it + if (responseData.length === 0) { + + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'DELETE', `/${folderKey}`, '', qs, {}, {}, region); + + responseData = { deleted: [ { 'Key': folderKey } ] }; + + } else { + // delete everything inside the folder + const body: IDataObject = { + Delete: { + '$': { + xmlns: 'http://s3.amazonaws.com/doc/2006-03-01/', + }, + Object: [], + }, + }; + + for (const childObject of responseData) { + //@ts-ignore + (body.Delete.Object as IDataObject[]).push({ + Key: childObject.Key as string + }); + } + + const builder = new Builder(); + const data = builder.buildObject(body); + + headers['Content-MD5'] = createHash('md5').update(data).digest('base64'); + + headers['Content-Type'] = 'application/xml'; + + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'POST', '/', data, { delete: '' } , headers, {}, region); + + responseData = { deleted: responseData.DeleteResult.Deleted }; + } + returnData.push(responseData); + } + //https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html + if (operation === 'getAll') { + const bucketName = this.getNodeParameter('bucketName', i) as string; + const returnAll = this.getNodeParameter('returnAll', 0) as boolean; + const options = this.getNodeParameter('options', 0) as IDataObject; + + if (options.folderKey) { + qs['prefix'] = options.folderKey as string; + } + + if (options.fetchOwner) { + qs['fetch-owner'] = options.fetchOwner as string; + } + + qs['list-type'] = 2; + + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' }); + + const region = responseData.LocationConstraint._; + + if (returnAll) { + responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '', '', qs, {}, {}, region); + } else { + qs.limit = this.getNodeParameter('limit', 0) as number; + responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '', '', qs, {}, {}, region); + } + if (Array.isArray(responseData)) { + responseData = responseData.filter((e: IDataObject) => (e.Key as string).endsWith('/') && e.Size === '0' && e.Key !== options.folderKey); + if (qs.limit) { + responseData = responseData.splice(0, qs.limit as number); + } + returnData.push.apply(returnData, responseData); + } + } + } + if (resource === 'file') { + //https://docs.aws.amazon.com/AmazonS3/latest/API/API_CopyObject.html + if (operation === 'copy') { + const sourcePath = this.getNodeParameter('sourcePath', i) as string; + const destinationPath = this.getNodeParameter('destinationPath', i) as string; + const additionalFields = this.getNodeParameter('additionalFields', i) as IDataObject; + + headers['x-amz-copy-source'] = sourcePath; + + if (additionalFields.requesterPays) { + headers['x-amz-request-payer'] = 'requester'; + } + if (additionalFields.storageClass) { + headers['x-amz-storage-class'] = (snakeCase(additionalFields.storageClass as string)).toUpperCase(); + } + if (additionalFields.acl) { + headers['x-amz-acl'] = paramCase(additionalFields.acl as string); + } + if (additionalFields.grantFullControl) { + headers['x-amz-grant-full-control'] = ''; + } + if (additionalFields.grantRead) { + headers['x-amz-grant-read'] = ''; + } + if (additionalFields.grantReadAcp) { + headers['x-amz-grant-read-acp'] = ''; + } + if (additionalFields.grantWriteAcp) { + headers['x-amz-grant-write-acp'] = ''; + } + if (additionalFields.lockLegalHold) { + headers['x-amz-object-lock-legal-hold'] = (additionalFields.lockLegalHold as boolean) ? 'ON' : 'OFF'; + } + if (additionalFields.lockMode) { + headers['x-amz-object-lock-mode'] = (additionalFields.lockMode as string).toUpperCase(); + } + if (additionalFields.lockRetainUntilDate) { + headers['x-amz-object-lock-retain-until-date'] = additionalFields.lockRetainUntilDate as string; + } + if (additionalFields.serverSideEncryption) { + headers['x-amz-server-side-encryption'] = additionalFields.serverSideEncryption as string; + } + if (additionalFields.encryptionAwsKmsKeyId) { + headers['x-amz-server-side-encryption-aws-kms-key-id'] = additionalFields.encryptionAwsKmsKeyId as string; + } + if (additionalFields.serverSideEncryptionContext) { + headers['x-amz-server-side-encryption-context'] = additionalFields.serverSideEncryptionContext as string; + } + if (additionalFields.serversideEncryptionCustomerAlgorithm) { + headers['x-amz-server-side-encryption-customer-algorithm'] = additionalFields.serversideEncryptionCustomerAlgorithm as string; + } + if (additionalFields.serversideEncryptionCustomerKey) { + headers['x-amz-server-side-encryption-customer-key'] = additionalFields.serversideEncryptionCustomerKey as string; + } + if (additionalFields.serversideEncryptionCustomerKeyMD5) { + headers['x-amz-server-side-encryption-customer-key-MD5'] = additionalFields.serversideEncryptionCustomerKeyMD5 as string; + } + if (additionalFields.taggingDirective) { + headers['x-amz-tagging-directive'] = (additionalFields.taggingDirective as string).toUpperCase(); + } + if (additionalFields.metadataDirective) { + headers['x-amz-metadata-directive'] = (additionalFields.metadataDirective as string).toUpperCase(); + } + + const destinationParts = destinationPath.split('/'); + + const bucketName = destinationParts[1]; + + const destination = `/${destinationParts.slice(2, destinationParts.length).join('/')}`; + + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' }); + + const region = responseData.LocationConstraint._; + + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'PUT', destination, '', qs, headers, {}, region); + returnData.push(responseData.CopyObjectResult); + + } + //https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html + if (operation === 'download') { + + const bucketName = this.getNodeParameter('bucketName', i) as string; + + const fileKey = this.getNodeParameter('fileKey', i) as string; + + const fileName = fileKey.split('/')[fileKey.split('/').length - 1]; + + if (fileKey.substring(fileKey.length - 1) === '/') { + throw new Error('Downloding a whole directory is not yet supported, please provide a file key'); + } + + let region = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' }); + + region = region.LocationConstraint._; + + const response = await s3ApiRequestREST.call(this, bucketName, 'GET', `/${fileKey}`, '', qs, {}, { encoding: null, resolveWithFullResponse: true }, region); + + let mimeType: string | undefined; + if (response.headers['content-type']) { + mimeType = response.headers['content-type']; + } + + const newItem: INodeExecutionData = { + json: items[i].json, + binary: {}, + }; + + if (items[i].binary !== undefined) { + // Create a shallow copy of the binary data so that the old + // data references which do not get changed still stay behind + // but the incoming data does not get changed. + Object.assign(newItem.binary, items[i].binary); + } + + items[i] = newItem; + + const dataPropertyNameDownload = this.getNodeParameter('binaryPropertyName', i) as string; + + const data = Buffer.from(response.body as string, 'utf8'); + + items[i].binary![dataPropertyNameDownload] = await this.helpers.prepareBinaryData(data as unknown as Buffer, fileName, mimeType); + } + //https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObject.html + if (operation === 'delete') { + const bucketName = this.getNodeParameter('bucketName', i) as string; + + const fileKey = this.getNodeParameter('fileKey', i) as string; + + const options = this.getNodeParameter('options', i) as IDataObject; + + if (options.versionId) { + qs.versionId = options.versionId as string; + } + + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' }); + + const region = responseData.LocationConstraint._; + + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'DELETE', `/${fileKey}`, '', qs, {}, {}, region); + + returnData.push({ success: true }); + } + //https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html + if (operation === 'getAll') { + const bucketName = this.getNodeParameter('bucketName', i) as string; + const returnAll = this.getNodeParameter('returnAll', 0) as boolean; + const options = this.getNodeParameter('options', 0) as IDataObject; + + if (options.folderKey) { + qs['prefix'] = options.folderKey as string; + } + + if (options.fetchOwner) { + qs['fetch-owner'] = options.fetchOwner as string; + } + + qs['delimiter'] = '/'; + + qs['list-type'] = 2; + + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' }); + + const region = responseData.LocationConstraint._; + + if (returnAll) { + responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '', '', qs, {}, {}, region); + } else { + qs.limit = this.getNodeParameter('limit', 0) as number; + responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '', '', qs, {}, {}, region); + responseData = responseData.splice(0, qs.limit); + } + if (Array.isArray(responseData)) { + responseData = responseData.filter((e: IDataObject) => !(e.Key as string).endsWith('/') && e.Size !== '0'); + if (qs.limit) { + responseData = responseData.splice(0, qs.limit as number); + } + returnData.push.apply(returnData, responseData); + } + } + //https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html + if (operation === 'upload') { + const bucketName = this.getNodeParameter('bucketName', i) as string; + const fileName = this.getNodeParameter('fileName', i) as string; + const isBinaryData = this.getNodeParameter('binaryData', i) as boolean; + const additionalFields = this.getNodeParameter('additionalFields', i) as IDataObject; + const tagsValues = (this.getNodeParameter('tagsUi', i) as IDataObject).tagsValues as IDataObject[]; + let path = '/'; + let body; + + if (additionalFields.requesterPays) { + headers['x-amz-request-payer'] = 'requester'; + } + + if (additionalFields.parentFolderKey) { + path = `/${additionalFields.parentFolderKey}/`; + } + if (additionalFields.storageClass) { + headers['x-amz-storage-class'] = (snakeCase(additionalFields.storageClass as string)).toUpperCase(); + } + if (additionalFields.acl) { + headers['x-amz-acl'] = paramCase(additionalFields.acl as string); + } + if (additionalFields.grantFullControl) { + headers['x-amz-grant-full-control'] = ''; + } + if (additionalFields.grantRead) { + headers['x-amz-grant-read'] = ''; + } + if (additionalFields.grantReadAcp) { + headers['x-amz-grant-read-acp'] = ''; + } + if (additionalFields.grantWriteAcp) { + headers['x-amz-grant-write-acp'] = ''; + } + if (additionalFields.lockLegalHold) { + headers['x-amz-object-lock-legal-hold'] = (additionalFields.lockLegalHold as boolean) ? 'ON' : 'OFF'; + } + if (additionalFields.lockMode) { + headers['x-amz-object-lock-mode'] = (additionalFields.lockMode as string).toUpperCase(); + } + if (additionalFields.lockRetainUntilDate) { + headers['x-amz-object-lock-retain-until-date'] = additionalFields.lockRetainUntilDate as string; + } + if (additionalFields.serverSideEncryption) { + headers['x-amz-server-side-encryption'] = additionalFields.serverSideEncryption as string; + } + if (additionalFields.encryptionAwsKmsKeyId) { + headers['x-amz-server-side-encryption-aws-kms-key-id'] = additionalFields.encryptionAwsKmsKeyId as string; + } + if (additionalFields.serverSideEncryptionContext) { + headers['x-amz-server-side-encryption-context'] = additionalFields.serverSideEncryptionContext as string; + } + if (additionalFields.serversideEncryptionCustomerAlgorithm) { + headers['x-amz-server-side-encryption-customer-algorithm'] = additionalFields.serversideEncryptionCustomerAlgorithm as string; + } + if (additionalFields.serversideEncryptionCustomerKey) { + headers['x-amz-server-side-encryption-customer-key'] = additionalFields.serversideEncryptionCustomerKey as string; + } + if (additionalFields.serversideEncryptionCustomerKeyMD5) { + headers['x-amz-server-side-encryption-customer-key-MD5'] = additionalFields.serversideEncryptionCustomerKeyMD5 as string; + } + if (tagsValues) { + const tags: string[] = []; + tagsValues.forEach((o: IDataObject) => { tags.push(`${o.key}=${o.value}`); }); + headers['x-amz-tagging'] = tags.join('&'); + } + + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' }); + + const region = responseData.LocationConstraint._; + + if (isBinaryData) { + const binaryPropertyName = this.getNodeParameter('binaryPropertyName', 0) as string; + + if (items[i].binary === undefined) { + throw new Error('No binary data exists on item!'); + } + + if ((items[i].binary as IBinaryKeyData)[binaryPropertyName] === undefined) { + throw new Error(`No binary data property "${binaryPropertyName}" does not exists on item!`); + } + + const binaryData = (items[i].binary as IBinaryKeyData)[binaryPropertyName]; + + body = Buffer.from(binaryData.data, BINARY_ENCODING) as Buffer; + + headers['Content-Type'] = binaryData.mimeType; + + headers['Content-MD5'] = createHash('md5').update(body).digest('base64'); + + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'PUT', `${path}${fileName || binaryData.fileName}`, body, qs, headers, {}, region); + + } else { + + const fileContent = this.getNodeParameter('fileContent', i) as string; + + body = Buffer.from(fileContent, 'utf8'); + + headers['Content-Type'] = 'text/html'; + + headers['Content-MD5'] = createHash('md5').update(fileContent).digest('base64'); + + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'PUT', `${path}${fileName}`, body, qs, headers, {}, region); + } + returnData.push({ success: true }); + } + } + } + if (resource === 'file' && operation === 'download') { + // For file downloads the files get attached to the existing items + return this.prepareOutputData(items); + } else { + return [this.helpers.returnJsonArray(returnData)]; + } + } +} diff --git a/packages/nodes-base/nodes/S3/generic-s3.png b/packages/nodes-base/nodes/S3/generic-s3.png new file mode 100644 index 0000000000..4f9faf6dd1 Binary files /dev/null and b/packages/nodes-base/nodes/S3/generic-s3.png differ diff --git a/packages/nodes-base/package.json b/packages/nodes-base/package.json index f8b3b6db92..5c759a70a7 100644 --- a/packages/nodes-base/package.json +++ b/packages/nodes-base/package.json @@ -52,6 +52,7 @@ "dist/credentials/CopperApi.credentials.js", "dist/credentials/CalendlyApi.credentials.js", "dist/credentials/CustomerIoApi.credentials.js", + "dist/credentials/CustomS3Endpoint.credentials.js", "dist/credentials/CrateDb.credentials.js", "dist/credentials/DisqusApi.credentials.js", "dist/credentials/DriftApi.credentials.js", @@ -314,6 +315,7 @@ "dist/nodes/Rocketchat/Rocketchat.node.js", "dist/nodes/RssFeedRead.node.js", "dist/nodes/Rundeck/Rundeck.node.js", + "dist/nodes/S3/S3.node.js", "dist/nodes/Salesforce/Salesforce.node.js", "dist/nodes/Set.node.js", "dist/nodes/Shopify/Shopify.node.js",