From 4b22df31ad292cff8330c2eef1084890caa9e83c Mon Sep 17 00:00:00 2001 From: Denis Palashevskii Date: Wed, 5 Aug 2020 14:40:45 +0400 Subject: [PATCH] Add support for custom S3 endpoint in AWS S3 node --- .../nodes-base/nodes/Aws/S3/AwsS3.node.ts | 113 +++++++++++++----- .../nodes/Aws/S3/GenericFunctions.ts | 55 +++++++-- 2 files changed, 124 insertions(+), 44 deletions(-) diff --git a/packages/nodes-base/nodes/Aws/S3/AwsS3.node.ts b/packages/nodes-base/nodes/Aws/S3/AwsS3.node.ts index 5af9ad3d33..5afa5dbff0 100644 --- a/packages/nodes-base/nodes/Aws/S3/AwsS3.node.ts +++ b/packages/nodes-base/nodes/Aws/S3/AwsS3.node.ts @@ -41,9 +41,9 @@ import { } from './FileDescription'; import { - awsApiRequestREST, - awsApiRequestSOAP, - awsApiRequestSOAPAllItems, + s3ApiRequestREST, + s3ApiRequestSOAP, + s3ApiRequestSOAPAllItems, } from './GenericFunctions'; export class AwsS3 implements INodeType { @@ -65,9 +65,44 @@ export class AwsS3 implements INodeType { { name: 'aws', required: true, - } + displayOptions: { + show: { + endpoint: [ + 'aws', + ], + }, + }, + }, + { + name: 'customS3Endpoint', + required: true, + displayOptions: { + show: { + endpoint: [ + 'customS3Endpoint', + ], + }, + }, + }, ], properties: [ + { + displayName: 'Endpoint', + name: 'endpoint', + type: 'options', + options: [ + { + name: 'AWS', + value: 'aws', + }, + { + name: 'Custom S3 endpoint', + value: 'customS3Endpoint', + }, + ], + default: 'aws', + description: 'The endpoint of S3 compatible service.', + }, { displayName: 'Resource', name: 'resource', @@ -113,7 +148,21 @@ export class AwsS3 implements INodeType { if (resource === 'bucket') { //https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateBucket.html if (operation === 'create') { - const credentials = this.getCredentials('aws'); + + let credentials; + + const endpointType = this.getNodeParameter('endpoint', 0); + + try { + if (endpointType === 'aws') { + credentials = this.getCredentials('aws'); + } else { + credentials = this.getCredentials('customS3Endpoint'); + } + } catch (error) { + throw new Error(error); + } + const name = this.getNodeParameter('name', i) as string; const additionalFields = this.getNodeParameter('additionalFields', i) as IDataObject; if (additionalFields.acl) { @@ -158,7 +207,7 @@ export class AwsS3 implements INodeType { const builder = new Builder(); data = builder.buildObject(body); } - responseData = await awsApiRequestSOAP.call(this, `${name}.s3`, 'PUT', '', data, qs, headers); + responseData = await s3ApiRequestSOAP.call(this, `${name}`, 'PUT', '', data, qs, headers); returnData.push({ success: true }); } @@ -166,10 +215,10 @@ export class AwsS3 implements INodeType { if (operation === 'getAll') { const returnAll = this.getNodeParameter('returnAll', 0) as boolean; if (returnAll) { - responseData = await awsApiRequestSOAPAllItems.call(this, 'ListAllMyBucketsResult.Buckets.Bucket', 's3', 'GET', ''); + responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListAllMyBucketsResult.Buckets.Bucket', '', 'GET', ''); } else { qs.limit = this.getNodeParameter('limit', 0) as number; - responseData = await awsApiRequestSOAPAllItems.call(this, 'ListAllMyBucketsResult.Buckets.Bucket', 's3', 'GET', '', '', qs); + responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListAllMyBucketsResult.Buckets.Bucket', '', 'GET', '', '', qs); responseData = responseData.slice(0, qs.limit); } returnData.push.apply(returnData, responseData); @@ -208,15 +257,15 @@ export class AwsS3 implements INodeType { qs['list-type'] = 2; - responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { location: '' }); + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' }); const region = responseData.LocationConstraint._ as string; if (returnAll) { - responseData = await awsApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', `${bucketName}.s3`, 'GET', '', '', qs, {}, {}, region); + responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '', '', qs, {}, {}, region); } else { qs['max-keys'] = this.getNodeParameter('limit', 0) as number; - responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', qs, {}, {}, region); + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', qs, {}, {}, region); responseData = responseData.ListBucketResult.Contents; } if (Array.isArray(responseData)) { @@ -243,11 +292,11 @@ export class AwsS3 implements INodeType { if (additionalFields.storageClass) { headers['x-amz-storage-class'] = (snakeCase(additionalFields.storageClass as string)).toUpperCase(); } - responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { location: '' }); + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' }); const region = responseData.LocationConstraint._; - responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'PUT', path, '', qs, headers, {}, region); + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'PUT', path, '', qs, headers, {}, region); returnData.push({ success: true }); } //https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html @@ -255,16 +304,16 @@ export class AwsS3 implements INodeType { const bucketName = this.getNodeParameter('bucketName', i) as string; const folderKey = this.getNodeParameter('folderKey', i) as string; - responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { location: '' }); + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' }); const region = responseData.LocationConstraint._; - responseData = await awsApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', `${bucketName}.s3`, 'GET', '/', '', { 'list-type': 2, prefix: folderKey }, {}, {}, region); + responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '/', '', { 'list-type': 2, prefix: folderKey }, {}, {}, region); // folder empty then just delete it if (responseData.length === 0) { - responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'DELETE', `/${folderKey}`, '', qs, {}, {}, region); + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'DELETE', `/${folderKey}`, '', qs, {}, {}, region); responseData = { deleted: [ { 'Key': folderKey } ] }; @@ -293,7 +342,7 @@ export class AwsS3 implements INodeType { headers['Content-Type'] = 'application/xml'; - responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'POST', '/', data, { delete: '' } , headers, {}, region); + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'POST', '/', data, { delete: '' } , headers, {}, region); responseData = { deleted: responseData.DeleteResult.Deleted }; } @@ -315,15 +364,15 @@ export class AwsS3 implements INodeType { qs['list-type'] = 2; - responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { location: '' }); + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' }); const region = responseData.LocationConstraint._; if (returnAll) { - responseData = await awsApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', `${bucketName}.s3`, 'GET', '', '', qs, {}, {}, region); + responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '', '', qs, {}, {}, region); } else { qs.limit = this.getNodeParameter('limit', 0) as number; - responseData = await awsApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', `${bucketName}.s3`, 'GET', '', '', qs, {}, {}, region); + responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '', '', qs, {}, {}, region); } if (Array.isArray(responseData)) { responseData = responseData.filter((e: IDataObject) => (e.Key as string).endsWith('/') && e.Size === '0' && e.Key !== options.folderKey); @@ -404,11 +453,11 @@ export class AwsS3 implements INodeType { const destination = `/${destinationParts.slice(2, destinationParts.length).join('/')}`; - responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { location: '' }); + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' }); const region = responseData.LocationConstraint._; - responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'PUT', destination, '', qs, headers, {}, region); + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'PUT', destination, '', qs, headers, {}, region); returnData.push(responseData.CopyObjectResult); } @@ -425,11 +474,11 @@ export class AwsS3 implements INodeType { throw new Error('Downloding a whole directory is not yet supported, please provide a file key'); } - let region = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { location: '' }); + let region = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' }); region = region.LocationConstraint._; - const response = await awsApiRequestREST.call(this, `${bucketName}.s3`, 'GET', `/${fileKey}`, '', qs, {}, { encoding: null, resolveWithFullResponse: true }, region); + const response = await s3ApiRequestREST.call(this, bucketName, 'GET', `/${fileKey}`, '', qs, {}, { encoding: null, resolveWithFullResponse: true }, region); let mimeType: string | undefined; if (response.headers['content-type']) { @@ -468,11 +517,11 @@ export class AwsS3 implements INodeType { qs.versionId = options.versionId as string; } - responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { location: '' }); + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' }); const region = responseData.LocationConstraint._; - responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'DELETE', `/${fileKey}`, '', qs, {}, {}, region); + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'DELETE', `/${fileKey}`, '', qs, {}, {}, region); returnData.push({ success: true }); } @@ -494,15 +543,15 @@ export class AwsS3 implements INodeType { qs['list-type'] = 2; - responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { location: '' }); + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' }); const region = responseData.LocationConstraint._; if (returnAll) { - responseData = await awsApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', `${bucketName}.s3`, 'GET', '', '', qs, {}, {}, region); + responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '', '', qs, {}, {}, region); } else { qs.limit = this.getNodeParameter('limit', 0) as number; - responseData = await awsApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', `${bucketName}.s3`, 'GET', '', '', qs, {}, {}, region); + responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '', '', qs, {}, {}, region); responseData = responseData.splice(0, qs.limit); } if (Array.isArray(responseData)) { @@ -581,7 +630,7 @@ export class AwsS3 implements INodeType { headers['x-amz-tagging'] = tags.join('&'); } - responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { location: '' }); + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' }); const region = responseData.LocationConstraint._; @@ -604,7 +653,7 @@ export class AwsS3 implements INodeType { headers['Content-MD5'] = createHash('md5').update(body).digest('base64'); - responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'PUT', `${path}${fileName || binaryData.fileName}`, body, qs, headers, {}, region); + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'PUT', `${path}${fileName || binaryData.fileName}`, body, qs, headers, {}, region); } else { @@ -616,7 +665,7 @@ export class AwsS3 implements INodeType { headers['Content-MD5'] = createHash('md5').update(fileContent).digest('base64'); - responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'PUT', `${path}${fileName}`, body, qs, headers, {}, region); + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'PUT', `${path}${fileName}`, body, qs, headers, {}, region); } returnData.push({ success: true }); } diff --git a/packages/nodes-base/nodes/Aws/S3/GenericFunctions.ts b/packages/nodes-base/nodes/Aws/S3/GenericFunctions.ts index 92d620d102..40ecb96949 100644 --- a/packages/nodes-base/nodes/Aws/S3/GenericFunctions.ts +++ b/packages/nodes-base/nodes/Aws/S3/GenericFunctions.ts @@ -25,16 +25,47 @@ import { IDataObject, } from 'n8n-workflow'; -export async function awsApiRequest(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions | IWebhookFunctions, service: string, method: string, path: string, body?: string | Buffer, query: IDataObject = {}, headers?: object, option: IDataObject = {}, region?: string): Promise { // tslint:disable-line:no-any - const credentials = this.getCredentials('aws'); +export async function s3ApiRequest(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions | IWebhookFunctions, bucket: string, method: string, path: string, body?: string | Buffer, query: IDataObject = {}, headers?: object, option: IDataObject = {}, region?: string): Promise { // tslint:disable-line:no-any + + let credentials; + + const endpointType = this.getNodeParameter('endpoint', 0); + + try { + if (endpointType === 'aws') { + credentials = this.getCredentials('aws'); + } else { + credentials = this.getCredentials('customS3Endpoint'); + } + } catch (error) { + throw new Error(error); + } + if (credentials === undefined) { throw new Error('No credentials got returned!'); } - const endpoint = `${service}.${region || credentials.region}.amazonaws.com`; + let endpoint = endpointType === 'aws' ? `${bucket}.s3.${region || credentials.region}.amazonaws.com` : credentials.endpoint; + + // Using path-style for non-AWS services + if (bucket && endpointType === 'customS3Endpoint') { + if (credentials.forcePathStyle) { + path = `/${bucket}${path}`; + } else { + endpoint = `${bucket}.${endpoint}`; + } + } // Sign AWS API request with the user credentials - const signOpts = {headers: headers || {}, host: endpoint, method, path: `${path}?${queryToString(query).replace(/\+/g, '%2B')}`, body}; + const signOpts = { + headers: headers || {}, + region: region || credentials.region, + host: endpoint, + method, + path: `${path}?${queryToString(query).replace(/\+/g, '%2B')}`, + service: 's3', + body + }; sign(signOpts, { accessKeyId: `${credentials.accessKeyId}`, secretAccessKey: `${credentials.secretAccessKey}`}); @@ -42,7 +73,7 @@ export async function awsApiRequest(this: IHookFunctions | IExecuteFunctions | I headers: signOpts.headers, method, qs: query, - uri: `https://${endpoint}${signOpts.path}`, + uri: `https://${endpoint}${path}`, body: signOpts.body, }; @@ -52,7 +83,7 @@ export async function awsApiRequest(this: IHookFunctions | IExecuteFunctions | I try { return await this.helpers.request!(options); } catch (error) { - const errorMessage = error.response.body.message || error.response.body.Message || error.message; + const errorMessage = error.response?.body.message || error.response?.body.Message || error.message; if (error.statusCode === 403) { if (errorMessage === 'The security token included in the request is invalid.') { @@ -66,8 +97,8 @@ export async function awsApiRequest(this: IHookFunctions | IExecuteFunctions | I } } -export async function awsApiRequestREST(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions, service: string, method: string, path: string, body?: string, query: IDataObject = {}, headers?: object, options: IDataObject = {}, region?: string): Promise { // tslint:disable-line:no-any - const response = await awsApiRequest.call(this, service, method, path, body, query, headers, options, region); +export async function s3ApiRequestREST(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions, bucket: string, method: string, path: string, body?: string, query: IDataObject = {}, headers?: object, options: IDataObject = {}, region?: string): Promise { // tslint:disable-line:no-any + const response = await s3ApiRequest.call(this, bucket, method, path, body, query, headers, options, region); try { return JSON.parse(response); } catch (e) { @@ -75,8 +106,8 @@ export async function awsApiRequestREST(this: IHookFunctions | IExecuteFunctions } } -export async function awsApiRequestSOAP(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions | IWebhookFunctions, service: string, method: string, path: string, body?: string | Buffer, query: IDataObject = {}, headers?: object, option: IDataObject = {}, region?: string): Promise { // tslint:disable-line:no-any - const response = await awsApiRequest.call(this, service, method, path, body, query, headers, option, region); +export async function s3ApiRequestSOAP(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions | IWebhookFunctions, bucket: string, method: string, path: string, body?: string | Buffer, query: IDataObject = {}, headers?: object, option: IDataObject = {}, region?: string): Promise { // tslint:disable-line:no-any + const response = await s3ApiRequest.call(this, bucket, method, path, body, query, headers, option, region); try { return await new Promise((resolve, reject) => { parseString(response, { explicitArray: false }, (err, data) => { @@ -91,14 +122,14 @@ export async function awsApiRequestSOAP(this: IHookFunctions | IExecuteFunctions } } -export async function awsApiRequestSOAPAllItems(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions | IWebhookFunctions, propertyName: string, service: string, method: string, path: string, body?: string, query: IDataObject = {}, headers: IDataObject = {}, option: IDataObject = {}, region?: string): Promise { // tslint:disable-line:no-any +export async function s3ApiRequestSOAPAllItems(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions | IWebhookFunctions, propertyName: string, service: string, method: string, path: string, body?: string, query: IDataObject = {}, headers: IDataObject = {}, option: IDataObject = {}, region?: string): Promise { // tslint:disable-line:no-any const returnData: IDataObject[] = []; let responseData; do { - responseData = await awsApiRequestSOAP.call(this, service, method, path, body, query, headers, option, region); + responseData = await s3ApiRequestSOAP.call(this, service, method, path, body, query, headers, option, region); //https://forums.aws.amazon.com/thread.jspa?threadID=55746 if (get(responseData, `${propertyName.split('.')[0]}.NextContinuationToken`)) {