From 823ae846bfae1ca29b91dd002dadb9075682568d Mon Sep 17 00:00:00 2001 From: Denis Palashevskii Date: Wed, 5 Aug 2020 00:34:22 +0400 Subject: [PATCH 1/8] Add Custom S3 Endpoint credentials --- .../CustomS3Endpoint.credentials.ts | 45 +++++++++++++++++++ packages/nodes-base/package.json | 1 + 2 files changed, 46 insertions(+) create mode 100644 packages/nodes-base/credentials/CustomS3Endpoint.credentials.ts diff --git a/packages/nodes-base/credentials/CustomS3Endpoint.credentials.ts b/packages/nodes-base/credentials/CustomS3Endpoint.credentials.ts new file mode 100644 index 0000000000..266a38fddf --- /dev/null +++ b/packages/nodes-base/credentials/CustomS3Endpoint.credentials.ts @@ -0,0 +1,45 @@ +import { + ICredentialType, + NodePropertyTypes, +} from 'n8n-workflow'; + + +export class CustomS3Endpoint implements ICredentialType { + name = 'customS3Endpoint'; + displayName = 'Custom S3'; + properties = [ + { + displayName: 'S3 endpoint', + name: 'endpoint', + type: 'string' as NodePropertyTypes, + default: '' + }, + { + displayName: 'Region', + name: 'region', + type: 'string' as NodePropertyTypes, + default: 'us-east-1', + }, + { + displayName: 'Access Key Id', + name: 'accessKeyId', + type: 'string' as NodePropertyTypes, + default: '', + }, + { + displayName: 'Secret Access Key', + name: 'secretAccessKey', + type: 'string' as NodePropertyTypes, + default: '', + typeOptions: { + password: true, + }, + }, + { + displayName: 'Force path style', + name: 'forcePathStyle', + type: 'boolean' as NodePropertyTypes, + default: false + }, + ]; +} diff --git a/packages/nodes-base/package.json b/packages/nodes-base/package.json index 6f22c97458..31248624b1 100644 --- a/packages/nodes-base/package.json +++ b/packages/nodes-base/package.json @@ -49,6 +49,7 @@ "dist/credentials/CopperApi.credentials.js", "dist/credentials/CalendlyApi.credentials.js", "dist/credentials/CustomerIoApi.credentials.js", + "dist/credentials/CustomS3Endpoint.credentials.js", "dist/credentials/CrateDb.credentials.js", "dist/credentials/DisqusApi.credentials.js", "dist/credentials/DriftApi.credentials.js", From 4b22df31ad292cff8330c2eef1084890caa9e83c Mon Sep 17 00:00:00 2001 From: Denis Palashevskii Date: Wed, 5 Aug 2020 14:40:45 +0400 Subject: [PATCH 2/8] Add support for custom S3 endpoint in AWS S3 node --- .../nodes-base/nodes/Aws/S3/AwsS3.node.ts | 113 +++++++++++++----- .../nodes/Aws/S3/GenericFunctions.ts | 55 +++++++-- 2 files changed, 124 insertions(+), 44 deletions(-) diff --git a/packages/nodes-base/nodes/Aws/S3/AwsS3.node.ts b/packages/nodes-base/nodes/Aws/S3/AwsS3.node.ts index 5af9ad3d33..5afa5dbff0 100644 --- a/packages/nodes-base/nodes/Aws/S3/AwsS3.node.ts +++ b/packages/nodes-base/nodes/Aws/S3/AwsS3.node.ts @@ -41,9 +41,9 @@ import { } from './FileDescription'; import { - awsApiRequestREST, - awsApiRequestSOAP, - awsApiRequestSOAPAllItems, + s3ApiRequestREST, + s3ApiRequestSOAP, + s3ApiRequestSOAPAllItems, } from './GenericFunctions'; export class AwsS3 implements INodeType { @@ -65,9 +65,44 @@ export class AwsS3 implements INodeType { { name: 'aws', required: true, - } + displayOptions: { + show: { + endpoint: [ + 'aws', + ], + }, + }, + }, + { + name: 'customS3Endpoint', + required: true, + displayOptions: { + show: { + endpoint: [ + 'customS3Endpoint', + ], + }, + }, + }, ], properties: [ + { + displayName: 'Endpoint', + name: 'endpoint', + type: 'options', + options: [ + { + name: 'AWS', + value: 'aws', + }, + { + name: 'Custom S3 endpoint', + value: 'customS3Endpoint', + }, + ], + default: 'aws', + description: 'The endpoint of S3 compatible service.', + }, { displayName: 'Resource', name: 'resource', @@ -113,7 +148,21 @@ export class AwsS3 implements INodeType { if (resource === 'bucket') { //https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateBucket.html if (operation === 'create') { - const credentials = this.getCredentials('aws'); + + let credentials; + + const endpointType = this.getNodeParameter('endpoint', 0); + + try { + if (endpointType === 'aws') { + credentials = this.getCredentials('aws'); + } else { + credentials = this.getCredentials('customS3Endpoint'); + } + } catch (error) { + throw new Error(error); + } + const name = this.getNodeParameter('name', i) as string; const additionalFields = this.getNodeParameter('additionalFields', i) as IDataObject; if (additionalFields.acl) { @@ -158,7 +207,7 @@ export class AwsS3 implements INodeType { const builder = new Builder(); data = builder.buildObject(body); } - responseData = await awsApiRequestSOAP.call(this, `${name}.s3`, 'PUT', '', data, qs, headers); + responseData = await s3ApiRequestSOAP.call(this, `${name}`, 'PUT', '', data, qs, headers); returnData.push({ success: true }); } @@ -166,10 +215,10 @@ export class AwsS3 implements INodeType { if (operation === 'getAll') { const returnAll = this.getNodeParameter('returnAll', 0) as boolean; if (returnAll) { - responseData = await awsApiRequestSOAPAllItems.call(this, 'ListAllMyBucketsResult.Buckets.Bucket', 's3', 'GET', ''); + responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListAllMyBucketsResult.Buckets.Bucket', '', 'GET', ''); } else { qs.limit = this.getNodeParameter('limit', 0) as number; - responseData = await awsApiRequestSOAPAllItems.call(this, 'ListAllMyBucketsResult.Buckets.Bucket', 's3', 'GET', '', '', qs); + responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListAllMyBucketsResult.Buckets.Bucket', '', 'GET', '', '', qs); responseData = responseData.slice(0, qs.limit); } returnData.push.apply(returnData, responseData); @@ -208,15 +257,15 @@ export class AwsS3 implements INodeType { qs['list-type'] = 2; - responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { location: '' }); + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' }); const region = responseData.LocationConstraint._ as string; if (returnAll) { - responseData = await awsApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', `${bucketName}.s3`, 'GET', '', '', qs, {}, {}, region); + responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '', '', qs, {}, {}, region); } else { qs['max-keys'] = this.getNodeParameter('limit', 0) as number; - responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', qs, {}, {}, region); + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', qs, {}, {}, region); responseData = responseData.ListBucketResult.Contents; } if (Array.isArray(responseData)) { @@ -243,11 +292,11 @@ export class AwsS3 implements INodeType { if (additionalFields.storageClass) { headers['x-amz-storage-class'] = (snakeCase(additionalFields.storageClass as string)).toUpperCase(); } - responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { location: '' }); + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' }); const region = responseData.LocationConstraint._; - responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'PUT', path, '', qs, headers, {}, region); + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'PUT', path, '', qs, headers, {}, region); returnData.push({ success: true }); } //https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html @@ -255,16 +304,16 @@ export class AwsS3 implements INodeType { const bucketName = this.getNodeParameter('bucketName', i) as string; const folderKey = this.getNodeParameter('folderKey', i) as string; - responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { location: '' }); + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' }); const region = responseData.LocationConstraint._; - responseData = await awsApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', `${bucketName}.s3`, 'GET', '/', '', { 'list-type': 2, prefix: folderKey }, {}, {}, region); + responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '/', '', { 'list-type': 2, prefix: folderKey }, {}, {}, region); // folder empty then just delete it if (responseData.length === 0) { - responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'DELETE', `/${folderKey}`, '', qs, {}, {}, region); + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'DELETE', `/${folderKey}`, '', qs, {}, {}, region); responseData = { deleted: [ { 'Key': folderKey } ] }; @@ -293,7 +342,7 @@ export class AwsS3 implements INodeType { headers['Content-Type'] = 'application/xml'; - responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'POST', '/', data, { delete: '' } , headers, {}, region); + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'POST', '/', data, { delete: '' } , headers, {}, region); responseData = { deleted: responseData.DeleteResult.Deleted }; } @@ -315,15 +364,15 @@ export class AwsS3 implements INodeType { qs['list-type'] = 2; - responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { location: '' }); + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' }); const region = responseData.LocationConstraint._; if (returnAll) { - responseData = await awsApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', `${bucketName}.s3`, 'GET', '', '', qs, {}, {}, region); + responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '', '', qs, {}, {}, region); } else { qs.limit = this.getNodeParameter('limit', 0) as number; - responseData = await awsApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', `${bucketName}.s3`, 'GET', '', '', qs, {}, {}, region); + responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '', '', qs, {}, {}, region); } if (Array.isArray(responseData)) { responseData = responseData.filter((e: IDataObject) => (e.Key as string).endsWith('/') && e.Size === '0' && e.Key !== options.folderKey); @@ -404,11 +453,11 @@ export class AwsS3 implements INodeType { const destination = `/${destinationParts.slice(2, destinationParts.length).join('/')}`; - responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { location: '' }); + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' }); const region = responseData.LocationConstraint._; - responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'PUT', destination, '', qs, headers, {}, region); + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'PUT', destination, '', qs, headers, {}, region); returnData.push(responseData.CopyObjectResult); } @@ -425,11 +474,11 @@ export class AwsS3 implements INodeType { throw new Error('Downloding a whole directory is not yet supported, please provide a file key'); } - let region = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { location: '' }); + let region = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' }); region = region.LocationConstraint._; - const response = await awsApiRequestREST.call(this, `${bucketName}.s3`, 'GET', `/${fileKey}`, '', qs, {}, { encoding: null, resolveWithFullResponse: true }, region); + const response = await s3ApiRequestREST.call(this, bucketName, 'GET', `/${fileKey}`, '', qs, {}, { encoding: null, resolveWithFullResponse: true }, region); let mimeType: string | undefined; if (response.headers['content-type']) { @@ -468,11 +517,11 @@ export class AwsS3 implements INodeType { qs.versionId = options.versionId as string; } - responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { location: '' }); + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' }); const region = responseData.LocationConstraint._; - responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'DELETE', `/${fileKey}`, '', qs, {}, {}, region); + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'DELETE', `/${fileKey}`, '', qs, {}, {}, region); returnData.push({ success: true }); } @@ -494,15 +543,15 @@ export class AwsS3 implements INodeType { qs['list-type'] = 2; - responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { location: '' }); + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' }); const region = responseData.LocationConstraint._; if (returnAll) { - responseData = await awsApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', `${bucketName}.s3`, 'GET', '', '', qs, {}, {}, region); + responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '', '', qs, {}, {}, region); } else { qs.limit = this.getNodeParameter('limit', 0) as number; - responseData = await awsApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', `${bucketName}.s3`, 'GET', '', '', qs, {}, {}, region); + responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '', '', qs, {}, {}, region); responseData = responseData.splice(0, qs.limit); } if (Array.isArray(responseData)) { @@ -581,7 +630,7 @@ export class AwsS3 implements INodeType { headers['x-amz-tagging'] = tags.join('&'); } - responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { location: '' }); + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' }); const region = responseData.LocationConstraint._; @@ -604,7 +653,7 @@ export class AwsS3 implements INodeType { headers['Content-MD5'] = createHash('md5').update(body).digest('base64'); - responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'PUT', `${path}${fileName || binaryData.fileName}`, body, qs, headers, {}, region); + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'PUT', `${path}${fileName || binaryData.fileName}`, body, qs, headers, {}, region); } else { @@ -616,7 +665,7 @@ export class AwsS3 implements INodeType { headers['Content-MD5'] = createHash('md5').update(fileContent).digest('base64'); - responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'PUT', `${path}${fileName}`, body, qs, headers, {}, region); + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'PUT', `${path}${fileName}`, body, qs, headers, {}, region); } returnData.push({ success: true }); } diff --git a/packages/nodes-base/nodes/Aws/S3/GenericFunctions.ts b/packages/nodes-base/nodes/Aws/S3/GenericFunctions.ts index 92d620d102..40ecb96949 100644 --- a/packages/nodes-base/nodes/Aws/S3/GenericFunctions.ts +++ b/packages/nodes-base/nodes/Aws/S3/GenericFunctions.ts @@ -25,16 +25,47 @@ import { IDataObject, } from 'n8n-workflow'; -export async function awsApiRequest(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions | IWebhookFunctions, service: string, method: string, path: string, body?: string | Buffer, query: IDataObject = {}, headers?: object, option: IDataObject = {}, region?: string): Promise { // tslint:disable-line:no-any - const credentials = this.getCredentials('aws'); +export async function s3ApiRequest(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions | IWebhookFunctions, bucket: string, method: string, path: string, body?: string | Buffer, query: IDataObject = {}, headers?: object, option: IDataObject = {}, region?: string): Promise { // tslint:disable-line:no-any + + let credentials; + + const endpointType = this.getNodeParameter('endpoint', 0); + + try { + if (endpointType === 'aws') { + credentials = this.getCredentials('aws'); + } else { + credentials = this.getCredentials('customS3Endpoint'); + } + } catch (error) { + throw new Error(error); + } + if (credentials === undefined) { throw new Error('No credentials got returned!'); } - const endpoint = `${service}.${region || credentials.region}.amazonaws.com`; + let endpoint = endpointType === 'aws' ? `${bucket}.s3.${region || credentials.region}.amazonaws.com` : credentials.endpoint; + + // Using path-style for non-AWS services + if (bucket && endpointType === 'customS3Endpoint') { + if (credentials.forcePathStyle) { + path = `/${bucket}${path}`; + } else { + endpoint = `${bucket}.${endpoint}`; + } + } // Sign AWS API request with the user credentials - const signOpts = {headers: headers || {}, host: endpoint, method, path: `${path}?${queryToString(query).replace(/\+/g, '%2B')}`, body}; + const signOpts = { + headers: headers || {}, + region: region || credentials.region, + host: endpoint, + method, + path: `${path}?${queryToString(query).replace(/\+/g, '%2B')}`, + service: 's3', + body + }; sign(signOpts, { accessKeyId: `${credentials.accessKeyId}`, secretAccessKey: `${credentials.secretAccessKey}`}); @@ -42,7 +73,7 @@ export async function awsApiRequest(this: IHookFunctions | IExecuteFunctions | I headers: signOpts.headers, method, qs: query, - uri: `https://${endpoint}${signOpts.path}`, + uri: `https://${endpoint}${path}`, body: signOpts.body, }; @@ -52,7 +83,7 @@ export async function awsApiRequest(this: IHookFunctions | IExecuteFunctions | I try { return await this.helpers.request!(options); } catch (error) { - const errorMessage = error.response.body.message || error.response.body.Message || error.message; + const errorMessage = error.response?.body.message || error.response?.body.Message || error.message; if (error.statusCode === 403) { if (errorMessage === 'The security token included in the request is invalid.') { @@ -66,8 +97,8 @@ export async function awsApiRequest(this: IHookFunctions | IExecuteFunctions | I } } -export async function awsApiRequestREST(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions, service: string, method: string, path: string, body?: string, query: IDataObject = {}, headers?: object, options: IDataObject = {}, region?: string): Promise { // tslint:disable-line:no-any - const response = await awsApiRequest.call(this, service, method, path, body, query, headers, options, region); +export async function s3ApiRequestREST(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions, bucket: string, method: string, path: string, body?: string, query: IDataObject = {}, headers?: object, options: IDataObject = {}, region?: string): Promise { // tslint:disable-line:no-any + const response = await s3ApiRequest.call(this, bucket, method, path, body, query, headers, options, region); try { return JSON.parse(response); } catch (e) { @@ -75,8 +106,8 @@ export async function awsApiRequestREST(this: IHookFunctions | IExecuteFunctions } } -export async function awsApiRequestSOAP(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions | IWebhookFunctions, service: string, method: string, path: string, body?: string | Buffer, query: IDataObject = {}, headers?: object, option: IDataObject = {}, region?: string): Promise { // tslint:disable-line:no-any - const response = await awsApiRequest.call(this, service, method, path, body, query, headers, option, region); +export async function s3ApiRequestSOAP(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions | IWebhookFunctions, bucket: string, method: string, path: string, body?: string | Buffer, query: IDataObject = {}, headers?: object, option: IDataObject = {}, region?: string): Promise { // tslint:disable-line:no-any + const response = await s3ApiRequest.call(this, bucket, method, path, body, query, headers, option, region); try { return await new Promise((resolve, reject) => { parseString(response, { explicitArray: false }, (err, data) => { @@ -91,14 +122,14 @@ export async function awsApiRequestSOAP(this: IHookFunctions | IExecuteFunctions } } -export async function awsApiRequestSOAPAllItems(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions | IWebhookFunctions, propertyName: string, service: string, method: string, path: string, body?: string, query: IDataObject = {}, headers: IDataObject = {}, option: IDataObject = {}, region?: string): Promise { // tslint:disable-line:no-any +export async function s3ApiRequestSOAPAllItems(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions | IWebhookFunctions, propertyName: string, service: string, method: string, path: string, body?: string, query: IDataObject = {}, headers: IDataObject = {}, option: IDataObject = {}, region?: string): Promise { // tslint:disable-line:no-any const returnData: IDataObject[] = []; let responseData; do { - responseData = await awsApiRequestSOAP.call(this, service, method, path, body, query, headers, option, region); + responseData = await s3ApiRequestSOAP.call(this, service, method, path, body, query, headers, option, region); //https://forums.aws.amazon.com/thread.jspa?threadID=55746 if (get(responseData, `${propertyName.split('.')[0]}.NextContinuationToken`)) { From a3c5a971e50874cbb6ce743c0c77dbb6df18f5fe Mon Sep 17 00:00:00 2001 From: Denis Palashevskii Date: Wed, 5 Aug 2020 15:22:57 +0400 Subject: [PATCH 3/8] Add support for custom endpoint URL scheme (both HTTP and HTTPS) --- .../nodes-base/nodes/Aws/S3/GenericFunctions.ts | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/packages/nodes-base/nodes/Aws/S3/GenericFunctions.ts b/packages/nodes-base/nodes/Aws/S3/GenericFunctions.ts index 40ecb96949..1fe0e7b98e 100644 --- a/packages/nodes-base/nodes/Aws/S3/GenericFunctions.ts +++ b/packages/nodes-base/nodes/Aws/S3/GenericFunctions.ts @@ -25,6 +25,8 @@ import { IDataObject, } from 'n8n-workflow'; +import { URL } from 'url'; + export async function s3ApiRequest(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions | IWebhookFunctions, bucket: string, method: string, path: string, body?: string | Buffer, query: IDataObject = {}, headers?: object, option: IDataObject = {}, region?: string): Promise { // tslint:disable-line:no-any let credentials; @@ -45,22 +47,27 @@ export async function s3ApiRequest(this: IHookFunctions | IExecuteFunctions | IL throw new Error('No credentials got returned!'); } - let endpoint = endpointType === 'aws' ? `${bucket}.s3.${region || credentials.region}.amazonaws.com` : credentials.endpoint; + if (endpointType === "customS3Endpoint" && !(credentials.endpoint as string).startsWith('http')) { + throw new Error('HTTP(S) Scheme is required in endpoint definition'); + } + + const endpoint = new URL(endpointType === 'aws' ? `https://${bucket}.s3.${region || credentials.region}.amazonaws.com` : credentials.endpoint as string); - // Using path-style for non-AWS services if (bucket && endpointType === 'customS3Endpoint') { if (credentials.forcePathStyle) { path = `/${bucket}${path}`; } else { - endpoint = `${bucket}.${endpoint}`; + endpoint.host = `${bucket}.${endpoint.host}`; } } + endpoint.pathname = path; + // Sign AWS API request with the user credentials const signOpts = { headers: headers || {}, region: region || credentials.region, - host: endpoint, + host: endpoint.host, method, path: `${path}?${queryToString(query).replace(/\+/g, '%2B')}`, service: 's3', @@ -73,7 +80,7 @@ export async function s3ApiRequest(this: IHookFunctions | IExecuteFunctions | IL headers: signOpts.headers, method, qs: query, - uri: `https://${endpoint}${path}`, + uri: endpoint, body: signOpts.body, }; From 77eb0f4955cd81bd3a2af954549c3999cad7d36f Mon Sep 17 00:00:00 2001 From: Denis Palashevskii Date: Tue, 18 Aug 2020 16:56:53 +0400 Subject: [PATCH 4/8] Create generic S3 node --- .../nodes-base/nodes/Aws/S3/AwsS3.node.ts | 43 +- .../nodes/Aws/S3/GenericFunctions.ts | 19 +- .../nodes-base/nodes/S3/BucketDescription.ts | 327 +++++++ .../nodes-base/nodes/S3/FileDescription.ts | 922 ++++++++++++++++++ .../nodes-base/nodes/S3/FolderDescription.ts | 278 ++++++ .../nodes-base/nodes/S3/GenericFunctions.ts | 155 +++ packages/nodes-base/nodes/S3/S3.node.ts | 640 ++++++++++++ packages/nodes-base/nodes/S3/generic-s3.png | Bin 0 -> 5600 bytes packages/nodes-base/package.json | 1 + 9 files changed, 2325 insertions(+), 60 deletions(-) create mode 100644 packages/nodes-base/nodes/S3/BucketDescription.ts create mode 100644 packages/nodes-base/nodes/S3/FileDescription.ts create mode 100644 packages/nodes-base/nodes/S3/FolderDescription.ts create mode 100644 packages/nodes-base/nodes/S3/GenericFunctions.ts create mode 100644 packages/nodes-base/nodes/S3/S3.node.ts create mode 100644 packages/nodes-base/nodes/S3/generic-s3.png diff --git a/packages/nodes-base/nodes/Aws/S3/AwsS3.node.ts b/packages/nodes-base/nodes/Aws/S3/AwsS3.node.ts index 5afa5dbff0..34122f1225 100644 --- a/packages/nodes-base/nodes/Aws/S3/AwsS3.node.ts +++ b/packages/nodes-base/nodes/Aws/S3/AwsS3.node.ts @@ -65,44 +65,9 @@ export class AwsS3 implements INodeType { { name: 'aws', required: true, - displayOptions: { - show: { - endpoint: [ - 'aws', - ], - }, - }, - }, - { - name: 'customS3Endpoint', - required: true, - displayOptions: { - show: { - endpoint: [ - 'customS3Endpoint', - ], - }, - }, }, ], properties: [ - { - displayName: 'Endpoint', - name: 'endpoint', - type: 'options', - options: [ - { - name: 'AWS', - value: 'aws', - }, - { - name: 'Custom S3 endpoint', - value: 'customS3Endpoint', - }, - ], - default: 'aws', - description: 'The endpoint of S3 compatible service.', - }, { displayName: 'Resource', name: 'resource', @@ -151,14 +116,8 @@ export class AwsS3 implements INodeType { let credentials; - const endpointType = this.getNodeParameter('endpoint', 0); - try { - if (endpointType === 'aws') { - credentials = this.getCredentials('aws'); - } else { - credentials = this.getCredentials('customS3Endpoint'); - } + credentials = this.getCredentials('aws'); } catch (error) { throw new Error(error); } diff --git a/packages/nodes-base/nodes/Aws/S3/GenericFunctions.ts b/packages/nodes-base/nodes/Aws/S3/GenericFunctions.ts index 1fe0e7b98e..bf0839a461 100644 --- a/packages/nodes-base/nodes/Aws/S3/GenericFunctions.ts +++ b/packages/nodes-base/nodes/Aws/S3/GenericFunctions.ts @@ -31,14 +31,8 @@ export async function s3ApiRequest(this: IHookFunctions | IExecuteFunctions | IL let credentials; - const endpointType = this.getNodeParameter('endpoint', 0); - try { - if (endpointType === 'aws') { credentials = this.getCredentials('aws'); - } else { - credentials = this.getCredentials('customS3Endpoint'); - } } catch (error) { throw new Error(error); } @@ -47,19 +41,8 @@ export async function s3ApiRequest(this: IHookFunctions | IExecuteFunctions | IL throw new Error('No credentials got returned!'); } - if (endpointType === "customS3Endpoint" && !(credentials.endpoint as string).startsWith('http')) { - throw new Error('HTTP(S) Scheme is required in endpoint definition'); - } - const endpoint = new URL(endpointType === 'aws' ? `https://${bucket}.s3.${region || credentials.region}.amazonaws.com` : credentials.endpoint as string); - - if (bucket && endpointType === 'customS3Endpoint') { - if (credentials.forcePathStyle) { - path = `/${bucket}${path}`; - } else { - endpoint.host = `${bucket}.${endpoint.host}`; - } - } + const endpoint = new URL(`https://${bucket}.s3.${region || credentials.region}.amazonaws.com`); endpoint.pathname = path; diff --git a/packages/nodes-base/nodes/S3/BucketDescription.ts b/packages/nodes-base/nodes/S3/BucketDescription.ts new file mode 100644 index 0000000000..a6c8d67cd1 --- /dev/null +++ b/packages/nodes-base/nodes/S3/BucketDescription.ts @@ -0,0 +1,327 @@ +import { + INodeProperties, +} from 'n8n-workflow'; + +export const bucketOperations = [ + { + displayName: 'Operation', + name: 'operation', + type: 'options', + displayOptions: { + show: { + resource: [ + 'bucket', + ], + }, + }, + options: [ + { + name: 'Create', + value: 'create', + description: 'Create a bucket', + }, + { + name: 'Get All', + value: 'getAll', + description: 'Get all buckets', + }, + { + name: 'Search', + value: 'search', + description: 'Search within a bucket', + }, + ], + default: 'create', + description: 'The operation to perform.', + }, +] as INodeProperties[]; + +export const bucketFields = [ + +/* -------------------------------------------------------------------------- */ +/* bucket:create */ +/* -------------------------------------------------------------------------- */ + { + displayName: 'Name', + name: 'name', + type: 'string', + required: true, + default: '', + displayOptions: { + show: { + resource: [ + 'bucket', + ], + operation: [ + 'create', + ], + }, + }, + description: 'A succinct description of the nature, symptoms, cause, or effect of the bucket.', + }, + { + displayName: 'Additional Fields', + name: 'additionalFields', + type: 'collection', + placeholder: 'Add Field', + displayOptions: { + show: { + resource: [ + 'bucket', + ], + operation: [ + 'create', + ], + }, + }, + default: {}, + options: [ + { + displayName: 'ACL', + name: 'acl', + type: 'options', + options: [ + { + name: 'Authenticated Read', + value: 'authenticatedRead', + }, + { + name: 'Private', + value: 'Private', + }, + { + name: 'Public Read', + value: 'publicRead', + }, + { + name: 'Public Read Write', + value: 'publicReadWrite', + }, + ], + default: '', + description: 'The canned ACL to apply to the bucket.', + }, + { + displayName: 'Bucket Object Lock Enabled', + name: 'bucketObjectLockEnabled', + type: 'boolean', + default: false, + description: 'Specifies whether you want S3 Object Lock to be enabled for the new bucket.', + }, + { + displayName: 'Grant Full Control', + name: 'grantFullControl', + type: 'boolean', + default: false, + description: 'Allows grantee the read, write, read ACP, and write ACP permissions on the bucket.', + }, + { + displayName: 'Grant Read', + name: 'grantRead', + type: 'boolean', + default: false, + description: 'Allows grantee to list the objects in the bucket.', + }, + { + displayName: 'Grant Read ACP', + name: 'grantReadAcp', + type: 'boolean', + default: false, + description: 'Allows grantee to read the bucket ACL.', + }, + { + displayName: 'Grant Write', + name: 'grantWrite', + type: 'boolean', + default: false, + description: 'Allows grantee to create, overwrite, and delete any object in the bucket.', + }, + { + displayName: 'Grant Write ACP', + name: 'grantWriteAcp', + type: 'boolean', + default: false, + description: 'Allows grantee to write the ACL for the applicable bucket.', + }, + { + displayName: 'Region', + name: 'region', + type: 'string', + default: '', + description: 'Region you want to create the bucket in, by default the buckets are created on the region defined on the credentials.', + }, + ], + }, +/* -------------------------------------------------------------------------- */ +/* bucket:getAll */ +/* -------------------------------------------------------------------------- */ + { + displayName: 'Return All', + name: 'returnAll', + type: 'boolean', + displayOptions: { + show: { + operation: [ + 'getAll', + ], + resource: [ + 'bucket', + ], + }, + }, + default: false, + description: 'If all results should be returned or only up to a given limit.', + }, + { + displayName: 'Limit', + name: 'limit', + type: 'number', + displayOptions: { + show: { + operation: [ + 'getAll', + ], + resource: [ + 'bucket', + ], + returnAll: [ + false, + ], + }, + }, + typeOptions: { + minValue: 1, + maxValue: 500, + }, + default: 100, + description: 'How many results to return.', + }, +/* -------------------------------------------------------------------------- */ +/* bucket:search */ +/* -------------------------------------------------------------------------- */ + { + displayName: 'Bucket Name', + name: 'bucketName', + type: 'string', + required: true, + default: '', + displayOptions: { + show: { + resource: [ + 'bucket', + ], + operation: [ + 'search', + ], + }, + }, + }, + { + displayName: 'Return All', + name: 'returnAll', + type: 'boolean', + displayOptions: { + show: { + operation: [ + 'search', + ], + resource: [ + 'bucket', + ], + }, + }, + default: false, + description: 'If all results should be returned or only up to a given limit.', + }, + { + displayName: 'Limit', + name: 'limit', + type: 'number', + displayOptions: { + show: { + operation: [ + 'search', + ], + resource: [ + 'bucket', + ], + returnAll: [ + false, + ], + }, + }, + typeOptions: { + minValue: 1, + maxValue: 500, + }, + default: 100, + description: 'How many results to return.', + }, + { + displayName: 'Additional Fields', + name: 'additionalFields', + type: 'collection', + placeholder: 'Add Field', + displayOptions: { + show: { + resource: [ + 'bucket', + ], + operation: [ + 'search', + ], + }, + }, + default: {}, + options: [ + { + displayName: 'Delimiter', + name: 'delimiter', + type: 'string', + default: '', + description: 'A delimiter is a character you use to group keys.', + }, + { + displayName: 'Encoding Type', + name: 'encodingType', + type: 'options', + options: [ + { + name: 'URL', + value: 'url', + }, + ], + default: '', + description: 'Encoding type used by Amazon S3 to encode object keys in the response.', + }, + { + displayName: 'Fetch Owner', + name: 'fetchOwner', + type: 'boolean', + default: false, + description: 'The owner field is not present in listV2 by default, if you want to return owner field with each key in the result then set the fetch owner field to true.', + }, + { + displayName: 'Prefix', + name: 'prefix', + type: 'string', + default: '', + description: 'Limits the response to keys that begin with the specified prefix.', + }, + { + displayName: 'Requester Pays', + name: 'requesterPays', + type: 'boolean', + default: false, + description: 'Weather the requester will pay for requests and data transfer. While Requester Pays is enabled, anonymous access to this bucket is disabled.', + }, + { + displayName: 'Start After', + name: 'startAfter', + type: 'string', + default: '', + description: 'StartAfter is where you want Amazon S3 to start listing from. Amazon S3 starts listing after this specified key', + }, + ], + }, +] as INodeProperties[]; diff --git a/packages/nodes-base/nodes/S3/FileDescription.ts b/packages/nodes-base/nodes/S3/FileDescription.ts new file mode 100644 index 0000000000..208b847b3d --- /dev/null +++ b/packages/nodes-base/nodes/S3/FileDescription.ts @@ -0,0 +1,922 @@ +import { + INodeProperties, +} from 'n8n-workflow'; + +export const fileOperations = [ + { + displayName: 'Operation', + name: 'operation', + type: 'options', + displayOptions: { + show: { + resource: [ + 'file', + ], + }, + }, + options: [ + { + name: 'Copy', + value: 'copy', + description: 'Copy a file', + }, + { + name: 'Delete', + value: 'delete', + description: 'Delete a file', + }, + { + name: 'Download', + value: 'download', + description: 'Download a file', + }, + { + name: 'Get All', + value: 'getAll', + description: 'Get all files', + }, + { + name: 'Upload', + value: 'upload', + description: 'Upload a file', + }, + ], + default: 'download', + description: 'The operation to perform.', + }, +] as INodeProperties[]; + +export const fileFields = [ + +/* -------------------------------------------------------------------------- */ +/* file:copy */ +/* -------------------------------------------------------------------------- */ + { + displayName: 'Source Path', + name: 'sourcePath', + type: 'string', + required: true, + default: '', + placeholder: '/bucket/my-image.jpg', + displayOptions: { + show: { + resource: [ + 'file', + ], + operation: [ + 'copy', + ], + }, + }, + description: 'The name of the source bucket and key name of the source object, separated by a slash (/)', + }, + { + displayName: 'Destination Path', + name: 'destinationPath', + type: 'string', + required: true, + default: '', + placeholder: '/bucket/my-second-image.jpg', + displayOptions: { + show: { + resource: [ + 'file', + ], + operation: [ + 'copy', + ], + }, + }, + description: 'The name of the destination bucket and key name of the destination object, separated by a slash (/)', + }, + { + displayName: 'Additional Fields', + name: 'additionalFields', + type: 'collection', + placeholder: 'Add Field', + displayOptions: { + show: { + resource: [ + 'file', + ], + operation: [ + 'copy', + ], + }, + }, + default: {}, + options: [ + { + displayName: 'ACL', + name: 'acl', + type: 'options', + options: [ + { + name: 'Authenticated Read', + value: 'authenticatedRead' + }, + { + name: 'AWS Exec Read', + value: 'awsExecRead' + }, + { + name: 'Bucket Owner Full Control', + value: 'bucketOwnerFullControl' + }, + { + name: 'Bucket Owner Read', + value: 'bucketOwnerRead' + }, + { + name: 'Private', + value: 'private', + }, + { + name: 'Public Read', + value: 'publicRead' + }, + { + name: 'Public Read Write', + value: 'publicReadWrite' + }, + ], + default: 'private', + description: 'The canned ACL to apply to the object.' + }, + { + displayName: 'Grant Full Control', + name: 'grantFullControl', + type: 'boolean', + default: false, + description: 'Gives the grantee READ, READ_ACP, and WRITE_ACP permissions on the object.', + }, + { + displayName: 'Grant Read', + name: 'grantRead', + type: 'boolean', + default: false, + description: 'Allows grantee to read the object data and its metadata.', + }, + { + displayName: 'Grant Read ACP', + name: 'grantReadAcp', + type: 'boolean', + default: false, + description: 'Allows grantee to read the object ACL.', + }, + { + displayName: 'Grant Write ACP', + name: 'grantWriteAcp', + type: 'boolean', + default: false, + description: 'Allows grantee to write the ACL for the applicable object.', + }, + { + displayName: 'Lock Legal Hold', + name: 'lockLegalHold', + type: 'boolean', + default: false, + description: 'Specifies whether a legal hold will be applied to this object', + }, + { + displayName: 'Lock Mode', + name: 'lockMode', + type: 'options', + options: [ + { + name: 'Governance', + value: 'governance', + }, + { + name: 'Compliance', + value: 'compliance', + }, + ], + default: '', + description: 'The Object Lock mode that you want to apply to this object.', + }, + { + displayName: 'Lock Retain Until Date', + name: 'lockRetainUntilDate', + type: 'dateTime', + default: '', + description: `The date and time when you want this object's Object Lock to expire.`, + }, + { + displayName: 'Metadata Directive', + name: 'metadataDirective', + type: 'options', + options: [ + { + name: 'Copy', + value: 'copy', + }, + { + name: 'Replace', + value: 'replace', + }, + ], + default: '', + description: 'Specifies whether the metadata is copied from the source object or replaced with metadata provided in the request.', + }, + { + displayName: 'Requester Pays', + name: 'requesterPays', + type: 'boolean', + default: false, + description: 'Weather the requester will pay for requests and data transfer. While Requester Pays is enabled, anonymous access to this bucket is disabled.', + }, + { + displayName: 'Server Side Encryption', + name: 'serverSideEncryption', + type: 'options', + options: [ + { + name: 'AES256', + value: 'AES256', + }, + { + name: 'AWS:KMS', + value: 'aws:kms', + }, + ], + default: '', + description: 'The server-side encryption algorithm used when storing this object in Amazon S3', + }, + { + displayName: 'Server Side Encryption Context', + name: 'serverSideEncryptionContext', + type: 'string', + default: '', + description: 'Specifies the AWS KMS Encryption Context to use for object encryption', + }, + { + displayName: 'Server Side Encryption AWS KMS Key ID', + name: 'encryptionAwsKmsKeyId', + type: 'string', + default: '', + description: 'If x-amz-server-side-encryption is present and has the value of aws:kms', + }, + { + displayName: 'Server Side Encryption Customer Algorithm', + name: 'serversideEncryptionCustomerAlgorithm', + type: 'string', + default: '', + description: 'Specifies the algorithm to use to when encrypting the object (for example, AES256).', + }, + { + displayName: 'Server Side Encryption Customer Key', + name: 'serversideEncryptionCustomerKey', + type: 'string', + default: '', + description: 'Specifies the customer-provided encryption key for Amazon S3 to use in encrypting data', + }, + { + displayName: 'Server Side Encryption Customer Key MD5', + name: 'serversideEncryptionCustomerKeyMD5', + type: 'string', + default: '', + description: 'Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321.', + }, + { + displayName: 'Storage Class', + name: 'storageClass', + type: 'options', + options: [ + { + name: 'Deep Archive', + value: 'deepArchive', + }, + { + name: 'Intelligent Tiering', + value: 'intelligentTiering', + }, + { + name: 'One Zone IA', + value: 'onezoneIA', + }, + { + name: 'Glacier', + value: 'glacier', + }, + { + name: 'Standard', + value: 'standard', + }, + { + name: 'Standard IA', + value: 'standardIA', + }, + ], + default: 'standard', + description: 'Amazon S3 storage classes.', + }, + { + displayName: 'Tagging Directive', + name: 'taggingDirective', + type: 'options', + options: [ + { + name: 'Copy', + value: 'copy', + }, + { + name: 'Replace', + value: 'replace', + }, + ], + default: '', + description: 'Specifies whether the metadata is copied from the source object or replaced with metadata provided in the request.', + }, + ], + }, +/* -------------------------------------------------------------------------- */ +/* file:upload */ +/* -------------------------------------------------------------------------- */ + { + displayName: 'Bucket Name', + name: 'bucketName', + type: 'string', + required: true, + default: '', + displayOptions: { + show: { + resource: [ + 'file', + ], + operation: [ + 'upload', + ], + }, + }, + }, + { + displayName: 'File Name', + name: 'fileName', + type: 'string', + default: '', + placeholder: 'hello.txt', + required: true, + displayOptions: { + show: { + resource: [ + 'file', + ], + operation: [ + 'upload', + ], + binaryData: [ + false, + ], + }, + }, + }, + { + displayName: 'File Name', + name: 'fileName', + type: 'string', + default: '', + displayOptions: { + show: { + resource: [ + 'file', + ], + operation: [ + 'upload', + ], + binaryData: [ + true, + ], + }, + }, + description: 'If not set the binary data filename will be used.', + }, + { + displayName: 'Binary Data', + name: 'binaryData', + type: 'boolean', + default: true, + displayOptions: { + show: { + operation: [ + 'upload' + ], + resource: [ + 'file', + ], + }, + }, + description: 'If the data to upload should be taken from binary field.', + }, + { + displayName: 'File Content', + name: 'fileContent', + type: 'string', + default: '', + displayOptions: { + show: { + operation: [ + 'upload' + ], + resource: [ + 'file', + ], + binaryData: [ + false + ], + }, + }, + placeholder: '', + description: 'The text content of the file to upload.', + }, + { + displayName: 'Binary Property', + name: 'binaryPropertyName', + type: 'string', + default: 'data', + required: true, + displayOptions: { + show: { + operation: [ + 'upload' + ], + resource: [ + 'file', + ], + binaryData: [ + true + ], + }, + + }, + placeholder: '', + description: 'Name of the binary property which contains
the data for the file to be uploaded.', + }, + { + displayName: 'Additional Fields', + name: 'additionalFields', + type: 'collection', + placeholder: 'Add Field', + displayOptions: { + show: { + resource: [ + 'file', + ], + operation: [ + 'upload', + ], + }, + }, + default: {}, + options: [ + { + displayName: 'ACL', + name: 'acl', + type: 'options', + options: [ + { + name: 'Authenticated Read', + value: 'authenticatedRead' + }, + { + name: 'AWS Exec Read', + value: 'awsExecRead' + }, + { + name: 'Bucket Owner Full Control', + value: 'bucketOwnerFullControl' + }, + { + name: 'Bucket Owner Read', + value: 'bucketOwnerRead' + }, + { + name: 'Private', + value: 'private', + }, + { + name: 'Public Read', + value: 'publicRead' + }, + { + name: 'Public Read Write', + value: 'publicReadWrite' + }, + ], + default: 'private', + description: 'The canned ACL to apply to the object.' + }, + { + displayName: 'Grant Full Control', + name: 'grantFullControl', + type: 'boolean', + default: false, + description: 'Gives the grantee READ, READ_ACP, and WRITE_ACP permissions on the object.', + }, + { + displayName: 'Grant Read', + name: 'grantRead', + type: 'boolean', + default: false, + description: 'Allows grantee to read the object data and its metadata.', + }, + { + displayName: 'Grant Read ACP', + name: 'grantReadAcp', + type: 'boolean', + default: false, + description: 'Allows grantee to read the object ACL.', + }, + { + displayName: 'Grant Write ACP', + name: 'grantWriteAcp', + type: 'boolean', + default: false, + description: 'Allows grantee to write the ACL for the applicable object.', + }, + { + displayName: 'Lock Legal Hold', + name: 'lockLegalHold', + type: 'boolean', + default: false, + description: 'Specifies whether a legal hold will be applied to this object', + }, + { + displayName: 'Lock Mode', + name: 'lockMode', + type: 'options', + options: [ + { + name: 'Governance', + value: 'governance', + }, + { + name: 'Compliance', + value: 'compliance', + }, + ], + default: '', + description: 'The Object Lock mode that you want to apply to this object.', + }, + { + displayName: 'Lock Retain Until Date', + name: 'lockRetainUntilDate', + type: 'dateTime', + default: '', + description: `The date and time when you want this object's Object Lock to expire.`, + }, + { + displayName: 'Parent Folder Key', + name: 'parentFolderKey', + type: 'string', + default: '', + description: 'Parent file you want to create the file in', + }, + { + displayName: 'Requester Pays', + name: 'requesterPays', + type: 'boolean', + default: false, + description: 'Weather the requester will pay for requests and data transfer. While Requester Pays is enabled, anonymous access to this bucket is disabled.', + }, + { + displayName: 'Server Side Encryption', + name: 'serverSideEncryption', + type: 'options', + options: [ + { + name: 'AES256', + value: 'AES256', + }, + { + name: 'AWS:KMS', + value: 'aws:kms', + }, + ], + default: '', + description: 'The server-side encryption algorithm used when storing this object in Amazon S3', + }, + { + displayName: 'Server Side Encryption Context', + name: 'serverSideEncryptionContext', + type: 'string', + default: '', + description: 'Specifies the AWS KMS Encryption Context to use for object encryption', + }, + { + displayName: 'Server Side Encryption AWS KMS Key ID', + name: 'encryptionAwsKmsKeyId', + type: 'string', + default: '', + description: 'If x-amz-server-side-encryption is present and has the value of aws:kms', + }, + { + displayName: 'Server Side Encryption Customer Algorithm', + name: 'serversideEncryptionCustomerAlgorithm', + type: 'string', + default: '', + description: 'Specifies the algorithm to use to when encrypting the object (for example, AES256).', + }, + { + displayName: 'Server Side Encryption Customer Key', + name: 'serversideEncryptionCustomerKey', + type: 'string', + default: '', + description: 'Specifies the customer-provided encryption key for Amazon S3 to use in encrypting data', + }, + { + displayName: 'Server Side Encryption Customer Key MD5', + name: 'serversideEncryptionCustomerKeyMD5', + type: 'string', + default: '', + description: 'Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321.', + }, + { + displayName: 'Storage Class', + name: 'storageClass', + type: 'options', + options: [ + { + name: 'Deep Archive', + value: 'deepArchive', + }, + { + name: 'Intelligent Tiering', + value: 'intelligentTiering', + }, + { + name: 'One Zone IA', + value: 'onezoneIA', + }, + { + name: 'Glacier', + value: 'glacier', + }, + { + name: 'Standard', + value: 'standard', + }, + { + name: 'Standard IA', + value: 'standardIA', + }, + ], + default: 'standard', + description: 'Amazon S3 storage classes.', + }, + ], + }, + { + displayName: 'Tags', + name: 'tagsUi', + placeholder: 'Add Tag', + type: 'fixedCollection', + default: '', + typeOptions: { + multipleValues: true, + }, + displayOptions: { + show: { + resource: [ + 'file', + ], + operation: [ + 'upload', + ], + }, + }, + options: [ + { + name: 'tagsValues', + displayName: 'Tag', + values: [ + { + displayName: 'Key', + name: 'key', + type: 'string', + default: '', + description: '', + }, + { + displayName: 'Value', + name: 'value', + type: 'string', + default: '', + description: '', + }, + ], + } + ], + description: 'Optional extra headers to add to the message (most headers are allowed).', + }, +/* -------------------------------------------------------------------------- */ +/* file:download */ +/* -------------------------------------------------------------------------- */ + { + displayName: 'Bucket Name', + name: 'bucketName', + type: 'string', + required: true, + default: '', + displayOptions: { + show: { + resource: [ + 'file', + ], + operation: [ + 'download', + ], + }, + }, + }, + { + displayName: 'File Key', + name: 'fileKey', + type: 'string', + required: true, + default: '', + displayOptions: { + show: { + resource: [ + 'file', + ], + operation: [ + 'download', + ], + }, + }, + }, + { + displayName: 'Binary Property', + name: 'binaryPropertyName', + type: 'string', + required: true, + default: 'data', + displayOptions: { + show: { + operation: [ + 'download' + ], + resource: [ + 'file', + ], + }, + }, + description: 'Name of the binary property to which to
write the data of the read file.', + }, +/* -------------------------------------------------------------------------- */ +/* file:delete */ +/* -------------------------------------------------------------------------- */ + { + displayName: 'Bucket Name', + name: 'bucketName', + type: 'string', + required: true, + default: '', + displayOptions: { + show: { + resource: [ + 'file', + ], + operation: [ + 'delete', + ], + }, + }, + }, + { + displayName: 'File Key', + name: 'fileKey', + type: 'string', + required: true, + default: '', + displayOptions: { + show: { + resource: [ + 'file', + ], + operation: [ + 'delete', + ], + }, + }, + }, + { + displayName: 'Options', + name: 'options', + type: 'collection', + placeholder: 'Add Field', + default: {}, + displayOptions: { + show: { + resource: [ + 'file', + ], + operation: [ + 'delete', + ], + }, + }, + options: [ + { + displayName: 'Version ID', + name: 'versionId', + type: 'string', + default: '', + }, + ], + }, +/* -------------------------------------------------------------------------- */ +/* file:getAll */ +/* -------------------------------------------------------------------------- */ + { + displayName: 'Bucket Name', + name: 'bucketName', + type: 'string', + required: true, + default: '', + displayOptions: { + show: { + resource: [ + 'file', + ], + operation: [ + 'getAll', + ], + }, + }, + }, + { + displayName: 'Return All', + name: 'returnAll', + type: 'boolean', + displayOptions: { + show: { + operation: [ + 'getAll', + ], + resource: [ + 'file', + ], + }, + }, + default: false, + description: 'If all results should be returned or only up to a given limit.', + }, + { + displayName: 'Limit', + name: 'limit', + type: 'number', + displayOptions: { + show: { + operation: [ + 'getAll', + ], + resource: [ + 'file', + ], + returnAll: [ + false, + ], + }, + }, + typeOptions: { + minValue: 1, + maxValue: 500, + }, + default: 100, + description: 'How many results to return.', + }, + { + displayName: 'Options', + name: 'options', + type: 'collection', + placeholder: 'Add Field', + default: {}, + displayOptions: { + show: { + resource: [ + 'file', + ], + operation: [ + 'getAll', + ], + }, + }, + options: [ + { + displayName: 'Fetch Owner', + name: 'fetchOwner', + type: 'boolean', + default: false, + description: 'The owner field is not present in listV2 by default, if you want to return owner field with each key in the result then set the fetch owner field to true.', + }, + { + displayName: 'Folder Key', + name: 'folderKey', + type: 'string', + default: '', + }, + ], + }, +] as INodeProperties[]; diff --git a/packages/nodes-base/nodes/S3/FolderDescription.ts b/packages/nodes-base/nodes/S3/FolderDescription.ts new file mode 100644 index 0000000000..2884206d66 --- /dev/null +++ b/packages/nodes-base/nodes/S3/FolderDescription.ts @@ -0,0 +1,278 @@ +import { + INodeProperties, +} from 'n8n-workflow'; + +export const folderOperations = [ + { + displayName: 'Operation', + name: 'operation', + type: 'options', + displayOptions: { + show: { + resource: [ + 'folder', + ], + }, + }, + options: [ + { + name: 'Create', + value: 'create', + description: 'Create a folder', + }, + { + name: 'Delete', + value: 'delete', + description: 'Delete a folder', + }, + { + name: 'Get All', + value: 'getAll', + description: 'Get all folders', + }, + ], + default: 'create', + description: 'The operation to perform.', + }, +] as INodeProperties[]; + +export const folderFields = [ + +/* -------------------------------------------------------------------------- */ +/* folder:create */ +/* -------------------------------------------------------------------------- */ + { + displayName: 'Bucket Name', + name: 'bucketName', + type: 'string', + required: true, + default: '', + displayOptions: { + show: { + resource: [ + 'folder', + ], + operation: [ + 'create', + ], + }, + }, + }, + { + displayName: 'Folder Name', + name: 'folderName', + type: 'string', + required: true, + default: '', + displayOptions: { + show: { + resource: [ + 'folder', + ], + operation: [ + 'create', + ], + }, + }, + }, + { + displayName: 'Additional Fields', + name: 'additionalFields', + type: 'collection', + placeholder: 'Add Field', + displayOptions: { + show: { + resource: [ + 'folder', + ], + operation: [ + 'create', + ], + }, + }, + default: {}, + options: [ + { + displayName: 'Parent Folder Key', + name: 'parentFolderKey', + type: 'string', + default: '', + description: 'Parent folder you want to create the folder in' + }, + { + displayName: 'Requester Pays', + name: 'requesterPays', + type: 'boolean', + default: false, + description: 'Weather the requester will pay for requests and data transfer. While Requester Pays is enabled, anonymous access to this bucket is disabled.', + }, + { + displayName: 'Storage Class', + name: 'storageClass', + type: 'options', + options: [ + { + name: 'Deep Archive', + value: 'deepArchive', + }, + { + name: 'Intelligent Tiering', + value: 'intelligentTiering', + }, + { + name: 'One Zone IA', + value: 'onezoneIA', + }, + { + name: 'Glacier', + value: 'glacier', + }, + { + name: 'Reduced Redundancy', + value: 'RecudedRedundancy', + }, + { + name: 'Standard', + value: 'standard', + }, + { + name: 'Standard IA', + value: 'standardIA', + }, + ], + default: 'standard', + description: 'Amazon S3 storage classes.' + }, + ], + }, +/* -------------------------------------------------------------------------- */ +/* folder:delete */ +/* -------------------------------------------------------------------------- */ + { + displayName: 'Bucket Name', + name: 'bucketName', + type: 'string', + required: true, + default: '', + displayOptions: { + show: { + resource: [ + 'folder', + ], + operation: [ + 'delete', + ], + }, + }, + }, + { + displayName: 'Folder Key', + name: 'folderKey', + type: 'string', + required: true, + default: '', + displayOptions: { + show: { + resource: [ + 'folder', + ], + operation: [ + 'delete', + ], + }, + }, + }, +/* -------------------------------------------------------------------------- */ +/* folder:getAll */ +/* -------------------------------------------------------------------------- */ + { + displayName: 'Bucket Name', + name: 'bucketName', + type: 'string', + required: true, + default: '', + displayOptions: { + show: { + resource: [ + 'folder', + ], + operation: [ + 'getAll', + ], + }, + }, + }, + { + displayName: 'Return All', + name: 'returnAll', + type: 'boolean', + displayOptions: { + show: { + operation: [ + 'getAll', + ], + resource: [ + 'folder', + ], + }, + }, + default: false, + description: 'If all results should be returned or only up to a given limit.', + }, + { + displayName: 'Limit', + name: 'limit', + type: 'number', + displayOptions: { + show: { + operation: [ + 'getAll', + ], + resource: [ + 'folder', + ], + returnAll: [ + false, + ], + }, + }, + typeOptions: { + minValue: 1, + maxValue: 500, + }, + default: 100, + description: 'How many results to return.', + }, + { + displayName: 'Options', + name: 'options', + type: 'collection', + placeholder: 'Add Field', + default: {}, + displayOptions: { + show: { + resource: [ + 'folder', + ], + operation: [ + 'getAll', + ], + }, + }, + options: [ + { + displayName: 'Fetch Owner', + name: 'fetchOwner', + type: 'boolean', + default: false, + description: 'The owner field is not present in listV2 by default, if you want to return owner field with each key in the result then set the fetch owner field to true.', + }, + { + displayName: 'Folder Key', + name: 'folderKey', + type: 'string', + default: '', + }, + ], + }, +] as INodeProperties[]; diff --git a/packages/nodes-base/nodes/S3/GenericFunctions.ts b/packages/nodes-base/nodes/S3/GenericFunctions.ts new file mode 100644 index 0000000000..00f59b3f70 --- /dev/null +++ b/packages/nodes-base/nodes/S3/GenericFunctions.ts @@ -0,0 +1,155 @@ +import { + sign, +} from 'aws4'; + +import { + get, +} from 'lodash'; + +import { + OptionsWithUri, +} from 'request'; + +import { + parseString, +} from 'xml2js'; + +import { + IExecuteFunctions, + IHookFunctions, + ILoadOptionsFunctions, + IWebhookFunctions, +} from 'n8n-core'; + +import { + IDataObject, + } from 'n8n-workflow'; + +import { URL } from 'url'; + +export async function s3ApiRequest(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions | IWebhookFunctions, bucket: string, method: string, path: string, body?: string | Buffer, query: IDataObject = {}, headers?: object, option: IDataObject = {}, region?: string): Promise { // tslint:disable-line:no-any + + let credentials; + + credentials = this.getCredentials('customS3Endpoint'); + + if (credentials === undefined) { + throw new Error('No credentials got returned!'); + } + + if (!(credentials.endpoint as string).startsWith('http')) { + throw new Error('HTTP(S) Scheme is required in endpoint definition'); + } + + const endpoint = new URL(credentials.endpoint as string); + + if (bucket) { + if (credentials.forcePathStyle) { + path = `/${bucket}${path}`; + } else { + endpoint.host = `${bucket}.${endpoint.host}`; + } + } + + endpoint.pathname = path; + + // Sign AWS API request with the user credentials + const signOpts = { + headers: headers || {}, + region: region || credentials.region, + host: endpoint.host, + method, + path: `${path}?${queryToString(query).replace(/\+/g, '%2B')}`, + service: 's3', + body + }; + + sign(signOpts, { accessKeyId: `${credentials.accessKeyId}`, secretAccessKey: `${credentials.secretAccessKey}`}); + + const options: OptionsWithUri = { + headers: signOpts.headers, + method, + qs: query, + uri: endpoint, + body: signOpts.body, + }; + + if (Object.keys(option).length !== 0) { + Object.assign(options, option); + } + try { + return await this.helpers.request!(options); + } catch (error) { + const errorMessage = error.response?.body.message || error.response?.body.Message || error.message; + + if (error.statusCode === 403) { + if (errorMessage === 'The security token included in the request is invalid.') { + throw new Error('The S3 credentials are not valid!'); + } else if (errorMessage.startsWith('The request signature we calculated does not match the signature you provided')) { + throw new Error('The S3 credentials are not valid!'); + } + } + + throw new Error(`S3 error response [${error.statusCode}]: ${errorMessage}`); + } +} + +export async function s3ApiRequestREST(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions, bucket: string, method: string, path: string, body?: string, query: IDataObject = {}, headers?: object, options: IDataObject = {}, region?: string): Promise { // tslint:disable-line:no-any + const response = await s3ApiRequest.call(this, bucket, method, path, body, query, headers, options, region); + try { + return JSON.parse(response); + } catch (e) { + return response; + } +} + +export async function s3ApiRequestSOAP(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions | IWebhookFunctions, bucket: string, method: string, path: string, body?: string | Buffer, query: IDataObject = {}, headers?: object, option: IDataObject = {}, region?: string): Promise { // tslint:disable-line:no-any + const response = await s3ApiRequest.call(this, bucket, method, path, body, query, headers, option, region); + try { + return await new Promise((resolve, reject) => { + parseString(response, { explicitArray: false }, (err, data) => { + if (err) { + return reject(err); + } + resolve(data); + }); + }); + } catch (e) { + return e; + } +} + +export async function s3ApiRequestSOAPAllItems(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions | IWebhookFunctions, propertyName: string, service: string, method: string, path: string, body?: string, query: IDataObject = {}, headers: IDataObject = {}, option: IDataObject = {}, region?: string): Promise { // tslint:disable-line:no-any + + const returnData: IDataObject[] = []; + + let responseData; + + do { + responseData = await s3ApiRequestSOAP.call(this, service, method, path, body, query, headers, option, region); + + //https://forums.aws.amazon.com/thread.jspa?threadID=55746 + if (get(responseData, `${propertyName.split('.')[0]}.NextContinuationToken`)) { + query['continuation-token'] = get(responseData, `${propertyName.split('.')[0]}.NextContinuationToken`); + } + if (get(responseData, propertyName)) { + if (Array.isArray(get(responseData, propertyName))) { + returnData.push.apply(returnData, get(responseData, propertyName)); + } else { + returnData.push(get(responseData, propertyName)); + } + } + if (query.limit && query.limit <= returnData.length) { + return returnData; + } + } while ( + get(responseData, `${propertyName.split('.')[0]}.IsTruncated`) !== undefined && + get(responseData, `${propertyName.split('.')[0]}.IsTruncated`) !== 'false' + ); + + return returnData; +} + +function queryToString(params: IDataObject) { + return Object.keys(params).map(key => key + '=' + params[key]).join('&'); +} diff --git a/packages/nodes-base/nodes/S3/S3.node.ts b/packages/nodes-base/nodes/S3/S3.node.ts new file mode 100644 index 0000000000..9eba6ab6d3 --- /dev/null +++ b/packages/nodes-base/nodes/S3/S3.node.ts @@ -0,0 +1,640 @@ + +import { + snakeCase, + paramCase, +} from 'change-case'; + +import { + createHash, +} from 'crypto'; + +import { + Builder, +} from 'xml2js'; + +import { + BINARY_ENCODING, + IExecuteFunctions, +} from 'n8n-core'; + +import { + IBinaryKeyData, + IDataObject, + INodeExecutionData, + INodeType, + INodeTypeDescription, +} from 'n8n-workflow'; + +import { + bucketFields, + bucketOperations, +} from './BucketDescription'; + +import { + folderFields, + folderOperations, +} from './FolderDescription'; + +import { + fileFields, + fileOperations, +} from './FileDescription'; + +import { + s3ApiRequestREST, + s3ApiRequestSOAP, + s3ApiRequestSOAPAllItems, +} from './GenericFunctions'; + +export class S3 implements INodeType { + description: INodeTypeDescription = { + displayName: 'S3', + name: 'S3', + icon: 'file:generic-s3.png', + group: ['output'], + version: 1, + subtitle: '={{$parameter["operation"] + ": " + $parameter["resource"]}}', + description: 'Sends data to any S3-compatible services', + defaults: { + name: 'S3', + color: '#d05b4b', + }, + inputs: ['main'], + outputs: ['main'], + credentials: [ + { + name: 'customS3Endpoint', + required: true, + }, + ], + properties: [ + { + displayName: 'Resource', + name: 'resource', + type: 'options', + options: [ + { + name: 'Bucket', + value: 'bucket', + }, + { + name: 'File', + value: 'file', + }, + { + name: 'Folder', + value: 'folder', + }, + ], + default: 'file', + description: 'The operation to perform.', + }, + // BUCKET + ...bucketOperations, + ...bucketFields, + // FOLDER + ...folderOperations, + ...folderFields, + // UPLOAD + ...fileOperations, + ...fileFields, + ], + }; + + async execute(this: IExecuteFunctions): Promise { + const items = this.getInputData(); + const returnData: IDataObject[] = []; + const qs: IDataObject = {}; + const headers: IDataObject = {}; + let responseData; + const resource = this.getNodeParameter('resource', 0) as string; + const operation = this.getNodeParameter('operation', 0) as string; + for (let i = 0; i < items.length; i++) { + if (resource === 'bucket') { + //https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateBucket.html + if (operation === 'create') { + + let credentials; + + try { + credentials = this.getCredentials('customS3Endpoint'); + } catch (error) { + throw new Error(error); + } + + const name = this.getNodeParameter('name', i) as string; + const additionalFields = this.getNodeParameter('additionalFields', i) as IDataObject; + if (additionalFields.acl) { + headers['x-amz-acl'] = paramCase(additionalFields.acl as string); + } + if (additionalFields.bucketObjectLockEnabled) { + headers['x-amz-bucket-object-lock-enabled'] = additionalFields.bucketObjectLockEnabled as boolean; + } + if (additionalFields.grantFullControl) { + headers['x-amz-grant-full-control'] = ''; + } + if (additionalFields.grantRead) { + headers['x-amz-grant-read'] = ''; + } + if (additionalFields.grantReadAcp) { + headers['x-amz-grant-read-acp'] = ''; + } + if (additionalFields.grantWrite) { + headers['x-amz-grant-write'] = ''; + } + if (additionalFields.grantWriteAcp) { + headers['x-amz-grant-write-acp'] = ''; + } + let region = credentials!.region as string; + + if (additionalFields.region) { + region = additionalFields.region as string; + } + + const body: IDataObject = { + CreateBucketConfiguration: { + '$': { + xmlns: 'http://s3.amazonaws.com/doc/2006-03-01/', + }, + } + }; + let data = ''; + // if credentials has the S3 defaul region (us-east-1) the body (XML) does not have to be sent. + if (region !== 'us-east-1') { + // @ts-ignore + body.CreateBucketConfiguration.LocationConstraint = [region]; + const builder = new Builder(); + data = builder.buildObject(body); + } + responseData = await s3ApiRequestSOAP.call(this, `${name}`, 'PUT', '', data, qs, headers); + + returnData.push({ success: true }); + } + //https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListBuckets.html + if (operation === 'getAll') { + const returnAll = this.getNodeParameter('returnAll', 0) as boolean; + if (returnAll) { + responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListAllMyBucketsResult.Buckets.Bucket', '', 'GET', ''); + } else { + qs.limit = this.getNodeParameter('limit', 0) as number; + responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListAllMyBucketsResult.Buckets.Bucket', '', 'GET', '', '', qs); + responseData = responseData.slice(0, qs.limit); + } + returnData.push.apply(returnData, responseData); + } + + //https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html + if (operation === 'search') { + const bucketName = this.getNodeParameter('bucketName', i) as string; + const returnAll = this.getNodeParameter('returnAll', 0) as boolean; + const additionalFields = this.getNodeParameter('additionalFields', 0) as IDataObject; + + if (additionalFields.prefix) { + qs['prefix'] = additionalFields.prefix as string; + } + + if (additionalFields.encodingType) { + qs['encoding-type'] = additionalFields.encodingType as string; + } + + if (additionalFields.delmiter) { + qs['delimiter'] = additionalFields.delmiter as string; + } + + if (additionalFields.fetchOwner) { + qs['fetch-owner'] = additionalFields.fetchOwner as string; + } + + if (additionalFields.startAfter) { + qs['start-after'] = additionalFields.startAfter as string; + } + + if (additionalFields.requesterPays) { + qs['x-amz-request-payer'] = 'requester'; + } + + qs['list-type'] = 2; + + + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' }); + + const region = responseData.LocationConstraint._ as string; + + if (returnAll) { + responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '', '', qs, {}, {}, region); + } else { + qs['max-keys'] = this.getNodeParameter('limit', 0) as number; + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', qs, {}, {}, region); + responseData = responseData.ListBucketResult.Contents; + } + if (Array.isArray(responseData)) { + returnData.push.apply(returnData, responseData); + } else { + returnData.push(responseData); + } + } + } + if (resource === 'folder') { + //https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html + if (operation === 'create') { + const bucketName = this.getNodeParameter('bucketName', i) as string; + const folderName = this.getNodeParameter('folderName', i) as string; + const additionalFields = this.getNodeParameter('additionalFields', i) as IDataObject; + let path = `/${folderName}/`; + + if (additionalFields.requesterPays) { + headers['x-amz-request-payer'] = 'requester'; + } + if (additionalFields.parentFolderKey) { + path = `/${additionalFields.parentFolderKey}${folderName}/`; + } + if (additionalFields.storageClass) { + headers['x-amz-storage-class'] = (snakeCase(additionalFields.storageClass as string)).toUpperCase(); + } + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' }); + + const region = responseData.LocationConstraint._; + + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'PUT', path, '', qs, headers, {}, region); + returnData.push({ success: true }); + } + //https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html + if (operation === 'delete') { + const bucketName = this.getNodeParameter('bucketName', i) as string; + const folderKey = this.getNodeParameter('folderKey', i) as string; + + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' }); + + const region = responseData.LocationConstraint._; + + responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '/', '', { 'list-type': 2, prefix: folderKey }, {}, {}, region); + + // folder empty then just delete it + if (responseData.length === 0) { + + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'DELETE', `/${folderKey}`, '', qs, {}, {}, region); + + responseData = { deleted: [ { 'Key': folderKey } ] }; + + } else { + // delete everything inside the folder + const body: IDataObject = { + Delete: { + '$': { + xmlns: 'http://s3.amazonaws.com/doc/2006-03-01/', + }, + Object: [], + }, + }; + + for (const childObject of responseData) { + //@ts-ignore + (body.Delete.Object as IDataObject[]).push({ + Key: childObject.Key as string + }); + } + + const builder = new Builder(); + const data = builder.buildObject(body); + + headers['Content-MD5'] = createHash('md5').update(data).digest('base64'); + + headers['Content-Type'] = 'application/xml'; + + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'POST', '/', data, { delete: '' } , headers, {}, region); + + responseData = { deleted: responseData.DeleteResult.Deleted }; + } + returnData.push(responseData); + } + //https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html + if (operation === 'getAll') { + const bucketName = this.getNodeParameter('bucketName', i) as string; + const returnAll = this.getNodeParameter('returnAll', 0) as boolean; + const options = this.getNodeParameter('options', 0) as IDataObject; + + if (options.folderKey) { + qs['prefix'] = options.folderKey as string; + } + + if (options.fetchOwner) { + qs['fetch-owner'] = options.fetchOwner as string; + } + + qs['list-type'] = 2; + + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' }); + + const region = responseData.LocationConstraint._; + + if (returnAll) { + responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '', '', qs, {}, {}, region); + } else { + qs.limit = this.getNodeParameter('limit', 0) as number; + responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '', '', qs, {}, {}, region); + } + if (Array.isArray(responseData)) { + responseData = responseData.filter((e: IDataObject) => (e.Key as string).endsWith('/') && e.Size === '0' && e.Key !== options.folderKey); + if (qs.limit) { + responseData = responseData.splice(0, qs.limit as number); + } + returnData.push.apply(returnData, responseData); + } + } + } + if (resource === 'file') { + //https://docs.aws.amazon.com/AmazonS3/latest/API/API_CopyObject.html + if (operation === 'copy') { + const sourcePath = this.getNodeParameter('sourcePath', i) as string; + const destinationPath = this.getNodeParameter('destinationPath', i) as string; + const additionalFields = this.getNodeParameter('additionalFields', i) as IDataObject; + + headers['x-amz-copy-source'] = sourcePath; + + if (additionalFields.requesterPays) { + headers['x-amz-request-payer'] = 'requester'; + } + if (additionalFields.storageClass) { + headers['x-amz-storage-class'] = (snakeCase(additionalFields.storageClass as string)).toUpperCase(); + } + if (additionalFields.acl) { + headers['x-amz-acl'] = paramCase(additionalFields.acl as string); + } + if (additionalFields.grantFullControl) { + headers['x-amz-grant-full-control'] = ''; + } + if (additionalFields.grantRead) { + headers['x-amz-grant-read'] = ''; + } + if (additionalFields.grantReadAcp) { + headers['x-amz-grant-read-acp'] = ''; + } + if (additionalFields.grantWriteAcp) { + headers['x-amz-grant-write-acp'] = ''; + } + if (additionalFields.lockLegalHold) { + headers['x-amz-object-lock-legal-hold'] = (additionalFields.lockLegalHold as boolean) ? 'ON' : 'OFF'; + } + if (additionalFields.lockMode) { + headers['x-amz-object-lock-mode'] = (additionalFields.lockMode as string).toUpperCase(); + } + if (additionalFields.lockRetainUntilDate) { + headers['x-amz-object-lock-retain-until-date'] = additionalFields.lockRetainUntilDate as string; + } + if (additionalFields.serverSideEncryption) { + headers['x-amz-server-side-encryption'] = additionalFields.serverSideEncryption as string; + } + if (additionalFields.encryptionAwsKmsKeyId) { + headers['x-amz-server-side-encryption-aws-kms-key-id'] = additionalFields.encryptionAwsKmsKeyId as string; + } + if (additionalFields.serverSideEncryptionContext) { + headers['x-amz-server-side-encryption-context'] = additionalFields.serverSideEncryptionContext as string; + } + if (additionalFields.serversideEncryptionCustomerAlgorithm) { + headers['x-amz-server-side-encryption-customer-algorithm'] = additionalFields.serversideEncryptionCustomerAlgorithm as string; + } + if (additionalFields.serversideEncryptionCustomerKey) { + headers['x-amz-server-side-encryption-customer-key'] = additionalFields.serversideEncryptionCustomerKey as string; + } + if (additionalFields.serversideEncryptionCustomerKeyMD5) { + headers['x-amz-server-side-encryption-customer-key-MD5'] = additionalFields.serversideEncryptionCustomerKeyMD5 as string; + } + if (additionalFields.taggingDirective) { + headers['x-amz-tagging-directive'] = (additionalFields.taggingDirective as string).toUpperCase(); + } + if (additionalFields.metadataDirective) { + headers['x-amz-metadata-directive'] = (additionalFields.metadataDirective as string).toUpperCase(); + } + + const destinationParts = destinationPath.split('/'); + + const bucketName = destinationParts[1]; + + const destination = `/${destinationParts.slice(2, destinationParts.length).join('/')}`; + + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' }); + + const region = responseData.LocationConstraint._; + + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'PUT', destination, '', qs, headers, {}, region); + returnData.push(responseData.CopyObjectResult); + + } + //https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html + if (operation === 'download') { + + const bucketName = this.getNodeParameter('bucketName', i) as string; + + const fileKey = this.getNodeParameter('fileKey', i) as string; + + const fileName = fileKey.split('/')[fileKey.split('/').length - 1]; + + if (fileKey.substring(fileKey.length - 1) === '/') { + throw new Error('Downloding a whole directory is not yet supported, please provide a file key'); + } + + let region = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' }); + + region = region.LocationConstraint._; + + const response = await s3ApiRequestREST.call(this, bucketName, 'GET', `/${fileKey}`, '', qs, {}, { encoding: null, resolveWithFullResponse: true }, region); + + let mimeType: string | undefined; + if (response.headers['content-type']) { + mimeType = response.headers['content-type']; + } + + const newItem: INodeExecutionData = { + json: items[i].json, + binary: {}, + }; + + if (items[i].binary !== undefined) { + // Create a shallow copy of the binary data so that the old + // data references which do not get changed still stay behind + // but the incoming data does not get changed. + Object.assign(newItem.binary, items[i].binary); + } + + items[i] = newItem; + + const dataPropertyNameDownload = this.getNodeParameter('binaryPropertyName', i) as string; + + const data = Buffer.from(response.body as string, 'utf8'); + + items[i].binary![dataPropertyNameDownload] = await this.helpers.prepareBinaryData(data as unknown as Buffer, fileName, mimeType); + } + //https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObject.html + if (operation === 'delete') { + const bucketName = this.getNodeParameter('bucketName', i) as string; + + const fileKey = this.getNodeParameter('fileKey', i) as string; + + const options = this.getNodeParameter('options', i) as IDataObject; + + if (options.versionId) { + qs.versionId = options.versionId as string; + } + + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' }); + + const region = responseData.LocationConstraint._; + + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'DELETE', `/${fileKey}`, '', qs, {}, {}, region); + + returnData.push({ success: true }); + } + //https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html + if (operation === 'getAll') { + const bucketName = this.getNodeParameter('bucketName', i) as string; + const returnAll = this.getNodeParameter('returnAll', 0) as boolean; + const options = this.getNodeParameter('options', 0) as IDataObject; + + if (options.folderKey) { + qs['prefix'] = options.folderKey as string; + } + + if (options.fetchOwner) { + qs['fetch-owner'] = options.fetchOwner as string; + } + + qs['delimiter'] = '/'; + + qs['list-type'] = 2; + + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' }); + + const region = responseData.LocationConstraint._; + + if (returnAll) { + responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '', '', qs, {}, {}, region); + } else { + qs.limit = this.getNodeParameter('limit', 0) as number; + responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '', '', qs, {}, {}, region); + responseData = responseData.splice(0, qs.limit); + } + if (Array.isArray(responseData)) { + responseData = responseData.filter((e: IDataObject) => !(e.Key as string).endsWith('/') && e.Size !== '0'); + if (qs.limit) { + responseData = responseData.splice(0, qs.limit as number); + } + returnData.push.apply(returnData, responseData); + } + } + //https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html + if (operation === 'upload') { + const bucketName = this.getNodeParameter('bucketName', i) as string; + const fileName = this.getNodeParameter('fileName', i) as string; + const isBinaryData = this.getNodeParameter('binaryData', i) as boolean; + const additionalFields = this.getNodeParameter('additionalFields', i) as IDataObject; + const tagsValues = (this.getNodeParameter('tagsUi', i) as IDataObject).tagsValues as IDataObject[]; + let path = '/'; + let body; + + if (additionalFields.requesterPays) { + headers['x-amz-request-payer'] = 'requester'; + } + + if (additionalFields.parentFolderKey) { + path = `/${additionalFields.parentFolderKey}/`; + } + if (additionalFields.storageClass) { + headers['x-amz-storage-class'] = (snakeCase(additionalFields.storageClass as string)).toUpperCase(); + } + if (additionalFields.acl) { + headers['x-amz-acl'] = paramCase(additionalFields.acl as string); + } + if (additionalFields.grantFullControl) { + headers['x-amz-grant-full-control'] = ''; + } + if (additionalFields.grantRead) { + headers['x-amz-grant-read'] = ''; + } + if (additionalFields.grantReadAcp) { + headers['x-amz-grant-read-acp'] = ''; + } + if (additionalFields.grantWriteAcp) { + headers['x-amz-grant-write-acp'] = ''; + } + if (additionalFields.lockLegalHold) { + headers['x-amz-object-lock-legal-hold'] = (additionalFields.lockLegalHold as boolean) ? 'ON' : 'OFF'; + } + if (additionalFields.lockMode) { + headers['x-amz-object-lock-mode'] = (additionalFields.lockMode as string).toUpperCase(); + } + if (additionalFields.lockRetainUntilDate) { + headers['x-amz-object-lock-retain-until-date'] = additionalFields.lockRetainUntilDate as string; + } + if (additionalFields.serverSideEncryption) { + headers['x-amz-server-side-encryption'] = additionalFields.serverSideEncryption as string; + } + if (additionalFields.encryptionAwsKmsKeyId) { + headers['x-amz-server-side-encryption-aws-kms-key-id'] = additionalFields.encryptionAwsKmsKeyId as string; + } + if (additionalFields.serverSideEncryptionContext) { + headers['x-amz-server-side-encryption-context'] = additionalFields.serverSideEncryptionContext as string; + } + if (additionalFields.serversideEncryptionCustomerAlgorithm) { + headers['x-amz-server-side-encryption-customer-algorithm'] = additionalFields.serversideEncryptionCustomerAlgorithm as string; + } + if (additionalFields.serversideEncryptionCustomerKey) { + headers['x-amz-server-side-encryption-customer-key'] = additionalFields.serversideEncryptionCustomerKey as string; + } + if (additionalFields.serversideEncryptionCustomerKeyMD5) { + headers['x-amz-server-side-encryption-customer-key-MD5'] = additionalFields.serversideEncryptionCustomerKeyMD5 as string; + } + if (tagsValues) { + const tags: string[] = []; + tagsValues.forEach((o: IDataObject) => { tags.push(`${o.key}=${o.value}`); }); + headers['x-amz-tagging'] = tags.join('&'); + } + + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' }); + + const region = responseData.LocationConstraint._; + + if (isBinaryData) { + const binaryPropertyName = this.getNodeParameter('binaryPropertyName', 0) as string; + + if (items[i].binary === undefined) { + throw new Error('No binary data exists on item!'); + } + + if ((items[i].binary as IBinaryKeyData)[binaryPropertyName] === undefined) { + throw new Error(`No binary data property "${binaryPropertyName}" does not exists on item!`); + } + + const binaryData = (items[i].binary as IBinaryKeyData)[binaryPropertyName]; + + body = Buffer.from(binaryData.data, BINARY_ENCODING) as Buffer; + + headers['Content-Type'] = binaryData.mimeType; + + headers['Content-MD5'] = createHash('md5').update(body).digest('base64'); + + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'PUT', `${path}${fileName || binaryData.fileName}`, body, qs, headers, {}, region); + + } else { + + const fileContent = this.getNodeParameter('fileContent', i) as string; + + body = Buffer.from(fileContent, 'utf8'); + + headers['Content-Type'] = 'text/html'; + + headers['Content-MD5'] = createHash('md5').update(fileContent).digest('base64'); + + responseData = await s3ApiRequestSOAP.call(this, bucketName, 'PUT', `${path}${fileName}`, body, qs, headers, {}, region); + } + returnData.push({ success: true }); + } + } + } + if (resource === 'file' && operation === 'download') { + // For file downloads the files get attached to the existing items + return this.prepareOutputData(items); + } else { + return [this.helpers.returnJsonArray(returnData)]; + } + } +} diff --git a/packages/nodes-base/nodes/S3/generic-s3.png b/packages/nodes-base/nodes/S3/generic-s3.png new file mode 100644 index 0000000000000000000000000000000000000000..4f9faf6dd14612bf493cae796f551cab0d3d7e4e GIT binary patch literal 5600 zcmeHLYcy2t-yX&(jgVtW#yrU6Fc^u+m@tJz^PrI9U^2s)5yOnbU?`!S<&a}iNF_qb zX@pTBq#TA&l2a%tLi|T>>v`U_-t~Uy!~5Y~t9$Le@4fHebzl2;U%z{;eXSjfu{049 z+$#tG0EEm;(N>%uvwaHiajprXOLsY4Y_N?T#R^LW`}&d35r{Z2CCC>C#?c7p000_0 zAfZF3MFV8Kcn$IwM$Wun_gE6(O~Hgof)fvNSr&Z)EYSNoD8EWb`gng~$m1LLtI}+< z5@NYk$FCncV1Bj3?5_8Ls|5WJW?sRbF#PPB?)w7d#&z}A>X&Ab_J&liw0PSksMi?Z z4}X^)-sK3@H+X%CfZcGO z1QHUmSl0vr6FiU*+d~!_7QV(fJi#>B4`&^0Y2zO3?XKqmK^X|@(-0g29~=b>ruh&_ z{sSRApn5Led9!x_ErlqCEL8$o$ktkT28p;2!ZHoWmK;!(~ z{RqAk0+|Hf=ES;@11LxcgcAq)&1}5~2x#ZL0|TB_pRQuz$+3t^`}30D()f>Df8_ct1%3$ep6E%2Y+_5VvQ!GDb%j>K_uRE}*s(^R~dWBGXejm&HWI4xAbBZlJ+m(9?IHZ=Bp zjswwgARxMFs}n>nM1O{)JhrREnK!m(C>(m!!Po#jU)}X%cRE z8mATqZ+Ux6K}tiqGmHLhXfD6H-+6t^TUhmZ;Xj9CYT;1mYFzQRNMn{NL<>ypSuF`2 z0w6aHBW*In5}`)z_byGzMX1yqN=^~r(dmfTYaH*ZRw!u=yv_Ywbdsr6z~dNs8kBG% zZZBja>g3RVV7$CAgRg>Llx~MSD%_*ki`Dg9S1mDqSC4Ti5TBybu?yU_#CCHuD`k>l z7xhTC)_Pj9<#+*~GPj^&Asj0);hG4Jt~jEQz(xq}GumwGsVXk4$ZO1XILFU??qgx6 zno5y52miF^)7_Y}*9RRF>{(`=_C2EGQcqFM^>;OD2fv{&qX% zw3P;M+_ZrkPOtNv@0s7-Y-DOdG#Y?wMXD}GqGyacKNHvDq{###-n(LIxjUnt@5m66|M~B9B6y=2n+lX}h}GSDz5?KXRcRkFL!av0k`# zc0eie#b!lrA$?x|rAVcak~ylTcpgt~S7bc3Tg7oVWDcH2`4RRE)Ka0V)=W9bnDB_Q4K4ni zPNW`Rz1J$WD7|%I%{=eSqlQuq?Zn;a@K;E_;?55yc!e%1N?dp~XanuyO$T=Hn5#-f#W702 z=Muj#AUIQQ_i=f9<7ln|;Su+;OH8L4L^_&ZZ2pK`M+~TBv?ejlh^Iqn1(+z`Sfu?U zx+~K+$hQ7E$`&9RX8kf(&M#KHN2rP$#dTI>k>5Qk_Qj%sPVt2Eu?c4dTUQdmKykJH z?S9{zMNcm+hdwysy!9Dcp?NjlqT4LkFC~Wn(!E6Ax2!T@Y-GY*9G*1+Nr@W_Xd}l2-64@)m8czN?MR|q6fUZnOx{wyR2GZ+(76(#qPHNV0s-bzak!eX}FhC~X{|eZ|Q^61*oh`9;>F z=)rLNg*uH)=?YY8&7Ot6HG!U;XY&@D`65k|=mn>4-TJaZf%z(s0cQ=$Fb70~_l-XWl)3W(Rm z!x4)_;l#aio!=VolzreeVM-ZZ{Vu)Q>;Ge8H6CvNrVpjX!f16Kj-4dc{Kzg7Ep8fRMmLps29idHTiPh?h?UnsZdTUX}MDa4>7#jVZ|7(YYv&$U?x<5%i(ATVUj$2vj;#+p%ZdJ5L})ep(qSuyGp*tE z=atrT5qb>0Ybqc;vd>L{wfjmQCHb2p@wU8+OI#aY|B@NPqz*GmV01qTIR~fXUn`9o zH<@dWjx%?~4acqk2W865HJjPz+g@^SPA|OPvr=7O6agE1pfsl`ehj!d`$Z<^iudR0 zF_-q+lS7ip-@jI=!$ISkS?0$b+*;jO%*Gt-{BS+X%c z^I7em1Fk9(8>sK1in38>x{z}!S~Igun=P8#EU(g<^5u|^8vt@n=LQd2pq}=L!kXXO zX#Q$*)iPA~wua=}YzLi_jqiG24Sg*I;&)xKopDL2=-F`k(tE|}`_+ftU#y0|Uty0v z_Jf`xUJH%hT8@_>KDzPYV9Ph@TLJxC+F?7_-(4!-TKciS@vG5r5qQ6TZ%ab>kyiXc zc=UG;H)db9{#N2~?#61|U%j!NJ&xL8A^lt@(}V*UqjfK--CIlAr}RT21`-0`ls!pU z0VgSw)diLIpp7A5&YaA%Y5ROT2Uhy9w?OXZdhs*ohj$!VQm98>kS#jwZ_QVSCI!-k z)o)Z*K-CGu7n)LBjV+*u2hy#qL?(%g8b0$43VbQ{NKcC(&9aUM=E=K(H|mEqeJ_f4 zhCH4Sino4^goU?x48k%+m_0LHH*QSb3OrZW;2HBR@4Eh#0iBQCmllGcU&Mp^zG)1^ zWr{Xu`|~j|IcP+vg=@w=CgP*27ob>{nfK-^kmXWMr;%j#^X&qR$jK;;I-HL&)|9jbb+ZVFDB}w~LusaPovA zEq8n+B7g5SMh&`Ne(`)T`yuuArd# Date: Tue, 18 Aug 2020 17:11:20 +0400 Subject: [PATCH 5/8] Revert AWS S3 node to its original state --- .../nodes-base/nodes/Aws/S3/AwsS3.node.ts | 72 +++++++++---------- .../nodes/Aws/S3/GenericFunctions.ts | 45 ++++-------- 2 files changed, 44 insertions(+), 73 deletions(-) diff --git a/packages/nodes-base/nodes/Aws/S3/AwsS3.node.ts b/packages/nodes-base/nodes/Aws/S3/AwsS3.node.ts index 34122f1225..5af9ad3d33 100644 --- a/packages/nodes-base/nodes/Aws/S3/AwsS3.node.ts +++ b/packages/nodes-base/nodes/Aws/S3/AwsS3.node.ts @@ -41,9 +41,9 @@ import { } from './FileDescription'; import { - s3ApiRequestREST, - s3ApiRequestSOAP, - s3ApiRequestSOAPAllItems, + awsApiRequestREST, + awsApiRequestSOAP, + awsApiRequestSOAPAllItems, } from './GenericFunctions'; export class AwsS3 implements INodeType { @@ -65,7 +65,7 @@ export class AwsS3 implements INodeType { { name: 'aws', required: true, - }, + } ], properties: [ { @@ -113,15 +113,7 @@ export class AwsS3 implements INodeType { if (resource === 'bucket') { //https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateBucket.html if (operation === 'create') { - - let credentials; - - try { - credentials = this.getCredentials('aws'); - } catch (error) { - throw new Error(error); - } - + const credentials = this.getCredentials('aws'); const name = this.getNodeParameter('name', i) as string; const additionalFields = this.getNodeParameter('additionalFields', i) as IDataObject; if (additionalFields.acl) { @@ -166,7 +158,7 @@ export class AwsS3 implements INodeType { const builder = new Builder(); data = builder.buildObject(body); } - responseData = await s3ApiRequestSOAP.call(this, `${name}`, 'PUT', '', data, qs, headers); + responseData = await awsApiRequestSOAP.call(this, `${name}.s3`, 'PUT', '', data, qs, headers); returnData.push({ success: true }); } @@ -174,10 +166,10 @@ export class AwsS3 implements INodeType { if (operation === 'getAll') { const returnAll = this.getNodeParameter('returnAll', 0) as boolean; if (returnAll) { - responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListAllMyBucketsResult.Buckets.Bucket', '', 'GET', ''); + responseData = await awsApiRequestSOAPAllItems.call(this, 'ListAllMyBucketsResult.Buckets.Bucket', 's3', 'GET', ''); } else { qs.limit = this.getNodeParameter('limit', 0) as number; - responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListAllMyBucketsResult.Buckets.Bucket', '', 'GET', '', '', qs); + responseData = await awsApiRequestSOAPAllItems.call(this, 'ListAllMyBucketsResult.Buckets.Bucket', 's3', 'GET', '', '', qs); responseData = responseData.slice(0, qs.limit); } returnData.push.apply(returnData, responseData); @@ -216,15 +208,15 @@ export class AwsS3 implements INodeType { qs['list-type'] = 2; - responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' }); + responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { location: '' }); const region = responseData.LocationConstraint._ as string; if (returnAll) { - responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '', '', qs, {}, {}, region); + responseData = await awsApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', `${bucketName}.s3`, 'GET', '', '', qs, {}, {}, region); } else { qs['max-keys'] = this.getNodeParameter('limit', 0) as number; - responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', qs, {}, {}, region); + responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', qs, {}, {}, region); responseData = responseData.ListBucketResult.Contents; } if (Array.isArray(responseData)) { @@ -251,11 +243,11 @@ export class AwsS3 implements INodeType { if (additionalFields.storageClass) { headers['x-amz-storage-class'] = (snakeCase(additionalFields.storageClass as string)).toUpperCase(); } - responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' }); + responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { location: '' }); const region = responseData.LocationConstraint._; - responseData = await s3ApiRequestSOAP.call(this, bucketName, 'PUT', path, '', qs, headers, {}, region); + responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'PUT', path, '', qs, headers, {}, region); returnData.push({ success: true }); } //https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html @@ -263,16 +255,16 @@ export class AwsS3 implements INodeType { const bucketName = this.getNodeParameter('bucketName', i) as string; const folderKey = this.getNodeParameter('folderKey', i) as string; - responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' }); + responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { location: '' }); const region = responseData.LocationConstraint._; - responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '/', '', { 'list-type': 2, prefix: folderKey }, {}, {}, region); + responseData = await awsApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', `${bucketName}.s3`, 'GET', '/', '', { 'list-type': 2, prefix: folderKey }, {}, {}, region); // folder empty then just delete it if (responseData.length === 0) { - responseData = await s3ApiRequestSOAP.call(this, bucketName, 'DELETE', `/${folderKey}`, '', qs, {}, {}, region); + responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'DELETE', `/${folderKey}`, '', qs, {}, {}, region); responseData = { deleted: [ { 'Key': folderKey } ] }; @@ -301,7 +293,7 @@ export class AwsS3 implements INodeType { headers['Content-Type'] = 'application/xml'; - responseData = await s3ApiRequestSOAP.call(this, bucketName, 'POST', '/', data, { delete: '' } , headers, {}, region); + responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'POST', '/', data, { delete: '' } , headers, {}, region); responseData = { deleted: responseData.DeleteResult.Deleted }; } @@ -323,15 +315,15 @@ export class AwsS3 implements INodeType { qs['list-type'] = 2; - responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' }); + responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { location: '' }); const region = responseData.LocationConstraint._; if (returnAll) { - responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '', '', qs, {}, {}, region); + responseData = await awsApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', `${bucketName}.s3`, 'GET', '', '', qs, {}, {}, region); } else { qs.limit = this.getNodeParameter('limit', 0) as number; - responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '', '', qs, {}, {}, region); + responseData = await awsApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', `${bucketName}.s3`, 'GET', '', '', qs, {}, {}, region); } if (Array.isArray(responseData)) { responseData = responseData.filter((e: IDataObject) => (e.Key as string).endsWith('/') && e.Size === '0' && e.Key !== options.folderKey); @@ -412,11 +404,11 @@ export class AwsS3 implements INodeType { const destination = `/${destinationParts.slice(2, destinationParts.length).join('/')}`; - responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' }); + responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { location: '' }); const region = responseData.LocationConstraint._; - responseData = await s3ApiRequestSOAP.call(this, bucketName, 'PUT', destination, '', qs, headers, {}, region); + responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'PUT', destination, '', qs, headers, {}, region); returnData.push(responseData.CopyObjectResult); } @@ -433,11 +425,11 @@ export class AwsS3 implements INodeType { throw new Error('Downloding a whole directory is not yet supported, please provide a file key'); } - let region = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' }); + let region = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { location: '' }); region = region.LocationConstraint._; - const response = await s3ApiRequestREST.call(this, bucketName, 'GET', `/${fileKey}`, '', qs, {}, { encoding: null, resolveWithFullResponse: true }, region); + const response = await awsApiRequestREST.call(this, `${bucketName}.s3`, 'GET', `/${fileKey}`, '', qs, {}, { encoding: null, resolveWithFullResponse: true }, region); let mimeType: string | undefined; if (response.headers['content-type']) { @@ -476,11 +468,11 @@ export class AwsS3 implements INodeType { qs.versionId = options.versionId as string; } - responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' }); + responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { location: '' }); const region = responseData.LocationConstraint._; - responseData = await s3ApiRequestSOAP.call(this, bucketName, 'DELETE', `/${fileKey}`, '', qs, {}, {}, region); + responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'DELETE', `/${fileKey}`, '', qs, {}, {}, region); returnData.push({ success: true }); } @@ -502,15 +494,15 @@ export class AwsS3 implements INodeType { qs['list-type'] = 2; - responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' }); + responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { location: '' }); const region = responseData.LocationConstraint._; if (returnAll) { - responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '', '', qs, {}, {}, region); + responseData = await awsApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', `${bucketName}.s3`, 'GET', '', '', qs, {}, {}, region); } else { qs.limit = this.getNodeParameter('limit', 0) as number; - responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '', '', qs, {}, {}, region); + responseData = await awsApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', `${bucketName}.s3`, 'GET', '', '', qs, {}, {}, region); responseData = responseData.splice(0, qs.limit); } if (Array.isArray(responseData)) { @@ -589,7 +581,7 @@ export class AwsS3 implements INodeType { headers['x-amz-tagging'] = tags.join('&'); } - responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' }); + responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { location: '' }); const region = responseData.LocationConstraint._; @@ -612,7 +604,7 @@ export class AwsS3 implements INodeType { headers['Content-MD5'] = createHash('md5').update(body).digest('base64'); - responseData = await s3ApiRequestSOAP.call(this, bucketName, 'PUT', `${path}${fileName || binaryData.fileName}`, body, qs, headers, {}, region); + responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'PUT', `${path}${fileName || binaryData.fileName}`, body, qs, headers, {}, region); } else { @@ -624,7 +616,7 @@ export class AwsS3 implements INodeType { headers['Content-MD5'] = createHash('md5').update(fileContent).digest('base64'); - responseData = await s3ApiRequestSOAP.call(this, bucketName, 'PUT', `${path}${fileName}`, body, qs, headers, {}, region); + responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'PUT', `${path}${fileName}`, body, qs, headers, {}, region); } returnData.push({ success: true }); } diff --git a/packages/nodes-base/nodes/Aws/S3/GenericFunctions.ts b/packages/nodes-base/nodes/Aws/S3/GenericFunctions.ts index bf0839a461..92d620d102 100644 --- a/packages/nodes-base/nodes/Aws/S3/GenericFunctions.ts +++ b/packages/nodes-base/nodes/Aws/S3/GenericFunctions.ts @@ -25,37 +25,16 @@ import { IDataObject, } from 'n8n-workflow'; -import { URL } from 'url'; - -export async function s3ApiRequest(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions | IWebhookFunctions, bucket: string, method: string, path: string, body?: string | Buffer, query: IDataObject = {}, headers?: object, option: IDataObject = {}, region?: string): Promise { // tslint:disable-line:no-any - - let credentials; - - try { - credentials = this.getCredentials('aws'); - } catch (error) { - throw new Error(error); - } - +export async function awsApiRequest(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions | IWebhookFunctions, service: string, method: string, path: string, body?: string | Buffer, query: IDataObject = {}, headers?: object, option: IDataObject = {}, region?: string): Promise { // tslint:disable-line:no-any + const credentials = this.getCredentials('aws'); if (credentials === undefined) { throw new Error('No credentials got returned!'); } - - const endpoint = new URL(`https://${bucket}.s3.${region || credentials.region}.amazonaws.com`); - - endpoint.pathname = path; + const endpoint = `${service}.${region || credentials.region}.amazonaws.com`; // Sign AWS API request with the user credentials - const signOpts = { - headers: headers || {}, - region: region || credentials.region, - host: endpoint.host, - method, - path: `${path}?${queryToString(query).replace(/\+/g, '%2B')}`, - service: 's3', - body - }; + const signOpts = {headers: headers || {}, host: endpoint, method, path: `${path}?${queryToString(query).replace(/\+/g, '%2B')}`, body}; sign(signOpts, { accessKeyId: `${credentials.accessKeyId}`, secretAccessKey: `${credentials.secretAccessKey}`}); @@ -63,7 +42,7 @@ export async function s3ApiRequest(this: IHookFunctions | IExecuteFunctions | IL headers: signOpts.headers, method, qs: query, - uri: endpoint, + uri: `https://${endpoint}${signOpts.path}`, body: signOpts.body, }; @@ -73,7 +52,7 @@ export async function s3ApiRequest(this: IHookFunctions | IExecuteFunctions | IL try { return await this.helpers.request!(options); } catch (error) { - const errorMessage = error.response?.body.message || error.response?.body.Message || error.message; + const errorMessage = error.response.body.message || error.response.body.Message || error.message; if (error.statusCode === 403) { if (errorMessage === 'The security token included in the request is invalid.') { @@ -87,8 +66,8 @@ export async function s3ApiRequest(this: IHookFunctions | IExecuteFunctions | IL } } -export async function s3ApiRequestREST(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions, bucket: string, method: string, path: string, body?: string, query: IDataObject = {}, headers?: object, options: IDataObject = {}, region?: string): Promise { // tslint:disable-line:no-any - const response = await s3ApiRequest.call(this, bucket, method, path, body, query, headers, options, region); +export async function awsApiRequestREST(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions, service: string, method: string, path: string, body?: string, query: IDataObject = {}, headers?: object, options: IDataObject = {}, region?: string): Promise { // tslint:disable-line:no-any + const response = await awsApiRequest.call(this, service, method, path, body, query, headers, options, region); try { return JSON.parse(response); } catch (e) { @@ -96,8 +75,8 @@ export async function s3ApiRequestREST(this: IHookFunctions | IExecuteFunctions } } -export async function s3ApiRequestSOAP(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions | IWebhookFunctions, bucket: string, method: string, path: string, body?: string | Buffer, query: IDataObject = {}, headers?: object, option: IDataObject = {}, region?: string): Promise { // tslint:disable-line:no-any - const response = await s3ApiRequest.call(this, bucket, method, path, body, query, headers, option, region); +export async function awsApiRequestSOAP(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions | IWebhookFunctions, service: string, method: string, path: string, body?: string | Buffer, query: IDataObject = {}, headers?: object, option: IDataObject = {}, region?: string): Promise { // tslint:disable-line:no-any + const response = await awsApiRequest.call(this, service, method, path, body, query, headers, option, region); try { return await new Promise((resolve, reject) => { parseString(response, { explicitArray: false }, (err, data) => { @@ -112,14 +91,14 @@ export async function s3ApiRequestSOAP(this: IHookFunctions | IExecuteFunctions } } -export async function s3ApiRequestSOAPAllItems(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions | IWebhookFunctions, propertyName: string, service: string, method: string, path: string, body?: string, query: IDataObject = {}, headers: IDataObject = {}, option: IDataObject = {}, region?: string): Promise { // tslint:disable-line:no-any +export async function awsApiRequestSOAPAllItems(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions | IWebhookFunctions, propertyName: string, service: string, method: string, path: string, body?: string, query: IDataObject = {}, headers: IDataObject = {}, option: IDataObject = {}, region?: string): Promise { // tslint:disable-line:no-any const returnData: IDataObject[] = []; let responseData; do { - responseData = await s3ApiRequestSOAP.call(this, service, method, path, body, query, headers, option, region); + responseData = await awsApiRequestSOAP.call(this, service, method, path, body, query, headers, option, region); //https://forums.aws.amazon.com/thread.jspa?threadID=55746 if (get(responseData, `${propertyName.split('.')[0]}.NextContinuationToken`)) { From 76ab07d60f114f3dd6158b0c9c5c38e69dd436e7 Mon Sep 17 00:00:00 2001 From: Denis Palashevskii Date: Tue, 1 Sep 2020 19:51:45 +0400 Subject: [PATCH 6/8] Remove code duplication --- .../nodes-base/nodes/S3/BucketDescription.ts | 327 ------- .../nodes-base/nodes/S3/FileDescription.ts | 922 ------------------ .../nodes-base/nodes/S3/FolderDescription.ts | 278 ------ packages/nodes-base/nodes/S3/S3.node.ts | 6 +- 4 files changed, 3 insertions(+), 1530 deletions(-) delete mode 100644 packages/nodes-base/nodes/S3/BucketDescription.ts delete mode 100644 packages/nodes-base/nodes/S3/FileDescription.ts delete mode 100644 packages/nodes-base/nodes/S3/FolderDescription.ts diff --git a/packages/nodes-base/nodes/S3/BucketDescription.ts b/packages/nodes-base/nodes/S3/BucketDescription.ts deleted file mode 100644 index a6c8d67cd1..0000000000 --- a/packages/nodes-base/nodes/S3/BucketDescription.ts +++ /dev/null @@ -1,327 +0,0 @@ -import { - INodeProperties, -} from 'n8n-workflow'; - -export const bucketOperations = [ - { - displayName: 'Operation', - name: 'operation', - type: 'options', - displayOptions: { - show: { - resource: [ - 'bucket', - ], - }, - }, - options: [ - { - name: 'Create', - value: 'create', - description: 'Create a bucket', - }, - { - name: 'Get All', - value: 'getAll', - description: 'Get all buckets', - }, - { - name: 'Search', - value: 'search', - description: 'Search within a bucket', - }, - ], - default: 'create', - description: 'The operation to perform.', - }, -] as INodeProperties[]; - -export const bucketFields = [ - -/* -------------------------------------------------------------------------- */ -/* bucket:create */ -/* -------------------------------------------------------------------------- */ - { - displayName: 'Name', - name: 'name', - type: 'string', - required: true, - default: '', - displayOptions: { - show: { - resource: [ - 'bucket', - ], - operation: [ - 'create', - ], - }, - }, - description: 'A succinct description of the nature, symptoms, cause, or effect of the bucket.', - }, - { - displayName: 'Additional Fields', - name: 'additionalFields', - type: 'collection', - placeholder: 'Add Field', - displayOptions: { - show: { - resource: [ - 'bucket', - ], - operation: [ - 'create', - ], - }, - }, - default: {}, - options: [ - { - displayName: 'ACL', - name: 'acl', - type: 'options', - options: [ - { - name: 'Authenticated Read', - value: 'authenticatedRead', - }, - { - name: 'Private', - value: 'Private', - }, - { - name: 'Public Read', - value: 'publicRead', - }, - { - name: 'Public Read Write', - value: 'publicReadWrite', - }, - ], - default: '', - description: 'The canned ACL to apply to the bucket.', - }, - { - displayName: 'Bucket Object Lock Enabled', - name: 'bucketObjectLockEnabled', - type: 'boolean', - default: false, - description: 'Specifies whether you want S3 Object Lock to be enabled for the new bucket.', - }, - { - displayName: 'Grant Full Control', - name: 'grantFullControl', - type: 'boolean', - default: false, - description: 'Allows grantee the read, write, read ACP, and write ACP permissions on the bucket.', - }, - { - displayName: 'Grant Read', - name: 'grantRead', - type: 'boolean', - default: false, - description: 'Allows grantee to list the objects in the bucket.', - }, - { - displayName: 'Grant Read ACP', - name: 'grantReadAcp', - type: 'boolean', - default: false, - description: 'Allows grantee to read the bucket ACL.', - }, - { - displayName: 'Grant Write', - name: 'grantWrite', - type: 'boolean', - default: false, - description: 'Allows grantee to create, overwrite, and delete any object in the bucket.', - }, - { - displayName: 'Grant Write ACP', - name: 'grantWriteAcp', - type: 'boolean', - default: false, - description: 'Allows grantee to write the ACL for the applicable bucket.', - }, - { - displayName: 'Region', - name: 'region', - type: 'string', - default: '', - description: 'Region you want to create the bucket in, by default the buckets are created on the region defined on the credentials.', - }, - ], - }, -/* -------------------------------------------------------------------------- */ -/* bucket:getAll */ -/* -------------------------------------------------------------------------- */ - { - displayName: 'Return All', - name: 'returnAll', - type: 'boolean', - displayOptions: { - show: { - operation: [ - 'getAll', - ], - resource: [ - 'bucket', - ], - }, - }, - default: false, - description: 'If all results should be returned or only up to a given limit.', - }, - { - displayName: 'Limit', - name: 'limit', - type: 'number', - displayOptions: { - show: { - operation: [ - 'getAll', - ], - resource: [ - 'bucket', - ], - returnAll: [ - false, - ], - }, - }, - typeOptions: { - minValue: 1, - maxValue: 500, - }, - default: 100, - description: 'How many results to return.', - }, -/* -------------------------------------------------------------------------- */ -/* bucket:search */ -/* -------------------------------------------------------------------------- */ - { - displayName: 'Bucket Name', - name: 'bucketName', - type: 'string', - required: true, - default: '', - displayOptions: { - show: { - resource: [ - 'bucket', - ], - operation: [ - 'search', - ], - }, - }, - }, - { - displayName: 'Return All', - name: 'returnAll', - type: 'boolean', - displayOptions: { - show: { - operation: [ - 'search', - ], - resource: [ - 'bucket', - ], - }, - }, - default: false, - description: 'If all results should be returned or only up to a given limit.', - }, - { - displayName: 'Limit', - name: 'limit', - type: 'number', - displayOptions: { - show: { - operation: [ - 'search', - ], - resource: [ - 'bucket', - ], - returnAll: [ - false, - ], - }, - }, - typeOptions: { - minValue: 1, - maxValue: 500, - }, - default: 100, - description: 'How many results to return.', - }, - { - displayName: 'Additional Fields', - name: 'additionalFields', - type: 'collection', - placeholder: 'Add Field', - displayOptions: { - show: { - resource: [ - 'bucket', - ], - operation: [ - 'search', - ], - }, - }, - default: {}, - options: [ - { - displayName: 'Delimiter', - name: 'delimiter', - type: 'string', - default: '', - description: 'A delimiter is a character you use to group keys.', - }, - { - displayName: 'Encoding Type', - name: 'encodingType', - type: 'options', - options: [ - { - name: 'URL', - value: 'url', - }, - ], - default: '', - description: 'Encoding type used by Amazon S3 to encode object keys in the response.', - }, - { - displayName: 'Fetch Owner', - name: 'fetchOwner', - type: 'boolean', - default: false, - description: 'The owner field is not present in listV2 by default, if you want to return owner field with each key in the result then set the fetch owner field to true.', - }, - { - displayName: 'Prefix', - name: 'prefix', - type: 'string', - default: '', - description: 'Limits the response to keys that begin with the specified prefix.', - }, - { - displayName: 'Requester Pays', - name: 'requesterPays', - type: 'boolean', - default: false, - description: 'Weather the requester will pay for requests and data transfer. While Requester Pays is enabled, anonymous access to this bucket is disabled.', - }, - { - displayName: 'Start After', - name: 'startAfter', - type: 'string', - default: '', - description: 'StartAfter is where you want Amazon S3 to start listing from. Amazon S3 starts listing after this specified key', - }, - ], - }, -] as INodeProperties[]; diff --git a/packages/nodes-base/nodes/S3/FileDescription.ts b/packages/nodes-base/nodes/S3/FileDescription.ts deleted file mode 100644 index 208b847b3d..0000000000 --- a/packages/nodes-base/nodes/S3/FileDescription.ts +++ /dev/null @@ -1,922 +0,0 @@ -import { - INodeProperties, -} from 'n8n-workflow'; - -export const fileOperations = [ - { - displayName: 'Operation', - name: 'operation', - type: 'options', - displayOptions: { - show: { - resource: [ - 'file', - ], - }, - }, - options: [ - { - name: 'Copy', - value: 'copy', - description: 'Copy a file', - }, - { - name: 'Delete', - value: 'delete', - description: 'Delete a file', - }, - { - name: 'Download', - value: 'download', - description: 'Download a file', - }, - { - name: 'Get All', - value: 'getAll', - description: 'Get all files', - }, - { - name: 'Upload', - value: 'upload', - description: 'Upload a file', - }, - ], - default: 'download', - description: 'The operation to perform.', - }, -] as INodeProperties[]; - -export const fileFields = [ - -/* -------------------------------------------------------------------------- */ -/* file:copy */ -/* -------------------------------------------------------------------------- */ - { - displayName: 'Source Path', - name: 'sourcePath', - type: 'string', - required: true, - default: '', - placeholder: '/bucket/my-image.jpg', - displayOptions: { - show: { - resource: [ - 'file', - ], - operation: [ - 'copy', - ], - }, - }, - description: 'The name of the source bucket and key name of the source object, separated by a slash (/)', - }, - { - displayName: 'Destination Path', - name: 'destinationPath', - type: 'string', - required: true, - default: '', - placeholder: '/bucket/my-second-image.jpg', - displayOptions: { - show: { - resource: [ - 'file', - ], - operation: [ - 'copy', - ], - }, - }, - description: 'The name of the destination bucket and key name of the destination object, separated by a slash (/)', - }, - { - displayName: 'Additional Fields', - name: 'additionalFields', - type: 'collection', - placeholder: 'Add Field', - displayOptions: { - show: { - resource: [ - 'file', - ], - operation: [ - 'copy', - ], - }, - }, - default: {}, - options: [ - { - displayName: 'ACL', - name: 'acl', - type: 'options', - options: [ - { - name: 'Authenticated Read', - value: 'authenticatedRead' - }, - { - name: 'AWS Exec Read', - value: 'awsExecRead' - }, - { - name: 'Bucket Owner Full Control', - value: 'bucketOwnerFullControl' - }, - { - name: 'Bucket Owner Read', - value: 'bucketOwnerRead' - }, - { - name: 'Private', - value: 'private', - }, - { - name: 'Public Read', - value: 'publicRead' - }, - { - name: 'Public Read Write', - value: 'publicReadWrite' - }, - ], - default: 'private', - description: 'The canned ACL to apply to the object.' - }, - { - displayName: 'Grant Full Control', - name: 'grantFullControl', - type: 'boolean', - default: false, - description: 'Gives the grantee READ, READ_ACP, and WRITE_ACP permissions on the object.', - }, - { - displayName: 'Grant Read', - name: 'grantRead', - type: 'boolean', - default: false, - description: 'Allows grantee to read the object data and its metadata.', - }, - { - displayName: 'Grant Read ACP', - name: 'grantReadAcp', - type: 'boolean', - default: false, - description: 'Allows grantee to read the object ACL.', - }, - { - displayName: 'Grant Write ACP', - name: 'grantWriteAcp', - type: 'boolean', - default: false, - description: 'Allows grantee to write the ACL for the applicable object.', - }, - { - displayName: 'Lock Legal Hold', - name: 'lockLegalHold', - type: 'boolean', - default: false, - description: 'Specifies whether a legal hold will be applied to this object', - }, - { - displayName: 'Lock Mode', - name: 'lockMode', - type: 'options', - options: [ - { - name: 'Governance', - value: 'governance', - }, - { - name: 'Compliance', - value: 'compliance', - }, - ], - default: '', - description: 'The Object Lock mode that you want to apply to this object.', - }, - { - displayName: 'Lock Retain Until Date', - name: 'lockRetainUntilDate', - type: 'dateTime', - default: '', - description: `The date and time when you want this object's Object Lock to expire.`, - }, - { - displayName: 'Metadata Directive', - name: 'metadataDirective', - type: 'options', - options: [ - { - name: 'Copy', - value: 'copy', - }, - { - name: 'Replace', - value: 'replace', - }, - ], - default: '', - description: 'Specifies whether the metadata is copied from the source object or replaced with metadata provided in the request.', - }, - { - displayName: 'Requester Pays', - name: 'requesterPays', - type: 'boolean', - default: false, - description: 'Weather the requester will pay for requests and data transfer. While Requester Pays is enabled, anonymous access to this bucket is disabled.', - }, - { - displayName: 'Server Side Encryption', - name: 'serverSideEncryption', - type: 'options', - options: [ - { - name: 'AES256', - value: 'AES256', - }, - { - name: 'AWS:KMS', - value: 'aws:kms', - }, - ], - default: '', - description: 'The server-side encryption algorithm used when storing this object in Amazon S3', - }, - { - displayName: 'Server Side Encryption Context', - name: 'serverSideEncryptionContext', - type: 'string', - default: '', - description: 'Specifies the AWS KMS Encryption Context to use for object encryption', - }, - { - displayName: 'Server Side Encryption AWS KMS Key ID', - name: 'encryptionAwsKmsKeyId', - type: 'string', - default: '', - description: 'If x-amz-server-side-encryption is present and has the value of aws:kms', - }, - { - displayName: 'Server Side Encryption Customer Algorithm', - name: 'serversideEncryptionCustomerAlgorithm', - type: 'string', - default: '', - description: 'Specifies the algorithm to use to when encrypting the object (for example, AES256).', - }, - { - displayName: 'Server Side Encryption Customer Key', - name: 'serversideEncryptionCustomerKey', - type: 'string', - default: '', - description: 'Specifies the customer-provided encryption key for Amazon S3 to use in encrypting data', - }, - { - displayName: 'Server Side Encryption Customer Key MD5', - name: 'serversideEncryptionCustomerKeyMD5', - type: 'string', - default: '', - description: 'Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321.', - }, - { - displayName: 'Storage Class', - name: 'storageClass', - type: 'options', - options: [ - { - name: 'Deep Archive', - value: 'deepArchive', - }, - { - name: 'Intelligent Tiering', - value: 'intelligentTiering', - }, - { - name: 'One Zone IA', - value: 'onezoneIA', - }, - { - name: 'Glacier', - value: 'glacier', - }, - { - name: 'Standard', - value: 'standard', - }, - { - name: 'Standard IA', - value: 'standardIA', - }, - ], - default: 'standard', - description: 'Amazon S3 storage classes.', - }, - { - displayName: 'Tagging Directive', - name: 'taggingDirective', - type: 'options', - options: [ - { - name: 'Copy', - value: 'copy', - }, - { - name: 'Replace', - value: 'replace', - }, - ], - default: '', - description: 'Specifies whether the metadata is copied from the source object or replaced with metadata provided in the request.', - }, - ], - }, -/* -------------------------------------------------------------------------- */ -/* file:upload */ -/* -------------------------------------------------------------------------- */ - { - displayName: 'Bucket Name', - name: 'bucketName', - type: 'string', - required: true, - default: '', - displayOptions: { - show: { - resource: [ - 'file', - ], - operation: [ - 'upload', - ], - }, - }, - }, - { - displayName: 'File Name', - name: 'fileName', - type: 'string', - default: '', - placeholder: 'hello.txt', - required: true, - displayOptions: { - show: { - resource: [ - 'file', - ], - operation: [ - 'upload', - ], - binaryData: [ - false, - ], - }, - }, - }, - { - displayName: 'File Name', - name: 'fileName', - type: 'string', - default: '', - displayOptions: { - show: { - resource: [ - 'file', - ], - operation: [ - 'upload', - ], - binaryData: [ - true, - ], - }, - }, - description: 'If not set the binary data filename will be used.', - }, - { - displayName: 'Binary Data', - name: 'binaryData', - type: 'boolean', - default: true, - displayOptions: { - show: { - operation: [ - 'upload' - ], - resource: [ - 'file', - ], - }, - }, - description: 'If the data to upload should be taken from binary field.', - }, - { - displayName: 'File Content', - name: 'fileContent', - type: 'string', - default: '', - displayOptions: { - show: { - operation: [ - 'upload' - ], - resource: [ - 'file', - ], - binaryData: [ - false - ], - }, - }, - placeholder: '', - description: 'The text content of the file to upload.', - }, - { - displayName: 'Binary Property', - name: 'binaryPropertyName', - type: 'string', - default: 'data', - required: true, - displayOptions: { - show: { - operation: [ - 'upload' - ], - resource: [ - 'file', - ], - binaryData: [ - true - ], - }, - - }, - placeholder: '', - description: 'Name of the binary property which contains
the data for the file to be uploaded.', - }, - { - displayName: 'Additional Fields', - name: 'additionalFields', - type: 'collection', - placeholder: 'Add Field', - displayOptions: { - show: { - resource: [ - 'file', - ], - operation: [ - 'upload', - ], - }, - }, - default: {}, - options: [ - { - displayName: 'ACL', - name: 'acl', - type: 'options', - options: [ - { - name: 'Authenticated Read', - value: 'authenticatedRead' - }, - { - name: 'AWS Exec Read', - value: 'awsExecRead' - }, - { - name: 'Bucket Owner Full Control', - value: 'bucketOwnerFullControl' - }, - { - name: 'Bucket Owner Read', - value: 'bucketOwnerRead' - }, - { - name: 'Private', - value: 'private', - }, - { - name: 'Public Read', - value: 'publicRead' - }, - { - name: 'Public Read Write', - value: 'publicReadWrite' - }, - ], - default: 'private', - description: 'The canned ACL to apply to the object.' - }, - { - displayName: 'Grant Full Control', - name: 'grantFullControl', - type: 'boolean', - default: false, - description: 'Gives the grantee READ, READ_ACP, and WRITE_ACP permissions on the object.', - }, - { - displayName: 'Grant Read', - name: 'grantRead', - type: 'boolean', - default: false, - description: 'Allows grantee to read the object data and its metadata.', - }, - { - displayName: 'Grant Read ACP', - name: 'grantReadAcp', - type: 'boolean', - default: false, - description: 'Allows grantee to read the object ACL.', - }, - { - displayName: 'Grant Write ACP', - name: 'grantWriteAcp', - type: 'boolean', - default: false, - description: 'Allows grantee to write the ACL for the applicable object.', - }, - { - displayName: 'Lock Legal Hold', - name: 'lockLegalHold', - type: 'boolean', - default: false, - description: 'Specifies whether a legal hold will be applied to this object', - }, - { - displayName: 'Lock Mode', - name: 'lockMode', - type: 'options', - options: [ - { - name: 'Governance', - value: 'governance', - }, - { - name: 'Compliance', - value: 'compliance', - }, - ], - default: '', - description: 'The Object Lock mode that you want to apply to this object.', - }, - { - displayName: 'Lock Retain Until Date', - name: 'lockRetainUntilDate', - type: 'dateTime', - default: '', - description: `The date and time when you want this object's Object Lock to expire.`, - }, - { - displayName: 'Parent Folder Key', - name: 'parentFolderKey', - type: 'string', - default: '', - description: 'Parent file you want to create the file in', - }, - { - displayName: 'Requester Pays', - name: 'requesterPays', - type: 'boolean', - default: false, - description: 'Weather the requester will pay for requests and data transfer. While Requester Pays is enabled, anonymous access to this bucket is disabled.', - }, - { - displayName: 'Server Side Encryption', - name: 'serverSideEncryption', - type: 'options', - options: [ - { - name: 'AES256', - value: 'AES256', - }, - { - name: 'AWS:KMS', - value: 'aws:kms', - }, - ], - default: '', - description: 'The server-side encryption algorithm used when storing this object in Amazon S3', - }, - { - displayName: 'Server Side Encryption Context', - name: 'serverSideEncryptionContext', - type: 'string', - default: '', - description: 'Specifies the AWS KMS Encryption Context to use for object encryption', - }, - { - displayName: 'Server Side Encryption AWS KMS Key ID', - name: 'encryptionAwsKmsKeyId', - type: 'string', - default: '', - description: 'If x-amz-server-side-encryption is present and has the value of aws:kms', - }, - { - displayName: 'Server Side Encryption Customer Algorithm', - name: 'serversideEncryptionCustomerAlgorithm', - type: 'string', - default: '', - description: 'Specifies the algorithm to use to when encrypting the object (for example, AES256).', - }, - { - displayName: 'Server Side Encryption Customer Key', - name: 'serversideEncryptionCustomerKey', - type: 'string', - default: '', - description: 'Specifies the customer-provided encryption key for Amazon S3 to use in encrypting data', - }, - { - displayName: 'Server Side Encryption Customer Key MD5', - name: 'serversideEncryptionCustomerKeyMD5', - type: 'string', - default: '', - description: 'Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321.', - }, - { - displayName: 'Storage Class', - name: 'storageClass', - type: 'options', - options: [ - { - name: 'Deep Archive', - value: 'deepArchive', - }, - { - name: 'Intelligent Tiering', - value: 'intelligentTiering', - }, - { - name: 'One Zone IA', - value: 'onezoneIA', - }, - { - name: 'Glacier', - value: 'glacier', - }, - { - name: 'Standard', - value: 'standard', - }, - { - name: 'Standard IA', - value: 'standardIA', - }, - ], - default: 'standard', - description: 'Amazon S3 storage classes.', - }, - ], - }, - { - displayName: 'Tags', - name: 'tagsUi', - placeholder: 'Add Tag', - type: 'fixedCollection', - default: '', - typeOptions: { - multipleValues: true, - }, - displayOptions: { - show: { - resource: [ - 'file', - ], - operation: [ - 'upload', - ], - }, - }, - options: [ - { - name: 'tagsValues', - displayName: 'Tag', - values: [ - { - displayName: 'Key', - name: 'key', - type: 'string', - default: '', - description: '', - }, - { - displayName: 'Value', - name: 'value', - type: 'string', - default: '', - description: '', - }, - ], - } - ], - description: 'Optional extra headers to add to the message (most headers are allowed).', - }, -/* -------------------------------------------------------------------------- */ -/* file:download */ -/* -------------------------------------------------------------------------- */ - { - displayName: 'Bucket Name', - name: 'bucketName', - type: 'string', - required: true, - default: '', - displayOptions: { - show: { - resource: [ - 'file', - ], - operation: [ - 'download', - ], - }, - }, - }, - { - displayName: 'File Key', - name: 'fileKey', - type: 'string', - required: true, - default: '', - displayOptions: { - show: { - resource: [ - 'file', - ], - operation: [ - 'download', - ], - }, - }, - }, - { - displayName: 'Binary Property', - name: 'binaryPropertyName', - type: 'string', - required: true, - default: 'data', - displayOptions: { - show: { - operation: [ - 'download' - ], - resource: [ - 'file', - ], - }, - }, - description: 'Name of the binary property to which to
write the data of the read file.', - }, -/* -------------------------------------------------------------------------- */ -/* file:delete */ -/* -------------------------------------------------------------------------- */ - { - displayName: 'Bucket Name', - name: 'bucketName', - type: 'string', - required: true, - default: '', - displayOptions: { - show: { - resource: [ - 'file', - ], - operation: [ - 'delete', - ], - }, - }, - }, - { - displayName: 'File Key', - name: 'fileKey', - type: 'string', - required: true, - default: '', - displayOptions: { - show: { - resource: [ - 'file', - ], - operation: [ - 'delete', - ], - }, - }, - }, - { - displayName: 'Options', - name: 'options', - type: 'collection', - placeholder: 'Add Field', - default: {}, - displayOptions: { - show: { - resource: [ - 'file', - ], - operation: [ - 'delete', - ], - }, - }, - options: [ - { - displayName: 'Version ID', - name: 'versionId', - type: 'string', - default: '', - }, - ], - }, -/* -------------------------------------------------------------------------- */ -/* file:getAll */ -/* -------------------------------------------------------------------------- */ - { - displayName: 'Bucket Name', - name: 'bucketName', - type: 'string', - required: true, - default: '', - displayOptions: { - show: { - resource: [ - 'file', - ], - operation: [ - 'getAll', - ], - }, - }, - }, - { - displayName: 'Return All', - name: 'returnAll', - type: 'boolean', - displayOptions: { - show: { - operation: [ - 'getAll', - ], - resource: [ - 'file', - ], - }, - }, - default: false, - description: 'If all results should be returned or only up to a given limit.', - }, - { - displayName: 'Limit', - name: 'limit', - type: 'number', - displayOptions: { - show: { - operation: [ - 'getAll', - ], - resource: [ - 'file', - ], - returnAll: [ - false, - ], - }, - }, - typeOptions: { - minValue: 1, - maxValue: 500, - }, - default: 100, - description: 'How many results to return.', - }, - { - displayName: 'Options', - name: 'options', - type: 'collection', - placeholder: 'Add Field', - default: {}, - displayOptions: { - show: { - resource: [ - 'file', - ], - operation: [ - 'getAll', - ], - }, - }, - options: [ - { - displayName: 'Fetch Owner', - name: 'fetchOwner', - type: 'boolean', - default: false, - description: 'The owner field is not present in listV2 by default, if you want to return owner field with each key in the result then set the fetch owner field to true.', - }, - { - displayName: 'Folder Key', - name: 'folderKey', - type: 'string', - default: '', - }, - ], - }, -] as INodeProperties[]; diff --git a/packages/nodes-base/nodes/S3/FolderDescription.ts b/packages/nodes-base/nodes/S3/FolderDescription.ts deleted file mode 100644 index 2884206d66..0000000000 --- a/packages/nodes-base/nodes/S3/FolderDescription.ts +++ /dev/null @@ -1,278 +0,0 @@ -import { - INodeProperties, -} from 'n8n-workflow'; - -export const folderOperations = [ - { - displayName: 'Operation', - name: 'operation', - type: 'options', - displayOptions: { - show: { - resource: [ - 'folder', - ], - }, - }, - options: [ - { - name: 'Create', - value: 'create', - description: 'Create a folder', - }, - { - name: 'Delete', - value: 'delete', - description: 'Delete a folder', - }, - { - name: 'Get All', - value: 'getAll', - description: 'Get all folders', - }, - ], - default: 'create', - description: 'The operation to perform.', - }, -] as INodeProperties[]; - -export const folderFields = [ - -/* -------------------------------------------------------------------------- */ -/* folder:create */ -/* -------------------------------------------------------------------------- */ - { - displayName: 'Bucket Name', - name: 'bucketName', - type: 'string', - required: true, - default: '', - displayOptions: { - show: { - resource: [ - 'folder', - ], - operation: [ - 'create', - ], - }, - }, - }, - { - displayName: 'Folder Name', - name: 'folderName', - type: 'string', - required: true, - default: '', - displayOptions: { - show: { - resource: [ - 'folder', - ], - operation: [ - 'create', - ], - }, - }, - }, - { - displayName: 'Additional Fields', - name: 'additionalFields', - type: 'collection', - placeholder: 'Add Field', - displayOptions: { - show: { - resource: [ - 'folder', - ], - operation: [ - 'create', - ], - }, - }, - default: {}, - options: [ - { - displayName: 'Parent Folder Key', - name: 'parentFolderKey', - type: 'string', - default: '', - description: 'Parent folder you want to create the folder in' - }, - { - displayName: 'Requester Pays', - name: 'requesterPays', - type: 'boolean', - default: false, - description: 'Weather the requester will pay for requests and data transfer. While Requester Pays is enabled, anonymous access to this bucket is disabled.', - }, - { - displayName: 'Storage Class', - name: 'storageClass', - type: 'options', - options: [ - { - name: 'Deep Archive', - value: 'deepArchive', - }, - { - name: 'Intelligent Tiering', - value: 'intelligentTiering', - }, - { - name: 'One Zone IA', - value: 'onezoneIA', - }, - { - name: 'Glacier', - value: 'glacier', - }, - { - name: 'Reduced Redundancy', - value: 'RecudedRedundancy', - }, - { - name: 'Standard', - value: 'standard', - }, - { - name: 'Standard IA', - value: 'standardIA', - }, - ], - default: 'standard', - description: 'Amazon S3 storage classes.' - }, - ], - }, -/* -------------------------------------------------------------------------- */ -/* folder:delete */ -/* -------------------------------------------------------------------------- */ - { - displayName: 'Bucket Name', - name: 'bucketName', - type: 'string', - required: true, - default: '', - displayOptions: { - show: { - resource: [ - 'folder', - ], - operation: [ - 'delete', - ], - }, - }, - }, - { - displayName: 'Folder Key', - name: 'folderKey', - type: 'string', - required: true, - default: '', - displayOptions: { - show: { - resource: [ - 'folder', - ], - operation: [ - 'delete', - ], - }, - }, - }, -/* -------------------------------------------------------------------------- */ -/* folder:getAll */ -/* -------------------------------------------------------------------------- */ - { - displayName: 'Bucket Name', - name: 'bucketName', - type: 'string', - required: true, - default: '', - displayOptions: { - show: { - resource: [ - 'folder', - ], - operation: [ - 'getAll', - ], - }, - }, - }, - { - displayName: 'Return All', - name: 'returnAll', - type: 'boolean', - displayOptions: { - show: { - operation: [ - 'getAll', - ], - resource: [ - 'folder', - ], - }, - }, - default: false, - description: 'If all results should be returned or only up to a given limit.', - }, - { - displayName: 'Limit', - name: 'limit', - type: 'number', - displayOptions: { - show: { - operation: [ - 'getAll', - ], - resource: [ - 'folder', - ], - returnAll: [ - false, - ], - }, - }, - typeOptions: { - minValue: 1, - maxValue: 500, - }, - default: 100, - description: 'How many results to return.', - }, - { - displayName: 'Options', - name: 'options', - type: 'collection', - placeholder: 'Add Field', - default: {}, - displayOptions: { - show: { - resource: [ - 'folder', - ], - operation: [ - 'getAll', - ], - }, - }, - options: [ - { - displayName: 'Fetch Owner', - name: 'fetchOwner', - type: 'boolean', - default: false, - description: 'The owner field is not present in listV2 by default, if you want to return owner field with each key in the result then set the fetch owner field to true.', - }, - { - displayName: 'Folder Key', - name: 'folderKey', - type: 'string', - default: '', - }, - ], - }, -] as INodeProperties[]; diff --git a/packages/nodes-base/nodes/S3/S3.node.ts b/packages/nodes-base/nodes/S3/S3.node.ts index 9eba6ab6d3..4272adb5b2 100644 --- a/packages/nodes-base/nodes/S3/S3.node.ts +++ b/packages/nodes-base/nodes/S3/S3.node.ts @@ -28,17 +28,17 @@ import { import { bucketFields, bucketOperations, -} from './BucketDescription'; +} from '../Aws/S3/BucketDescription'; import { folderFields, folderOperations, -} from './FolderDescription'; +} from '../Aws/S3/FolderDescription'; import { fileFields, fileOperations, -} from './FileDescription'; +} from '../Aws/S3/FileDescription'; import { s3ApiRequestREST, From 2f9bab571d5e95e41cb7408f1d13d1f5a945e328 Mon Sep 17 00:00:00 2001 From: Denis Palashevskii Date: Wed, 2 Sep 2020 00:29:56 +0400 Subject: [PATCH 7/8] Trim whitespaces in AWS credentials --- packages/nodes-base/nodes/Aws/S3/GenericFunctions.ts | 2 +- packages/nodes-base/nodes/S3/GenericFunctions.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/nodes-base/nodes/Aws/S3/GenericFunctions.ts b/packages/nodes-base/nodes/Aws/S3/GenericFunctions.ts index 92d620d102..fd7fe838f6 100644 --- a/packages/nodes-base/nodes/Aws/S3/GenericFunctions.ts +++ b/packages/nodes-base/nodes/Aws/S3/GenericFunctions.ts @@ -36,7 +36,7 @@ export async function awsApiRequest(this: IHookFunctions | IExecuteFunctions | I // Sign AWS API request with the user credentials const signOpts = {headers: headers || {}, host: endpoint, method, path: `${path}?${queryToString(query).replace(/\+/g, '%2B')}`, body}; - sign(signOpts, { accessKeyId: `${credentials.accessKeyId}`, secretAccessKey: `${credentials.secretAccessKey}`}); + sign(signOpts, { accessKeyId: `${credentials.accessKeyId}`.trim(), secretAccessKey: `${credentials.secretAccessKey}`.trim()}); const options: OptionsWithUri = { headers: signOpts.headers, diff --git a/packages/nodes-base/nodes/S3/GenericFunctions.ts b/packages/nodes-base/nodes/S3/GenericFunctions.ts index 00f59b3f70..093b7cfb71 100644 --- a/packages/nodes-base/nodes/S3/GenericFunctions.ts +++ b/packages/nodes-base/nodes/S3/GenericFunctions.ts @@ -64,7 +64,7 @@ export async function s3ApiRequest(this: IHookFunctions | IExecuteFunctions | IL body }; - sign(signOpts, { accessKeyId: `${credentials.accessKeyId}`, secretAccessKey: `${credentials.secretAccessKey}`}); + sign(signOpts, { accessKeyId: `${credentials.accessKeyId}`.trim(), secretAccessKey: `${credentials.secretAccessKey}`.trim()}); const options: OptionsWithUri = { headers: signOpts.headers, From de50e2479d7a6f46720e66b9fffe57e14cd02591 Mon Sep 17 00:00:00 2001 From: Denis Palashevskii Date: Wed, 2 Sep 2020 00:41:51 +0400 Subject: [PATCH 8/8] Trim whitespaces in the AWS credentials in other nodes --- packages/nodes-base/nodes/Aws/GenericFunctions.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/nodes-base/nodes/Aws/GenericFunctions.ts b/packages/nodes-base/nodes/Aws/GenericFunctions.ts index ef334b1d67..880c0a02bb 100644 --- a/packages/nodes-base/nodes/Aws/GenericFunctions.ts +++ b/packages/nodes-base/nodes/Aws/GenericFunctions.ts @@ -20,7 +20,7 @@ export async function awsApiRequest(this: IHookFunctions | IExecuteFunctions | I // Sign AWS API request with the user credentials const signOpts = { headers: headers || {}, host: endpoint, method, path, body }; - sign(signOpts, { accessKeyId: `${credentials.accessKeyId}`, secretAccessKey: `${credentials.secretAccessKey}` }); + sign(signOpts, { accessKeyId: `${credentials.accessKeyId}`.trim(), secretAccessKey: `${credentials.secretAccessKey}`.trim() }); const options: OptionsWithUri = { headers: signOpts.headers,