mudhorn/n8n
1
0
Fork 0
mirror of https://github.com/n8n-io/n8n.git synced 2024-11-10 22:54:05 -08:00
Code Issues Actions Packages Projects Releases Wiki Activity

🔀 Merge branch 'RicardoE105-feature/aws-s3'

Browse source
This commit is contained in:
Jan Oberhauser 2020-04-22 11:29:17 +02:00
parent 400bff58a3 b24537b4a9
commit c2d7474d19
4 changed files with 94 additions and 54 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

91
packages/nodes-base/nodes/Aws/S3/AwsS3.node.ts
View file

@ -137,6 +137,11 @@ export class AwsS3 implements INodeType {
if (additionalFields.grantWriteAcp) { if (additionalFields.grantWriteAcp) {
headers['x-amz-grant-write-acp']=additionalFields.grantWriteAcp; headers['x-amz-grant-write-acp']=additionalFields.grantWriteAcp;
} }
let region = credentials!.region as string;
if (additionalFields.region) {
region = additionalFields.region as string;
}
const body: IDataObject = { const body: IDataObject = {
CreateBucketConfiguration: { CreateBucketConfiguration: {
@ -147,9 +152,9 @@ export class AwsS3 implements INodeType {
}; };
let data = ''; let data = '';
// if credentials has the S3 defaul region (us-east-1) the body (XML) does not have to be sent. // if credentials has the S3 defaul region (us-east-1) the body (XML) does not have to be sent.
if (credentials!.region !== 'us-east-1') { if (region !== 'us-east-1') {
// @ts-ignore // @ts-ignore
body.CreateBucketConfiguration.LocationConstraint = [credentials!.region]; body.CreateBucketConfiguration.LocationConstraint = [region];
const builder = new Builder(); const builder = new Builder();
data = builder.buildObject(body); data = builder.buildObject(body);
} }
@ -169,6 +174,7 @@ export class AwsS3 implements INodeType {
} }
returnData.push.apply(returnData, responseData); returnData.push.apply(returnData, responseData);
} }
//https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html //https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html
if (operation === 'search') { if (operation === 'search') {
const bucketName = this.getNodeParameter('bucketName', i) as string; const bucketName = this.getNodeParameter('bucketName', i) as string;
@ -201,11 +207,16 @@ export class AwsS3 implements INodeType {
qs['list-type'] = 2; qs['list-type'] = 2;
responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { location: '' });
const region = responseData.LocationConstraint._ as string;
if (returnAll) { if (returnAll) {
responseData = await awsApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', `${bucketName}.s3`, 'GET', '', '', qs); responseData = await awsApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', `${bucketName}.s3`, 'GET', '', '', qs, {}, {}, region);
} else { } else {
qs['max-keys'] = this.getNodeParameter('limit', 0) as number; qs['max-keys'] = this.getNodeParameter('limit', 0) as number;
responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', qs); responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', qs, {}, {}, region);
responseData = responseData.ListBucketResult.Contents; responseData = responseData.ListBucketResult.Contents;
} }
if (Array.isArray(responseData)) { if (Array.isArray(responseData)) {
@ -232,7 +243,11 @@ export class AwsS3 implements INodeType {
if (additionalFields.storageClass) { if (additionalFields.storageClass) {
headers['x-amz-storage-class'] = (snakeCase(additionalFields.storageClass as string)).toUpperCase(); headers['x-amz-storage-class'] = (snakeCase(additionalFields.storageClass as string)).toUpperCase();
} }
responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'PUT', path, '', qs, headers); responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { location: '' });
const region = responseData.LocationConstraint._;
responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'PUT', path, '', qs, headers, {}, region);
returnData.push({ success: true }); returnData.push({ success: true });
} }
//https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html //https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html
@ -240,12 +255,16 @@ export class AwsS3 implements INodeType {
const bucketName = this.getNodeParameter('bucketName', i) as string; const bucketName = this.getNodeParameter('bucketName', i) as string;
const folderKey = this.getNodeParameter('folderKey', i) as string; const folderKey = this.getNodeParameter('folderKey', i) as string;
responseData = await awsApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', `${bucketName}.s3`, 'GET', '/', '', { 'list-type': 2, prefix: folderKey }); responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { location: '' });
const region = responseData.LocationConstraint._;
responseData = await awsApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', `${bucketName}.s3`, 'GET', '/', '', { 'list-type': 2, prefix: folderKey }, {}, {}, region);
// folder empty then just delete it // folder empty then just delete it
if (responseData.length === 0) { if (responseData.length === 0) {
responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'DELETE', `/${folderKey}`, '', qs); responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'DELETE', `/${folderKey}`, '', qs, {}, {}, region);
responseData = { deleted: [ { 'Key': folderKey } ] }; responseData = { deleted: [ { 'Key': folderKey } ] };
@ -274,7 +293,7 @@ export class AwsS3 implements INodeType {
headers['Content-Type'] = 'application/xml'; headers['Content-Type'] = 'application/xml';
responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'POST', '/', data, { delete: '' } , headers); responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'POST', '/', data, { delete: '' } , headers, {}, region);
responseData = { deleted: responseData.DeleteResult.Deleted }; responseData = { deleted: responseData.DeleteResult.Deleted };
} }
@ -296,11 +315,15 @@ export class AwsS3 implements INodeType {
qs['list-type'] = 2; qs['list-type'] = 2;
responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { location: '' });
const region = responseData.LocationConstraint._;
if (returnAll) { if (returnAll) {
responseData = await awsApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', `${bucketName}.s3`, 'GET', '', '', qs); responseData = await awsApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', `${bucketName}.s3`, 'GET', '', '', qs, {}, {}, region);
} else { } else {
qs.limit = this.getNodeParameter('limit', 0) as number; qs.limit = this.getNodeParameter('limit', 0) as number;
responseData = await awsApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', `${bucketName}.s3`, 'GET', '', '', qs); responseData = await awsApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', `${bucketName}.s3`, 'GET', '', '', qs, {}, {}, region);
} }
if (Array.isArray(responseData)) { if (Array.isArray(responseData)) {
responseData = responseData.filter((e: IDataObject) => (e.Key as string).endsWith('/') && e.Size === '0' && e.Key !== options.folderKey); responseData = responseData.filter((e: IDataObject) => (e.Key as string).endsWith('/') && e.Size === '0' && e.Key !== options.folderKey);
@ -314,12 +337,11 @@ export class AwsS3 implements INodeType {
if (resource === 'file') { if (resource === 'file') {
//https://docs.aws.amazon.com/AmazonS3/latest/API/API_CopyObject.html //https://docs.aws.amazon.com/AmazonS3/latest/API/API_CopyObject.html
if (operation === 'copy') { if (operation === 'copy') {
const source = this.getNodeParameter('source', i) as string; const sourcePath = this.getNodeParameter('sourcePath', i) as string;
const bucketName = this.getNodeParameter('bucketName', i) as string; const destinationPath = this.getNodeParameter('destinationPath', i) as string;
const destination = this.getNodeParameter('destination', i) as string;
const additionalFields = this.getNodeParameter('additionalFields', i) as IDataObject; const additionalFields = this.getNodeParameter('additionalFields', i) as IDataObject;
headers['x-amz-copy-source'] = source; headers['x-amz-copy-source'] = sourcePath;
if (additionalFields.requesterPays) { if (additionalFields.requesterPays) {
headers['x-amz-request-payer'] = 'requester'; headers['x-amz-request-payer'] = 'requester';
@ -376,7 +398,17 @@ export class AwsS3 implements INodeType {
headers['x-amz-metadata-directive'] = (additionalFields.metadataDirective as string).toUpperCase(); headers['x-amz-metadata-directive'] = (additionalFields.metadataDirective as string).toUpperCase();
} }
responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'PUT', `${destination}`, '', qs, headers); const destinationParts = destinationPath.split('/');
const bucketName = destinationParts[1];
const destination = `/${destinationParts.slice(2, destinationParts.length).join('/')}`;
responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { location: '' });
const region = responseData.LocationConstraint._;
responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'PUT', destination, '', qs, headers, {}, region);
returnData.push(responseData.CopyObjectResult); returnData.push(responseData.CopyObjectResult);
} }
@ -393,7 +425,11 @@ export class AwsS3 implements INodeType {
throw new Error('Downloding a whole directory is not yet supported, please provide a file key'); throw new Error('Downloding a whole directory is not yet supported, please provide a file key');
} }
const response = await awsApiRequestREST.call(this, `${bucketName}.s3`, 'GET', `/${fileKey}`, '', qs, {}, { encoding: null, resolveWithFullResponse: true }); let region = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { location: '' });
region = region.LocationConstraint._;
const response = await awsApiRequestREST.call(this, `${bucketName}.s3`, 'GET', `/${fileKey}`, '', qs, {}, { encoding: null, resolveWithFullResponse: true }, region);
let mimeType: string | undefined; let mimeType: string | undefined;
if (response.headers['content-type']) { if (response.headers['content-type']) {
@ -432,7 +468,11 @@ export class AwsS3 implements INodeType {
qs.versionId = options.versionId as string; qs.versionId = options.versionId as string;
} }
responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'DELETE', `/${fileKey}`, '', qs); responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { location: '' });
const region = responseData.LocationConstraint._;
responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'DELETE', `/${fileKey}`, '', qs, {}, {}, region);
returnData.push({ success: true }); returnData.push({ success: true });
} }
@ -454,11 +494,15 @@ export class AwsS3 implements INodeType {
qs['list-type'] = 2; qs['list-type'] = 2;
responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { location: '' });
const region = responseData.LocationConstraint._;
if (returnAll) { if (returnAll) {
responseData = await awsApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', `${bucketName}.s3`, 'GET', '', '', qs); responseData = await awsApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', `${bucketName}.s3`, 'GET', '', '', qs, {}, {}, region);
} else { } else {
qs.limit = this.getNodeParameter('limit', 0) as number; qs.limit = this.getNodeParameter('limit', 0) as number;
responseData = await awsApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', `${bucketName}.s3`, 'GET', '', '', qs); responseData = await awsApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', `${bucketName}.s3`, 'GET', '', '', qs, {}, {}, region);
responseData = responseData.splice(0, qs.limit); responseData = responseData.splice(0, qs.limit);
} }
if (Array.isArray(responseData)) { if (Array.isArray(responseData)) {
@ -536,6 +580,11 @@ export class AwsS3 implements INodeType {
tagsValues.forEach((o: IDataObject) => { tags.push(`${o.key}=${o.value}`); }); tagsValues.forEach((o: IDataObject) => { tags.push(`${o.key}=${o.value}`); });
headers['x-amz-tagging'] = tags.join('&'); headers['x-amz-tagging'] = tags.join('&');
} }
responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { location: '' });
const region = responseData.LocationConstraint._;
if (isBinaryData) { if (isBinaryData) {
const binaryPropertyName = this.getNodeParameter('binaryPropertyName', 0) as string; const binaryPropertyName = this.getNodeParameter('binaryPropertyName', 0) as string;
@ -555,7 +604,7 @@ export class AwsS3 implements INodeType {
headers['Content-MD5'] = createHash('md5').update(body).digest('base64'); headers['Content-MD5'] = createHash('md5').update(body).digest('base64');
responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'PUT', `${path}${fileName || binaryData.fileName}`, body, qs, headers); responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'PUT', `${path}${fileName || binaryData.fileName}`, body, qs, headers, {}, region);
} else { } else {
@ -567,7 +616,7 @@ export class AwsS3 implements INodeType {
headers['Content-MD5'] = createHash('md5').update(fileContent).digest('base64'); headers['Content-MD5'] = createHash('md5').update(fileContent).digest('base64');
responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'PUT', `${path}${fileName}`, body, qs, headers); responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'PUT', `${path}${fileName}`, body, qs, headers, {}, region);
} }
returnData.push({ success: true }); returnData.push({ success: true });
} }

7
packages/nodes-base/nodes/Aws/S3/BucketDescription.ts
View file

@ -143,6 +143,13 @@ export const bucketFields = [
default: false, default: false,
description: 'Allows grantee to write the ACL for the applicable bucket.', description: 'Allows grantee to write the ACL for the applicable bucket.',
}, },
{
displayName: 'Region',
name: 'region',
type: 'string',
default: '',
description: 'Region you want to create the bucket in, by default the buckets are created on the region defined on the credentials.',
},
], ],
}, },
/* -------------------------------------------------------------------------- */ /* -------------------------------------------------------------------------- */

32
packages/nodes-base/nodes/Aws/S3/FileDescription.ts
View file

@ -52,8 +52,8 @@ export const fileFields = [
/* file:copy */ /* file:copy */
/* -------------------------------------------------------------------------- */ /* -------------------------------------------------------------------------- */
{ {
displayName: 'Source', displayName: 'Source Path',
name: 'source', name: 'sourcePath',
type: 'string', type: 'string',
required: true, required: true,
default: '', default: '',
@ -71,11 +71,12 @@ export const fileFields = [
description: 'The name of the source bucket and key name of the source object, separated by a slash (/)', description: 'The name of the source bucket and key name of the source object, separated by a slash (/)',
}, },
{ {
displayName: 'Bucket Name', displayName: 'Destination Path',
name: 'bucketName', name: 'destinationPath',
type: 'string', type: 'string',
required: true, required: true,
default: '', default: '',
placeholder: '/bucket/my-second-image.jpg',
displayOptions: { displayOptions: {
show: { show: {
resource: [ resource: [
@ -86,26 +87,7 @@ export const fileFields = [
], ],
}, },
}, },
description: 'The name of the destination bucket.', description: 'The name of the destination bucket and key name of the destination object, separated by a slash (/)',
},
{
displayName: 'Destination',
name: 'destination',
type: 'string',
required: true,
default: '',
placeholder: '/my-second-image.jpg',
displayOptions: {
show: {
resource: [
'file',
],
operation: [
'copy',
],
},
},
description: 'The key of the destination object.',
}, },
{ {
displayName: 'Additional Fields', displayName: 'Additional Fields',
@ -413,7 +395,7 @@ export const fileFields = [
displayName: 'Binary Data', displayName: 'Binary Data',
name: 'binaryData', name: 'binaryData',
type: 'boolean', type: 'boolean',
default: false, default: true,
displayOptions: { displayOptions: {
show: { show: {
operation: [ operation: [

18
packages/nodes-base/nodes/Aws/S3/GenericFunctions.ts
View file

@ -25,12 +25,13 @@ import {
IDataObject, IDataObject,
} from 'n8n-workflow'; } from 'n8n-workflow';
export async function awsApiRequest(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions | IWebhookFunctions, service: string, method: string, path: string, body?: string | Buffer, query: IDataObject = {}, headers?: object, option: IDataObject = {}): Promise<any> { // tslint:disable-line:no-any export async function awsApiRequest(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions | IWebhookFunctions, service: string, method: string, path: string, body?: string | Buffer, query: IDataObject = {}, headers?: object, option: IDataObject = {}, region?: string): Promise<any> { // tslint:disable-line:no-any
const credentials = this.getCredentials('aws'); const credentials = this.getCredentials('aws');
if (credentials === undefined) { if (credentials === undefined) {
throw new Error('No credentials got returned!'); throw new Error('No credentials got returned!');
} }
const endpoint = `${service}.${credentials.region}.amazonaws.com`;
const endpoint = `${service}.${region || credentials.region}.amazonaws.com`;
// Sign AWS API request with the user credentials // Sign AWS API request with the user credentials
const signOpts = {headers: headers || {}, host: endpoint, method, path: `${path}?${queryToString(query).replace(/\+/g, '%2B')}`, body}; const signOpts = {headers: headers || {}, host: endpoint, method, path: `${path}?${queryToString(query).replace(/\+/g, '%2B')}`, body};
@ -44,6 +45,7 @@ export async function awsApiRequest(this: IHookFunctions | IExecuteFunctions | I
uri: `https://${endpoint}${signOpts.path}`, uri: `https://${endpoint}${signOpts.path}`,
body: signOpts.body, body: signOpts.body,
}; };
if (Object.keys(option).length !== 0) { if (Object.keys(option).length !== 0) {
Object.assign(options, option); Object.assign(options, option);
} }
@ -64,8 +66,8 @@ export async function awsApiRequest(this: IHookFunctions | IExecuteFunctions | I
} }
} }
export async function awsApiRequestREST(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions, service: string, method: string, path: string, body?: string, query: IDataObject = {}, headers?: object, options: IDataObject = {}): Promise<any> { // tslint:disable-line:no-any export async function awsApiRequestREST(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions, service: string, method: string, path: string, body?: string, query: IDataObject = {}, headers?: object, options: IDataObject = {}, region?: string): Promise<any> { // tslint:disable-line:no-any
const response = await awsApiRequest.call(this, service, method, path, body, query, headers, options); const response = await awsApiRequest.call(this, service, method, path, body, query, headers, options, region);
try { try {
return JSON.parse(response); return JSON.parse(response);
} catch (e) { } catch (e) {
@ -73,8 +75,8 @@ export async function awsApiRequestREST(this: IHookFunctions | IExecuteFunctions
} }
} }
export async function awsApiRequestSOAP(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions | IWebhookFunctions, service: string, method: string, path: string, body?: string | Buffer, query: IDataObject = {}, headers?: object): Promise<any> { // tslint:disable-line:no-any export async function awsApiRequestSOAP(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions | IWebhookFunctions, service: string, method: string, path: string, body?: string | Buffer, query: IDataObject = {}, headers?: object, option: IDataObject = {}, region?: string): Promise<any> { // tslint:disable-line:no-any
const response = await awsApiRequest.call(this, service, method, path, body, query, headers); const response = await awsApiRequest.call(this, service, method, path, body, query, headers, option, region);
try { try {
return await new Promise((resolve, reject) => { return await new Promise((resolve, reject) => {
parseString(response, { explicitArray: false }, (err, data) => { parseString(response, { explicitArray: false }, (err, data) => {
@ -89,14 +91,14 @@ export async function awsApiRequestSOAP(this: IHookFunctions | IExecuteFunctions
} }
} }
export async function awsApiRequestSOAPAllItems(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions | IWebhookFunctions, propertyName: string, service: string, method: string, path: string, body?: string, query: IDataObject = {}, headers: IDataObject = {}): Promise<any> { // tslint:disable-line:no-any export async function awsApiRequestSOAPAllItems(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions | IWebhookFunctions, propertyName: string, service: string, method: string, path: string, body?: string, query: IDataObject = {}, headers: IDataObject = {}, option: IDataObject = {}, region?: string): Promise<any> { // tslint:disable-line:no-any
const returnData: IDataObject[] = []; const returnData: IDataObject[] = [];
let responseData; let responseData;
do { do {
responseData = await awsApiRequestSOAP.call(this, service, method, path, body, query, headers); responseData = await awsApiRequestSOAP.call(this, service, method, path, body, query, headers, option, region);
//https://forums.aws.amazon.com/thread.jspa?threadID=55746 //https://forums.aws.amazon.com/thread.jspa?threadID=55746
if (get(responseData, `${propertyName.split('.')[0]}.NextContinuationToken`)) { if (get(responseData, `${propertyName.split('.')[0]}.NextContinuationToken`)) {