mirror of
https://github.com/n8n-io/n8n.git
synced 2024-11-10 14:44:05 -08:00
Add support for custom S3 endpoint in AWS S3 node
This commit is contained in:
Denis Palashevskii
parent
823ae846bf
commit
4b22df31ad
|
|
@ -41,9 +41,9 @@ import {
|
|||
} from './FileDescription';
|
||||
|
||||
import {
|
||||
awsApiRequestREST,
|
||||
awsApiRequestSOAP,
|
||||
awsApiRequestSOAPAllItems,
|
||||
s3ApiRequestREST,
|
||||
s3ApiRequestSOAP,
|
||||
s3ApiRequestSOAPAllItems,
|
||||
} from './GenericFunctions';
|
||||
|
||||
export class AwsS3 implements INodeType {
|
||||
|
|
@ -65,9 +65,44 @@ export class AwsS3 implements INodeType {
|
|||
{
|
||||
name: 'aws',
|
||||
required: true,
|
||||
}
|
||||
displayOptions: {
|
||||
show: {
|
||||
endpoint: [
|
||||
'aws',
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'customS3Endpoint',
|
||||
required: true,
|
||||
displayOptions: {
|
||||
show: {
|
||||
endpoint: [
|
||||
'customS3Endpoint',
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
properties: [
|
||||
{
|
||||
displayName: 'Endpoint',
|
||||
name: 'endpoint',
|
||||
type: 'options',
|
||||
options: [
|
||||
{
|
||||
name: 'AWS',
|
||||
value: 'aws',
|
||||
},
|
||||
{
|
||||
name: 'Custom S3 endpoint',
|
||||
value: 'customS3Endpoint',
|
||||
},
|
||||
],
|
||||
default: 'aws',
|
||||
description: 'The endpoint of S3 compatible service.',
|
||||
},
|
||||
{
|
||||
displayName: 'Resource',
|
||||
name: 'resource',
|
||||
|
|
@ -113,7 +148,21 @@ export class AwsS3 implements INodeType {
|
|||
if (resource === 'bucket') {
|
||||
//https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateBucket.html
|
||||
if (operation === 'create') {
|
||||
const credentials = this.getCredentials('aws');
|
||||
|
||||
let credentials;
|
||||
|
||||
const endpointType = this.getNodeParameter('endpoint', 0);
|
||||
|
||||
try {
|
||||
if (endpointType === 'aws') {
|
||||
credentials = this.getCredentials('aws');
|
||||
} else {
|
||||
credentials = this.getCredentials('customS3Endpoint');
|
||||
}
|
||||
} catch (error) {
|
||||
throw new Error(error);
|
||||
}
|
||||
|
||||
const name = this.getNodeParameter('name', i) as string;
|
||||
const additionalFields = this.getNodeParameter('additionalFields', i) as IDataObject;
|
||||
if (additionalFields.acl) {
|
||||
|
|
@ -158,7 +207,7 @@ export class AwsS3 implements INodeType {
|
|||
const builder = new Builder();
|
||||
data = builder.buildObject(body);
|
||||
}
|
||||
responseData = await awsApiRequestSOAP.call(this, `${name}.s3`, 'PUT', '', data, qs, headers);
|
||||
responseData = await s3ApiRequestSOAP.call(this, `${name}`, 'PUT', '', data, qs, headers);
|
||||
|
||||
returnData.push({ success: true });
|
||||
}
|
||||
|
|
@ -166,10 +215,10 @@ export class AwsS3 implements INodeType {
|
|||
if (operation === 'getAll') {
|
||||
const returnAll = this.getNodeParameter('returnAll', 0) as boolean;
|
||||
if (returnAll) {
|
||||
responseData = await awsApiRequestSOAPAllItems.call(this, 'ListAllMyBucketsResult.Buckets.Bucket', 's3', 'GET', '');
|
||||
responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListAllMyBucketsResult.Buckets.Bucket', '', 'GET', '');
|
||||
} else {
|
||||
qs.limit = this.getNodeParameter('limit', 0) as number;
|
||||
responseData = await awsApiRequestSOAPAllItems.call(this, 'ListAllMyBucketsResult.Buckets.Bucket', 's3', 'GET', '', '', qs);
|
||||
responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListAllMyBucketsResult.Buckets.Bucket', '', 'GET', '', '', qs);
|
||||
responseData = responseData.slice(0, qs.limit);
|
||||
}
|
||||
returnData.push.apply(returnData, responseData);
|
||||
|
|
@ -208,15 +257,15 @@ export class AwsS3 implements INodeType {
|
|||
qs['list-type'] = 2;
|
||||
|
||||
|
||||
responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { location: '' });
|
||||
responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' });
|
||||
|
||||
const region = responseData.LocationConstraint._ as string;
|
||||
|
||||
if (returnAll) {
|
||||
responseData = await awsApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', `${bucketName}.s3`, 'GET', '', '', qs, {}, {}, region);
|
||||
responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '', '', qs, {}, {}, region);
|
||||
} else {
|
||||
qs['max-keys'] = this.getNodeParameter('limit', 0) as number;
|
||||
responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', qs, {}, {}, region);
|
||||
responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', qs, {}, {}, region);
|
||||
responseData = responseData.ListBucketResult.Contents;
|
||||
}
|
||||
if (Array.isArray(responseData)) {
|
||||
|
|
@ -243,11 +292,11 @@ export class AwsS3 implements INodeType {
|
|||
if (additionalFields.storageClass) {
|
||||
headers['x-amz-storage-class'] = (snakeCase(additionalFields.storageClass as string)).toUpperCase();
|
||||
}
|
||||
responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { location: '' });
|
||||
responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' });
|
||||
|
||||
const region = responseData.LocationConstraint._;
|
||||
|
||||
responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'PUT', path, '', qs, headers, {}, region);
|
||||
responseData = await s3ApiRequestSOAP.call(this, bucketName, 'PUT', path, '', qs, headers, {}, region);
|
||||
returnData.push({ success: true });
|
||||
}
|
||||
//https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html
|
||||
|
|
@ -255,16 +304,16 @@ export class AwsS3 implements INodeType {
|
|||
const bucketName = this.getNodeParameter('bucketName', i) as string;
|
||||
const folderKey = this.getNodeParameter('folderKey', i) as string;
|
||||
|
||||
responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { location: '' });
|
||||
responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' });
|
||||
|
||||
const region = responseData.LocationConstraint._;
|
||||
|
||||
responseData = await awsApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', `${bucketName}.s3`, 'GET', '/', '', { 'list-type': 2, prefix: folderKey }, {}, {}, region);
|
||||
responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '/', '', { 'list-type': 2, prefix: folderKey }, {}, {}, region);
|
||||
|
||||
// folder empty then just delete it
|
||||
if (responseData.length === 0) {
|
||||
|
||||
responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'DELETE', `/${folderKey}`, '', qs, {}, {}, region);
|
||||
responseData = await s3ApiRequestSOAP.call(this, bucketName, 'DELETE', `/${folderKey}`, '', qs, {}, {}, region);
|
||||
|
||||
responseData = { deleted: [ { 'Key': folderKey } ] };
|
||||
|
||||
|
|
@ -293,7 +342,7 @@ export class AwsS3 implements INodeType {
|
|||
|
||||
headers['Content-Type'] = 'application/xml';
|
||||
|
||||
responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'POST', '/', data, { delete: '' } , headers, {}, region);
|
||||
responseData = await s3ApiRequestSOAP.call(this, bucketName, 'POST', '/', data, { delete: '' } , headers, {}, region);
|
||||
|
||||
responseData = { deleted: responseData.DeleteResult.Deleted };
|
||||
}
|
||||
|
|
@ -315,15 +364,15 @@ export class AwsS3 implements INodeType {
|
|||
|
||||
qs['list-type'] = 2;
|
||||
|
||||
responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { location: '' });
|
||||
responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' });
|
||||
|
||||
const region = responseData.LocationConstraint._;
|
||||
|
||||
if (returnAll) {
|
||||
responseData = await awsApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', `${bucketName}.s3`, 'GET', '', '', qs, {}, {}, region);
|
||||
responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '', '', qs, {}, {}, region);
|
||||
} else {
|
||||
qs.limit = this.getNodeParameter('limit', 0) as number;
|
||||
responseData = await awsApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', `${bucketName}.s3`, 'GET', '', '', qs, {}, {}, region);
|
||||
responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '', '', qs, {}, {}, region);
|
||||
}
|
||||
if (Array.isArray(responseData)) {
|
||||
responseData = responseData.filter((e: IDataObject) => (e.Key as string).endsWith('/') && e.Size === '0' && e.Key !== options.folderKey);
|
||||
|
|
@ -404,11 +453,11 @@ export class AwsS3 implements INodeType {
|
|||
|
||||
const destination = `/${destinationParts.slice(2, destinationParts.length).join('/')}`;
|
||||
|
||||
responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { location: '' });
|
||||
responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' });
|
||||
|
||||
const region = responseData.LocationConstraint._;
|
||||
|
||||
responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'PUT', destination, '', qs, headers, {}, region);
|
||||
responseData = await s3ApiRequestSOAP.call(this, bucketName, 'PUT', destination, '', qs, headers, {}, region);
|
||||
returnData.push(responseData.CopyObjectResult);
|
||||
|
||||
}
|
||||
|
|
@ -425,11 +474,11 @@ export class AwsS3 implements INodeType {
|
|||
throw new Error('Downloding a whole directory is not yet supported, please provide a file key');
|
||||
}
|
||||
|
||||
let region = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { location: '' });
|
||||
let region = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' });
|
||||
|
||||
region = region.LocationConstraint._;
|
||||
|
||||
const response = await awsApiRequestREST.call(this, `${bucketName}.s3`, 'GET', `/${fileKey}`, '', qs, {}, { encoding: null, resolveWithFullResponse: true }, region);
|
||||
const response = await s3ApiRequestREST.call(this, bucketName, 'GET', `/${fileKey}`, '', qs, {}, { encoding: null, resolveWithFullResponse: true }, region);
|
||||
|
||||
let mimeType: string | undefined;
|
||||
if (response.headers['content-type']) {
|
||||
|
|
@ -468,11 +517,11 @@ export class AwsS3 implements INodeType {
|
|||
qs.versionId = options.versionId as string;
|
||||
}
|
||||
|
||||
responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { location: '' });
|
||||
responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' });
|
||||
|
||||
const region = responseData.LocationConstraint._;
|
||||
|
||||
responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'DELETE', `/${fileKey}`, '', qs, {}, {}, region);
|
||||
responseData = await s3ApiRequestSOAP.call(this, bucketName, 'DELETE', `/${fileKey}`, '', qs, {}, {}, region);
|
||||
|
||||
returnData.push({ success: true });
|
||||
}
|
||||
|
|
@ -494,15 +543,15 @@ export class AwsS3 implements INodeType {
|
|||
|
||||
qs['list-type'] = 2;
|
||||
|
||||
responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { location: '' });
|
||||
responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' });
|
||||
|
||||
const region = responseData.LocationConstraint._;
|
||||
|
||||
if (returnAll) {
|
||||
responseData = await awsApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', `${bucketName}.s3`, 'GET', '', '', qs, {}, {}, region);
|
||||
responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '', '', qs, {}, {}, region);
|
||||
} else {
|
||||
qs.limit = this.getNodeParameter('limit', 0) as number;
|
||||
responseData = await awsApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', `${bucketName}.s3`, 'GET', '', '', qs, {}, {}, region);
|
||||
responseData = await s3ApiRequestSOAPAllItems.call(this, 'ListBucketResult.Contents', bucketName, 'GET', '', '', qs, {}, {}, region);
|
||||
responseData = responseData.splice(0, qs.limit);
|
||||
}
|
||||
if (Array.isArray(responseData)) {
|
||||
|
|
@ -581,7 +630,7 @@ export class AwsS3 implements INodeType {
|
|||
headers['x-amz-tagging'] = tags.join('&');
|
||||
}
|
||||
|
||||
responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'GET', '', '', { location: '' });
|
||||
responseData = await s3ApiRequestSOAP.call(this, bucketName, 'GET', '', '', { location: '' });
|
||||
|
||||
const region = responseData.LocationConstraint._;
|
||||
|
||||
|
|
@ -604,7 +653,7 @@ export class AwsS3 implements INodeType {
|
|||
|
||||
headers['Content-MD5'] = createHash('md5').update(body).digest('base64');
|
||||
|
||||
responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'PUT', `${path}${fileName || binaryData.fileName}`, body, qs, headers, {}, region);
|
||||
responseData = await s3ApiRequestSOAP.call(this, bucketName, 'PUT', `${path}${fileName || binaryData.fileName}`, body, qs, headers, {}, region);
|
||||
|
||||
} else {
|
||||
|
||||
|
|
@ -616,7 +665,7 @@ export class AwsS3 implements INodeType {
|
|||
|
||||
headers['Content-MD5'] = createHash('md5').update(fileContent).digest('base64');
|
||||
|
||||
responseData = await awsApiRequestSOAP.call(this, `${bucketName}.s3`, 'PUT', `${path}${fileName}`, body, qs, headers, {}, region);
|
||||
responseData = await s3ApiRequestSOAP.call(this, bucketName, 'PUT', `${path}${fileName}`, body, qs, headers, {}, region);
|
||||
}
|
||||
returnData.push({ success: true });
|
||||
}
|
||||
|
|
|
|||
|
|
@ -25,16 +25,47 @@ import {
|
|||
IDataObject,
|
||||
} from 'n8n-workflow';
|
||||
|
||||
export async function awsApiRequest(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions | IWebhookFunctions, service: string, method: string, path: string, body?: string | Buffer, query: IDataObject = {}, headers?: object, option: IDataObject = {}, region?: string): Promise<any> { // tslint:disable-line:no-any
|
||||
const credentials = this.getCredentials('aws');
|
||||
export async function s3ApiRequest(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions | IWebhookFunctions, bucket: string, method: string, path: string, body?: string | Buffer, query: IDataObject = {}, headers?: object, option: IDataObject = {}, region?: string): Promise<any> { // tslint:disable-line:no-any
|
||||
|
||||
let credentials;
|
||||
|
||||
const endpointType = this.getNodeParameter('endpoint', 0);
|
||||
|
||||
try {
|
||||
if (endpointType === 'aws') {
|
||||
credentials = this.getCredentials('aws');
|
||||
} else {
|
||||
credentials = this.getCredentials('customS3Endpoint');
|
||||
}
|
||||
} catch (error) {
|
||||
throw new Error(error);
|
||||
}
|
||||
|
||||
if (credentials === undefined) {
|
||||
throw new Error('No credentials got returned!');
|
||||
}
|
||||
|
||||
const endpoint = `${service}.${region || credentials.region}.amazonaws.com`;
|
||||
let endpoint = endpointType === 'aws' ? `${bucket}.s3.${region || credentials.region}.amazonaws.com` : credentials.endpoint;
|
||||
|
||||
// Using path-style for non-AWS services
|
||||
if (bucket && endpointType === 'customS3Endpoint') {
|
||||
if (credentials.forcePathStyle) {
|
||||
path = `/${bucket}${path}`;
|
||||
} else {
|
||||
endpoint = `${bucket}.${endpoint}`;
|
||||
}
|
||||
}
|
||||
|
||||
// Sign AWS API request with the user credentials
|
||||
const signOpts = {headers: headers || {}, host: endpoint, method, path: `${path}?${queryToString(query).replace(/\+/g, '%2B')}`, body};
|
||||
const signOpts = {
|
||||
headers: headers || {},
|
||||
region: region || credentials.region,
|
||||
host: endpoint,
|
||||
method,
|
||||
path: `${path}?${queryToString(query).replace(/\+/g, '%2B')}`,
|
||||
service: 's3',
|
||||
body
|
||||
};
|
||||
|
||||
sign(signOpts, { accessKeyId: `${credentials.accessKeyId}`, secretAccessKey: `${credentials.secretAccessKey}`});
|
||||
|
||||
|
|
@ -42,7 +73,7 @@ export async function awsApiRequest(this: IHookFunctions | IExecuteFunctions | I
|
|||
headers: signOpts.headers,
|
||||
method,
|
||||
qs: query,
|
||||
uri: `https://${endpoint}${signOpts.path}`,
|
||||
uri: `https://${endpoint}${path}`,
|
||||
body: signOpts.body,
|
||||
};
|
||||
|
||||
|
|
@ -52,7 +83,7 @@ export async function awsApiRequest(this: IHookFunctions | IExecuteFunctions | I
|
|||
try {
|
||||
return await this.helpers.request!(options);
|
||||
} catch (error) {
|
||||
const errorMessage = error.response.body.message || error.response.body.Message || error.message;
|
||||
const errorMessage = error.response?.body.message || error.response?.body.Message || error.message;
|
||||
|
||||
if (error.statusCode === 403) {
|
||||
if (errorMessage === 'The security token included in the request is invalid.') {
|
||||
|
|
@ -66,8 +97,8 @@ export async function awsApiRequest(this: IHookFunctions | IExecuteFunctions | I
|
|||
}
|
||||
}
|
||||
|
||||
export async function awsApiRequestREST(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions, service: string, method: string, path: string, body?: string, query: IDataObject = {}, headers?: object, options: IDataObject = {}, region?: string): Promise<any> { // tslint:disable-line:no-any
|
||||
const response = await awsApiRequest.call(this, service, method, path, body, query, headers, options, region);
|
||||
export async function s3ApiRequestREST(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions, bucket: string, method: string, path: string, body?: string, query: IDataObject = {}, headers?: object, options: IDataObject = {}, region?: string): Promise<any> { // tslint:disable-line:no-any
|
||||
const response = await s3ApiRequest.call(this, bucket, method, path, body, query, headers, options, region);
|
||||
try {
|
||||
return JSON.parse(response);
|
||||
} catch (e) {
|
||||
|
|
@ -75,8 +106,8 @@ export async function awsApiRequestREST(this: IHookFunctions | IExecuteFunctions
|
|||
}
|
||||
}
|
||||
|
||||
export async function awsApiRequestSOAP(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions | IWebhookFunctions, service: string, method: string, path: string, body?: string | Buffer, query: IDataObject = {}, headers?: object, option: IDataObject = {}, region?: string): Promise<any> { // tslint:disable-line:no-any
|
||||
const response = await awsApiRequest.call(this, service, method, path, body, query, headers, option, region);
|
||||
export async function s3ApiRequestSOAP(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions | IWebhookFunctions, bucket: string, method: string, path: string, body?: string | Buffer, query: IDataObject = {}, headers?: object, option: IDataObject = {}, region?: string): Promise<any> { // tslint:disable-line:no-any
|
||||
const response = await s3ApiRequest.call(this, bucket, method, path, body, query, headers, option, region);
|
||||
try {
|
||||
return await new Promise((resolve, reject) => {
|
||||
parseString(response, { explicitArray: false }, (err, data) => {
|
||||
|
|
@ -91,14 +122,14 @@ export async function awsApiRequestSOAP(this: IHookFunctions | IExecuteFunctions
|
|||
}
|
||||
}
|
||||
|
||||
export async function awsApiRequestSOAPAllItems(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions | IWebhookFunctions, propertyName: string, service: string, method: string, path: string, body?: string, query: IDataObject = {}, headers: IDataObject = {}, option: IDataObject = {}, region?: string): Promise<any> { // tslint:disable-line:no-any
|
||||
export async function s3ApiRequestSOAPAllItems(this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions | IWebhookFunctions, propertyName: string, service: string, method: string, path: string, body?: string, query: IDataObject = {}, headers: IDataObject = {}, option: IDataObject = {}, region?: string): Promise<any> { // tslint:disable-line:no-any
|
||||
|
||||
const returnData: IDataObject[] = [];
|
||||
|
||||
let responseData;
|
||||
|
||||
do {
|
||||
responseData = await awsApiRequestSOAP.call(this, service, method, path, body, query, headers, option, region);
|
||||
responseData = await s3ApiRequestSOAP.call(this, service, method, path, body, query, headers, option, region);
|
||||
|
||||
//https://forums.aws.amazon.com/thread.jspa?threadID=55746
|
||||
if (get(responseData, `${propertyName.split('.')[0]}.NextContinuationToken`)) {
|
||||
|
|
|
|||
Loading…
Reference in a new issue