N8N-4134 Add AWS cred testing and http custom calls with credentials (#3924)

* ✨ Add Aws testing and http custom api

packages/core/src/NodeExecuteFunctions.ts

@@ -1423,7 +1423,6 @@ export async function requestWithAuthentication(
 			node,
 			additionalData.timezone,
 		);
-
 		return await proxyRequestToAxios(requestOptions as IDataObject);
 	} catch (error) {
 		try {

packages/nodes-base/credentials/Aws.credentials.ts

@@ -1,4 +1,12 @@
-import { ICredentialType, INodeProperties } from 'n8n-workflow';
+import { Request, sign } from 'aws4';
+import { ICredentialTestRequest, IHttpRequestMethods } from 'n8n-workflow';
+import {
+	ICredentialDataDecryptedObject,
+	ICredentialType,
+	IDataObject,
+	IHttpRequestOptions,
+	INodeProperties,
+} from 'n8n-workflow';
 
 export const regions = [
 	{
@@ -259,4 +267,98 @@ export class Aws implements ICredentialType {
 			placeholder: 'https://s3.{region}.amazonaws.com',
 		},
 	];
+
+	async authenticate(
+		credentials: ICredentialDataDecryptedObject,
+		requestOptions: IHttpRequestOptions,
+	): Promise<IHttpRequestOptions> {
+		let endpoint;
+		let service = requestOptions.qs?.service;
+		let path = requestOptions.qs?.path;
+		const method = requestOptions.method;
+		const body = requestOptions.body;
+		let region = credentials.region;
+		const query = requestOptions.qs?.query as IDataObject;
+		if (!requestOptions.baseURL && !requestOptions.url) {
+			if (service === 'lambda' && credentials.lambdaEndpoint) {
+				endpoint = credentials.lambdaEndpoint;
+			} else if (service === 'sns' && credentials.snsEndpoint) {
+				endpoint = credentials.snsEndpoint;
+			} else if (service === 'sqs' && credentials.sqsEndpoint) {
+				endpoint = credentials.sqsEndpoint;
+			} else if (service === 's3' && credentials.s3Endpoint) {
+				endpoint = credentials.s3Endpoint;
+			} else if (service === 'ses' && credentials.sesEndpoint) {
+				endpoint = credentials.sesEndpoint;
+			} else if (service === 'rekognition' && credentials.rekognitionEndpoint) {
+				endpoint = credentials.rekognitionEndpoint;
+			} else if (service === 'sqs' && credentials.sqsEndpoint) {
+				endpoint = credentials.sqsEndpoint;
+			} else if (service) {
+				endpoint = `https://${service}.${credentials.region}.amazonaws.com`;
+			}
+			endpoint = new URL((endpoint as string).replace('{region}', credentials.region as string));
+		} else {
+			// If no endpoint is set, we try to decompose the path and use the default endpoint
+			const customUrl = new URL(requestOptions.baseURL! + requestOptions.url!);
+			service = customUrl.hostname.split('.')[0] as string;
+			region = customUrl.hostname.split('.')[1] as string;
+			if (service === 'sts') {
+				try {
+					customUrl.searchParams.set('Action', 'GetCallerIdentity');
+					customUrl.searchParams.append('Version', '2011-06-15');
+				} catch (err) {
+					console.log(err);
+				}
+			}
+			path = customUrl.pathname as string;
+			endpoint = customUrl;
+		}
+		if (service === 's3' && credentials.s3Endpoint) {
+			path = `${endpoint.pathname}?${queryToString(query).replace(/\+/g, '%2B')}`;
+		}
+		const signOpts = {
+			headers: requestOptions.headers,
+			host: endpoint.host,
+			method,
+			path,
+			body,
+			region,
+		} as Request;
+
+		const securityHeaders = {
+			accessKeyId: `${credentials.accessKeyId}`.trim(),
+			secretAccessKey: `${credentials.secretAccessKey}`.trim(),
+			sessionToken: credentials.temporaryCredentials
+				? `${credentials.sessionToken}`.trim()
+				: undefined,
+		};
+		try {
+			sign(signOpts, securityHeaders);
+		} catch (err) {
+			console.log(err);
+		}
+		const options: IHttpRequestOptions = {
+			headers: signOpts.headers,
+			method,
+			url: endpoint.origin + path,
+			body: signOpts.body,
+		};
+
+		return options;
+	}
+
+	test: ICredentialTestRequest = {
+		request: {
+			baseURL: '=https://sts.{{$credentials.region}}.amazonaws.com',
+			url: '?Action=GetCallerIdentity&Version=2011-06-15',
+			method: 'POST',
+		},
+	};
+}
+
+function queryToString(params: IDataObject) {
+	return Object.keys(params)
+		.map((key) => key + '=' + params[key])
+		.join('&');
 }

packages/nodes-base/nodes/Aws/Comprehend/GenericFunctions.ts

@@ -13,22 +13,7 @@ import {
 	IWebhookFunctions,
 } from 'n8n-core';
 
-import { ICredentialDataDecryptedObject, NodeApiError, NodeOperationError } from 'n8n-workflow';
+import { ICredentialDataDecryptedObject, IHttpRequestOptions, NodeApiError, NodeOperationError } from 'n8n-workflow';
-
-function getEndpointForService(
-	service: string,
-	credentials: ICredentialDataDecryptedObject,
-): string {
-	let endpoint;
-	if (service === 'lambda' && credentials.lambdaEndpoint) {
-		endpoint = credentials.lambdaEndpoint;
-	} else if (service === 'sns' && credentials.snsEndpoint) {
-		endpoint = credentials.snsEndpoint;
-	} else {
-		endpoint = `https://${service}.${credentials.region}.amazonaws.com`;
-	}
-	return (endpoint as string).replace('{region}', credentials.region as string);
-}
 
 export async function awsApiRequest(
 	this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions | IWebhookFunctions,
@@ -41,30 +26,19 @@ export async function awsApiRequest(
 ): Promise<any> {
 	const credentials = await this.getCredentials('aws');
 
-	// Concatenate path and instantiate URL object so it parses correctly query strings
+	const requestOptions = {
-	const endpoint = new URL(getEndpointForService(service, credentials) + path);
+		qs: {
-
+			service,
-	// Sign AWS API request with the user credentials
+			path,
-	const signOpts = { headers: headers || {}, host: endpoint.host, method, path, body } as Request;
+		},
-	const securityHeaders = {
-		accessKeyId: `${credentials.accessKeyId}`.trim(),
-		secretAccessKey: `${credentials.secretAccessKey}`.trim(),
-		sessionToken: credentials.temporaryCredentials
-			? `${credentials.sessionToken}`.trim()
-			: undefined,
-	};
-
-	sign(signOpts, securityHeaders);
-
-	const options: OptionsWithUri = {
-		headers: signOpts.headers,
 		method,
-		uri: endpoint.href,
+		body,
-		body: signOpts.body,
+		url: '',
-	};
+		headers,
-
+		region: credentials?.region as string,
+	} as IHttpRequestOptions;
 	try {
-		return await this.helpers.request!(options);
+		return await this.helpers.requestWithAuthentication.call(this, 'aws', requestOptions);
 	} catch (error) {
 		throw new NodeApiError(this.getNode(), error); // no XML parsing needed
 	}

packages/nodes-base/nodes/Aws/DynamoDB/GenericFunctions.ts

@@ -1,7 +1,3 @@
-import { URL } from 'url';
-
-import { sign } from 'aws4';
-
 import {
 	IExecuteFunctions,
 	IHookFunctions,
@@ -9,25 +5,15 @@ import {
 	IWebhookFunctions,
 } from 'n8n-core';
 
-import { ICredentialDataDecryptedObject, IDataObject, INodeExecutionData } from 'n8n-workflow';
+import {
+	ICredentialDataDecryptedObject,
+	IDataObject,
+	IHttpRequestOptions,
+	INodeExecutionData,
+} from 'n8n-workflow';
 
 import { IRequestBody } from './types';
 
-function getEndpointForService(
-	service: string,
-	credentials: ICredentialDataDecryptedObject,
-): string {
-	let endpoint;
-	if (service === 'lambda' && credentials.lambdaEndpoint) {
-		endpoint = credentials.lambdaEndpoint;
-	} else if (service === 'sns' && credentials.snsEndpoint) {
-		endpoint = credentials.snsEndpoint;
-	} else {
-		endpoint = `https://${service}.${credentials.region}.amazonaws.com`;
-	}
-	return (endpoint as string).replace('{region}', credentials.region as string);
-}
-
 export async function awsApiRequest(
 	this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions | IWebhookFunctions,
 	service: string,
@@ -38,32 +24,22 @@ export async function awsApiRequest(
 	// tslint:disable-next-line:no-any
 ): Promise<any> {
 	const credentials = await this.getCredentials('aws');
-
+	const requestOptions = {
-	// Concatenate path and instantiate URL object so it parses correctly query strings
+		qs: {
-	const endpoint = new URL(getEndpointForService(service, credentials) + path);
-	const securityHeaders = {
-		accessKeyId: `${credentials.accessKeyId}`.trim(),
-		secretAccessKey: `${credentials.secretAccessKey}`.trim(),
-		sessionToken: credentials.temporaryCredentials
-			? `${credentials.sessionToken}`.trim()
-			: undefined,
-	};
-	const options = sign(
-		{
-			// @ts-ignore
-			uri: endpoint,
 			service,
-			region: credentials.region as string,
+			path,
-			method,
-			path: '/',
-			headers: { ...headers },
-			body: JSON.stringify(body),
 		},
-		securityHeaders,
+		method,
-	);
+		body: JSON.stringify(body),
+		url: '',
+		headers,
+		region: credentials?.region as string,
+	} as IHttpRequestOptions;
 
 	try {
-		return JSON.parse(await this.helpers.request!(options));
+		return JSON.parse(
+			await this.helpers.requestWithAuthentication.call(this, 'aws', requestOptions),
+		);
 	} catch (error) {
 		const errorMessage =
 			(error.response && error.response.body && error.response.body.message) ||

packages/nodes-base/nodes/Aws/GenericFunctions.ts

@@ -10,24 +10,7 @@ import {
 	IWebhookFunctions,
 } from 'n8n-core';
 
-import { ICredentialDataDecryptedObject, NodeApiError, NodeOperationError } from 'n8n-workflow';
+import { ICredentialDataDecryptedObject, IHttpRequestOptions, NodeApiError, NodeOperationError } from 'n8n-workflow';
-
-function getEndpointForService(
-	service: string,
-	credentials: ICredentialDataDecryptedObject,
-): string {
-	let endpoint;
-	if (service === 'lambda' && credentials.lambdaEndpoint) {
-		endpoint = credentials.lambdaEndpoint;
-	} else if (service === 'sns' && credentials.snsEndpoint) {
-		endpoint = credentials.snsEndpoint;
-	} else if (service === 'sqs' && credentials.sqsEndpoint) {
-		endpoint = credentials.sqsEndpoint;
-	} else {
-		endpoint = `https://${service}.${credentials.region}.amazonaws.com`;
-	}
-	return (endpoint as string).replace('{region}', credentials.region as string);
-}
 
 export async function awsApiRequest(
 	this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions | IWebhookFunctions,
@@ -39,31 +22,20 @@ export async function awsApiRequest(
 	// tslint:disable-next-line:no-any
 ): Promise<any> {
 	const credentials = await this.getCredentials('aws');
-
+	const requestOptions = {
-	// Concatenate path and instantiate URL object so it parses correctly query strings
+		qs: {
-	const endpoint = new URL(getEndpointForService(service, credentials) + path);
+			service,
-
+			path,
-	// Sign AWS API request with the user credentials
+		},
-	const signOpts = { headers: headers || {}, host: endpoint.host, method, path, body } as Request;
-	const securityHeaders = {
-		accessKeyId: `${credentials.accessKeyId}`.trim(),
-		secretAccessKey: `${credentials.secretAccessKey}`.trim(),
-		sessionToken: credentials.temporaryCredentials
-			? `${credentials.sessionToken}`.trim()
-			: undefined,
-	};
-
-	sign(signOpts, securityHeaders);
-
-	const options: OptionsWithUri = {
-		headers: signOpts.headers,
 		method,
-		uri: endpoint.href,
+		body: JSON.stringify(body),
-		body: signOpts.body,
+		url: '',
-	};
+		headers,
+		region: credentials?.region as string,
+	} as IHttpRequestOptions;
 
 	try {
-		return await this.helpers.request!(options);
+		return await this.helpers.requestWithAuthentication.call(this,'aws', requestOptions);
 	} catch (error) {
 		throw new NodeApiError(this.getNode(), error, { parseXml: true });
 	}

packages/nodes-base/nodes/Aws/Rekognition/GenericFunctions.ts

@@ -15,7 +15,7 @@ import {
 	IWebhookFunctions,
 } from 'n8n-core';
 
-import { IDataObject, NodeApiError, NodeOperationError } from 'n8n-workflow';
+import { IDataObject, IHttpRequestOptions, NodeApiError, NodeOperationError } from 'n8n-workflow';
 
 import { pascalCase } from 'change-case';
 
@@ -33,37 +33,23 @@ export async function awsApiRequest(
 ): Promise<any> {
 	const credentials = await this.getCredentials('aws');
 
-	const endpoint = new URL(
+	const requestOptions = {
-		(((credentials.rekognitionEndpoint as string) || '').replace(
+		qs: {
-			'{region}',
+			service,
-			credentials.region as string,
+			path,
-		) || `https://${service}.${credentials.region}.amazonaws.com`) + path,
+		},
-	);
-
-	// Sign AWS API request with the user credentials
-	const signOpts = { headers: headers || {}, host: endpoint.host, method, path, body } as Request;
-	const securityHeaders = {
-		accessKeyId: `${credentials.accessKeyId}`.trim(),
-		secretAccessKey: `${credentials.secretAccessKey}`.trim(),
-		sessionToken: credentials.temporaryCredentials
-			? `${credentials.sessionToken}`.trim()
-			: undefined,
-	};
-
-	sign(signOpts, securityHeaders);
-
-	const options: OptionsWithUri = {
-		headers: signOpts.headers,
 		method,
-		uri: endpoint.href,
+		body,
-		body: signOpts.body,
+		url: '',
-	};
+		headers,
+		region: credentials?.region as string,
+	} as IHttpRequestOptions;
 
 	if (Object.keys(option).length !== 0) {
-		Object.assign(options, option);
+		Object.assign(requestOptions, option);
 	}
 	try {
-		return await this.helpers.request!(options);
+		return await this.helpers.requestWithAuthentication.call(this, 'aws', requestOptions);
 	} catch (error) {
 		throw new NodeApiError(this.getNode(), error);
 	}

packages/nodes-base/nodes/Aws/S3/GenericFunctions.ts

@@ -15,7 +15,7 @@ import {
 	IWebhookFunctions,
 } from 'n8n-core';
 
-import { IDataObject, JsonObject, NodeApiError, NodeOperationError } from 'n8n-workflow';
+import { IDataObject, IHttpRequestOptions, JsonObject, NodeApiError, NodeOperationError } from 'n8n-workflow';
 
 export async function awsApiRequest(
 	this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions | IWebhookFunctions,
@@ -30,43 +30,24 @@ export async function awsApiRequest(
 	// tslint:disable-next-line:no-any
 ): Promise<any> {
 	const credentials = await this.getCredentials('aws');
-
+	const requestOptions = {
-	const endpoint = new URL(
+		qs: {
-		(((credentials.s3Endpoint as string) || '').replace('{region}', credentials.region as string) ||
+			service,
-			`https://${service}.${credentials.region}.amazonaws.com`) + path,
+			path,
-	);
+			query,
-
+		},
-	// Sign AWS API request with the user credentials
-	const signOpts = {
-		headers: headers || {},
-		host: endpoint.host,
 		method,
-		path: `${endpoint.pathname}?${queryToString(query).replace(/\+/g, '%2B')}`,
+		body: JSON.stringify(body),
-		body,
+		url: '',
-	} as Request;
+		headers,
-	const securityHeaders = {
+		//region: credentials?.region as string,
-		accessKeyId: `${credentials.accessKeyId}`.trim(),
+	} as IHttpRequestOptions;
-		secretAccessKey: `${credentials.secretAccessKey}`.trim(),
-		sessionToken: credentials.temporaryCredentials
-			? `${credentials.sessionToken}`.trim()
-			: undefined,
-	};
-
-	sign(signOpts, securityHeaders);
-
-	const options: OptionsWithUri = {
-		headers: signOpts.headers,
-		method,
-		qs: query,
-		uri: endpoint.href,
-		body: signOpts.body,
-	};
 
 	if (Object.keys(option).length !== 0) {
-		Object.assign(options, option);
+		Object.assign(requestOptions, option);
 	}
 	try {
-		return await this.helpers.request!(options);
+		return await this.helpers.requestWithAuthentication.call(this, 'aws', requestOptions);
 	} catch (error) {
 		throw new NodeApiError(this.getNode(), error as JsonObject);
 	}

packages/nodes-base/nodes/Aws/SES/GenericFunctions.ts

@@ -13,7 +13,7 @@ import {
 	IWebhookFunctions,
 } from 'n8n-core';
 
-import { IDataObject, NodeApiError, NodeOperationError } from 'n8n-workflow';
+import { IDataObject, IHttpRequestOptions, NodeApiError, NodeOperationError } from 'n8n-workflow';
 
 import { get } from 'lodash';
 
@@ -28,35 +28,20 @@ export async function awsApiRequest(
 ): Promise<any> {
 	const credentials = await this.getCredentials('aws');
 
-	const endpoint = new URL(
+	const requestOptions = {
-		(((credentials.sesEndpoint as string) || '').replace(
+		qs: {
-			'{region}',
+			service,
-			credentials.region as string,
+			path,
-		) || `https://${service}.${credentials.region}.amazonaws.com`) + path,
+		},
-	);
-
-	// Sign AWS API request with the user credentials
-
-	const signOpts = { headers: headers || {}, host: endpoint.host, method, path, body } as Request;
-	const securityHeaders = {
-		accessKeyId: `${credentials.accessKeyId}`.trim(),
-		secretAccessKey: `${credentials.secretAccessKey}`.trim(),
-		sessionToken: credentials.temporaryCredentials
-			? `${credentials.sessionToken}`.trim()
-			: undefined,
-	};
-
-	sign(signOpts, securityHeaders);
-
-	const options: OptionsWithUri = {
-		headers: signOpts.headers,
 		method,
-		uri: endpoint.href,
+		body: JSON.stringify(body),
-		body: signOpts.body as string,
+		url: '',
-	};
+		headers,
+		region: credentials?.region as string,
+	} as IHttpRequestOptions;
 
 	try {
-		return await this.helpers.request!(options);
+		return await this.helpers.requestWithAuthentication.call(this,'aws', requestOptions);
 	} catch (error) {
 		throw new NodeApiError(this.getNode(), error, { parseXml: true });
 	}

packages/nodes-base/nodes/Aws/Textract/AwsTextract.node.ts

@@ -38,7 +38,6 @@ export class AwsTextract implements INodeType {
 			{
 				name: 'aws',
 				required: true,
-				testedBy: 'awsTextractApiCredentialTest',
 			},
 		],
 		properties: [

packages/nodes-base/nodes/Aws/Textract/GenericFunctions.ts

@@ -16,6 +16,7 @@ import {
 import {
 	ICredentialDataDecryptedObject,
 	ICredentialTestFunctions,
+	IHttpRequestOptions,
 	NodeApiError,
 	NodeOperationError,
 } from 'n8n-workflow';
@@ -46,30 +47,20 @@ export async function awsApiRequest(
 ): Promise<any> {
 	const credentials = await this.getCredentials('aws');
 
-	// Concatenate path and instantiate URL object so it parses correctly query strings
+	const requestOptions = {
-	const endpoint = new URL(getEndpointForService(service, credentials) + path);
+		qs: {
-
+			service,
-	// Sign AWS API request with the user credentials
+			path,
-	const signOpts = { headers: headers || {}, host: endpoint.host, method, path, body } as Request;
+		},
-	const securityHeaders = {
-		accessKeyId: `${credentials.accessKeyId}`.trim(),
-		secretAccessKey: `${credentials.secretAccessKey}`.trim(),
-		sessionToken: credentials.temporaryCredentials
-			? `${credentials.sessionToken}`.trim()
-			: undefined,
-	};
-
-	sign(signOpts, securityHeaders);
-
-	const options: OptionsWithUri = {
-		headers: signOpts.headers,
 		method,
-		uri: endpoint.href,
+		body,
-		body: signOpts.body,
+		url: '',
-	};
+		headers,
+		region: credentials?.region as string,
+	} as IHttpRequestOptions;
 
 	try {
-		return await this.helpers.request!(options);
+		return await this.helpers.requestWithAuthentication.call(this, 'aws', requestOptions);
 	} catch (error) {
 		if (error?.response?.data || error?.response?.body) {
 			const errorMessage = error?.response?.data || error?.response?.body;