feat(core): Introduce AWS secrets manager as external secrets store (#8982)

packages/cli/package.json

@@ -61,6 +61,7 @@
   ],
   "devDependencies": {
     "@redocly/cli": "^1.6.0",
+    "@types/aws4": "^1.5.1",
     "@types/basic-auth": "^1.1.3",
     "@types/bcryptjs": "^2.4.2",
     "@types/compression": "1.0.1",
@@ -103,6 +104,7 @@
     "@rudderstack/rudder-sdk-node": "2.0.7",
     "@sentry/integrations": "7.87.0",
     "@sentry/node": "7.87.0",
+    "aws4": "1.11.0",
     "axios": "1.6.7",
     "basic-auth": "2.0.1",
     "bcryptjs": "2.4.3",

packages/cli/src/ExternalSecrets/ExternalSecretsManager.ee.ts

@@ -204,7 +204,7 @@ export class ExternalSecretsManager {
 		return Object.keys(this.providers);
 	}
 
-	getSecret(provider: string, name: string): IDataObject | undefined {
+	getSecret(provider: string, name: string) {
 		return this.getProvider(provider)?.getSecret(name);
 	}
 

packages/cli/src/ExternalSecrets/ExternalSecretsProviders.ee.ts

@@ -2,10 +2,12 @@ import type { SecretsProvider } from '@/Interfaces';
 import { Service } from 'typedi';
 import { InfisicalProvider } from './providers/infisical';
 import { VaultProvider } from './providers/vault';
+import { AwsSecretsManager } from './providers/aws-secrets/aws-secrets-manager';
 
 @Service()
 export class ExternalSecretsProviders {
 	providers: Record<string, { new (): SecretsProvider }> = {
+		awsSecretsManager: AwsSecretsManager,
 		infisical: InfisicalProvider,
 		vault: VaultProvider,
 	};

packages/cli/src/ExternalSecrets/providers/aws-secrets/aws-secrets-client.ts

@@ -0,0 +1,151 @@
+import axios from 'axios';
+import * as aws4 from 'aws4';
+import type { AxiosRequestConfig } from 'axios';
+import type { Request as Aws4Options } from 'aws4';
+import type {
+	AwsSecretsManagerContext,
+	ConnectionTestResult,
+	Secret,
+	SecretsNamesPage,
+	SecretsPage,
+	AwsSecretsClientSettings,
+} from './types';
+
+export class AwsSecretsClient {
+	private settings: AwsSecretsClientSettings = {
+		region: '',
+		host: '',
+		url: '',
+		accessKeyId: '',
+		secretAccessKey: '',
+	};
+
+	constructor(settings: AwsSecretsManagerContext['settings']) {
+		const { region, accessKeyId, secretAccessKey } = settings;
+
+		this.settings = {
+			region,
+			host: `secretsmanager.${region}.amazonaws.com`,
+			url: `https://secretsmanager.${region}.amazonaws.com`,
+			accessKeyId,
+			secretAccessKey,
+		};
+	}
+
+	/**
+	 * Check whether the client can connect to AWS Secrets Manager.
+	 */
+	async checkConnection(): ConnectionTestResult {
+		try {
+			await this.fetchSecretsNamesPage();
+			return [true];
+		} catch (e) {
+			const error = e instanceof Error ? e : new Error(`${e}`);
+			return [false, error.message];
+		}
+	}
+
+	/**
+	 * Fetch all secrets from AWS Secrets Manager.
+	 */
+	async fetchAllSecrets() {
+		const secrets: Secret[] = [];
+
+		const allSecretsNames = await this.fetchAllSecretsNames();
+
+		const batches = this.batch(allSecretsNames);
+
+		for (const batch of batches) {
+			const page = await this.fetchSecretsPage(batch);
+
+			secrets.push(
+				...page.SecretValues.map((s) => ({ secretName: s.Name, secretValue: s.SecretString })),
+			);
+		}
+
+		return secrets;
+	}
+
+	private batch<T>(arr: T[], size = 20): T[][] {
+		return Array.from({ length: Math.ceil(arr.length / size) }, (_, index) =>
+			arr.slice(index * size, (index + 1) * size),
+		);
+	}
+
+	private toRequestOptions(
+		action: 'ListSecrets' | 'BatchGetSecretValue',
+		body: string,
+	): Aws4Options {
+		return {
+			method: 'POST',
+			service: 'secretsmanager',
+			region: this.settings.region,
+			host: this.settings.host,
+			headers: {
+				'X-Amz-Target': `secretsmanager.${action}`,
+				'Content-Type': 'application/x-amz-json-1.1',
+			},
+			body,
+		};
+	}
+
+	/**
+	 * @doc https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_BatchGetSecretValue.html
+	 */
+	private async fetchSecretsPage(secretsNames: string[], nextToken?: string) {
+		const body = JSON.stringify(
+			nextToken
+				? { SecretIdList: secretsNames, NextToken: nextToken }
+				: { SecretIdList: secretsNames },
+		);
+
+		const options = this.toRequestOptions('BatchGetSecretValue', body);
+		const { headers } = aws4.sign(options, this.settings);
+
+		const config: AxiosRequestConfig = {
+			method: 'POST',
+			url: this.settings.url,
+			headers,
+			data: body,
+		};
+
+		const response = await axios.request<SecretsPage>(config);
+
+		return response.data;
+	}
+
+	private async fetchAllSecretsNames() {
+		const names: string[] = [];
+
+		let nextToken: string | undefined;
+
+		do {
+			const page = await this.fetchSecretsNamesPage(nextToken);
+			names.push(...page.SecretList.map((s) => s.Name));
+			nextToken = page.NextToken;
+		} while (nextToken);
+
+		return names;
+	}
+
+	/**
+	 * @doc https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_ListSecrets.html
+	 */
+	private async fetchSecretsNamesPage(nextToken?: string) {
+		const body = JSON.stringify(nextToken ? { NextToken: nextToken } : {});
+
+		const options = this.toRequestOptions('ListSecrets', body);
+		const { headers } = aws4.sign(options, this.settings);
+
+		const config: AxiosRequestConfig = {
+			method: 'POST',
+			url: this.settings.url,
+			headers,
+			data: body,
+		};
+
+		const response = await axios.request<SecretsNamesPage>(config);
+
+		return response.data;
+	}
+}

packages/cli/src/ExternalSecrets/providers/aws-secrets/aws-secrets-manager.ts

@@ -0,0 +1,131 @@
+import { AwsSecretsClient } from './aws-secrets-client';
+import { UnknownAuthTypeError } from '@/errors/unknown-auth-type.error';
+import { EXTERNAL_SECRETS_NAME_REGEX } from '@/ExternalSecrets/constants';
+import type { SecretsProvider, SecretsProviderState } from '@/Interfaces';
+import type { INodeProperties } from 'n8n-workflow';
+import type { AwsSecretsManagerContext } from './types';
+
+export class AwsSecretsManager implements SecretsProvider {
+	name = 'awsSecretsManager';
+
+	displayName = 'AWS Secrets Manager';
+
+	state: SecretsProviderState = 'initializing';
+
+	properties: INodeProperties[] = [
+		{
+			displayName:
+				'Need help filling out these fields? <a href="https://docs.n8n.io/external-secrets/#connect-n8n-to-your-secrets-store" target="_blank">Open docs</a>',
+			name: 'notice',
+			type: 'notice',
+			default: '',
+			noDataExpression: true,
+		},
+		{
+			displayName: 'Region',
+			name: 'region',
+			type: 'string',
+			default: '',
+			required: true,
+			placeholder: 'e.g. eu-west-3',
+			noDataExpression: true,
+		},
+		{
+			displayName: 'Authentication Method',
+			name: 'authMethod',
+			type: 'options',
+			options: [
+				{
+					name: 'IAM User',
+					value: 'iamUser',
+					description:
+						'Credentials for IAM user having <code>secretsmanager:ListSecrets</code> and <code>secretsmanager:BatchGetSecretValue</code> permissions. <a href="https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html" target="_blank">Learn more</a>',
+				},
+			],
+			default: 'iamUser',
+			required: true,
+			noDataExpression: true,
+		},
+		{
+			displayName: 'Access Key ID',
+			name: 'accessKeyId',
+			type: 'string',
+			default: '',
+			required: true,
+			placeholder: 'e.g. ACHXUQMBAQEVTE2RKMWP',
+			noDataExpression: true,
+			displayOptions: {
+				show: {
+					authMethod: ['iamUser'],
+				},
+			},
+		},
+		{
+			displayName: 'Secret Access Key',
+			name: 'secretAccessKey',
+			type: 'string',
+			default: '',
+			required: true,
+			placeholder: 'e.g. cbmjrH/xNAjPwlQR3i/1HRSDD+esQX/Lan3gcmBc',
+			typeOptions: { password: true },
+			noDataExpression: true,
+			displayOptions: {
+				show: {
+					authMethod: ['iamUser'],
+				},
+			},
+		},
+	];
+
+	private cachedSecrets: Record<string, string> = {};
+
+	private client: AwsSecretsClient;
+
+	async init(context: AwsSecretsManagerContext) {
+		this.assertAuthType(context);
+
+		this.client = new AwsSecretsClient(context.settings);
+	}
+
+	async test() {
+		return await this.client.checkConnection();
+	}
+
+	async connect() {
+		const [wasSuccessful] = await this.test();
+
+		this.state = wasSuccessful ? 'connected' : 'error';
+	}
+
+	async disconnect() {
+		return;
+	}
+
+	async update() {
+		const secrets = await this.client.fetchAllSecrets();
+
+		const supportedSecrets = secrets.filter((s) => EXTERNAL_SECRETS_NAME_REGEX.test(s.secretName));
+
+		this.cachedSecrets = Object.fromEntries(
+			supportedSecrets.map((s) => [s.secretName, s.secretValue]),
+		);
+	}
+
+	getSecret(name: string) {
+		return this.cachedSecrets[name];
+	}
+
+	hasSecret(name: string) {
+		return name in this.cachedSecrets;
+	}
+
+	getSecretNames() {
+		return Object.keys(this.cachedSecrets);
+	}
+
+	private assertAuthType(context: AwsSecretsManagerContext) {
+		if (context.settings.authMethod === 'iamUser') return;
+
+		throw new UnknownAuthTypeError(context.settings.authMethod);
+	}
+}

packages/cli/src/ExternalSecrets/providers/aws-secrets/types.ts

@@ -0,0 +1,50 @@
+import type { SecretsProviderSettings } from '@/Interfaces';
+
+export type SecretsNamesPage = {
+	NextToken?: string;
+	SecretList: SecretName[];
+};
+
+export type SecretsPage = {
+	NextToken?: string;
+	SecretValues: SecretValue[];
+};
+
+type SecretName = {
+	ARN: string;
+	CreatedDate: number;
+	LastAccessedDate: number;
+	LastChangedDate: number;
+	Name: string;
+	Tags: string[];
+};
+
+type SecretValue = {
+	ARN: string;
+	CreatedDate: number;
+	Name: string;
+	SecretString: string;
+	VersionId: string;
+};
+
+export type Secret = {
+	secretName: string;
+	secretValue: string;
+};
+
+export type ConnectionTestResult = Promise<[boolean] | [boolean, string]>;
+
+export type AwsSecretsManagerContext = SecretsProviderSettings<{
+	region: string;
+	authMethod: 'iamUser';
+	accessKeyId: string;
+	secretAccessKey: string;
+}>;
+
+export type AwsSecretsClientSettings = {
+	region: string;
+	host: string;
+	url: string;
+	accessKeyId: string;
+	secretAccessKey: string;
+};

packages/cli/src/Interfaces.ts

@@ -669,7 +669,7 @@ export abstract class SecretsProvider {
 	abstract disconnect(): Promise<void>;
 	abstract update(): Promise<void>;
 	abstract test(): Promise<[boolean] | [boolean, string]>;
-	abstract getSecret(name: string): IDataObject | undefined;
+	abstract getSecret(name: string): unknown;
 	abstract hasSecret(name: string): boolean;
 	abstract getSecretNames(): string[];
 }

packages/cli/src/SecretsHelpers.ts

@@ -1,4 +1,4 @@
-import type { IDataObject, SecretsHelpersBase } from 'n8n-workflow';
+import type { SecretsHelpersBase } from 'n8n-workflow';
 import { Service } from 'typedi';
 import { ExternalSecretsManager } from './ExternalSecrets/ExternalSecretsManager.ee';
 
@@ -19,7 +19,7 @@ export class SecretsHelper implements SecretsHelpersBase {
 		}
 	}
 
-	getSecret(provider: string, name: string): IDataObject | undefined {
+	getSecret(provider: string, name: string) {
 		return this.service.getSecret(provider, name);
 	}
 

packages/cli/src/errors/unknown-auth-type.error.ts

@@ -0,0 +1,7 @@
+import { ApplicationError } from 'n8n-workflow';
+
+export class UnknownAuthTypeError extends ApplicationError {
+	constructor(authType: string) {
+		super('Unknown auth type', { extra: { authType } });
+	}
+}

packages/core/src/Secrets.ts

@@ -35,7 +35,7 @@ export function getSecretsProxy(additionalData: IWorkflowExecuteAdditionalData):
 					return new Proxy(
 						{},
 						{
-							get(target2, secretName): IDataObject | undefined {
+							get(target2, secretName) {
 								if (typeof secretName !== 'string') {
 									return;
 								}
@@ -47,7 +47,7 @@ export function getSecretsProxy(additionalData: IWorkflowExecuteAdditionalData):
 								}
 								const retValue = secretsHelpers.getSecret(providerName, secretName);
 								if (typeof retValue === 'object' && retValue !== null) {
-									return buildSecretsValueProxy(retValue) as IDataObject;
+									return buildSecretsValueProxy(retValue as IDataObject);
 								}
 								return retValue;
 							},

packages/editor-ui/src/assets/images/aws-secrets-manager.svg

@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg width="80px" height="80px" viewBox="0 0 80 80" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+    <!-- Generator: Sketch 64 (93537) - https://sketch.com -->
+    <title>Icon-Architecture/64/Arch_AWS-Secrets-Manager_64</title>
+    <desc>Created with Sketch.</desc>
+    <defs>
+        <linearGradient x1="0%" y1="100%" x2="100%" y2="0%" id="linearGradient-1">
+            <stop stop-color="#BD0816" offset="0%"></stop>
+            <stop stop-color="#FF5252" offset="100%"></stop>
+        </linearGradient>
+    </defs>
+    <g id="Icon-Architecture/64/Arch_AWS-Secrets-Manager_64" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
+        <g id="Icon-Architecture-BG/64/Security-Identity-Compliance" fill="url(#linearGradient-1)">
+            <rect id="Rectangle" x="0" y="0" width="80" height="80"></rect>
+        </g>
+        <path d="M38.76,43.36 C38.76,44.044 39.317,44.6 40,44.6 C40.684,44.6 41.24,44.044 41.24,43.36 C41.24,42.676 40.684,42.12 40,42.12 C39.317,42.12 38.76,42.676 38.76,43.36 L38.76,43.36 Z M36.76,43.36 C36.76,41.573 38.213,40.12 40,40.12 C41.787,40.12 43.24,41.573 43.24,43.36 C43.24,44.796 42.296,46.002 41,46.426 L41,49 L39,49 L39,46.426 C37.704,46.002 36.76,44.796 36.76,43.36 L36.76,43.36 Z M49,38 L31,38 L31,51 L49,51 L49,48 L46,48 L46,46 L49,46 L49,43 L46,43 L46,41 L49,41 L49,38 Z M34,36 L45.999,36 L46,31 C46.001,28.384 43.143,26.002 40.004,26 L40.001,26 C38.472,26 36.928,26.574 35.763,27.575 C34.643,28.537 34,29.786 34,31.001 L34,36 Z M48,31.001 L47.999,36 L50,36 C50.553,36 51,36.448 51,37 L51,52 C51,52.552 50.553,53 50,53 L30,53 C29.447,53 29,52.552 29,52 L29,37 C29,36.448 29.447,36 30,36 L32,36 L32,31 C32.001,29.202 32.897,27.401 34.459,26.058 C35.982,24.75 38.001,24 40.001,24 L40.004,24 C44.265,24.002 48.001,27.273 48,31.001 L48,31.001 Z M19.207,55.049 L20.828,53.877 C18.093,50.097 16.581,45.662 16.396,41 L19,41 L19,39 L16.399,39 C16.598,34.366 18.108,29.957 20.828,26.198 L19.207,25.025 C16.239,29.128 14.599,33.942 14.399,39 L12,39 L12,41 L14.396,41 C14.582,46.086 16.224,50.926 19.207,55.049 L19.207,55.049 Z M53.838,59.208 C50.069,61.936 45.648,63.446 41,63.639 L41,61 L39,61 L39,63.639 C34.352,63.447 29.93,61.937 26.159,59.208 L24.988,60.828 C29.1,63.805 33.928,65.445 39,65.639 L39,68 L41,68 L41,65.639 C46.072,65.445 50.898,63.805 55.01,60.828 L53.838,59.208 Z M26.159,20.866 C29.93,18.138 34.352,16.628 39,16.436 L39,19 L41,19 L41,16.436 C45.648,16.628 50.069,18.138 53.838,20.866 L55.01,19.246 C50.898,16.27 46.072,14.63 41,14.436 L41,12 L39,12 L39,14.436 C33.928,14.629 29.1,16.269 24.988,19.246 L26.159,20.866 Z M65.599,39 C65.399,33.942 63.759,29.128 60.79,25.025 L59.169,26.198 C61.89,29.957 63.4,34.366 63.599,39 L61,39 L61,41 L63.602,41 C63.416,45.662 61.905,50.097 59.169,53.877 L60.79,55.049 C63.774,50.926 65.415,46.086 65.602,41 L68,41 L68,39 L65.599,39 Z M56.386,25.064 L64.226,17.224 L62.812,15.81 L54.972,23.65 L56.386,25.064 Z M23.612,55.01 L15.772,62.85 L17.186,64.264 L25.026,56.424 L23.612,55.01 Z M28.666,27.253 L13.825,12.413 L12.411,13.827 L27.252,28.667 L28.666,27.253 Z M54.193,52.78 L67.586,66.173 L66.172,67.587 L52.779,54.194 L54.193,52.78 Z" id="AWS-Secrets-Manager_Icon_64_Squid" fill="#FFFFFF"></path>
+    </g>
+</svg>

packages/editor-ui/src/components/ExternalSecretsProviderCard.ee.vue

@@ -54,7 +54,7 @@ const canConnect = computed(() => {
 });
 
 const formattedDate = computed((provider: ExternalSecretsProvider) => {
-	return DateTime.fromISO(props.provider.connectedAt).toFormat('dd LLL yyyy');
+	return DateTime.fromISO(props.provider.connectedAt ?? new Date()).toFormat('dd LLL yyyy');
 });
 
 onMounted(() => {

packages/editor-ui/src/components/ExternalSecretsProviderImage.ee.vue

@@ -5,6 +5,7 @@ import { computed } from 'vue';
 import infisical from '../assets/images/infisical.webp';
 import doppler from '../assets/images/doppler.webp';
 import vault from '../assets/images/hashicorp.webp';
+import awsSecretsManager from '../assets/images/aws-secrets-manager.svg';
 
 const props = defineProps({
 	provider: {
@@ -19,6 +20,7 @@ const image = computed(
 			doppler,
 			infisical,
 			vault,
+			awsSecretsManager,
 		})[props.provider.name],
 );
 </script>

packages/workflow/src/Interfaces.ts

@@ -2587,7 +2587,7 @@ export interface SecretsHelpersBase {
 	update(): Promise<void>;
 	waitForInit(): Promise<void>;
 
-	getSecret(provider: string, name: string): IDataObject | undefined;
+	getSecret(provider: string, name: string): unknown;
 	hasSecret(provider: string, name: string): boolean;
 	hasProvider(provider: string): boolean;
 	listProviders(): string[];

pnpm-lock.yaml

@@ -468,6 +468,9 @@ importers:
       '@sentry/node':
         specifier: 7.87.0
         version: 7.87.0
+      aws4:
+        specifier: 1.11.0
+        version: 1.11.0
       axios:
         specifier: 1.6.7
         version: 1.6.7(debug@3.2.7)
@@ -706,6 +709,9 @@ importers:
       '@redocly/cli':
         specifier: ^1.6.0
         version: 1.6.0
+      '@types/aws4':
+        specifier: ^1.5.1
+        version: 1.11.2
       '@types/basic-auth':
         specifier: ^1.1.3
         version: 1.1.3
@@ -8979,7 +8985,7 @@ packages:
       ts-dedent: 2.2.0
       type-fest: 2.19.0
       vue: 3.4.21(typescript@5.4.2)
-      vue-component-type-helpers: 2.0.6
+      vue-component-type-helpers: 2.0.7
     transitivePeerDependencies:
       - encoding
       - supports-color
@@ -25647,8 +25653,8 @@ packages:
     resolution: {integrity: sha512-0vOfAtI67UjeO1G6UiX5Kd76CqaQ67wrRZiOe7UAb9Jm6GzlUr/fC7CV90XfwapJRjpCMaZFhv1V0ajWRmE9Dg==}
     dev: true
 
-  /vue-component-type-helpers@2.0.6:
+  /vue-component-type-helpers@2.0.7:
-    resolution: {integrity: sha512-qdGXCtoBrwqk1BT6r2+1Wcvl583ZVkuSZ3or7Y1O2w5AvWtlvvxwjGhmz5DdPJS9xqRdDlgTJ/38ehWnEi0tFA==}
+    resolution: {integrity: sha512-7e12Evdll7JcTIocojgnCgwocX4WzIYStGClBQ+QuWPinZo/vQolv2EMq4a3lg16TKfwWafLimG77bxb56UauA==}
     dev: true
 
   /vue-demi@0.14.5(vue@3.4.21):