refactor(core): Alllow using S3 compatible object stores over http (#12812)

packages/@n8n/config/src/configs/external-storage.config.ts

@@ -23,11 +23,14 @@ class S3CredentialsConfig {
 }
 
 @Config
-class S3Config {
+export class S3Config {
 	/** Host of the n8n bucket in S3-compatible external storage @example "s3.us-east-1.amazonaws.com" */
 	@Env('N8N_EXTERNAL_STORAGE_S3_HOST')
 	host: string = '';
 
+	@Env('N8N_EXTERNAL_STORAGE_S3_PROTOCOL')
+	protocol: 'http' | 'https' = 'https';
+
 	@Nested
 	bucket: S3BucketConfig;
 

packages/@n8n/config/src/index.ts

@@ -30,6 +30,7 @@ export { TaskRunnersConfig } from './configs/runners.config';
 export { SecurityConfig } from './configs/security.config';
 export { ExecutionsConfig } from './configs/executions.config';
 export { FrontendBetaFeatures, FrontendConfig } from './configs/frontend.config';
+export { S3Config } from './configs/external-storage.config';
 export { LOG_SCOPES } from './configs/logging.config';
 export type { LogScope } from './configs/logging.config';
 

packages/@n8n/config/test/config.test.ts

@@ -138,6 +138,7 @@ describe('GlobalConfig', () => {
 		externalStorage: {
 			s3: {
 				host: '',
+				protocol: 'https',
 				bucket: {
 					name: '',
 					region: '',

packages/cli/src/commands/base-command.ts

@@ -189,42 +189,10 @@ export abstract class BaseCommand extends Command {
 	private async _initObjectStoreService(options = { isReadOnly: false }) {
 		const objectStoreService = Container.get(ObjectStoreService);
 
-		const { host, bucket, credentials } = this.globalConfig.externalStorage.s3;
-
-		if (host === '') {
-			throw new ApplicationError(
-				'External storage host not configured. Please set `N8N_EXTERNAL_STORAGE_S3_HOST`.',
-			);
-		}
-
-		if (bucket.name === '') {
-			throw new ApplicationError(
-				'External storage bucket name not configured. Please set `N8N_EXTERNAL_STORAGE_S3_BUCKET_NAME`.',
-			);
-		}
-
-		if (bucket.region === '') {
-			throw new ApplicationError(
-				'External storage bucket region not configured. Please set `N8N_EXTERNAL_STORAGE_S3_BUCKET_REGION`.',
-			);
-		}
-
-		if (credentials.accessKey === '') {
-			throw new ApplicationError(
-				'External storage access key not configured. Please set `N8N_EXTERNAL_STORAGE_S3_ACCESS_KEY`.',
-			);
-		}
-
-		if (credentials.accessSecret === '') {
-			throw new ApplicationError(
-				'External storage access secret not configured. Please set `N8N_EXTERNAL_STORAGE_S3_ACCESS_SECRET`.',
-			);
-		}
-
 		this.logger.debug('Initializing object store service');
 
 		try {
-			await objectStoreService.init(host, bucket, credentials);
+			await objectStoreService.init();
 			objectStoreService.setReadonly(options.isReadOnly);
 
 			this.logger.debug('Object store init completed');

packages/core/src/binary-data/object-store/__tests__/object-store.service.test.ts

@@ -1,3 +1,4 @@
+import type { S3Config } from '@n8n/config';
 import axios from 'axios';
 import { mock } from 'jest-mock-extended';
 import { Readable } from 'stream';
@@ -18,6 +19,12 @@ const mockError = new Error('Something went wrong!');
 const fileId =
 	'workflows/ObogjVbqpNOQpiyV/executions/999/binary_data/71f6209b-5d48-41a2-a224-80d529d8bb32';
 const mockBuffer = Buffer.from('Test data');
+const s3Config = mock<S3Config>({
+	host: mockHost,
+	bucket: mockBucket,
+	credentials: mockCredentials,
+	protocol: 'https',
+});
 
 const toDeletionXml = (filename: string) => `<Delete>
 <Object><Key>${filename}</Key></Object>
@@ -25,10 +32,13 @@ const toDeletionXml = (filename: string) => `<Delete>
 
 let objectStoreService: ObjectStoreService;
 
+const now = new Date('2024-02-01T01:23:45.678Z');
+jest.useFakeTimers({ now });
+
 beforeEach(async () => {
-	objectStoreService = new ObjectStoreService(mock());
+	objectStoreService = new ObjectStoreService(mock(), s3Config);
 	mockAxios.request.mockResolvedValueOnce({ status: 200 }); // for checkConnection
-	await objectStoreService.init(mockHost, mockBucket, mockCredentials);
+	await objectStoreService.init();
 	jest.restoreAllMocks();
 });
 
@@ -40,17 +50,17 @@ describe('checkConnection()', () => {
 
 		await objectStoreService.checkConnection();
 
-		expect(mockAxios.request).toHaveBeenCalledWith(
+		expect(mockAxios.request).toHaveBeenCalledWith({
-			expect.objectContaining({
+			method: 'HEAD',
-				method: 'HEAD',
+			url: 'https://s3.us-east-1.amazonaws.com/test-bucket',
-				url: `https://${mockHost}/${mockBucket.name}`,
+			headers: {
-				headers: expect.objectContaining({
+				Host: 's3.us-east-1.amazonaws.com',
-					'X-Amz-Content-Sha256': expect.any(String),
+				'X-Amz-Content-Sha256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855',
-					'X-Amz-Date': expect.any(String),
+				'X-Amz-Date': '20240201T012345Z',
-					Authorization: expect.any(String),
+				Authorization:
-				}),
+					'AWS4-HMAC-SHA256 Credential=mock-access-key/20240201/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=a5240c11a706e9e6c60e7033a848fc934911b12330e5a4609b0b943f97d9781b',
-			}),
+			},
-		);
+		});
 	});
 
 	it('should throw an error on request failure', async () => {
@@ -70,18 +80,17 @@ describe('getMetadata()', () => {
 
 		await objectStoreService.getMetadata(fileId);
 
-		expect(mockAxios.request).toHaveBeenCalledWith(
+		expect(mockAxios.request).toHaveBeenCalledWith({
-			expect.objectContaining({
+			method: 'HEAD',
-				method: 'HEAD',
+			url: `${mockUrl}/${fileId}`,
-				url: `${mockUrl}/${fileId}`,
+			headers: {
-				headers: expect.objectContaining({
+				Host: mockHost,
-					Host: mockHost,
+				'X-Amz-Content-Sha256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855',
-					'X-Amz-Content-Sha256': expect.any(String),
+				'X-Amz-Date': '20240201T012345Z',
-					'X-Amz-Date': expect.any(String),
+				Authorization:
-					Authorization: expect.any(String),
+					'AWS4-HMAC-SHA256 Credential=mock-access-key/20240201/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=60e11c39580ad7dd3a3d549523e7115cdff018540f24c6412ed40053e52a21d0',
-				}),
+			},
-			}),
+		});
-		);
 	});
 
 	it('should throw an error on request failure', async () => {
@@ -101,19 +110,22 @@ describe('put()', () => {
 
 		await objectStoreService.put(fileId, mockBuffer, metadata);
 
-		expect(mockAxios.request).toHaveBeenCalledWith(
+		expect(mockAxios.request).toHaveBeenCalledWith({
-			expect.objectContaining({
+			method: 'PUT',
-				method: 'PUT',
+			url: 'https://s3.us-east-1.amazonaws.com/test-bucket/workflows/ObogjVbqpNOQpiyV/executions/999/binary_data/71f6209b-5d48-41a2-a224-80d529d8bb32',
-				url: `${mockUrl}/${fileId}`,
+			headers: {
-				headers: expect.objectContaining({
+				'Content-Length': 9,
-					'Content-Length': mockBuffer.length,
+				'Content-MD5': 'yh6gLBC3w39CW5t92G1eEQ==',
-					'Content-MD5': expect.any(String),
+				'x-amz-meta-filename': 'file.txt',
-					'x-amz-meta-filename': metadata.fileName,
+				'Content-Type': 'text/plain',
-					'Content-Type': metadata.mimeType,
+				Host: 's3.us-east-1.amazonaws.com',
-				}),
+				'X-Amz-Content-Sha256': 'e27c8214be8b7cf5bccc7c08247e3cb0c1514a48ee1f63197fe4ef3ef51d7e6f',
-				data: mockBuffer,
+				'X-Amz-Date': '20240201T012345Z',
-			}),
+				Authorization:
-		);
+					'AWS4-HMAC-SHA256 Credential=mock-access-key/20240201/us-east-1/s3/aws4_request, SignedHeaders=content-length;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-meta-filename, Signature=6b0fbb51a35dbfa73ac79a964ffc7203b40517a062efc5b01f5f9b7ad553fa7a',
+			},
+			data: mockBuffer,
+		});
 	});
 
 	it('should block if read-only', async () => {
@@ -152,13 +164,18 @@ describe('get()', () => {
 
 		const result = await objectStoreService.get(fileId, { mode: 'buffer' });
 
-		expect(mockAxios.request).toHaveBeenCalledWith(
+		expect(mockAxios.request).toHaveBeenCalledWith({
-			expect.objectContaining({
+			method: 'GET',
-				method: 'GET',
+			url: `${mockUrl}/${fileId}`,
-				url: `${mockUrl}/${fileId}`,
+			responseType: 'arraybuffer',
-				responseType: 'arraybuffer',
+			headers: {
-			}),
+				Authorization:
-		);
+					'AWS4-HMAC-SHA256 Credential=mock-access-key/20240201/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=5f69680786e0ad9f0a0324eb5e4b8fe8c78562afc924489ea423632a2ad2187d',
+				Host: 's3.us-east-1.amazonaws.com',
+				'X-Amz-Content-Sha256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855',
+				'X-Amz-Date': '20240201T012345Z',
+			},
+		});
 
 		expect(Buffer.isBuffer(result)).toBe(true);
 	});
@@ -168,13 +185,18 @@ describe('get()', () => {
 
 		const result = await objectStoreService.get(fileId, { mode: 'stream' });
 
-		expect(mockAxios.request).toHaveBeenCalledWith(
+		expect(mockAxios.request).toHaveBeenCalledWith({
-			expect.objectContaining({
+			method: 'GET',
-				method: 'GET',
+			url: `${mockUrl}/${fileId}`,
-				url: `${mockUrl}/${fileId}`,
+			responseType: 'stream',
-				responseType: 'stream',
+			headers: {
-			}),
+				Authorization:
-		);
+					'AWS4-HMAC-SHA256 Credential=mock-access-key/20240201/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=3ef579ebe2ae89303a89c0faf3ce8ef8e907295dc538d59e95bcf35481c0d03e',
+				Host: 's3.us-east-1.amazonaws.com',
+				'X-Amz-Content-Sha256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855',
+				'X-Amz-Date': '20240201T012345Z',
+			},
+		});
 
 		expect(result instanceof Readable).toBe(true);
 	});
@@ -194,12 +216,17 @@ describe('deleteOne()', () => {
 
 		await objectStoreService.deleteOne(fileId);
 
-		expect(mockAxios.request).toHaveBeenCalledWith(
+		expect(mockAxios.request).toHaveBeenCalledWith({
-			expect.objectContaining({
+			method: 'DELETE',
-				method: 'DELETE',
+			url: `${mockUrl}/${fileId}`,
-				url: `${mockUrl}/${fileId}`,
+			headers: {
-			}),
+				Authorization:
-		);
+					'AWS4-HMAC-SHA256 Credential=mock-access-key/20240201/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=4ad61b1b4da335c6c49772d28e54a301f787d199c9403055b217f890f7aec7fc',
+				Host: 's3.us-east-1.amazonaws.com',
+				'X-Amz-Content-Sha256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855',
+				'X-Amz-Date': '20240201T012345Z',
+			},
+		});
 	});
 
 	it('should throw an error on request failure', async () => {
@@ -232,19 +259,21 @@ describe('deleteMany()', () => {
 
 		await objectStoreService.deleteMany(prefix);
 
-		expect(objectStoreService.list).toHaveBeenCalledWith(prefix);
+		expect(mockAxios.request).toHaveBeenCalledWith({
-		expect(mockAxios.request).toHaveBeenCalledWith(
+			method: 'POST',
-			expect.objectContaining({
+			url: `${mockUrl}?delete=`,
-				method: 'POST',
+			headers: {
-				url: `${mockUrl}/?delete`,
+				'Content-Type': 'application/xml',
-				headers: expect.objectContaining({
+				'Content-Length': 55,
-					'Content-Type': 'application/xml',
+				'Content-MD5': 'ybYDrpQxwYvNIGBQs7PJNA==',
-					'Content-Length': expect.any(Number),
+				Host: 's3.us-east-1.amazonaws.com',
-					'Content-MD5': expect.any(String),
+				'X-Amz-Content-Sha256': '5708e5c935cb75eb528e41ef1548e08b26c5b3b7504b67dc911abc1ff1881f76',
-				}),
+				'X-Amz-Date': '20240201T012345Z',
-				data: toDeletionXml(fileName),
+				Authorization:
-			}),
+					'AWS4-HMAC-SHA256 Credential=mock-access-key/20240201/us-east-1/s3/aws4_request, SignedHeaders=content-length;content-md5;content-type;host;x-amz-content-sha256;x-amz-date, Signature=039168f10927b31624f3a5edae8eb4c89405f7c594eb2d6e00257c1462363f99',
-		);
+			},
+			data: toDeletionXml(fileName),
+		});
 	});
 
 	it('should not send a deletion request if no prefix match', async () => {

packages/core/src/binary-data/object-store/object-store.service.ee.ts

@@ -1,6 +1,7 @@
+import { S3Config } from '@n8n/config';
 import { Service } from '@n8n/di';
 import { sign } from 'aws4';
-import type { Request as Aws4Options, Credentials as Aws4Credentials } from 'aws4';
+import type { Request as Aws4Options } from 'aws4';
 import axios from 'axios';
 import type { AxiosRequestConfig, AxiosResponse, InternalAxiosRequestConfig, Method } from 'axios';
 import { ApplicationError } from 'n8n-workflow';
@@ -9,43 +10,41 @@ import type { Readable } from 'stream';
 
 import { Logger } from '@/logging/logger';
 
-import type {
+import type { ListPage, MetadataResponseHeaders, RawListPage, RequestOptions } from './types';
-	Bucket,
-	ConfigSchemaCredentials,
-	ListPage,
-	MetadataResponseHeaders,
-	RawListPage,
-	RequestOptions,
-} from './types';
 import { isStream, parseXml, writeBlockedMessage } from './utils';
 import type { BinaryData } from '../types';
 
 @Service()
 export class ObjectStoreService {
-	private host = '';
+	private baseUrl: URL;
-
-	private bucket: Bucket = { region: '', name: '' };
-
-	private credentials: Aws4Credentials = { accessKeyId: '', secretAccessKey: '' };
 
 	private isReady = false;
 
 	private isReadOnly = false;
 
-	constructor(private readonly logger: Logger) {}
+	constructor(
+		private readonly logger: Logger,
+		private readonly s3Config: S3Config,
+	) {
+		const { host, bucket, protocol } = s3Config;
 
-	async init(host: string, bucket: Bucket, credentials: ConfigSchemaCredentials) {
+		if (host === '') {
-		this.host = host;
+			throw new ApplicationError(
-		this.bucket.name = bucket.name;
+				'External storage host not configured. Please set `N8N_EXTERNAL_STORAGE_S3_HOST`.',
-		this.bucket.region = bucket.region;
+			);
+		}
 
-		this.credentials = {
+		if (bucket.name === '') {
-			accessKeyId: credentials.accessKey,
+			throw new ApplicationError(
-			secretAccessKey: credentials.accessSecret,
+				'External storage bucket name not configured. Please set `N8N_EXTERNAL_STORAGE_S3_BUCKET_NAME`.',
-		};
+			);
+		}
 
+		this.baseUrl = new URL(`${protocol}://${host}/${bucket.name}`);
+	}
+
+	async init() {
 		await this.checkConnection();
-
 		this.setReady(true);
 	}
 
@@ -65,7 +64,7 @@ export class ObjectStoreService {
 	async checkConnection() {
 		if (this.isReady) return;
 
-		return await this.request('HEAD', this.host, this.bucket.name);
+		return await this.request('HEAD', '');
 	}
 
 	/**
@@ -84,9 +83,7 @@ export class ObjectStoreService {
 		if (metadata.fileName) headers['x-amz-meta-filename'] = metadata.fileName;
 		if (metadata.mimeType) headers['Content-Type'] = metadata.mimeType;
 
-		const path = `/${this.bucket.name}/${filename}`;
+		return await this.request('PUT', filename, { headers, body: buffer });
-
-		return await this.request('PUT', this.host, path, { headers, body: buffer });
 	}
 
 	/**
@@ -97,9 +94,7 @@ export class ObjectStoreService {
 	async get(fileId: string, { mode }: { mode: 'buffer' }): Promise<Buffer>;
 	async get(fileId: string, { mode }: { mode: 'stream' }): Promise<Readable>;
 	async get(fileId: string, { mode }: { mode: 'stream' | 'buffer' }) {
-		const path = `${this.bucket.name}/${fileId}`;
+		const { data } = await this.request('GET', fileId, {
-
-		const { data } = await this.request('GET', this.host, path, {
 			responseType: mode === 'buffer' ? 'arraybuffer' : 'stream',
 		});
 
@@ -116,9 +111,7 @@ export class ObjectStoreService {
 	 * @doc https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingMetadata.html
 	 */
 	async getMetadata(fileId: string) {
-		const path = `${this.bucket.name}/${fileId}`;
+		const response = await this.request('HEAD', fileId);
-
-		const response = await this.request('HEAD', this.host, path);
 
 		return response.headers as MetadataResponseHeaders;
 	}
@@ -129,9 +122,7 @@ export class ObjectStoreService {
 	 * @doc https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html
 	 */
 	async deleteOne(fileId: string) {
-		const path = `${this.bucket.name}/${fileId}`;
+		return await this.request('DELETE', fileId);
-
-		return await this.request('DELETE', this.host, path);
 	}
 
 	/**
@@ -154,9 +145,7 @@ export class ObjectStoreService {
 			'Content-MD5': createHash('md5').update(body).digest('base64'),
 		};
 
-		const path = `${this.bucket.name}/?delete`;
+		return await this.request('POST', '', { headers, body, qs: { delete: '' } });
-
-		return await this.request('POST', this.host, path, { headers, body });
 	}
 
 	/**
@@ -192,7 +181,7 @@ export class ObjectStoreService {
 
 		if (nextPageToken) qs['continuation-token'] = nextPageToken;
 
-		const { data } = await this.request('GET', this.host, this.bucket.name, { qs });
+		const { data } = await this.request('GET', '', { qs });
 
 		if (typeof data !== 'string') {
 			throw new TypeError(`Expected XML string but received ${typeof data}`);
@@ -215,18 +204,6 @@ export class ObjectStoreService {
 		return page as ListPage;
 	}
 
-	private toPath(rawPath: string, qs?: Record<string, string | number>) {
-		const path = rawPath.startsWith('/') ? rawPath : `/${rawPath}`;
-
-		if (!qs) return path;
-
-		const qsParams = Object.entries(qs)
-			.map(([key, value]) => `${key}=${value}`)
-			.join('&');
-
-		return path.concat(`?${qsParams}`);
-	}
-
 	private async blockWrite(filename: string): Promise<AxiosResponse> {
 		const logMessage = writeBlockedMessage(filename);
 
@@ -243,28 +220,37 @@ export class ObjectStoreService {
 
 	private async request<T>(
 		method: Method,
-		host: string,
 		rawPath = '',
 		{ qs, headers, body, responseType }: RequestOptions = {},
 	) {
-		const path = this.toPath(rawPath, qs);
+		const url = new URL(this.baseUrl);
+		if (rawPath && rawPath !== '/') {
+			url.pathname = `${url.pathname}/${rawPath}`;
+		}
+		Object.entries(qs ?? {}).forEach(([key, value]) => {
+			url.searchParams.set(key, String(value));
+		});
 
 		const optionsToSign: Aws4Options = {
 			method,
 			service: 's3',
-			region: this.bucket.region,
+			region: this.s3Config.bucket.region,
-			host,
+			host: this.s3Config.host,
-			path,
+			path: `${url.pathname}${url.search}`,
 		};
 
 		if (headers) optionsToSign.headers = headers;
 		if (body) optionsToSign.body = body;
 
-		const signedOptions = sign(optionsToSign, this.credentials);
+		const { accessKey, accessSecret } = this.s3Config.credentials;
+		const signedOptions = sign(optionsToSign, {
+			accessKeyId: accessKey,
+			secretAccessKey: accessSecret,
+		});
 
 		const config: AxiosRequestConfig = {
 			method,
-			url: `https://${host}${path}`,
+			url: url.toString(),
 			headers: signedOptions.headers,
 		};
 

packages/core/src/binary-data/object-store/types.ts

@@ -24,8 +24,6 @@ type Item = {
 
 export type ListPage = Omit<RawListPage['listBucketResult'], 'contents'> & { contents: Item[] };
 
-export type Bucket = { region: string; name: string };
-
 export type RequestOptions = {
 	qs?: Record<string, string | number>;
 	headers?: Record<string, string | number>;
@@ -38,5 +36,3 @@ export type MetadataResponseHeaders = AxiosResponseHeaders & {
 	'content-type'?: string;
 	'x-amz-meta-filename'?: string;
 } & BinaryData.PreWriteMetadata;
-
-export type ConfigSchemaCredentials = { accessKey: string; accessSecret: string };