diff --git a/packages/@n8n/config/src/configs/external-storage.config.ts b/packages/@n8n/config/src/configs/external-storage.config.ts index 6e5fbd64d8..aff2447d40 100644 --- a/packages/@n8n/config/src/configs/external-storage.config.ts +++ b/packages/@n8n/config/src/configs/external-storage.config.ts @@ -23,11 +23,14 @@ class S3CredentialsConfig { } @Config -class S3Config { +export class S3Config { /** Host of the n8n bucket in S3-compatible external storage @example "s3.us-east-1.amazonaws.com" */ @Env('N8N_EXTERNAL_STORAGE_S3_HOST') host: string = ''; + @Env('N8N_EXTERNAL_STORAGE_S3_PROTOCOL') + protocol: 'http' | 'https' = 'https'; + @Nested bucket: S3BucketConfig; diff --git a/packages/@n8n/config/src/index.ts b/packages/@n8n/config/src/index.ts index f462ef9424..00abdf6d7c 100644 --- a/packages/@n8n/config/src/index.ts +++ b/packages/@n8n/config/src/index.ts @@ -30,6 +30,7 @@ export { TaskRunnersConfig } from './configs/runners.config'; export { SecurityConfig } from './configs/security.config'; export { ExecutionsConfig } from './configs/executions.config'; export { FrontendBetaFeatures, FrontendConfig } from './configs/frontend.config'; +export { S3Config } from './configs/external-storage.config'; export { LOG_SCOPES } from './configs/logging.config'; export type { LogScope } from './configs/logging.config'; diff --git a/packages/@n8n/config/test/config.test.ts b/packages/@n8n/config/test/config.test.ts index d9499d7849..aa115364d9 100644 --- a/packages/@n8n/config/test/config.test.ts +++ b/packages/@n8n/config/test/config.test.ts @@ -138,6 +138,7 @@ describe('GlobalConfig', () => { externalStorage: { s3: { host: '', + protocol: 'https', bucket: { name: '', region: '', diff --git a/packages/cli/src/commands/base-command.ts b/packages/cli/src/commands/base-command.ts index 9bfd992b3d..5e07b8e350 100644 --- a/packages/cli/src/commands/base-command.ts +++ b/packages/cli/src/commands/base-command.ts @@ -189,42 +189,10 @@ export abstract class BaseCommand extends Command { private async _initObjectStoreService(options = { isReadOnly: false }) { const objectStoreService = Container.get(ObjectStoreService); - const { host, bucket, credentials } = this.globalConfig.externalStorage.s3; - - if (host === '') { - throw new ApplicationError( - 'External storage host not configured. Please set `N8N_EXTERNAL_STORAGE_S3_HOST`.', - ); - } - - if (bucket.name === '') { - throw new ApplicationError( - 'External storage bucket name not configured. Please set `N8N_EXTERNAL_STORAGE_S3_BUCKET_NAME`.', - ); - } - - if (bucket.region === '') { - throw new ApplicationError( - 'External storage bucket region not configured. Please set `N8N_EXTERNAL_STORAGE_S3_BUCKET_REGION`.', - ); - } - - if (credentials.accessKey === '') { - throw new ApplicationError( - 'External storage access key not configured. Please set `N8N_EXTERNAL_STORAGE_S3_ACCESS_KEY`.', - ); - } - - if (credentials.accessSecret === '') { - throw new ApplicationError( - 'External storage access secret not configured. Please set `N8N_EXTERNAL_STORAGE_S3_ACCESS_SECRET`.', - ); - } - this.logger.debug('Initializing object store service'); try { - await objectStoreService.init(host, bucket, credentials); + await objectStoreService.init(); objectStoreService.setReadonly(options.isReadOnly); this.logger.debug('Object store init completed'); diff --git a/packages/core/src/binary-data/object-store/__tests__/object-store.service.test.ts b/packages/core/src/binary-data/object-store/__tests__/object-store.service.test.ts index f5d2924eb5..ca8f4c0c94 100644 --- a/packages/core/src/binary-data/object-store/__tests__/object-store.service.test.ts +++ b/packages/core/src/binary-data/object-store/__tests__/object-store.service.test.ts @@ -1,3 +1,4 @@ +import type { S3Config } from '@n8n/config'; import axios from 'axios'; import { mock } from 'jest-mock-extended'; import { Readable } from 'stream'; @@ -18,6 +19,12 @@ const mockError = new Error('Something went wrong!'); const fileId = 'workflows/ObogjVbqpNOQpiyV/executions/999/binary_data/71f6209b-5d48-41a2-a224-80d529d8bb32'; const mockBuffer = Buffer.from('Test data'); +const s3Config = mock({ + host: mockHost, + bucket: mockBucket, + credentials: mockCredentials, + protocol: 'https', +}); const toDeletionXml = (filename: string) => ` ${filename} @@ -25,10 +32,13 @@ const toDeletionXml = (filename: string) => ` let objectStoreService: ObjectStoreService; +const now = new Date('2024-02-01T01:23:45.678Z'); +jest.useFakeTimers({ now }); + beforeEach(async () => { - objectStoreService = new ObjectStoreService(mock()); + objectStoreService = new ObjectStoreService(mock(), s3Config); mockAxios.request.mockResolvedValueOnce({ status: 200 }); // for checkConnection - await objectStoreService.init(mockHost, mockBucket, mockCredentials); + await objectStoreService.init(); jest.restoreAllMocks(); }); @@ -40,17 +50,17 @@ describe('checkConnection()', () => { await objectStoreService.checkConnection(); - expect(mockAxios.request).toHaveBeenCalledWith( - expect.objectContaining({ - method: 'HEAD', - url: `https://${mockHost}/${mockBucket.name}`, - headers: expect.objectContaining({ - 'X-Amz-Content-Sha256': expect.any(String), - 'X-Amz-Date': expect.any(String), - Authorization: expect.any(String), - }), - }), - ); + expect(mockAxios.request).toHaveBeenCalledWith({ + method: 'HEAD', + url: 'https://s3.us-east-1.amazonaws.com/test-bucket', + headers: { + Host: 's3.us-east-1.amazonaws.com', + 'X-Amz-Content-Sha256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', + 'X-Amz-Date': '20240201T012345Z', + Authorization: + 'AWS4-HMAC-SHA256 Credential=mock-access-key/20240201/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=a5240c11a706e9e6c60e7033a848fc934911b12330e5a4609b0b943f97d9781b', + }, + }); }); it('should throw an error on request failure', async () => { @@ -70,18 +80,17 @@ describe('getMetadata()', () => { await objectStoreService.getMetadata(fileId); - expect(mockAxios.request).toHaveBeenCalledWith( - expect.objectContaining({ - method: 'HEAD', - url: `${mockUrl}/${fileId}`, - headers: expect.objectContaining({ - Host: mockHost, - 'X-Amz-Content-Sha256': expect.any(String), - 'X-Amz-Date': expect.any(String), - Authorization: expect.any(String), - }), - }), - ); + expect(mockAxios.request).toHaveBeenCalledWith({ + method: 'HEAD', + url: `${mockUrl}/${fileId}`, + headers: { + Host: mockHost, + 'X-Amz-Content-Sha256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', + 'X-Amz-Date': '20240201T012345Z', + Authorization: + 'AWS4-HMAC-SHA256 Credential=mock-access-key/20240201/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=60e11c39580ad7dd3a3d549523e7115cdff018540f24c6412ed40053e52a21d0', + }, + }); }); it('should throw an error on request failure', async () => { @@ -101,19 +110,22 @@ describe('put()', () => { await objectStoreService.put(fileId, mockBuffer, metadata); - expect(mockAxios.request).toHaveBeenCalledWith( - expect.objectContaining({ - method: 'PUT', - url: `${mockUrl}/${fileId}`, - headers: expect.objectContaining({ - 'Content-Length': mockBuffer.length, - 'Content-MD5': expect.any(String), - 'x-amz-meta-filename': metadata.fileName, - 'Content-Type': metadata.mimeType, - }), - data: mockBuffer, - }), - ); + expect(mockAxios.request).toHaveBeenCalledWith({ + method: 'PUT', + url: 'https://s3.us-east-1.amazonaws.com/test-bucket/workflows/ObogjVbqpNOQpiyV/executions/999/binary_data/71f6209b-5d48-41a2-a224-80d529d8bb32', + headers: { + 'Content-Length': 9, + 'Content-MD5': 'yh6gLBC3w39CW5t92G1eEQ==', + 'x-amz-meta-filename': 'file.txt', + 'Content-Type': 'text/plain', + Host: 's3.us-east-1.amazonaws.com', + 'X-Amz-Content-Sha256': 'e27c8214be8b7cf5bccc7c08247e3cb0c1514a48ee1f63197fe4ef3ef51d7e6f', + 'X-Amz-Date': '20240201T012345Z', + Authorization: + 'AWS4-HMAC-SHA256 Credential=mock-access-key/20240201/us-east-1/s3/aws4_request, SignedHeaders=content-length;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-meta-filename, Signature=6b0fbb51a35dbfa73ac79a964ffc7203b40517a062efc5b01f5f9b7ad553fa7a', + }, + data: mockBuffer, + }); }); it('should block if read-only', async () => { @@ -152,13 +164,18 @@ describe('get()', () => { const result = await objectStoreService.get(fileId, { mode: 'buffer' }); - expect(mockAxios.request).toHaveBeenCalledWith( - expect.objectContaining({ - method: 'GET', - url: `${mockUrl}/${fileId}`, - responseType: 'arraybuffer', - }), - ); + expect(mockAxios.request).toHaveBeenCalledWith({ + method: 'GET', + url: `${mockUrl}/${fileId}`, + responseType: 'arraybuffer', + headers: { + Authorization: + 'AWS4-HMAC-SHA256 Credential=mock-access-key/20240201/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=5f69680786e0ad9f0a0324eb5e4b8fe8c78562afc924489ea423632a2ad2187d', + Host: 's3.us-east-1.amazonaws.com', + 'X-Amz-Content-Sha256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', + 'X-Amz-Date': '20240201T012345Z', + }, + }); expect(Buffer.isBuffer(result)).toBe(true); }); @@ -168,13 +185,18 @@ describe('get()', () => { const result = await objectStoreService.get(fileId, { mode: 'stream' }); - expect(mockAxios.request).toHaveBeenCalledWith( - expect.objectContaining({ - method: 'GET', - url: `${mockUrl}/${fileId}`, - responseType: 'stream', - }), - ); + expect(mockAxios.request).toHaveBeenCalledWith({ + method: 'GET', + url: `${mockUrl}/${fileId}`, + responseType: 'stream', + headers: { + Authorization: + 'AWS4-HMAC-SHA256 Credential=mock-access-key/20240201/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=3ef579ebe2ae89303a89c0faf3ce8ef8e907295dc538d59e95bcf35481c0d03e', + Host: 's3.us-east-1.amazonaws.com', + 'X-Amz-Content-Sha256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', + 'X-Amz-Date': '20240201T012345Z', + }, + }); expect(result instanceof Readable).toBe(true); }); @@ -194,12 +216,17 @@ describe('deleteOne()', () => { await objectStoreService.deleteOne(fileId); - expect(mockAxios.request).toHaveBeenCalledWith( - expect.objectContaining({ - method: 'DELETE', - url: `${mockUrl}/${fileId}`, - }), - ); + expect(mockAxios.request).toHaveBeenCalledWith({ + method: 'DELETE', + url: `${mockUrl}/${fileId}`, + headers: { + Authorization: + 'AWS4-HMAC-SHA256 Credential=mock-access-key/20240201/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=4ad61b1b4da335c6c49772d28e54a301f787d199c9403055b217f890f7aec7fc', + Host: 's3.us-east-1.amazonaws.com', + 'X-Amz-Content-Sha256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', + 'X-Amz-Date': '20240201T012345Z', + }, + }); }); it('should throw an error on request failure', async () => { @@ -232,19 +259,21 @@ describe('deleteMany()', () => { await objectStoreService.deleteMany(prefix); - expect(objectStoreService.list).toHaveBeenCalledWith(prefix); - expect(mockAxios.request).toHaveBeenCalledWith( - expect.objectContaining({ - method: 'POST', - url: `${mockUrl}/?delete`, - headers: expect.objectContaining({ - 'Content-Type': 'application/xml', - 'Content-Length': expect.any(Number), - 'Content-MD5': expect.any(String), - }), - data: toDeletionXml(fileName), - }), - ); + expect(mockAxios.request).toHaveBeenCalledWith({ + method: 'POST', + url: `${mockUrl}?delete=`, + headers: { + 'Content-Type': 'application/xml', + 'Content-Length': 55, + 'Content-MD5': 'ybYDrpQxwYvNIGBQs7PJNA==', + Host: 's3.us-east-1.amazonaws.com', + 'X-Amz-Content-Sha256': '5708e5c935cb75eb528e41ef1548e08b26c5b3b7504b67dc911abc1ff1881f76', + 'X-Amz-Date': '20240201T012345Z', + Authorization: + 'AWS4-HMAC-SHA256 Credential=mock-access-key/20240201/us-east-1/s3/aws4_request, SignedHeaders=content-length;content-md5;content-type;host;x-amz-content-sha256;x-amz-date, Signature=039168f10927b31624f3a5edae8eb4c89405f7c594eb2d6e00257c1462363f99', + }, + data: toDeletionXml(fileName), + }); }); it('should not send a deletion request if no prefix match', async () => { diff --git a/packages/core/src/binary-data/object-store/object-store.service.ee.ts b/packages/core/src/binary-data/object-store/object-store.service.ee.ts index 508477d50e..5561bd61db 100644 --- a/packages/core/src/binary-data/object-store/object-store.service.ee.ts +++ b/packages/core/src/binary-data/object-store/object-store.service.ee.ts @@ -1,6 +1,7 @@ +import { S3Config } from '@n8n/config'; import { Service } from '@n8n/di'; import { sign } from 'aws4'; -import type { Request as Aws4Options, Credentials as Aws4Credentials } from 'aws4'; +import type { Request as Aws4Options } from 'aws4'; import axios from 'axios'; import type { AxiosRequestConfig, AxiosResponse, InternalAxiosRequestConfig, Method } from 'axios'; import { ApplicationError } from 'n8n-workflow'; @@ -9,43 +10,41 @@ import type { Readable } from 'stream'; import { Logger } from '@/logging/logger'; -import type { - Bucket, - ConfigSchemaCredentials, - ListPage, - MetadataResponseHeaders, - RawListPage, - RequestOptions, -} from './types'; +import type { ListPage, MetadataResponseHeaders, RawListPage, RequestOptions } from './types'; import { isStream, parseXml, writeBlockedMessage } from './utils'; import type { BinaryData } from '../types'; @Service() export class ObjectStoreService { - private host = ''; - - private bucket: Bucket = { region: '', name: '' }; - - private credentials: Aws4Credentials = { accessKeyId: '', secretAccessKey: '' }; + private baseUrl: URL; private isReady = false; private isReadOnly = false; - constructor(private readonly logger: Logger) {} + constructor( + private readonly logger: Logger, + private readonly s3Config: S3Config, + ) { + const { host, bucket, protocol } = s3Config; - async init(host: string, bucket: Bucket, credentials: ConfigSchemaCredentials) { - this.host = host; - this.bucket.name = bucket.name; - this.bucket.region = bucket.region; + if (host === '') { + throw new ApplicationError( + 'External storage host not configured. Please set `N8N_EXTERNAL_STORAGE_S3_HOST`.', + ); + } - this.credentials = { - accessKeyId: credentials.accessKey, - secretAccessKey: credentials.accessSecret, - }; + if (bucket.name === '') { + throw new ApplicationError( + 'External storage bucket name not configured. Please set `N8N_EXTERNAL_STORAGE_S3_BUCKET_NAME`.', + ); + } + this.baseUrl = new URL(`${protocol}://${host}/${bucket.name}`); + } + + async init() { await this.checkConnection(); - this.setReady(true); } @@ -65,7 +64,7 @@ export class ObjectStoreService { async checkConnection() { if (this.isReady) return; - return await this.request('HEAD', this.host, this.bucket.name); + return await this.request('HEAD', ''); } /** @@ -84,9 +83,7 @@ export class ObjectStoreService { if (metadata.fileName) headers['x-amz-meta-filename'] = metadata.fileName; if (metadata.mimeType) headers['Content-Type'] = metadata.mimeType; - const path = `/${this.bucket.name}/${filename}`; - - return await this.request('PUT', this.host, path, { headers, body: buffer }); + return await this.request('PUT', filename, { headers, body: buffer }); } /** @@ -97,9 +94,7 @@ export class ObjectStoreService { async get(fileId: string, { mode }: { mode: 'buffer' }): Promise; async get(fileId: string, { mode }: { mode: 'stream' }): Promise; async get(fileId: string, { mode }: { mode: 'stream' | 'buffer' }) { - const path = `${this.bucket.name}/${fileId}`; - - const { data } = await this.request('GET', this.host, path, { + const { data } = await this.request('GET', fileId, { responseType: mode === 'buffer' ? 'arraybuffer' : 'stream', }); @@ -116,9 +111,7 @@ export class ObjectStoreService { * @doc https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingMetadata.html */ async getMetadata(fileId: string) { - const path = `${this.bucket.name}/${fileId}`; - - const response = await this.request('HEAD', this.host, path); + const response = await this.request('HEAD', fileId); return response.headers as MetadataResponseHeaders; } @@ -129,9 +122,7 @@ export class ObjectStoreService { * @doc https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html */ async deleteOne(fileId: string) { - const path = `${this.bucket.name}/${fileId}`; - - return await this.request('DELETE', this.host, path); + return await this.request('DELETE', fileId); } /** @@ -154,9 +145,7 @@ export class ObjectStoreService { 'Content-MD5': createHash('md5').update(body).digest('base64'), }; - const path = `${this.bucket.name}/?delete`; - - return await this.request('POST', this.host, path, { headers, body }); + return await this.request('POST', '', { headers, body, qs: { delete: '' } }); } /** @@ -192,7 +181,7 @@ export class ObjectStoreService { if (nextPageToken) qs['continuation-token'] = nextPageToken; - const { data } = await this.request('GET', this.host, this.bucket.name, { qs }); + const { data } = await this.request('GET', '', { qs }); if (typeof data !== 'string') { throw new TypeError(`Expected XML string but received ${typeof data}`); @@ -215,18 +204,6 @@ export class ObjectStoreService { return page as ListPage; } - private toPath(rawPath: string, qs?: Record) { - const path = rawPath.startsWith('/') ? rawPath : `/${rawPath}`; - - if (!qs) return path; - - const qsParams = Object.entries(qs) - .map(([key, value]) => `${key}=${value}`) - .join('&'); - - return path.concat(`?${qsParams}`); - } - private async blockWrite(filename: string): Promise { const logMessage = writeBlockedMessage(filename); @@ -243,28 +220,37 @@ export class ObjectStoreService { private async request( method: Method, - host: string, rawPath = '', { qs, headers, body, responseType }: RequestOptions = {}, ) { - const path = this.toPath(rawPath, qs); + const url = new URL(this.baseUrl); + if (rawPath && rawPath !== '/') { + url.pathname = `${url.pathname}/${rawPath}`; + } + Object.entries(qs ?? {}).forEach(([key, value]) => { + url.searchParams.set(key, String(value)); + }); const optionsToSign: Aws4Options = { method, service: 's3', - region: this.bucket.region, - host, - path, + region: this.s3Config.bucket.region, + host: this.s3Config.host, + path: `${url.pathname}${url.search}`, }; if (headers) optionsToSign.headers = headers; if (body) optionsToSign.body = body; - const signedOptions = sign(optionsToSign, this.credentials); + const { accessKey, accessSecret } = this.s3Config.credentials; + const signedOptions = sign(optionsToSign, { + accessKeyId: accessKey, + secretAccessKey: accessSecret, + }); const config: AxiosRequestConfig = { method, - url: `https://${host}${path}`, + url: url.toString(), headers: signedOptions.headers, }; diff --git a/packages/core/src/binary-data/object-store/types.ts b/packages/core/src/binary-data/object-store/types.ts index 49726f5c43..20390cf243 100644 --- a/packages/core/src/binary-data/object-store/types.ts +++ b/packages/core/src/binary-data/object-store/types.ts @@ -24,8 +24,6 @@ type Item = { export type ListPage = Omit & { contents: Item[] }; -export type Bucket = { region: string; name: string }; - export type RequestOptions = { qs?: Record; headers?: Record; @@ -38,5 +36,3 @@ export type MetadataResponseHeaders = AxiosResponseHeaders & { 'content-type'?: string; 'x-amz-meta-filename'?: string; } & BinaryData.PreWriteMetadata; - -export type ConfigSchemaCredentials = { accessKey: string; accessSecret: string };