From ddc0f571169b4a4fcdcf5b06a3b0405de8702a9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Tue, 26 Mar 2024 19:09:46 +0100 Subject: [PATCH] refactor(core): Store SSH key pair for source control in DB settings (#8965) --- .../1711390882123-MoveSshKeysToDatabase.ts | 106 ++++++++++++++++++ .../src/databases/migrations/mysqldb/index.ts | 2 + .../databases/migrations/postgresdb/index.ts | 2 + .../src/databases/migrations/sqlite/index.ts | 2 + .../sourceControl.controller.ee.ts | 6 +- .../sourceControl/sourceControl.service.ee.ts | 3 +- .../sourceControlGit.service.ee.ts | 13 +-- .../sourceControlPreferences.service.ee.ts | 100 +++++++++++++---- 8 files changed, 201 insertions(+), 33 deletions(-) create mode 100644 packages/cli/src/databases/migrations/common/1711390882123-MoveSshKeysToDatabase.ts diff --git a/packages/cli/src/databases/migrations/common/1711390882123-MoveSshKeysToDatabase.ts b/packages/cli/src/databases/migrations/common/1711390882123-MoveSshKeysToDatabase.ts new file mode 100644 index 0000000000..b02450b3cc --- /dev/null +++ b/packages/cli/src/databases/migrations/common/1711390882123-MoveSshKeysToDatabase.ts @@ -0,0 +1,106 @@ +import path from 'node:path'; +import { readFile, writeFile, rm } from 'node:fs/promises'; +import Container from 'typedi'; +import { Cipher, InstanceSettings } from 'n8n-core'; +import { jsonParse } from 'n8n-workflow'; +import type { MigrationContext, ReversibleMigration } from '@db/types'; + +/** + * Move SSH key pair from file system to database, to enable SSH connections + * when running n8n in multiple containers - mains, webhooks, workers, etc. + */ +export class MoveSshKeysToDatabase1711390882123 implements ReversibleMigration { + private readonly settingsKey = 'features.sourceControl.sshKeys'; + + private readonly privateKeyPath = path.join( + Container.get(InstanceSettings).n8nFolder, + 'ssh', + 'key', + ); + + private readonly publicKeyPath = this.privateKeyPath + '.pub'; + + private readonly cipher = Container.get(Cipher); + + async up({ escape, runQuery, logger, migrationName }: MigrationContext) { + let privateKey, publicKey; + + try { + [privateKey, publicKey] = await Promise.all([ + readFile(this.privateKeyPath, { encoding: 'utf8' }), + readFile(this.publicKeyPath, { encoding: 'utf8' }), + ]); + } catch { + logger.info(`[${migrationName}] No SSH keys in filesystem, skipping`); + return; + } + + const settings = escape.tableName('settings'); + + const rows: Array<{ value: string }> = await runQuery( + `SELECT value FROM ${settings} WHERE key = '${this.settingsKey}';`, + ); + + if (rows.length === 1) { + logger.info(`[${migrationName}] SSH keys already in database, skipping`); + return; + } + + const value = JSON.stringify({ + encryptedPrivateKey: this.cipher.encrypt(privateKey), + publicKey, + }); + + await runQuery( + `INSERT INTO ${settings} (key, value) VALUES ('${this.settingsKey}', '${value}');`, + ); + + try { + await Promise.all([rm(this.privateKeyPath), rm(this.publicKeyPath)]); + } catch (e) { + const error = e instanceof Error ? e : new Error(`${e}`); + logger.error( + `[${migrationName}] Failed to remove SSH keys from filesystem: ${error.message}`, + ); + } + } + + async down({ escape, runQuery, logger, migrationName }: MigrationContext) { + const settings = escape.tableName('settings'); + + const rows: Array<{ value: string }> = await runQuery( + `SELECT value FROM ${settings} WHERE key = '${this.settingsKey}';`, + ); + + if (rows.length !== 1) { + logger.info(`[${migrationName}] No SSH keys in database, skipping revert`); + return; + } + + const [row] = rows; + + type KeyPair = { publicKey: string; encryptedPrivateKey: string }; + + const dbKeyPair = jsonParse(row.value, { fallbackValue: null }); + + if (!dbKeyPair) { + logger.info(`[${migrationName}] Malformed SSH keys in database, skipping revert`); + return; + } + + const privateKey = this.cipher.decrypt(dbKeyPair.encryptedPrivateKey); + const { publicKey } = dbKeyPair; + + try { + await Promise.all([ + writeFile(this.privateKeyPath, privateKey, { encoding: 'utf8', mode: 0o600 }), + writeFile(this.publicKeyPath, publicKey, { encoding: 'utf8', mode: 0o600 }), + ]); + } catch { + logger.error(`[${migrationName}] Failed to write SSH keys to filesystem, skipping revert`); + return; + } + + await runQuery(`DELETE ${settings} WHERE WHERE key = 'features.sourceControl.sshKeys';`); + } +} diff --git a/packages/cli/src/databases/migrations/mysqldb/index.ts b/packages/cli/src/databases/migrations/mysqldb/index.ts index b8878cab43..89a6a2b0ee 100644 --- a/packages/cli/src/databases/migrations/mysqldb/index.ts +++ b/packages/cli/src/databases/migrations/mysqldb/index.ts @@ -53,6 +53,7 @@ import { ModifyWorkflowHistoryNodesAndConnections1695829275184 } from '../common import { AddGlobalAdminRole1700571993961 } from '../common/1700571993961-AddGlobalAdminRole'; import { DropRoleMapping1705429061930 } from '../common/1705429061930-DropRoleMapping'; import { RemoveFailedExecutionStatus1711018413374 } from '../common/1711018413374-RemoveFailedExecutionStatus'; +import { MoveSshKeysToDatabase1711390882123 } from '../common/1711390882123-MoveSshKeysToDatabase'; export const mysqlMigrations: Migration[] = [ InitialMigration1588157391238, @@ -109,4 +110,5 @@ export const mysqlMigrations: Migration[] = [ AddGlobalAdminRole1700571993961, DropRoleMapping1705429061930, RemoveFailedExecutionStatus1711018413374, + MoveSshKeysToDatabase1711390882123, ]; diff --git a/packages/cli/src/databases/migrations/postgresdb/index.ts b/packages/cli/src/databases/migrations/postgresdb/index.ts index de177ebbc7..3b572c2e57 100644 --- a/packages/cli/src/databases/migrations/postgresdb/index.ts +++ b/packages/cli/src/databases/migrations/postgresdb/index.ts @@ -52,6 +52,7 @@ import { ModifyWorkflowHistoryNodesAndConnections1695829275184 } from '../common import { AddGlobalAdminRole1700571993961 } from '../common/1700571993961-AddGlobalAdminRole'; import { DropRoleMapping1705429061930 } from '../common/1705429061930-DropRoleMapping'; import { RemoveFailedExecutionStatus1711018413374 } from '../common/1711018413374-RemoveFailedExecutionStatus'; +import { MoveSshKeysToDatabase1711390882123 } from '../common/1711390882123-MoveSshKeysToDatabase'; export const postgresMigrations: Migration[] = [ InitialMigration1587669153312, @@ -107,4 +108,5 @@ export const postgresMigrations: Migration[] = [ AddGlobalAdminRole1700571993961, DropRoleMapping1705429061930, RemoveFailedExecutionStatus1711018413374, + MoveSshKeysToDatabase1711390882123, ]; diff --git a/packages/cli/src/databases/migrations/sqlite/index.ts b/packages/cli/src/databases/migrations/sqlite/index.ts index 2e0bfca4a7..bc25048b53 100644 --- a/packages/cli/src/databases/migrations/sqlite/index.ts +++ b/packages/cli/src/databases/migrations/sqlite/index.ts @@ -50,6 +50,7 @@ import { ModifyWorkflowHistoryNodesAndConnections1695829275184 } from '../common import { AddGlobalAdminRole1700571993961 } from '../common/1700571993961-AddGlobalAdminRole'; import { DropRoleMapping1705429061930 } from './1705429061930-DropRoleMapping'; import { RemoveFailedExecutionStatus1711018413374 } from '../common/1711018413374-RemoveFailedExecutionStatus'; +import { MoveSshKeysToDatabase1711390882123 } from '../common/1711390882123-MoveSshKeysToDatabase'; const sqliteMigrations: Migration[] = [ InitialMigration1588102412422, @@ -103,6 +104,7 @@ const sqliteMigrations: Migration[] = [ AddGlobalAdminRole1700571993961, DropRoleMapping1705429061930, RemoveFailedExecutionStatus1711018413374, + MoveSshKeysToDatabase1711390882123, ]; export { sqliteMigrations }; diff --git a/packages/cli/src/environments/sourceControl/sourceControl.controller.ee.ts b/packages/cli/src/environments/sourceControl/sourceControl.controller.ee.ts index 0cd0dde9a9..77d7330799 100644 --- a/packages/cli/src/environments/sourceControl/sourceControl.controller.ee.ts +++ b/packages/cli/src/environments/sourceControl/sourceControl.controller.ee.ts @@ -28,7 +28,8 @@ export class SourceControlController { @Get('/preferences', { middlewares: [sourceControlLicensedMiddleware], skipAuth: true }) async getPreferences(): Promise { // returns the settings with the privateKey property redacted - return this.sourceControlPreferencesService.getPreferences(); + const publicKey = await this.sourceControlPreferencesService.getPublicKey(); + return { ...this.sourceControlPreferencesService.getPreferences(), publicKey }; } @Post('/preferences', { middlewares: [sourceControlLicensedMiddleware] }) @@ -238,7 +239,8 @@ export class SourceControlController { try { const keyPairType = req.body.keyGeneratorType; const result = await this.sourceControlPreferencesService.generateAndSaveKeyPair(keyPairType); - return result; + const publicKey = await this.sourceControlPreferencesService.getPublicKey(); + return { ...result, publicKey }; } catch (error) { throw new BadRequestError((error as { message: string }).message); } diff --git a/packages/cli/src/environments/sourceControl/sourceControl.service.ee.ts b/packages/cli/src/environments/sourceControl/sourceControl.service.ee.ts index 189d3b9e77..759c69f74b 100644 --- a/packages/cli/src/environments/sourceControl/sourceControl.service.ee.ts +++ b/packages/cli/src/environments/sourceControl/sourceControl.service.ee.ts @@ -39,6 +39,7 @@ import { ApplicationError } from 'n8n-workflow'; @Service() export class SourceControlService { + /** Path to SSH private key in filesystem. */ private sshKeyName: string; private sshFolder: string; @@ -112,7 +113,7 @@ export class SourceControlService { }); await this.sourceControlExportService.deleteRepositoryFolder(); if (!options.keepKeyPair) { - await this.sourceControlPreferencesService.deleteKeyPairFiles(); + await this.sourceControlPreferencesService.deleteKeyPair(); } this.gitService.resetService(); return this.sourceControlPreferencesService.sourceControlPreferences; diff --git a/packages/cli/src/environments/sourceControl/sourceControlGit.service.ee.ts b/packages/cli/src/environments/sourceControl/sourceControlGit.service.ee.ts index 64690a632d..0c270b2a88 100644 --- a/packages/cli/src/environments/sourceControl/sourceControlGit.service.ee.ts +++ b/packages/cli/src/environments/sourceControl/sourceControlGit.service.ee.ts @@ -23,6 +23,7 @@ import type { User } from '@db/entities/User'; import { Logger } from '@/Logger'; import { ApplicationError } from 'n8n-workflow'; import { OwnershipService } from '@/services/ownership.service'; +import { SourceControlPreferencesService } from './sourceControlPreferences.service.ee'; @Service() export class SourceControlGitService { @@ -33,6 +34,7 @@ export class SourceControlGitService { constructor( private readonly logger: Logger, private readonly ownershipService: OwnershipService, + private readonly sourceControlPreferencesService: SourceControlPreferencesService, ) {} /** @@ -66,12 +68,7 @@ export class SourceControlGitService { sshFolder: string; sshKeyName: string; }): Promise { - const { - sourceControlPreferences: sourceControlPreferences, - gitFolder, - sshKeyName, - sshFolder, - } = options; + const { sourceControlPreferences: sourceControlPreferences, gitFolder, sshFolder } = options; this.logger.debug('GitService.init'); if (this.git !== null) { return; @@ -82,8 +79,10 @@ export class SourceControlGitService { sourceControlFoldersExistCheck([gitFolder, sshFolder]); + const privateKeyPath = await this.sourceControlPreferencesService.getPrivateKeyPath(); + const sshKnownHosts = path.join(sshFolder, 'known_hosts'); - const sshCommand = `ssh -o UserKnownHostsFile=${sshKnownHosts} -o StrictHostKeyChecking=no -i ${sshKeyName}`; + const sshCommand = `ssh -o UserKnownHostsFile=${sshKnownHosts} -o StrictHostKeyChecking=no -i ${privateKeyPath}`; this.gitOptions = { baseDir: gitFolder, diff --git a/packages/cli/src/environments/sourceControl/sourceControlPreferences.service.ee.ts b/packages/cli/src/environments/sourceControl/sourceControlPreferences.service.ee.ts index 4da3221ab5..085df4c5bc 100644 --- a/packages/cli/src/environments/sourceControl/sourceControlPreferences.service.ee.ts +++ b/packages/cli/src/environments/sourceControl/sourceControlPreferences.service.ee.ts @@ -1,15 +1,16 @@ +import os from 'node:os'; +import { writeFile, chmod, readFile } from 'node:fs/promises'; import Container, { Service } from 'typedi'; import { SourceControlPreferences } from './types/sourceControlPreferences'; import type { ValidationError } from 'class-validator'; import { validate } from 'class-validator'; -import { readFileSync as fsReadFileSync, existsSync as fsExistsSync } from 'fs'; import { writeFile as fsWriteFile, rm as fsRm } from 'fs/promises'; import { generateSshKeyPair, isSourceControlLicensed, sourceControlFoldersExistCheck, } from './sourceControlHelper.ee'; -import { InstanceSettings } from 'n8n-core'; +import { Cipher, InstanceSettings } from 'n8n-core'; import { ApplicationError, jsonParse } from 'n8n-workflow'; import { SOURCE_CONTROL_SSH_FOLDER, @@ -36,6 +37,7 @@ export class SourceControlPreferencesService { constructor( instanceSettings: InstanceSettings, private readonly logger: Logger, + private readonly cipher: Cipher, ) { this.sshFolder = path.join(instanceSettings.n8nFolder, SOURCE_CONTROL_SSH_FOLDER); this.gitFolder = path.join(instanceSettings.n8nFolder, SOURCE_CONTROL_GIT_FOLDER); @@ -46,7 +48,6 @@ export class SourceControlPreferencesService { return { ...this._sourceControlPreferences, connected: this._sourceControlPreferences.connected ?? false, - publicKey: this.getPublicKey(), }; } @@ -66,24 +67,71 @@ export class SourceControlPreferencesService { ); } - getPublicKey(): string { + private async getKeyPairFromDatabase() { + const dbSetting = await Container.get(SettingsRepository).findByKey( + 'features.sourceControl.sshKeys', + ); + + if (!dbSetting?.value) return null; + + type KeyPair = { publicKey: string; encryptedPrivateKey: string }; + + return jsonParse(dbSetting.value, { fallbackValue: null }); + } + + private async getPrivateKeyFromDatabase() { + const dbKeyPair = await this.getKeyPairFromDatabase(); + + if (!dbKeyPair) return null; + + return this.cipher.decrypt(dbKeyPair.encryptedPrivateKey); + } + + private async getPublicKeyFromDatabase() { + const dbKeyPair = await this.getKeyPairFromDatabase(); + + if (!dbKeyPair) return null; + + return dbKeyPair.publicKey; + } + + async getPrivateKeyPath() { + const dbPrivateKey = await this.getPrivateKeyFromDatabase(); + + if (dbPrivateKey) { + const tempFilePath = path.join(os.tmpdir(), 'ssh_private_key_temp'); + + await writeFile(tempFilePath, dbPrivateKey); + + await chmod(tempFilePath, 0o600); + + return tempFilePath; + } + + return this.sshKeyName; // fall back to key in filesystem + } + + async getPublicKey() { try { - return fsReadFileSync(this.sshKeyName + '.pub', { encoding: 'utf8' }); - } catch (error) { - this.logger.error(`Failed to read public key: ${(error as Error).message}`); + const dbPublicKey = await this.getPublicKeyFromDatabase(); + + if (dbPublicKey) return dbPublicKey; + + return await readFile(this.sshKeyName + '.pub', { encoding: 'utf8' }); + } catch (e) { + const error = e instanceof Error ? e : new Error(`${e}`); + this.logger.error(`Failed to read SSH public key: ${error.message}`); } return ''; } - hasKeyPairFiles(): boolean { - return fsExistsSync(this.sshKeyName) && fsExistsSync(this.sshKeyName + '.pub'); - } - - async deleteKeyPairFiles(): Promise { + async deleteKeyPair() { try { await fsRm(this.sshFolder, { recursive: true }); - } catch (error) { - this.logger.error(`Failed to delete ssh folder: ${(error as Error).message}`); + await Container.get(SettingsRepository).delete({ key: 'features.sourceControl.sshKeys' }); + } catch (e) { + const error = e instanceof Error ? e : new Error(`${e}`); + this.logger.error(`Failed to delete SSH key pair: ${error.message}`); } } @@ -108,13 +156,27 @@ export class SourceControlPreferencesService { }); await fsWriteFile(this.sshKeyName, keyPair.privateKey, { encoding: 'utf8', mode: 0o600 }); } catch (error) { - throw new ApplicationError('Failed to save key pair', { cause: error }); + throw new ApplicationError('Failed to save key pair to disk', { cause: error }); } } // update preferences only after generating key pair to prevent endless loop if (keyPairType !== this.getPreferences().keyGeneratorType) { await this.setPreferences({ keyGeneratorType: keyPairType }); } + + try { + await Container.get(SettingsRepository).save({ + key: 'features.sourceControl.sshKeys', + value: JSON.stringify({ + encryptedPrivateKey: this.cipher.encrypt(keyPair.privateKey), + publicKey: keyPair.publicKey, + }), + loadOnStartup: true, + }); + } catch (error) { + throw new ApplicationError('Failed to write key pair to database', { cause: error }); + } + return this.getPreferences(); } @@ -161,14 +223,6 @@ export class SourceControlPreferencesService { preferences: Partial, saveToDb = true, ): Promise { - sourceControlFoldersExistCheck([this.gitFolder, this.sshFolder]); - if (!this.hasKeyPairFiles()) { - const keyPairType = - preferences.keyGeneratorType ?? - (config.get('sourceControl.defaultKeyPairType') as KeyPairType); - this.logger.debug(`No key pair files found, generating new pair using type: ${keyPairType}`); - await this.generateAndSaveKeyPair(keyPairType); - } this.sourceControlPreferences = preferences; if (saveToDb) { const settingsValue = JSON.stringify(this._sourceControlPreferences);