diff --git a/packages/cli/package.json b/packages/cli/package.json index 2cc9396c4b..9c9b788222 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -139,6 +139,7 @@ "formidable": "^3.5.0", "google-timezones-json": "^1.1.0", "handlebars": "4.7.7", + "infisical-node": "^1.3.0", "inquirer": "^7.0.1", "ioredis": "^5.2.4", "json-diff": "^1.0.6", diff --git a/packages/cli/src/CredentialsHelper.ts b/packages/cli/src/CredentialsHelper.ts index 303b7a0fcb..f8bfc0cc8a 100644 --- a/packages/cli/src/CredentialsHelper.ts +++ b/packages/cli/src/CredentialsHelper.ts @@ -31,6 +31,7 @@ import type { IHttpRequestHelper, INodeTypeData, INodeTypes, + IWorkflowExecuteAdditionalData, ICredentialTestFunctions, } from 'n8n-workflow'; import { @@ -342,6 +343,7 @@ export class CredentialsHelper extends ICredentialsHelper { * @param {boolean} [raw] Return the data as supplied without defaults or overwrites */ async getDecrypted( + additionalData: IWorkflowExecuteAdditionalData, nodeCredentials: INodeCredentialsDetails, type: string, mode: WorkflowExecuteMode, @@ -356,12 +358,18 @@ export class CredentialsHelper extends ICredentialsHelper { return decryptedDataOriginal; } + await additionalData?.secretsHelpers?.waitForInit(); + + const canUseSecrets = await this.credentialOwnedByOwner(nodeCredentials); + return this.applyDefaultsAndOverwrites( + additionalData, decryptedDataOriginal, type, mode, defaultTimezone, expressionResolveValues, + canUseSecrets, ); } @@ -369,11 +377,13 @@ export class CredentialsHelper extends ICredentialsHelper { * Applies credential default data and overwrites */ applyDefaultsAndOverwrites( + additionalData: IWorkflowExecuteAdditionalData, decryptedDataOriginal: ICredentialDataDecryptedObject, type: string, mode: WorkflowExecuteMode, defaultTimezone: string, expressionResolveValues?: ICredentialsExpressionResolveValues, + canUseSecrets?: boolean, ): ICredentialDataDecryptedObject { const credentialsProperties = this.getCredentialsProperties(type); @@ -395,6 +405,10 @@ export class CredentialsHelper extends ICredentialsHelper { decryptedData.oauthTokenData = decryptedDataOriginal.oauthTokenData; } + const additionalKeys = NodeExecuteFunctions.getAdditionalKeys(additionalData, mode, null, { + secretsEnabled: canUseSecrets, + }); + if (expressionResolveValues) { const timezone = expressionResolveValues.workflow.settings.timezone ?? defaultTimezone; @@ -408,7 +422,7 @@ export class CredentialsHelper extends ICredentialsHelper { expressionResolveValues.connectionInputData, mode, timezone, - {}, + additionalKeys, undefined, false, decryptedData, @@ -431,7 +445,7 @@ export class CredentialsHelper extends ICredentialsHelper { decryptedData as INodeParameters, mode, defaultTimezone, - {}, + additionalKeys, undefined, undefined, decryptedData, @@ -573,10 +587,24 @@ export class CredentialsHelper extends ICredentialsHelper { } if (credentialsDecrypted.data) { - credentialsDecrypted.data = CredentialsOverwrites().applyOverwrite( - credentialType, - credentialsDecrypted.data, - ); + try { + const additionalData = await WorkflowExecuteAdditionalData.getBase(user.id); + credentialsDecrypted.data = this.applyDefaultsAndOverwrites( + additionalData, + credentialsDecrypted.data, + credentialType, + 'internal' as WorkflowExecuteMode, + additionalData.timezone, + undefined, + user.isOwner, + ); + } catch (error) { + Logger.debug('Credential test failed', error); + return { + status: 'Error', + message: error.message.toString(), + }; + } } if (typeof credentialTestFunction === 'function') { @@ -759,6 +787,36 @@ export class CredentialsHelper extends ICredentialsHelper { message: 'Connection successful!', }; } + + async credentialOwnedByOwner(nodeCredential: INodeCredentialsDetails): Promise { + if (!nodeCredential.id) { + return false; + } + + const credential = await Db.collections.SharedCredentials.findOne({ + where: { + role: { + scope: 'credential', + name: 'owner', + }, + user: { + globalRole: { + scope: 'global', + name: 'owner', + }, + }, + credentials: { + id: nodeCredential.id, + }, + }, + }); + + if (!credential) { + return false; + } + + return true; + } } /** diff --git a/packages/cli/src/ExternalSecrets/ExternalSecrets.controller.ee.ts b/packages/cli/src/ExternalSecrets/ExternalSecrets.controller.ee.ts new file mode 100644 index 0000000000..10eed808a6 --- /dev/null +++ b/packages/cli/src/ExternalSecrets/ExternalSecrets.controller.ee.ts @@ -0,0 +1,102 @@ +import { Authorized, Get, Post, RestController } from '@/decorators'; +import { ExternalSecretsRequest } from '@/requests'; +import { NotFoundError } from '@/ResponseHelper'; +import { Response } from 'express'; +import { Service } from 'typedi'; +import { ProviderNotFoundError, ExternalSecretsService } from './ExternalSecrets.service.ee'; + +@Service() +@Authorized(['global', 'owner']) +@RestController('/external-secrets') +export class ExternalSecretsController { + constructor(private readonly secretsService: ExternalSecretsService) {} + + @Get('/providers') + async getProviders() { + return this.secretsService.getProviders(); + } + + @Get('/providers/:provider') + async getProvider(req: ExternalSecretsRequest.GetProvider) { + const providerName = req.params.provider; + try { + return this.secretsService.getProvider(providerName); + } catch (e) { + if (e instanceof ProviderNotFoundError) { + throw new NotFoundError(`Could not find provider "${e.providerName}"`); + } + throw e; + } + } + + @Post('/providers/:provider/test') + async testProviderSettings(req: ExternalSecretsRequest.TestProviderSettings, res: Response) { + const providerName = req.params.provider; + try { + const result = await this.secretsService.testProviderSettings(providerName, req.body); + if (result.success) { + res.statusCode = 200; + } else { + res.statusCode = 400; + } + return result; + } catch (e) { + if (e instanceof ProviderNotFoundError) { + throw new NotFoundError(`Could not find provider "${e.providerName}"`); + } + throw e; + } + } + + @Post('/providers/:provider') + async setProviderSettings(req: ExternalSecretsRequest.SetProviderSettings) { + const providerName = req.params.provider; + try { + await this.secretsService.saveProviderSettings(providerName, req.body, req.user.id); + } catch (e) { + if (e instanceof ProviderNotFoundError) { + throw new NotFoundError(`Could not find provider "${e.providerName}"`); + } + throw e; + } + return {}; + } + + @Post('/providers/:provider/connect') + async setProviderConnected(req: ExternalSecretsRequest.SetProviderConnected) { + const providerName = req.params.provider; + try { + await this.secretsService.saveProviderConnected(providerName, req.body.connected); + } catch (e) { + if (e instanceof ProviderNotFoundError) { + throw new NotFoundError(`Could not find provider "${e.providerName}"`); + } + throw e; + } + return {}; + } + + @Post('/providers/:provider/update') + async updateProvider(req: ExternalSecretsRequest.UpdateProvider, res: Response) { + const providerName = req.params.provider; + try { + const resp = await this.secretsService.updateProvider(providerName); + if (resp) { + res.statusCode = 200; + } else { + res.statusCode = 400; + } + return { updated: resp }; + } catch (e) { + if (e instanceof ProviderNotFoundError) { + throw new NotFoundError(`Could not find provider "${e.providerName}"`); + } + throw e; + } + } + + @Get('/secrets') + getSecretNames() { + return this.secretsService.getAllSecrets(); + } +} diff --git a/packages/cli/src/ExternalSecrets/ExternalSecrets.service.ee.ts b/packages/cli/src/ExternalSecrets/ExternalSecrets.service.ee.ts new file mode 100644 index 0000000000..9952c8a9b8 --- /dev/null +++ b/packages/cli/src/ExternalSecrets/ExternalSecrets.service.ee.ts @@ -0,0 +1,154 @@ +import { CREDENTIAL_BLANKING_VALUE } from '@/constants'; +import type { SecretsProvider } from '@/Interfaces'; +import type { ExternalSecretsRequest } from '@/requests'; +import type { IDataObject } from 'n8n-workflow'; +import { deepCopy } from 'n8n-workflow'; +import Container, { Service } from 'typedi'; +import { ExternalSecretsManager } from './ExternalSecretsManager.ee'; + +export class ProviderNotFoundError extends Error { + constructor(public providerName: string) { + super(undefined); + } +} + +@Service() +export class ExternalSecretsService { + getProvider(providerName: string): ExternalSecretsRequest.GetProviderResponse | null { + const providerAndSettings = + Container.get(ExternalSecretsManager).getProviderWithSettings(providerName); + if (!providerAndSettings) { + throw new ProviderNotFoundError(providerName); + } + const { provider, settings } = providerAndSettings; + return { + displayName: provider.displayName, + name: provider.name, + icon: provider.name, + state: provider.state, + connected: settings.connected, + connectedAt: settings.connectedAt, + properties: provider.properties, + data: this.redact(settings.settings, provider), + }; + } + + async getProviders() { + return Container.get(ExternalSecretsManager) + .getProvidersWithSettings() + .map(({ provider, settings }) => ({ + displayName: provider.displayName, + name: provider.name, + icon: provider.name, + state: provider.state, + connected: !!settings.connected, + connectedAt: settings.connectedAt, + data: this.redact(settings.settings, provider), + })); + } + + // Take data and replace all sensitive values with a sentinel value. + // This will replace password fields and oauth data. + redact(data: IDataObject, provider: SecretsProvider): IDataObject { + const copiedData = deepCopy(data || {}); + + const properties = provider.properties; + + for (const dataKey of Object.keys(copiedData)) { + // The frontend only cares that this value isn't falsy. + if (dataKey === 'oauthTokenData') { + copiedData[dataKey] = CREDENTIAL_BLANKING_VALUE; + continue; + } + const prop = properties.find((v) => v.name === dataKey); + if (!prop) { + continue; + } + + if ( + prop.typeOptions?.password && + (!(copiedData[dataKey] as string).startsWith('=') || prop.noDataExpression) + ) { + copiedData[dataKey] = CREDENTIAL_BLANKING_VALUE; + } + } + + return copiedData; + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + private unredactRestoreValues(unmerged: any, replacement: any) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-argument + for (const [key, value] of Object.entries(unmerged)) { + if (value === CREDENTIAL_BLANKING_VALUE) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access + unmerged[key] = replacement[key]; + } else if ( + typeof value === 'object' && + value !== null && + key in replacement && + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + typeof replacement[key] === 'object' && + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + replacement[key] !== null + ) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + this.unredactRestoreValues(value, replacement[key]); + } + } + } + + // Take unredacted data (probably from the DB) and merge it with + // redacted data to create an unredacted version. + unredact(redactedData: IDataObject, savedData: IDataObject): IDataObject { + // Replace any blank sentinel values with their saved version + const mergedData = deepCopy(redactedData ?? {}); + this.unredactRestoreValues(mergedData, savedData); + return mergedData; + } + + async saveProviderSettings(providerName: string, data: IDataObject, userId: string) { + const providerAndSettings = + Container.get(ExternalSecretsManager).getProviderWithSettings(providerName); + if (!providerAndSettings) { + throw new ProviderNotFoundError(providerName); + } + const { settings } = providerAndSettings; + const newData = this.unredact(data, settings.settings); + await Container.get(ExternalSecretsManager).setProviderSettings(providerName, newData, userId); + } + + async saveProviderConnected(providerName: string, connected: boolean) { + const providerAndSettings = + Container.get(ExternalSecretsManager).getProviderWithSettings(providerName); + if (!providerAndSettings) { + throw new ProviderNotFoundError(providerName); + } + await Container.get(ExternalSecretsManager).setProviderConnected(providerName, connected); + return this.getProvider(providerName); + } + + getAllSecrets(): Record { + return Container.get(ExternalSecretsManager).getAllSecretNames(); + } + + async testProviderSettings(providerName: string, data: IDataObject) { + const providerAndSettings = + Container.get(ExternalSecretsManager).getProviderWithSettings(providerName); + if (!providerAndSettings) { + throw new ProviderNotFoundError(providerName); + } + const { settings } = providerAndSettings; + const newData = this.unredact(data, settings.settings); + return Container.get(ExternalSecretsManager).testProviderSettings(providerName, newData); + } + + async updateProvider(providerName: string) { + const providerAndSettings = + Container.get(ExternalSecretsManager).getProviderWithSettings(providerName); + if (!providerAndSettings) { + throw new ProviderNotFoundError(providerName); + } + return Container.get(ExternalSecretsManager).updateProvider(providerName); + } +} diff --git a/packages/cli/src/ExternalSecrets/ExternalSecretsManager.ee.ts b/packages/cli/src/ExternalSecrets/ExternalSecretsManager.ee.ts new file mode 100644 index 0000000000..6a0b422377 --- /dev/null +++ b/packages/cli/src/ExternalSecrets/ExternalSecretsManager.ee.ts @@ -0,0 +1,381 @@ +import { SettingsRepository } from '@/databases/repositories'; +import type { + ExternalSecretsSettings, + SecretsProvider, + SecretsProviderSettings, +} from '@/Interfaces'; + +import { UserSettings } from 'n8n-core'; +import Container, { Service } from 'typedi'; + +import { AES, enc } from 'crypto-js'; +import { getLogger } from '@/Logger'; + +import type { IDataObject } from 'n8n-workflow'; +import { + EXTERNAL_SECRETS_INITIAL_BACKOFF, + EXTERNAL_SECRETS_MAX_BACKOFF, + EXTERNAL_SECRETS_UPDATE_INTERVAL, +} from './constants'; +import { License } from '@/License'; +import { InternalHooks } from '@/InternalHooks'; +import { ExternalSecretsProviders } from './ExternalSecretsProviders.ee'; + +const logger = getLogger(); + +@Service() +export class ExternalSecretsManager { + private providers: Record = {}; + + private initializingPromise?: Promise; + + private cachedSettings: ExternalSecretsSettings = {}; + + initialized = false; + + updateInterval: NodeJS.Timer; + + initRetryTimeouts: Record = {}; + + constructor( + private settingsRepo: SettingsRepository, + private license: License, + private secretsProviders: ExternalSecretsProviders, + ) {} + + async init(): Promise { + if (!this.initialized) { + if (!this.initializingPromise) { + this.initializingPromise = new Promise(async (resolve) => { + await this.internalInit(); + this.initialized = true; + resolve(); + this.initializingPromise = undefined; + this.updateInterval = setInterval( + async () => this.updateSecrets(), + EXTERNAL_SECRETS_UPDATE_INTERVAL, + ); + }); + } + return this.initializingPromise; + } + } + + shutdown() { + clearInterval(this.updateInterval); + Object.values(this.providers).forEach((p) => { + // Disregard any errors as we're shutting down anyway + void p.disconnect().catch(() => {}); + }); + Object.values(this.initRetryTimeouts).forEach((v) => clearTimeout(v)); + } + + private async getEncryptionKey(): Promise { + return UserSettings.getEncryptionKey(); + } + + private decryptSecretsSettings(value: string, encryptionKey: string): ExternalSecretsSettings { + const decryptedData = AES.decrypt(value, encryptionKey); + + try { + return JSON.parse(decryptedData.toString(enc.Utf8)) as ExternalSecretsSettings; + } catch (e) { + throw new Error( + 'External Secrets Settings could not be decrypted. The likely reason is that a different "encryptionKey" was used to encrypt the data.', + ); + } + } + + private async getDecryptedSettings( + settingsRepo: SettingsRepository, + ): Promise { + const encryptedSettings = await settingsRepo.getEncryptedSecretsProviderSettings(); + if (encryptedSettings === null) { + return null; + } + const encryptionKey = await this.getEncryptionKey(); + return this.decryptSecretsSettings(encryptedSettings, encryptionKey); + } + + private async internalInit() { + const settings = await this.getDecryptedSettings(this.settingsRepo); + if (!settings) { + return; + } + const providers: Array = ( + await Promise.allSettled( + Object.entries(settings).map(async ([name, providerSettings]) => + this.initProvider(name, providerSettings), + ), + ) + ).map((i) => (i.status === 'rejected' ? null : i.value)); + this.providers = Object.fromEntries( + (providers.filter((p) => p !== null) as SecretsProvider[]).map((s) => [s.name, s]), + ); + this.cachedSettings = settings; + await this.updateSecrets(); + } + + private async initProvider( + name: string, + providerSettings: SecretsProviderSettings, + currentBackoff = EXTERNAL_SECRETS_INITIAL_BACKOFF, + ) { + const providerClass = this.secretsProviders.getProvider(name); + if (!providerClass) { + return null; + } + const provider: SecretsProvider = new providerClass(); + + try { + await provider.init(providerSettings); + } catch (e) { + logger.error( + `Error initializing secrets provider ${provider.displayName} (${provider.name}).`, + ); + this.retryInitWithBackoff(name, currentBackoff); + return provider; + } + + try { + if (providerSettings.connected) { + await provider.connect(); + } + } catch (e) { + try { + await provider.disconnect(); + } catch {} + logger.error( + `Error initializing secrets provider ${provider.displayName} (${provider.name}).`, + ); + this.retryInitWithBackoff(name, currentBackoff); + return provider; + } + + return provider; + } + + private retryInitWithBackoff(name: string, currentBackoff: number) { + if (name in this.initRetryTimeouts) { + clearTimeout(this.initRetryTimeouts[name]); + delete this.initRetryTimeouts[name]; + } + this.initRetryTimeouts[name] = setTimeout(() => { + delete this.initRetryTimeouts[name]; + if (this.providers[name] && this.providers[name].state !== 'error') { + return; + } + void this.reloadProvider(name, Math.min(currentBackoff * 2, EXTERNAL_SECRETS_MAX_BACKOFF)); + }, currentBackoff); + } + + async updateSecrets() { + if (!this.license.isExternalSecretsEnabled()) { + return; + } + await Promise.allSettled( + Object.entries(this.providers).map(async ([k, p]) => { + try { + if (this.cachedSettings[k].connected && p.state === 'connected') { + await p.update(); + } + } catch { + logger.error(`Error updating secrets provider ${p.displayName} (${p.name}).`); + } + }), + ); + } + + getProvider(provider: string): SecretsProvider | undefined { + return this.providers[provider]; + } + + hasProvider(provider: string): boolean { + return provider in this.providers; + } + + getProviderNames(): string[] | undefined { + return Object.keys(this.providers); + } + + getSecret(provider: string, name: string): IDataObject | undefined { + return this.getProvider(provider)?.getSecret(name); + } + + hasSecret(provider: string, name: string): boolean { + return this.getProvider(provider)?.hasSecret(name) ?? false; + } + + getSecretNames(provider: string): string[] | undefined { + return this.getProvider(provider)?.getSecretNames(); + } + + getAllSecretNames(): Record { + return Object.fromEntries( + Object.keys(this.providers).map((provider) => [ + provider, + this.getSecretNames(provider) ?? [], + ]), + ); + } + + getProvidersWithSettings(): Array<{ + provider: SecretsProvider; + settings: SecretsProviderSettings; + }> { + return Object.entries(this.secretsProviders.getAllProviders()).map(([k, c]) => ({ + provider: this.getProvider(k) ?? new c(), + settings: this.cachedSettings[k] ?? {}, + })); + } + + getProviderWithSettings(provider: string): + | { + provider: SecretsProvider; + settings: SecretsProviderSettings; + } + | undefined { + const providerConstructor = this.secretsProviders.getProvider(provider); + if (!providerConstructor) { + return undefined; + } + return { + provider: this.getProvider(provider) ?? new providerConstructor(), + settings: this.cachedSettings[provider] ?? {}, + }; + } + + async reloadProvider(provider: string, backoff = EXTERNAL_SECRETS_INITIAL_BACKOFF) { + if (provider in this.providers) { + await this.providers[provider].disconnect(); + delete this.providers[provider]; + } + const newProvider = await this.initProvider(provider, this.cachedSettings[provider], backoff); + if (newProvider) { + this.providers[provider] = newProvider; + } + } + + async setProviderSettings(provider: string, data: IDataObject, userId?: string) { + let isNewProvider = false; + let settings = await this.getDecryptedSettings(this.settingsRepo); + if (!settings) { + settings = {}; + } + if (!(provider in settings)) { + isNewProvider = true; + } + settings[provider] = { + connected: settings[provider]?.connected ?? false, + connectedAt: settings[provider]?.connectedAt ?? new Date(), + settings: data, + }; + + await this.saveAndSetSettings(settings, this.settingsRepo); + this.cachedSettings = settings; + await this.reloadProvider(provider); + + void this.trackProviderSave(provider, isNewProvider, userId); + } + + async setProviderConnected(provider: string, connected: boolean) { + let settings = await this.getDecryptedSettings(this.settingsRepo); + if (!settings) { + settings = {}; + } + settings[provider] = { + connected, + connectedAt: connected ? new Date() : settings[provider]?.connectedAt ?? null, + settings: settings[provider]?.settings ?? {}, + }; + + await this.saveAndSetSettings(settings, this.settingsRepo); + this.cachedSettings = settings; + await this.reloadProvider(provider); + await this.updateSecrets(); + } + + private async trackProviderSave(vaultType: string, isNew: boolean, userId?: string) { + let testResult: [boolean] | [boolean, string] | undefined; + try { + testResult = await this.getProvider(vaultType)?.test(); + } catch {} + void Container.get(InternalHooks).onExternalSecretsProviderSettingsSaved({ + user_id: userId, + vault_type: vaultType, + is_new: isNew, + is_valid: testResult?.[0] ?? false, + error_message: testResult?.[1], + }); + } + + encryptSecretsSettings(settings: ExternalSecretsSettings, encryptionKey: string): string { + return AES.encrypt(JSON.stringify(settings), encryptionKey).toString(); + } + + async saveAndSetSettings(settings: ExternalSecretsSettings, settingsRepo: SettingsRepository) { + const encryptionKey = await this.getEncryptionKey(); + const encryptedSettings = this.encryptSecretsSettings(settings, encryptionKey); + await settingsRepo.saveEncryptedSecretsProviderSettings(encryptedSettings); + } + + async testProviderSettings( + provider: string, + data: IDataObject, + ): Promise<{ + success: boolean; + testState: 'connected' | 'tested' | 'error'; + error?: string; + }> { + let testProvider: SecretsProvider | null = null; + try { + testProvider = await this.initProvider(provider, { + connected: true, + connectedAt: new Date(), + settings: data, + }); + if (!testProvider) { + return { + success: false, + testState: 'error', + }; + } + const [success, error] = await testProvider.test(); + let testState: 'connected' | 'tested' | 'error' = 'error'; + if (success && this.cachedSettings[provider]?.connected) { + testState = 'connected'; + } else if (success) { + testState = 'tested'; + } + return { + success, + testState, + error, + }; + } catch { + return { + success: false, + testState: 'error', + }; + } finally { + if (testProvider) { + await testProvider.disconnect(); + } + } + } + + async updateProvider(provider: string): Promise { + if (!this.license.isExternalSecretsEnabled()) { + return false; + } + if (!this.providers[provider] || this.providers[provider].state !== 'connected') { + return false; + } + try { + await this.providers[provider].update(); + return true; + } catch { + return false; + } + } +} diff --git a/packages/cli/src/ExternalSecrets/ExternalSecretsProviders.ee.ts b/packages/cli/src/ExternalSecrets/ExternalSecretsProviders.ee.ts new file mode 100644 index 0000000000..a0e9353699 --- /dev/null +++ b/packages/cli/src/ExternalSecrets/ExternalSecretsProviders.ee.ts @@ -0,0 +1,24 @@ +import type { SecretsProvider } from '@/Interfaces'; +import { Service } from 'typedi'; +import { InfisicalProvider } from './providers/infisical'; +import { VaultProvider } from './providers/vault'; + +@Service() +export class ExternalSecretsProviders { + providers: Record = { + infisical: InfisicalProvider, + vault: VaultProvider, + }; + + getProvider(name: string): { new (): SecretsProvider } | null { + return this.providers[name] ?? null; + } + + hasProvider(name: string) { + return name in this.providers; + } + + getAllProviders() { + return this.providers; + } +} diff --git a/packages/cli/src/ExternalSecrets/constants.ts b/packages/cli/src/ExternalSecrets/constants.ts new file mode 100644 index 0000000000..534b4ceb72 --- /dev/null +++ b/packages/cli/src/ExternalSecrets/constants.ts @@ -0,0 +1,6 @@ +export const EXTERNAL_SECRETS_DB_KEY = 'feature.externalSecrets'; +export const EXTERNAL_SECRETS_UPDATE_INTERVAL = 5 * 60 * 1000; +export const EXTERNAL_SECRETS_INITIAL_BACKOFF = 10 * 1000; +export const EXTERNAL_SECRETS_MAX_BACKOFF = 5 * 60 * 1000; + +export const EXTERNAL_SECRETS_NAME_REGEX = /^[a-zA-Z0-9_]+$/; diff --git a/packages/cli/src/ExternalSecrets/externalSecretsHelper.ee.ts b/packages/cli/src/ExternalSecrets/externalSecretsHelper.ee.ts new file mode 100644 index 0000000000..54885664ff --- /dev/null +++ b/packages/cli/src/ExternalSecrets/externalSecretsHelper.ee.ts @@ -0,0 +1,7 @@ +import { License } from '@/License'; +import Container from 'typedi'; + +export function isExternalSecretsEnabled() { + const license = Container.get(License); + return license.isExternalSecretsEnabled(); +} diff --git a/packages/cli/src/ExternalSecrets/providers/infisical.ts b/packages/cli/src/ExternalSecrets/providers/infisical.ts new file mode 100644 index 0000000000..39a0eb92d4 --- /dev/null +++ b/packages/cli/src/ExternalSecrets/providers/infisical.ts @@ -0,0 +1,153 @@ +import type { SecretsProvider, SecretsProviderSettings, SecretsProviderState } from '@/Interfaces'; +import InfisicalClient from 'infisical-node'; +import { populateClientWorkspaceConfigsHelper } from 'infisical-node/lib/helpers/key'; +import { getServiceTokenData } from 'infisical-node/lib/api/serviceTokenData'; +import type { IDataObject, INodeProperties } from 'n8n-workflow'; +import { EXTERNAL_SECRETS_NAME_REGEX } from '../constants'; + +export interface InfisicalSettings { + token: string; + siteURL: string; + cacheTTL: number; + debug: boolean; +} + +interface InfisicalSecret { + secretName: string; + secretValue?: string; +} + +interface InfisicalServiceToken { + environment?: string; + scopes?: Array<{ environment: string; path: string }>; +} + +export class InfisicalProvider implements SecretsProvider { + properties: INodeProperties[] = [ + { + displayName: + 'Need help filling out these fields? Open docs', + name: 'notice', + type: 'notice', + default: '', + }, + { + displayName: 'Service Token', + name: 'token', + type: 'string', + hint: 'The Infisical Service Token with read access', + default: '', + required: true, + placeholder: 'e.g. st.64ae963e1874ea.374226a166439dce.39557e4a1b7bdd82', + noDataExpression: true, + typeOptions: { password: true }, + }, + { + displayName: 'Site URL', + name: 'siteURL', + type: 'string', + hint: "The absolute URL of the Infisical instance. Change it only if you're self-hosting Infisical.", + required: true, + noDataExpression: true, + placeholder: 'https://app.infisical.com', + default: 'https://app.infisical.com', + }, + ]; + + displayName = 'Infisical'; + + name = 'infisical'; + + state: SecretsProviderState = 'initializing'; + + private cachedSecrets: Record = {}; + + private client: InfisicalClient; + + private settings: InfisicalSettings; + + private environment: string; + + async init(settings: SecretsProviderSettings): Promise { + this.settings = settings.settings as unknown as InfisicalSettings; + } + + async update(): Promise { + if (!this.client) { + throw new Error('Updated attempted on Infisical when initialization failed'); + } + if (!(await this.test())[0]) { + throw new Error('Infisical provider test failed during update'); + } + const secrets = (await this.client.getAllSecrets({ + environment: this.environment, + path: '/', + attachToProcessEnv: false, + includeImports: true, + })) as InfisicalSecret[]; + const newCache = Object.fromEntries( + secrets.map((s) => [s.secretName, s.secretValue]), + ) as Record; + if (Object.keys(newCache).length === 1 && '' in newCache) { + this.cachedSecrets = {}; + } else { + this.cachedSecrets = newCache; + } + } + + async connect(): Promise { + this.client = new InfisicalClient(this.settings); + if ((await this.test())[0]) { + try { + this.environment = await this.getEnvironment(); + this.state = 'connected'; + } catch { + this.state = 'error'; + } + } else { + this.state = 'error'; + } + } + + async getEnvironment(): Promise { + const serviceTokenData = (await getServiceTokenData( + // eslint-disable-next-line @typescript-eslint/no-unsafe-argument + this.client.clientConfig, + )) as InfisicalServiceToken; + if (serviceTokenData.environment) { + return serviceTokenData.environment; + } + if (serviceTokenData.scopes) { + return serviceTokenData.scopes[0].environment; + } + throw new Error("Couldn't find environment for Infisical"); + } + + async test(): Promise<[boolean] | [boolean, string]> { + if (!this.client) { + return [false, 'Client not initialized']; + } + try { + await populateClientWorkspaceConfigsHelper(this.client.clientConfig); + return [true]; + } catch (e) { + return [false]; + } + } + + async disconnect(): Promise { + // + } + + getSecret(name: string): IDataObject { + return this.cachedSecrets[name] as unknown as IDataObject; + } + + getSecretNames(): string[] { + return Object.keys(this.cachedSecrets).filter((k) => EXTERNAL_SECRETS_NAME_REGEX.test(k)); + } + + hasSecret(name: string): boolean { + return name in this.cachedSecrets; + } +} diff --git a/packages/cli/src/ExternalSecrets/providers/vault.ts b/packages/cli/src/ExternalSecrets/providers/vault.ts new file mode 100644 index 0000000000..65ef4a9acf --- /dev/null +++ b/packages/cli/src/ExternalSecrets/providers/vault.ts @@ -0,0 +1,559 @@ +import type { SecretsProviderSettings, SecretsProviderState } from '@/Interfaces'; +import { SecretsProvider } from '@/Interfaces'; +import type { IDataObject, INodeProperties } from 'n8n-workflow'; +import type { AxiosInstance, AxiosResponse } from 'axios'; +import axios from 'axios'; +import { getLogger } from '@/Logger'; +import { EXTERNAL_SECRETS_NAME_REGEX } from '../constants'; + +const logger = getLogger(); + +type VaultAuthMethod = 'token' | 'usernameAndPassword' | 'appRole'; + +interface VaultSettings { + url: string; + namespace?: string; + authMethod: VaultAuthMethod; + + // Token + token: string; + renewToken: boolean; + + // Username and Password + username: string; + password: string; + + // AppRole + roleId: string; + secretId: string; +} + +interface VaultResponse { + data: T; +} + +interface VaultTokenInfo { + accessor: string; + creation_time: number; + creation_ttl: number; + display_name: string; + entity_id: string; + expire_time: string | null; + explicit_max_ttl: number; + id: string; + issue_time: string; + meta: Record; + num_uses: number; + orphan: boolean; + path: string; + policies: string[]; + ttl: number; + renewable: boolean; + type: 'kv' | string; +} + +interface VaultMount { + accessor: string; + config: Record; + description: string; + external_entropy_access: boolean; + local: boolean; + options: Record; + plugin_version: string; + running_plugin_version: string; + running_sha256: string; + seal_wrap: number; + type: string; + uuid: string; +} + +interface VaultMountsResp { + [path: string]: VaultMount; +} + +interface VaultUserPassLoginResp { + auth: { + client_token: string; + }; +} + +type VaultAppRoleResp = VaultUserPassLoginResp; + +interface VaultSecretList { + keys: string[]; +} + +export class VaultProvider extends SecretsProvider { + properties: INodeProperties[] = [ + { + displayName: + 'Need help filling out these fields? Open docs', + name: 'notice', + type: 'notice', + default: '', + }, + { + displayName: 'Vault URL', + name: 'url', + type: 'string', + required: true, + noDataExpression: true, + placeholder: 'e.g. https://example.com/v1/', + default: '', + }, + { + displayName: 'Vault Namespace (optional)', + name: 'namespace', + type: 'string', + hint: 'Leave blank if not using namespaces', + required: false, + noDataExpression: true, + placeholder: 'e.g. admin', + default: '', + }, + { + displayName: 'Authentication Method', + name: 'authMethod', + type: 'options', + required: true, + noDataExpression: true, + options: [ + { name: 'Token', value: 'token' }, + { name: 'Username and Password', value: 'usernameAndPassword' }, + { name: 'AppRole', value: 'appRole' }, + ], + default: 'token', + }, + + // Token Auth + { + displayName: 'Token', + name: 'token', + type: 'string', + default: '', + required: true, + noDataExpression: true, + placeholder: 'e.g. hvs.2OCsZxZA6Z9lChbt0janOOZI', + typeOptions: { password: true }, + displayOptions: { + show: { + authMethod: ['token'], + }, + }, + }, + // { + // displayName: 'Renew Token', + // name: 'renewToken', + // description: + // 'Try to renew Vault token. This will update the settings on this provider when doing so.', + // type: 'boolean', + // noDataExpression: true, + // default: true, + // displayOptions: { + // show: { + // authMethod: ['token'], + // }, + // }, + // }, + + // Username and Password + { + displayName: 'Username', + name: 'username', + type: 'string', + default: '', + required: true, + noDataExpression: true, + placeholder: 'Username', + displayOptions: { + show: { + authMethod: ['usernameAndPassword'], + }, + }, + }, + { + displayName: 'Password', + name: 'password', + type: 'string', + default: '', + required: true, + noDataExpression: true, + placeholder: '***************', + typeOptions: { password: true }, + displayOptions: { + show: { + authMethod: ['usernameAndPassword'], + }, + }, + }, + + // Username and Password + { + displayName: 'Role ID', + name: 'roleId', + type: 'string', + default: '', + required: true, + noDataExpression: true, + placeholder: '59d6d1ca-47bb-4e7e-a40b-8be3bc5a0ba8', + displayOptions: { + show: { + authMethod: ['appRole'], + }, + }, + }, + { + displayName: 'Secret ID', + name: 'secretId', + type: 'string', + default: '', + required: true, + noDataExpression: true, + placeholder: '84896a0c-1347-aa90-a4f6-aca8b7558780', + typeOptions: { password: true }, + displayOptions: { + show: { + authMethod: ['appRole'], + }, + }, + }, + ]; + + displayName = 'HashiCorp Vault'; + + name = 'vault'; + + state: SecretsProviderState = 'initializing'; + + private cachedSecrets: Record = {}; + + private settings: VaultSettings; + + #currentToken: string | null = null; + + #tokenInfo: VaultTokenInfo | null = null; + + #http: AxiosInstance; + + private refreshTimeout: NodeJS.Timer | null; + + private refreshAbort = new AbortController(); + + async init(settings: SecretsProviderSettings): Promise { + this.settings = settings.settings as unknown as VaultSettings; + + const baseURL = new URL(this.settings.url); + + this.#http = axios.create({ baseURL: baseURL.toString() }); + if (this.settings.namespace) { + this.#http.interceptors.request.use((config) => { + return { + ...config, + // eslint-disable-next-line @typescript-eslint/naming-convention, @typescript-eslint/no-unsafe-assignment + headers: { ...config.headers, 'X-Vault-Namespace': this.settings.namespace }, + }; + }); + } + this.#http.interceptors.request.use((config) => { + if (!this.#currentToken) { + return config; + } + // eslint-disable-next-line @typescript-eslint/naming-convention, @typescript-eslint/no-unsafe-assignment + return { ...config, headers: { ...config.headers, 'X-Vault-Token': this.#currentToken } }; + }); + } + + async connect(): Promise { + if (this.settings.authMethod === 'token') { + this.#currentToken = this.settings.token; + } else if (this.settings.authMethod === 'usernameAndPassword') { + try { + this.#currentToken = await this.authUsernameAndPassword( + this.settings.username, + this.settings.password, + ); + } catch { + this.state = 'error'; + logger.error('Failed to connect to Vault using Username and Password credentials.'); + return; + } + } else if (this.settings.authMethod === 'appRole') { + try { + this.#currentToken = await this.authAppRole(this.settings.roleId, this.settings.secretId); + } catch { + this.state = 'error'; + logger.error('Failed to connect to Vault using AppRole credentials.'); + return; + } + } + try { + if (!(await this.test())[0]) { + this.state = 'error'; + } else { + this.state = 'connected'; + + [this.#tokenInfo] = await this.getTokenInfo(); + this.setupTokenRefresh(); + } + } catch (e) { + this.state = 'error'; + logger.error('Failed credentials test on Vault connect.'); + } + + try { + await this.update(); + } catch { + logger.warn('Failed to update Vault secrets'); + } + } + + async disconnect(): Promise { + if (this.refreshTimeout !== null) { + clearTimeout(this.refreshTimeout); + } + this.refreshAbort.abort(); + } + + private setupTokenRefresh() { + if (!this.#tokenInfo) { + return; + } + // Token never expires + if (this.#tokenInfo.expire_time === null) { + return; + } + // Token can't be renewed + if (!this.#tokenInfo.renewable) { + return; + } + + const expireDate = new Date(this.#tokenInfo.expire_time); + setTimeout(this.tokenRefresh, (expireDate.valueOf() - Date.now()) / 2); + } + + private tokenRefresh = async () => { + if (this.refreshAbort.signal.aborted) { + return; + } + try { + // We don't actually care about the result of this since it doesn't + // return an expire_time + await this.#http.post('auth/token/renew-self'); + + [this.#tokenInfo] = await this.getTokenInfo(); + + if (!this.#tokenInfo) { + logger.error('Failed to fetch token info during renewal. Cancelling all future renewals.'); + return; + } + + if (this.refreshAbort.signal.aborted) { + return; + } + + this.setupTokenRefresh(); + } catch { + logger.error('Failed to renew Vault token. Attempting to reconnect.'); + void this.connect(); + } + }; + + private async authUsernameAndPassword( + username: string, + password: string, + ): Promise { + try { + const resp = await this.#http.request({ + method: 'POST', + url: `auth/userpass/login/${username}`, + responseType: 'json', + data: { password }, + }); + + return resp.data.auth.client_token; + } catch { + return null; + } + } + + private async authAppRole(roleId: string, secretId: string): Promise { + try { + const resp = await this.#http.request({ + method: 'POST', + url: 'auth/approle/login', + responseType: 'json', + data: { role_id: roleId, secret_id: secretId }, + }); + + return resp.data.auth.client_token; + } catch (e) { + return null; + } + } + + private async getTokenInfo(): Promise<[VaultTokenInfo | null, AxiosResponse]> { + const resp = await this.#http.request>({ + method: 'GET', + url: 'auth/token/lookup-self', + responseType: 'json', + validateStatus: () => true, + }); + + if (resp.status !== 200 || !resp.data.data) { + return [null, resp]; + } + return [resp.data.data, resp]; + } + + private async getKVSecrets( + mountPath: string, + kvVersion: string, + path: string, + ): Promise<[string, IDataObject] | null> { + let listPath = mountPath; + if (kvVersion === '2') { + listPath += 'metadata/'; + } + listPath += path; + let listResp: AxiosResponse>; + try { + listResp = await this.#http.request>({ + url: listPath, + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-explicit-any + method: 'LIST' as any, + }); + } catch { + return null; + } + const data = Object.fromEntries( + ( + await Promise.allSettled( + listResp.data.data.keys.map(async (key): Promise<[string, IDataObject] | null> => { + if (key.endsWith('/')) { + return this.getKVSecrets(mountPath, kvVersion, path + key); + } + let secretPath = mountPath; + if (kvVersion === '2') { + secretPath += 'data/'; + } + secretPath += path + key; + try { + const secretResp = await this.#http.get>(secretPath); + return [ + key, + kvVersion === '2' + ? (secretResp.data.data.data as IDataObject) + : secretResp.data.data, + ]; + } catch { + return null; + } + }), + ) + ) + .map((i) => (i.status === 'rejected' ? null : i.value)) + .filter((v) => v !== null) as Array<[string, IDataObject]>, + ); + const name = path.substring(0, path.length - 1); + return [name, data]; + } + + async update(): Promise { + const mounts = await this.#http.get>('sys/mounts'); + + const kvs = Object.entries(mounts.data.data).filter(([, v]) => v.type === 'kv'); + + const secrets = Object.fromEntries( + ( + await Promise.all( + kvs.map(async ([basePath, data]): Promise<[string, IDataObject] | null> => { + const value = await this.getKVSecrets(basePath, data.options.version as string, ''); + if (value === null) { + return null; + } + return [basePath.substring(0, basePath.length - 1), value[1]]; + }), + ) + ).filter((v) => v !== null) as Array<[string, IDataObject]>, + ); + this.cachedSecrets = secrets; + } + + async test(): Promise<[boolean] | [boolean, string]> { + try { + const [token, tokenResp] = await this.getTokenInfo(); + + if (token === null) { + if (tokenResp.status === 404) { + return [false, 'Could not find auth path. Try adding /v1/ to the end of your base URL.']; + } + return [false, 'Invalid credentials']; + } + + const resp = await this.#http.request>({ + method: 'GET', + url: 'sys/mounts', + responseType: 'json', + validateStatus: () => true, + }); + + if (resp.status === 403) { + return [ + false, + "Couldn't list mounts. Please give these credentials 'read' access to sys/mounts.", + ]; + } else if (resp.status !== 200) { + return [ + false, + "Couldn't list mounts but wasn't a permissions issue. Please consult your Vault admin.", + ]; + } + + return [true]; + } catch (e) { + if (axios.isAxiosError(e)) { + if (e.code === 'ECONNREFUSED') { + return [ + false, + 'Connection refused. Please check the host and port of the server are correct.', + ]; + } + } + + return [false]; + } + } + + getSecret(name: string): IDataObject { + return this.cachedSecrets[name]; + } + + hasSecret(name: string): boolean { + return name in this.cachedSecrets; + } + + getSecretNames(): string[] { + const getKeys = ([k, v]: [string, IDataObject]): string[] => { + if (!EXTERNAL_SECRETS_NAME_REGEX.test(k)) { + return []; + } + if (typeof v === 'object') { + const keys: string[] = []; + for (const key of Object.keys(v)) { + if (!EXTERNAL_SECRETS_NAME_REGEX.test(key)) { + continue; + } + const value = v[key]; + if (typeof value === 'object' && value !== null) { + keys.push(...getKeys([key, value as IDataObject]).map((ok) => `${k}.${ok}`)); + } else { + keys.push(`${k}.${key}`); + } + } + return keys; + } + return [k]; + }; + return Object.entries(this.cachedSecrets).flatMap(getKeys); + } +} diff --git a/packages/cli/src/Interfaces.ts b/packages/cli/src/Interfaces.ts index 4de5209a0f..83c2664332 100644 --- a/packages/cli/src/Interfaces.ts +++ b/packages/cli/src/Interfaces.ts @@ -21,6 +21,7 @@ import type { ExecutionStatus, IExecutionsSummary, FeatureFlags, + INodeProperties, IUserSettings, IHttpRequestMethods, } from 'n8n-workflow'; @@ -460,6 +461,13 @@ export interface IInternalHooksClass { onApiKeyCreated(apiKeyDeletedData: { user: User; public_api: boolean }): Promise; onApiKeyDeleted(apiKeyDeletedData: { user: User; public_api: boolean }): Promise; onVariableCreated(createData: { variable_type: string }): Promise; + onExternalSecretsProviderSettingsSaved(saveData: { + user_id?: string; + vault_type: string; + is_valid: boolean; + is_new: boolean; + error_message?: string; + }): Promise; } export interface IVersionNotificationSettings { @@ -779,4 +787,35 @@ export interface N8nApp { export type UserSettings = Pick; +export interface SecretsProviderSettings { + connected: boolean; + connectedAt: Date | null; + settings: T; +} + +export interface ExternalSecretsSettings { + [key: string]: SecretsProviderSettings; +} + +export type SecretsProviderState = 'initializing' | 'connected' | 'error'; + +export abstract class SecretsProvider { + displayName: string; + + name: string; + + properties: INodeProperties[]; + + state: SecretsProviderState; + + abstract init(settings: SecretsProviderSettings): Promise; + abstract connect(): Promise; + abstract disconnect(): Promise; + abstract update(): Promise; + abstract test(): Promise<[boolean] | [boolean, string]>; + abstract getSecret(name: string): IDataObject | undefined; + abstract hasSecret(name: string): boolean; + abstract getSecretNames(): string[]; +} + export type N8nInstanceType = 'main' | 'webhook' | 'worker'; diff --git a/packages/cli/src/InternalHooks.ts b/packages/cli/src/InternalHooks.ts index ebd3bc2570..a099d6fd76 100644 --- a/packages/cli/src/InternalHooks.ts +++ b/packages/cli/src/InternalHooks.ts @@ -1086,4 +1086,14 @@ export class InternalHooks implements IInternalHooksClass { }): Promise { return this.telemetry.track('User finished push via UI', data); } + + async onExternalSecretsProviderSettingsSaved(saveData: { + user_id?: string | undefined; + vault_type: string; + is_valid: boolean; + is_new: boolean; + error_message?: string | undefined; + }): Promise { + return this.telemetry.track('User updated external secrets settings', saveData); + } } diff --git a/packages/cli/src/License.ts b/packages/cli/src/License.ts index 267419da9f..b296c3a333 100644 --- a/packages/cli/src/License.ts +++ b/packages/cli/src/License.ts @@ -140,6 +140,10 @@ export class License { return this.isFeatureEnabled(LICENSE_FEATURES.SOURCE_CONTROL); } + isExternalSecretsEnabled() { + return this.isFeatureEnabled(LICENSE_FEATURES.EXTERNAL_SECRETS); + } + isWorkflowHistoryLicensed() { return this.isFeatureEnabled(LICENSE_FEATURES.WORKFLOW_HISTORY); } diff --git a/packages/cli/src/SecretsHelpers.ts b/packages/cli/src/SecretsHelpers.ts new file mode 100644 index 0000000000..d90a0e84c8 --- /dev/null +++ b/packages/cli/src/SecretsHelpers.ts @@ -0,0 +1,41 @@ +import type { IDataObject, SecretsHelpersBase } from 'n8n-workflow'; +import { Service } from 'typedi'; +import { ExternalSecretsManager } from './ExternalSecrets/ExternalSecretsManager.ee'; + +@Service() +export class SecretsHelper implements SecretsHelpersBase { + constructor(private service: ExternalSecretsManager) {} + + async update() { + if (!this.service.initialized) { + await this.service.init(); + } + await this.service.updateSecrets(); + } + + async waitForInit() { + if (!this.service.initialized) { + await this.service.init(); + } + } + + getSecret(provider: string, name: string): IDataObject | undefined { + return this.service.getSecret(provider, name); + } + + hasSecret(provider: string, name: string): boolean { + return this.service.hasSecret(provider, name); + } + + hasProvider(provider: string): boolean { + return this.service.hasProvider(provider); + } + + listProviders(): string[] { + return this.service.getProviderNames() ?? []; + } + + listSecrets(provider: string): string[] { + return this.service.getSecretNames(provider) ?? []; + } +} diff --git a/packages/cli/src/Server.ts b/packages/cli/src/Server.ts index 372071ac19..ebc16f9a61 100644 --- a/packages/cli/src/Server.ts +++ b/packages/cli/src/Server.ts @@ -99,6 +99,7 @@ import { WorkflowStatisticsController, } from '@/controllers'; +import { ExternalSecretsController } from '@/ExternalSecrets/ExternalSecrets.controller.ee'; import { executionsController } from '@/executions/executions.controller'; import { isApiEnabled, loadPublicApiVersions } from '@/PublicApi'; import { @@ -163,6 +164,7 @@ import { isLdapCurrentAuthenticationMethod, isSamlCurrentAuthenticationMethod, } from './sso/ssoHelpers'; +import { isExternalSecretsEnabled } from './ExternalSecrets/externalSecretsHelper.ee'; import { isSourceControlLicensed } from '@/environments/sourceControl/sourceControlHelper.ee'; import { SourceControlService } from '@/environments/sourceControl/sourceControl.service.ee'; import { SourceControlController } from '@/environments/sourceControl/sourceControl.controller.ee'; @@ -314,6 +316,7 @@ export class Server extends AbstractServer { variables: false, sourceControl: false, auditLogs: false, + externalSecrets: false, showNonProdBanner: false, debugInEditor: false, }, @@ -451,6 +454,7 @@ export class Server extends AbstractServer { advancedExecutionFilters: isAdvancedExecutionFiltersEnabled(), variables: isVariablesEnabled(), sourceControl: isSourceControlLicensed(), + externalSecrets: isExternalSecretsEnabled(), showNonProdBanner: Container.get(License).isFeatureEnabled( LICENSE_FEATURES.SHOW_NON_PROD_BANNER, ), @@ -526,6 +530,7 @@ export class Server extends AbstractServer { Container.get(SamlController), Container.get(SourceControlController), Container.get(WorkflowStatisticsController), + Container.get(ExternalSecretsController), ]; if (isLdapEnabled()) { @@ -929,10 +934,13 @@ export class Server extends AbstractServer { throw new ResponseHelper.InternalServerError(error.message); } + const additionalData = await WorkflowExecuteAdditionalData.getBase(req.user.id); + const mode: WorkflowExecuteMode = 'internal'; const timezone = config.getEnv('generic.timezone'); const credentialsHelper = new CredentialsHelper(encryptionKey); const decryptedDataOriginal = await credentialsHelper.getDecrypted( + additionalData, credential as INodeCredentialsDetails, credential.type, mode, @@ -941,6 +949,7 @@ export class Server extends AbstractServer { ); const oauthCredentials = credentialsHelper.applyDefaultsAndOverwrites( + additionalData, decryptedDataOriginal, credential.type, mode, @@ -1075,10 +1084,13 @@ export class Server extends AbstractServer { throw new ResponseHelper.InternalServerError(error.message); } + const additionalData = await WorkflowExecuteAdditionalData.getBase(req.user.id); + const mode: WorkflowExecuteMode = 'internal'; const timezone = config.getEnv('generic.timezone'); const credentialsHelper = new CredentialsHelper(encryptionKey); const decryptedDataOriginal = await credentialsHelper.getDecrypted( + additionalData, credential as INodeCredentialsDetails, credential.type, mode, @@ -1086,6 +1098,7 @@ export class Server extends AbstractServer { true, ); const oauthCredentials = credentialsHelper.applyDefaultsAndOverwrites( + additionalData, decryptedDataOriginal, credential.type, mode, diff --git a/packages/cli/src/WorkflowExecuteAdditionalData.ts b/packages/cli/src/WorkflowExecuteAdditionalData.ts index 4c8dacf76b..5760cce3d1 100644 --- a/packages/cli/src/WorkflowExecuteAdditionalData.ts +++ b/packages/cli/src/WorkflowExecuteAdditionalData.ts @@ -65,6 +65,7 @@ import { InternalHooks } from '@/InternalHooks'; import type { ExecutionMetadata } from '@db/entities/ExecutionMetadata'; import { ExecutionRepository } from '@db/repositories'; import { EventsService } from '@/services/events.service'; +import { SecretsHelper } from './SecretsHelpers'; import { OwnershipService } from './services/ownership.service'; const ERROR_TRIGGER_TYPE = config.getEnv('nodes.errorTriggerType'); @@ -1167,6 +1168,7 @@ export async function getBase( userId, setExecutionStatus, variables, + secretsHelpers: Container.get(SecretsHelper), }; } diff --git a/packages/cli/src/commands/BaseCommand.ts b/packages/cli/src/commands/BaseCommand.ts index abf7257eda..78be7cc23f 100644 --- a/packages/cli/src/commands/BaseCommand.ts +++ b/packages/cli/src/commands/BaseCommand.ts @@ -20,6 +20,7 @@ import type { IExternalHooksClass } from '@/Interfaces'; import { InternalHooks } from '@/InternalHooks'; import { PostHogClient } from '@/posthog'; import { License } from '@/License'; +import { ExternalSecretsManager } from '@/ExternalSecrets/ExternalSecretsManager.ee'; export abstract class BaseCommand extends Command { protected logger = LoggerProxy.init(getLogger()); @@ -134,6 +135,11 @@ export abstract class BaseCommand extends Command { } } + async initExternalSecrets() { + const secretsManager = Container.get(ExternalSecretsManager); + await secretsManager.init(); + } + async finally(error: Error | undefined) { if (inTest || this.id === 'start') return; if (Db.connectionState.connected) { diff --git a/packages/cli/src/commands/start.ts b/packages/cli/src/commands/start.ts index 84afe4770c..a21a93e21f 100644 --- a/packages/cli/src/commands/start.ts +++ b/packages/cli/src/commands/start.ts @@ -195,6 +195,7 @@ export class Start extends BaseCommand { await this.initLicense(); await this.initBinaryManager(); await this.initExternalHooks(); + await this.initExternalSecrets(); if (!config.getEnv('endpoints.disableUi')) { await this.generateStaticAssets(); diff --git a/packages/cli/src/commands/webhook.ts b/packages/cli/src/commands/webhook.ts index c627d0d328..7d5bf46302 100644 --- a/packages/cli/src/commands/webhook.ts +++ b/packages/cli/src/commands/webhook.ts @@ -80,6 +80,7 @@ export class Webhook extends BaseCommand { await this.initLicense(); await this.initBinaryManager(); await this.initExternalHooks(); + await this.initExternalSecrets(); } async run() { diff --git a/packages/cli/src/commands/worker.ts b/packages/cli/src/commands/worker.ts index 5702dc2d35..dfaf0e34c0 100644 --- a/packages/cli/src/commands/worker.ts +++ b/packages/cli/src/commands/worker.ts @@ -239,6 +239,7 @@ export class Worker extends BaseCommand { await this.initLicense(); await this.initBinaryManager(); await this.initExternalHooks(); + await this.initExternalSecrets(); } async run() { diff --git a/packages/cli/src/constants.ts b/packages/cli/src/constants.ts index 88808c0491..8d9bbb9e14 100644 --- a/packages/cli/src/constants.ts +++ b/packages/cli/src/constants.ts @@ -77,6 +77,7 @@ export const LICENSE_FEATURES = { VARIABLES: 'feat:variables', SOURCE_CONTROL: 'feat:sourceControl', API_DISABLED: 'feat:apiDisabled', + EXTERNAL_SECRETS: 'feat:externalSecrets', SHOW_NON_PROD_BANNER: 'feat:showNonProdBanner', WORKFLOW_HISTORY: 'feat:workflowHistory', DEBUG_IN_EDITOR: 'feat:debugInEditor', diff --git a/packages/cli/src/controllers/e2e.controller.ts b/packages/cli/src/controllers/e2e.controller.ts index d58e21f4f4..57ec0316d4 100644 --- a/packages/cli/src/controllers/e2e.controller.ts +++ b/packages/cli/src/controllers/e2e.controller.ts @@ -64,6 +64,7 @@ export class E2EController { [LICENSE_FEATURES.SOURCE_CONTROL]: false, [LICENSE_FEATURES.VARIABLES]: false, [LICENSE_FEATURES.API_DISABLED]: false, + [LICENSE_FEATURES.EXTERNAL_SECRETS]: false, [LICENSE_FEATURES.SHOW_NON_PROD_BANNER]: false, [LICENSE_FEATURES.WORKFLOW_HISTORY]: false, [LICENSE_FEATURES.DEBUG_IN_EDITOR]: false, diff --git a/packages/cli/src/credentials/credentials.service.ts b/packages/cli/src/credentials/credentials.service.ts index 94be5b9fb6..0d6c67c554 100644 --- a/packages/cli/src/credentials/credentials.service.ts +++ b/packages/cli/src/credentials/credentials.service.ts @@ -309,7 +309,10 @@ export class CredentialsService { if (!prop) { continue; } - if (prop.typeOptions?.password) { + if ( + prop.typeOptions?.password && + (!(copiedData[dataKey] as string).startsWith('={{') || prop.noDataExpression) + ) { if (copiedData[dataKey].toString().length > 0) { copiedData[dataKey] = CREDENTIAL_BLANKING_VALUE; } else { diff --git a/packages/cli/src/credentials/oauth2Credential.api.ts b/packages/cli/src/credentials/oauth2Credential.api.ts index bcb3892ddf..dda9b486e9 100644 --- a/packages/cli/src/credentials/oauth2Credential.api.ts +++ b/packages/cli/src/credentials/oauth2Credential.api.ts @@ -34,6 +34,8 @@ import config from '@/config'; import { getInstanceBaseUrl } from '@/UserManagement/UserManagementHelper'; import { Container } from 'typedi'; +import * as WorkflowExecuteAdditionalData from '@/WorkflowExecuteAdditionalData'; + export const oauth2CredentialController = express.Router(); /** @@ -81,12 +83,15 @@ oauth2CredentialController.get( throw new ResponseHelper.InternalServerError((error as Error).message); } + const additionalData = await WorkflowExecuteAdditionalData.getBase(req.user.id); + const credentialType = (credential as unknown as ICredentialsEncrypted).type; const mode: WorkflowExecuteMode = 'internal'; const timezone = config.getEnv('generic.timezone'); const credentialsHelper = new CredentialsHelper(encryptionKey); const decryptedDataOriginal = await credentialsHelper.getDecrypted( + additionalData, credential as INodeCredentialsDetails, credentialType, mode, @@ -107,6 +112,7 @@ oauth2CredentialController.get( } const oauthCredentials = credentialsHelper.applyDefaultsAndOverwrites( + additionalData, decryptedDataOriginal, credentialType, mode, @@ -223,11 +229,13 @@ oauth2CredentialController.get( } const encryptionKey = await UserSettings.getEncryptionKey(); + const additionalData = await WorkflowExecuteAdditionalData.getBase(state.cid); const mode: WorkflowExecuteMode = 'internal'; const timezone = config.getEnv('generic.timezone'); const credentialsHelper = new CredentialsHelper(encryptionKey); const decryptedDataOriginal = await credentialsHelper.getDecrypted( + additionalData, credential as INodeCredentialsDetails, (credential as unknown as ICredentialsEncrypted).type, mode, @@ -235,6 +243,7 @@ oauth2CredentialController.get( true, ); const oauthCredentials = credentialsHelper.applyDefaultsAndOverwrites( + additionalData, decryptedDataOriginal, (credential as unknown as ICredentialsEncrypted).type, mode, diff --git a/packages/cli/src/databases/repositories/settings.repository.ts b/packages/cli/src/databases/repositories/settings.repository.ts index 82630bffdb..a213ee78a3 100644 --- a/packages/cli/src/databases/repositories/settings.repository.ts +++ b/packages/cli/src/databases/repositories/settings.repository.ts @@ -1,3 +1,4 @@ +import { EXTERNAL_SECRETS_DB_KEY } from '@/ExternalSecrets/constants'; import { Service } from 'typedi'; import { DataSource, Repository } from 'typeorm'; import { ErrorReporterProxy as ErrorReporter } from 'n8n-workflow'; @@ -10,6 +11,21 @@ export class SettingsRepository extends Repository { super(Settings, dataSource.manager); } + async getEncryptedSecretsProviderSettings(): Promise { + return (await this.findOne({ where: { key: EXTERNAL_SECRETS_DB_KEY } }))?.value ?? null; + } + + async saveEncryptedSecretsProviderSettings(data: string): Promise { + await this.upsert( + { + key: EXTERNAL_SECRETS_DB_KEY, + value: data, + loadOnStartup: false, + }, + ['key'], + ); + } + async dismissBanner({ bannerName }: { bannerName: string }): Promise<{ success: boolean }> { const key = 'ui.banners.dismissed'; const dismissedBannersSetting = await this.findOneBy({ key }); diff --git a/packages/cli/src/eventbus/MessageEventBusDestination/MessageEventBusDestinationWebhook.ee.ts b/packages/cli/src/eventbus/MessageEventBusDestination/MessageEventBusDestinationWebhook.ee.ts index 2a97f4471d..6af27c1b76 100644 --- a/packages/cli/src/eventbus/MessageEventBusDestination/MessageEventBusDestinationWebhook.ee.ts +++ b/packages/cli/src/eventbus/MessageEventBusDestination/MessageEventBusDestinationWebhook.ee.ts @@ -15,6 +15,7 @@ import type { MessageEventBusDestinationOptions, MessageEventBusDestinationWebhookParameterItem, MessageEventBusDestinationWebhookParameterOptions, + IWorkflowExecuteAdditionalData, } from 'n8n-workflow'; import { CredentialsHelper } from '@/CredentialsHelper'; import { UserSettings } from 'n8n-core'; @@ -24,6 +25,7 @@ import { isLogStreamingEnabled } from '../MessageEventBus/MessageEventBusHelper' import { eventMessageGenericDestinationTestEvent } from '../EventMessageClasses/EventMessageGeneric'; import { MessageEventBus } from '../MessageEventBus/MessageEventBus'; import type { MessageWithCallback } from '../MessageEventBus/MessageEventBus'; +import * as SecretsHelpers from '@/ExternalSecrets/externalSecretsHelper.ee'; export const isMessageEventBusDestinationWebhookOptions = ( candidate: unknown, @@ -108,6 +110,7 @@ export class MessageEventBusDestinationWebhook if (foundCredential) { const timezone = config.getEnv('generic.timezone'); const credentialsDecrypted = await this.credentialsHelper?.getDecrypted( + { secretsHelpers: SecretsHelpers } as unknown as IWorkflowExecuteAdditionalData, foundCredential[1], foundCredential[0], 'internal', diff --git a/packages/cli/src/requests.ts b/packages/cli/src/requests.ts index ecee712f22..fea69cd652 100644 --- a/packages/cli/src/requests.ts +++ b/packages/cli/src/requests.ts @@ -4,6 +4,7 @@ import type { IConnections, ICredentialDataDecryptedObject, ICredentialNodeAccess, + IDataObject, INode, INodeCredentialTestRequest, IPinData, @@ -14,7 +15,13 @@ import type { import { IsBoolean, IsEmail, IsOptional, IsString, Length } from 'class-validator'; import { NoXss } from '@db/utils/customValidators'; -import type { PublicUser, IExecutionDeleteFilter, IWorkflowDb } from '@/Interfaces'; +import type { + PublicUser, + IExecutionDeleteFilter, + IWorkflowDb, + SecretsProvider, + SecretsProviderState, +} from '@/Interfaces'; import type { Role } from '@db/entities/Role'; import type { User } from '@db/entities/User'; import type { UserManagementMailer } from '@/UserManagement/email'; @@ -497,3 +504,25 @@ export declare namespace VariablesRequest { type Update = AuthenticatedRequest<{ id: string }, {}, CreateUpdatePayload, {}>; type Delete = Get; } + +export declare namespace ExternalSecretsRequest { + type GetProviderResponse = Pick & { + icon: string; + connected: boolean; + connectedAt: Date | null; + state: SecretsProviderState; + data: IDataObject; + }; + + type GetProviders = AuthenticatedRequest; + type GetProvider = AuthenticatedRequest<{ provider: string }, GetProviderResponse>; + type SetProviderSettings = AuthenticatedRequest<{ provider: string }, {}, IDataObject>; + type TestProviderSettings = SetProviderSettings; + type SetProviderConnected = AuthenticatedRequest< + { provider: string }, + {}, + { connected: boolean } + >; + + type UpdateProvider = AuthenticatedRequest<{ provider: string }>; +} diff --git a/packages/cli/test/integration/ExternalSecrets/externalSecrets.api.test.ts b/packages/cli/test/integration/ExternalSecrets/externalSecrets.api.test.ts new file mode 100644 index 0000000000..8ea55e7e73 --- /dev/null +++ b/packages/cli/test/integration/ExternalSecrets/externalSecrets.api.test.ts @@ -0,0 +1,370 @@ +import type { SuperAgentTest } from 'supertest'; +import { License } from '@/License'; +import * as testDb from '../shared/testDb'; +import * as utils from '../shared/utils/'; +import type { ExternalSecretsSettings, SecretsProviderState } from '@/Interfaces'; +import { UserSettings } from 'n8n-core'; +import { SettingsRepository } from '@/databases/repositories/settings.repository'; +import Container from 'typedi'; +import { AES, enc } from 'crypto-js'; +import { ExternalSecretsProviders } from '@/ExternalSecrets/ExternalSecretsProviders.ee'; +import { + DummyProvider, + FailedProvider, + MockProviders, + TestFailProvider, +} from '../../shared/ExternalSecrets/utils'; +import config from '@/config'; +import { ExternalSecretsManager } from '@/ExternalSecrets/ExternalSecretsManager.ee'; +import { CREDENTIAL_BLANKING_VALUE } from '@/constants'; +import type { IDataObject } from 'n8n-workflow'; + +let authOwnerAgent: SuperAgentTest; +let authMemberAgent: SuperAgentTest; + +const licenseLike = utils.mockInstance(License, { + isExternalSecretsEnabled: jest.fn().mockReturnValue(true), + isWithinUsersLimit: jest.fn().mockReturnValue(true), +}); + +const mockProvidersInstance = new MockProviders(); +let providersMock: ExternalSecretsProviders = utils.mockInstance( + ExternalSecretsProviders, + mockProvidersInstance, +); + +const testServer = utils.setupTestServer({ endpointGroups: ['externalSecrets'] }); + +const connectedDate = '2023-08-01T12:32:29.000Z'; + +async function setExternalSecretsSettings(settings: ExternalSecretsSettings) { + const encryptionKey = await UserSettings.getEncryptionKey(); + return Container.get(SettingsRepository).saveEncryptedSecretsProviderSettings( + AES.encrypt(JSON.stringify(settings), encryptionKey).toString(), + ); +} + +async function getExternalSecretsSettings(): Promise { + const encryptionKey = await UserSettings.getEncryptionKey(); + const encSettings = await Container.get(SettingsRepository).getEncryptedSecretsProviderSettings(); + if (encSettings === null) { + return null; + } + return JSON.parse(AES.decrypt(encSettings, encryptionKey).toString(enc.Utf8)); +} + +const resetManager = async () => { + Container.get(ExternalSecretsManager).shutdown(); + Container.set( + ExternalSecretsManager, + new ExternalSecretsManager( + Container.get(SettingsRepository), + licenseLike, + mockProvidersInstance, + ), + ); + + await Container.get(ExternalSecretsManager).init(); +}; + +const getDummyProviderData = ({ + data, + includeProperties, + connected, + state, + connectedAt, + displayName, +}: { + data?: IDataObject; + includeProperties?: boolean; + connected?: boolean; + state?: SecretsProviderState; + connectedAt?: string | null; + displayName?: string; +} = {}) => { + const dummy: IDataObject = { + connected: connected ?? true, + connectedAt: connectedAt === undefined ? connectedDate : connectedAt, + data: data ?? {}, + name: 'dummy', + displayName: displayName ?? 'Dummy Provider', + icon: 'dummy', + state: state ?? 'connected', + }; + + if (includeProperties) { + dummy.properties = new DummyProvider().properties; + } + + return dummy; +}; + +beforeAll(async () => { + await utils.initEncryptionKey(); + + const owner = await testDb.createOwner(); + authOwnerAgent = testServer.authAgentFor(owner); + const member = await testDb.createUser(); + authMemberAgent = testServer.authAgentFor(member); + config.set('userManagement.isInstanceOwnerSetUp', true); +}); + +beforeEach(async () => { + licenseLike.isExternalSecretsEnabled.mockReturnValue(true); + + mockProvidersInstance.setProviders({ + dummy: DummyProvider, + }); + + await setExternalSecretsSettings({ + dummy: { + connected: true, + connectedAt: new Date(connectedDate), + settings: {}, + }, + }); + + await resetManager(); +}); + +afterEach(async () => { + Container.get(ExternalSecretsManager).shutdown(); +}); + +describe('GET /external-secrets/providers', () => { + test('can retrieve providers as owner', async () => { + const resp = await authOwnerAgent.get('/external-secrets/providers'); + expect(resp.body).toEqual({ + data: [getDummyProviderData()], + }); + }); + + test('can not retrieve providers as non-owner', async () => { + const resp = await authMemberAgent.get('/external-secrets/providers'); + expect(resp.status).toBe(403); + }); + + test('does obscure passwords', async () => { + await setExternalSecretsSettings({ + dummy: { + connected: true, + connectedAt: new Date(connectedDate), + settings: { + username: 'testuser', + password: 'testpass', + }, + }, + }); + + await resetManager(); + + const resp = await authOwnerAgent.get('/external-secrets/providers'); + expect(resp.body).toEqual({ + data: [ + getDummyProviderData({ + data: { + username: 'testuser', + password: CREDENTIAL_BLANKING_VALUE, + }, + }), + ], + }); + }); +}); + +describe('GET /external-secrets/providers/:provider', () => { + test('can retrieve provider as owner', async () => { + const resp = await authOwnerAgent.get('/external-secrets/providers/dummy'); + expect(resp.body.data).toEqual(getDummyProviderData({ includeProperties: true })); + }); + + test('can not retrieve provider as non-owner', async () => { + const resp = await authMemberAgent.get('/external-secrets/providers/dummy'); + expect(resp.status).toBe(403); + }); + + test('does obscure passwords', async () => { + await setExternalSecretsSettings({ + dummy: { + connected: true, + connectedAt: new Date(connectedDate), + settings: { + username: 'testuser', + password: 'testpass', + }, + }, + }); + + await resetManager(); + + const resp = await authOwnerAgent.get('/external-secrets/providers/dummy'); + expect(resp.body.data).toEqual( + getDummyProviderData({ + data: { + username: 'testuser', + password: CREDENTIAL_BLANKING_VALUE, + }, + includeProperties: true, + }), + ); + }); +}); + +describe('POST /external-secrets/providers/:provider', () => { + test('can update provider settings', async () => { + const testData = { + username: 'testuser', + other: 'testother', + }; + const resp = await authOwnerAgent.post('/external-secrets/providers/dummy').send(testData); + expect(resp.status).toBe(200); + + const confirmResp = await authOwnerAgent.get('/external-secrets/providers/dummy'); + expect(confirmResp.body.data).toEqual( + getDummyProviderData({ data: testData, includeProperties: true }), + ); + }); + + test('can update provider settings with blanking value', async () => { + await setExternalSecretsSettings({ + dummy: { + connected: true, + connectedAt: new Date(connectedDate), + settings: { + username: 'testuser', + password: 'testpass', + }, + }, + }); + + await resetManager(); + + const testData = { + username: 'newuser', + password: CREDENTIAL_BLANKING_VALUE, + }; + const resp = await authOwnerAgent.post('/external-secrets/providers/dummy').send(testData); + expect(resp.status).toBe(200); + + const confirmResp = await authOwnerAgent.get('/external-secrets/providers/dummy'); + expect((await getExternalSecretsSettings())?.dummy.settings).toEqual({ + username: 'newuser', + password: 'testpass', + }); + }); +}); + +describe('POST /external-secrets/providers/:provider/connect', () => { + test('can change provider connected state', async () => { + const testData = { + connected: false, + }; + const resp = await authOwnerAgent + .post('/external-secrets/providers/dummy/connect') + .send(testData); + expect(resp.status).toBe(200); + + const confirmResp = await authOwnerAgent.get('/external-secrets/providers/dummy'); + expect(confirmResp.body.data).toEqual( + getDummyProviderData({ + includeProperties: true, + connected: false, + state: 'initializing', + }), + ); + }); +}); + +describe('POST /external-secrets/providers/:provider/test', () => { + test('can test provider', async () => { + const testData = { + username: 'testuser', + other: 'testother', + }; + const resp = await authOwnerAgent.post('/external-secrets/providers/dummy/test').send(testData); + expect(resp.status).toBe(200); + expect(resp.body.data.success).toBe(true); + expect(resp.body.data.testState).toBe('connected'); + }); + + test('can test provider fail', async () => { + mockProvidersInstance.setProviders({ + dummy: TestFailProvider, + }); + + await resetManager(); + + const testData = { + username: 'testuser', + other: 'testother', + }; + const resp = await authOwnerAgent.post('/external-secrets/providers/dummy/test').send(testData); + expect(resp.status).toBe(400); + expect(resp.body.data.success).toBe(false); + expect(resp.body.data.testState).toBe('error'); + }); +}); + +describe('POST /external-secrets/providers/:provider/update', () => { + test('can update provider', async () => { + const updateSpy = jest.spyOn( + Container.get(ExternalSecretsManager).getProvider('dummy')!, + 'update', + ); + + const resp = await authOwnerAgent.post('/external-secrets/providers/dummy/update'); + expect(resp.status).toBe(200); + expect(resp.body.data).toEqual({ updated: true }); + expect(updateSpy).toBeCalled(); + }); + + test('can not update errored provider', async () => { + mockProvidersInstance.setProviders({ + dummy: FailedProvider, + }); + + await resetManager(); + + const updateSpy = jest.spyOn( + Container.get(ExternalSecretsManager).getProvider('dummy')!, + 'update', + ); + + const resp = await authOwnerAgent.post('/external-secrets/providers/dummy/update'); + expect(resp.status).toBe(400); + expect(resp.body.data).toEqual({ updated: false }); + expect(updateSpy).not.toBeCalled(); + }); + + test('can not update provider without a valid license', async () => { + const updateSpy = jest.spyOn( + Container.get(ExternalSecretsManager).getProvider('dummy')!, + 'update', + ); + + licenseLike.isExternalSecretsEnabled.mockReturnValue(false); + + const resp = await authOwnerAgent.post('/external-secrets/providers/dummy/update'); + expect(resp.status).toBe(400); + expect(resp.body.data).toEqual({ updated: false }); + expect(updateSpy).not.toBeCalled(); + }); +}); + +describe('GET /external-secrets/secrets', () => { + test('can get secret names as owner', async () => { + const resp = await authOwnerAgent.get('/external-secrets/secrets'); + expect(resp.status).toBe(200); + expect(resp.body.data).toEqual({ + dummy: ['test1', 'test2'], + }); + }); + + test('can not get secret names as non-owner', async () => { + const resp = await authMemberAgent.get('/external-secrets/secrets'); + expect(resp.status).toBe(403); + expect(resp.body.data).not.toEqual({ + dummy: ['test1', 'test2'], + }); + }); +}); diff --git a/packages/cli/test/integration/shared/types.ts b/packages/cli/test/integration/shared/types.ts index b1fa62a11e..1e4e20c6f0 100644 --- a/packages/cli/test/integration/shared/types.ts +++ b/packages/cli/test/integration/shared/types.ts @@ -26,6 +26,7 @@ export type EndpointGroup = | 'license' | 'variables' | 'tags' + | 'externalSecrets' | 'mfa' | 'metrics'; diff --git a/packages/cli/test/integration/shared/utils/testServer.ts b/packages/cli/test/integration/shared/utils/testServer.ts index c6ed5426c9..26fa2b81a0 100644 --- a/packages/cli/test/integration/shared/utils/testServer.ts +++ b/packages/cli/test/integration/shared/utils/testServer.ts @@ -50,6 +50,7 @@ import * as testDb from '../../shared/testDb'; import { AUTHLESS_ENDPOINTS, PUBLIC_API_REST_PATH_SEGMENT, REST_PATH_SEGMENT } from '../constants'; import type { EndpointGroup, SetupProps, TestServer } from '../types'; import { mockInstance } from './mocking'; +import { ExternalSecretsController } from '@/ExternalSecrets/ExternalSecrets.controller.ee'; import { MfaService } from '@/Mfa/mfa.service'; import { TOTPService } from '@/Mfa/totp.service'; import { UserSettings } from 'n8n-core'; @@ -285,6 +286,9 @@ export const setupTestServer = ({ case 'tags': registerController(app, config, Container.get(TagsController)); break; + case 'externalSecrets': + registerController(app, config, Container.get(ExternalSecretsController)); + break; } } } diff --git a/packages/cli/test/shared/ExternalSecrets/utils.ts b/packages/cli/test/shared/ExternalSecrets/utils.ts new file mode 100644 index 0000000000..068e22576e --- /dev/null +++ b/packages/cli/test/shared/ExternalSecrets/utils.ts @@ -0,0 +1,215 @@ +import { SecretsProvider } from '@/Interfaces'; +import type { SecretsProviderSettings, SecretsProviderState } from '@/Interfaces'; +import type { IDataObject, INodeProperties } from 'n8n-workflow'; + +export class MockProviders { + providers: Record = { + dummy: DummyProvider, + }; + + setProviders(providers: Record) { + this.providers = providers; + } + + getProvider(name: string): { new (): SecretsProvider } | null { + return this.providers[name] ?? null; + } + + hasProvider(name: string) { + return name in this.providers; + } + + getAllProviders() { + return this.providers; + } +} + +export class DummyProvider extends SecretsProvider { + properties: INodeProperties[] = [ + { + name: 'username', + displayName: 'Username', + type: 'string', + default: '', + required: true, + }, + { + name: 'other', + displayName: 'Other', + type: 'string', + default: '', + }, + { + name: 'password', + displayName: 'Password', + type: 'string', + default: '', + typeOptions: { + password: true, + }, + }, + ]; + + secrets: Record = {}; + + displayName = 'Dummy Provider'; + + name = 'dummy'; + + state: SecretsProviderState = 'initializing'; + + _updateSecrets: Record = { + test1: 'value1', + test2: 'value2', + }; + + async init(settings: SecretsProviderSettings): Promise {} + + async connect(): Promise { + this.state = 'connected'; + } + + async disconnect(): Promise {} + + async update(): Promise { + this.secrets = this._updateSecrets; + } + + async test(): Promise<[boolean] | [boolean, string]> { + return [true]; + } + + getSecret(name: string): IDataObject | undefined { + return this.secrets[name] as unknown as IDataObject | undefined; + } + + hasSecret(name: string): boolean { + return name in this.secrets; + } + + getSecretNames(): string[] { + return Object.keys(this.secrets); + } +} + +export class ErrorProvider extends SecretsProvider { + secrets: Record = {}; + + displayName = 'Error Provider'; + + name = 'dummy'; + + state: SecretsProviderState = 'initializing'; + + async init(settings: SecretsProviderSettings): Promise { + throw new Error(); + } + + async connect(): Promise { + this.state = 'error'; + throw new Error(); + } + + async disconnect(): Promise { + throw new Error(); + } + + async update(): Promise { + throw new Error(); + } + + async test(): Promise<[boolean] | [boolean, string]> { + throw new Error(); + } + + getSecret(name: string): IDataObject | undefined { + throw new Error(); + } + + hasSecret(name: string): boolean { + throw new Error(); + } + + getSecretNames(): string[] { + throw new Error(); + } +} + +export class FailedProvider extends SecretsProvider { + secrets: Record = {}; + + displayName = 'Failed Provider'; + + name = 'dummy'; + + state: SecretsProviderState = 'initializing'; + + async init(settings: SecretsProviderSettings): Promise {} + + async connect(): Promise { + this.state = 'error'; + } + + async disconnect(): Promise {} + + async update(): Promise {} + + async test(): Promise<[boolean] | [boolean, string]> { + return [true]; + } + + getSecret(name: string): IDataObject | undefined { + return this.secrets[name] as unknown as IDataObject | undefined; + } + + hasSecret(name: string): boolean { + return name in this.secrets; + } + + getSecretNames(): string[] { + return Object.keys(this.secrets); + } +} + +export class TestFailProvider extends SecretsProvider { + secrets: Record = {}; + + displayName = 'Test Failed Provider'; + + name = 'dummy'; + + state: SecretsProviderState = 'initializing'; + + _updateSecrets: Record = { + test1: 'value1', + test2: 'value2', + }; + + async init(settings: SecretsProviderSettings): Promise {} + + async connect(): Promise { + this.state = 'connected'; + } + + async disconnect(): Promise {} + + async update(): Promise { + this.secrets = this._updateSecrets; + } + + async test(): Promise<[boolean] | [boolean, string]> { + return [false]; + } + + getSecret(name: string): IDataObject | undefined { + return this.secrets[name] as unknown as IDataObject | undefined; + } + + hasSecret(name: string): boolean { + return name in this.secrets; + } + + getSecretNames(): string[] { + return Object.keys(this.secrets); + } +} diff --git a/packages/cli/test/unit/ActiveWorkflowRunner.test.ts b/packages/cli/test/unit/ActiveWorkflowRunner.test.ts index c9b3e6ad09..0622dac9a2 100644 --- a/packages/cli/test/unit/ActiveWorkflowRunner.test.ts +++ b/packages/cli/test/unit/ActiveWorkflowRunner.test.ts @@ -24,6 +24,7 @@ import { mockInstance } from '../integration/shared/utils/'; import { Push } from '@/push'; import { ActiveExecutions } from '@/ActiveExecutions'; import { NodeTypes } from '@/NodeTypes'; +import { SecretsHelper } from '@/SecretsHelpers'; import { WebhookService } from '@/services/webhook.service'; import { VariablesService } from '../../src/environments/variables/variables.service'; @@ -159,6 +160,7 @@ describe('ActiveWorkflowRunner', () => { Container.set(LoadNodesAndCredentials, nodesAndCredentials); Container.set(VariablesService, mockVariablesService); mockInstance(Push); + mockInstance(SecretsHelper); }); beforeEach(() => { diff --git a/packages/cli/test/unit/ExternalSecrets/ExternalSecretsManager.test.ts b/packages/cli/test/unit/ExternalSecrets/ExternalSecretsManager.test.ts new file mode 100644 index 0000000000..0789b87d67 --- /dev/null +++ b/packages/cli/test/unit/ExternalSecrets/ExternalSecretsManager.test.ts @@ -0,0 +1,194 @@ +import type { SettingsRepository } from '@/databases/repositories'; +import type { ExternalSecretsSettings } from '@/Interfaces'; +import { License } from '@/License'; +import { ExternalSecretsManager } from '@/ExternalSecrets/ExternalSecretsManager.ee'; +import { ExternalSecretsProviders } from '@/ExternalSecrets/ExternalSecretsProviders.ee'; +import { mock } from 'jest-mock-extended'; +import { UserSettings } from 'n8n-core'; +import Container from 'typedi'; +import { mockInstance } from '../../integration/shared/utils'; +import { + DummyProvider, + ErrorProvider, + FailedProvider, + MockProviders, +} from '../../shared/ExternalSecrets/utils'; +import { AES, enc } from 'crypto-js'; +import { InternalHooks } from '@/InternalHooks'; + +const connectedDate = '2023-08-01T12:32:29.000Z'; +const encryptionKey = 'testkey'; +let settings: string | null = null; +const mockProvidersInstance = new MockProviders(); +const settingsRepo = mock({ + async getEncryptedSecretsProviderSettings() { + return settings; + }, + async saveEncryptedSecretsProviderSettings(data) { + settings = data; + }, +}); +let licenseMock: License; +let providersMock: ExternalSecretsProviders; +let manager: ExternalSecretsManager | undefined; + +const createMockSettings = (settings: ExternalSecretsSettings): string => { + return AES.encrypt(JSON.stringify(settings), encryptionKey).toString(); +}; + +const decryptSettings = (settings: string) => { + return JSON.parse(AES.decrypt(settings ?? '', encryptionKey).toString(enc.Utf8)); +}; + +describe('External Secrets Manager', () => { + beforeAll(() => { + jest + .spyOn(UserSettings, 'getEncryptionKey') + .mockReturnValue(new Promise((resolve) => resolve(encryptionKey))); + providersMock = mockInstance(ExternalSecretsProviders, mockProvidersInstance); + licenseMock = mockInstance(License, { + isExternalSecretsEnabled() { + return true; + }, + }); + mockInstance(InternalHooks); + }); + + beforeEach(() => { + mockProvidersInstance.setProviders({ + dummy: DummyProvider, + }); + settings = createMockSettings({ + dummy: { connected: true, connectedAt: new Date(connectedDate), settings: {} }, + }); + + Container.remove(ExternalSecretsManager); + }); + + afterEach(() => { + manager?.shutdown(); + jest.useRealTimers(); + }); + + test('should get secret', async () => { + manager = new ExternalSecretsManager(settingsRepo, licenseMock, providersMock); + + await manager.init(); + + expect(manager.getSecret('dummy', 'test1')).toBe('value1'); + }); + + test('should not throw errors during init', async () => { + mockProvidersInstance.setProviders({ + dummy: ErrorProvider, + }); + manager = new ExternalSecretsManager(settingsRepo, licenseMock, providersMock); + + expect(async () => manager!.init()).not.toThrow(); + }); + + test('should not throw errors during shutdown', async () => { + mockProvidersInstance.setProviders({ + dummy: ErrorProvider, + }); + manager = new ExternalSecretsManager(settingsRepo, licenseMock, providersMock); + + await manager.init(); + expect(() => manager!.shutdown()).not.toThrow(); + manager = undefined; + }); + + test('should save provider settings', async () => { + manager = new ExternalSecretsManager(settingsRepo, licenseMock, providersMock); + + const settingsSpy = jest.spyOn(settingsRepo, 'saveEncryptedSecretsProviderSettings'); + + await manager.init(); + + await manager.setProviderSettings('dummy', { + test: 'value', + }); + + expect(decryptSettings(settingsSpy.mock.calls[0][0])).toEqual({ + dummy: { + connected: true, + connectedAt: connectedDate, + settings: { + test: 'value', + }, + }, + }); + }); + + test('should call provider update functions on a timer', async () => { + jest.useFakeTimers(); + manager = new ExternalSecretsManager(settingsRepo, licenseMock, providersMock); + + await manager.init(); + + const updateSpy = jest.spyOn(manager.getProvider('dummy')!, 'update'); + + expect(updateSpy).toBeCalledTimes(0); + + jest.runOnlyPendingTimers(); + + expect(updateSpy).toBeCalledTimes(1); + }); + + test('should not call provider update functions if the not licensed', async () => { + jest.useFakeTimers(); + + manager = new ExternalSecretsManager( + settingsRepo, + mock({ + isExternalSecretsEnabled() { + return false; + }, + }), + providersMock, + ); + + await manager.init(); + + const updateSpy = jest.spyOn(manager.getProvider('dummy')!, 'update'); + + expect(updateSpy).toBeCalledTimes(0); + + jest.runOnlyPendingTimers(); + + expect(updateSpy).toBeCalledTimes(0); + }); + + test('should not call provider update functions if the provider has an error', async () => { + jest.useFakeTimers(); + + mockProvidersInstance.setProviders({ + dummy: FailedProvider, + }); + manager = new ExternalSecretsManager(settingsRepo, licenseMock, providersMock); + + await manager.init(); + + const updateSpy = jest.spyOn(manager.getProvider('dummy')!, 'update'); + + expect(updateSpy).toBeCalledTimes(0); + + jest.runOnlyPendingTimers(); + + expect(updateSpy).toBeCalledTimes(0); + }); + + test('should reinitialize a provider when save provider settings', async () => { + manager = new ExternalSecretsManager(settingsRepo, licenseMock, providersMock); + + await manager.init(); + + const dummyInitSpy = jest.spyOn(DummyProvider.prototype, 'init'); + + await manager.setProviderSettings('dummy', { + test: 'value', + }); + + expect(dummyInitSpy).toBeCalledTimes(1); + }); +}); diff --git a/packages/core/src/NodeExecuteFunctions.ts b/packages/core/src/NodeExecuteFunctions.ts index 11c1844a71..af000e91cc 100644 --- a/packages/core/src/NodeExecuteFunctions.ts +++ b/packages/core/src/NodeExecuteFunctions.ts @@ -136,6 +136,7 @@ import { setAllWorkflowExecutionMetadata, setWorkflowExecutionMetadata, } from './WorkflowExecutionMetadata'; +import { getSecretsProxy } from './Secrets'; import { getUserN8nFolderPath } from './UserSettings'; axios.defaults.timeout = 300000; @@ -1683,6 +1684,7 @@ export function getAdditionalKeys( additionalData: IWorkflowExecuteAdditionalData, mode: WorkflowExecuteMode, runExecutionData: IRunExecutionData | null, + options?: { secretsEnabled?: boolean }, ): IWorkflowDataProxyAdditionalKeys { const executionId = additionalData.executionId || PLACEHOLDER_EMPTY_EXECUTION_ID; const resumeUrl = `${additionalData.webhookWaitingBaseUrl}/${executionId}`; @@ -1723,6 +1725,7 @@ export function getAdditionalKeys( : undefined, }, $vars: additionalData.variables, + $secrets: options?.secretsEnabled ? getSecretsProxy(additionalData) : undefined, // deprecated $executionId: executionId, @@ -1858,6 +1861,7 @@ export async function getCredentials( // } const decryptedDataObject = await additionalData.credentialsHelper.getDecrypted( + additionalData, nodeCredentials, type, mode, diff --git a/packages/core/src/Secrets.ts b/packages/core/src/Secrets.ts new file mode 100644 index 0000000000..508af6ada3 --- /dev/null +++ b/packages/core/src/Secrets.ts @@ -0,0 +1,76 @@ +import type { IDataObject, IWorkflowExecuteAdditionalData } from 'n8n-workflow'; +import { ExpressionError } from 'n8n-workflow'; + +function buildSecretsValueProxy(value: IDataObject): unknown { + return new Proxy(value, { + get(target, valueName) { + if (typeof valueName !== 'string') { + return; + } + if (!(valueName in value)) { + throw new ExpressionError('Could not load secrets', { + description: + 'The credential in use tries to use secret from an external store that could not be found', + }); + } + const retValue = value[valueName]; + if (typeof retValue === 'object' && retValue !== null) { + return buildSecretsValueProxy(retValue as IDataObject); + } + return retValue; + }, + }); +} + +export function getSecretsProxy(additionalData: IWorkflowExecuteAdditionalData): IDataObject { + const secretsHelpers = additionalData.secretsHelpers; + return new Proxy( + {}, + { + get(target, providerName) { + if (typeof providerName !== 'string') { + return {}; + } + if (secretsHelpers.hasProvider(providerName)) { + return new Proxy( + {}, + { + get(target2, secretName): IDataObject | undefined { + if (typeof secretName !== 'string') { + return; + } + if (!secretsHelpers.hasSecret(providerName, secretName)) { + throw new ExpressionError('Could not load secrets', { + description: + 'The credential in use tries to use secret from an external store that could not be found', + }); + } + const retValue = secretsHelpers.getSecret(providerName, secretName); + if (typeof retValue === 'object' && retValue !== null) { + return buildSecretsValueProxy(retValue) as IDataObject; + } + return retValue; + }, + set() { + return false; + }, + ownKeys() { + return secretsHelpers.listSecrets(providerName); + }, + }, + ); + } + throw new ExpressionError('Could not load secrets', { + description: + 'The credential in use pulls secrets from an external store that is not reachable', + }); + }, + set() { + return false; + }, + ownKeys() { + return secretsHelpers.listProviders(); + }, + }, + ); +} diff --git a/packages/design-system/src/components/N8nCallout/Callout.vue b/packages/design-system/src/components/N8nCallout/Callout.vue index 53545563ab..639380773f 100644 --- a/packages/design-system/src/components/N8nCallout/Callout.vue +++ b/packages/design-system/src/components/N8nCallout/Callout.vue @@ -24,7 +24,7 @@ const CALLOUT_DEFAULT_ICONS: { [key: string]: string } = { info: 'info-circle', success: 'check-circle', warning: 'exclamation-triangle', - danger: 'times-circle', + danger: 'exclamation-triangle', }; export default defineComponent({ diff --git a/packages/design-system/src/components/N8nCallout/__tests__/__snapshots__/Callout.spec.ts.snap b/packages/design-system/src/components/N8nCallout/__tests__/__snapshots__/Callout.spec.ts.snap index 3e04a43141..a2f82a3ef9 100644 --- a/packages/design-system/src/components/N8nCallout/__tests__/__snapshots__/Callout.spec.ts.snap +++ b/packages/design-system/src/components/N8nCallout/__tests__/__snapshots__/Callout.spec.ts.snap @@ -27,7 +27,7 @@ exports[`components > N8nCallout > should render danger theme correctly 1`] = ` "
- +
 
diff --git a/packages/design-system/src/css/loading.scss b/packages/design-system/src/css/loading.scss index 9244291640..d13db8be39 100644 --- a/packages/design-system/src/css/loading.scss +++ b/packages/design-system/src/css/loading.scss @@ -26,7 +26,7 @@ position: fixed; .el-loading-spinner { - margin-top: #{- var.$loading-fullscreen-spinner-size * 0.5}; + transform: translateY(-50%); .circular { height: var.$loading-fullscreen-spinner-size; @@ -38,7 +38,7 @@ @include mixins.b(loading-spinner) { top: 50%; - margin-top: #{- var.$loading-spinner-size * 0.5}; + transform: translateY(-50%); width: 100%; text-align: center; position: absolute; diff --git a/packages/design-system/src/css/utilities/_link.scss b/packages/design-system/src/css/utilities/_link.scss new file mode 100644 index 0000000000..4a65d43ae8 --- /dev/null +++ b/packages/design-system/src/css/utilities/_link.scss @@ -0,0 +1,10 @@ +.overlay-link::after { + content: ''; + position: absolute; + top: 0; + right: 0; + bottom: 0; + left: 0; + z-index: 1; + pointer-events: auto; +} diff --git a/packages/design-system/src/css/utilities/index.scss b/packages/design-system/src/css/utilities/index.scss index 73ac65908a..e79fb2651a 100644 --- a/packages/design-system/src/css/utilities/index.scss +++ b/packages/design-system/src/css/utilities/index.scss @@ -1,4 +1,5 @@ @import 'float'; +@import 'link'; @import 'list'; @import 'spacing'; @import 'typography'; diff --git a/packages/editor-ui/src/Interface.ts b/packages/editor-ui/src/Interface.ts index c2dece42d7..8b0a607d60 100644 --- a/packages/editor-ui/src/Interface.ts +++ b/packages/editor-ui/src/Interface.ts @@ -44,6 +44,7 @@ import type { } from './constants'; import type { BulkCommand, Undoable } from '@/models/history'; import type { PartialBy } from '@/utils/typeHelpers'; +import type { INodeProperties } from 'n8n-workflow'; export * from 'n8n-design-system/types'; @@ -1542,6 +1543,26 @@ export interface InstanceUsage { export type CloudPlanAndUsageData = Cloud.PlanData & { usage: InstanceUsage }; +export interface ExternalSecretsProviderSecret { + key: string; +} + +export type ExternalSecretsProviderData = Record; + +export interface ExternalSecretsProvider { + icon: string; + name: string; + displayName: string; + connected: boolean; + connectedAt: string | false; + state: 'connected' | 'tested' | 'initializing' | 'error'; + data?: ExternalSecretsProviderData; +} + +export interface ExternalSecretsProviderWithProperties extends ExternalSecretsProvider { + properties: INodeProperties[]; +} + export type CloudUpdateLinkSourceType = | 'canvas-nav' | 'custom-data-filter' diff --git a/packages/editor-ui/src/api/externalSecrets.ee.ts b/packages/editor-ui/src/api/externalSecrets.ee.ts new file mode 100644 index 0000000000..76a1e189f9 --- /dev/null +++ b/packages/editor-ui/src/api/externalSecrets.ee.ts @@ -0,0 +1,58 @@ +import type { + IRestApiContext, + ExternalSecretsProvider, + ExternalSecretsProviderWithProperties, +} from '@/Interface'; +import { makeRestApiRequest } from '@/utils'; + +export const getExternalSecrets = async ( + context: IRestApiContext, +): Promise> => { + return makeRestApiRequest(context, 'GET', '/external-secrets/secrets'); +}; + +export const getExternalSecretsProviders = async ( + context: IRestApiContext, +): Promise => { + return makeRestApiRequest(context, 'GET', '/external-secrets/providers'); +}; + +export const getExternalSecretsProvider = async ( + context: IRestApiContext, + id: string, +): Promise => { + return makeRestApiRequest(context, 'GET', `/external-secrets/providers/${id}`); +}; + +export const testExternalSecretsProviderConnection = async ( + context: IRestApiContext, + id: string, + data: ExternalSecretsProvider['data'], +): Promise<{ testState: ExternalSecretsProvider['state'] }> => { + return makeRestApiRequest(context, 'POST', `/external-secrets/providers/${id}/test`, data); +}; + +export const updateProvider = async ( + context: IRestApiContext, + id: string, + data: ExternalSecretsProvider['data'], +): Promise => { + return makeRestApiRequest(context, 'POST', `/external-secrets/providers/${id}`, data); +}; + +export const reloadProvider = async ( + context: IRestApiContext, + id: string, +): Promise<{ updated: boolean }> => { + return makeRestApiRequest(context, 'POST', `/external-secrets/providers/${id}/update`); +}; + +export const connectProvider = async ( + context: IRestApiContext, + id: string, + connected: boolean, +): Promise => { + return makeRestApiRequest(context, 'POST', `/external-secrets/providers/${id}/connect`, { + connected, + }); +}; diff --git a/packages/editor-ui/src/assets/images/doppler.webp b/packages/editor-ui/src/assets/images/doppler.webp new file mode 100644 index 0000000000..7bd3440637 Binary files /dev/null and b/packages/editor-ui/src/assets/images/doppler.webp differ diff --git a/packages/editor-ui/src/assets/images/hashicorp.webp b/packages/editor-ui/src/assets/images/hashicorp.webp new file mode 100644 index 0000000000..ca52275380 Binary files /dev/null and b/packages/editor-ui/src/assets/images/hashicorp.webp differ diff --git a/packages/editor-ui/src/assets/images/infisical.webp b/packages/editor-ui/src/assets/images/infisical.webp new file mode 100644 index 0000000000..abe9a20de3 Binary files /dev/null and b/packages/editor-ui/src/assets/images/infisical.webp differ diff --git a/packages/editor-ui/src/components/CodeNodeEditor/completer.ts b/packages/editor-ui/src/components/CodeNodeEditor/completer.ts index dfcb2b6f14..9d64f92a49 100644 --- a/packages/editor-ui/src/components/CodeNodeEditor/completer.ts +++ b/packages/editor-ui/src/components/CodeNodeEditor/completer.ts @@ -172,6 +172,7 @@ export const completerExtension = defineComponent({ if (value === '$execution') return this.executionCompletions(context, variable); if (value === '$vars') return this.variablesCompletions(context, variable); + if (value === '$workflow') return this.workflowCompletions(context, variable); if (value === '$prevNode') return this.prevNodeCompletions(context, variable); diff --git a/packages/editor-ui/src/components/CodeNodeEditor/completions/secrets.completions.ts b/packages/editor-ui/src/components/CodeNodeEditor/completions/secrets.completions.ts new file mode 100644 index 0000000000..d3eec85bc0 --- /dev/null +++ b/packages/editor-ui/src/components/CodeNodeEditor/completions/secrets.completions.ts @@ -0,0 +1,54 @@ +import Vue from 'vue'; +import { addVarType } from '../utils'; +import type { Completion, CompletionContext, CompletionResult } from '@codemirror/autocomplete'; +import type { CodeNodeEditorMixin } from '../types'; +import { useExternalSecretsStore } from '@/stores'; + +const escape = (str: string) => str.replace('$', '\\$'); + +export const secretsCompletions = (Vue as CodeNodeEditorMixin).extend({ + methods: { + /** + * Complete `$secrets.` to `$secrets.providerName` and `$secrets.providerName.secretName`. + */ + secretsCompletions(context: CompletionContext, matcher = '$secrets'): CompletionResult | null { + const pattern = new RegExp(`${escape(matcher)}\..*`); + const preCursor = context.matchBefore(pattern); + + if (!preCursor || (preCursor.from === preCursor.to && !context.explicit)) return null; + + const provider = preCursor.text.split('.')[1]; + const externalSecretsStore = useExternalSecretsStore(); + let options: Completion[]; + + const optionsForObject = (leftSide: string, object: object): Completion[] => { + return Object.entries(object).flatMap(([key, value]) => { + if (typeof value === 'object' && value !== null) { + return optionsForObject(`${leftSide}.${key}`, value); + } + return { + label: `${leftSide}.${key}`, + info: '*******', + }; + }); + }; + + if (provider) { + options = optionsForObject( + `${matcher}.${provider}`, + externalSecretsStore.secretsAsObject[provider], + ); + } else { + options = Object.keys(externalSecretsStore.secretsAsObject).map((provider) => ({ + label: `${matcher}.${provider}`, + info: JSON.stringify(externalSecretsStore.secretsAsObject[provider]), + })); + } + + return { + from: preCursor.from, + options: options.map(addVarType), + }; + }, + }, +}); diff --git a/packages/editor-ui/src/components/CodeNodeEditor/completions/variables.completions.ts b/packages/editor-ui/src/components/CodeNodeEditor/completions/variables.completions.ts index c7a6983b6e..278d401545 100644 --- a/packages/editor-ui/src/components/CodeNodeEditor/completions/variables.completions.ts +++ b/packages/editor-ui/src/components/CodeNodeEditor/completions/variables.completions.ts @@ -8,7 +8,7 @@ const escape = (str: string) => str.replace('$', '\\$'); export const variablesCompletions = defineComponent({ methods: { /** - * Complete `$workflow.` to `.id .name .active`. + * Complete `$vars.` to `$vars.VAR_NAME`. */ variablesCompletions(context: CompletionContext, matcher = '$vars'): CompletionResult | null { const pattern = new RegExp(`${escape(matcher)}\..*`); diff --git a/packages/editor-ui/src/components/CredentialEdit/CredentialConfig.vue b/packages/editor-ui/src/components/CredentialEdit/CredentialConfig.vue index 8a75cfeb2b..37b8e7f4c7 100644 --- a/packages/editor-ui/src/components/CredentialEdit/CredentialConfig.vue +++ b/packages/editor-ui/src/components/CredentialEdit/CredentialConfig.vue @@ -126,6 +126,17 @@ {{ $locale.baseText('credentialEdit.credentialConfig.missingCredentialType') }} + + + +
@@ -152,10 +163,12 @@ import { useNodeTypesStore } from '@/stores/nodeTypes.store'; import type { ICredentialsResponse } from '@/Interface'; import AuthTypeSelector from '@/components/CredentialEdit/AuthTypeSelector.vue'; import GoogleAuthButton from './GoogleAuthButton.vue'; +import EnterpriseEdition from '@/components/EnterpriseEdition.ee.vue'; export default defineComponent({ name: 'CredentialConfig', components: { + EnterpriseEdition, AuthTypeSelector, Banner, CopyInput, diff --git a/packages/editor-ui/src/components/CredentialEdit/CredentialEdit.vue b/packages/editor-ui/src/components/CredentialEdit/CredentialEdit.vue index 05a06cbf4d..22a5e8f3ef 100644 --- a/packages/editor-ui/src/components/CredentialEdit/CredentialEdit.vue +++ b/packages/editor-ui/src/components/CredentialEdit/CredentialEdit.vue @@ -157,6 +157,8 @@ import { getNodeCredentialForSelectedAuthType, updateNodeAuthType, isCredentialModalState, + isExpression, + isTestableExpression, } from '@/utils'; import { externalHooks } from '@/mixins/externalHooks'; @@ -370,12 +372,13 @@ export default defineComponent({ } const { ownedBy, sharedWith, ...credentialData } = this.credentialData; - const hasExpressions = Object.values(credentialData).reduce( + const hasUntestableExpressions = Object.values(credentialData).reduce( (accu: boolean, value: CredentialInformation) => - accu || (typeof value === 'string' && value.startsWith('=')), + accu || + (typeof value === 'string' && isExpression(value) && !isTestableExpression(value)), false, ); - if (hasExpressions) { + if (hasUntestableExpressions) { return false; } @@ -445,8 +448,14 @@ export default defineComponent({ return false; } - if (property.type === 'number' && typeof this.credentialData[property.name] !== 'number') { - return false; + if (property.type === 'number') { + const isExpression = + typeof this.credentialData[property.name] === 'string' && + this.credentialData[property.name].startsWith('='); + + if (typeof this.credentialData[property.name] !== 'number' && !isExpression) { + return false; + } } } return true; @@ -835,12 +844,17 @@ export default defineComponent({ this.testedSuccessfully = false; } + const usesExternalSecrets = Object.entries(credentialDetails.data || {}).some(([, value]) => + /=.*\{\{[^}]*\$secrets\.[^}]+}}.*/.test(`${value}`), + ); + const trackProperties: ITelemetryTrackProperties = { credential_type: credentialDetails.type, workflow_id: this.workflowsStore.workflowId, credential_id: credential.id, is_complete: !!this.requiredPropertiesFilled, is_new: isNewCredential, + uses_external_secrets: usesExternalSecrets, }; if (this.isOAuthType) { diff --git a/packages/editor-ui/src/components/ExpressionParameterInput.vue b/packages/editor-ui/src/components/ExpressionParameterInput.vue index 8567f95123..ad9093f180 100644 --- a/packages/editor-ui/src/components/ExpressionParameterInput.vue +++ b/packages/editor-ui/src/components/ExpressionParameterInput.vue @@ -19,6 +19,7 @@ :isReadOnly="isReadOnly" :targetItem="hoveringItem" :isSingleLine="isForRecordLocator" + :additionalData="additionalExpressionData" :path="path" @focus="onFocus" @blur="onBlur" @@ -34,7 +35,6 @@ data-test-id="expander" /> - import { mapStores } from 'pinia'; +import type { PropType } from 'vue'; import { defineComponent } from 'vue'; import { useNDVStore } from '@/stores/ndv.store'; @@ -57,6 +58,7 @@ import { createExpressionTelemetryPayload } from '@/utils/telemetryUtils'; import type { Segment } from '@/types/expressions'; import type { TargetItem } from '@/Interface'; +import type { IDataObject } from 'n8n-workflow'; type InlineExpressionEditorInputRef = InstanceType; @@ -88,6 +90,10 @@ export default defineComponent({ type: Boolean, default: false, }, + additionalExpressionData: { + type: Object as PropType, + default: () => ({}), + }, }, computed: { ...mapStores(useNDVStore, useWorkflowsStore), diff --git a/packages/editor-ui/src/components/ExternalSecretsProviderCard.ee.vue b/packages/editor-ui/src/components/ExternalSecretsProviderCard.ee.vue new file mode 100644 index 0000000000..ed4934733c --- /dev/null +++ b/packages/editor-ui/src/components/ExternalSecretsProviderCard.ee.vue @@ -0,0 +1,185 @@ + + + + + diff --git a/packages/editor-ui/src/components/ExternalSecretsProviderConnectionSwitch.ee.vue b/packages/editor-ui/src/components/ExternalSecretsProviderConnectionSwitch.ee.vue new file mode 100644 index 0000000000..d56ead9a86 --- /dev/null +++ b/packages/editor-ui/src/components/ExternalSecretsProviderConnectionSwitch.ee.vue @@ -0,0 +1,116 @@ + + + + + diff --git a/packages/editor-ui/src/components/ExternalSecretsProviderImage.ee.vue b/packages/editor-ui/src/components/ExternalSecretsProviderImage.ee.vue new file mode 100644 index 0000000000..e70742a4a7 --- /dev/null +++ b/packages/editor-ui/src/components/ExternalSecretsProviderImage.ee.vue @@ -0,0 +1,28 @@ + + + diff --git a/packages/editor-ui/src/components/ExternalSecretsProviderModal.ee.vue b/packages/editor-ui/src/components/ExternalSecretsProviderModal.ee.vue new file mode 100644 index 0000000000..64c2d54d7b --- /dev/null +++ b/packages/editor-ui/src/components/ExternalSecretsProviderModal.ee.vue @@ -0,0 +1,331 @@ + + + + + + + diff --git a/packages/editor-ui/src/components/InlineExpressionEditor/InlineExpressionEditorInput.vue b/packages/editor-ui/src/components/InlineExpressionEditor/InlineExpressionEditorInput.vue index 902729dbe1..fc6434aa63 100644 --- a/packages/editor-ui/src/components/InlineExpressionEditor/InlineExpressionEditorInput.vue +++ b/packages/editor-ui/src/components/InlineExpressionEditor/InlineExpressionEditorInput.vue @@ -4,6 +4,7 @@ + + diff --git a/packages/editor-ui/src/views/__tests__/SettingsExternalSecrets.test.ts b/packages/editor-ui/src/views/__tests__/SettingsExternalSecrets.test.ts new file mode 100644 index 0000000000..d6969d5261 --- /dev/null +++ b/packages/editor-ui/src/views/__tests__/SettingsExternalSecrets.test.ts @@ -0,0 +1,60 @@ +import { createTestingPinia } from '@pinia/testing'; +import { merge } from 'lodash-es'; +import { EnterpriseEditionFeature, STORES } from '@/constants'; +import { SETTINGS_STORE_DEFAULT_STATE } from '@/__tests__/utils'; +import SettingsExternalSecrets from '@/views/SettingsExternalSecrets.vue'; +import { useExternalSecretsStore } from '@/stores/externalSecrets.ee.store'; +import { createComponentRenderer } from '@/__tests__/render'; +import { useSettingsStore } from '@/stores'; +import { setupServer } from '@/__tests__/server'; + +let pinia: ReturnType; +let externalSecretsStore: ReturnType; +let settingsStore: ReturnType; +let server: ReturnType; + +const renderComponent = createComponentRenderer(SettingsExternalSecrets); + +describe('SettingsExternalSecrets', () => { + beforeAll(() => { + server = setupServer(); + }); + + beforeEach(async () => { + pinia = createTestingPinia({ + initialState: { + [STORES.SETTINGS]: { + settings: merge({}, SETTINGS_STORE_DEFAULT_STATE.settings), + }, + }, + }); + externalSecretsStore = useExternalSecretsStore(pinia); + settingsStore = useSettingsStore(); + + await settingsStore.getSettings(); + }); + + afterEach(() => { + vi.clearAllMocks(); + }); + + afterAll(() => { + server.shutdown(); + }); + + it('should render paywall state when there is no license', () => { + const { getByTestId, queryByTestId } = renderComponent({ pinia }); + + expect(queryByTestId('external-secrets-content-licensed')).not.toBeInTheDocument(); + expect(getByTestId('external-secrets-content-unlicensed')).toBeInTheDocument(); + }); + + it('should render licensed content', () => { + settingsStore.settings.enterprise[EnterpriseEditionFeature.ExternalSecrets] = true; + + const { getByTestId, queryByTestId } = renderComponent({ pinia }); + + expect(getByTestId('external-secrets-content-licensed')).toBeInTheDocument(); + expect(queryByTestId('external-secrets-content-unlicensed')).not.toBeInTheDocument(); + }); +}); diff --git a/packages/nodes-base/test/nodes/Helpers.ts b/packages/nodes-base/test/nodes/Helpers.ts index c038f7621d..5e3e4a0c7b 100644 --- a/packages/nodes-base/test/nodes/Helpers.ts +++ b/packages/nodes-base/test/nodes/Helpers.ts @@ -123,6 +123,7 @@ export class CredentialsHelper extends ICredentialsHelper { } async getDecrypted( + additionalData: IWorkflowExecuteAdditionalData, nodeCredentials: INodeCredentialsDetails, type: string, ): Promise { diff --git a/packages/workflow/src/Expression.ts b/packages/workflow/src/Expression.ts index a99040b000..6d1ab313c9 100644 --- a/packages/workflow/src/Expression.ts +++ b/packages/workflow/src/Expression.ts @@ -2,6 +2,7 @@ import * as tmpl from '@n8n_io/riot-tmpl'; import { DateTime, Duration, Interval } from 'luxon'; import type { + IDataObject, IExecuteData, INode, INodeExecutionData, @@ -66,8 +67,8 @@ export class Expression { this.workflow = workflow; } - static resolveWithoutWorkflow(expression: string) { - return tmpl.tmpl(expression, {}); + static resolveWithoutWorkflow(expression: string, data: IDataObject = {}) { + return tmpl.tmpl(expression, data); } /** diff --git a/packages/workflow/src/Interfaces.ts b/packages/workflow/src/Interfaces.ts index cba00dbd34..84f142fa05 100644 --- a/packages/workflow/src/Interfaces.ts +++ b/packages/workflow/src/Interfaces.ts @@ -225,6 +225,7 @@ export abstract class ICredentialsHelper { ): Promise; abstract getDecrypted( + additionalData: IWorkflowExecuteAdditionalData, nodeCredentials: INodeCredentialsDetails, type: string, mode: WorkflowExecuteMode, @@ -1775,6 +1776,7 @@ export interface IWorkflowExecuteAdditionalData { executionTimeoutTimestamp?: number; userId: string; variables: IDataObject; + secretsHelpers: SecretsHelpersBase; } export type WorkflowExecuteMode = @@ -2185,6 +2187,7 @@ export interface IN8nUISettings { variables: boolean; sourceControl: boolean; auditLogs: boolean; + externalSecrets: boolean; showNonProdBanner: boolean; debugInEditor: boolean; }; @@ -2206,4 +2209,15 @@ export interface IN8nUISettings { }; } +export interface SecretsHelpersBase { + update(): Promise; + waitForInit(): Promise; + + getSecret(provider: string, name: string): IDataObject | undefined; + hasSecret(provider: string, name: string): boolean; + hasProvider(provider: string): boolean; + listProviders(): string[]; + listSecrets(provider: string): string[]; +} + export type BannerName = 'V1' | 'TRIAL_OVER' | 'TRIAL' | 'NON_PRODUCTION_LICENSE'; diff --git a/packages/workflow/src/index.ts b/packages/workflow/src/index.ts index f2a6ac66e6..7fe6de45dd 100644 --- a/packages/workflow/src/index.ts +++ b/packages/workflow/src/index.ts @@ -44,6 +44,7 @@ export { } from './type-guards'; export { ExpressionExtensions } from './Extensions'; +export * as ExpressionParser from './Extensions/ExpressionParser'; export { NativeMethods } from './NativeMethods'; export type { DocMetadata, NativeDoc } from './Extensions'; diff --git a/packages/workflow/test/Helpers.ts b/packages/workflow/test/Helpers.ts index ad2b8c64b7..fededf3777 100644 --- a/packages/workflow/test/Helpers.ts +++ b/packages/workflow/test/Helpers.ts @@ -125,6 +125,7 @@ export class CredentialsHelper extends ICredentialsHelper { } async getDecrypted( + additionalData: IWorkflowExecuteAdditionalData, nodeCredentials: INodeCredentialsDetails, type: string, ): Promise { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2277df9914..24d3216059 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -305,6 +305,9 @@ importers: handlebars: specifier: 4.7.7 version: 4.7.7 + infisical-node: + specifier: ^1.3.0 + version: 1.3.0 inquirer: specifier: ^7.0.1 version: 7.3.3 @@ -8218,6 +8221,16 @@ packages: transitivePeerDependencies: - debug + /axios@1.4.0: + resolution: {integrity: sha512-S4XCWMEmzvo64T9GfvQDOXgYRDJ/wsSZc7Jvdgx5u1sd0JwsuPLqb3SYmusag+edF6ziyMensPVqLTSc1PiSEA==} + dependencies: + follow-redirects: 1.15.2(debug@4.3.4) + form-data: 4.0.0 + proxy-from-env: 1.1.0 + transitivePeerDependencies: + - debug + dev: false + /babel-core@7.0.0-bridge.0(@babel/core@7.22.9): resolution: {integrity: sha512-poPX9mZH/5CSanm50Q+1toVci6pv5KSRv/5TWCwtzQS5XEwn40BcCrgIeMFWP9CKKIniKXNxoIOnOq4VVlGXhg==} peerDependencies: @@ -12918,6 +12931,17 @@ packages: dev: false optional: true + /infisical-node@1.3.0: + resolution: {integrity: sha512-tTnnExRAO/ZyqiRdnSlBisErNToYWgtunMWh+8opClEt5qjX7l6HC/b4oGo2AuR2Pf41IR+oqo+dzkM1TCvlUA==} + dependencies: + axios: 1.4.0 + dotenv: 16.0.3 + tweetnacl: 1.0.3 + tweetnacl-util: 0.15.1 + transitivePeerDependencies: + - debug + dev: false + /inflected@2.1.0: resolution: {integrity: sha512-hAEKNxvHf2Iq3H60oMBHkB4wl5jn3TPF3+fXek/sRwAB5gP9xWs4r7aweSF95f99HFoz69pnZTcu8f0SIHV18w==} dev: true @@ -20081,9 +20105,17 @@ packages: turbo-windows-arm64: 1.10.12 dev: true + /tweetnacl-util@0.15.1: + resolution: {integrity: sha512-RKJBIj8lySrShN4w6i/BonWp2Z/uxwC3h4y7xsRrpP59ZboCd0GpEVsOnMDYLMmKBpYhb5TgHzZXy7wTfYFBRw==} + dev: false + /tweetnacl@0.14.5: resolution: {integrity: sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==} + /tweetnacl@1.0.3: + resolution: {integrity: sha512-6rt+RN7aOi1nGMyC4Xa5DdYiukl2UWCbcJft7YhxReBGQD7OAM8Pbxw6YMo4r2diNEA8FEmu32YOn9rhaiE5yw==} + dev: false + /type-check@0.3.2: resolution: {integrity: sha512-ZCmOJdvOWDBYJlzAoFkC+Q0+bUyEOS1ltgp1MGU03fqHG+dbi9tBFU2Rd9QKiDZFAYrhPh2JUf7rZRIuHRKtOg==} engines: {node: '>= 0.8.0'}