From 3137de2585f93753a50fb5e41a8c39128b73002a Mon Sep 17 00:00:00 2001 From: Ben Hesseldieck <1849459+BHesseldieck@users.noreply.github.com> Date: Thu, 14 Oct 2021 00:21:00 +0200 Subject: [PATCH] :zap: Change credentials structure (#2139) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * :sparkles: change FE to handle new object type * 🚸 improve UX of handling invalid credentials * 🚧 WIP * :art: fix typescript issues * 🐘 add migrations for all supported dbs * ✏️ add description to migrations * :zap: add credential update on import * :zap: resolve after merge issues * :shirt: fix lint issues * :zap: check credentials on workflow create/update * update interface * :shirt: fix ts issues * :zap: adaption to new credentials UI * :bug: intialize cache on BE for credentials check * :bug: fix undefined oldCredentials * :bug: fix deleting credential * :bug: fix check for undefined keys * :bug: fix disabling edit in execution * :art: just show credential name on execution view * ✏️ remove TODO * :zap: implement review suggestions * :zap: add cache to getCredentialsByType * ⏪ use getter instead of cache * ✏️ fix variable name typo * 🐘 include waiting nodes to migrations * :bug: fix reverting migrations command * :zap: update typeorm command * :sparkles: create db:revert command * 👕 fix lint error Co-authored-by: Mutasem --- packages/cli/commands/db/revert.ts | 61 +++++ packages/cli/commands/export/credentials.ts | 3 +- packages/cli/commands/import/workflow.ts | 43 +++- packages/cli/commands/start.ts | 10 +- packages/cli/migrations/ormconfig.ts | 6 +- packages/cli/package.json | 2 +- packages/cli/src/CredentialsHelper.ts | 44 ++-- packages/cli/src/GenericHelpers.ts | 40 ---- packages/cli/src/Server.ts | 90 +++---- packages/cli/src/WorkflowCredentials.ts | 27 ++- packages/cli/src/WorkflowHelpers.ts | 108 +++++++++ .../databases/entities/CredentialsEntity.ts | 35 ++- .../src/databases/entities/ExecutionEntity.ts | 20 +- .../cli/src/databases/entities/TagEntity.ts | 17 +- .../src/databases/entities/WorkflowEntity.ts | 37 ++- ...1630451444017-UpdateWorkflowCredentials.ts | 215 +++++++++++++++++ .../src/databases/mysqldb/migrations/index.ts | 2 + ...1630419189837-UpdateWorkflowCredentials.ts | 223 ++++++++++++++++++ .../databases/postgresdb/migrations/index.ts | 2 + ...1630330987096-UpdateWorkflowCredentials.ts | 215 +++++++++++++++++ .../src/databases/sqlite/migrations/index.ts | 2 + packages/cli/src/databases/utils.ts | 42 ---- packages/core/src/Credentials.ts | 1 + packages/core/src/NodeExecuteFunctions.ts | 43 ++-- packages/core/test/Credentials.test.ts | 11 +- packages/core/test/Helpers.ts | 12 +- packages/editor-ui/src/Interface.ts | 2 +- .../CredentialEdit/CredentialEdit.vue | 2 + .../src/components/NodeCredentials.vue | 94 ++++++-- .../src/components/mixins/nodeHelpers.ts | 31 ++- packages/editor-ui/src/modules/credentials.ts | 8 +- packages/editor-ui/src/store.ts | 26 ++ packages/editor-ui/src/views/NodeView.vue | 51 +++- packages/nodes-base/nodes/Wait.node.ts | 1 + packages/nodes-base/nodes/Webhook.node.ts | 1 + packages/workflow/src/Interfaces.ts | 42 +++- 36 files changed, 1318 insertions(+), 251 deletions(-) create mode 100644 packages/cli/commands/db/revert.ts create mode 100644 packages/cli/src/databases/mysqldb/migrations/1630451444017-UpdateWorkflowCredentials.ts create mode 100644 packages/cli/src/databases/postgresdb/migrations/1630419189837-UpdateWorkflowCredentials.ts create mode 100644 packages/cli/src/databases/sqlite/migrations/1630330987096-UpdateWorkflowCredentials.ts delete mode 100644 packages/cli/src/databases/utils.ts diff --git a/packages/cli/commands/db/revert.ts b/packages/cli/commands/db/revert.ts new file mode 100644 index 0000000000..585027cf06 --- /dev/null +++ b/packages/cli/commands/db/revert.ts @@ -0,0 +1,61 @@ +/* eslint-disable @typescript-eslint/no-unsafe-member-access */ +/* eslint-disable no-console */ +import { Command, flags } from '@oclif/command'; +import { Connection, ConnectionOptions, createConnection } from 'typeorm'; +import { LoggerProxy } from 'n8n-workflow'; + +import { getLogger } from '../../src/Logger'; + +import { Db } from '../../src'; + +export class DbRevertMigrationCommand extends Command { + static description = 'Revert last database migration'; + + static examples = ['$ n8n db:revert']; + + static flags = { + help: flags.help({ char: 'h' }), + }; + + // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types + async run() { + const logger = getLogger(); + LoggerProxy.init(logger); + + // eslint-disable-next-line @typescript-eslint/no-shadow, @typescript-eslint/no-unused-vars + const { flags } = this.parse(DbRevertMigrationCommand); + + let connection: Connection | undefined; + try { + await Db.init(); + connection = Db.collections.Credentials?.manager.connection; + + if (!connection) { + throw new Error(`No database connection available.`); + } + + const connectionOptions: ConnectionOptions = Object.assign(connection.options, { + subscribers: [], + synchronize: false, + migrationsRun: false, + dropSchema: false, + logging: ['query', 'error', 'schema'], + }); + + // close connection in order to reconnect with updated options + await connection.close(); + connection = await createConnection(connectionOptions); + + await connection.undoLastMigration(); + await connection.close(); + } catch (error) { + if (connection) await connection.close(); + + console.error('Error reverting last migration. See log messages for details.'); + logger.error(error.message); + this.exit(1); + } + + this.exit(); + } +} diff --git a/packages/cli/commands/export/credentials.ts b/packages/cli/commands/export/credentials.ts index 9ee488fe0e..b8ebb95ec9 100644 --- a/packages/cli/commands/export/credentials.ts +++ b/packages/cli/commands/export/credentials.ts @@ -129,7 +129,8 @@ export class ExportCredentialsCommand extends Command { for (let i = 0; i < credentials.length; i++) { const { name, type, nodesAccess, data } = credentials[i]; - const credential = new Credentials(name, type, nodesAccess, data); + const id = credentials[i].id as string; + const credential = new Credentials({ id, name }, type, nodesAccess, data); const plainData = credential.getData(encryptionKey); (credentials[i] as ICredentialsDecryptedDb).data = plainData; } diff --git a/packages/cli/commands/import/workflow.ts b/packages/cli/commands/import/workflow.ts index 7fb94c1039..e27ac2dd16 100644 --- a/packages/cli/commands/import/workflow.ts +++ b/packages/cli/commands/import/workflow.ts @@ -2,14 +2,14 @@ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ import { Command, flags } from '@oclif/command'; -import { LoggerProxy } from 'n8n-workflow'; +import { INode, INodeCredentialsDetails, LoggerProxy } from 'n8n-workflow'; import * as fs from 'fs'; import * as glob from 'fast-glob'; import * as path from 'path'; import { UserSettings } from 'n8n-core'; import { getLogger } from '../../src/Logger'; -import { Db } from '../../src'; +import { Db, ICredentialsDb } from '../../src'; export class ImportWorkflowsCommand extends Command { static description = 'Import workflows'; @@ -30,6 +30,32 @@ export class ImportWorkflowsCommand extends Command { }), }; + private transformCredentials(node: INode, credentialsEntities: ICredentialsDb[]) { + if (node.credentials) { + const allNodeCredentials = Object.entries(node.credentials); + // eslint-disable-next-line no-restricted-syntax + for (const [type, name] of allNodeCredentials) { + if (typeof name === 'string') { + const nodeCredentials: INodeCredentialsDetails = { + id: null, + name, + }; + + const matchingCredentials = credentialsEntities.filter( + (credentials) => credentials.name === name && credentials.type === type, + ); + + if (matchingCredentials.length === 1) { + nodeCredentials.id = matchingCredentials[0].id.toString(); + } + + // eslint-disable-next-line no-param-reassign + node.credentials[type] = nodeCredentials; + } + } + } + } + // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types async run() { const logger = getLogger(); @@ -57,6 +83,7 @@ export class ImportWorkflowsCommand extends Command { // Make sure the settings exist await UserSettings.prepareUserSettings(); + const credentialsEntities = (await Db.collections.Credentials?.find()) ?? []; let i; if (flags.separate) { const files = await glob( @@ -64,6 +91,12 @@ export class ImportWorkflowsCommand extends Command { ); for (i = 0; i < files.length; i++) { const workflow = JSON.parse(fs.readFileSync(files[i], { encoding: 'utf8' })); + if (credentialsEntities.length > 0) { + // eslint-disable-next-line + workflow.nodes.forEach((node: INode) => { + this.transformCredentials(node, credentialsEntities); + }); + } // eslint-disable-next-line no-await-in-loop, @typescript-eslint/no-non-null-assertion await Db.collections.Workflow!.save(workflow); } @@ -75,6 +108,12 @@ export class ImportWorkflowsCommand extends Command { } for (i = 0; i < fileContents.length; i++) { + if (credentialsEntities.length > 0) { + // eslint-disable-next-line + fileContents[i].nodes.forEach((node: INode) => { + this.transformCredentials(node, credentialsEntities); + }); + } // eslint-disable-next-line no-await-in-loop, @typescript-eslint/no-non-null-assertion await Db.collections.Workflow!.save(fileContents[i]); } diff --git a/packages/cli/commands/start.ts b/packages/cli/commands/start.ts index 78550f7037..33f83cac7a 100644 --- a/packages/cli/commands/start.ts +++ b/packages/cli/commands/start.ts @@ -37,7 +37,7 @@ import { getLogger } from '../src/Logger'; const open = require('open'); let activeWorkflowRunner: ActiveWorkflowRunner.ActiveWorkflowRunner | undefined; -let processExistCode = 0; +let processExitCode = 0; export class Start extends Command { static description = 'Starts n8n. Makes Web-UI available and starts active workflows'; @@ -92,7 +92,7 @@ export class Start extends Command { setTimeout(() => { // In case that something goes wrong with shutdown we // kill after max. 30 seconds no matter what - process.exit(processExistCode); + process.exit(processExitCode); }, 30000); const skipWebhookDeregistration = config.get( @@ -133,7 +133,7 @@ export class Start extends Command { console.error('There was an error shutting down n8n.', error); } - process.exit(processExistCode); + process.exit(processExitCode); } async run() { @@ -160,7 +160,7 @@ export class Start extends Command { const startDbInitPromise = Db.init().catch((error: Error) => { logger.error(`There was an error initializing DB: "${error.message}"`); - processExistCode = 1; + processExitCode = 1; // @ts-ignore process.emit('SIGINT'); process.exit(1); @@ -355,7 +355,7 @@ export class Start extends Command { // eslint-disable-next-line @typescript-eslint/restrict-template-expressions this.error(`There was an error: ${error.message}`); - processExistCode = 1; + processExitCode = 1; // @ts-ignore process.emit('SIGINT'); } diff --git a/packages/cli/migrations/ormconfig.ts b/packages/cli/migrations/ormconfig.ts index efcfce2cc3..1b83c806e1 100644 --- a/packages/cli/migrations/ormconfig.ts +++ b/packages/cli/migrations/ormconfig.ts @@ -9,7 +9,7 @@ module.exports = [ logging: true, entities: Object.values(entities), database: path.join(UserSettings.getUserN8nFolderPath(), 'database.sqlite'), - migrations: ['./src/databases/sqlite/migrations/*.ts'], + migrations: ['./src/databases/sqlite/migrations/index.ts'], subscribers: ['./src/databases/sqlite/subscribers/*.ts'], cli: { entitiesDir: './src/databases/entities', @@ -28,7 +28,7 @@ module.exports = [ database: 'n8n', schema: 'public', entities: Object.values(entities), - migrations: ['./src/databases/postgresdb/migrations/*.ts'], + migrations: ['./src/databases/postgresdb/migrations/index.ts'], subscribers: ['src/subscriber/**/*.ts'], cli: { entitiesDir: './src/databases/entities', @@ -46,7 +46,7 @@ module.exports = [ port: '3306', logging: false, entities: Object.values(entities), - migrations: ['./src/databases/mysqldb/migrations/*.ts'], + migrations: ['./src/databases/mysqldb/migrations/index.ts'], subscribers: ['src/subscriber/**/*.ts'], cli: { entitiesDir: './src/databases/entities', diff --git a/packages/cli/package.json b/packages/cli/package.json index 60155aabd8..d87958f778 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -31,7 +31,7 @@ "start:windows": "cd bin && n8n", "test": "jest", "watch": "tsc --watch", - "typeorm": "ts-node ./node_modules/typeorm/cli.js" + "typeorm": "ts-node ../../node_modules/typeorm/cli.js" }, "bin": { "n8n": "./bin/n8n" diff --git a/packages/cli/src/CredentialsHelper.ts b/packages/cli/src/CredentialsHelper.ts index 5a101e7984..f16f094bb6 100644 --- a/packages/cli/src/CredentialsHelper.ts +++ b/packages/cli/src/CredentialsHelper.ts @@ -5,6 +5,7 @@ import { ICredentialsExpressionResolveValues, ICredentialsHelper, INode, + INodeCredentialsDetails, INodeParameters, INodeProperties, INodeType, @@ -39,30 +40,32 @@ export class CredentialsHelper extends ICredentialsHelper { /** * Returns the credentials instance * - * @param {string} name Name of the credentials to return instance of + * @param {INodeCredentialsDetails} nodeCredentials id and name to return instance of * @param {string} type Type of the credentials to return instance of * @returns {Credentials} * @memberof CredentialsHelper */ - async getCredentials(name: string, type: string): Promise { - const credentialsDb = await Db.collections.Credentials?.find({ type }); - - if (credentialsDb === undefined || credentialsDb.length === 0) { - throw new Error(`No credentials of type "${type}" exist.`); + async getCredentials( + nodeCredentials: INodeCredentialsDetails, + type: string, + ): Promise { + if (!nodeCredentials.id) { + throw new Error(`Credentials "${nodeCredentials.name}" for type "${type}" don't have an ID.`); } - // eslint-disable-next-line @typescript-eslint/no-shadow - const credential = credentialsDb.find((credential) => credential.name === name); + const credentials = await Db.collections.Credentials?.findOne({ id: nodeCredentials.id, type }); - if (credential === undefined) { - throw new Error(`No credentials with name "${name}" exist for type "${type}".`); + if (!credentials) { + throw new Error( + `Credentials with ID "${nodeCredentials.id}" don't exist for type "${type}".`, + ); } return new Credentials( - credential.name, - credential.type, - credential.nodesAccess, - credential.data, + { id: credentials.id.toString(), name: credentials.name }, + credentials.type, + credentials.nodesAccess, + credentials.data, ); } @@ -101,21 +104,20 @@ export class CredentialsHelper extends ICredentialsHelper { /** * Returns the decrypted credential data with applied overwrites * - * @param {string} name Name of the credentials to return data of + * @param {INodeCredentialsDetails} nodeCredentials id and name to return instance of * @param {string} type Type of the credentials to return data of * @param {boolean} [raw] Return the data as supplied without defaults or overwrites * @returns {ICredentialDataDecryptedObject} * @memberof CredentialsHelper */ async getDecrypted( - name: string, + nodeCredentials: INodeCredentialsDetails, type: string, mode: WorkflowExecuteMode, raw?: boolean, expressionResolveValues?: ICredentialsExpressionResolveValues, ): Promise { - const credentials = await this.getCredentials(name, type); - + const credentials = await this.getCredentials(nodeCredentials, type); const decryptedDataOriginal = credentials.getData(this.encryptionKey); if (raw === true) { @@ -228,12 +230,12 @@ export class CredentialsHelper extends ICredentialsHelper { * @memberof CredentialsHelper */ async updateCredentials( - name: string, + nodeCredentials: INodeCredentialsDetails, type: string, data: ICredentialDataDecryptedObject, ): Promise { // eslint-disable-next-line @typescript-eslint/await-thenable - const credentials = await this.getCredentials(name, type); + const credentials = await this.getCredentials(nodeCredentials, type); if (Db.collections.Credentials === null) { // The first time executeWorkflow gets called the Database has @@ -251,7 +253,7 @@ export class CredentialsHelper extends ICredentialsHelper { // Save the credentials in DB const findQuery = { - name, + id: credentials.id, type, }; diff --git a/packages/cli/src/GenericHelpers.ts b/packages/cli/src/GenericHelpers.ts index 7fe8c50ac4..73a2310929 100644 --- a/packages/cli/src/GenericHelpers.ts +++ b/packages/cli/src/GenericHelpers.ts @@ -7,7 +7,6 @@ import * as express from 'express'; import { join as pathJoin } from 'path'; import { readFile as fsReadFile } from 'fs/promises'; -import { readFileSync as fsReadFileSync } from 'fs'; import { IDataObject } from 'n8n-workflow'; import * as config from '../config'; @@ -137,45 +136,6 @@ export async function getConfigValue( return data; } -/** - * Gets value from config with support for "_FILE" environment variables synchronously - * - * @export - * @param {string} configKey The key of the config data to get - * @returns {(string | boolean | number | undefined)} - */ -export function getConfigValueSync(configKey: string): string | boolean | number | undefined { - // Get the environment variable - const configSchema = config.getSchema(); - // @ts-ignore - const currentSchema = extractSchemaForKey(configKey, configSchema._cvtProperties as IDataObject); - // Check if environment variable is defined for config key - if (currentSchema.env === undefined) { - // No environment variable defined, so return value from config - return config.get(configKey); - } - - // Check if special file enviroment variable exists - const fileEnvironmentVariable = process.env[`${currentSchema.env}_FILE`]; - if (fileEnvironmentVariable === undefined) { - // Does not exist, so return value from config - return config.get(configKey); - } - - let data; - try { - data = fsReadFileSync(fileEnvironmentVariable, 'utf8'); - } catch (error) { - if (error.code === 'ENOENT') { - throw new Error(`The file "${fileEnvironmentVariable}" could not be found.`); - } - - throw error; - } - - return data; -} - /** * Generate a unique name for a workflow or credentials entity. * diff --git a/packages/cli/src/Server.ts b/packages/cli/src/Server.ts index a23c5eedb9..f19c6a922f 100644 --- a/packages/cli/src/Server.ts +++ b/packages/cli/src/Server.ts @@ -66,6 +66,7 @@ import { ICredentialType, IDataObject, INodeCredentials, + INodeCredentialsDetails, INodeParameters, INodePropertyOptions, INodeType, @@ -642,6 +643,9 @@ class App { }); } + // check credentials for old format + await WorkflowHelpers.replaceInvalidCredentials(newWorkflow); + await this.externalHooks.run('workflow.create', [newWorkflow]); await WorkflowHelpers.validateWorkflow(newWorkflow); @@ -782,6 +786,9 @@ class App { const { id } = req.params; updateData.id = id; + // check credentials for old format + await WorkflowHelpers.replaceInvalidCredentials(updateData as WorkflowEntity); + await this.externalHooks.run('workflow.update', [updateData]); const isActive = await this.activeWorkflowRunner.isActive(id); @@ -1293,26 +1300,9 @@ class App { throw new Error('Credentials have to have a name set!'); } - // Check if credentials with the same name and type exist already - const findQuery = { - where: { - name: incomingData.name, - type: incomingData.type, - }, - } as FindOneOptions; - - const checkResult = await Db.collections.Credentials!.findOne(findQuery); - if (checkResult !== undefined) { - throw new ResponseHelper.ResponseError( - `Credentials with the same type and name exist already.`, - undefined, - 400, - ); - } - // Encrypt the data const credentials = new Credentials( - incomingData.name, + { id: null, name: incomingData.name }, incomingData.type, incomingData.nodesAccess, ); @@ -1321,10 +1311,6 @@ class App { await this.externalHooks.run('credentials.create', [newCredentialsData]); - // Add special database related data - - // TODO: also add user automatically depending on who is logged in, if anybody is logged in - // Save the credentials in DB const result = await Db.collections.Credentials!.save(newCredentialsData); result.data = incomingData.data; @@ -1445,24 +1431,6 @@ class App { } } - // Check if credentials with the same name and type exist already - const findQuery = { - where: { - id: Not(id), - name: incomingData.name, - type: incomingData.type, - }, - } as FindOneOptions; - - const checkResult = await Db.collections.Credentials!.findOne(findQuery); - if (checkResult !== undefined) { - throw new ResponseHelper.ResponseError( - `Credentials with the same type and name exist already.`, - undefined, - 400, - ); - } - const encryptionKey = await UserSettings.getEncryptionKey(); if (encryptionKey === undefined) { throw new Error('No encryption key got found to encrypt the credentials!'); @@ -1479,7 +1447,7 @@ class App { } const currentlySavedCredentials = new Credentials( - result.name, + result as INodeCredentialsDetails, result.type, result.nodesAccess, result.data, @@ -1494,7 +1462,7 @@ class App { // Encrypt the data const credentials = new Credentials( - incomingData.name, + { id, name: incomingData.name }, incomingData.type, incomingData.nodesAccess, ); @@ -1563,7 +1531,7 @@ class App { } const credentials = new Credentials( - result.name, + result as INodeCredentialsDetails, result.type, result.nodesAccess, result.data, @@ -1707,7 +1675,7 @@ class App { const mode: WorkflowExecuteMode = 'internal'; const credentialsHelper = new CredentialsHelper(encryptionKey); const decryptedDataOriginal = await credentialsHelper.getDecrypted( - result.name, + result as INodeCredentialsDetails, result.type, mode, true, @@ -1766,7 +1734,11 @@ class App { }`; // Encrypt the data - const credentials = new Credentials(result.name, result.type, result.nodesAccess); + const credentials = new Credentials( + result as INodeCredentialsDetails, + result.type, + result.nodesAccess, + ); credentials.setData(decryptedDataOriginal, encryptionKey); const newCredentialsData = credentials.getDataToSave() as unknown as ICredentialsDb; @@ -1823,13 +1795,13 @@ class App { // Decrypt the currently saved credentials const workflowCredentials: IWorkflowCredentials = { [result.type]: { - [result.name]: result as ICredentialsEncrypted, + [result.id.toString()]: result as ICredentialsEncrypted, }, }; const mode: WorkflowExecuteMode = 'internal'; const credentialsHelper = new CredentialsHelper(encryptionKey); const decryptedDataOriginal = await credentialsHelper.getDecrypted( - result.name, + result as INodeCredentialsDetails, result.type, mode, true, @@ -1868,7 +1840,11 @@ class App { decryptedDataOriginal.oauthTokenData = oauthTokenJson; - const credentials = new Credentials(result.name, result.type, result.nodesAccess); + const credentials = new Credentials( + result as INodeCredentialsDetails, + result.type, + result.nodesAccess, + ); credentials.setData(decryptedDataOriginal, encryptionKey); const newCredentialsData = credentials.getDataToSave() as unknown as ICredentialsDb; // Add special database related data @@ -1913,7 +1889,7 @@ class App { const mode: WorkflowExecuteMode = 'internal'; const credentialsHelper = new CredentialsHelper(encryptionKey); const decryptedDataOriginal = await credentialsHelper.getDecrypted( - result.name, + result as INodeCredentialsDetails, result.type, mode, true, @@ -1950,7 +1926,11 @@ class App { const oAuthObj = new clientOAuth2(oAuthOptions); // Encrypt the data - const credentials = new Credentials(result.name, result.type, result.nodesAccess); + const credentials = new Credentials( + result as INodeCredentialsDetails, + result.type, + result.nodesAccess, + ); decryptedDataOriginal.csrfSecret = csrfSecret; credentials.setData(decryptedDataOriginal, encryptionKey); @@ -2039,14 +2019,14 @@ class App { // Decrypt the currently saved credentials const workflowCredentials: IWorkflowCredentials = { [result.type]: { - [result.name]: result as ICredentialsEncrypted, + [result.id.toString()]: result as ICredentialsEncrypted, }, }; const mode: WorkflowExecuteMode = 'internal'; const credentialsHelper = new CredentialsHelper(encryptionKey); const decryptedDataOriginal = await credentialsHelper.getDecrypted( - result.name, + result as INodeCredentialsDetails, result.type, mode, true, @@ -2128,7 +2108,11 @@ class App { _.unset(decryptedDataOriginal, 'csrfSecret'); - const credentials = new Credentials(result.name, result.type, result.nodesAccess); + const credentials = new Credentials( + result as INodeCredentialsDetails, + result.type, + result.nodesAccess, + ); credentials.setData(decryptedDataOriginal, encryptionKey); const newCredentialsData = credentials.getDataToSave() as unknown as ICredentialsDb; // Add special database related data diff --git a/packages/cli/src/WorkflowCredentials.ts b/packages/cli/src/WorkflowCredentials.ts index 622f1d9b0e..a52232866f 100644 --- a/packages/cli/src/WorkflowCredentials.ts +++ b/packages/cli/src/WorkflowCredentials.ts @@ -10,7 +10,7 @@ export async function WorkflowCredentials(nodes: INode[]): Promise { + const { nodes } = workflow; + if (!nodes) return workflow; + + // caching + const credentialsByName: Record> = {}; + const credentialsById: Record> = {}; + + // for loop to run DB fetches sequential and use cache to keep pressure off DB + // trade-off: longer response time for less DB queries + /* eslint-disable no-await-in-loop */ + for (const node of nodes) { + if (!node.credentials || node.disabled) { + continue; + } + // extract credentials types + const allNodeCredentials = Object.entries(node.credentials); + for (const [nodeCredentialType, nodeCredentials] of allNodeCredentials) { + // Check if Node applies old credentials style + if (typeof nodeCredentials === 'string' || nodeCredentials.id === null) { + const name = typeof nodeCredentials === 'string' ? nodeCredentials : nodeCredentials.name; + // init cache for type + if (!credentialsByName[nodeCredentialType]) { + credentialsByName[nodeCredentialType] = {}; + } + if (credentialsByName[nodeCredentialType][name] === undefined) { + const credentials = await Db.collections.Credentials?.find({ + name, + type: nodeCredentialType, + }); + // if credential name-type combination is unique, use it + if (credentials?.length === 1) { + credentialsByName[nodeCredentialType][name] = { + id: credentials[0].id.toString(), + name: credentials[0].name, + }; + node.credentials[nodeCredentialType] = credentialsByName[nodeCredentialType][name]; + continue; + } + + // nothing found - add invalid credentials to cache to prevent further DB checks + credentialsByName[nodeCredentialType][name] = { + id: null, + name, + }; + } else { + // get credentials from cache + node.credentials[nodeCredentialType] = credentialsByName[nodeCredentialType][name]; + } + continue; + } + + // Node has credentials with an ID + + // init cache for type + if (!credentialsById[nodeCredentialType]) { + credentialsById[nodeCredentialType] = {}; + } + + // check if credentials for ID-type are not yet cached + if (credentialsById[nodeCredentialType][nodeCredentials.id] === undefined) { + // check first if ID-type combination exists + const credentials = await Db.collections.Credentials?.findOne({ + id: nodeCredentials.id, + type: nodeCredentialType, + }); + if (credentials) { + credentialsById[nodeCredentialType][nodeCredentials.id] = { + id: credentials.id.toString(), + name: credentials.name, + }; + node.credentials[nodeCredentialType] = + credentialsById[nodeCredentialType][nodeCredentials.id]; + continue; + } + // no credentials found for ID, check if some exist for name + const credsByName = await Db.collections.Credentials?.find({ + name: nodeCredentials.name, + type: nodeCredentialType, + }); + // if credential name-type combination is unique, take it + if (credsByName?.length === 1) { + // add found credential to cache + credentialsById[nodeCredentialType][credsByName[0].id] = { + id: credsByName[0].id.toString(), + name: credsByName[0].name, + }; + node.credentials[nodeCredentialType] = + credentialsById[nodeCredentialType][credsByName[0].id]; + continue; + } + + // nothing found - add invalid credentials to cache to prevent further DB checks + credentialsById[nodeCredentialType][nodeCredentials.id] = nodeCredentials; + continue; + } + + // get credentials from cache + node.credentials[nodeCredentialType] = + credentialsById[nodeCredentialType][nodeCredentials.id]; + } + } + /* eslint-enable no-await-in-loop */ + return workflow; +} + // TODO: Deduplicate `validateWorkflow` and `throwDuplicateEntryError` with TagHelpers? // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types diff --git a/packages/cli/src/databases/entities/CredentialsEntity.ts b/packages/cli/src/databases/entities/CredentialsEntity.ts index 5cf65fb0c3..50b6f168e6 100644 --- a/packages/cli/src/databases/entities/CredentialsEntity.ts +++ b/packages/cli/src/databases/entities/CredentialsEntity.ts @@ -11,9 +11,40 @@ import { PrimaryGeneratedColumn, UpdateDateColumn, } from 'typeorm'; -import { getTimestampSyntax, resolveDataType } from '../utils'; -import { ICredentialsDb } from '../..'; +import config = require('../../../config'); +import { DatabaseType, ICredentialsDb } from '../..'; + +function resolveDataType(dataType: string) { + const dbType = config.get('database.type') as DatabaseType; + + const typeMap: { [key in DatabaseType]: { [key: string]: string } } = { + sqlite: { + json: 'simple-json', + }, + postgresdb: { + datetime: 'timestamptz', + }, + mysqldb: {}, + mariadb: {}, + }; + + return typeMap[dbType][dataType] ?? dataType; +} + +// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types +function getTimestampSyntax() { + const dbType = config.get('database.type') as DatabaseType; + + const map: { [key in DatabaseType]: string } = { + sqlite: "STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')", + postgresdb: 'CURRENT_TIMESTAMP(3)', + mysqldb: 'CURRENT_TIMESTAMP(3)', + mariadb: 'CURRENT_TIMESTAMP(3)', + }; + + return map[dbType]; +} @Entity() export class CredentialsEntity implements ICredentialsDb { diff --git a/packages/cli/src/databases/entities/ExecutionEntity.ts b/packages/cli/src/databases/entities/ExecutionEntity.ts index 564ec9a4e3..1777727b91 100644 --- a/packages/cli/src/databases/entities/ExecutionEntity.ts +++ b/packages/cli/src/databases/entities/ExecutionEntity.ts @@ -2,9 +2,25 @@ import { WorkflowExecuteMode } from 'n8n-workflow'; import { Column, ColumnOptions, Entity, Index, PrimaryGeneratedColumn } from 'typeorm'; -import { IExecutionFlattedDb, IWorkflowDb } from '../..'; +import config = require('../../../config'); +import { DatabaseType, IExecutionFlattedDb, IWorkflowDb } from '../..'; -import { resolveDataType } from '../utils'; +function resolveDataType(dataType: string) { + const dbType = config.get('database.type') as DatabaseType; + + const typeMap: { [key in DatabaseType]: { [key: string]: string } } = { + sqlite: { + json: 'simple-json', + }, + postgresdb: { + datetime: 'timestamptz', + }, + mysqldb: {}, + mariadb: {}, + }; + + return typeMap[dbType][dataType] ?? dataType; +} @Entity() export class ExecutionEntity implements IExecutionFlattedDb { diff --git a/packages/cli/src/databases/entities/TagEntity.ts b/packages/cli/src/databases/entities/TagEntity.ts index 445a104af9..a845313ea7 100644 --- a/packages/cli/src/databases/entities/TagEntity.ts +++ b/packages/cli/src/databases/entities/TagEntity.ts @@ -12,9 +12,24 @@ import { } from 'typeorm'; import { IsDate, IsOptional, IsString, Length } from 'class-validator'; +import config = require('../../../config'); +import { DatabaseType } from '../../index'; import { ITagDb } from '../../Interfaces'; import { WorkflowEntity } from './WorkflowEntity'; -import { getTimestampSyntax } from '../utils'; + +// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types +function getTimestampSyntax() { + const dbType = config.get('database.type') as DatabaseType; + + const map: { [key in DatabaseType]: string } = { + sqlite: "STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')", + postgresdb: 'CURRENT_TIMESTAMP(3)', + mysqldb: 'CURRENT_TIMESTAMP(3)', + mariadb: 'CURRENT_TIMESTAMP(3)', + }; + + return map[dbType]; +} @Entity() export class TagEntity implements ITagDb { diff --git a/packages/cli/src/databases/entities/WorkflowEntity.ts b/packages/cli/src/databases/entities/WorkflowEntity.ts index 88eb44d7ff..b07c4e6a06 100644 --- a/packages/cli/src/databases/entities/WorkflowEntity.ts +++ b/packages/cli/src/databases/entities/WorkflowEntity.ts @@ -17,12 +17,41 @@ import { UpdateDateColumn, } from 'typeorm'; -import { IWorkflowDb } from '../..'; - -import { getTimestampSyntax, resolveDataType } from '../utils'; - +import config = require('../../../config'); +import { DatabaseType, IWorkflowDb } from '../..'; import { TagEntity } from './TagEntity'; +function resolveDataType(dataType: string) { + const dbType = config.get('database.type') as DatabaseType; + + const typeMap: { [key in DatabaseType]: { [key: string]: string } } = { + sqlite: { + json: 'simple-json', + }, + postgresdb: { + datetime: 'timestamptz', + }, + mysqldb: {}, + mariadb: {}, + }; + + return typeMap[dbType][dataType] ?? dataType; +} + +// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types +function getTimestampSyntax() { + const dbType = config.get('database.type') as DatabaseType; + + const map: { [key in DatabaseType]: string } = { + sqlite: "STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')", + postgresdb: 'CURRENT_TIMESTAMP(3)', + mysqldb: 'CURRENT_TIMESTAMP(3)', + mariadb: 'CURRENT_TIMESTAMP(3)', + }; + + return map[dbType]; +} + @Entity() export class WorkflowEntity implements IWorkflowDb { @PrimaryGeneratedColumn() diff --git a/packages/cli/src/databases/mysqldb/migrations/1630451444017-UpdateWorkflowCredentials.ts b/packages/cli/src/databases/mysqldb/migrations/1630451444017-UpdateWorkflowCredentials.ts new file mode 100644 index 0000000000..0012ee0aa1 --- /dev/null +++ b/packages/cli/src/databases/mysqldb/migrations/1630451444017-UpdateWorkflowCredentials.ts @@ -0,0 +1,215 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; +import config = require('../../../../config'); + +// replacing the credentials in workflows and execution +// `nodeType: name` changes to `nodeType: { id, name }` + +export class UpdateWorkflowCredentials1630451444017 implements MigrationInterface { + name = 'UpdateWorkflowCredentials1630451444017'; + + public async up(queryRunner: QueryRunner): Promise { + const tablePrefix = config.get('database.tablePrefix'); + + const credentialsEntities = await queryRunner.query(` + SELECT id, name, type + FROM ${tablePrefix}credentials_entity + `); + + const workflows = await queryRunner.query(` + SELECT id, nodes + FROM ${tablePrefix}workflow_entity + `); + // @ts-ignore + workflows.forEach(async (workflow) => { + const nodes = workflow.nodes; + let credentialsUpdated = false; + // @ts-ignore + nodes.forEach((node) => { + if (node.credentials) { + const allNodeCredentials = Object.entries(node.credentials); + for (const [type, name] of allNodeCredentials) { + if (typeof name === 'string') { + // @ts-ignore + const matchingCredentials = credentialsEntities.find( + // @ts-ignore + (credentials) => credentials.name === name && credentials.type === type, + ); + node.credentials[type] = { id: matchingCredentials?.id.toString() || null, name }; + credentialsUpdated = true; + } + } + } + }); + if (credentialsUpdated) { + const [updateQuery, updateParams] = queryRunner.connection.driver.escapeQueryWithParameters( + ` + UPDATE ${tablePrefix}workflow_entity + SET nodes = :nodes + WHERE id = '${workflow.id}' + `, + { nodes: JSON.stringify(nodes) }, + {}, + ); + + await queryRunner.query(updateQuery, updateParams); + } + }); + + const waitingExecutions = await queryRunner.query(` + SELECT id, workflowData + FROM ${tablePrefix}execution_entity + WHERE waitTill IS NOT NULL AND finished = 0 + `); + + const retryableExecutions = await queryRunner.query(` + SELECT id, workflowData + FROM ${tablePrefix}execution_entity + WHERE waitTill IS NULL AND finished = 0 AND mode != 'retry' + ORDER BY startedAt DESC + LIMIT 200 + `); + + [...waitingExecutions, ...retryableExecutions].forEach(async (execution) => { + const data = execution.workflowData; + let credentialsUpdated = false; + // @ts-ignore + data.nodes.forEach((node) => { + if (node.credentials) { + const allNodeCredentials = Object.entries(node.credentials); + for (const [type, name] of allNodeCredentials) { + if (typeof name === 'string') { + // @ts-ignore + const matchingCredentials = credentialsEntities.find( + // @ts-ignore + (credentials) => credentials.name === name && credentials.type === type, + ); + node.credentials[type] = { id: matchingCredentials?.id.toString() || null, name }; + credentialsUpdated = true; + } + } + } + }); + if (credentialsUpdated) { + const [updateQuery, updateParams] = queryRunner.connection.driver.escapeQueryWithParameters( + ` + UPDATE ${tablePrefix}execution_entity + SET workflowData = :data + WHERE id = '${execution.id}' + `, + { data: JSON.stringify(data) }, + {}, + ); + + await queryRunner.query(updateQuery, updateParams); + } + }); + } + + public async down(queryRunner: QueryRunner): Promise { + const tablePrefix = config.get('database.tablePrefix'); + + const credentialsEntities = await queryRunner.query(` + SELECT id, name, type + FROM ${tablePrefix}credentials_entity + `); + + const workflows = await queryRunner.query(` + SELECT id, nodes + FROM ${tablePrefix}workflow_entity + `); + // @ts-ignore + workflows.forEach(async (workflow) => { + const nodes = workflow.nodes; + let credentialsUpdated = false; + // @ts-ignore + nodes.forEach((node) => { + if (node.credentials) { + const allNodeCredentials = Object.entries(node.credentials); + for (const [type, creds] of allNodeCredentials) { + if (typeof creds === 'object') { + // @ts-ignore + const matchingCredentials = credentialsEntities.find( + // @ts-ignore + (credentials) => credentials.id === creds.id && credentials.type === type, + ); + if (matchingCredentials) { + node.credentials[type] = matchingCredentials.name; + } else { + // @ts-ignore + node.credentials[type] = creds.name; + } + credentialsUpdated = true; + } + } + } + }); + if (credentialsUpdated) { + const [updateQuery, updateParams] = queryRunner.connection.driver.escapeQueryWithParameters( + ` + UPDATE ${tablePrefix}workflow_entity + SET nodes = :nodes + WHERE id = '${workflow.id}' + `, + { nodes: JSON.stringify(nodes) }, + {}, + ); + + await queryRunner.query(updateQuery, updateParams); + } + }); + + const waitingExecutions = await queryRunner.query(` + SELECT id, workflowData + FROM ${tablePrefix}execution_entity + WHERE waitTill IS NOT NULL AND finished = 0 + `); + + const retryableExecutions = await queryRunner.query(` + SELECT id, workflowData + FROM ${tablePrefix}execution_entity + WHERE waitTill IS NULL AND finished = 0 AND mode != 'retry' + ORDER BY startedAt DESC + LIMIT 200 + `); + + [...waitingExecutions, ...retryableExecutions].forEach(async (execution) => { + const data = execution.workflowData; + let credentialsUpdated = false; + // @ts-ignore + data.nodes.forEach((node) => { + if (node.credentials) { + const allNodeCredentials = Object.entries(node.credentials); + for (const [type, creds] of allNodeCredentials) { + if (typeof creds === 'object') { + // @ts-ignore + const matchingCredentials = credentialsEntities.find( + // @ts-ignore + (credentials) => credentials.id === creds.id && credentials.type === type, + ); + if (matchingCredentials) { + node.credentials[type] = matchingCredentials.name; + } else { + // @ts-ignore + node.credentials[type] = creds.name; + } + credentialsUpdated = true; + } + } + } + }); + if (credentialsUpdated) { + const [updateQuery, updateParams] = queryRunner.connection.driver.escapeQueryWithParameters( + ` + UPDATE ${tablePrefix}execution_entity + SET workflowData = :data + WHERE id = '${execution.id}' + `, + { data: JSON.stringify(data) }, + {}, + ); + + await queryRunner.query(updateQuery, updateParams); + } + }); + } +} diff --git a/packages/cli/src/databases/mysqldb/migrations/index.ts b/packages/cli/src/databases/mysqldb/migrations/index.ts index b48bc58aff..0d3501eb3f 100644 --- a/packages/cli/src/databases/mysqldb/migrations/index.ts +++ b/packages/cli/src/databases/mysqldb/migrations/index.ts @@ -9,6 +9,7 @@ import { CreateTagEntity1617268711084 } from './1617268711084-CreateTagEntity'; import { UniqueWorkflowNames1620826335440 } from './1620826335440-UniqueWorkflowNames'; import { CertifyCorrectCollation1623936588000 } from './1623936588000-CertifyCorrectCollation'; import { AddWaitColumnId1626183952959 } from './1626183952959-AddWaitColumn'; +import { UpdateWorkflowCredentials1630451444017 } from './1630451444017-UpdateWorkflowCredentials'; export const mysqlMigrations = [ InitialMigration1588157391238, @@ -22,4 +23,5 @@ export const mysqlMigrations = [ UniqueWorkflowNames1620826335440, CertifyCorrectCollation1623936588000, AddWaitColumnId1626183952959, + UpdateWorkflowCredentials1630451444017, ]; diff --git a/packages/cli/src/databases/postgresdb/migrations/1630419189837-UpdateWorkflowCredentials.ts b/packages/cli/src/databases/postgresdb/migrations/1630419189837-UpdateWorkflowCredentials.ts new file mode 100644 index 0000000000..357d7c2974 --- /dev/null +++ b/packages/cli/src/databases/postgresdb/migrations/1630419189837-UpdateWorkflowCredentials.ts @@ -0,0 +1,223 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; +import config = require('../../../../config'); + +// replacing the credentials in workflows and execution +// `nodeType: name` changes to `nodeType: { id, name }` + +export class UpdateWorkflowCredentials1630419189837 implements MigrationInterface { + name = 'UpdateWorkflowCredentials1630419189837'; + + public async up(queryRunner: QueryRunner): Promise { + let tablePrefix = config.get('database.tablePrefix'); + const schema = config.get('database.postgresdb.schema'); + if (schema) { + tablePrefix = schema + '.' + tablePrefix; + } + + const credentialsEntities = await queryRunner.query(` + SELECT id, name, type + FROM ${tablePrefix}credentials_entity + `); + + const workflows = await queryRunner.query(` + SELECT id, nodes + FROM ${tablePrefix}workflow_entity + `); + // @ts-ignore + workflows.forEach(async (workflow) => { + const nodes = workflow.nodes; + let credentialsUpdated = false; + // @ts-ignore + nodes.forEach((node) => { + if (node.credentials) { + const allNodeCredentials = Object.entries(node.credentials); + for (const [type, name] of allNodeCredentials) { + if (typeof name === 'string') { + // @ts-ignore + const matchingCredentials = credentialsEntities.find( + // @ts-ignore + (credentials) => credentials.name === name && credentials.type === type, + ); + node.credentials[type] = { id: matchingCredentials?.id.toString() || null, name }; + credentialsUpdated = true; + } + } + } + }); + if (credentialsUpdated) { + const [updateQuery, updateParams] = queryRunner.connection.driver.escapeQueryWithParameters( + ` + UPDATE ${tablePrefix}workflow_entity + SET nodes = :nodes + WHERE id = '${workflow.id}' + `, + { nodes: JSON.stringify(nodes) }, + {}, + ); + + await queryRunner.query(updateQuery, updateParams); + } + }); + + const waitingExecutions = await queryRunner.query(` + SELECT id, "workflowData" + FROM ${tablePrefix}execution_entity + WHERE "waitTill" IS NOT NULL AND finished = FALSE + `); + + const retryableExecutions = await queryRunner.query(` + SELECT id, "workflowData" + FROM ${tablePrefix}execution_entity + WHERE "waitTill" IS NULL AND finished = FALSE AND mode != 'retry' + ORDER BY "startedAt" DESC + LIMIT 200 + `); + + [...waitingExecutions, ...retryableExecutions].forEach(async (execution) => { + const data = execution.workflowData; + let credentialsUpdated = false; + // @ts-ignore + data.nodes.forEach((node) => { + if (node.credentials) { + const allNodeCredentials = Object.entries(node.credentials); + for (const [type, name] of allNodeCredentials) { + if (typeof name === 'string') { + // @ts-ignore + const matchingCredentials = credentialsEntities.find( + // @ts-ignore + (credentials) => credentials.name === name && credentials.type === type, + ); + node.credentials[type] = { id: matchingCredentials?.id.toString() || null, name }; + credentialsUpdated = true; + } + } + } + }); + if (credentialsUpdated) { + const [updateQuery, updateParams] = queryRunner.connection.driver.escapeQueryWithParameters( + ` + UPDATE ${tablePrefix}execution_entity + SET "workflowData" = :data + WHERE id = '${execution.id}' + `, + { data: JSON.stringify(data) }, + {}, + ); + + await queryRunner.query(updateQuery, updateParams); + } + }); + } + + public async down(queryRunner: QueryRunner): Promise { + let tablePrefix = config.get('database.tablePrefix'); + const schema = config.get('database.postgresdb.schema'); + if (schema) { + tablePrefix = schema + '.' + tablePrefix; + } + + const credentialsEntities = await queryRunner.query(` + SELECT id, name, type + FROM ${tablePrefix}credentials_entity + `); + + const workflows = await queryRunner.query(` + SELECT id, nodes + FROM ${tablePrefix}workflow_entity + `); + // @ts-ignore + workflows.forEach(async (workflow) => { + const nodes = workflow.nodes; + let credentialsUpdated = false; + // @ts-ignore + nodes.forEach((node) => { + if (node.credentials) { + const allNodeCredentials = Object.entries(node.credentials); + for (const [type, creds] of allNodeCredentials) { + if (typeof creds === 'object') { + // @ts-ignore + const matchingCredentials = credentialsEntities.find( + // @ts-ignore + (credentials) => credentials.id === creds.id && credentials.type === type, + ); + if (matchingCredentials) { + node.credentials[type] = matchingCredentials.name; + } else { + // @ts-ignore + node.credentials[type] = creds.name; + } + credentialsUpdated = true; + } + } + } + }); + if (credentialsUpdated) { + const [updateQuery, updateParams] = queryRunner.connection.driver.escapeQueryWithParameters( + ` + UPDATE ${tablePrefix}workflow_entity + SET nodes = :nodes + WHERE id = '${workflow.id}' + `, + { nodes: JSON.stringify(nodes) }, + {}, + ); + + await queryRunner.query(updateQuery, updateParams); + } + }); + + const waitingExecutions = await queryRunner.query(` + SELECT id, "workflowData" + FROM ${tablePrefix}execution_entity + WHERE "waitTill" IS NOT NULL AND finished = FALSE + `); + + const retryableExecutions = await queryRunner.query(` + SELECT id, "workflowData" + FROM ${tablePrefix}execution_entity + WHERE "waitTill" IS NULL AND finished = FALSE AND mode != 'retry' + ORDER BY "startedAt" DESC + LIMIT 200 + `); + + [...waitingExecutions, ...retryableExecutions].forEach(async (execution) => { + const data = execution.workflowData; + let credentialsUpdated = false; + // @ts-ignore + data.nodes.forEach((node) => { + if (node.credentials) { + const allNodeCredentials = Object.entries(node.credentials); + for (const [type, creds] of allNodeCredentials) { + if (typeof creds === 'object') { + // @ts-ignore + const matchingCredentials = credentialsEntities.find( + // @ts-ignore + (credentials) => credentials.id === creds.id && credentials.type === type, + ); + if (matchingCredentials) { + node.credentials[type] = matchingCredentials.name; + } else { + // @ts-ignore + node.credentials[type] = creds.name; + } + credentialsUpdated = true; + } + } + } + }); + if (credentialsUpdated) { + const [updateQuery, updateParams] = queryRunner.connection.driver.escapeQueryWithParameters( + ` + UPDATE ${tablePrefix}execution_entity + SET "workflowData" = :data + WHERE id = '${execution.id}' + `, + { data: JSON.stringify(data) }, + {}, + ); + + await queryRunner.query(updateQuery, updateParams); + } + }); + } +} diff --git a/packages/cli/src/databases/postgresdb/migrations/index.ts b/packages/cli/src/databases/postgresdb/migrations/index.ts index 83983dd039..f885c97851 100644 --- a/packages/cli/src/databases/postgresdb/migrations/index.ts +++ b/packages/cli/src/databases/postgresdb/migrations/index.ts @@ -6,6 +6,7 @@ import { MakeStoppedAtNullable1607431743768 } from './1607431743768-MakeStoppedA import { CreateTagEntity1617270242566 } from './1617270242566-CreateTagEntity'; import { UniqueWorkflowNames1620824779533 } from './1620824779533-UniqueWorkflowNames'; import { AddwaitTill1626176912946 } from './1626176912946-AddwaitTill'; +import { UpdateWorkflowCredentials1630419189837 } from './1630419189837-UpdateWorkflowCredentials'; export const postgresMigrations = [ InitialMigration1587669153312, @@ -16,4 +17,5 @@ export const postgresMigrations = [ CreateTagEntity1617270242566, UniqueWorkflowNames1620824779533, AddwaitTill1626176912946, + UpdateWorkflowCredentials1630419189837, ]; diff --git a/packages/cli/src/databases/sqlite/migrations/1630330987096-UpdateWorkflowCredentials.ts b/packages/cli/src/databases/sqlite/migrations/1630330987096-UpdateWorkflowCredentials.ts new file mode 100644 index 0000000000..f2a6f0a19a --- /dev/null +++ b/packages/cli/src/databases/sqlite/migrations/1630330987096-UpdateWorkflowCredentials.ts @@ -0,0 +1,215 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; +import config = require('../../../../config'); + +// replacing the credentials in workflows and execution +// `nodeType: name` changes to `nodeType: { id, name }` + +export class UpdateWorkflowCredentials1630330987096 implements MigrationInterface { + name = 'UpdateWorkflowCredentials1630330987096'; + + public async up(queryRunner: QueryRunner): Promise { + const tablePrefix = config.get('database.tablePrefix'); + + const credentialsEntities = await queryRunner.query(` + SELECT id, name, type + FROM "${tablePrefix}credentials_entity" + `); + + const workflows = await queryRunner.query(` + SELECT id, nodes + FROM "${tablePrefix}workflow_entity" + `); + // @ts-ignore + workflows.forEach(async (workflow) => { + const nodes = JSON.parse(workflow.nodes); + let credentialsUpdated = false; + // @ts-ignore + nodes.forEach((node) => { + if (node.credentials) { + const allNodeCredentials = Object.entries(node.credentials); + for (const [type, name] of allNodeCredentials) { + if (typeof name === 'string') { + // @ts-ignore + const matchingCredentials = credentialsEntities.find( + // @ts-ignore + (credentials) => credentials.name === name && credentials.type === type, + ); + node.credentials[type] = { id: matchingCredentials?.id || null, name }; + credentialsUpdated = true; + } + } + } + }); + if (credentialsUpdated) { + const [updateQuery, updateParams] = queryRunner.connection.driver.escapeQueryWithParameters( + ` + UPDATE "${tablePrefix}workflow_entity" + SET nodes = :nodes + WHERE id = '${workflow.id}' + `, + { nodes: JSON.stringify(nodes) }, + {}, + ); + + await queryRunner.query(updateQuery, updateParams); + } + }); + + const waitingExecutions = await queryRunner.query(` + SELECT id, "workflowData" + FROM "${tablePrefix}execution_entity" + WHERE "waitTill" IS NOT NULL AND finished = 0 + `); + + const retryableExecutions = await queryRunner.query(` + SELECT id, "workflowData" + FROM "${tablePrefix}execution_entity" + WHERE "waitTill" IS NULL AND finished = 0 AND mode != 'retry' + ORDER BY "startedAt" DESC + LIMIT 200 + `); + + [...waitingExecutions, ...retryableExecutions].forEach(async (execution) => { + const data = JSON.parse(execution.workflowData); + let credentialsUpdated = false; + // @ts-ignore + data.nodes.forEach((node) => { + if (node.credentials) { + const allNodeCredentials = Object.entries(node.credentials); + for (const [type, name] of allNodeCredentials) { + if (typeof name === 'string') { + // @ts-ignore + const matchingCredentials = credentialsEntities.find( + // @ts-ignore + (credentials) => credentials.name === name && credentials.type === type, + ); + node.credentials[type] = { id: matchingCredentials?.id || null, name }; + credentialsUpdated = true; + } + } + } + }); + if (credentialsUpdated) { + const [updateQuery, updateParams] = queryRunner.connection.driver.escapeQueryWithParameters( + ` + UPDATE "${tablePrefix}execution_entity" + SET "workflowData" = :data + WHERE id = '${execution.id}' + `, + { data: JSON.stringify(data) }, + {}, + ); + + await queryRunner.query(updateQuery, updateParams); + } + }); + } + + public async down(queryRunner: QueryRunner): Promise { + const tablePrefix = config.get('database.tablePrefix'); + + const credentialsEntities = await queryRunner.query(` + SELECT id, name, type + FROM "${tablePrefix}credentials_entity" + `); + + const workflows = await queryRunner.query(` + SELECT id, nodes + FROM "${tablePrefix}workflow_entity" + `); + // @ts-ignore + workflows.forEach(async (workflow) => { + const nodes = JSON.parse(workflow.nodes); + let credentialsUpdated = false; + // @ts-ignore + nodes.forEach((node) => { + if (node.credentials) { + const allNodeCredentials = Object.entries(node.credentials); + for (const [type, creds] of allNodeCredentials) { + if (typeof creds === 'object') { + // @ts-ignore + const matchingCredentials = credentialsEntities.find( + // @ts-ignore + (credentials) => credentials.id === creds.id && credentials.type === type, + ); + if (matchingCredentials) { + node.credentials[type] = matchingCredentials.name; + } else { + // @ts-ignore + node.credentials[type] = creds.name; + } + credentialsUpdated = true; + } + } + } + }); + if (credentialsUpdated) { + const [updateQuery, updateParams] = queryRunner.connection.driver.escapeQueryWithParameters( + ` + UPDATE "${tablePrefix}workflow_entity" + SET nodes = :nodes + WHERE id = '${workflow.id}' + `, + { nodes: JSON.stringify(nodes) }, + {}, + ); + + await queryRunner.query(updateQuery, updateParams); + } + }); + + const waitingExecutions = await queryRunner.query(` + SELECT id, "workflowData" + FROM "${tablePrefix}execution_entity" + WHERE "waitTill" IS NOT NULL AND finished = 0 + `); + + const retryableExecutions = await queryRunner.query(` + SELECT id, "workflowData" + FROM "${tablePrefix}execution_entity" + WHERE "waitTill" IS NULL AND finished = 0 AND mode != 'retry' + ORDER BY "startedAt" DESC + LIMIT 200 + `); + + [...waitingExecutions, ...retryableExecutions].forEach(async (execution) => { + const data = JSON.parse(execution.workflowData); + let credentialsUpdated = false; + // @ts-ignore + data.nodes.forEach((node) => { + if (node.credentials) { + const allNodeCredentials = Object.entries(node.credentials); + for (const [type, creds] of allNodeCredentials) { + if (typeof creds === 'object') { + // @ts-ignore + const matchingCredentials = credentialsEntities.find( + // @ts-ignore + (credentials) => credentials.id === creds.id && credentials.type === type, + ); + if (matchingCredentials) { + node.credentials[type] = matchingCredentials.name; + } else { + // @ts-ignore + node.credentials[type] = creds.name; + } + credentialsUpdated = true; + } + } + } + }); + if (credentialsUpdated) { + const [updateQuery, updateParams] = queryRunner.connection.driver.escapeQueryWithParameters( + ` + UPDATE "${tablePrefix}execution_entity" + SET "workflowData" = :data + WHERE id = '${execution.id}' + `, + { data: JSON.stringify(data) }, + {}, + ); + + await queryRunner.query(updateQuery, updateParams); + } + }); + } +} diff --git a/packages/cli/src/databases/sqlite/migrations/index.ts b/packages/cli/src/databases/sqlite/migrations/index.ts index 64038d9e30..0e1907d2cc 100644 --- a/packages/cli/src/databases/sqlite/migrations/index.ts +++ b/packages/cli/src/databases/sqlite/migrations/index.ts @@ -6,6 +6,7 @@ import { MakeStoppedAtNullable1607431743769 } from './1607431743769-MakeStoppedA import { CreateTagEntity1617213344594 } from './1617213344594-CreateTagEntity'; import { UniqueWorkflowNames1620821879465 } from './1620821879465-UniqueWorkflowNames'; import { AddWaitColumn1621707690587 } from './1621707690587-AddWaitColumn'; +import { UpdateWorkflowCredentials1630330987096 } from './1630330987096-UpdateWorkflowCredentials'; export const sqliteMigrations = [ InitialMigration1588102412422, @@ -16,4 +17,5 @@ export const sqliteMigrations = [ CreateTagEntity1617213344594, UniqueWorkflowNames1620821879465, AddWaitColumn1621707690587, + UpdateWorkflowCredentials1630330987096, ]; diff --git a/packages/cli/src/databases/utils.ts b/packages/cli/src/databases/utils.ts deleted file mode 100644 index 3a0e9b08c3..0000000000 --- a/packages/cli/src/databases/utils.ts +++ /dev/null @@ -1,42 +0,0 @@ -/* eslint-disable import/no-cycle */ -import { DatabaseType } from '../index'; -import { getConfigValueSync } from '../GenericHelpers'; - -/** - * Resolves the data type for the used database type - * - * @export - * @param {string} dataType - * @returns {string} - */ -// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types -export function resolveDataType(dataType: string) { - const dbType = getConfigValueSync('database.type') as DatabaseType; - - const typeMap: { [key in DatabaseType]: { [key: string]: string } } = { - sqlite: { - json: 'simple-json', - }, - postgresdb: { - datetime: 'timestamptz', - }, - mysqldb: {}, - mariadb: {}, - }; - - return typeMap[dbType][dataType] ?? dataType; -} - -// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types -export function getTimestampSyntax() { - const dbType = getConfigValueSync('database.type') as DatabaseType; - - const map: { [key in DatabaseType]: string } = { - sqlite: "STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')", - postgresdb: 'CURRENT_TIMESTAMP(3)', - mysqldb: 'CURRENT_TIMESTAMP(3)', - mariadb: 'CURRENT_TIMESTAMP(3)', - }; - - return map[dbType]; -} diff --git a/packages/core/src/Credentials.ts b/packages/core/src/Credentials.ts index 2ba8b10634..293a35a8c7 100644 --- a/packages/core/src/Credentials.ts +++ b/packages/core/src/Credentials.ts @@ -98,6 +98,7 @@ export class Credentials extends ICredentials { } return { + id: this.id, name: this.name, type: this.type, data: this.data, diff --git a/packages/core/src/NodeExecuteFunctions.ts b/packages/core/src/NodeExecuteFunctions.ts index 9e7f1bb747..54be53f1e7 100644 --- a/packages/core/src/NodeExecuteFunctions.ts +++ b/packages/core/src/NodeExecuteFunctions.ts @@ -738,16 +738,20 @@ export async function requestOAuth2( credentials.oauthTokenData = newToken.data; - // Find the name of the credentials + // Find the credentials if (!node.credentials || !node.credentials[credentialsType]) { throw new Error( `The node "${node.name}" does not have credentials of type "${credentialsType}"!`, ); } - const name = node.credentials[credentialsType]; + const nodeCredentials = node.credentials[credentialsType]; // Save the refreshed token - await additionalData.credentialsHelper.updateCredentials(name, credentialsType, credentials); + await additionalData.credentialsHelper.updateCredentials( + nodeCredentials, + credentialsType, + credentials, + ); Logger.debug( `OAuth2 token for "${credentialsType}" used by node "${node.name}" has been saved to database successfully.`, @@ -955,25 +959,26 @@ export async function getCredentials( } as ICredentialsExpressionResolveValues; } - let name = node.credentials[type]; + const nodeCredentials = node.credentials[type]; - if (name.charAt(0) === '=') { - // If the credential name is an expression resolve it - const additionalKeys = getAdditionalKeys(additionalData); - name = workflow.expression.getParameterValue( - name, - runExecutionData || null, - runIndex || 0, - itemIndex || 0, - node.name, - connectionInputData || [], - mode, - additionalKeys, - ) as string; - } + // TODO: solve using credentials via expression + // if (name.charAt(0) === '=') { + // // If the credential name is an expression resolve it + // const additionalKeys = getAdditionalKeys(additionalData); + // name = workflow.expression.getParameterValue( + // name, + // runExecutionData || null, + // runIndex || 0, + // itemIndex || 0, + // node.name, + // connectionInputData || [], + // mode, + // additionalKeys, + // ) as string; + // } const decryptedDataObject = await additionalData.credentialsHelper.getDecrypted( - name, + nodeCredentials, type, mode, false, diff --git a/packages/core/test/Credentials.test.ts b/packages/core/test/Credentials.test.ts index c5fe7f57c8..860c842341 100644 --- a/packages/core/test/Credentials.test.ts +++ b/packages/core/test/Credentials.test.ts @@ -3,7 +3,7 @@ import { Credentials } from '../src'; describe('Credentials', () => { describe('without nodeType set', () => { test('should be able to set and read key data without initial data set', () => { - const credentials = new Credentials('testName', 'testType', []); + const credentials = new Credentials({ id: null, name: 'testName' }, 'testType', []); const key = 'key1'; const password = 'password'; @@ -23,7 +23,12 @@ describe('Credentials', () => { const initialData = 4321; const initialDataEncoded = 'U2FsdGVkX1+0baznXt+Ag/ub8A2kHLyoLxn/rR9h4XQ='; - const credentials = new Credentials('testName', 'testType', [], initialDataEncoded); + const credentials = new Credentials( + { id: null, name: 'testName' }, + 'testType', + [], + initialDataEncoded, + ); const newData = 1234; @@ -46,7 +51,7 @@ describe('Credentials', () => { }, ]; - const credentials = new Credentials('testName', 'testType', nodeAccess); + const credentials = new Credentials({ id: null, name: 'testName' }, 'testType', nodeAccess); const key = 'key1'; const password = 'password'; diff --git a/packages/core/test/Helpers.ts b/packages/core/test/Helpers.ts index 74305631d1..387ac67a85 100644 --- a/packages/core/test/Helpers.ts +++ b/packages/core/test/Helpers.ts @@ -5,6 +5,7 @@ import { ICredentialsHelper, IDataObject, IExecuteWorkflowInfo, + INodeCredentialsDetails, INodeExecutionData, INodeParameters, INodeType, @@ -22,18 +23,21 @@ import { import { Credentials, IDeferredPromise, IExecuteFunctions } from '../src'; export class CredentialsHelper extends ICredentialsHelper { - getDecrypted(name: string, type: string): Promise { + getDecrypted( + nodeCredentials: INodeCredentialsDetails, + type: string, + ): Promise { return new Promise((res) => res({})); } - getCredentials(name: string, type: string): Promise { + getCredentials(nodeCredentials: INodeCredentialsDetails, type: string): Promise { return new Promise((res) => { - res(new Credentials('', '', [], '')); + res(new Credentials({ id: null, name: '' }, '', [], '')); }); } async updateCredentials( - name: string, + nodeCredentials: INodeCredentialsDetails, type: string, data: ICredentialDataDecryptedObject, ): Promise {} diff --git a/packages/editor-ui/src/Interface.ts b/packages/editor-ui/src/Interface.ts index 8181647da4..70ccf59ee6 100644 --- a/packages/editor-ui/src/Interface.ts +++ b/packages/editor-ui/src/Interface.ts @@ -249,7 +249,7 @@ export interface IActivationError { } export interface ICredentialsResponse extends ICredentialsEncrypted { - id?: string; + id: string; createdAt: number | string; updatedAt: number | string; } diff --git a/packages/editor-ui/src/components/CredentialEdit/CredentialEdit.vue b/packages/editor-ui/src/components/CredentialEdit/CredentialEdit.vue index 9532ad4073..105507699f 100644 --- a/packages/editor-ui/src/components/CredentialEdit/CredentialEdit.vue +++ b/packages/editor-ui/src/components/CredentialEdit/CredentialEdit.vue @@ -557,6 +557,7 @@ export default mixins(showMessage, nodeHelpers).extend({ ); const details: ICredentialsDecrypted = { + id: this.credentialId, name: this.credentialName, type: this.credentialTypeName!, data: data as unknown as ICredentialDataDecryptedObject, @@ -605,6 +606,7 @@ export default mixins(showMessage, nodeHelpers).extend({ ); const credentialDetails: ICredentialsDecrypted = { + id: this.credentialId, name: this.credentialName, type: this.credentialTypeName!, data: data as unknown as ICredentialDataDecryptedObject, diff --git a/packages/editor-ui/src/components/NodeCredentials.vue b/packages/editor-ui/src/components/NodeCredentials.vue index fb6c0b651e..ea81a4c252 100644 --- a/packages/editor-ui/src/components/NodeCredentials.vue +++ b/packages/editor-ui/src/components/NodeCredentials.vue @@ -9,15 +9,15 @@ {{credentialTypeNames[credentialTypeDescription.name]}}: - +
- + + :value="item.id">
- - - + + + + + + @@ -49,12 +52,14 @@