From 0a53c957c4d69270e10058cdd384e47c8e3c987e Mon Sep 17 00:00:00 2001 From: Michael Kret <88898367+michael-radency@users.noreply.github.com> Date: Wed, 12 Apr 2023 17:24:17 +0300 Subject: [PATCH] feat(MySQL Node): Overhaul --- .../credentials/MySql.credentials.ts | 115 ++++ packages/nodes-base/nodes/MySql/MySql.node.ts | 420 +------------- .../nodes/MySql/test/v2/operations.test.ts | 511 ++++++++++++++++++ .../nodes/MySql/test/v2/runQueries.test.ts | 178 ++++++ .../nodes/MySql/test/v2/utils.test.ts | 150 +++++ .../nodes/MySql/{ => v1}/GenericFunctions.ts | 0 .../nodes-base/nodes/MySql/v1/MySqlV1.node.ts | 424 +++++++++++++++ .../nodes-base/nodes/MySql/v2/MySqlV2.node.ts | 32 ++ .../MySql/v2/actions/common.descriptions.ts | 361 +++++++++++++ .../v2/actions/database/Database.resource.ts | 76 +++ .../actions/database/deleteTable.operation.ts | 137 +++++ .../database/executeQuery.operation.ts | 89 +++ .../v2/actions/database/insert.operation.ts | 227 ++++++++ .../v2/actions/database/select.operation.ts | 131 +++++ .../v2/actions/database/update.operation.ts | 195 +++++++ .../v2/actions/database/upsert.operation.ts | 199 +++++++ .../nodes/MySql/v2/actions/node.type.ts | 9 + .../nodes/MySql/v2/actions/router.ts | 66 +++ .../MySql/v2/actions/versionDescription.ts | 42 ++ .../nodes/MySql/v2/helpers/interfaces.ts | 28 + .../nodes/MySql/v2/helpers/utils.ts | 414 ++++++++++++++ .../nodes/MySql/v2/methods/credentialTest.ts | 44 ++ .../nodes/MySql/v2/methods/index.ts | 3 + .../nodes/MySql/v2/methods/listSearch.ts | 44 ++ .../nodes/MySql/v2/methods/loadOptions.ts | 64 +++ .../nodes/MySql/v2/transport/index.ts | 139 +++++ packages/nodes-base/test/nodes/Helpers.ts | 33 +- 27 files changed, 3729 insertions(+), 402 deletions(-) create mode 100644 packages/nodes-base/nodes/MySql/test/v2/operations.test.ts create mode 100644 packages/nodes-base/nodes/MySql/test/v2/runQueries.test.ts create mode 100644 packages/nodes-base/nodes/MySql/test/v2/utils.test.ts rename packages/nodes-base/nodes/MySql/{ => v1}/GenericFunctions.ts (100%) create mode 100644 packages/nodes-base/nodes/MySql/v1/MySqlV1.node.ts create mode 100644 packages/nodes-base/nodes/MySql/v2/MySqlV2.node.ts create mode 100644 packages/nodes-base/nodes/MySql/v2/actions/common.descriptions.ts create mode 100644 packages/nodes-base/nodes/MySql/v2/actions/database/Database.resource.ts create mode 100644 packages/nodes-base/nodes/MySql/v2/actions/database/deleteTable.operation.ts create mode 100644 packages/nodes-base/nodes/MySql/v2/actions/database/executeQuery.operation.ts create mode 100644 packages/nodes-base/nodes/MySql/v2/actions/database/insert.operation.ts create mode 100644 packages/nodes-base/nodes/MySql/v2/actions/database/select.operation.ts create mode 100644 packages/nodes-base/nodes/MySql/v2/actions/database/update.operation.ts create mode 100644 packages/nodes-base/nodes/MySql/v2/actions/database/upsert.operation.ts create mode 100644 packages/nodes-base/nodes/MySql/v2/actions/node.type.ts create mode 100644 packages/nodes-base/nodes/MySql/v2/actions/router.ts create mode 100644 packages/nodes-base/nodes/MySql/v2/actions/versionDescription.ts create mode 100644 packages/nodes-base/nodes/MySql/v2/helpers/interfaces.ts create mode 100644 packages/nodes-base/nodes/MySql/v2/helpers/utils.ts create mode 100644 packages/nodes-base/nodes/MySql/v2/methods/credentialTest.ts create mode 100644 packages/nodes-base/nodes/MySql/v2/methods/index.ts create mode 100644 packages/nodes-base/nodes/MySql/v2/methods/listSearch.ts create mode 100644 packages/nodes-base/nodes/MySql/v2/methods/loadOptions.ts create mode 100644 packages/nodes-base/nodes/MySql/v2/transport/index.ts diff --git a/packages/nodes-base/credentials/MySql.credentials.ts b/packages/nodes-base/credentials/MySql.credentials.ts index 93845b4c22..f7efaf06e0 100644 --- a/packages/nodes-base/credentials/MySql.credentials.ts +++ b/packages/nodes-base/credentials/MySql.credentials.ts @@ -97,5 +97,120 @@ export class MySql implements ICredentialType { type: 'string', default: '', }, + { + displayName: 'SSH Tunnel', + name: 'sshTunnel', + type: 'boolean', + default: false, + }, + { + displayName: 'SSH Authenticate with', + name: 'sshAuthenticateWith', + type: 'options', + default: 'password', + options: [ + { + name: 'Password', + value: 'password', + }, + { + name: 'Private Key', + value: 'privateKey', + }, + ], + displayOptions: { + show: { + sshTunnel: [true], + }, + }, + }, + { + displayName: 'SSH Host', + name: 'sshHost', + type: 'string', + default: 'localhost', + displayOptions: { + show: { + sshTunnel: [true], + }, + }, + }, + { + displayName: 'SSH Port', + name: 'sshPort', + type: 'number', + default: 22, + displayOptions: { + show: { + sshTunnel: [true], + }, + }, + }, + { + displayName: 'SSH MySQL Port', + name: 'sshMysqlPort', + type: 'number', + default: 3306, + displayOptions: { + show: { + sshTunnel: [true], + }, + }, + }, + { + displayName: 'SSH User', + name: 'sshUser', + type: 'string', + default: 'root', + displayOptions: { + show: { + sshTunnel: [true], + }, + }, + }, + { + displayName: 'SSH Password', + name: 'sshPassword', + type: 'string', + typeOptions: { + password: true, + }, + default: '', + displayOptions: { + show: { + sshTunnel: [true], + sshAuthenticateWith: ['password'], + }, + }, + }, + { + displayName: 'Private Key', + name: 'privateKey', + type: 'string', + typeOptions: { + rows: 4, + password: true, + }, + default: '', + displayOptions: { + show: { + sshTunnel: [true], + sshAuthenticateWith: ['privateKey'], + }, + }, + }, + { + displayName: 'Passphrase', + name: 'passphrase', + type: 'string', + default: '', + description: 'Passphase used to create the key, if no passphase was used leave empty', + displayOptions: { + show: { + sshTunnel: [true], + sshAuthenticateWith: ['privateKey'], + }, + }, + }, ]; } diff --git a/packages/nodes-base/nodes/MySql/MySql.node.ts b/packages/nodes-base/nodes/MySql/MySql.node.ts index 3c3e68def8..1d055b5ee5 100644 --- a/packages/nodes-base/nodes/MySql/MySql.node.ts +++ b/packages/nodes-base/nodes/MySql/MySql.node.ts @@ -1,407 +1,25 @@ -import type { - IExecuteFunctions, - ICredentialDataDecryptedObject, - ICredentialsDecrypted, - ICredentialTestFunctions, - IDataObject, - INodeCredentialTestResult, - INodeExecutionData, - INodeType, - INodeTypeDescription, -} from 'n8n-workflow'; -import { NodeOperationError } from 'n8n-workflow'; -// @ts-ignore -import type mysql2 from 'mysql2/promise'; +import type { INodeTypeBaseDescription, IVersionedNodeType } from 'n8n-workflow'; +import { VersionedNodeType } from 'n8n-workflow'; -import { copyInputItems, createConnection, searchTables } from './GenericFunctions'; +import { MySqlV1 } from './v1/MySqlV1.node'; +import { MySqlV2 } from './v2/MySqlV2.node'; -export class MySql implements INodeType { - description: INodeTypeDescription = { - displayName: 'MySQL', - name: 'mySql', - icon: 'file:mysql.svg', - group: ['input'], - version: 1, - description: 'Get, add and update data in MySQL', - defaults: { - name: 'MySQL', - }, - inputs: ['main'], - outputs: ['main'], - credentials: [ - { - name: 'mySql', - required: true, - testedBy: 'mysqlConnectionTest', - }, - ], - properties: [ - { - displayName: 'Operation', - name: 'operation', - type: 'options', - noDataExpression: true, - options: [ - { - name: 'Execute Query', - value: 'executeQuery', - description: 'Execute an SQL query', - action: 'Execute a SQL query', - }, - { - name: 'Insert', - value: 'insert', - description: 'Insert rows in database', - action: 'Insert rows in database', - }, - { - name: 'Update', - value: 'update', - description: 'Update rows in database', - action: 'Update rows in database', - }, - ], - default: 'insert', - }, +export class MySql extends VersionedNodeType { + constructor() { + const baseDescription: INodeTypeBaseDescription = { + displayName: 'MySQL', + name: 'mySql', + icon: 'file:mysql.svg', + group: ['input'], + defaultVersion: 2, + description: 'Get, add and update data in MySQL', + }; - // ---------------------------------- - // executeQuery - // ---------------------------------- - { - displayName: 'Query', - name: 'query', - type: 'string', - displayOptions: { - show: { - operation: ['executeQuery'], - }, - }, - default: '', - placeholder: 'SELECT id, name FROM product WHERE id < 40', - required: true, - description: 'The SQL query to execute', - }, + const nodeVersions: IVersionedNodeType['nodeVersions'] = { + 1: new MySqlV1(baseDescription), + 2: new MySqlV2(baseDescription), + }; - // ---------------------------------- - // insert - // ---------------------------------- - { - displayName: 'Table', - name: 'table', - type: 'resourceLocator', - default: { mode: 'list', value: '' }, - required: true, - modes: [ - { - displayName: 'From List', - name: 'list', - type: 'list', - placeholder: 'Select a Table...', - typeOptions: { - searchListMethod: 'searchTables', - searchFilterRequired: false, - searchable: true, - }, - }, - { - displayName: 'Name', - name: 'name', - type: 'string', - placeholder: 'table_name', - }, - ], - displayOptions: { - show: { - operation: ['insert'], - }, - }, - description: 'Name of the table in which to insert data to', - }, - { - displayName: 'Columns', - name: 'columns', - type: 'string', - displayOptions: { - show: { - operation: ['insert'], - }, - }, - requiresDataPath: 'multiple', - default: '', - placeholder: 'id,name,description', - description: - 'Comma-separated list of the properties which should used as columns for the new rows', - }, - { - displayName: 'Options', - name: 'options', - type: 'collection', - displayOptions: { - show: { - operation: ['insert'], - }, - }, - default: {}, - placeholder: 'Add modifiers', - description: 'Modifiers for INSERT statement', - options: [ - { - displayName: 'Ignore', - name: 'ignore', - type: 'boolean', - default: true, - description: - 'Whether to ignore any ignorable errors that occur while executing the INSERT statement', - }, - { - displayName: 'Priority', - name: 'priority', - type: 'options', - options: [ - { - name: 'Low Prioirity', - value: 'LOW_PRIORITY', - description: - 'Delays execution of the INSERT until no other clients are reading from the table', - }, - { - name: 'High Priority', - value: 'HIGH_PRIORITY', - description: - 'Overrides the effect of the --low-priority-updates option if the server was started with that option. It also causes concurrent inserts not to be used.', - }, - ], - default: 'LOW_PRIORITY', - description: - 'Ignore any ignorable errors that occur while executing the INSERT statement', - }, - ], - }, - - // ---------------------------------- - // update - // ---------------------------------- - { - displayName: 'Table', - name: 'table', - type: 'resourceLocator', - default: { mode: 'list', value: '' }, - required: true, - modes: [ - { - displayName: 'From List', - name: 'list', - type: 'list', - placeholder: 'Select a Table...', - typeOptions: { - searchListMethod: 'searchTables', - searchFilterRequired: false, - searchable: true, - }, - }, - { - displayName: 'Name', - name: 'name', - type: 'string', - placeholder: 'table_name', - }, - ], - displayOptions: { - show: { - operation: ['update'], - }, - }, - description: 'Name of the table in which to update data in', - }, - { - displayName: 'Update Key', - name: 'updateKey', - type: 'string', - displayOptions: { - show: { - operation: ['update'], - }, - }, - default: 'id', - required: true, - // eslint-disable-next-line n8n-nodes-base/node-param-description-miscased-id - description: - 'Name of the property which decides which rows in the database should be updated. Normally that would be "id".', - }, - { - displayName: 'Columns', - name: 'columns', - type: 'string', - requiresDataPath: 'multiple', - displayOptions: { - show: { - operation: ['update'], - }, - }, - default: '', - placeholder: 'name,description', - description: - 'Comma-separated list of the properties which should used as columns for rows to update', - }, - ], - }; - - methods = { - credentialTest: { - async mysqlConnectionTest( - this: ICredentialTestFunctions, - credential: ICredentialsDecrypted, - ): Promise { - const credentials = credential.data as ICredentialDataDecryptedObject; - try { - const connection = await createConnection(credentials); - await connection.end(); - } catch (error) { - return { - status: 'Error', - message: error.message, - }; - } - return { - status: 'OK', - message: 'Connection successful!', - }; - }, - }, - listSearch: { - searchTables, - }, - }; - - async execute(this: IExecuteFunctions): Promise { - const credentials = await this.getCredentials('mySql'); - const connection = await createConnection(credentials); - const items = this.getInputData(); - const operation = this.getNodeParameter('operation', 0); - let returnItems: INodeExecutionData[] = []; - - if (operation === 'executeQuery') { - // ---------------------------------- - // executeQuery - // ---------------------------------- - - try { - const queryQueue = items.map(async (item, index) => { - const rawQuery = this.getNodeParameter('query', index) as string; - - return connection.query(rawQuery); - }); - - returnItems = ((await Promise.all(queryQueue)) as mysql2.OkPacket[][]).reduce( - (collection, result, index) => { - const [rows] = result; - - const executionData = this.helpers.constructExecutionMetaData( - this.helpers.returnJsonArray(rows as unknown as IDataObject[]), - { itemData: { item: index } }, - ); - - collection.push(...executionData); - - return collection; - }, - [] as INodeExecutionData[], - ); - } catch (error) { - if (this.continueOnFail()) { - returnItems = this.helpers.returnJsonArray({ error: error.message }); - } else { - await connection.end(); - throw error; - } - } - } else if (operation === 'insert') { - // ---------------------------------- - // insert - // ---------------------------------- - - try { - const table = this.getNodeParameter('table', 0, '', { extractValue: true }) as string; - const columnString = this.getNodeParameter('columns', 0) as string; - const columns = columnString.split(',').map((column) => column.trim()); - const insertItems = copyInputItems(items, columns); - const insertPlaceholder = `(${columns.map((_column) => '?').join(',')})`; - const options = this.getNodeParameter('options', 0); - const insertIgnore = options.ignore as boolean; - const insertPriority = options.priority as string; - - const insertSQL = `INSERT ${insertPriority || ''} ${ - insertIgnore ? 'IGNORE' : '' - } INTO ${table}(${columnString}) VALUES ${items - .map((_item) => insertPlaceholder) - .join(',')};`; - const queryItems = insertItems.reduce( - (collection: IDataObject[], item) => - collection.concat(Object.values(item) as IDataObject[]), - [], - ); - - const queryResult = await connection.query(insertSQL, queryItems); - - returnItems = this.helpers.returnJsonArray(queryResult[0] as unknown as IDataObject); - } catch (error) { - if (this.continueOnFail()) { - returnItems = this.helpers.returnJsonArray({ error: error.message }); - } else { - await connection.end(); - throw error; - } - } - } else if (operation === 'update') { - // ---------------------------------- - // update - // ---------------------------------- - - try { - const table = this.getNodeParameter('table', 0, '', { extractValue: true }) as string; - const updateKey = this.getNodeParameter('updateKey', 0) as string; - const columnString = this.getNodeParameter('columns', 0) as string; - const columns = columnString.split(',').map((column) => column.trim()); - - if (!columns.includes(updateKey)) { - columns.unshift(updateKey); - } - - const updateItems = copyInputItems(items, columns); - const updateSQL = `UPDATE ${table} SET ${columns - .map((column) => `${column} = ?`) - .join(',')} WHERE ${updateKey} = ?;`; - const queryQueue = updateItems.map(async (item) => - connection.query(updateSQL, Object.values(item).concat(item[updateKey])), - ); - const queryResult = await Promise.all(queryQueue); - returnItems = this.helpers.returnJsonArray( - queryResult.map((result) => result[0]) as unknown as IDataObject[], - ); - } catch (error) { - if (this.continueOnFail()) { - returnItems = this.helpers.returnJsonArray({ error: error.message }); - } else { - await connection.end(); - throw error; - } - } - } else { - if (this.continueOnFail()) { - returnItems = this.helpers.returnJsonArray({ - error: `The operation "${operation}" is not supported!`, - }); - } else { - await connection.end(); - throw new NodeOperationError( - this.getNode(), - `The operation "${operation}" is not supported!`, - ); - } - } - - await connection.end(); - - return this.prepareOutputData(returnItems); + super(nodeVersions, baseDescription); } } diff --git a/packages/nodes-base/nodes/MySql/test/v2/operations.test.ts b/packages/nodes-base/nodes/MySql/test/v2/operations.test.ts new file mode 100644 index 0000000000..8f24baf498 --- /dev/null +++ b/packages/nodes-base/nodes/MySql/test/v2/operations.test.ts @@ -0,0 +1,511 @@ +import type { IDataObject, INode } from 'n8n-workflow'; + +import { createMockExecuteFunction } from '../../../../test/nodes/Helpers'; + +import * as deleteTable from '../../v2/actions/database/deleteTable.operation'; +import * as executeQuery from '../../v2/actions/database/executeQuery.operation'; +import * as insert from '../../v2/actions/database/insert.operation'; +import * as select from '../../v2/actions/database/select.operation'; +import * as update from '../../v2/actions/database/update.operation'; +import * as upsert from '../../v2/actions/database/upsert.operation'; + +import type { Mysql2Pool, QueryRunner } from '../../v2/helpers/interfaces'; +import { configureQueryRunner } from '../../v2/helpers/utils'; + +import mysql2 from 'mysql2/promise'; + +const mySqlMockNode: INode = { + id: '1', + name: 'MySQL node', + typeVersion: 2, + type: 'n8n-nodes-base.mySql', + position: [60, 760], + parameters: { + operation: 'select', + }, +}; + +const fakeConnection = { + format(query: string, values: any[]) { + return mysql2.format(query, values); + }, + query: jest.fn(async (_query = ''): Promise => Promise.resolve([{}])), + release: jest.fn(), + beginTransaction: jest.fn(), + commit: jest.fn(), + rollback: jest.fn(), +}; + +const createFakePool = (connection: IDataObject) => { + return { + getConnection() { + return connection; + }, + query: jest.fn(async () => Promise.resolve([{}])), + } as unknown as Mysql2Pool; +}; + +const emptyInputItems = [{ json: {}, pairedItem: { item: 0, input: undefined } }]; + +describe('Test MySql V2, operations', () => { + afterEach(() => { + jest.clearAllMocks(); + }); + + it('should have all operations', () => { + expect(deleteTable.execute).toBeDefined(); + expect(deleteTable.description).toBeDefined(); + expect(executeQuery.execute).toBeDefined(); + expect(executeQuery.description).toBeDefined(); + expect(insert.execute).toBeDefined(); + expect(insert.description).toBeDefined(); + expect(select.execute).toBeDefined(); + expect(select.description).toBeDefined(); + expect(update.execute).toBeDefined(); + expect(update.description).toBeDefined(); + expect(upsert.execute).toBeDefined(); + expect(upsert.description).toBeDefined(); + }); + + it('deleteTable: drop, should call runQueries with', async () => { + const nodeParameters: IDataObject = { + operation: 'deleteTable', + table: { + __rl: true, + value: 'test_table', + mode: 'list', + cachedResultName: 'test_table', + }, + deleteCommand: 'drop', + options: {}, + }; + + const nodeOptions = nodeParameters.options as IDataObject; + + const pool = createFakePool(fakeConnection); + + const poolQuerySpy = jest.spyOn(pool, 'query'); + + const fakeExecuteFunction = createMockExecuteFunction(nodeParameters, mySqlMockNode); + + const runQueries: QueryRunner = configureQueryRunner.call( + fakeExecuteFunction, + nodeOptions, + pool, + ); + + const result = await deleteTable.execute.call(fakeExecuteFunction, emptyInputItems, runQueries); + + expect(result).toBeDefined(); + expect(result).toEqual([{ json: { success: true } }]); + + expect(poolQuerySpy).toBeCalledTimes(1); + expect(poolQuerySpy).toBeCalledWith('DROP TABLE IF EXISTS `test_table`'); + }); + + it('deleteTable: truncate, should call runQueries with', async () => { + const nodeParameters: IDataObject = { + operation: 'deleteTable', + table: { + __rl: true, + value: 'test_table', + mode: 'list', + cachedResultName: 'test_table', + }, + deleteCommand: 'truncate', + options: {}, + }; + + const nodeOptions = nodeParameters.options as IDataObject; + + const pool = createFakePool(fakeConnection); + + const poolQuerySpy = jest.spyOn(pool, 'query'); + + const fakeExecuteFunction = createMockExecuteFunction(nodeParameters, mySqlMockNode); + + const runQueries: QueryRunner = configureQueryRunner.call( + fakeExecuteFunction, + nodeOptions, + pool, + ); + + const result = await deleteTable.execute.call(fakeExecuteFunction, emptyInputItems, runQueries); + + expect(result).toBeDefined(); + expect(result).toEqual([{ json: { success: true } }]); + + expect(poolQuerySpy).toBeCalledTimes(1); + expect(poolQuerySpy).toBeCalledWith('TRUNCATE TABLE `test_table`'); + }); + + it('deleteTable: delete, should call runQueries with', async () => { + const nodeParameters: IDataObject = { + operation: 'deleteTable', + table: { + __rl: true, + value: 'test_table', + mode: 'list', + cachedResultName: 'test_table', + }, + deleteCommand: 'delete', + where: { + values: [ + { + column: 'id', + condition: 'equal', + value: '1', + }, + { + column: 'name', + condition: 'LIKE', + value: 'some%', + }, + ], + }, + options: {}, + }; + + const nodeOptions = nodeParameters.options as IDataObject; + + const pool = createFakePool(fakeConnection); + + const poolQuerySpy = jest.spyOn(pool, 'query'); + + const fakeExecuteFunction = createMockExecuteFunction(nodeParameters, mySqlMockNode); + + const runQueries: QueryRunner = configureQueryRunner.call( + fakeExecuteFunction, + nodeOptions, + pool, + ); + + const result = await deleteTable.execute.call(fakeExecuteFunction, emptyInputItems, runQueries); + + expect(result).toBeDefined(); + expect(result).toEqual([{ json: { success: true } }]); + + expect(poolQuerySpy).toBeCalledTimes(1); + expect(poolQuerySpy).toBeCalledWith( + "DELETE FROM `test_table` WHERE `id` = '1' AND `name` LIKE 'some%'", + ); + }); + + it('executeQuery, should call runQueries with', async () => { + const nodeParameters: IDataObject = { + operation: 'executeQuery', + query: + "DROP TABLE IF EXISTS $1:name;\ncreate table $1:name (id INT, name TEXT);\ninsert into $1:name (id, name) values (1, 'test 1');\nselect * from $1:name;\n", + options: { + queryBatching: 'independently', + queryReplacement: 'test_table', + }, + }; + + const nodeOptions = nodeParameters.options as IDataObject; + + const fakeConnectionCopy = { ...fakeConnection }; + + fakeConnectionCopy.query = jest.fn(async (query?: string) => { + const result = []; + console.log(query); + if (query?.toLowerCase().includes('select')) { + result.push([{ id: 1, name: 'test 1' }]); + } else { + result.push({}); + } + return Promise.resolve(result); + }); + const pool = createFakePool(fakeConnectionCopy); + + const connectionQuerySpy = jest.spyOn(fakeConnectionCopy, 'query'); + + const fakeExecuteFunction = createMockExecuteFunction(nodeParameters, mySqlMockNode); + + const runQueries: QueryRunner = configureQueryRunner.call( + fakeExecuteFunction, + nodeOptions, + pool, + ); + + const result = await executeQuery.execute.call( + fakeExecuteFunction, + emptyInputItems, + runQueries, + nodeOptions, + ); + + expect(result).toBeDefined(); + expect(result).toEqual([ + { + json: { + id: 1, + name: 'test 1', + }, + pairedItem: { + item: 0, + }, + }, + ]); + + expect(connectionQuerySpy).toBeCalledTimes(4); + expect(connectionQuerySpy).toBeCalledWith('DROP TABLE IF EXISTS `test_table`'); + expect(connectionQuerySpy).toBeCalledWith('create table `test_table` (id INT, name TEXT)'); + expect(connectionQuerySpy).toBeCalledWith( + "insert into `test_table` (id, name) values (1, 'test 1')", + ); + expect(connectionQuerySpy).toBeCalledWith('select * from `test_table`'); + }); + + it('select, should call runQueries with', async () => { + const nodeParameters: IDataObject = { + operation: 'select', + table: { + __rl: true, + value: 'test_table', + mode: 'list', + cachedResultName: 'test_table', + }, + limit: 2, + where: { + values: [ + { + column: 'id', + condition: '>', + value: '1', + }, + { + column: 'name', + value: 'test', + }, + ], + }, + combineConditions: 'OR', + sort: { + values: [ + { + column: 'id', + direction: 'DESC', + }, + ], + }, + options: { + queryBatching: 'transaction', + detailedOutput: false, + }, + }; + + const nodeOptions = nodeParameters.options as IDataObject; + + const pool = createFakePool(fakeConnection); + + const connectionQuerySpy = jest.spyOn(fakeConnection, 'query'); + + const fakeExecuteFunction = createMockExecuteFunction(nodeParameters, mySqlMockNode); + + const runQueries: QueryRunner = configureQueryRunner.call( + fakeExecuteFunction, + nodeOptions, + pool, + ); + + const result = await select.execute.call(fakeExecuteFunction, emptyInputItems, runQueries); + + expect(result).toBeDefined(); + expect(result).toEqual([{ json: { success: true } }]); + + const connectionBeginTransactionSpy = jest.spyOn(fakeConnection, 'beginTransaction'); + const connectionCommitSpy = jest.spyOn(fakeConnection, 'commit'); + + expect(connectionBeginTransactionSpy).toBeCalledTimes(1); + + expect(connectionQuerySpy).toBeCalledTimes(1); + expect(connectionQuerySpy).toBeCalledWith( + "SELECT * FROM `test_table` WHERE `id` > 1 OR `name` undefined 'test' ORDER BY `id` DESC LIMIT 2", + ); + + expect(connectionCommitSpy).toBeCalledTimes(1); + }); + + it('insert, should call runQueries with', async () => { + const nodeParameters: IDataObject = { + table: { + __rl: true, + value: 'test_table', + mode: 'list', + cachedResultName: 'test_table', + }, + dataMode: 'defineBelow', + valuesToSend: { + values: [ + { + column: 'id', + value: '2', + }, + { + column: 'name', + value: 'name 2', + }, + ], + }, + options: { + queryBatching: 'independently', + priority: 'HIGH_PRIORITY', + detailedOutput: false, + skipOnConflict: true, + }, + }; + + const nodeOptions = nodeParameters.options as IDataObject; + + const pool = createFakePool(fakeConnection); + + const connectionQuerySpy = jest.spyOn(fakeConnection, 'query'); + + const fakeExecuteFunction = createMockExecuteFunction(nodeParameters, mySqlMockNode); + + const runQueries: QueryRunner = configureQueryRunner.call( + fakeExecuteFunction, + nodeOptions, + pool, + ); + + const result = await insert.execute.call( + fakeExecuteFunction, + emptyInputItems, + runQueries, + nodeOptions, + ); + + expect(result).toBeDefined(); + expect(result).toEqual([{ json: { success: true } }]); + + expect(connectionQuerySpy).toBeCalledTimes(1); + expect(connectionQuerySpy).toBeCalledWith( + "INSERT HIGH_PRIORITY IGNORE INTO `test_table` (`id`, `name`) VALUES ('2','name 2')", + ); + }); + + it('update, should call runQueries with', async () => { + const nodeParameters: IDataObject = { + operation: 'update', + table: { + __rl: true, + value: 'test_table', + mode: 'list', + cachedResultName: 'test_table', + }, + dataMode: 'autoMapInputData', + columnToMatchOn: 'id', + options: { + queryBatching: 'independently', + }, + }; + + const nodeOptions = nodeParameters.options as IDataObject; + + const pool = createFakePool(fakeConnection); + + const connectionQuerySpy = jest.spyOn(fakeConnection, 'query'); + + const fakeExecuteFunction = createMockExecuteFunction(nodeParameters, mySqlMockNode); + + const runQueries: QueryRunner = configureQueryRunner.call( + fakeExecuteFunction, + nodeOptions, + pool, + ); + + const inputItems = [ + { + json: { + id: 42, + name: 'test 4', + }, + }, + { + json: { + id: 88, + name: 'test 88', + }, + }, + ]; + + const result = await update.execute.call( + fakeExecuteFunction, + inputItems, + runQueries, + nodeOptions, + ); + + expect(result).toBeDefined(); + expect(result).toEqual([{ json: { success: true } }, { json: { success: true } }]); + + expect(connectionQuerySpy).toBeCalledTimes(2); + expect(connectionQuerySpy).toBeCalledWith( + "UPDATE `test_table` SET `name` = 'test 4' WHERE `id` = 42", + ); + expect(connectionQuerySpy).toBeCalledWith( + "UPDATE `test_table` SET `name` = 'test 88' WHERE `id` = 88", + ); + }); + + it('upsert, should call runQueries with', async () => { + const nodeParameters: IDataObject = { + operation: 'upsert', + table: { + __rl: true, + value: 'test_table', + mode: 'list', + cachedResultName: 'test_table', + }, + columnToMatchOn: 'id', + dataMode: 'autoMapInputData', + options: {}, + }; + + const nodeOptions = nodeParameters.options as IDataObject; + + const pool = createFakePool(fakeConnection); + + const poolQuerySpy = jest.spyOn(pool, 'query'); + + const fakeExecuteFunction = createMockExecuteFunction(nodeParameters, mySqlMockNode); + + const runQueries: QueryRunner = configureQueryRunner.call( + fakeExecuteFunction, + nodeOptions, + pool, + ); + + const inputItems = [ + { + json: { + id: 42, + name: 'test 4', + }, + }, + { + json: { + id: 88, + name: 'test 88', + }, + }, + ]; + + const result = await upsert.execute.call( + fakeExecuteFunction, + inputItems, + runQueries, + nodeOptions, + ); + + expect(result).toBeDefined(); + expect(result).toEqual([{ json: { success: true } }]); + + expect(poolQuerySpy).toBeCalledTimes(1); + expect(poolQuerySpy).toBeCalledWith( + "INSERT INTO `test_table`(`id`, `name`) VALUES(42,'test 4') ON DUPLICATE KEY UPDATE `name` = 'test 4';INSERT INTO `test_table`(`id`, `name`) VALUES(88,'test 88') ON DUPLICATE KEY UPDATE `name` = 'test 88'", + ); + }); +}); diff --git a/packages/nodes-base/nodes/MySql/test/v2/runQueries.test.ts b/packages/nodes-base/nodes/MySql/test/v2/runQueries.test.ts new file mode 100644 index 0000000000..73a0eb3418 --- /dev/null +++ b/packages/nodes-base/nodes/MySql/test/v2/runQueries.test.ts @@ -0,0 +1,178 @@ +import { createMockExecuteFunction } from '../../../../test/nodes/Helpers'; + +import { configureQueryRunner } from '../../v2/helpers/utils'; +import type { Mysql2Pool, QueryRunner } from '../../v2/helpers/interfaces'; +import { BATCH_MODE } from '../../v2/helpers/interfaces'; + +import type { IDataObject, INode } from 'n8n-workflow'; + +import mysql2 from 'mysql2/promise'; + +const mySqlMockNode: INode = { + id: '1', + name: 'MySQL node', + typeVersion: 2, + type: 'n8n-nodes-base.mySql', + position: [60, 760], + parameters: { + operation: 'select', + }, +}; + +const fakeConnection = { + format(query: string, values: any[]) { + return mysql2.format(query, values); + }, + query: jest.fn(async () => Promise.resolve([{}])), + release: jest.fn(), + beginTransaction: jest.fn(), + commit: jest.fn(), + rollback: jest.fn(), +}; + +const createFakePool = (connection: IDataObject) => { + return { + getConnection() { + return connection; + }, + query: jest.fn(async () => Promise.resolve([{}])), + } as unknown as Mysql2Pool; +}; + +describe('Test MySql V2, runQueries', () => { + afterEach(() => { + jest.clearAllMocks(); + }); + + it('should execute in "Single" mode, should return success true', async () => { + const nodeOptions: IDataObject = { queryBatching: BATCH_MODE.SINGLE }; + + const pool = createFakePool(fakeConnection); + const fakeExecuteFunction = createMockExecuteFunction({}, mySqlMockNode); + + const runQueries: QueryRunner = configureQueryRunner.call( + fakeExecuteFunction, + nodeOptions, + pool, + ); + + const poolGetConnectionSpy = jest.spyOn(pool, 'getConnection'); + const poolQuerySpy = jest.spyOn(pool, 'query'); + const connectionReleaseSpy = jest.spyOn(fakeConnection, 'release'); + const connectionFormatSpy = jest.spyOn(fakeConnection, 'format'); + + const result = await runQueries([ + { query: 'SELECT * FROM my_table WHERE id = ?', values: [55] }, + ]); + + expect(result).toBeDefined(); + expect(result).toHaveLength(1); + expect(result).toEqual([{ json: { success: true } }]); + + expect(poolGetConnectionSpy).toBeCalledTimes(1); + + expect(connectionReleaseSpy).toBeCalledTimes(1); + + expect(poolQuerySpy).toBeCalledTimes(1); + expect(poolQuerySpy).toBeCalledWith('SELECT * FROM my_table WHERE id = 55'); + + expect(connectionFormatSpy).toBeCalledTimes(1); + expect(connectionFormatSpy).toBeCalledWith('SELECT * FROM my_table WHERE id = ?', [55]); + }); + + it('should execute in "independently" mode, should return success true', async () => { + const nodeOptions: IDataObject = { queryBatching: BATCH_MODE.INDEPENDENTLY }; + + const pool = createFakePool(fakeConnection); + + const fakeExecuteFunction = createMockExecuteFunction({}, mySqlMockNode); + + const runQueries: QueryRunner = configureQueryRunner.call( + fakeExecuteFunction, + nodeOptions, + pool, + ); + + const poolGetConnectionSpy = jest.spyOn(pool, 'getConnection'); + + const connectionReleaseSpy = jest.spyOn(fakeConnection, 'release'); + const connectionFormatSpy = jest.spyOn(fakeConnection, 'format'); + const connectionQuerySpy = jest.spyOn(fakeConnection, 'query'); + + const result = await runQueries([ + { + query: 'SELECT * FROM my_table WHERE id = ?; SELECT * FROM my_table WHERE id = ?', + values: [55, 42], + }, + ]); + + expect(result).toBeDefined(); + expect(result).toHaveLength(1); + expect(result).toEqual([{ json: { success: true } }]); + + expect(poolGetConnectionSpy).toBeCalledTimes(1); + + expect(connectionQuerySpy).toBeCalledTimes(2); + expect(connectionQuerySpy).toBeCalledWith('SELECT * FROM my_table WHERE id = 55'); + expect(connectionQuerySpy).toBeCalledWith('SELECT * FROM my_table WHERE id = 42'); + + expect(connectionFormatSpy).toBeCalledTimes(1); + expect(connectionFormatSpy).toBeCalledWith( + 'SELECT * FROM my_table WHERE id = ?; SELECT * FROM my_table WHERE id = ?', + [55, 42], + ); + + expect(connectionReleaseSpy).toBeCalledTimes(1); + }); + + it('should execute in "transaction" mode, should return success true', async () => { + const nodeOptions: IDataObject = { queryBatching: BATCH_MODE.TRANSACTION }; + + const pool = createFakePool(fakeConnection); + + const fakeExecuteFunction = createMockExecuteFunction({}, mySqlMockNode); + + const runQueries: QueryRunner = configureQueryRunner.call( + fakeExecuteFunction, + nodeOptions, + pool, + ); + + const poolGetConnectionSpy = jest.spyOn(pool, 'getConnection'); + + const connectionReleaseSpy = jest.spyOn(fakeConnection, 'release'); + const connectionFormatSpy = jest.spyOn(fakeConnection, 'format'); + const connectionQuerySpy = jest.spyOn(fakeConnection, 'query'); + const connectionBeginTransactionSpy = jest.spyOn(fakeConnection, 'beginTransaction'); + const connectionCommitSpy = jest.spyOn(fakeConnection, 'commit'); + + const result = await runQueries([ + { + query: 'SELECT * FROM my_table WHERE id = ?; SELECT * FROM my_table WHERE id = ?', + values: [55, 42], + }, + ]); + + expect(result).toBeDefined(); + expect(result).toHaveLength(1); + expect(result).toEqual([{ json: { success: true } }]); + + expect(poolGetConnectionSpy).toBeCalledTimes(1); + + expect(connectionBeginTransactionSpy).toBeCalledTimes(1); + + expect(connectionQuerySpy).toBeCalledTimes(2); + expect(connectionQuerySpy).toBeCalledWith('SELECT * FROM my_table WHERE id = 55'); + expect(connectionQuerySpy).toBeCalledWith('SELECT * FROM my_table WHERE id = 42'); + + expect(connectionFormatSpy).toBeCalledTimes(1); + expect(connectionFormatSpy).toBeCalledWith( + 'SELECT * FROM my_table WHERE id = ?; SELECT * FROM my_table WHERE id = ?', + [55, 42], + ); + + expect(connectionCommitSpy).toBeCalledTimes(1); + + expect(connectionReleaseSpy).toBeCalledTimes(1); + }); +}); diff --git a/packages/nodes-base/nodes/MySql/test/v2/utils.test.ts b/packages/nodes-base/nodes/MySql/test/v2/utils.test.ts new file mode 100644 index 0000000000..15c7bd2847 --- /dev/null +++ b/packages/nodes-base/nodes/MySql/test/v2/utils.test.ts @@ -0,0 +1,150 @@ +import type { INode } from 'n8n-workflow'; +import type { SortRule, WhereClause } from '../../v2/helpers/interfaces'; + +import { + prepareQueryAndReplacements, + wrapData, + addWhereClauses, + addSortRules, + replaceEmptyStringsByNulls, +} from '../../v2/helpers/utils'; + +const mySqlMockNode: INode = { + id: '1', + name: 'MySQL node', + typeVersion: 2, + type: 'n8n-nodes-base.mySql', + position: [60, 760], + parameters: { + operation: 'select', + }, +}; + +describe('Test MySql V2, prepareQueryAndReplacements', () => { + it('should transform query and values', () => { + const preparedQuery = prepareQueryAndReplacements( + 'SELECT * FROM $1:name WHERE id = $2 AND name = $4 AND $3:name = 28', + ['table', 15, 'age', 'Name'], + ); + expect(preparedQuery).toBeDefined(); + expect(preparedQuery.query).toEqual( + 'SELECT * FROM `table` WHERE id = ? AND name = ? AND `age` = 28', + ); + expect(preparedQuery.values.length).toEqual(2); + expect(preparedQuery.values[0]).toEqual(15); + expect(preparedQuery.values[1]).toEqual('Name'); + }); +}); + +describe('Test MySql V2, wrapData', () => { + it('should wrap object in json', () => { + const data = { + id: 1, + name: 'Name', + }; + const wrappedData = wrapData(data); + expect(wrappedData).toBeDefined(); + expect(wrappedData).toEqual([{ json: data }]); + }); + it('should wrap each object in array in json', () => { + const data = [ + { + id: 1, + name: 'Name', + }, + { + id: 2, + name: 'Name 2', + }, + ]; + const wrappedData = wrapData(data); + expect(wrappedData).toBeDefined(); + expect(wrappedData).toEqual([{ json: data[0] }, { json: data[1] }]); + }); + it('json key from source should be inside json', () => { + const data = { + json: { + id: 1, + name: 'Name', + }, + }; + const wrappedData = wrapData(data); + expect(wrappedData).toBeDefined(); + expect(wrappedData).toEqual([{ json: data }]); + expect(Object.keys(wrappedData[0].json)).toContain('json'); + }); +}); + +describe('Test MySql V2, addWhereClauses', () => { + it('add where clauses to query', () => { + const whereClauses: WhereClause[] = [ + { column: 'species', condition: 'equal', value: 'dog' }, + { column: 'name', condition: 'equal', value: 'Hunter' }, + ]; + const [query, values] = addWhereClauses( + mySqlMockNode, + 0, + 'SELECT * FROM `pet`', + whereClauses, + [], + ); + expect(query).toEqual('SELECT * FROM `pet` WHERE `species` = ? AND `name` = ?'); + expect(values.length).toEqual(2); + expect(values[0]).toEqual('dog'); + expect(values[1]).toEqual('Hunter'); + }); + it('add where clauses to query combined by OR', () => { + const whereClauses: WhereClause[] = [ + { column: 'species', condition: 'equal', value: 'dog' }, + { column: 'name', condition: 'equal', value: 'Hunter' }, + ]; + const [query, values] = addWhereClauses( + mySqlMockNode, + 0, + 'SELECT * FROM `pet`', + whereClauses, + [], + 'OR', + ); + expect(query).toEqual('SELECT * FROM `pet` WHERE `species` = ? OR `name` = ?'); + expect(values.length).toEqual(2); + expect(values[0]).toEqual('dog'); + expect(values[1]).toEqual('Hunter'); + }); +}); + +describe('Test MySql V2, addSortRules', () => { + it('should add ORDER by', () => { + const sortRules: SortRule[] = [ + { column: 'name', direction: 'ASC' }, + { column: 'age', direction: 'DESC' }, + ]; + const [query, values] = addSortRules('SELECT * FROM `pet`', sortRules, []); + + expect(query).toEqual('SELECT * FROM `pet` ORDER BY `name` ASC, `age` DESC'); + expect(values.length).toEqual(0); + }); +}); + +describe('Test MySql V2, replaceEmptyStringsByNulls', () => { + it('should replace empty strings', () => { + const data = [ + { json: { id: 1, name: '' } }, + { json: { id: '', name: '' } }, + { json: { id: null, data: '' } }, + ]; + const replacedData = replaceEmptyStringsByNulls(data, true); + expect(replacedData).toBeDefined(); + expect(replacedData).toEqual([ + { json: { id: 1, name: null } }, + { json: { id: null, name: null } }, + { json: { id: null, data: null } }, + ]); + }); + it('should not replace empty strings', () => { + const data = [{ json: { id: 1, name: '' } }]; + const replacedData = replaceEmptyStringsByNulls(data); + expect(replacedData).toBeDefined(); + expect(replacedData).toEqual([{ json: { id: 1, name: '' } }]); + }); +}); diff --git a/packages/nodes-base/nodes/MySql/GenericFunctions.ts b/packages/nodes-base/nodes/MySql/v1/GenericFunctions.ts similarity index 100% rename from packages/nodes-base/nodes/MySql/GenericFunctions.ts rename to packages/nodes-base/nodes/MySql/v1/GenericFunctions.ts diff --git a/packages/nodes-base/nodes/MySql/v1/MySqlV1.node.ts b/packages/nodes-base/nodes/MySql/v1/MySqlV1.node.ts new file mode 100644 index 0000000000..31abec978e --- /dev/null +++ b/packages/nodes-base/nodes/MySql/v1/MySqlV1.node.ts @@ -0,0 +1,424 @@ +/* eslint-disable n8n-nodes-base/node-filename-against-convention */ +import type { + ICredentialDataDecryptedObject, + ICredentialsDecrypted, + ICredentialTestFunctions, + IDataObject, + INodeCredentialTestResult, + INodeExecutionData, + INodeType, + INodeTypeBaseDescription, + INodeTypeDescription, +} from 'n8n-workflow'; +import { NodeOperationError } from 'n8n-workflow'; + +import type mysql2 from 'mysql2/promise'; + +import { copyInputItems, createConnection, searchTables } from './GenericFunctions'; +import type { IExecuteFunctions } from 'n8n-core'; + +const versionDescription: INodeTypeDescription = { + displayName: 'MySQL', + name: 'mySql', + icon: 'file:mysql.svg', + group: ['input'], + version: 1, + description: 'Get, add and update data in MySQL', + defaults: { + name: 'MySQL', + }, + inputs: ['main'], + outputs: ['main'], + credentials: [ + { + name: 'mySql', + required: true, + testedBy: 'mysqlConnectionTest', + }, + ], + properties: [ + { + displayName: 'Version 1', + name: 'versionNotice', + type: 'notice', + default: '', + }, + { + displayName: 'Operation', + name: 'operation', + type: 'options', + noDataExpression: true, + options: [ + { + name: 'Execute Query', + value: 'executeQuery', + description: 'Execute an SQL query', + action: 'Execute a SQL query', + }, + { + name: 'Insert', + value: 'insert', + description: 'Insert rows in database', + action: 'Insert rows in database', + }, + { + name: 'Update', + value: 'update', + description: 'Update rows in database', + action: 'Update rows in database', + }, + ], + default: 'insert', + }, + + // ---------------------------------- + // executeQuery + // ---------------------------------- + { + displayName: 'Query', + name: 'query', + type: 'string', + displayOptions: { + show: { + operation: ['executeQuery'], + }, + }, + default: '', + placeholder: 'SELECT id, name FROM product WHERE id < 40', + required: true, + description: 'The SQL query to execute', + }, + + // ---------------------------------- + // insert + // ---------------------------------- + { + displayName: 'Table', + name: 'table', + type: 'resourceLocator', + default: { mode: 'list', value: '' }, + required: true, + modes: [ + { + displayName: 'From List', + name: 'list', + type: 'list', + placeholder: 'Select a Table...', + typeOptions: { + searchListMethod: 'searchTables', + searchFilterRequired: false, + searchable: true, + }, + }, + { + displayName: 'Name', + name: 'name', + type: 'string', + placeholder: 'table_name', + }, + ], + displayOptions: { + show: { + operation: ['insert'], + }, + }, + description: 'Name of the table in which to insert data to', + }, + { + displayName: 'Columns', + name: 'columns', + type: 'string', + displayOptions: { + show: { + operation: ['insert'], + }, + }, + requiresDataPath: 'multiple', + default: '', + placeholder: 'id,name,description', + description: + 'Comma-separated list of the properties which should used as columns for the new rows', + }, + { + displayName: 'Options', + name: 'options', + type: 'collection', + displayOptions: { + show: { + operation: ['insert'], + }, + }, + default: {}, + placeholder: 'Add modifiers', + description: 'Modifiers for INSERT statement', + options: [ + { + displayName: 'Ignore', + name: 'ignore', + type: 'boolean', + default: true, + description: + 'Whether to ignore any ignorable errors that occur while executing the INSERT statement', + }, + { + displayName: 'Priority', + name: 'priority', + type: 'options', + options: [ + { + name: 'Low Prioirity', + value: 'LOW_PRIORITY', + description: + 'Delays execution of the INSERT until no other clients are reading from the table', + }, + { + name: 'High Priority', + value: 'HIGH_PRIORITY', + description: + 'Overrides the effect of the --low-priority-updates option if the server was started with that option. It also causes concurrent inserts not to be used.', + }, + ], + default: 'LOW_PRIORITY', + description: + 'Ignore any ignorable errors that occur while executing the INSERT statement', + }, + ], + }, + + // ---------------------------------- + // update + // ---------------------------------- + { + displayName: 'Table', + name: 'table', + type: 'resourceLocator', + default: { mode: 'list', value: '' }, + required: true, + modes: [ + { + displayName: 'From List', + name: 'list', + type: 'list', + placeholder: 'Select a Table...', + typeOptions: { + searchListMethod: 'searchTables', + searchFilterRequired: false, + searchable: true, + }, + }, + { + displayName: 'Name', + name: 'name', + type: 'string', + placeholder: 'table_name', + }, + ], + displayOptions: { + show: { + operation: ['update'], + }, + }, + description: 'Name of the table in which to update data in', + }, + { + displayName: 'Update Key', + name: 'updateKey', + type: 'string', + displayOptions: { + show: { + operation: ['update'], + }, + }, + default: 'id', + required: true, + // eslint-disable-next-line n8n-nodes-base/node-param-description-miscased-id + description: + 'Name of the property which decides which rows in the database should be updated. Normally that would be "id".', + }, + { + displayName: 'Columns', + name: 'columns', + type: 'string', + requiresDataPath: 'multiple', + displayOptions: { + show: { + operation: ['update'], + }, + }, + default: '', + placeholder: 'name,description', + description: + 'Comma-separated list of the properties which should used as columns for rows to update', + }, + ], +}; + +export class MySqlV1 implements INodeType { + description: INodeTypeDescription; + + constructor(baseDescription: INodeTypeBaseDescription) { + this.description = { + ...baseDescription, + ...versionDescription, + }; + } + + methods = { + credentialTest: { + async mysqlConnectionTest( + this: ICredentialTestFunctions, + credential: ICredentialsDecrypted, + ): Promise { + const credentials = credential.data as ICredentialDataDecryptedObject; + try { + const connection = await createConnection(credentials); + await connection.end(); + } catch (error) { + return { + status: 'Error', + message: error.message, + }; + } + return { + status: 'OK', + message: 'Connection successful!', + }; + }, + }, + listSearch: { + searchTables, + }, + }; + + async execute(this: IExecuteFunctions): Promise { + const credentials = await this.getCredentials('mySql'); + const connection = await createConnection(credentials); + const items = this.getInputData(); + const operation = this.getNodeParameter('operation', 0); + let returnItems: INodeExecutionData[] = []; + + if (operation === 'executeQuery') { + // ---------------------------------- + // executeQuery + // ---------------------------------- + + try { + const queryQueue = items.map(async (item, index) => { + const rawQuery = this.getNodeParameter('query', index) as string; + + return connection.query(rawQuery); + }); + + returnItems = ((await Promise.all(queryQueue)) as mysql2.OkPacket[][]).reduce( + (collection, result, index) => { + const [rows] = result; + + const executionData = this.helpers.constructExecutionMetaData( + this.helpers.returnJsonArray(rows as unknown as IDataObject[]), + { itemData: { item: index } }, + ); + + collection.push(...executionData); + + return collection; + }, + [] as INodeExecutionData[], + ); + } catch (error) { + if (this.continueOnFail()) { + returnItems = this.helpers.returnJsonArray({ error: error.message }); + } else { + await connection.end(); + throw error; + } + } + } else if (operation === 'insert') { + // ---------------------------------- + // insert + // ---------------------------------- + + try { + const table = this.getNodeParameter('table', 0, '', { extractValue: true }) as string; + const columnString = this.getNodeParameter('columns', 0) as string; + const columns = columnString.split(',').map((column) => column.trim()); + const insertItems = copyInputItems(items, columns); + const insertPlaceholder = `(${columns.map((_column) => '?').join(',')})`; + const options = this.getNodeParameter('options', 0); + const insertIgnore = options.ignore as boolean; + const insertPriority = options.priority as string; + + const insertSQL = `INSERT ${insertPriority || ''} ${ + insertIgnore ? 'IGNORE' : '' + } INTO ${table}(${columnString}) VALUES ${items + .map((_item) => insertPlaceholder) + .join(',')};`; + const queryItems = insertItems.reduce( + (collection: IDataObject[], item) => + collection.concat(Object.values(item) as IDataObject[]), + [], + ); + + const queryResult = await connection.query(insertSQL, queryItems); + + returnItems = this.helpers.returnJsonArray(queryResult[0] as unknown as IDataObject); + } catch (error) { + if (this.continueOnFail()) { + returnItems = this.helpers.returnJsonArray({ error: error.message }); + } else { + await connection.end(); + throw error; + } + } + } else if (operation === 'update') { + // ---------------------------------- + // update + // ---------------------------------- + + try { + const table = this.getNodeParameter('table', 0, '', { extractValue: true }) as string; + const updateKey = this.getNodeParameter('updateKey', 0) as string; + const columnString = this.getNodeParameter('columns', 0) as string; + const columns = columnString.split(',').map((column) => column.trim()); + + if (!columns.includes(updateKey)) { + columns.unshift(updateKey); + } + + const updateItems = copyInputItems(items, columns); + const updateSQL = `UPDATE ${table} SET ${columns + .map((column) => `${column} = ?`) + .join(',')} WHERE ${updateKey} = ?;`; + const queryQueue = updateItems.map(async (item) => + connection.query(updateSQL, Object.values(item).concat(item[updateKey])), + ); + const queryResult = await Promise.all(queryQueue); + returnItems = this.helpers.returnJsonArray( + queryResult.map((result) => result[0]) as unknown as IDataObject[], + ); + } catch (error) { + if (this.continueOnFail()) { + returnItems = this.helpers.returnJsonArray({ error: error.message }); + } else { + await connection.end(); + throw error; + } + } + } else { + if (this.continueOnFail()) { + returnItems = this.helpers.returnJsonArray({ + error: `The operation "${operation}" is not supported!`, + }); + } else { + await connection.end(); + throw new NodeOperationError( + this.getNode(), + `The operation "${operation}" is not supported!`, + ); + } + } + + await connection.end(); + + return this.prepareOutputData(returnItems); + } +} diff --git a/packages/nodes-base/nodes/MySql/v2/MySqlV2.node.ts b/packages/nodes-base/nodes/MySql/v2/MySqlV2.node.ts new file mode 100644 index 0000000000..13696dc468 --- /dev/null +++ b/packages/nodes-base/nodes/MySql/v2/MySqlV2.node.ts @@ -0,0 +1,32 @@ +/* eslint-disable n8n-nodes-base/node-filename-against-convention */ +import type { + INodeExecutionData, + INodeType, + INodeTypeBaseDescription, + INodeTypeDescription, +} from 'n8n-workflow'; + +import type { IExecuteFunctions } from 'n8n-core'; + +import { listSearch, credentialTest, loadOptions } from './methods'; + +import { versionDescription } from './actions/versionDescription'; + +import { router } from './actions/router'; + +export class MySqlV2 implements INodeType { + description: INodeTypeDescription; + + constructor(baseDescription: INodeTypeBaseDescription) { + this.description = { + ...baseDescription, + ...versionDescription, + }; + } + + methods = { listSearch, loadOptions, credentialTest }; + + async execute(this: IExecuteFunctions): Promise { + return router.call(this); + } +} diff --git a/packages/nodes-base/nodes/MySql/v2/actions/common.descriptions.ts b/packages/nodes-base/nodes/MySql/v2/actions/common.descriptions.ts new file mode 100644 index 0000000000..69a4187731 --- /dev/null +++ b/packages/nodes-base/nodes/MySql/v2/actions/common.descriptions.ts @@ -0,0 +1,361 @@ +import type { INodeProperties } from 'n8n-workflow'; +import { BATCH_MODE, SINGLE } from '../helpers/interfaces'; + +export const tableRLC: INodeProperties = { + displayName: 'Table', + name: 'table', + type: 'resourceLocator', + default: { mode: 'list', value: '' }, + required: true, + description: 'The table you want to work on', + modes: [ + { + displayName: 'From List', + name: 'list', + type: 'list', + placeholder: 'Select a Table...', + typeOptions: { + searchListMethod: 'searchTables', + searchable: true, + }, + }, + { + displayName: 'Name', + name: 'name', + type: 'string', + placeholder: 'table_name', + }, + ], +}; + +export const optionsCollection: INodeProperties = { + displayName: 'Options', + name: 'options', + type: 'collection', + default: {}, + placeholder: 'Add Option', + options: [ + { + displayName: 'Connection Timeout', + name: 'connectionTimeoutMillis', + type: 'number', + default: 30, + description: 'Number of milliseconds reserved for connecting to the database', + typeOptions: { + minValue: 1, + }, + }, + { + displayName: 'Connections Limit', + name: 'connectionLimit', + type: 'number', + default: 10, + typeOptions: { + minValue: 1, + }, + description: + 'Maximum amount of connections to the database, setting high value can lead to performance issues and potential database crashes', + }, + { + displayName: 'Query Batching', + name: 'queryBatching', + type: 'options', + noDataExpression: true, + description: 'The way queries should be sent to the database', + options: [ + { + name: 'Single Query', + value: BATCH_MODE.SINGLE, + description: 'A single query for all incoming items', + }, + { + name: 'Independently', + value: BATCH_MODE.INDEPENDENTLY, + description: 'Execute one query per incoming item of the run', + }, + { + name: 'Transaction', + value: BATCH_MODE.TRANSACTION, + description: + 'Execute all queries in a transaction, if a failure occurs, all changes are rolled back', + }, + ], + default: SINGLE, + }, + { + displayName: 'Query Parameters', + name: 'queryReplacement', + type: 'string', + default: '', + placeholder: 'e.g. value1,value2,value3', + description: + 'Comma-separated list of the values you want to use as query parameters. You can drag the values from the input panel on the left. More info', + hint: 'Comma-separated list of values: reference them in your query as $1, $2, $3…', + displayOptions: { + show: { '/operation': ['executeQuery'] }, + }, + }, + { + // eslint-disable-next-line n8n-nodes-base/node-param-display-name-wrong-for-dynamic-multi-options + displayName: 'Output Columns', + name: 'outputColumns', + type: 'multiOptions', + description: + 'Choose from the list, or specify IDs using an expression', + typeOptions: { + loadOptionsMethod: 'getColumnsMultiOptions', + loadOptionsDependsOn: ['table.value'], + }, + default: [], + displayOptions: { + show: { '/operation': ['select'] }, + }, + }, + { + displayName: 'Output Large-Format Numbers As', + name: 'largeNumbersOutput', + type: 'options', + options: [ + { + name: 'Numbers', + value: 'numbers', + }, + { + name: 'Text', + value: 'text', + description: + 'Use this if you expect numbers longer than 16 digits (otherwise numbers may be incorrect)', + }, + ], + hint: 'Applies to NUMERIC and BIGINT columns only', + default: 'text', + displayOptions: { + show: { '/operation': ['select', 'executeQuery'] }, + }, + }, + { + displayName: 'Priority', + name: 'priority', + type: 'options', + options: [ + { + name: 'Low Prioirity', + value: 'LOW_PRIORITY', + description: + 'Delays execution of the INSERT until no other clients are reading from the table', + }, + { + name: 'High Priority', + value: 'HIGH_PRIORITY', + description: + 'Overrides the effect of the --low-priority-updates option if the server was started with that option. It also causes concurrent inserts not to be used.', + }, + ], + default: 'LOW_PRIORITY', + description: 'Ignore any ignorable errors that occur while executing the INSERT statement', + displayOptions: { + show: { + '/operation': ['insert'], + }, + }, + }, + { + displayName: 'Replace Empty Strings with NULL', + name: 'replaceEmptyStrings', + type: 'boolean', + default: false, + description: + 'Whether to replace empty strings with NULL in input, could be useful when data come from spreadsheet', + displayOptions: { + show: { + '/operation': ['insert', 'update', 'upsert', 'executeQuery'], + }, + }, + }, + { + displayName: 'Select Distinct', + name: 'selectDistinct', + type: 'boolean', + default: false, + description: 'Whether to remove these duplicate rows', + displayOptions: { + show: { + '/operation': ['select'], + }, + }, + }, + { + displayName: 'Output Query Execution Details', + name: 'detailedOutput', + type: 'boolean', + default: false, + description: + 'Whether to show in output details of the ofexecuted query for each statement, or just confirmation of success', + }, + { + displayName: 'Skip on Conflict', + name: 'skipOnConflict', + type: 'boolean', + default: false, + description: + 'Whether to skip the row and do not throw error if a unique constraint or exclusion constraint is violated', + displayOptions: { + show: { + '/operation': ['insert'], + }, + }, + }, + ], +}; + +export const selectRowsFixedCollection: INodeProperties = { + displayName: 'Select Rows', + name: 'where', + type: 'fixedCollection', + typeOptions: { + multipleValues: true, + }, + placeholder: 'Add Condition', + default: {}, + description: 'If not set, all rows will be selected', + options: [ + { + displayName: 'Values', + name: 'values', + values: [ + { + // eslint-disable-next-line n8n-nodes-base/node-param-display-name-wrong-for-dynamic-options + displayName: 'Column', + name: 'column', + type: 'options', + description: + 'Choose from the list, or specify an ID using an expression', + default: '', + placeholder: 'e.g. ID', + typeOptions: { + loadOptionsMethod: 'getColumns', + loadOptionsDependsOn: ['schema.value', 'table.value'], + }, + }, + { + displayName: 'Operator', + name: 'condition', + type: 'options', + description: + "The operator to check the column against. When using 'LIKE' operator percent sign ( %) matches zero or more characters, underscore ( _ ) matches any single character.", + // eslint-disable-next-line n8n-nodes-base/node-param-options-type-unsorted-items + options: [ + { + name: 'Equal', + value: 'equal', + }, + { + name: 'Not Equal', + value: '!=', + }, + { + name: 'Like', + value: 'LIKE', + }, + { + name: 'Greater Than', + value: '>', + }, + { + name: 'Less Than', + value: '<', + }, + { + name: 'Greater Than Or Equal', + value: '>=', + }, + { + name: 'Less Than Or Equal', + value: '<=', + }, + { + name: 'Is Null', + value: 'IS NULL', + }, + ], + default: 'equal', + }, + { + displayName: 'Value', + name: 'value', + type: 'string', + default: '', + }, + ], + }, + ], +}; + +export const sortFixedCollection: INodeProperties = { + displayName: 'Sort', + name: 'sort', + type: 'fixedCollection', + typeOptions: { + multipleValues: true, + }, + placeholder: 'Add Sort Rule', + default: {}, + options: [ + { + displayName: 'Values', + name: 'values', + values: [ + { + // eslint-disable-next-line n8n-nodes-base/node-param-display-name-wrong-for-dynamic-options + displayName: 'Column', + name: 'column', + type: 'options', + description: + 'Choose from the list, or specify an ID using an expression', + default: '', + typeOptions: { + loadOptionsMethod: 'getColumns', + loadOptionsDependsOn: ['schema.value', 'table.value'], + }, + }, + { + displayName: 'Direction', + name: 'direction', + type: 'options', + options: [ + { + name: 'ASC', + value: 'ASC', + }, + { + name: 'DESC', + value: 'DESC', + }, + ], + default: 'ASC', + }, + ], + }, + ], +}; + +export const combineConditionsCollection: INodeProperties = { + displayName: 'Combine Conditions', + name: 'combineConditions', + type: 'options', + description: + 'How to combine the conditions defined in "Select Rows": AND requires all conditions to be true, OR requires at least one condition to be true', + options: [ + { + name: 'AND', + value: 'AND', + description: 'Only rows that meet all the conditions are selected', + }, + { + name: 'OR', + value: 'OR', + description: 'Rows that meet at least one condition are selected', + }, + ], + default: 'AND', +}; diff --git a/packages/nodes-base/nodes/MySql/v2/actions/database/Database.resource.ts b/packages/nodes-base/nodes/MySql/v2/actions/database/Database.resource.ts new file mode 100644 index 0000000000..88e93664e0 --- /dev/null +++ b/packages/nodes-base/nodes/MySql/v2/actions/database/Database.resource.ts @@ -0,0 +1,76 @@ +import type { INodeProperties } from 'n8n-workflow'; + +import * as deleteTable from './deleteTable.operation'; +import * as executeQuery from './executeQuery.operation'; +import * as insert from './insert.operation'; +import * as select from './select.operation'; +import * as update from './update.operation'; +import * as upsert from './upsert.operation'; +import { tableRLC } from '../common.descriptions'; + +export { deleteTable, executeQuery, insert, select, update, upsert }; + +export const description: INodeProperties[] = [ + { + displayName: 'Operation', + name: 'operation', + type: 'options', + noDataExpression: true, + options: [ + { + name: 'Delete', + value: 'deleteTable', + description: 'Delete an entire table or rows in a table', + action: 'Delete table or rows', + }, + { + name: 'Execute SQL', + value: 'executeQuery', + description: 'Execute an SQL query', + action: 'Execute a SQL query', + }, + { + name: 'Insert', + value: 'insert', + description: 'Insert rows in a table', + action: 'Insert rows in a table', + }, + { + // eslint-disable-next-line n8n-nodes-base/node-param-option-name-wrong-for-upsert + name: 'Insert or Update', + value: 'upsert', + // eslint-disable-next-line n8n-nodes-base/node-param-description-wrong-for-upsert + description: 'Insert or update rows in a table', + action: 'Insert or update rows in a table', + }, + { + name: 'Select', + value: 'select', + description: 'Select rows from a table', + action: 'Select rows from a table', + }, + { + name: 'Update', + value: 'update', + description: 'Update rows in a table', + action: 'Update rows in a table', + }, + ], + displayOptions: { + show: { + resource: ['database'], + }, + }, + default: 'insert', + }, + { + ...tableRLC, + displayOptions: { hide: { operation: ['executeQuery'] } }, + }, + ...deleteTable.description, + ...executeQuery.description, + ...insert.description, + ...select.description, + ...update.description, + ...upsert.description, +]; diff --git a/packages/nodes-base/nodes/MySql/v2/actions/database/deleteTable.operation.ts b/packages/nodes-base/nodes/MySql/v2/actions/database/deleteTable.operation.ts new file mode 100644 index 0000000000..09cadea313 --- /dev/null +++ b/packages/nodes-base/nodes/MySql/v2/actions/database/deleteTable.operation.ts @@ -0,0 +1,137 @@ +import type { IExecuteFunctions } from 'n8n-core'; +import type { IDataObject, INodeExecutionData, INodeProperties } from 'n8n-workflow'; +import { NodeOperationError } from 'n8n-workflow'; + +import type { + QueryRunner, + QueryValues, + QueryWithValues, + WhereClause, +} from '../../helpers/interfaces'; + +import { updateDisplayOptions } from '../../../../../utils/utilities'; + +import { addWhereClauses } from '../../helpers/utils'; + +import { + optionsCollection, + selectRowsFixedCollection, + combineConditionsCollection, +} from '../common.descriptions'; + +const properties: INodeProperties[] = [ + { + displayName: 'Command', + name: 'deleteCommand', + type: 'options', + default: 'truncate', + options: [ + { + name: 'Truncate', + value: 'truncate', + description: "Only removes the table's data and preserves the table's structure", + }, + { + name: 'Delete', + value: 'delete', + description: + "Delete the rows that match the 'Select Rows' conditions below. If no selection is made, all rows in the table are deleted.", + }, + { + name: 'Drop', + value: 'drop', + description: "Deletes the table's data and also the table's structure permanently", + }, + ], + }, + { + ...selectRowsFixedCollection, + displayOptions: { + show: { + deleteCommand: ['delete'], + }, + }, + }, + { + ...combineConditionsCollection, + displayOptions: { + show: { + deleteCommand: ['delete'], + }, + }, + }, + optionsCollection, +]; + +const displayOptions = { + show: { + resource: ['database'], + operation: ['deleteTable'], + }, + hide: { + table: [''], + }, +}; + +export const description = updateDisplayOptions(displayOptions, properties); + +export async function execute( + this: IExecuteFunctions, + inputItems: INodeExecutionData[], + runQueries: QueryRunner, +): Promise { + let returnData: INodeExecutionData[] = []; + + const queries: QueryWithValues[] = []; + + for (let i = 0; i < inputItems.length; i++) { + const table = this.getNodeParameter('table', i, undefined, { + extractValue: true, + }) as string; + + const deleteCommand = this.getNodeParameter('deleteCommand', i) as string; + + let query = ''; + let values: QueryValues = []; + + if (deleteCommand === 'drop') { + query = `DROP TABLE IF EXISTS \`${table}\``; + } + + if (deleteCommand === 'truncate') { + query = `TRUNCATE TABLE \`${table}\``; + } + + if (deleteCommand === 'delete') { + const whereClauses = + ((this.getNodeParameter('where', i, []) as IDataObject).values as WhereClause[]) || []; + + const combineConditions = this.getNodeParameter('combineConditions', i, 'AND') as string; + + [query, values] = addWhereClauses( + this.getNode(), + i, + `DELETE FROM \`${table}\``, + whereClauses, + values, + combineConditions, + ); + } + + if (query === '') { + throw new NodeOperationError( + this.getNode(), + 'Invalid delete command, only drop, delete and truncate are supported ', + { itemIndex: i }, + ); + } + + const queryWithValues = { query, values }; + + queries.push(queryWithValues); + } + + returnData = await runQueries(queries); + + return returnData; +} diff --git a/packages/nodes-base/nodes/MySql/v2/actions/database/executeQuery.operation.ts b/packages/nodes-base/nodes/MySql/v2/actions/database/executeQuery.operation.ts new file mode 100644 index 0000000000..bf247596d3 --- /dev/null +++ b/packages/nodes-base/nodes/MySql/v2/actions/database/executeQuery.operation.ts @@ -0,0 +1,89 @@ +import type { IExecuteFunctions } from 'n8n-core'; +import type { IDataObject, INodeExecutionData, INodeProperties } from 'n8n-workflow'; +import { NodeOperationError } from 'n8n-workflow'; + +import type { QueryRunner, QueryWithValues } from '../../helpers/interfaces'; + +import { updateDisplayOptions } from '../../../../../utils/utilities'; + +import { prepareQueryAndReplacements, replaceEmptyStringsByNulls } from '../../helpers/utils'; + +import { optionsCollection } from '../common.descriptions'; + +const properties: INodeProperties[] = [ + { + displayName: 'Query', + name: 'query', + type: 'string', + default: '', + placeholder: 'e.g. SELECT id, name FROM product WHERE id < 40', + required: true, + description: + "The SQL query to execute. You can use n8n expressions and $1, $2, $3, etc to refer to the 'Query Parameters' set in options below.", + typeOptions: { + rows: 3, + }, + hint: 'Prefer using query parameters over n8n expressions to avoid SQL injection attacks', + }, + { + displayName: ` + To use query parameters in your SQL query, reference them as $1, $2, $3, etc in the corresponding order. More info. + `, + name: 'notice', + type: 'notice', + default: '', + }, + optionsCollection, +]; + +const displayOptions = { + show: { + resource: ['database'], + operation: ['executeQuery'], + }, +}; + +export const description = updateDisplayOptions(displayOptions, properties); + +export async function execute( + this: IExecuteFunctions, + inputItems: INodeExecutionData[], + runQueries: QueryRunner, + nodeOptions: IDataObject, +): Promise { + let returnData: INodeExecutionData[] = []; + const items = replaceEmptyStringsByNulls(inputItems, nodeOptions.replaceEmptyStrings as boolean); + + const queries: QueryWithValues[] = []; + + for (let i = 0; i < items.length; i++) { + const rawQuery = this.getNodeParameter('query', i) as string; + + const options = this.getNodeParameter('options', i, {}); + + let values; + let queryReplacement = options.queryReplacement || []; + + if (typeof queryReplacement === 'string') { + queryReplacement = queryReplacement.split(',').map((entry) => entry.trim()); + } + + if (Array.isArray(queryReplacement)) { + values = queryReplacement as IDataObject[]; + } else { + throw new NodeOperationError( + this.getNode(), + 'Query Replacement must be a string of comma-separated values, or an array of values', + { itemIndex: i }, + ); + } + + const preparedQuery = prepareQueryAndReplacements(rawQuery, values); + + queries.push(preparedQuery); + } + + returnData = await runQueries(queries); + + return returnData; +} diff --git a/packages/nodes-base/nodes/MySql/v2/actions/database/insert.operation.ts b/packages/nodes-base/nodes/MySql/v2/actions/database/insert.operation.ts new file mode 100644 index 0000000000..ce5c523c66 --- /dev/null +++ b/packages/nodes-base/nodes/MySql/v2/actions/database/insert.operation.ts @@ -0,0 +1,227 @@ +import type { IExecuteFunctions } from 'n8n-core'; +import type { IDataObject, INodeExecutionData, INodeProperties } from 'n8n-workflow'; + +import type { + QueryMode, + QueryRunner, + QueryValues, + QueryWithValues, +} from '../../helpers/interfaces'; + +import { AUTO_MAP, BATCH_MODE, DATA_MODE } from '../../helpers/interfaces'; + +import { updateDisplayOptions } from '../../../../../utils/utilities'; + +import { copyInputItems, replaceEmptyStringsByNulls } from '../../helpers/utils'; + +import { optionsCollection } from '../common.descriptions'; + +const properties: INodeProperties[] = [ + { + displayName: 'Data Mode', + name: 'dataMode', + type: 'options', + options: [ + { + name: 'Auto-Map Input Data to Columns', + value: DATA_MODE.AUTO_MAP, + description: 'Use when node input properties names exactly match the table column names', + }, + { + name: 'Map Each Column Manually', + value: DATA_MODE.MANUAL, + description: 'Set the value for each destination column manually', + }, + ], + default: AUTO_MAP, + description: + 'Whether to map node input properties and the table data automatically or manually', + }, + { + displayName: ` + In this mode, make sure incoming data fields are named the same as the columns in your table. If needed, use a 'Set' node before this node to change the field names. + `, + name: 'notice', + type: 'notice', + default: '', + displayOptions: { + show: { + dataMode: [DATA_MODE.AUTO_MAP], + }, + }, + }, + { + displayName: 'Values to Send', + name: 'valuesToSend', + placeholder: 'Add Value', + type: 'fixedCollection', + typeOptions: { + multipleValueButtonText: 'Add Value', + multipleValues: true, + }, + displayOptions: { + show: { + dataMode: [DATA_MODE.MANUAL], + }, + }, + default: {}, + options: [ + { + displayName: 'Values', + name: 'values', + values: [ + { + // eslint-disable-next-line n8n-nodes-base/node-param-display-name-wrong-for-dynamic-options + displayName: 'Column', + name: 'column', + type: 'options', + description: + 'Choose from the list, or specify an ID using an expression', + typeOptions: { + loadOptionsMethod: 'getColumns', + loadOptionsDependsOn: ['table.value'], + }, + default: [], + }, + { + displayName: 'Value', + name: 'value', + type: 'string', + default: '', + }, + ], + }, + ], + }, + optionsCollection, +]; + +const displayOptions = { + show: { + resource: ['database'], + operation: ['insert'], + }, + hide: { + table: [''], + }, +}; + +export const description = updateDisplayOptions(displayOptions, properties); + +export async function execute( + this: IExecuteFunctions, + inputItems: INodeExecutionData[], + runQueries: QueryRunner, + nodeOptions: IDataObject, +): Promise { + let returnData: INodeExecutionData[] = []; + const items = replaceEmptyStringsByNulls(inputItems, nodeOptions.replaceEmptyStrings as boolean); + + const table = this.getNodeParameter('table', 0, '', { extractValue: true }) as string; + + const dataMode = this.getNodeParameter('dataMode', 0) as string; + const queryBatching = (nodeOptions.queryBatching as QueryMode) || BATCH_MODE.SINGLE; + + const queries: QueryWithValues[] = []; + + if (queryBatching === BATCH_MODE.SINGLE) { + let columns: string[] = []; + let insertItems: IDataObject[] = []; + + const priority = (nodeOptions.priority as string) || ''; + const ignore = (nodeOptions.skipOnConflict as boolean) ? 'IGNORE' : ''; + + if (dataMode === DATA_MODE.AUTO_MAP) { + columns = [ + ...new Set( + items.reduce((acc, item) => { + const itemColumns = Object.keys(item.json); + + return acc.concat(itemColumns); + }, [] as string[]), + ), + ]; + insertItems = copyInputItems(items, columns); + } + + if (dataMode === DATA_MODE.MANUAL) { + for (let i = 0; i < items.length; i++) { + const valuesToSend = (this.getNodeParameter('valuesToSend', i, []) as IDataObject) + .values as IDataObject[]; + + const item = valuesToSend.reduce((acc, { column, value }) => { + acc[column as string] = value; + return acc; + }, {} as IDataObject); + + insertItems.push(item); + } + columns = [ + ...new Set( + insertItems.reduce((acc, item) => { + const itemColumns = Object.keys(item); + + return acc.concat(itemColumns); + }, [] as string[]), + ), + ]; + } + + const escapedColumns = columns.map((column) => `\`${column}\``).join(', '); + const placeholder = `(${columns.map(() => '?').join(',')})`; + const replacements = items.map(() => placeholder).join(','); + + const query = `INSERT ${priority} ${ignore} INTO \`${table}\` (${escapedColumns}) VALUES ${replacements}`; + + const values = insertItems.reduce( + (acc: IDataObject[], item) => acc.concat(Object.values(item) as IDataObject[]), + [], + ); + + queries.push({ query, values }); + } else { + for (let i = 0; i < items.length; i++) { + let columns: string[] = []; + let insertItem: IDataObject = {}; + + const options = this.getNodeParameter('options', i); + const priority = (options.priority as string) || ''; + const ignore = (options.skipOnConflict as boolean) ? 'IGNORE' : ''; + + if (dataMode === DATA_MODE.AUTO_MAP) { + columns = Object.keys(items[i].json); + insertItem = columns.reduce((acc, key) => { + if (columns.includes(key)) { + acc[key] = items[i].json[key]; + } + return acc; + }, {} as IDataObject); + } + + if (dataMode === DATA_MODE.MANUAL) { + const valuesToSend = (this.getNodeParameter('valuesToSend', i, []) as IDataObject) + .values as IDataObject[]; + + insertItem = valuesToSend.reduce((acc, { column, value }) => { + acc[column as string] = value; + return acc; + }, {} as IDataObject); + + columns = Object.keys(insertItem); + } + + const escapedColumns = columns.map((column) => `\`${column}\``).join(', '); + const placeholder = `(${columns.map(() => '?').join(',')})`; + + const query = `INSERT ${priority} ${ignore} INTO \`${table}\` (${escapedColumns}) VALUES ${placeholder};`; + + const values = Object.values(insertItem) as QueryValues; + + queries.push({ query, values }); + } + } + + returnData = await runQueries(queries); + + return returnData; +} diff --git a/packages/nodes-base/nodes/MySql/v2/actions/database/select.operation.ts b/packages/nodes-base/nodes/MySql/v2/actions/database/select.operation.ts new file mode 100644 index 0000000000..5afbe31513 --- /dev/null +++ b/packages/nodes-base/nodes/MySql/v2/actions/database/select.operation.ts @@ -0,0 +1,131 @@ +import type { IExecuteFunctions } from 'n8n-core'; +import type { IDataObject, INodeExecutionData, INodeProperties } from 'n8n-workflow'; + +import type { + QueryRunner, + QueryValues, + QueryWithValues, + SortRule, + WhereClause, +} from '../../helpers/interfaces'; + +import { updateDisplayOptions } from '../../../../../utils/utilities'; + +import { addSortRules, addWhereClauses } from '../../helpers/utils'; + +import { + optionsCollection, + sortFixedCollection, + selectRowsFixedCollection, + combineConditionsCollection, +} from '../common.descriptions'; + +const properties: INodeProperties[] = [ + { + displayName: 'Return All', + name: 'returnAll', + type: 'boolean', + default: false, + description: 'Whether to return all results or only up to a given limit', + displayOptions: { + show: { + resource: ['event'], + operation: ['getAll'], + }, + }, + }, + { + displayName: 'Limit', + name: 'limit', + type: 'number', + default: 50, + description: 'Max number of results to return', + typeOptions: { + minValue: 1, + }, + displayOptions: { + show: { + returnAll: [false], + }, + }, + }, + selectRowsFixedCollection, + combineConditionsCollection, + sortFixedCollection, + optionsCollection, +]; + +const displayOptions = { + show: { + resource: ['database'], + operation: ['select'], + }, + hide: { + table: [''], + }, +}; + +export const description = updateDisplayOptions(displayOptions, properties); + +export async function execute( + this: IExecuteFunctions, + inputItems: INodeExecutionData[], + runQueries: QueryRunner, +): Promise { + let returnData: INodeExecutionData[] = []; + + const queries: QueryWithValues[] = []; + + for (let i = 0; i < inputItems.length; i++) { + const table = this.getNodeParameter('table', i, undefined, { + extractValue: true, + }) as string; + + const outputColumns = this.getNodeParameter('options.outputColumns', i, ['*']) as string[]; + const selectDistinct = this.getNodeParameter('options.selectDistinct', i, false) as boolean; + + let query = ''; + const SELECT = selectDistinct ? 'SELECT DISTINCT' : 'SELECT'; + + if (outputColumns.includes('*')) { + query = `${SELECT} * FROM \`${table}\``; + } else { + const escapedColumns = outputColumns.map((column) => `\`${column}\``).join(', '); + query = `${SELECT} ${escapedColumns} FROM \`${table}\``; + } + + let values: QueryValues = []; + + const whereClauses = + ((this.getNodeParameter('where', i, []) as IDataObject).values as WhereClause[]) || []; + + const combineConditions = this.getNodeParameter('combineConditions', i, 'AND') as string; + + [query, values] = addWhereClauses( + this.getNode(), + i, + query, + whereClauses, + values, + combineConditions, + ); + + const sortRules = + ((this.getNodeParameter('sort', i, []) as IDataObject).values as SortRule[]) || []; + + [query, values] = addSortRules(query, sortRules, values); + + const returnAll = this.getNodeParameter('returnAll', i, false); + if (!returnAll) { + const limit = this.getNodeParameter('limit', i, 50); + query += ' LIMIT ?'; + values.push(limit); + } + + queries.push({ query, values }); + } + + returnData = await runQueries(queries); + + return returnData; +} diff --git a/packages/nodes-base/nodes/MySql/v2/actions/database/update.operation.ts b/packages/nodes-base/nodes/MySql/v2/actions/database/update.operation.ts new file mode 100644 index 0000000000..b3620e0c4b --- /dev/null +++ b/packages/nodes-base/nodes/MySql/v2/actions/database/update.operation.ts @@ -0,0 +1,195 @@ +import type { IExecuteFunctions } from 'n8n-core'; +import type { IDataObject, INodeExecutionData, INodeProperties } from 'n8n-workflow'; + +import type { QueryRunner, QueryValues, QueryWithValues } from '../../helpers/interfaces'; +import { AUTO_MAP, DATA_MODE } from '../../helpers/interfaces'; + +import { updateDisplayOptions } from '../../../../../utils/utilities'; + +import { replaceEmptyStringsByNulls } from '../../helpers/utils'; + +import { optionsCollection } from '../common.descriptions'; + +const properties: INodeProperties[] = [ + { + displayName: 'Data Mode', + name: 'dataMode', + type: 'options', + options: [ + { + name: 'Auto-Map Input Data to Columns', + value: DATA_MODE.AUTO_MAP, + description: 'Use when node input properties names exactly match the table column names', + }, + { + name: 'Map Each Column Below', + value: DATA_MODE.MANUAL, + description: 'Set the value for each destination column manually', + }, + ], + default: AUTO_MAP, + description: + 'Whether to map node input properties and the table data automatically or manually', + }, + { + displayName: ` + In this mode, make sure incoming data fields are named the same as the columns in your table. If needed, use a 'Set' node before this node to change the field names. + `, + name: 'notice', + type: 'notice', + default: '', + displayOptions: { + show: { + dataMode: [DATA_MODE.AUTO_MAP], + }, + }, + }, + { + // eslint-disable-next-line n8n-nodes-base/node-param-display-name-miscased, n8n-nodes-base/node-param-display-name-wrong-for-dynamic-options + displayName: 'Column to Match On', + name: 'columnToMatchOn', + type: 'options', + required: true, + description: + 'Choose from the list, or specify an ID using an expression', + typeOptions: { + loadOptionsMethod: 'getColumns', + loadOptionsDependsOn: ['schema.value', 'table.value'], + }, + default: '', + hint: "Used to find the correct row to update. Doesn't get changed.", + }, + { + displayName: 'Value of Column to Match On', + name: 'valueToMatchOn', + type: 'string', + default: '', + description: + 'Rows with a value in the specified "Column to Match On" that corresponds to the value in this field will be updated', + displayOptions: { + show: { + dataMode: [DATA_MODE.MANUAL], + }, + }, + }, + { + displayName: 'Values to Send', + name: 'valuesToSend', + placeholder: 'Add Value', + type: 'fixedCollection', + typeOptions: { + multipleValueButtonText: 'Add Value', + multipleValues: true, + }, + displayOptions: { + show: { + dataMode: [DATA_MODE.MANUAL], + }, + }, + default: {}, + options: [ + { + displayName: 'Values', + name: 'values', + values: [ + { + // eslint-disable-next-line n8n-nodes-base/node-param-display-name-wrong-for-dynamic-options + displayName: 'Column', + name: 'column', + type: 'options', + description: + 'Choose from the list, or specify an ID using an expression', + typeOptions: { + loadOptionsMethod: 'getColumnsWithoutColumnToMatchOn', + loadOptionsDependsOn: ['schema.value', 'table.value'], + }, + default: [], + }, + { + displayName: 'Value', + name: 'value', + type: 'string', + default: '', + }, + ], + }, + ], + }, + optionsCollection, +]; + +const displayOptions = { + show: { + resource: ['database'], + operation: ['update'], + }, + hide: { + table: [''], + }, +}; + +export const description = updateDisplayOptions(displayOptions, properties); + +export async function execute( + this: IExecuteFunctions, + inputItems: INodeExecutionData[], + runQueries: QueryRunner, + nodeOptions: IDataObject, +): Promise { + let returnData: INodeExecutionData[] = []; + const items = replaceEmptyStringsByNulls(inputItems, nodeOptions.replaceEmptyStrings as boolean); + + const queries: QueryWithValues[] = []; + + for (let i = 0; i < items.length; i++) { + const table = this.getNodeParameter('table', i, undefined, { + extractValue: true, + }) as string; + + const columnToMatchOn = this.getNodeParameter('columnToMatchOn', i) as string; + + const dataMode = this.getNodeParameter('dataMode', i) as string; + + let item: IDataObject = {}; + let valueToMatchOn: string | IDataObject = ''; + + if (dataMode === DATA_MODE.AUTO_MAP) { + item = items[i].json; + valueToMatchOn = item[columnToMatchOn] as string; + } + + if (dataMode === DATA_MODE.MANUAL) { + const valuesToSend = (this.getNodeParameter('valuesToSend', i, []) as IDataObject) + .values as IDataObject[]; + + item = valuesToSend.reduce((acc, { column, value }) => { + acc[column as string] = value; + return acc; + }, {} as IDataObject); + + valueToMatchOn = this.getNodeParameter('valueToMatchOn', i) as string; + } + + const values: QueryValues = []; + + const updateColumns = Object.keys(item).filter((column) => column !== columnToMatchOn); + + const updates: string[] = []; + + for (const column of updateColumns) { + updates.push(`\`${column}\` = ?`); + values.push(item[column] as string); + } + + const condition = `\`${columnToMatchOn}\` = ?`; + values.push(valueToMatchOn); + + const query = `UPDATE \`${table}\` SET ${updates.join(', ')} WHERE ${condition}`; + + queries.push({ query, values }); + } + + returnData = await runQueries(queries); + + return returnData; +} diff --git a/packages/nodes-base/nodes/MySql/v2/actions/database/upsert.operation.ts b/packages/nodes-base/nodes/MySql/v2/actions/database/upsert.operation.ts new file mode 100644 index 0000000000..dd50f20ba1 --- /dev/null +++ b/packages/nodes-base/nodes/MySql/v2/actions/database/upsert.operation.ts @@ -0,0 +1,199 @@ +import type { IExecuteFunctions } from 'n8n-core'; +import type { IDataObject, INodeExecutionData, INodeProperties } from 'n8n-workflow'; + +import type { QueryRunner, QueryValues, QueryWithValues } from '../../helpers/interfaces'; +import { AUTO_MAP, DATA_MODE } from '../../helpers/interfaces'; + +import { updateDisplayOptions } from '../../../../../utils/utilities'; + +import { replaceEmptyStringsByNulls } from '../../helpers/utils'; + +import { optionsCollection } from '../common.descriptions'; + +const properties: INodeProperties[] = [ + { + displayName: 'Data Mode', + name: 'dataMode', + type: 'options', + options: [ + { + name: 'Auto-Map Input Data to Columns', + value: DATA_MODE.AUTO_MAP, + description: 'Use when node input properties names exactly match the table column names', + }, + { + name: 'Map Each Column Below', + value: DATA_MODE.MANUAL, + description: 'Set the value for each destination column manually', + }, + ], + default: AUTO_MAP, + description: + 'Whether to map node input properties and the table data automatically or manually', + }, + { + displayName: ` + In this mode, make sure incoming data fields are named the same as the columns in your table. If needed, use a 'Set' node before this node to change the field names. + `, + name: 'notice', + type: 'notice', + default: '', + displayOptions: { + show: { + dataMode: [DATA_MODE.AUTO_MAP], + }, + }, + }, + { + // eslint-disable-next-line n8n-nodes-base/node-param-display-name-miscased, n8n-nodes-base/node-param-display-name-wrong-for-dynamic-options + displayName: 'Column to Match On', + name: 'columnToMatchOn', + type: 'options', + required: true, + description: + 'The column to compare when finding the rows to update. Choose from the list, or specify an ID using an expression.', + typeOptions: { + loadOptionsMethod: 'getColumns', + loadOptionsDependsOn: ['schema.value', 'table.value'], + }, + default: '', + hint: "Used to find the correct row to update. Doesn't get changed. Has to be unique.", + }, + { + displayName: 'Value of Column to Match On', + name: 'valueToMatchOn', + type: 'string', + default: '', + description: + 'Rows with a value in the specified "Column to Match On" that corresponds to the value in this field will be updated. New rows will be created for non-matching items.', + displayOptions: { + show: { + dataMode: [DATA_MODE.MANUAL], + }, + }, + }, + { + displayName: 'Values to Send', + name: 'valuesToSend', + placeholder: 'Add Value', + type: 'fixedCollection', + typeOptions: { + multipleValueButtonText: 'Add Value', + multipleValues: true, + }, + displayOptions: { + show: { + dataMode: [DATA_MODE.MANUAL], + }, + }, + default: {}, + options: [ + { + displayName: 'Values', + name: 'values', + values: [ + { + // eslint-disable-next-line n8n-nodes-base/node-param-display-name-wrong-for-dynamic-options + displayName: 'Column', + name: 'column', + type: 'options', + description: + 'Choose from the list, or specify an ID using an expression', + typeOptions: { + loadOptionsMethod: 'getColumnsWithoutColumnToMatchOn', + loadOptionsDependsOn: ['schema.value', 'table.value'], + }, + default: [], + }, + { + displayName: 'Value', + name: 'value', + type: 'string', + default: '', + }, + ], + }, + ], + }, + optionsCollection, +]; + +const displayOptions = { + show: { + resource: ['database'], + operation: ['upsert'], + }, + hide: { + table: [''], + }, +}; + +export const description = updateDisplayOptions(displayOptions, properties); + +export async function execute( + this: IExecuteFunctions, + inputItems: INodeExecutionData[], + runQueries: QueryRunner, + nodeOptions: IDataObject, +): Promise { + let returnData: INodeExecutionData[] = []; + + const items = replaceEmptyStringsByNulls(inputItems, nodeOptions.replaceEmptyStrings as boolean); + + const queries: QueryWithValues[] = []; + + for (let i = 0; i < items.length; i++) { + const table = this.getNodeParameter('table', i, undefined, { + extractValue: true, + }) as string; + + const columnToMatchOn = this.getNodeParameter('columnToMatchOn', i) as string; + + const dataMode = this.getNodeParameter('dataMode', i) as string; + + let item: IDataObject = {}; + + if (dataMode === DATA_MODE.AUTO_MAP) { + item = items[i].json; + } + + if (dataMode === DATA_MODE.MANUAL) { + const valuesToSend = (this.getNodeParameter('valuesToSend', i, []) as IDataObject) + .values as IDataObject[]; + + item = valuesToSend.reduce((acc, { column, value }) => { + acc[column as string] = value; + return acc; + }, {} as IDataObject); + + item[columnToMatchOn] = this.getNodeParameter('valueToMatchOn', i) as string; + } + + const onConflict = 'ON DUPLICATE KEY UPDATE'; + + const columns = Object.keys(item); + const escapedColumns = columns.map((column) => `\`${column}\``).join(', '); + const placeholder = `${columns.map(() => '?').join(',')}`; + + const insertQuery = `INSERT INTO \`${table}\`(${escapedColumns}) VALUES(${placeholder})`; + + const values = Object.values(item) as QueryValues; + + const updateColumns = Object.keys(item).filter((column) => column !== columnToMatchOn); + + const updates: string[] = []; + + for (const column of updateColumns) { + updates.push(`\`${column}\` = ?`); + values.push(item[column] as string); + } + + const query = `${insertQuery} ${onConflict} ${updates.join(', ')}`; + + queries.push({ query, values }); + } + + returnData = await runQueries(queries); + + return returnData; +} diff --git a/packages/nodes-base/nodes/MySql/v2/actions/node.type.ts b/packages/nodes-base/nodes/MySql/v2/actions/node.type.ts new file mode 100644 index 0000000000..b571dc0952 --- /dev/null +++ b/packages/nodes-base/nodes/MySql/v2/actions/node.type.ts @@ -0,0 +1,9 @@ +import type { AllEntities, Entity } from 'n8n-workflow'; + +type MySQLMap = { + database: 'deleteTable' | 'executeQuery' | 'insert' | 'select' | 'update' | 'upsert'; +}; + +export type MySqlType = AllEntities; + +export type MySQLDatabaseType = Entity; diff --git a/packages/nodes-base/nodes/MySql/v2/actions/router.ts b/packages/nodes-base/nodes/MySql/v2/actions/router.ts new file mode 100644 index 0000000000..9aa5758672 --- /dev/null +++ b/packages/nodes-base/nodes/MySql/v2/actions/router.ts @@ -0,0 +1,66 @@ +import type { INodeExecutionData } from 'n8n-workflow'; +import { NodeOperationError } from 'n8n-workflow'; +import type { IExecuteFunctions } from 'n8n-core'; + +import { Client } from 'ssh2'; + +import type { MySqlType } from './node.type'; +import type { QueryRunner } from '../helpers/interfaces'; + +import * as database from './database/Database.resource'; + +import { createPool } from '../transport'; +import { configureQueryRunner } from '../helpers/utils'; + +export async function router(this: IExecuteFunctions): Promise { + let returnData: INodeExecutionData[] = []; + + const resource = this.getNodeParameter('resource', 0); + const operation = this.getNodeParameter('operation', 0); + const nodeOptions = this.getNodeParameter('options', 0); + + const credentials = await this.getCredentials('mySql'); + + let sshClient: Client | undefined = undefined; + + if (credentials.sshTunnel) { + sshClient = new Client(); + } + const pool = await createPool(credentials, nodeOptions, sshClient); + + const runQueries: QueryRunner = configureQueryRunner.call(this, nodeOptions, pool); + + const mysqlNodeData = { + resource, + operation, + } as MySqlType; + + try { + switch (mysqlNodeData.resource) { + case 'database': + const items = this.getInputData(); + + returnData = await database[mysqlNodeData.operation].execute.call( + this, + items, + runQueries, + nodeOptions, + ); + break; + default: + throw new NodeOperationError( + this.getNode(), + `The operation "${operation}" is not supported!`, + ); + } + } catch (error) { + throw error; + } finally { + if (sshClient) { + sshClient.end(); + } + await pool.end(); + } + + return this.prepareOutputData(returnData); +} diff --git a/packages/nodes-base/nodes/MySql/v2/actions/versionDescription.ts b/packages/nodes-base/nodes/MySql/v2/actions/versionDescription.ts new file mode 100644 index 0000000000..fd12c75352 --- /dev/null +++ b/packages/nodes-base/nodes/MySql/v2/actions/versionDescription.ts @@ -0,0 +1,42 @@ +/* eslint-disable n8n-nodes-base/node-filename-against-convention */ +import type { INodeTypeDescription } from 'n8n-workflow'; + +import * as database from './database/Database.resource'; + +export const versionDescription: INodeTypeDescription = { + displayName: 'MySQL', + name: 'mySql', + icon: 'file:mysql.svg', + group: ['input'], + version: 2, + subtitle: '={{ $parameter["operation"] }}', + description: 'Get, add and update data in MySQL', + defaults: { + name: 'MySQL', + }, + inputs: ['main'], + outputs: ['main'], + credentials: [ + { + name: 'mySql', + required: true, + testedBy: 'mysqlConnectionTest', + }, + ], + properties: [ + { + displayName: 'Resource', + name: 'resource', + type: 'hidden', + noDataExpression: true, + options: [ + { + name: 'Database', + value: 'database', + }, + ], + default: 'database', + }, + ...database.description, + ], +}; diff --git a/packages/nodes-base/nodes/MySql/v2/helpers/interfaces.ts b/packages/nodes-base/nodes/MySql/v2/helpers/interfaces.ts new file mode 100644 index 0000000000..1fef0f3bb4 --- /dev/null +++ b/packages/nodes-base/nodes/MySql/v2/helpers/interfaces.ts @@ -0,0 +1,28 @@ +import type mysql2 from 'mysql2/promise'; +import type { IDataObject, INodeExecutionData } from 'n8n-workflow'; + +export type Mysql2Connection = mysql2.Connection; +export type Mysql2Pool = mysql2.Pool; +export type Mysql2OkPacket = mysql2.OkPacket; + +export type QueryValues = Array; +export type QueryWithValues = { query: string; values: QueryValues }; + +export type QueryRunner = (queries: QueryWithValues[]) => Promise; + +export type WhereClause = { column: string; condition: string; value: string | number }; +export type SortRule = { column: string; direction: string }; + +export const AUTO_MAP = 'autoMapInputData'; +const MANUAL = 'defineBelow'; +export const DATA_MODE = { + AUTO_MAP, + MANUAL, +}; + +export const SINGLE = 'single'; +const TRANSACTION = 'transaction'; +const INDEPENDENTLY = 'independently'; +export const BATCH_MODE = { SINGLE, TRANSACTION, INDEPENDENTLY }; + +export type QueryMode = typeof SINGLE | typeof TRANSACTION | typeof INDEPENDENTLY; diff --git a/packages/nodes-base/nodes/MySql/v2/helpers/utils.ts b/packages/nodes-base/nodes/MySql/v2/helpers/utils.ts new file mode 100644 index 0000000000..7c72e47332 --- /dev/null +++ b/packages/nodes-base/nodes/MySql/v2/helpers/utils.ts @@ -0,0 +1,414 @@ +import type { + IDataObject, + IExecuteFunctions, + INode, + INodeExecutionData, + IPairedItemData, + NodeExecutionWithMetadata, +} from 'n8n-workflow'; + +import { NodeOperationError, deepCopy } from 'n8n-workflow'; + +import type { + Mysql2Pool, + QueryMode, + QueryValues, + QueryWithValues, + SortRule, + WhereClause, +} from './interfaces'; + +import { BATCH_MODE } from './interfaces'; + +export function copyInputItems(items: INodeExecutionData[], properties: string[]): IDataObject[] { + // Prepare the data to insert and copy it to be returned + let newItem: IDataObject; + return items.map((item) => { + newItem = {}; + for (const property of properties) { + if (item.json[property] === undefined) { + newItem[property] = null; + } else { + newItem[property] = deepCopy(item.json[property]); + } + } + return newItem; + }); +} + +export const prepareQueryAndReplacements = (rawQuery: string, replacements?: QueryValues) => { + if (replacements === undefined) { + return { query: rawQuery, values: [] }; + } + // in UI for replacements we use syntax identical to Postgres Query Replacement, but we need to convert it to mysql2 replacement syntax + let query: string = rawQuery; + const values: QueryValues = []; + + const regex = /\$(\d+)(?::name)?/g; + const matches = rawQuery.match(regex) || []; + + for (const match of matches) { + if (match.includes(':name')) { + const matchIndex = Number(match.replace('$', '').replace(':name', '')) - 1; + query = query.replace(match, `\`${replacements[matchIndex]}\``); + } else { + const matchIndex = Number(match.replace('$', '')) - 1; + query = query.replace(match, '?'); + values.push(replacements[matchIndex]); + } + } + + return { query, values }; +}; + +export function prepareErrorItem( + item: IDataObject, + error: IDataObject | NodeOperationError | Error, + index: number, +) { + return { + json: { message: error.message, item: { ...item }, itemIndex: index, error: { ...error } }, + pairedItem: { item: index }, + } as INodeExecutionData; +} + +export function parseMySqlError( + this: IExecuteFunctions, + error: any, + itemIndex = 0, + queries?: string[], +) { + let message: string = error.message; + const description = `sql: ${error.sql}, code: ${error.code}`; + + if ( + queries?.length && + (message || '').toLowerCase().includes('you have an error in your sql syntax') + ) { + let queryIndex = itemIndex; + const failedStatement = ((message.split("near '")[1] || '').split("' at")[0] || '').split( + ';', + )[0]; + + if (failedStatement) { + if (queryIndex === 0 && queries.length > 1) { + const failedQueryIndex = queries.findIndex((query) => query.includes(failedStatement)); + if (failedQueryIndex !== -1) { + queryIndex = failedQueryIndex; + } + } + const lines = queries[queryIndex].split('\n'); + + const failedLine = lines.findIndex((line) => line.includes(failedStatement)); + if (failedLine !== -1) { + message = `You have an error in your SQL syntax on line ${ + failedLine + 1 + } near '${failedStatement}'`; + } + } + } + + if ((error?.message as string).includes('ECONNREFUSED')) { + message = 'Connection refused'; + } + + return new NodeOperationError(this.getNode(), error as Error, { + message, + description, + itemIndex, + }); +} + +export function wrapData(data: IDataObject | IDataObject[]): INodeExecutionData[] { + if (!Array.isArray(data)) { + return [{ json: data }]; + } + return data.map((item) => ({ + json: item, + })); +} + +export function prepareOutput( + response: IDataObject[], + options: IDataObject, + statements: string[], + constructExecutionHelper: ( + inputData: INodeExecutionData[], + options: { + itemData: IPairedItemData | IPairedItemData[]; + }, + ) => NodeExecutionWithMetadata[], +) { + const returnData: INodeExecutionData[] = []; + + if (options.detailedOutput) { + response.forEach((entry, index) => { + const item = { + sql: statements[index], + data: entry, + }; + + const executionData = constructExecutionHelper(wrapData(item), { + itemData: { item: index }, + }); + + returnData.push(...executionData); + }); + } else { + response + .filter((entry) => Array.isArray(entry)) + .forEach((entry, index) => { + const executionData = constructExecutionHelper(wrapData(entry), { + itemData: { item: index }, + }); + + returnData.push(...executionData); + }); + } + + if (!returnData.length) { + returnData.push({ json: { success: true } }); + } + + return returnData; +} + +export function configureQueryRunner( + this: IExecuteFunctions, + options: IDataObject, + pool: Mysql2Pool, +) { + return async (queries: QueryWithValues[]) => { + if (queries.length === 0) { + return []; + } + + const returnData: INodeExecutionData[] = []; + const mode = (options.queryBatching as QueryMode) || BATCH_MODE.SINGLE; + + const connection = await pool.getConnection(); + + if (mode === BATCH_MODE.SINGLE) { + const formatedQueries = queries.map(({ query, values }) => connection.format(query, values)); + try { + //releasing connection after formating queries, otherwise pool.query() will fail with timeout + connection.release(); + + let singleQuery = ''; + if (formatedQueries.length > 1) { + singleQuery = formatedQueries.map((query) => query.trim().replace(/;$/, '')).join(';'); + } else { + singleQuery = formatedQueries[0]; + } + + let response: IDataObject | IDataObject[] = ( + await pool.query(singleQuery) + )[0] as unknown as IDataObject; + + if (!response) return []; + + const statements = singleQuery + .replace(/\n/g, '') + .split(';') + .filter((statement) => statement !== ''); + + if (Array.isArray(response)) { + if (statements.length === 1) response = [response]; + } else { + response = [response]; + } + + returnData.push( + ...prepareOutput(response, options, statements, this.helpers.constructExecutionMetaData), + ); + } catch (err) { + const error = parseMySqlError.call(this, err, 0, formatedQueries); + + if (!this.continueOnFail()) throw error; + returnData.push({ json: { message: error.message, error: { ...error } } }); + } + } else { + if (mode === BATCH_MODE.INDEPENDENTLY) { + let formatedQuery = ''; + for (const [index, queryWithValues] of queries.entries()) { + try { + const { query, values } = queryWithValues; + formatedQuery = connection.format(query, values); + const statements = formatedQuery.split(';').map((q) => q.trim()); + + const responses: IDataObject[] = []; + for (const statement of statements) { + if (statement === '') continue; + const response = (await connection.query(statement))[0] as unknown as IDataObject; + + responses.push(response); + } + + returnData.push( + ...prepareOutput( + responses, + options, + statements, + this.helpers.constructExecutionMetaData, + ), + ); + } catch (err) { + const error = parseMySqlError.call(this, err, index, [formatedQuery]); + + if (!this.continueOnFail()) { + connection.release(); + throw error; + } + returnData.push(prepareErrorItem(queries[index], error as Error, index)); + } + } + } + + if (mode === BATCH_MODE.TRANSACTION) { + await connection.beginTransaction(); + + let formatedQuery = ''; + for (const [index, queryWithValues] of queries.entries()) { + try { + const { query, values } = queryWithValues; + formatedQuery = connection.format(query, values); + const statements = formatedQuery.split(';').map((q) => q.trim()); + + const responses: IDataObject[] = []; + for (const statement of statements) { + if (statement === '') continue; + const response = (await connection.query(statement))[0] as unknown as IDataObject; + + responses.push(response); + } + + returnData.push( + ...prepareOutput( + responses, + options, + statements, + this.helpers.constructExecutionMetaData, + ), + ); + } catch (err) { + const error = parseMySqlError.call(this, err, index, [formatedQuery]); + + if (connection) { + await connection.rollback(); + connection.release(); + } + + if (!this.continueOnFail()) throw error; + returnData.push(prepareErrorItem(queries[index], error as Error, index)); + + // Return here because we already rolled back the transaction + return returnData; + } + } + + await connection.commit(); + } + + connection.release(); + } + + return returnData; + }; +} + +export function addWhereClauses( + node: INode, + itemIndex: number, + query: string, + clauses: WhereClause[], + replacements: QueryValues, + combineConditions?: string, +): [string, QueryValues] { + if (clauses.length === 0) return [query, replacements]; + + let combineWith = 'AND'; + + if (combineConditions === 'OR') { + combineWith = 'OR'; + } + + let whereQuery = ' WHERE'; + const values: QueryValues = []; + + clauses.forEach((clause, index) => { + if (clause.condition === 'equal') { + clause.condition = '='; + } + + if (['>', '<', '>=', '<='].includes(clause.condition)) { + const value = Number(clause.value); + + if (Number.isNaN(value)) { + throw new NodeOperationError( + node, + `Operator in entry ${index + 1} of 'Select Rows' works with numbers, but value ${ + clause.value + } is not a number`, + { + itemIndex, + }, + ); + } + + clause.value = value; + } + + let valueReplacement = ' '; + if (clause.condition !== 'IS NULL') { + valueReplacement = ' ?'; + values.push(clause.value); + } + + const operator = index === clauses.length - 1 ? '' : ` ${combineWith}`; + + whereQuery += ` \`${clause.column}\` ${clause.condition}${valueReplacement}${operator}`; + }); + + return [`${query}${whereQuery}`, replacements.concat(...values)]; +} + +export function addSortRules( + query: string, + rules: SortRule[], + replacements: QueryValues, +): [string, QueryValues] { + if (rules.length === 0) return [query, replacements]; + + let orderByQuery = ' ORDER BY'; + const values: string[] = []; + + rules.forEach((rule, index) => { + const endWith = index === rules.length - 1 ? '' : ','; + + orderByQuery += ` \`${rule.column}\` ${rule.direction}${endWith}`; + }); + + return [`${query}${orderByQuery}`, replacements.concat(...values)]; +} + +export function replaceEmptyStringsByNulls( + items: INodeExecutionData[], + replace?: boolean, +): INodeExecutionData[] { + if (!replace) return [...items]; + + const returnData: INodeExecutionData[] = items.map((item) => { + const newItem = { ...item }; + const keys = Object.keys(newItem.json); + + for (const key of keys) { + if (newItem.json[key] === '') { + newItem.json[key] = null; + } + } + + return newItem; + }); + + return returnData; +} diff --git a/packages/nodes-base/nodes/MySql/v2/methods/credentialTest.ts b/packages/nodes-base/nodes/MySql/v2/methods/credentialTest.ts new file mode 100644 index 0000000000..916d87e7af --- /dev/null +++ b/packages/nodes-base/nodes/MySql/v2/methods/credentialTest.ts @@ -0,0 +1,44 @@ +import type { + ICredentialDataDecryptedObject, + ICredentialsDecrypted, + ICredentialTestFunctions, + INodeCredentialTestResult, +} from 'n8n-workflow'; + +import { createPool } from '../transport'; + +import { Client } from 'ssh2'; + +export async function mysqlConnectionTest( + this: ICredentialTestFunctions, + credential: ICredentialsDecrypted, +): Promise { + const credentials = credential.data as ICredentialDataDecryptedObject; + + let sshClient: Client | undefined = undefined; + + if (credentials.sshTunnel) { + sshClient = new Client(); + } + const pool = await createPool(credentials, {}, sshClient); + + try { + const connection = await pool.getConnection(); + connection.release(); + } catch (error) { + return { + status: 'Error', + message: error.message, + }; + } finally { + if (sshClient) { + sshClient.end(); + } + await pool.end(); + } + + return { + status: 'OK', + message: 'Connection successful!', + }; +} diff --git a/packages/nodes-base/nodes/MySql/v2/methods/index.ts b/packages/nodes-base/nodes/MySql/v2/methods/index.ts new file mode 100644 index 0000000000..8d2d0278dd --- /dev/null +++ b/packages/nodes-base/nodes/MySql/v2/methods/index.ts @@ -0,0 +1,3 @@ +export * as credentialTest from './credentialTest'; +export * as listSearch from './listSearch'; +export * as loadOptions from './loadOptions'; diff --git a/packages/nodes-base/nodes/MySql/v2/methods/listSearch.ts b/packages/nodes-base/nodes/MySql/v2/methods/listSearch.ts new file mode 100644 index 0000000000..b5eb1526d1 --- /dev/null +++ b/packages/nodes-base/nodes/MySql/v2/methods/listSearch.ts @@ -0,0 +1,44 @@ +import type { IDataObject, ILoadOptionsFunctions, INodeListSearchResult } from 'n8n-workflow'; +import { createPool } from '../transport'; + +import { Client } from 'ssh2'; + +export async function searchTables(this: ILoadOptionsFunctions): Promise { + const credentials = await this.getCredentials('mySql'); + + const nodeOptions = this.getNodeParameter('options', 0) as IDataObject; + + let sshClient: Client | undefined = undefined; + + if (credentials.sshTunnel) { + sshClient = new Client(); + } + const pool = await createPool(credentials, nodeOptions, sshClient); + + try { + const connection = await pool.getConnection(); + + const query = 'SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE table_schema = ?'; + const values = [credentials.database as string]; + + const formatedQuery = connection.format(query, values); + + const response = (await connection.query(formatedQuery))[0]; + + connection.release(); + + const results = (response as IDataObject[]).map((table) => ({ + name: (table.table_name as string) || (table.TABLE_NAME as string), + value: (table.table_name as string) || (table.TABLE_NAME as string), + })); + + return { results }; + } catch (error) { + throw error; + } finally { + if (sshClient) { + sshClient.end(); + } + await pool.end(); + } +} diff --git a/packages/nodes-base/nodes/MySql/v2/methods/loadOptions.ts b/packages/nodes-base/nodes/MySql/v2/methods/loadOptions.ts new file mode 100644 index 0000000000..635329b6ef --- /dev/null +++ b/packages/nodes-base/nodes/MySql/v2/methods/loadOptions.ts @@ -0,0 +1,64 @@ +import type { IDataObject, ILoadOptionsFunctions, INodePropertyOptions } from 'n8n-workflow'; +import { createPool } from '../transport'; + +import { Client } from 'ssh2'; + +export async function getColumns(this: ILoadOptionsFunctions): Promise { + const credentials = await this.getCredentials('mySql'); + const nodeOptions = this.getNodeParameter('options', 0) as IDataObject; + + let sshClient: Client | undefined = undefined; + + if (credentials.sshTunnel) { + sshClient = new Client(); + } + const pool = await createPool(credentials, nodeOptions, sshClient); + + try { + const connection = await pool.getConnection(); + + const table = this.getNodeParameter('table', 0, { + extractValue: true, + }) as string; + + const columns = ( + await connection.query( + `SHOW COLUMNS FROM \`${table}\` FROM \`${credentials.database as string}\``, + ) + )[0] as IDataObject[]; + + connection.release(); + + return (columns || []).map((column: IDataObject) => ({ + name: column.Field as string, + value: column.Field as string, + // eslint-disable-next-line n8n-nodes-base/node-param-description-lowercase-first-char + description: `type: ${(column.Type as string).toUpperCase()}, nullable: ${ + column.Null as string + }`, + })); + } catch (error) { + throw error; + } finally { + if (sshClient) { + sshClient.end(); + } + await pool.end(); + } +} + +export async function getColumnsMultiOptions( + this: ILoadOptionsFunctions, +): Promise { + const returnData = await getColumns.call(this); + const returnAll = { name: '*', value: '*', description: 'All columns' }; + return [returnAll, ...returnData]; +} + +export async function getColumnsWithoutColumnToMatchOn( + this: ILoadOptionsFunctions, +): Promise { + const columnToMatchOn = this.getNodeParameter('columnToMatchOn') as string; + const returnData = await getColumns.call(this); + return returnData.filter((column) => column.value !== columnToMatchOn); +} diff --git a/packages/nodes-base/nodes/MySql/v2/transport/index.ts b/packages/nodes-base/nodes/MySql/v2/transport/index.ts new file mode 100644 index 0000000000..fd662c4b76 --- /dev/null +++ b/packages/nodes-base/nodes/MySql/v2/transport/index.ts @@ -0,0 +1,139 @@ +import type { ICredentialDataDecryptedObject, IDataObject } from 'n8n-workflow'; + +import mysql2 from 'mysql2/promise'; +import type { Client, ConnectConfig } from 'ssh2'; +import { rm, writeFile } from 'fs/promises'; + +import { file } from 'tmp-promise'; +import type { Mysql2Pool } from '../helpers/interfaces'; + +async function createSshConnectConfig(credentials: IDataObject) { + if (credentials.sshAuthenticateWith === 'password') { + return { + host: credentials.sshHost as string, + port: credentials.sshPort as number, + username: credentials.sshUser as string, + password: credentials.sshPassword as string, + } as ConnectConfig; + } else { + const { path } = await file({ prefix: 'n8n-ssh-' }); + await writeFile(path, credentials.privateKey as string); + + const options: ConnectConfig = { + host: credentials.host as string, + username: credentials.username as string, + port: credentials.port as number, + privateKey: path, + }; + + if (credentials.passphrase) { + options.passphrase = credentials.passphrase as string; + } + + return options; + } +} + +export async function createPool( + credentials: ICredentialDataDecryptedObject, + options?: IDataObject, + sshClient?: Client, +): Promise { + if (credentials === undefined) { + throw new Error('Credentials not selected, select or add new credentials'); + } + const { + ssl, + caCertificate, + clientCertificate, + clientPrivateKey, + sshTunnel, + sshHost, + sshUser, + sshPassword, + sshPort, + sshMysqlPort, + privateKey, + passphrase, + sshAuthenticateWith, + ...baseCredentials + } = credentials; + + if (ssl) { + baseCredentials.ssl = {}; + + if (caCertificate) { + baseCredentials.ssl.ca = caCertificate; + } + + if (clientCertificate || clientPrivateKey) { + baseCredentials.ssl.cert = clientCertificate; + baseCredentials.ssl.key = clientPrivateKey; + } + } + + const connectionOptions: mysql2.ConnectionOptions = { + ...baseCredentials, + multipleStatements: true, + supportBigNumbers: true, + }; + + if (options?.connectionLimit) { + connectionOptions.connectionLimit = options.connectionLimit as number; + } + + if (options?.connectTimeout) { + connectionOptions.connectTimeout = options.connectTimeout as number; + } + + if (options?.largeNumbersOutput === 'text') { + connectionOptions.bigNumberStrings = true; + } + + if (!sshTunnel) { + return mysql2.createPool(connectionOptions); + } else { + if (!sshClient) { + throw new Error('SSH Tunnel is enabled but no SSH Client was provided'); + } + + const tunnelConfig = await createSshConnectConfig(credentials); + + const forwardConfig = { + srcHost: '127.0.0.1', + srcPort: sshMysqlPort as number, + dstHost: credentials.host as string, + dstPort: credentials.port as number, + }; + + if (sshAuthenticateWith === 'privateKey') { + sshClient.on('end', async () => { + await rm(tunnelConfig.privateKey as string); + }); + } + + const poolSetup = new Promise((resolve, reject) => { + sshClient + .on('ready', () => { + sshClient.forwardOut( + forwardConfig.srcHost, + forwardConfig.srcPort, + forwardConfig.dstHost, + forwardConfig.dstPort, + (err, stream) => { + if (err) reject(err); + const updatedDbServer = { + ...connectionOptions, + stream, + }; + const connection = mysql2.createPool(updatedDbServer); + resolve(connection); + }, + ); + }) + .connect(tunnelConfig); + }); + + return poolSetup; + } +} diff --git a/packages/nodes-base/test/nodes/Helpers.ts b/packages/nodes-base/test/nodes/Helpers.ts index 09a4e9e44c..b1d0603ad6 100644 --- a/packages/nodes-base/test/nodes/Helpers.ts +++ b/packages/nodes-base/test/nodes/Helpers.ts @@ -1,11 +1,13 @@ import { readFileSync, readdirSync, mkdtempSync } from 'fs'; -import { BinaryDataManager, Credentials } from 'n8n-core'; +import { BinaryDataManager, Credentials, constructExecutionMetaData } from 'n8n-core'; import { ICredentialDataDecryptedObject, ICredentialsHelper, IDataObject, IDeferredPromise, + IExecuteFunctions, IExecuteWorkflowInfo, + IGetNodeParameterOptions, IHttpRequestHelper, IHttpRequestOptions, ILogger, @@ -29,6 +31,7 @@ import { WorkflowTestData } from './types'; import path from 'path'; import { tmpdir } from 'os'; import { isEmpty } from 'lodash'; +import { get } from 'lodash'; import { FAKE_CREDENTIALS_DATA } from './FakeCredentialsMap'; @@ -328,3 +331,31 @@ export const getWorkflowFilenames = (dirname: string) => { return workflows; }; + +export const createMockExecuteFunction = ( + nodeParameters: IDataObject, + nodeMock: INode, + continueBool = false, +) => { + const fakeExecuteFunction = { + getNodeParameter( + parameterName: string, + _itemIndex: number, + fallbackValue?: IDataObject | undefined, + options?: IGetNodeParameterOptions | undefined, + ) { + const parameter = options?.extractValue ? `${parameterName}.value` : parameterName; + return get(nodeParameters, parameter, fallbackValue); + }, + getNode() { + return nodeMock; + }, + continueOnFail() { + return continueBool; + }, + helpers: { + constructExecutionMetaData, + }, + } as unknown as IExecuteFunctions; + return fakeExecuteFunction; +};