From 7ecd5e59eca01ca2b1a01e0a3e3871bd5d322eea Mon Sep 17 00:00:00 2001 From: Michael Kret <88898367+michael-radency@users.noreply.github.com> Date: Tue, 21 Feb 2023 15:42:00 +0200 Subject: [PATCH] fix(Compare Datasets Node): UI tweaks and fixes --- .../CompareDatasets/CompareDatasets.node.ts | 26 +++- .../nodes/CompareDatasets/GenericFunctions.ts | 132 +++++------------- .../nodes/Merge/v2/GenericFunctions.ts | 93 +----------- .../nodes-base/test/utils/utilities.test.ts | 31 ++++ packages/nodes-base/utils/utilities.ts | 105 +++++++++++++- 5 files changed, 195 insertions(+), 192 deletions(-) create mode 100644 packages/nodes-base/test/utils/utilities.test.ts diff --git a/packages/nodes-base/nodes/CompareDatasets/CompareDatasets.node.ts b/packages/nodes-base/nodes/CompareDatasets/CompareDatasets.node.ts index 4acc603164..8cccc59d34 100644 --- a/packages/nodes-base/nodes/CompareDatasets/CompareDatasets.node.ts +++ b/packages/nodes-base/nodes/CompareDatasets/CompareDatasets.node.ts @@ -13,7 +13,7 @@ export class CompareDatasets implements INodeType { name: 'compareDatasets', icon: 'file:compare.svg', group: ['transform'], - version: 1, + version: [1, 2], description: 'Compare two inputs for changes', defaults: { name: 'Compare Datasets' }, // eslint-disable-next-line n8n-nodes-base/node-class-description-inputs-wrong-regular-node @@ -94,6 +94,19 @@ export class CompareDatasets implements INodeType { }, ], }, + { + displayName: 'Fuzzy Compare', + name: 'fuzzyCompare', + type: 'boolean', + default: false, + description: + "Whether to tolerate small type differences when comparing fields. E.g. the number 3 and the string '3' are treated as the same.", + displayOptions: { + show: { + '@version': [2], + }, + }, + }, { displayName: 'Prefer', name: 'preferWhenMix', @@ -155,6 +168,11 @@ export class CompareDatasets implements INodeType { default: false, description: "Whether to tolerate small type differences when comparing fields. E.g. the number 3 and the string '3' are treated as the same.", + displayOptions: { + hide: { + '@version': [2], + }, + }, }, { displayName: 'Disable Dot Notation', @@ -194,6 +212,12 @@ export class CompareDatasets implements INodeType { const options = this.getNodeParameter('options', 0, {}); + options.nodeVersion = this.getNode().typeVersion; + + if (options.nodeVersion === 2) { + options.fuzzyCompare = this.getNodeParameter('fuzzyCompare', 0, false) as boolean; + } + const input1 = checkInput( this.getInputData(0), matchFields.map((pair) => pair.field1), diff --git a/packages/nodes-base/nodes/CompareDatasets/GenericFunctions.ts b/packages/nodes-base/nodes/CompareDatasets/GenericFunctions.ts index dad2818323..9e6116c818 100644 --- a/packages/nodes-base/nodes/CompareDatasets/GenericFunctions.ts +++ b/packages/nodes-base/nodes/CompareDatasets/GenericFunctions.ts @@ -1,6 +1,7 @@ import type { IDataObject, INodeExecutionData } from 'n8n-workflow'; -import { jsonParse } from 'n8n-workflow'; -import { difference, get, intersection, isEmpty, isEqual, isNull, omit, set, union } from 'lodash'; + +import { difference, get, intersection, isEmpty, omit, set, union } from 'lodash'; +import { fuzzyCompare } from '../../utils/utilities'; type PairToMatch = { field1: string; @@ -14,11 +15,18 @@ type EntryMatches = { type CompareFunction = (a: T, b: U) => boolean; +const processNullishValueFunction = (version: number) => { + if (version === 2) { + return (value: T) => (value === undefined ? null : value); + } + return (value: T) => value || null; +}; + function compareItems( item1: INodeExecutionData, item2: INodeExecutionData, fieldsToMatch: PairToMatch[], - resolve: string, + options: IDataObject, skipFields: string[], isEntriesEqual: CompareFunction, ) { @@ -46,20 +54,28 @@ function compareItems( const skipped: IDataObject = {}; differentKeys.forEach((key) => { - switch (resolve) { + const processNullishValue = processNullishValueFunction(options.nodeVersion as number); + + switch (options.resolve) { case 'preferInput1': - different[key] = item1.json[key] || null; + different[key] = processNullishValue(item1.json[key]); break; case 'preferInput2': - different[key] = item2.json[key] || null; + different[key] = processNullishValue(item2.json[key]); break; default: - const input1 = item1.json[key] || null; - const input2 = item2.json[key] || null; + const input1 = processNullishValue(item1.json[key]); + const input2 = processNullishValue(item2.json[key]); + + let [firstInputName, secondInputName] = ['input1', 'input2']; + if (options.nodeVersion === 2) { + [firstInputName, secondInputName] = ['inputA', 'inputB']; + } + if (skipFields.includes(key)) { - skipped[key] = { input1, input2 }; + skipped[key] = { [firstInputName]: input1, [secondInputName]: input2 }; } else { - different[key] = { input1, input2 }; + different[key] = { [firstInputName]: input1, [secondInputName]: input2 }; } } }); @@ -160,93 +176,6 @@ function findFirstMatch( return [{ entry: data[index], index }]; } -const parseStringAndCompareToObject = (str: string, arr: IDataObject) => { - try { - const parsedArray = jsonParse(str); - return isEqual(parsedArray, arr); - } catch (error) { - return false; - } -}; - -function isFalsy(value: T) { - if (isNull(value)) return true; - if (typeof value === 'string' && value === '') return true; - if (Array.isArray(value) && value.length === 0) return true; - return false; -} - -const fuzzyCompare = - (options: IDataObject) => - (item1: T, item2: U) => { - //Fuzzy compare is disabled, so we do strict comparison - if (!options.fuzzyCompare) return isEqual(item1, item2); - - //Both types are the same, so we do strict comparison - if (!isNull(item1) && !isNull(item2) && typeof item1 === typeof item2) { - return isEqual(item1, item2); - } - - //Null, empty strings, empty arrays all treated as the same - if (isFalsy(item1) && isFalsy(item2)) return true; - - //When a field is missing in one branch and isFalsy() in another, treat them as matching - if (isFalsy(item1) && item2 === undefined) return true; - if (item1 === undefined && isFalsy(item2)) return true; - - //Compare numbers and strings representing that number - if (typeof item1 === 'number' && typeof item2 === 'string') { - return item1.toString() === item2; - } - - if (typeof item1 === 'string' && typeof item2 === 'number') { - return item1 === item2.toString(); - } - - //Compare objects/arrays and their stringified version - if (!isNull(item1) && typeof item1 === 'object' && typeof item2 === 'string') { - return parseStringAndCompareToObject(item2, item1 as IDataObject); - } - - if (!isNull(item2) && typeof item1 === 'string' && typeof item2 === 'object') { - return parseStringAndCompareToObject(item1, item2 as IDataObject); - } - - //Compare booleans and strings representing the boolean (’true’, ‘True’, ‘TRUE’) - if (typeof item1 === 'boolean' && typeof item2 === 'string') { - if (item1 === true && item2.toLocaleLowerCase() === 'true') return true; - if (item1 === false && item2.toLocaleLowerCase() === 'false') return true; - } - - if (typeof item2 === 'boolean' && typeof item1 === 'string') { - if (item2 === true && item1.toLocaleLowerCase() === 'true') return true; - if (item2 === false && item1.toLocaleLowerCase() === 'false') return true; - } - - //Compare booleans and the numbers/string 0 and 1 - if (typeof item1 === 'boolean' && typeof item2 === 'number') { - if (item1 === true && item2 === 1) return true; - if (item1 === false && item2 === 0) return true; - } - - if (typeof item2 === 'boolean' && typeof item1 === 'number') { - if (item2 === true && item1 === 1) return true; - if (item2 === false && item1 === 0) return true; - } - - if (typeof item1 === 'boolean' && typeof item2 === 'string') { - if (item1 === true && item2 === '1') return true; - if (item1 === false && item2 === '0') return true; - } - - if (typeof item2 === 'boolean' && typeof item1 === 'string') { - if (item2 === true && item1 === '1') return true; - if (item2 === false && item1 === '0') return true; - } - - return isEqual(item1, item2); - }; - export function findMatches( input1: INodeExecutionData[], input2: INodeExecutionData[], @@ -256,7 +185,12 @@ export function findMatches( const data1 = [...input1]; const data2 = [...input2]; - const isEntriesEqual = fuzzyCompare(options); + let compareVersion = 1; + if (options.nodeVersion === 2) { + compareVersion = 2; + } + + const isEntriesEqual = fuzzyCompare(options.fuzzyCompare as boolean, compareVersion); const disableDotNotation = (options.disableDotNotation as boolean) || false; const multipleMatches = (options.multipleMatches as string) || 'first'; const skipFields = ((options.skipFields as string) || '').split(',').map((field) => field.trim()); @@ -370,7 +304,7 @@ export function findMatches( entryMatches.entry, match, fieldsToMatch, - options.resolve as string, + options, skipFields, isEntriesEqual, ), diff --git a/packages/nodes-base/nodes/Merge/v2/GenericFunctions.ts b/packages/nodes-base/nodes/Merge/v2/GenericFunctions.ts index 7ceb7afc89..7a1a4a14bc 100644 --- a/packages/nodes-base/nodes/Merge/v2/GenericFunctions.ts +++ b/packages/nodes-base/nodes/Merge/v2/GenericFunctions.ts @@ -5,9 +5,9 @@ import type { INodeExecutionData, IPairedItemData, } from 'n8n-workflow'; -import { jsonParse } from 'n8n-workflow'; -import { assign, assignWith, get, isEqual, isNull, merge, mergeWith } from 'lodash'; +import { assign, assignWith, get, merge, mergeWith } from 'lodash'; +import { fuzzyCompare } from '../../../utils/utilities'; type PairToMatch = { field1: string; @@ -122,93 +122,6 @@ function findFirstMatch( return [{ entry: data[index], index }]; } -const parseStringAndCompareToObject = (str: string, arr: IDataObject) => { - try { - const parsedArray = jsonParse(str); - return isEqual(parsedArray, arr); - } catch (error) { - return false; - } -}; - -function isFalsy(value: T) { - if (isNull(value)) return true; - if (typeof value === 'string' && value === '') return true; - if (Array.isArray(value) && value.length === 0) return true; - return false; -} - -const fuzzyCompare = - (options: IDataObject) => - (item1: T, item2: U) => { - //Fuzzy compare is disabled, so we do strict comparison - if (!options.fuzzyCompare) return isEqual(item1, item2); - - //Both types are the same, so we do strict comparison - if (!isNull(item1) && !isNull(item2) && typeof item1 === typeof item2) { - return isEqual(item1, item2); - } - - //Null, empty strings, empty arrays all treated as the same - if (isFalsy(item1) && isFalsy(item2)) return true; - - //When a field is missing in one branch and isFalsy() in another, treat them as matching - if (isFalsy(item1) && item2 === undefined) return true; - if (item1 === undefined && isFalsy(item2)) return true; - - //Compare numbers and strings representing that number - if (typeof item1 === 'number' && typeof item2 === 'string') { - return item1.toString() === item2; - } - - if (typeof item1 === 'string' && typeof item2 === 'number') { - return item1 === item2.toString(); - } - - //Compare objects/arrays and their stringified version - if (!isNull(item1) && typeof item1 === 'object' && typeof item2 === 'string') { - return parseStringAndCompareToObject(item2, item1 as IDataObject); - } - - if (!isNull(item2) && typeof item1 === 'string' && typeof item2 === 'object') { - return parseStringAndCompareToObject(item1, item2 as IDataObject); - } - - //Compare booleans and strings representing the boolean (’true’, ‘True’, ‘TRUE’) - if (typeof item1 === 'boolean' && typeof item2 === 'string') { - if (item1 === true && item2.toLocaleLowerCase() === 'true') return true; - if (item1 === false && item2.toLocaleLowerCase() === 'false') return true; - } - - if (typeof item2 === 'boolean' && typeof item1 === 'string') { - if (item2 === true && item1.toLocaleLowerCase() === 'true') return true; - if (item2 === false && item1.toLocaleLowerCase() === 'false') return true; - } - - //Compare booleans and the numbers/string 0 and 1 - if (typeof item1 === 'boolean' && typeof item2 === 'number') { - if (item1 === true && item2 === 1) return true; - if (item1 === false && item2 === 0) return true; - } - - if (typeof item2 === 'boolean' && typeof item1 === 'number') { - if (item2 === true && item1 === 1) return true; - if (item2 === false && item1 === 0) return true; - } - - if (typeof item1 === 'boolean' && typeof item2 === 'string') { - if (item1 === true && item2 === '1') return true; - if (item1 === false && item2 === '0') return true; - } - - if (typeof item2 === 'boolean' && typeof item1 === 'string') { - if (item2 === true && item1 === '1') return true; - if (item2 === false && item1 === '0') return true; - } - - return isEqual(item1, item2); - }; - export function findMatches( input1: INodeExecutionData[], input2: INodeExecutionData[], @@ -222,7 +135,7 @@ export function findMatches( [data1, data2] = [data2, data1]; } - const isEntriesEqual = fuzzyCompare(options); + const isEntriesEqual = fuzzyCompare(options.fuzzyCompare as boolean); const disableDotNotation = options.disableDotNotation || false; const multipleMatches = (options.multipleMatches as string) || 'all'; diff --git a/packages/nodes-base/test/utils/utilities.test.ts b/packages/nodes-base/test/utils/utilities.test.ts new file mode 100644 index 0000000000..78fa5631bc --- /dev/null +++ b/packages/nodes-base/test/utils/utilities.test.ts @@ -0,0 +1,31 @@ +import { fuzzyCompare } from '../../utils/utilities'; + +//most test cases for fuzzyCompare are done in Compare Datasets node tests +describe('Test fuzzyCompare', () => { + it('should do strict comparison', () => { + const compareFunction = fuzzyCompare(false); + + expect(compareFunction(1, '1')).toEqual(false); + }); + + it('should do fuzzy comparison', () => { + const compareFunction = fuzzyCompare(true); + + expect(compareFunction(1, '1')).toEqual(true); + }); + + it('should treat null, 0 and "0" as equal', () => { + const compareFunction = fuzzyCompare(true, 2); + + expect(compareFunction(null, null)).toEqual(true); + expect(compareFunction(null, 0)).toEqual(true); + expect(compareFunction(null, '0')).toEqual(true); + }); + + it('should not treat null, 0 and "0" as equal', () => { + const compareFunction = fuzzyCompare(true); + + expect(compareFunction(null, 0)).toEqual(false); + expect(compareFunction(null, '0')).toEqual(false); + }); +}); diff --git a/packages/nodes-base/utils/utilities.ts b/packages/nodes-base/utils/utilities.ts index 6cf759bbe4..0ec385078f 100644 --- a/packages/nodes-base/utils/utilities.ts +++ b/packages/nodes-base/utils/utilities.ts @@ -1,6 +1,7 @@ -import { IDisplayOptions, INodeProperties } from 'n8n-workflow'; +import type { IDataObject, IDisplayOptions, INodeProperties } from 'n8n-workflow'; +import { jsonParse } from 'n8n-workflow'; -import { merge } from 'lodash'; +import { isEqual, isNull, merge } from 'lodash'; /** * Creates an array of elements split into groups the length of `size`. @@ -71,3 +72,103 @@ export function updateDisplayOptions( }; }); } + +function isFalsy(value: T) { + if (isNull(value)) return true; + if (typeof value === 'string' && value === '') return true; + if (Array.isArray(value) && value.length === 0) return true; + return false; +} + +const parseStringAndCompareToObject = (str: string, arr: IDataObject) => { + try { + const parsedArray = jsonParse(str); + return isEqual(parsedArray, arr); + } catch (error) { + return false; + } +}; + +export const fuzzyCompare = (useFuzzyCompare: boolean, compareVersion = 1) => { + if (!useFuzzyCompare) { + //Fuzzy compare is false we do strict comparison + return (item1: T, item2: U) => isEqual(item1, item2); + } + + return (item1: T, item2: U) => { + //Both types are the same, so we do strict comparison + if (!isNull(item1) && !isNull(item2) && typeof item1 === typeof item2) { + return isEqual(item1, item2); + } + + if (compareVersion >= 2) { + //Null, 0 and "0" treated as equal + if (isNull(item1) && (isNull(item2) || item2 === 0 || item2 === '0')) { + return true; + } + + if (isNull(item2) && (isNull(item1) || item1 === 0 || item1 === '0')) { + return true; + } + } + + //Null, empty strings, empty arrays all treated as the same + if (isFalsy(item1) && isFalsy(item2)) return true; + + //When a field is missing in one branch and isFalsy() in another, treat them as matching + if (isFalsy(item1) && item2 === undefined) return true; + if (item1 === undefined && isFalsy(item2)) return true; + + //Compare numbers and strings representing that number + if (typeof item1 === 'number' && typeof item2 === 'string') { + return item1.toString() === item2; + } + + if (typeof item1 === 'string' && typeof item2 === 'number') { + return item1 === item2.toString(); + } + + //Compare objects/arrays and their stringified version + if (!isNull(item1) && typeof item1 === 'object' && typeof item2 === 'string') { + return parseStringAndCompareToObject(item2, item1 as IDataObject); + } + + if (!isNull(item2) && typeof item1 === 'string' && typeof item2 === 'object') { + return parseStringAndCompareToObject(item1, item2 as IDataObject); + } + + //Compare booleans and strings representing the boolean (’true’, ‘True’, ‘TRUE’) + if (typeof item1 === 'boolean' && typeof item2 === 'string') { + if (item1 === true && item2.toLocaleLowerCase() === 'true') return true; + if (item1 === false && item2.toLocaleLowerCase() === 'false') return true; + } + + if (typeof item2 === 'boolean' && typeof item1 === 'string') { + if (item2 === true && item1.toLocaleLowerCase() === 'true') return true; + if (item2 === false && item1.toLocaleLowerCase() === 'false') return true; + } + + //Compare booleans and the numbers/string 0 and 1 + if (typeof item1 === 'boolean' && typeof item2 === 'number') { + if (item1 === true && item2 === 1) return true; + if (item1 === false && item2 === 0) return true; + } + + if (typeof item2 === 'boolean' && typeof item1 === 'number') { + if (item2 === true && item1 === 1) return true; + if (item2 === false && item1 === 0) return true; + } + + if (typeof item1 === 'boolean' && typeof item2 === 'string') { + if (item1 === true && item2 === '1') return true; + if (item1 === false && item2 === '0') return true; + } + + if (typeof item2 === 'boolean' && typeof item1 === 'string') { + if (item2 === true && item1 === '1') return true; + if (item2 === false && item1 === '0') return true; + } + + return isEqual(item1, item2); + }; +};