From 9615253155bec6b57344d790681c8a598fbc21c0 Mon Sep 17 00:00:00 2001 From: Michael Kret <88898367+michael-radency@users.noreply.github.com> Date: Wed, 4 Jan 2023 14:37:54 +0200 Subject: [PATCH] feat(Compare Datasets Node): Fuzzy compare option --- .../CompareDatasets/CompareDatasets.node.ts | 15 ++ .../nodes/CompareDatasets/GenericFunctions.ts | 131 ++++++++++++++++-- .../nodes/Merge/v2/GenericFunctions.ts | 104 +++++++++++++- .../nodes/Merge/v2/OptionsDescription.ts | 8 ++ 4 files changed, 244 insertions(+), 14 deletions(-) diff --git a/packages/nodes-base/nodes/CompareDatasets/CompareDatasets.node.ts b/packages/nodes-base/nodes/CompareDatasets/CompareDatasets.node.ts index b9f277bfc9..3bb53f0bd5 100644 --- a/packages/nodes-base/nodes/CompareDatasets/CompareDatasets.node.ts +++ b/packages/nodes-base/nodes/CompareDatasets/CompareDatasets.node.ts @@ -18,6 +18,13 @@ export class CompareDatasets implements INodeType { outputs: ['main', 'main', 'main', 'main'], outputNames: ['In A only', 'Same', 'Different', 'In B only'], properties: [ + { + displayName: + 'Items from different branches are paired together when the fields below match. If paired, the rest of the fields are compared to determine whether the items are the same or different', + name: 'infoBox', + type: 'notice', + default: '', + }, { displayName: 'Fields to Match', name: 'mergeByFields', @@ -132,6 +139,14 @@ export class CompareDatasets implements INodeType { description: "Fields that shouldn't be included when checking whether two items are the same", }, + { + displayName: 'Fuzzy Compare', + name: 'fuzzyCompare', + type: 'boolean', + default: false, + description: + "Whether to tolerate small type differences when comparing fields. E.g. the number 3 and the string '3' are treated as the same.", + }, { displayName: 'Disable Dot Notation', name: 'disableDotNotation', diff --git a/packages/nodes-base/nodes/CompareDatasets/GenericFunctions.ts b/packages/nodes-base/nodes/CompareDatasets/GenericFunctions.ts index 2324dc156e..959cf0f790 100644 --- a/packages/nodes-base/nodes/CompareDatasets/GenericFunctions.ts +++ b/packages/nodes-base/nodes/CompareDatasets/GenericFunctions.ts @@ -1,5 +1,5 @@ -import { IDataObject, INodeExecutionData } from 'n8n-workflow'; -import { difference, get, intersection, isEmpty, isEqual, omit, set, union } from 'lodash'; +import { IDataObject, INodeExecutionData, jsonParse } from 'n8n-workflow'; +import { difference, get, intersection, isEmpty, isEqual, isNull, omit, set, union } from 'lodash'; type PairToMatch = { field1: string; @@ -11,12 +11,15 @@ type EntryMatches = { matches: INodeExecutionData[]; }; +type CompareFunction = (a: T, b: U) => boolean; + function compareItems( item1: INodeExecutionData, item2: INodeExecutionData, fieldsToMatch: PairToMatch[], resolve: string, skipFields: string[], + isEntriesEqual: CompareFunction, ) { const keys = {} as IDataObject; fieldsToMatch.forEach((field) => { @@ -28,7 +31,7 @@ function compareItems( const intersectionKeys = intersection(keys1, keys2); const same = intersectionKeys.reduce((acc, key) => { - if (isEqual(item1.json[key], item2.json[key])) { + if (isEntriesEqual(item1.json[key], item2.json[key])) { acc[key] = item1.json[key]; } return acc; @@ -98,6 +101,7 @@ function findAllMatches( data: INodeExecutionData[], lookup: IDataObject, disableDotNotation: boolean, + isEntriesEqual: CompareFunction, ) { return data.reduce((acc, entry2, i) => { if (entry2 === undefined) return acc; @@ -112,7 +116,7 @@ function findAllMatches( entry2FieldValue = get(entry2.json, key); } - if (!isEqual(excpectedValue, entry2FieldValue)) { + if (!isEntriesEqual(excpectedValue, entry2FieldValue)) { return acc; } } @@ -128,6 +132,7 @@ function findFirstMatch( data: INodeExecutionData[], lookup: IDataObject, disableDotNotation: boolean, + isEntriesEqual: CompareFunction, ) { const index = data.findIndex((entry2) => { if (entry2 === undefined) return false; @@ -142,7 +147,7 @@ function findFirstMatch( entry2FieldValue = get(entry2.json, key); } - if (!isEqual(excpectedValue, entry2FieldValue)) { + if (!isEntriesEqual(excpectedValue, entry2FieldValue)) { return false; } } @@ -163,6 +168,7 @@ export function findMatches( const data1 = [...input1]; const data2 = [...input2]; + const isEntriesEqual = fuzzyCompare(options); const disableDotNotation = (options.disableDotNotation as boolean) || false; const multipleMatches = (options.multipleMatches as string) || 'first'; const skipFields = ((options.skipFields as string) || '').split(',').map((field) => field.trim()); @@ -197,8 +203,8 @@ export function findMatches( const foundedMatches = multipleMatches === 'all' - ? findAllMatches(data2, lookup, disableDotNotation) - : findFirstMatch(data2, lookup, disableDotNotation); + ? findAllMatches(data2, lookup, disableDotNotation, isEntriesEqual) + : findFirstMatch(data2, lookup, disableDotNotation, isEntriesEqual); const matches = foundedMatches.map((match) => match.entry) as INodeExecutionData[]; foundedMatches.map((match) => matchedInInput2.add(match.index as number)); @@ -230,8 +236,27 @@ export function findMatches( entryFromInput1 = omit(entryFromInput1, skipFields); entryFromInput2 = omit(entryFromInput2, skipFields); } - if (isEqual(entryFromInput1, entryFromInput2)) { - if (!entryCopy) entryCopy = match; + + let isItemsEqual = true; + if (options.fuzzyCompare) { + for (const key of Object.keys(entryFromInput1)) { + if (!isEntriesEqual(entryFromInput1[key], entryFromInput2[key])) { + isItemsEqual = false; + break; + } + } + } else { + isItemsEqual = isEntriesEqual(entryFromInput1, entryFromInput2); + } + + if (isItemsEqual) { + if (!entryCopy) { + if (options.fuzzyCompare && options.resolve === 'preferInput2') { + entryCopy = match; + } else { + entryCopy = entryMatches.entry; + } + } } else { switch (options.resolve) { case 'preferInput1': @@ -259,6 +284,7 @@ export function findMatches( fieldsToMatch, options.resolve as string, skipFields, + isEntriesEqual, ), ); } @@ -315,3 +341,90 @@ export function checkInput( } return input; } + +const fuzzyCompare = + (options: IDataObject) => + (item1: T, item2: U) => { + //Fuzzy compare is disabled, so we do strict comparison + if (!options.fuzzyCompare) return isEqual(item1, item2); + + //Both types are the same, so we do strict comparison + if (!isNull(item1) && !isNull(item2) && typeof item1 === typeof item2) { + return isEqual(item1, item2); + } + + //Null, empty strings, empty arrays all treated as the same + if (isFalsy(item1) && isFalsy(item2)) return true; + + //When a field is missing in one branch and isFalsy() in another, treat them as matching + if (isFalsy(item1) && item2 === undefined) return true; + if (item1 === undefined && isFalsy(item2)) return true; + + //Compare numbers and strings representing that number + if (typeof item1 === 'number' && typeof item2 === 'string') { + return item1.toString() === item2; + } + + if (typeof item1 === 'string' && typeof item2 === 'number') { + return item1 === item2.toString(); + } + + //Compare objects/arrays and their stringified version + if (!isNull(item1) && typeof item1 === 'object' && typeof item2 === 'string') { + return parseStringAndCompareToObject(item2, item1 as IDataObject); + } + + if (!isNull(item2) && typeof item1 === 'string' && typeof item2 === 'object') { + return parseStringAndCompareToObject(item1, item2 as IDataObject); + } + + //Compare booleans and strings representing the boolean (’true’, ‘True’, ‘TRUE’) + if (typeof item1 === 'boolean' && typeof item2 === 'string') { + if (item1 === true && item2.toLocaleLowerCase() === 'true') return true; + if (item1 === false && item2.toLocaleLowerCase() === 'false') return true; + } + + if (typeof item2 === 'boolean' && typeof item1 === 'string') { + if (item2 === true && item1.toLocaleLowerCase() === 'true') return true; + if (item2 === false && item1.toLocaleLowerCase() === 'false') return true; + } + + //Compare booleans and the numbers/string 0 and 1 + if (typeof item1 === 'boolean' && typeof item2 === 'number') { + if (item1 === true && item2 === 1) return true; + if (item1 === false && item2 === 0) return true; + } + + if (typeof item2 === 'boolean' && typeof item1 === 'number') { + if (item2 === true && item1 === 1) return true; + if (item2 === false && item1 === 0) return true; + } + + if (typeof item1 === 'boolean' && typeof item2 === 'string') { + if (item1 === true && item2 === '1') return true; + if (item1 === false && item2 === '0') return true; + } + + if (typeof item2 === 'boolean' && typeof item1 === 'string') { + if (item2 === true && item1 === '1') return true; + if (item2 === false && item1 === '0') return true; + } + + return isEqual(item1, item2); + }; + +const parseStringAndCompareToObject = (str: string, arr: IDataObject) => { + try { + const parsedArray = jsonParse(str); + return isEqual(parsedArray, arr); + } catch (error) { + return false; + } +}; + +function isFalsy(value: T) { + if (isNull(value)) return true; + if (typeof value === 'string' && value === '') return true; + if (Array.isArray(value) && value.length === 0) return true; + return false; +} diff --git a/packages/nodes-base/nodes/Merge/v2/GenericFunctions.ts b/packages/nodes-base/nodes/Merge/v2/GenericFunctions.ts index 44a9598549..37390aed36 100644 --- a/packages/nodes-base/nodes/Merge/v2/GenericFunctions.ts +++ b/packages/nodes-base/nodes/Merge/v2/GenericFunctions.ts @@ -4,9 +4,10 @@ import { IDataObject, INodeExecutionData, IPairedItemData, + jsonParse, } from 'n8n-workflow'; -import { assign, assignWith, get, isEqual, merge, mergeWith } from 'lodash'; +import { assign, assignWith, get, isEqual, isNull, merge, mergeWith } from 'lodash'; type PairToMatch = { field1: string; @@ -18,6 +19,7 @@ export type MatchFieldsOptions = { outputDataFrom: MatchFieldsOutput; multipleMatches: MultipleMatches; disableDotNotation: boolean; + fuzzyCompare?: boolean; }; export type ClashResolveOptions = { @@ -46,6 +48,8 @@ type EntryMatches = { matches: INodeExecutionData[]; }; +type CompareFunction = (a: T, b: U) => boolean; + export function addSuffixToEntriesKeys(data: INodeExecutionData[], suffix: string) { return data.map((entry) => { const json: IDataObject = {}; @@ -60,6 +64,7 @@ function findAllMatches( data: INodeExecutionData[], lookup: IDataObject, disableDotNotation: boolean, + isEntriesEqual: CompareFunction, ) { return data.reduce((acc, entry2, i) => { if (entry2 === undefined) return acc; @@ -74,7 +79,7 @@ function findAllMatches( entry2FieldValue = get(entry2.json, key); } - if (!isEqual(excpectedValue, entry2FieldValue)) { + if (!isEntriesEqual(excpectedValue, entry2FieldValue)) { return acc; } } @@ -90,6 +95,7 @@ function findFirstMatch( data: INodeExecutionData[], lookup: IDataObject, disableDotNotation: boolean, + isEntriesEqual: CompareFunction, ) { const index = data.findIndex((entry2) => { if (entry2 === undefined) return false; @@ -104,7 +110,7 @@ function findFirstMatch( entry2FieldValue = get(entry2.json, key); } - if (!isEqual(excpectedValue, entry2FieldValue)) { + if (!isEntriesEqual(excpectedValue, entry2FieldValue)) { return false; } } @@ -129,6 +135,7 @@ export function findMatches( [data1, data2] = [data2, data1]; } + const isEntriesEqual = fuzzyCompare(options); const disableDotNotation = options.disableDotNotation || false; const multipleMatches = (options.multipleMatches as string) || 'all'; @@ -163,8 +170,8 @@ export function findMatches( const foundedMatches = multipleMatches === 'all' - ? findAllMatches(data2, lookup, disableDotNotation) - : findFirstMatch(data2, lookup, disableDotNotation); + ? findAllMatches(data2, lookup, disableDotNotation, isEntriesEqual) + : findFirstMatch(data2, lookup, disableDotNotation, isEntriesEqual); const matches = foundedMatches.map((match) => match.entry) as INodeExecutionData[]; foundedMatches.map((match) => matchedInInput2.add(match.index as number)); @@ -367,3 +374,90 @@ export function addSourceField(data: INodeExecutionData[], sourceField: string) }; }); } + +const fuzzyCompare = + (options: IDataObject) => + (item1: T, item2: U) => { + //Fuzzy compare is disabled, so we do strict comparison + if (!options.fuzzyCompare) return isEqual(item1, item2); + + //Both types are the same, so we do strict comparison + if (!isNull(item1) && !isNull(item2) && typeof item1 === typeof item2) { + return isEqual(item1, item2); + } + + //Null, empty strings, empty arrays all treated as the same + if (isFalsy(item1) && isFalsy(item2)) return true; + + //When a field is missing in one branch and isFalsy() in another, treat them as matching + if (isFalsy(item1) && item2 === undefined) return true; + if (item1 === undefined && isFalsy(item2)) return true; + + //Compare numbers and strings representing that number + if (typeof item1 === 'number' && typeof item2 === 'string') { + return item1.toString() === item2; + } + + if (typeof item1 === 'string' && typeof item2 === 'number') { + return item1 === item2.toString(); + } + + //Compare objects/arrays and their stringified version + if (!isNull(item1) && typeof item1 === 'object' && typeof item2 === 'string') { + return parseStringAndCompareToObject(item2, item1 as IDataObject); + } + + if (!isNull(item2) && typeof item1 === 'string' && typeof item2 === 'object') { + return parseStringAndCompareToObject(item1, item2 as IDataObject); + } + + //Compare booleans and strings representing the boolean (’true’, ‘True’, ‘TRUE’) + if (typeof item1 === 'boolean' && typeof item2 === 'string') { + if (item1 === true && item2.toLocaleLowerCase() === 'true') return true; + if (item1 === false && item2.toLocaleLowerCase() === 'false') return true; + } + + if (typeof item2 === 'boolean' && typeof item1 === 'string') { + if (item2 === true && item1.toLocaleLowerCase() === 'true') return true; + if (item2 === false && item1.toLocaleLowerCase() === 'false') return true; + } + + //Compare booleans and the numbers/string 0 and 1 + if (typeof item1 === 'boolean' && typeof item2 === 'number') { + if (item1 === true && item2 === 1) return true; + if (item1 === false && item2 === 0) return true; + } + + if (typeof item2 === 'boolean' && typeof item1 === 'number') { + if (item2 === true && item1 === 1) return true; + if (item2 === false && item1 === 0) return true; + } + + if (typeof item1 === 'boolean' && typeof item2 === 'string') { + if (item1 === true && item2 === '1') return true; + if (item1 === false && item2 === '0') return true; + } + + if (typeof item2 === 'boolean' && typeof item1 === 'string') { + if (item2 === true && item1 === '1') return true; + if (item2 === false && item1 === '0') return true; + } + + return isEqual(item1, item2); + }; + +const parseStringAndCompareToObject = (str: string, arr: IDataObject) => { + try { + const parsedArray = jsonParse(str); + return isEqual(parsedArray, arr); + } catch (error) { + return false; + } +}; + +function isFalsy(value: T) { + if (isNull(value)) return true; + if (typeof value === 'string' && value === '') return true; + if (Array.isArray(value) && value.length === 0) return true; + return false; +} diff --git a/packages/nodes-base/nodes/Merge/v2/OptionsDescription.ts b/packages/nodes-base/nodes/Merge/v2/OptionsDescription.ts index f0118be450..e794e8f936 100644 --- a/packages/nodes-base/nodes/Merge/v2/OptionsDescription.ts +++ b/packages/nodes-base/nodes/Merge/v2/OptionsDescription.ts @@ -129,6 +129,14 @@ export const optionsDescription: INodeProperties[] = [ }, }, }, + { + displayName: 'Fuzzy Compare', + name: 'fuzzyCompare', + type: 'boolean', + default: false, + description: + "Whether to tolerate small type differences when comparing fields. E.g. the number 3 and the string '3' are treated as the same.", + }, { displayName: 'Include Any Unpaired Items', name: 'includeUnpaired',