feat(Compare Datasets Node): Fuzzy compare option

packages/nodes-base/nodes/CompareDatasets/CompareDatasets.node.ts

@@ -18,6 +18,13 @@ export class CompareDatasets implements INodeType {
 		outputs: ['main', 'main', 'main', 'main'],
 		outputNames: ['In A only', 'Same', 'Different', 'In B only'],
 		properties: [
+			{
+				displayName:
+					'Items from different branches are paired together when the fields below match. If paired, the rest of the fields are compared to determine whether the items are the same or different',
+				name: 'infoBox',
+				type: 'notice',
+				default: '',
+			},
 			{
 				displayName: 'Fields to Match',
 				name: 'mergeByFields',
@@ -132,6 +139,14 @@ export class CompareDatasets implements INodeType {
 						description:
 							"Fields that shouldn't be included when checking whether two items are the same",
 					},
+					{
+						displayName: 'Fuzzy Compare',
+						name: 'fuzzyCompare',
+						type: 'boolean',
+						default: false,
+						description:
+							"Whether to tolerate small type differences when comparing fields. E.g. the number 3 and the string '3' are treated as the same.",
+					},
 					{
 						displayName: 'Disable Dot Notation',
 						name: 'disableDotNotation',

packages/nodes-base/nodes/CompareDatasets/GenericFunctions.ts

@@ -1,5 +1,5 @@
-import { IDataObject, INodeExecutionData } from 'n8n-workflow';
-import { difference, get, intersection, isEmpty, isEqual, omit, set, union } from 'lodash';
+import { IDataObject, INodeExecutionData, jsonParse } from 'n8n-workflow';
+import { difference, get, intersection, isEmpty, isEqual, isNull, omit, set, union } from 'lodash';
 
 type PairToMatch = {
 	field1: string;
@@ -11,12 +11,15 @@ type EntryMatches = {
 	matches: INodeExecutionData[];
 };
 
+type CompareFunction = <T, U>(a: T, b: U) => boolean;
+
 function compareItems(
 	item1: INodeExecutionData,
 	item2: INodeExecutionData,
 	fieldsToMatch: PairToMatch[],
 	resolve: string,
 	skipFields: string[],
+	isEntriesEqual: CompareFunction,
 ) {
 	const keys = {} as IDataObject;
 	fieldsToMatch.forEach((field) => {
@@ -28,7 +31,7 @@ function compareItems(
 	const intersectionKeys = intersection(keys1, keys2);
 
 	const same = intersectionKeys.reduce((acc, key) => {
-		if (isEqual(item1.json[key], item2.json[key])) {
+		if (isEntriesEqual(item1.json[key], item2.json[key])) {
 			acc[key] = item1.json[key];
 		}
 		return acc;
@@ -98,6 +101,7 @@ function findAllMatches(
 	data: INodeExecutionData[],
 	lookup: IDataObject,
 	disableDotNotation: boolean,
+	isEntriesEqual: CompareFunction,
 ) {
 	return data.reduce((acc, entry2, i) => {
 		if (entry2 === undefined) return acc;
@@ -112,7 +116,7 @@ function findAllMatches(
 				entry2FieldValue = get(entry2.json, key);
 			}
 
-			if (!isEqual(excpectedValue, entry2FieldValue)) {
+			if (!isEntriesEqual(excpectedValue, entry2FieldValue)) {
 				return acc;
 			}
 		}
@@ -128,6 +132,7 @@ function findFirstMatch(
 	data: INodeExecutionData[],
 	lookup: IDataObject,
 	disableDotNotation: boolean,
+	isEntriesEqual: CompareFunction,
 ) {
 	const index = data.findIndex((entry2) => {
 		if (entry2 === undefined) return false;
@@ -142,7 +147,7 @@ function findFirstMatch(
 				entry2FieldValue = get(entry2.json, key);
 			}
 
-			if (!isEqual(excpectedValue, entry2FieldValue)) {
+			if (!isEntriesEqual(excpectedValue, entry2FieldValue)) {
 				return false;
 			}
 		}
@@ -163,6 +168,7 @@ export function findMatches(
 	const data1 = [...input1];
 	const data2 = [...input2];
 
+	const isEntriesEqual = fuzzyCompare(options);
 	const disableDotNotation = (options.disableDotNotation as boolean) || false;
 	const multipleMatches = (options.multipleMatches as string) || 'first';
 	const skipFields = ((options.skipFields as string) || '').split(',').map((field) => field.trim());
@@ -197,8 +203,8 @@ export function findMatches(
 
 		const foundedMatches =
 			multipleMatches === 'all'
-				? findAllMatches(data2, lookup, disableDotNotation)
-				: findFirstMatch(data2, lookup, disableDotNotation);
+				? findAllMatches(data2, lookup, disableDotNotation, isEntriesEqual)
+				: findFirstMatch(data2, lookup, disableDotNotation, isEntriesEqual);
 
 		const matches = foundedMatches.map((match) => match.entry) as INodeExecutionData[];
 		foundedMatches.map((match) => matchedInInput2.add(match.index as number));
@@ -230,8 +236,27 @@ export function findMatches(
 				entryFromInput1 = omit(entryFromInput1, skipFields);
 				entryFromInput2 = omit(entryFromInput2, skipFields);
 			}
-			if (isEqual(entryFromInput1, entryFromInput2)) {
-				if (!entryCopy) entryCopy = match;
+
+			let isItemsEqual = true;
+			if (options.fuzzyCompare) {
+				for (const key of Object.keys(entryFromInput1)) {
+					if (!isEntriesEqual(entryFromInput1[key], entryFromInput2[key])) {
+						isItemsEqual = false;
+						break;
+					}
+				}
+			} else {
+				isItemsEqual = isEntriesEqual(entryFromInput1, entryFromInput2);
+			}
+
+			if (isItemsEqual) {
+				if (!entryCopy) {
+					if (options.fuzzyCompare && options.resolve === 'preferInput2') {
+						entryCopy = match;
+					} else {
+						entryCopy = entryMatches.entry;
+					}
+				}
 			} else {
 				switch (options.resolve) {
 					case 'preferInput1':
@@ -259,6 +284,7 @@ export function findMatches(
 								fieldsToMatch,
 								options.resolve as string,
 								skipFields,
+								isEntriesEqual,
 							),
 						);
 				}
@@ -315,3 +341,90 @@ export function checkInput(
 	}
 	return input;
 }
+
+const fuzzyCompare =
+	(options: IDataObject) =>
+	<T, U>(item1: T, item2: U) => {
+		//Fuzzy compare is disabled, so we do strict comparison
+		if (!options.fuzzyCompare) return isEqual(item1, item2);
+
+		//Both types are the same, so we do strict comparison
+		if (!isNull(item1) && !isNull(item2) && typeof item1 === typeof item2) {
+			return isEqual(item1, item2);
+		}
+
+		//Null, empty strings, empty arrays all treated as the same
+		if (isFalsy(item1) && isFalsy(item2)) return true;
+
+		//When a field is missing in one branch and isFalsy() in another, treat them as matching
+		if (isFalsy(item1) && item2 === undefined) return true;
+		if (item1 === undefined && isFalsy(item2)) return true;
+
+		//Compare numbers and strings representing that number
+		if (typeof item1 === 'number' && typeof item2 === 'string') {
+			return item1.toString() === item2;
+		}
+
+		if (typeof item1 === 'string' && typeof item2 === 'number') {
+			return item1 === item2.toString();
+		}
+
+		//Compare objects/arrays and their stringified version
+		if (!isNull(item1) && typeof item1 === 'object' && typeof item2 === 'string') {
+			return parseStringAndCompareToObject(item2, item1 as IDataObject);
+		}
+
+		if (!isNull(item2) && typeof item1 === 'string' && typeof item2 === 'object') {
+			return parseStringAndCompareToObject(item1, item2 as IDataObject);
+		}
+
+		//Compare booleans and strings representing the boolean (’true’, ‘True’, ‘TRUE’)
+		if (typeof item1 === 'boolean' && typeof item2 === 'string') {
+			if (item1 === true && item2.toLocaleLowerCase() === 'true') return true;
+			if (item1 === false && item2.toLocaleLowerCase() === 'false') return true;
+		}
+
+		if (typeof item2 === 'boolean' && typeof item1 === 'string') {
+			if (item2 === true && item1.toLocaleLowerCase() === 'true') return true;
+			if (item2 === false && item1.toLocaleLowerCase() === 'false') return true;
+		}
+
+		//Compare booleans and the numbers/string 0 and 1
+		if (typeof item1 === 'boolean' && typeof item2 === 'number') {
+			if (item1 === true && item2 === 1) return true;
+			if (item1 === false && item2 === 0) return true;
+		}
+
+		if (typeof item2 === 'boolean' && typeof item1 === 'number') {
+			if (item2 === true && item1 === 1) return true;
+			if (item2 === false && item1 === 0) return true;
+		}
+
+		if (typeof item1 === 'boolean' && typeof item2 === 'string') {
+			if (item1 === true && item2 === '1') return true;
+			if (item1 === false && item2 === '0') return true;
+		}
+
+		if (typeof item2 === 'boolean' && typeof item1 === 'string') {
+			if (item2 === true && item1 === '1') return true;
+			if (item2 === false && item1 === '0') return true;
+		}
+
+		return isEqual(item1, item2);
+	};
+
+const parseStringAndCompareToObject = (str: string, arr: IDataObject) => {
+	try {
+		const parsedArray = jsonParse(str);
+		return isEqual(parsedArray, arr);
+	} catch (error) {
+		return false;
+	}
+};
+
+function isFalsy<T>(value: T) {
+	if (isNull(value)) return true;
+	if (typeof value === 'string' && value === '') return true;
+	if (Array.isArray(value) && value.length === 0) return true;
+	return false;
+}

packages/nodes-base/nodes/Merge/v2/GenericFunctions.ts

@@ -4,9 +4,10 @@ import {
 	IDataObject,
 	INodeExecutionData,
 	IPairedItemData,
+	jsonParse,
 } from 'n8n-workflow';
 
-import { assign, assignWith, get, isEqual, merge, mergeWith } from 'lodash';
+import { assign, assignWith, get, isEqual, isNull, merge, mergeWith } from 'lodash';
 
 type PairToMatch = {
 	field1: string;
@@ -18,6 +19,7 @@ export type MatchFieldsOptions = {
 	outputDataFrom: MatchFieldsOutput;
 	multipleMatches: MultipleMatches;
 	disableDotNotation: boolean;
+	fuzzyCompare?: boolean;
 };
 
 export type ClashResolveOptions = {
@@ -46,6 +48,8 @@ type EntryMatches = {
 	matches: INodeExecutionData[];
 };
 
+type CompareFunction = <T, U>(a: T, b: U) => boolean;
+
 export function addSuffixToEntriesKeys(data: INodeExecutionData[], suffix: string) {
 	return data.map((entry) => {
 		const json: IDataObject = {};
@@ -60,6 +64,7 @@ function findAllMatches(
 	data: INodeExecutionData[],
 	lookup: IDataObject,
 	disableDotNotation: boolean,
+	isEntriesEqual: CompareFunction,
 ) {
 	return data.reduce((acc, entry2, i) => {
 		if (entry2 === undefined) return acc;
@@ -74,7 +79,7 @@ function findAllMatches(
 				entry2FieldValue = get(entry2.json, key);
 			}
 
-			if (!isEqual(excpectedValue, entry2FieldValue)) {
+			if (!isEntriesEqual(excpectedValue, entry2FieldValue)) {
 				return acc;
 			}
 		}
@@ -90,6 +95,7 @@ function findFirstMatch(
 	data: INodeExecutionData[],
 	lookup: IDataObject,
 	disableDotNotation: boolean,
+	isEntriesEqual: CompareFunction,
 ) {
 	const index = data.findIndex((entry2) => {
 		if (entry2 === undefined) return false;
@@ -104,7 +110,7 @@ function findFirstMatch(
 				entry2FieldValue = get(entry2.json, key);
 			}
 
-			if (!isEqual(excpectedValue, entry2FieldValue)) {
+			if (!isEntriesEqual(excpectedValue, entry2FieldValue)) {
 				return false;
 			}
 		}
@@ -129,6 +135,7 @@ export function findMatches(
 		[data1, data2] = [data2, data1];
 	}
 
+	const isEntriesEqual = fuzzyCompare(options);
 	const disableDotNotation = options.disableDotNotation || false;
 	const multipleMatches = (options.multipleMatches as string) || 'all';
 
@@ -163,8 +170,8 @@ export function findMatches(
 
 		const foundedMatches =
 			multipleMatches === 'all'
-				? findAllMatches(data2, lookup, disableDotNotation)
-				: findFirstMatch(data2, lookup, disableDotNotation);
+				? findAllMatches(data2, lookup, disableDotNotation, isEntriesEqual)
+				: findFirstMatch(data2, lookup, disableDotNotation, isEntriesEqual);
 
 		const matches = foundedMatches.map((match) => match.entry) as INodeExecutionData[];
 		foundedMatches.map((match) => matchedInInput2.add(match.index as number));
@@ -367,3 +374,90 @@ export function addSourceField(data: INodeExecutionData[], sourceField: string)
 		};
 	});
 }
+
+const fuzzyCompare =
+	(options: IDataObject) =>
+	<T, U>(item1: T, item2: U) => {
+		//Fuzzy compare is disabled, so we do strict comparison
+		if (!options.fuzzyCompare) return isEqual(item1, item2);
+
+		//Both types are the same, so we do strict comparison
+		if (!isNull(item1) && !isNull(item2) && typeof item1 === typeof item2) {
+			return isEqual(item1, item2);
+		}
+
+		//Null, empty strings, empty arrays all treated as the same
+		if (isFalsy(item1) && isFalsy(item2)) return true;
+
+		//When a field is missing in one branch and isFalsy() in another, treat them as matching
+		if (isFalsy(item1) && item2 === undefined) return true;
+		if (item1 === undefined && isFalsy(item2)) return true;
+
+		//Compare numbers and strings representing that number
+		if (typeof item1 === 'number' && typeof item2 === 'string') {
+			return item1.toString() === item2;
+		}
+
+		if (typeof item1 === 'string' && typeof item2 === 'number') {
+			return item1 === item2.toString();
+		}
+
+		//Compare objects/arrays and their stringified version
+		if (!isNull(item1) && typeof item1 === 'object' && typeof item2 === 'string') {
+			return parseStringAndCompareToObject(item2, item1 as IDataObject);
+		}
+
+		if (!isNull(item2) && typeof item1 === 'string' && typeof item2 === 'object') {
+			return parseStringAndCompareToObject(item1, item2 as IDataObject);
+		}
+
+		//Compare booleans and strings representing the boolean (’true’, ‘True’, ‘TRUE’)
+		if (typeof item1 === 'boolean' && typeof item2 === 'string') {
+			if (item1 === true && item2.toLocaleLowerCase() === 'true') return true;
+			if (item1 === false && item2.toLocaleLowerCase() === 'false') return true;
+		}
+
+		if (typeof item2 === 'boolean' && typeof item1 === 'string') {
+			if (item2 === true && item1.toLocaleLowerCase() === 'true') return true;
+			if (item2 === false && item1.toLocaleLowerCase() === 'false') return true;
+		}
+
+		//Compare booleans and the numbers/string 0 and 1
+		if (typeof item1 === 'boolean' && typeof item2 === 'number') {
+			if (item1 === true && item2 === 1) return true;
+			if (item1 === false && item2 === 0) return true;
+		}
+
+		if (typeof item2 === 'boolean' && typeof item1 === 'number') {
+			if (item2 === true && item1 === 1) return true;
+			if (item2 === false && item1 === 0) return true;
+		}
+
+		if (typeof item1 === 'boolean' && typeof item2 === 'string') {
+			if (item1 === true && item2 === '1') return true;
+			if (item1 === false && item2 === '0') return true;
+		}
+
+		if (typeof item2 === 'boolean' && typeof item1 === 'string') {
+			if (item2 === true && item1 === '1') return true;
+			if (item2 === false && item1 === '0') return true;
+		}
+
+		return isEqual(item1, item2);
+	};
+
+const parseStringAndCompareToObject = (str: string, arr: IDataObject) => {
+	try {
+		const parsedArray = jsonParse(str);
+		return isEqual(parsedArray, arr);
+	} catch (error) {
+		return false;
+	}
+};
+
+function isFalsy<T>(value: T) {
+	if (isNull(value)) return true;
+	if (typeof value === 'string' && value === '') return true;
+	if (Array.isArray(value) && value.length === 0) return true;
+	return false;
+}

packages/nodes-base/nodes/Merge/v2/OptionsDescription.ts

@@ -129,6 +129,14 @@ export const optionsDescription: INodeProperties[] = [
 					},
 				},
 			},
+			{
+				displayName: 'Fuzzy Compare',
+				name: 'fuzzyCompare',
+				type: 'boolean',
+				default: false,
+				description:
+					"Whether to tolerate small type differences when comparing fields. E.g. the number 3 and the string '3' are treated as the same.",
+			},
 			{
 				displayName: 'Include Any Unpaired Items',
 				name: 'includeUnpaired',