feat(Compare Node): new node to compare two inputs

packages/nodes-base/nodes/CompareDatasets/CompareDatasets.node.json

@@ -0,0 +1,48 @@
+{
+	"node": "n8n-nodes-base.compareDatasets",
+	"nodeVersion": "1.0",
+	"codexVersion": "1.0",
+	"categories": ["Core Nodes"],
+	"resources": {
+		"primaryDocumentation": [
+			{
+				"url": "https://docs.n8n.io/nodes/n8n-nodes-base.compareDatasets/"
+			}
+		],
+		"generic": [
+			{
+				"label": "How to synchronize data between two systems (one-way vs. two-way sync",
+				"icon": "🏬",
+				"url": "https://n8n.io/blog/how-to-sync-data-between-two-systems/"
+			},
+			{
+				"label": "Supercharging your conference registration process with n8n",
+				"icon": "🎫",
+				"url": "https://n8n.io/blog/supercharging-your-conference-registration-process-with-n8n/"
+			},
+			{
+				"label": "Migrating Community Metrics to Orbit using n8n",
+				"icon": "📈",
+				"url": "https://n8n.io/blog/migrating-community-metrics-to-orbit-using-n8n/"
+			},
+			{
+				"label": "Build your own virtual assistant with n8n: A step by step guide",
+				"icon": "👦",
+				"url": "https://n8n.io/blog/build-your-own-virtual-assistant-with-n8n-a-step-by-step-guide/"
+			},
+			{
+				"label": "Sending Automated Congratulations with Google Sheets, Twilio, and n8n ",
+				"icon": "🙌",
+				"url": "https://n8n.io/blog/sending-automated-congratulations-with-google-sheets-twilio-and-n8n/"
+			},
+			{
+				"label": "7 no-code workflow automations for Amazon Web Services",
+				"url": "https://n8n.io/blog/aws-workflow-automation/"
+			}
+		]
+	},
+	"alias": ["Join", "Concatenate", "Compare", "Dataset", "Split"],
+	"subcategories": {
+		"Core Nodes": ["Flow"]
+	}
+}

packages/nodes-base/nodes/CompareDatasets/CompareDatasets.node.ts

@@ -0,0 +1,189 @@
+import { IExecuteFunctions } from 'n8n-core';
+import { IDataObject, INodeExecutionData, INodeType, INodeTypeDescription } from 'n8n-workflow';
+import { checkInput, checkMatchFieldsInput, findMatches } from './GenericFunctions';
+
+export class CompareDatasets implements INodeType {
+	description: INodeTypeDescription = {
+		displayName: 'Compare Datasets',
+		name: 'compareDatasets',
+		icon: 'file:compare.svg',
+		group: ['transform'],
+		version: 1,
+		description: 'Compare two inputs for changes',
+		defaults: { name: 'Compare Datasets' },
+		// eslint-disable-next-line n8n-nodes-base/node-class-description-inputs-wrong-regular-node
+		inputs: ['main', 'main'],
+		inputNames: ['Input 1', 'Input 2'],
+		// eslint-disable-next-line n8n-nodes-base/node-class-description-outputs-wrong
+		outputs: ['main', 'main', 'main', 'main'],
+		outputNames: ["'In 1 only'", "'Same'", "'Different'", "'In 2 only'"],
+		properties: [
+			{
+				displayName: 'Fields to Match',
+				name: 'mergeByFields',
+				type: 'fixedCollection',
+				placeholder: 'Add Fields to Match',
+				default: { values: [{ field1: '', field2: '' }] },
+				typeOptions: {
+					multipleValues: true,
+				},
+				options: [
+					{
+						displayName: 'Values',
+						name: 'values',
+						values: [
+							{
+								displayName: 'Input 1 Field',
+								name: 'field1',
+								type: 'string',
+								default: '',
+								// eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id
+								placeholder: 'e.g. id',
+								hint: ' Enter the field name as text',
+							},
+							{
+								displayName: 'Input 2 Field',
+								name: 'field2',
+								type: 'string',
+								default: '',
+								// eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id
+								placeholder: 'e.g. id',
+								hint: ' Enter the field name as text',
+							},
+						],
+					},
+				],
+			},
+			{
+				displayName: 'When There Are Differences',
+				name: 'resolve',
+				type: 'options',
+				default: 'preferInput2',
+				options: [
+					{
+						name: 'Use Input 1 Version',
+						value: 'preferInput1',
+					},
+					{
+						name: 'Use Input 2 Version',
+						value: 'preferInput2',
+					},
+					{
+						name: 'Use a Mix of Versions',
+						value: 'mix',
+						description: 'Output uses different inputs for different fields',
+					},
+					{
+						name: 'Include Both Versions',
+						value: 'includeBoth',
+						description: 'Output contains all data (but structure more complex)',
+					},
+				],
+			},
+			{
+				displayName: 'Prefer',
+				name: 'preferWhenMix',
+				type: 'options',
+				default: 'input1',
+				options: [
+					{
+						name: 'Input 1 Version',
+						value: 'input1',
+					},
+					{
+						name: 'Input 2 Version',
+						value: 'input2',
+					},
+				],
+				displayOptions: {
+					show: {
+						resolve: ['mix'],
+					},
+				},
+			},
+			{
+				displayName: 'For Everything Except',
+				name: 'exceptWhenMix',
+				type: 'string',
+				default: '',
+				// eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id
+				placeholder: 'e.d. id, country',
+				hint: 'Enter the names of the input fields as text, separated by commas',
+				displayOptions: {
+					show: {
+						resolve: ['mix'],
+					},
+				},
+			},
+			{
+				displayName: 'Options',
+				name: 'options',
+				type: 'collection',
+				placeholder: 'Add Option',
+				default: {},
+				options: [
+					{
+						displayName: 'Disable Dot Notation',
+						name: 'disableDotNotation',
+						type: 'boolean',
+						default: false,
+						description:
+							'Whether to disallow referencing child fields using `parent.child` in the field name',
+					},
+					{
+						displayName: 'Multiple Matches',
+						name: 'multipleMatches',
+						type: 'options',
+						default: 'first',
+						options: [
+							{
+								name: 'Include First Match Only',
+								value: 'first',
+								description: 'Only ever output a single item per match',
+							},
+							{
+								name: 'Include All Matches',
+								value: 'all',
+								description: 'Output multiple items if there are multiple matches',
+							},
+						],
+					},
+				],
+			},
+		],
+	};
+
+	async execute(this: IExecuteFunctions): Promise<INodeExecutionData[][]> {
+		const matchFields = checkMatchFieldsInput(
+			this.getNodeParameter('mergeByFields.values', 0, []) as IDataObject[],
+		);
+
+		const options = this.getNodeParameter('options', 0, {}) as IDataObject;
+
+		const input1 = checkInput(
+			this.getInputData(0),
+			matchFields.map((pair) => pair.field1 as string),
+			(options.disableDotNotation as boolean) || false,
+			'Input 1',
+		);
+
+		const input2 = checkInput(
+			this.getInputData(1),
+			matchFields.map((pair) => pair.field2 as string),
+			(options.disableDotNotation as boolean) || false,
+			'Input 2',
+		);
+
+		const resolve = this.getNodeParameter('resolve', 0, '') as string;
+		options.resolve = resolve;
+
+		if (resolve === 'mix') {
+			options.preferWhenMix = this.getNodeParameter('preferWhenMix', 0, '') as string;
+			options.exceptWhenMix = this.getNodeParameter('exceptWhenMix', 0, '') as string;
+		}
+
+		const matches = findMatches(input1, input2, matchFields, options);
+
+		return matches;
+	}
+}

packages/nodes-base/nodes/CompareDatasets/GenericFunctions.ts

@@ -0,0 +1,289 @@
+import { IDataObject, INodeExecutionData } from 'n8n-workflow';
+import { difference, get, intersection, isEmpty, isEqual, set, union } from 'lodash';
+
+type PairToMatch = {
+	field1: string;
+	field2: string;
+};
+
+type EntryMatches = {
+	entry: INodeExecutionData;
+	matches: INodeExecutionData[];
+};
+
+function compareItems(
+	item1: INodeExecutionData,
+	item2: INodeExecutionData,
+	fieldsToMatch: PairToMatch[],
+	resolve?: string,
+) {
+	const keys = {} as IDataObject;
+	fieldsToMatch.forEach((field) => {
+		keys[field.field1] = item1.json[field.field1];
+	});
+
+	const keys1 = Object.keys(item1.json);
+	const keys2 = Object.keys(item2.json);
+	const intersectionKeys = intersection(keys1, keys2);
+
+	const same = intersectionKeys.reduce((acc, key) => {
+		if (isEqual(item1.json[key], item2.json[key])) {
+			acc[key] = item1.json[key];
+		}
+		return acc;
+	}, {} as IDataObject);
+
+	const sameKeys = Object.keys(same);
+	const allUniqueKeys = union(keys1, keys2);
+	const differentKeys = difference(allUniqueKeys, sameKeys);
+
+	const different: IDataObject = {};
+
+	differentKeys.forEach((key) => {
+		switch (resolve) {
+			case 'preferInput1':
+				different[key] = item1.json[key] || null;
+				break;
+			case 'preferInput2':
+				different[key] = item2.json[key] || null;
+				break;
+			default:
+				const input1 = item1.json[key] || null;
+				const input2 = item2.json[key] || null;
+				different[key] = { input1, input2 };
+		}
+	});
+
+	return { json: { keys, same, different } } as INodeExecutionData;
+}
+
+function combineItems(
+	item1: INodeExecutionData,
+	item2: INodeExecutionData,
+	prefer: string,
+	except: string,
+	disableDotNotation: boolean,
+) {
+	let exceptFields: string[];
+	const [entry, match] = prefer === 'input1' ? [item1, item2] : [item2, item1];
+
+	if (except && Array.isArray(except) && except.length) {
+		exceptFields = except;
+	} else {
+		exceptFields = except ? except.split(',').map((field) => field.trim()) : [];
+	}
+
+	exceptFields.forEach((field) => {
+		entry.json[field] = match.json[field];
+		if (disableDotNotation) {
+			entry.json[field] = match.json[field];
+		} else {
+			const value = get(match.json, field) || null;
+			set(entry, `json.${field}`, value);
+		}
+	});
+
+	return entry;
+}
+
+function findAllMatches(
+	data: INodeExecutionData[],
+	lookup: IDataObject,
+	disableDotNotation: boolean,
+) {
+	return data.reduce((acc, entry2, i) => {
+		if (entry2 === undefined) return acc;
+
+		for (const key of Object.keys(lookup)) {
+			const excpectedValue = lookup[key];
+			let entry2FieldValue;
+
+			if (disableDotNotation) {
+				entry2FieldValue = entry2.json[key];
+			} else {
+				entry2FieldValue = get(entry2.json, key);
+			}
+
+			if (!isEqual(excpectedValue, entry2FieldValue)) {
+				return acc;
+			}
+		}
+
+		return acc.concat({
+			entry: entry2,
+			index: i,
+		});
+	}, [] as IDataObject[]);
+}
+
+function findFirstMatch(
+	data: INodeExecutionData[],
+	lookup: IDataObject,
+	disableDotNotation: boolean,
+) {
+	const index = data.findIndex((entry2) => {
+		if (entry2 === undefined) return false;
+
+		for (const key of Object.keys(lookup)) {
+			const excpectedValue = lookup[key];
+			let entry2FieldValue;
+
+			if (disableDotNotation) {
+				entry2FieldValue = entry2.json[key];
+			} else {
+				entry2FieldValue = get(entry2.json, key);
+			}
+
+			if (!isEqual(excpectedValue, entry2FieldValue)) {
+				return false;
+			}
+		}
+
+		return true;
+	});
+	if (index === -1) return [];
+
+	return [{ entry: data[index], index }];
+}
+
+export function findMatches(
+	input1: INodeExecutionData[],
+	input2: INodeExecutionData[],
+	fieldsToMatch: PairToMatch[],
+	options: IDataObject,
+) {
+	const data1 = [...input1];
+	const data2 = [...input2];
+
+	const disableDotNotation = (options.disableDotNotation as boolean) || false;
+	const multipleMatches = (options.multipleMatches as string) || 'first';
+
+	const filteredData = {
+		matched: [] as EntryMatches[],
+		unmatched1: [] as INodeExecutionData[],
+		unmatched2: [] as INodeExecutionData[],
+	};
+
+	const matchedInInput2 = new Set<number>();
+
+	matchesLoop: for (const entry of data1) {
+		const lookup: IDataObject = {};
+
+		fieldsToMatch.forEach((matchCase) => {
+			let valueToCompare;
+			if (disableDotNotation) {
+				valueToCompare = entry.json[matchCase.field1 as string];
+			} else {
+				valueToCompare = get(entry.json, matchCase.field1 as string);
+			}
+			lookup[matchCase.field2 as string] = valueToCompare;
+		});
+
+		for (const fieldValue of Object.values(lookup)) {
+			if (fieldValue === undefined) {
+				filteredData.unmatched1.push(entry);
+				continue matchesLoop;
+			}
+		}
+
+		const foundedMatches =
+			multipleMatches === 'all'
+				? findAllMatches(data2, lookup, disableDotNotation)
+				: findFirstMatch(data2, lookup, disableDotNotation);
+
+		const matches = foundedMatches.map((match) => match.entry) as INodeExecutionData[];
+		foundedMatches.map((match) => matchedInInput2.add(match.index as number));
+
+		if (matches.length) {
+			filteredData.matched.push({ entry, matches });
+		} else {
+			filteredData.unmatched1.push(entry);
+		}
+	}
+
+	data2.forEach((entry, i) => {
+		if (!matchedInInput2.has(i)) {
+			filteredData.unmatched2.push(entry);
+		}
+	});
+
+	const same: INodeExecutionData[] = [];
+	const different: INodeExecutionData[] = [];
+
+	filteredData.matched.forEach((entryMatches) => {
+		let entryCopy: INodeExecutionData | undefined;
+
+		entryMatches.matches.forEach((match) => {
+			if (isEqual(entryMatches.entry.json, match.json)) {
+				if (!entryCopy) entryCopy = match;
+			} else {
+				switch (options.resolve) {
+					case 'preferInput1':
+						different.push(entryMatches.entry);
+						break;
+					case 'preferInput2':
+						different.push(match);
+						break;
+					case 'mix':
+						different.push(
+							combineItems(
+								entryMatches.entry,
+								match,
+								options.preferWhenMix as string,
+								options.exceptWhenMix as string,
+								disableDotNotation,
+							),
+						);
+						break;
+					default:
+						different.push(
+							compareItems(entryMatches.entry, match, fieldsToMatch, options.resolve as string),
+						);
+				}
+			}
+		});
+		if (!isEmpty(entryCopy)) {
+			same.push(entryCopy);
+		}
+	});
+
+	return [filteredData.unmatched1, same, different, filteredData.unmatched2];
+}
+
+export function checkMatchFieldsInput(data: IDataObject[]) {
+	if (data.length === 1 && data[0].field1 === '' && data[0].field2 === '') {
+		throw new Error(
+			'You need to define at least one pair of fields in "Fields to Match" to match on',
+		);
+	}
+	for (const [index, pair] of data.entries()) {
+		if (pair.field1 === '' || pair.field2 === '') {
+			throw new Error(
+				`You need to define both fields in "Fields to Match" for pair ${index + 1},
+				 field 1 = '${pair.field1}'
+				 field 2 = '${pair.field2}'`,
+			);
+		}
+	}
+	return data as PairToMatch[];
+}
+
+export function checkInput(
+	input: INodeExecutionData[],
+	fields: string[],
+	disableDotNotation: boolean,
+	inputLabel: string,
+) {
+	for (const field of fields) {
+		const isPresent = (input || []).some((entry) => {
+			if (disableDotNotation) {
+				return entry.json.hasOwnProperty(field);
+			}
+			return get(entry.json, field, undefined) !== undefined;
+		});
+		if (!isPresent) {
+			throw new Error(`Field '${field}' is not present in any of items in '${inputLabel}'`);
+		}
+	}
+	return input;
+}

packages/nodes-base/nodes/CompareDatasets/compare.svg

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg width="384" height="512" viewBox="0 0 384 512" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+    <path id="plus" fill="#ed230d" stroke="none" d="M 352 448 L 32 448 C 14.31 448 0 462.309998 0 480 C 0 497.690002 14.31 511.100006 32 511.100006 L 352 511.100006 C 369.690002 511.100006 384 496.790009 384 480 C 384 463.209991 369.700012 448 352 448 Z"/>
+    <path id="minus" fill="#62f730" stroke="none" d="M 48 208 L 160 208 L 160 319.100006 C 160 336.790009 174.309998 350.200012 192 350.200012 C 209.690002 350.200012 224 335.890015 224 319.100006 L 224 208 L 336 208 C 353.690002 208 368 193.679993 368 175.98999 C 368 158.299988 353.690002 144 336 144 L 224 144 L 224 32 C 224 14.309998 209.690002 -0.01001 192 -0.01001 C 174.309998 -0.01001 160 14.329987 160 32.01001 L 160 144.01001 L 48 144.01001 C 30.309999 144.01001 16 158.320007 16 176 C 16 193.679993 30.309999 208 48 208 Z"/>
+</svg>

packages/nodes-base/package.json

@@ -401,6 +401,7 @@
       "dist/nodes/Coda/Coda.node.js",
       "dist/nodes/Code/Code.node.js",
       "dist/nodes/CoinGecko/CoinGecko.node.js",
+      "dist/nodes/CompareDatasets/CompareDatasets.node.js",
       "dist/nodes/Compression/Compression.node.js",
       "dist/nodes/Contentful/Contentful.node.js",
       "dist/nodes/ConvertKit/ConvertKit.node.js",