From 638d6f60d3f1abd7f647cfe2964259a889e6cd1a Mon Sep 17 00:00:00 2001 From: Michael Kret <88898367+michael-radency@users.noreply.github.com> Date: Tue, 18 Oct 2022 14:10:18 +0300 Subject: [PATCH] feat(Compare Node): new node to compare two inputs --- .../CompareDatasets/CompareDatasets.node.json | 48 +++ .../CompareDatasets/CompareDatasets.node.ts | 189 ++++++++++++ .../nodes/CompareDatasets/GenericFunctions.ts | 289 ++++++++++++++++++ .../nodes/CompareDatasets/compare.svg | 5 + packages/nodes-base/package.json | 1 + 5 files changed, 532 insertions(+) create mode 100644 packages/nodes-base/nodes/CompareDatasets/CompareDatasets.node.json create mode 100644 packages/nodes-base/nodes/CompareDatasets/CompareDatasets.node.ts create mode 100644 packages/nodes-base/nodes/CompareDatasets/GenericFunctions.ts create mode 100644 packages/nodes-base/nodes/CompareDatasets/compare.svg diff --git a/packages/nodes-base/nodes/CompareDatasets/CompareDatasets.node.json b/packages/nodes-base/nodes/CompareDatasets/CompareDatasets.node.json new file mode 100644 index 0000000000..504fab736c --- /dev/null +++ b/packages/nodes-base/nodes/CompareDatasets/CompareDatasets.node.json @@ -0,0 +1,48 @@ +{ + "node": "n8n-nodes-base.compareDatasets", + "nodeVersion": "1.0", + "codexVersion": "1.0", + "categories": ["Core Nodes"], + "resources": { + "primaryDocumentation": [ + { + "url": "https://docs.n8n.io/nodes/n8n-nodes-base.compareDatasets/" + } + ], + "generic": [ + { + "label": "How to synchronize data between two systems (one-way vs. two-way sync", + "icon": "🏬", + "url": "https://n8n.io/blog/how-to-sync-data-between-two-systems/" + }, + { + "label": "Supercharging your conference registration process with n8n", + "icon": "🎫", + "url": "https://n8n.io/blog/supercharging-your-conference-registration-process-with-n8n/" + }, + { + "label": "Migrating Community Metrics to Orbit using n8n", + "icon": "📈", + "url": "https://n8n.io/blog/migrating-community-metrics-to-orbit-using-n8n/" + }, + { + "label": "Build your own virtual assistant with n8n: A step by step guide", + "icon": "👦", + "url": "https://n8n.io/blog/build-your-own-virtual-assistant-with-n8n-a-step-by-step-guide/" + }, + { + "label": "Sending Automated Congratulations with Google Sheets, Twilio, and n8n ", + "icon": "🙌", + "url": "https://n8n.io/blog/sending-automated-congratulations-with-google-sheets-twilio-and-n8n/" + }, + { + "label": "7 no-code workflow automations for Amazon Web Services", + "url": "https://n8n.io/blog/aws-workflow-automation/" + } + ] + }, + "alias": ["Join", "Concatenate", "Compare", "Dataset", "Split"], + "subcategories": { + "Core Nodes": ["Flow"] + } +} diff --git a/packages/nodes-base/nodes/CompareDatasets/CompareDatasets.node.ts b/packages/nodes-base/nodes/CompareDatasets/CompareDatasets.node.ts new file mode 100644 index 0000000000..449551cd7f --- /dev/null +++ b/packages/nodes-base/nodes/CompareDatasets/CompareDatasets.node.ts @@ -0,0 +1,189 @@ +import { IExecuteFunctions } from 'n8n-core'; +import { IDataObject, INodeExecutionData, INodeType, INodeTypeDescription } from 'n8n-workflow'; +import { checkInput, checkMatchFieldsInput, findMatches } from './GenericFunctions'; + +export class CompareDatasets implements INodeType { + description: INodeTypeDescription = { + displayName: 'Compare Datasets', + name: 'compareDatasets', + icon: 'file:compare.svg', + group: ['transform'], + version: 1, + description: 'Compare two inputs for changes', + defaults: { name: 'Compare Datasets' }, + // eslint-disable-next-line n8n-nodes-base/node-class-description-inputs-wrong-regular-node + inputs: ['main', 'main'], + inputNames: ['Input 1', 'Input 2'], + // eslint-disable-next-line n8n-nodes-base/node-class-description-outputs-wrong + outputs: ['main', 'main', 'main', 'main'], + outputNames: ["'In 1 only'", "'Same'", "'Different'", "'In 2 only'"], + properties: [ + { + displayName: 'Fields to Match', + name: 'mergeByFields', + type: 'fixedCollection', + placeholder: 'Add Fields to Match', + default: { values: [{ field1: '', field2: '' }] }, + typeOptions: { + multipleValues: true, + }, + options: [ + { + displayName: 'Values', + name: 'values', + values: [ + { + displayName: 'Input 1 Field', + name: 'field1', + type: 'string', + default: '', + // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id + placeholder: 'e.g. id', + hint: ' Enter the field name as text', + }, + { + displayName: 'Input 2 Field', + name: 'field2', + type: 'string', + default: '', + // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id + placeholder: 'e.g. id', + hint: ' Enter the field name as text', + }, + ], + }, + ], + }, + { + displayName: 'When There Are Differences', + name: 'resolve', + type: 'options', + default: 'preferInput2', + options: [ + { + name: 'Use Input 1 Version', + value: 'preferInput1', + }, + { + name: 'Use Input 2 Version', + value: 'preferInput2', + }, + { + name: 'Use a Mix of Versions', + value: 'mix', + description: 'Output uses different inputs for different fields', + }, + { + name: 'Include Both Versions', + value: 'includeBoth', + description: 'Output contains all data (but structure more complex)', + }, + ], + }, + { + displayName: 'Prefer', + name: 'preferWhenMix', + type: 'options', + default: 'input1', + options: [ + { + name: 'Input 1 Version', + value: 'input1', + }, + { + name: 'Input 2 Version', + value: 'input2', + }, + ], + displayOptions: { + show: { + resolve: ['mix'], + }, + }, + }, + { + displayName: 'For Everything Except', + name: 'exceptWhenMix', + type: 'string', + default: '', + // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id + placeholder: 'e.d. id, country', + hint: 'Enter the names of the input fields as text, separated by commas', + displayOptions: { + show: { + resolve: ['mix'], + }, + }, + }, + { + displayName: 'Options', + name: 'options', + type: 'collection', + placeholder: 'Add Option', + default: {}, + options: [ + { + displayName: 'Disable Dot Notation', + name: 'disableDotNotation', + type: 'boolean', + default: false, + description: + 'Whether to disallow referencing child fields using `parent.child` in the field name', + }, + { + displayName: 'Multiple Matches', + name: 'multipleMatches', + type: 'options', + default: 'first', + options: [ + { + name: 'Include First Match Only', + value: 'first', + description: 'Only ever output a single item per match', + }, + { + name: 'Include All Matches', + value: 'all', + description: 'Output multiple items if there are multiple matches', + }, + ], + }, + ], + }, + ], + }; + + async execute(this: IExecuteFunctions): Promise { + const matchFields = checkMatchFieldsInput( + this.getNodeParameter('mergeByFields.values', 0, []) as IDataObject[], + ); + + const options = this.getNodeParameter('options', 0, {}) as IDataObject; + + const input1 = checkInput( + this.getInputData(0), + matchFields.map((pair) => pair.field1 as string), + (options.disableDotNotation as boolean) || false, + 'Input 1', + ); + + const input2 = checkInput( + this.getInputData(1), + matchFields.map((pair) => pair.field2 as string), + (options.disableDotNotation as boolean) || false, + 'Input 2', + ); + + const resolve = this.getNodeParameter('resolve', 0, '') as string; + options.resolve = resolve; + + if (resolve === 'mix') { + options.preferWhenMix = this.getNodeParameter('preferWhenMix', 0, '') as string; + options.exceptWhenMix = this.getNodeParameter('exceptWhenMix', 0, '') as string; + } + + const matches = findMatches(input1, input2, matchFields, options); + + return matches; + } +} diff --git a/packages/nodes-base/nodes/CompareDatasets/GenericFunctions.ts b/packages/nodes-base/nodes/CompareDatasets/GenericFunctions.ts new file mode 100644 index 0000000000..cd6e2c3aa7 --- /dev/null +++ b/packages/nodes-base/nodes/CompareDatasets/GenericFunctions.ts @@ -0,0 +1,289 @@ +import { IDataObject, INodeExecutionData } from 'n8n-workflow'; +import { difference, get, intersection, isEmpty, isEqual, set, union } from 'lodash'; + +type PairToMatch = { + field1: string; + field2: string; +}; + +type EntryMatches = { + entry: INodeExecutionData; + matches: INodeExecutionData[]; +}; + +function compareItems( + item1: INodeExecutionData, + item2: INodeExecutionData, + fieldsToMatch: PairToMatch[], + resolve?: string, +) { + const keys = {} as IDataObject; + fieldsToMatch.forEach((field) => { + keys[field.field1] = item1.json[field.field1]; + }); + + const keys1 = Object.keys(item1.json); + const keys2 = Object.keys(item2.json); + const intersectionKeys = intersection(keys1, keys2); + + const same = intersectionKeys.reduce((acc, key) => { + if (isEqual(item1.json[key], item2.json[key])) { + acc[key] = item1.json[key]; + } + return acc; + }, {} as IDataObject); + + const sameKeys = Object.keys(same); + const allUniqueKeys = union(keys1, keys2); + const differentKeys = difference(allUniqueKeys, sameKeys); + + const different: IDataObject = {}; + + differentKeys.forEach((key) => { + switch (resolve) { + case 'preferInput1': + different[key] = item1.json[key] || null; + break; + case 'preferInput2': + different[key] = item2.json[key] || null; + break; + default: + const input1 = item1.json[key] || null; + const input2 = item2.json[key] || null; + different[key] = { input1, input2 }; + } + }); + + return { json: { keys, same, different } } as INodeExecutionData; +} + +function combineItems( + item1: INodeExecutionData, + item2: INodeExecutionData, + prefer: string, + except: string, + disableDotNotation: boolean, +) { + let exceptFields: string[]; + const [entry, match] = prefer === 'input1' ? [item1, item2] : [item2, item1]; + + if (except && Array.isArray(except) && except.length) { + exceptFields = except; + } else { + exceptFields = except ? except.split(',').map((field) => field.trim()) : []; + } + + exceptFields.forEach((field) => { + entry.json[field] = match.json[field]; + if (disableDotNotation) { + entry.json[field] = match.json[field]; + } else { + const value = get(match.json, field) || null; + set(entry, `json.${field}`, value); + } + }); + + return entry; +} + +function findAllMatches( + data: INodeExecutionData[], + lookup: IDataObject, + disableDotNotation: boolean, +) { + return data.reduce((acc, entry2, i) => { + if (entry2 === undefined) return acc; + + for (const key of Object.keys(lookup)) { + const excpectedValue = lookup[key]; + let entry2FieldValue; + + if (disableDotNotation) { + entry2FieldValue = entry2.json[key]; + } else { + entry2FieldValue = get(entry2.json, key); + } + + if (!isEqual(excpectedValue, entry2FieldValue)) { + return acc; + } + } + + return acc.concat({ + entry: entry2, + index: i, + }); + }, [] as IDataObject[]); +} + +function findFirstMatch( + data: INodeExecutionData[], + lookup: IDataObject, + disableDotNotation: boolean, +) { + const index = data.findIndex((entry2) => { + if (entry2 === undefined) return false; + + for (const key of Object.keys(lookup)) { + const excpectedValue = lookup[key]; + let entry2FieldValue; + + if (disableDotNotation) { + entry2FieldValue = entry2.json[key]; + } else { + entry2FieldValue = get(entry2.json, key); + } + + if (!isEqual(excpectedValue, entry2FieldValue)) { + return false; + } + } + + return true; + }); + if (index === -1) return []; + + return [{ entry: data[index], index }]; +} + +export function findMatches( + input1: INodeExecutionData[], + input2: INodeExecutionData[], + fieldsToMatch: PairToMatch[], + options: IDataObject, +) { + const data1 = [...input1]; + const data2 = [...input2]; + + const disableDotNotation = (options.disableDotNotation as boolean) || false; + const multipleMatches = (options.multipleMatches as string) || 'first'; + + const filteredData = { + matched: [] as EntryMatches[], + unmatched1: [] as INodeExecutionData[], + unmatched2: [] as INodeExecutionData[], + }; + + const matchedInInput2 = new Set(); + + matchesLoop: for (const entry of data1) { + const lookup: IDataObject = {}; + + fieldsToMatch.forEach((matchCase) => { + let valueToCompare; + if (disableDotNotation) { + valueToCompare = entry.json[matchCase.field1 as string]; + } else { + valueToCompare = get(entry.json, matchCase.field1 as string); + } + lookup[matchCase.field2 as string] = valueToCompare; + }); + + for (const fieldValue of Object.values(lookup)) { + if (fieldValue === undefined) { + filteredData.unmatched1.push(entry); + continue matchesLoop; + } + } + + const foundedMatches = + multipleMatches === 'all' + ? findAllMatches(data2, lookup, disableDotNotation) + : findFirstMatch(data2, lookup, disableDotNotation); + + const matches = foundedMatches.map((match) => match.entry) as INodeExecutionData[]; + foundedMatches.map((match) => matchedInInput2.add(match.index as number)); + + if (matches.length) { + filteredData.matched.push({ entry, matches }); + } else { + filteredData.unmatched1.push(entry); + } + } + + data2.forEach((entry, i) => { + if (!matchedInInput2.has(i)) { + filteredData.unmatched2.push(entry); + } + }); + + const same: INodeExecutionData[] = []; + const different: INodeExecutionData[] = []; + + filteredData.matched.forEach((entryMatches) => { + let entryCopy: INodeExecutionData | undefined; + + entryMatches.matches.forEach((match) => { + if (isEqual(entryMatches.entry.json, match.json)) { + if (!entryCopy) entryCopy = match; + } else { + switch (options.resolve) { + case 'preferInput1': + different.push(entryMatches.entry); + break; + case 'preferInput2': + different.push(match); + break; + case 'mix': + different.push( + combineItems( + entryMatches.entry, + match, + options.preferWhenMix as string, + options.exceptWhenMix as string, + disableDotNotation, + ), + ); + break; + default: + different.push( + compareItems(entryMatches.entry, match, fieldsToMatch, options.resolve as string), + ); + } + } + }); + if (!isEmpty(entryCopy)) { + same.push(entryCopy); + } + }); + + return [filteredData.unmatched1, same, different, filteredData.unmatched2]; +} + +export function checkMatchFieldsInput(data: IDataObject[]) { + if (data.length === 1 && data[0].field1 === '' && data[0].field2 === '') { + throw new Error( + 'You need to define at least one pair of fields in "Fields to Match" to match on', + ); + } + for (const [index, pair] of data.entries()) { + if (pair.field1 === '' || pair.field2 === '') { + throw new Error( + `You need to define both fields in "Fields to Match" for pair ${index + 1}, + field 1 = '${pair.field1}' + field 2 = '${pair.field2}'`, + ); + } + } + return data as PairToMatch[]; +} + +export function checkInput( + input: INodeExecutionData[], + fields: string[], + disableDotNotation: boolean, + inputLabel: string, +) { + for (const field of fields) { + const isPresent = (input || []).some((entry) => { + if (disableDotNotation) { + return entry.json.hasOwnProperty(field); + } + return get(entry.json, field, undefined) !== undefined; + }); + if (!isPresent) { + throw new Error(`Field '${field}' is not present in any of items in '${inputLabel}'`); + } + } + return input; +} diff --git a/packages/nodes-base/nodes/CompareDatasets/compare.svg b/packages/nodes-base/nodes/CompareDatasets/compare.svg new file mode 100644 index 0000000000..652f53f5d7 --- /dev/null +++ b/packages/nodes-base/nodes/CompareDatasets/compare.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/packages/nodes-base/package.json b/packages/nodes-base/package.json index 685c9230f7..9dfc6ac9ce 100644 --- a/packages/nodes-base/package.json +++ b/packages/nodes-base/package.json @@ -401,6 +401,7 @@ "dist/nodes/Coda/Coda.node.js", "dist/nodes/Code/Code.node.js", "dist/nodes/CoinGecko/CoinGecko.node.js", + "dist/nodes/CompareDatasets/CompareDatasets.node.js", "dist/nodes/Compression/Compression.node.js", "dist/nodes/Contentful/Contentful.node.js", "dist/nodes/ConvertKit/ConvertKit.node.js",