feat(Compare Node): new node to compare two inputs

This commit is contained in:
Michael Kret 2022-10-18 14:10:18 +03:00 committed by GitHub
parent c74fdc7815
commit 638d6f60d3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 532 additions and 0 deletions

View file

@ -0,0 +1,48 @@
{
"node": "n8n-nodes-base.compareDatasets",
"nodeVersion": "1.0",
"codexVersion": "1.0",
"categories": ["Core Nodes"],
"resources": {
"primaryDocumentation": [
{
"url": "https://docs.n8n.io/nodes/n8n-nodes-base.compareDatasets/"
}
],
"generic": [
{
"label": "How to synchronize data between two systems (one-way vs. two-way sync",
"icon": "🏬",
"url": "https://n8n.io/blog/how-to-sync-data-between-two-systems/"
},
{
"label": "Supercharging your conference registration process with n8n",
"icon": "🎫",
"url": "https://n8n.io/blog/supercharging-your-conference-registration-process-with-n8n/"
},
{
"label": "Migrating Community Metrics to Orbit using n8n",
"icon": "📈",
"url": "https://n8n.io/blog/migrating-community-metrics-to-orbit-using-n8n/"
},
{
"label": "Build your own virtual assistant with n8n: A step by step guide",
"icon": "👦",
"url": "https://n8n.io/blog/build-your-own-virtual-assistant-with-n8n-a-step-by-step-guide/"
},
{
"label": "Sending Automated Congratulations with Google Sheets, Twilio, and n8n ",
"icon": "🙌",
"url": "https://n8n.io/blog/sending-automated-congratulations-with-google-sheets-twilio-and-n8n/"
},
{
"label": "7 no-code workflow automations for Amazon Web Services",
"url": "https://n8n.io/blog/aws-workflow-automation/"
}
]
},
"alias": ["Join", "Concatenate", "Compare", "Dataset", "Split"],
"subcategories": {
"Core Nodes": ["Flow"]
}
}

View file

@ -0,0 +1,189 @@
import { IExecuteFunctions } from 'n8n-core';
import { IDataObject, INodeExecutionData, INodeType, INodeTypeDescription } from 'n8n-workflow';
import { checkInput, checkMatchFieldsInput, findMatches } from './GenericFunctions';
export class CompareDatasets implements INodeType {
description: INodeTypeDescription = {
displayName: 'Compare Datasets',
name: 'compareDatasets',
icon: 'file:compare.svg',
group: ['transform'],
version: 1,
description: 'Compare two inputs for changes',
defaults: { name: 'Compare Datasets' },
// eslint-disable-next-line n8n-nodes-base/node-class-description-inputs-wrong-regular-node
inputs: ['main', 'main'],
inputNames: ['Input 1', 'Input 2'],
// eslint-disable-next-line n8n-nodes-base/node-class-description-outputs-wrong
outputs: ['main', 'main', 'main', 'main'],
outputNames: ["'In 1 only'", "'Same'", "'Different'", "'In 2 only'"],
properties: [
{
displayName: 'Fields to Match',
name: 'mergeByFields',
type: 'fixedCollection',
placeholder: 'Add Fields to Match',
default: { values: [{ field1: '', field2: '' }] },
typeOptions: {
multipleValues: true,
},
options: [
{
displayName: 'Values',
name: 'values',
values: [
{
displayName: 'Input 1 Field',
name: 'field1',
type: 'string',
default: '',
// eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id
placeholder: 'e.g. id',
hint: ' Enter the field name as text',
},
{
displayName: 'Input 2 Field',
name: 'field2',
type: 'string',
default: '',
// eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id
placeholder: 'e.g. id',
hint: ' Enter the field name as text',
},
],
},
],
},
{
displayName: 'When There Are Differences',
name: 'resolve',
type: 'options',
default: 'preferInput2',
options: [
{
name: 'Use Input 1 Version',
value: 'preferInput1',
},
{
name: 'Use Input 2 Version',
value: 'preferInput2',
},
{
name: 'Use a Mix of Versions',
value: 'mix',
description: 'Output uses different inputs for different fields',
},
{
name: 'Include Both Versions',
value: 'includeBoth',
description: 'Output contains all data (but structure more complex)',
},
],
},
{
displayName: 'Prefer',
name: 'preferWhenMix',
type: 'options',
default: 'input1',
options: [
{
name: 'Input 1 Version',
value: 'input1',
},
{
name: 'Input 2 Version',
value: 'input2',
},
],
displayOptions: {
show: {
resolve: ['mix'],
},
},
},
{
displayName: 'For Everything Except',
name: 'exceptWhenMix',
type: 'string',
default: '',
// eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id
placeholder: 'e.d. id, country',
hint: 'Enter the names of the input fields as text, separated by commas',
displayOptions: {
show: {
resolve: ['mix'],
},
},
},
{
displayName: 'Options',
name: 'options',
type: 'collection',
placeholder: 'Add Option',
default: {},
options: [
{
displayName: 'Disable Dot Notation',
name: 'disableDotNotation',
type: 'boolean',
default: false,
description:
'Whether to disallow referencing child fields using `parent.child` in the field name',
},
{
displayName: 'Multiple Matches',
name: 'multipleMatches',
type: 'options',
default: 'first',
options: [
{
name: 'Include First Match Only',
value: 'first',
description: 'Only ever output a single item per match',
},
{
name: 'Include All Matches',
value: 'all',
description: 'Output multiple items if there are multiple matches',
},
],
},
],
},
],
};
async execute(this: IExecuteFunctions): Promise<INodeExecutionData[][]> {
const matchFields = checkMatchFieldsInput(
this.getNodeParameter('mergeByFields.values', 0, []) as IDataObject[],
);
const options = this.getNodeParameter('options', 0, {}) as IDataObject;
const input1 = checkInput(
this.getInputData(0),
matchFields.map((pair) => pair.field1 as string),
(options.disableDotNotation as boolean) || false,
'Input 1',
);
const input2 = checkInput(
this.getInputData(1),
matchFields.map((pair) => pair.field2 as string),
(options.disableDotNotation as boolean) || false,
'Input 2',
);
const resolve = this.getNodeParameter('resolve', 0, '') as string;
options.resolve = resolve;
if (resolve === 'mix') {
options.preferWhenMix = this.getNodeParameter('preferWhenMix', 0, '') as string;
options.exceptWhenMix = this.getNodeParameter('exceptWhenMix', 0, '') as string;
}
const matches = findMatches(input1, input2, matchFields, options);
return matches;
}
}

View file

@ -0,0 +1,289 @@
import { IDataObject, INodeExecutionData } from 'n8n-workflow';
import { difference, get, intersection, isEmpty, isEqual, set, union } from 'lodash';
type PairToMatch = {
field1: string;
field2: string;
};
type EntryMatches = {
entry: INodeExecutionData;
matches: INodeExecutionData[];
};
function compareItems(
item1: INodeExecutionData,
item2: INodeExecutionData,
fieldsToMatch: PairToMatch[],
resolve?: string,
) {
const keys = {} as IDataObject;
fieldsToMatch.forEach((field) => {
keys[field.field1] = item1.json[field.field1];
});
const keys1 = Object.keys(item1.json);
const keys2 = Object.keys(item2.json);
const intersectionKeys = intersection(keys1, keys2);
const same = intersectionKeys.reduce((acc, key) => {
if (isEqual(item1.json[key], item2.json[key])) {
acc[key] = item1.json[key];
}
return acc;
}, {} as IDataObject);
const sameKeys = Object.keys(same);
const allUniqueKeys = union(keys1, keys2);
const differentKeys = difference(allUniqueKeys, sameKeys);
const different: IDataObject = {};
differentKeys.forEach((key) => {
switch (resolve) {
case 'preferInput1':
different[key] = item1.json[key] || null;
break;
case 'preferInput2':
different[key] = item2.json[key] || null;
break;
default:
const input1 = item1.json[key] || null;
const input2 = item2.json[key] || null;
different[key] = { input1, input2 };
}
});
return { json: { keys, same, different } } as INodeExecutionData;
}
function combineItems(
item1: INodeExecutionData,
item2: INodeExecutionData,
prefer: string,
except: string,
disableDotNotation: boolean,
) {
let exceptFields: string[];
const [entry, match] = prefer === 'input1' ? [item1, item2] : [item2, item1];
if (except && Array.isArray(except) && except.length) {
exceptFields = except;
} else {
exceptFields = except ? except.split(',').map((field) => field.trim()) : [];
}
exceptFields.forEach((field) => {
entry.json[field] = match.json[field];
if (disableDotNotation) {
entry.json[field] = match.json[field];
} else {
const value = get(match.json, field) || null;
set(entry, `json.${field}`, value);
}
});
return entry;
}
function findAllMatches(
data: INodeExecutionData[],
lookup: IDataObject,
disableDotNotation: boolean,
) {
return data.reduce((acc, entry2, i) => {
if (entry2 === undefined) return acc;
for (const key of Object.keys(lookup)) {
const excpectedValue = lookup[key];
let entry2FieldValue;
if (disableDotNotation) {
entry2FieldValue = entry2.json[key];
} else {
entry2FieldValue = get(entry2.json, key);
}
if (!isEqual(excpectedValue, entry2FieldValue)) {
return acc;
}
}
return acc.concat({
entry: entry2,
index: i,
});
}, [] as IDataObject[]);
}
function findFirstMatch(
data: INodeExecutionData[],
lookup: IDataObject,
disableDotNotation: boolean,
) {
const index = data.findIndex((entry2) => {
if (entry2 === undefined) return false;
for (const key of Object.keys(lookup)) {
const excpectedValue = lookup[key];
let entry2FieldValue;
if (disableDotNotation) {
entry2FieldValue = entry2.json[key];
} else {
entry2FieldValue = get(entry2.json, key);
}
if (!isEqual(excpectedValue, entry2FieldValue)) {
return false;
}
}
return true;
});
if (index === -1) return [];
return [{ entry: data[index], index }];
}
export function findMatches(
input1: INodeExecutionData[],
input2: INodeExecutionData[],
fieldsToMatch: PairToMatch[],
options: IDataObject,
) {
const data1 = [...input1];
const data2 = [...input2];
const disableDotNotation = (options.disableDotNotation as boolean) || false;
const multipleMatches = (options.multipleMatches as string) || 'first';
const filteredData = {
matched: [] as EntryMatches[],
unmatched1: [] as INodeExecutionData[],
unmatched2: [] as INodeExecutionData[],
};
const matchedInInput2 = new Set<number>();
matchesLoop: for (const entry of data1) {
const lookup: IDataObject = {};
fieldsToMatch.forEach((matchCase) => {
let valueToCompare;
if (disableDotNotation) {
valueToCompare = entry.json[matchCase.field1 as string];
} else {
valueToCompare = get(entry.json, matchCase.field1 as string);
}
lookup[matchCase.field2 as string] = valueToCompare;
});
for (const fieldValue of Object.values(lookup)) {
if (fieldValue === undefined) {
filteredData.unmatched1.push(entry);
continue matchesLoop;
}
}
const foundedMatches =
multipleMatches === 'all'
? findAllMatches(data2, lookup, disableDotNotation)
: findFirstMatch(data2, lookup, disableDotNotation);
const matches = foundedMatches.map((match) => match.entry) as INodeExecutionData[];
foundedMatches.map((match) => matchedInInput2.add(match.index as number));
if (matches.length) {
filteredData.matched.push({ entry, matches });
} else {
filteredData.unmatched1.push(entry);
}
}
data2.forEach((entry, i) => {
if (!matchedInInput2.has(i)) {
filteredData.unmatched2.push(entry);
}
});
const same: INodeExecutionData[] = [];
const different: INodeExecutionData[] = [];
filteredData.matched.forEach((entryMatches) => {
let entryCopy: INodeExecutionData | undefined;
entryMatches.matches.forEach((match) => {
if (isEqual(entryMatches.entry.json, match.json)) {
if (!entryCopy) entryCopy = match;
} else {
switch (options.resolve) {
case 'preferInput1':
different.push(entryMatches.entry);
break;
case 'preferInput2':
different.push(match);
break;
case 'mix':
different.push(
combineItems(
entryMatches.entry,
match,
options.preferWhenMix as string,
options.exceptWhenMix as string,
disableDotNotation,
),
);
break;
default:
different.push(
compareItems(entryMatches.entry, match, fieldsToMatch, options.resolve as string),
);
}
}
});
if (!isEmpty(entryCopy)) {
same.push(entryCopy);
}
});
return [filteredData.unmatched1, same, different, filteredData.unmatched2];
}
export function checkMatchFieldsInput(data: IDataObject[]) {
if (data.length === 1 && data[0].field1 === '' && data[0].field2 === '') {
throw new Error(
'You need to define at least one pair of fields in "Fields to Match" to match on',
);
}
for (const [index, pair] of data.entries()) {
if (pair.field1 === '' || pair.field2 === '') {
throw new Error(
`You need to define both fields in "Fields to Match" for pair ${index + 1},
field 1 = '${pair.field1}'
field 2 = '${pair.field2}'`,
);
}
}
return data as PairToMatch[];
}
export function checkInput(
input: INodeExecutionData[],
fields: string[],
disableDotNotation: boolean,
inputLabel: string,
) {
for (const field of fields) {
const isPresent = (input || []).some((entry) => {
if (disableDotNotation) {
return entry.json.hasOwnProperty(field);
}
return get(entry.json, field, undefined) !== undefined;
});
if (!isPresent) {
throw new Error(`Field '${field}' is not present in any of items in '${inputLabel}'`);
}
}
return input;
}

View file

@ -0,0 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
<svg width="384" height="512" viewBox="0 0 384 512" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
<path id="plus" fill="#ed230d" stroke="none" d="M 352 448 L 32 448 C 14.31 448 0 462.309998 0 480 C 0 497.690002 14.31 511.100006 32 511.100006 L 352 511.100006 C 369.690002 511.100006 384 496.790009 384 480 C 384 463.209991 369.700012 448 352 448 Z"/>
<path id="minus" fill="#62f730" stroke="none" d="M 48 208 L 160 208 L 160 319.100006 C 160 336.790009 174.309998 350.200012 192 350.200012 C 209.690002 350.200012 224 335.890015 224 319.100006 L 224 208 L 336 208 C 353.690002 208 368 193.679993 368 175.98999 C 368 158.299988 353.690002 144 336 144 L 224 144 L 224 32 C 224 14.309998 209.690002 -0.01001 192 -0.01001 C 174.309998 -0.01001 160 14.329987 160 32.01001 L 160 144.01001 L 48 144.01001 C 30.309999 144.01001 16 158.320007 16 176 C 16 193.679993 30.309999 208 48 208 Z"/>
</svg>

After

Width:  |  Height:  |  Size: 970 B

View file

@ -401,6 +401,7 @@
"dist/nodes/Coda/Coda.node.js",
"dist/nodes/Code/Code.node.js",
"dist/nodes/CoinGecko/CoinGecko.node.js",
"dist/nodes/CompareDatasets/CompareDatasets.node.js",
"dist/nodes/Compression/Compression.node.js",
"dist/nodes/Contentful/Contentful.node.js",
"dist/nodes/ConvertKit/ConvertKit.node.js",