fix(Remove Duplicates Node): Tolerate null fields (#9642)

packages/nodes-base/nodes/Transform/RemoveDuplicates/RemoveDuplicates.node.ts

@@ -10,7 +10,7 @@ import {
 	type INodeTypeDescription,
 } from 'n8n-workflow';
 import { prepareFieldsArray } from '../utils/utils';
-import { compareItems, flattenKeys } from './utils';
+import { compareItems, flattenKeys, validateInputData } from './utils';
 
 export class RemoveDuplicates implements INodeType {
 	description: INodeTypeDescription = {
@@ -19,7 +19,7 @@ export class RemoveDuplicates implements INodeType {
 		icon: 'file:removeDuplicates.svg',
 		group: ['transform'],
 		subtitle: '',
-		version: 1,
+		version: [1, 1.1],
 		description: 'Delete items with matching field values',
 		defaults: {
 			name: 'Remove Duplicates',
@@ -205,37 +205,7 @@ export class RemoveDuplicates implements INodeType {
 			return result;
 		});
 
-		for (const key of keys) {
+		validateInputData(this.getNode(), newItems, keys, disableDotNotation);
-			let type: any = undefined;
-			for (const item of newItems) {
-				if (key === '') {
-					throw new NodeOperationError(this.getNode(), 'Name of field to compare is blank');
-				}
-				const value = !disableDotNotation ? get(item.json, key) : item.json[key];
-				if (value === undefined && disableDotNotation && key.includes('.')) {
-					throw new NodeOperationError(
-						this.getNode(),
-						`'${key}' field is missing from some input items`,
-						{
-							description:
-								"If you're trying to use a nested field, make sure you turn off 'disable dot notation' in the node options",
-						},
-					);
-				} else if (value === undefined) {
-					throw new NodeOperationError(
-						this.getNode(),
-						`'${key}' field is missing from some input items`,
-					);
-				}
-				if (type !== undefined && value !== undefined && type !== typeof value) {
-					throw new NodeOperationError(this.getNode(), `'${key}' isn't always the same type`, {
-						description: 'The type of this field varies between items',
-					});
-				} else {
-					type = typeof value;
-				}
-			}
-		}
 
 		// collect the original indexes of items to be removed
 		const removedIndexes: number[] = [];

packages/nodes-base/nodes/Transform/RemoveDuplicates/test/RemoveDuplicates.test.ts

@@ -1,5 +1,62 @@
+import type { INode } from 'n8n-workflow';
+import { validateInputData } from '../utils';
 import { testWorkflows, getWorkflowFilenames } from '@test/nodes/Helpers';
 
 const workflows = getWorkflowFilenames(__dirname);
 
 describe('Test Remove Duplicates Node', () => testWorkflows(workflows));
+
+describe('Test Remove Duplicates Node, validateInputData util', () => {
+	test('Should throw error for version 1', () => {
+		expect(() =>
+			validateInputData(
+				{
+					name: 'Remove Duplicates',
+					type: 'n8n-nodes-base.removeDuplicates',
+					typeVersion: 1,
+				} as INode,
+				[
+					{ json: { country: 'uk' } },
+					{ json: { country: 'us' } },
+					{ json: { country: 'uk' } },
+					{ json: { country: null } },
+				],
+				['country'],
+				false,
+			),
+		).toThrow("'country' isn't always the same type");
+	});
+	test('Should ignore null values and not throw error for version grater than 1', () => {
+		expect(() =>
+			validateInputData(
+				{
+					name: 'Remove Duplicates',
+					type: 'n8n-nodes-base.removeDuplicates',
+					typeVersion: 1.1,
+				} as INode,
+				[
+					{ json: { country: 'uk' } },
+					{ json: { country: 'us' } },
+					{ json: { country: 'uk' } },
+					{ json: { country: null } },
+				],
+				['country'],
+				false,
+			),
+		).not.toThrow();
+	});
+	test('Should throw error for different types, version grater than 1', () => {
+		expect(() =>
+			validateInputData(
+				{
+					name: 'Remove Duplicates',
+					type: 'n8n-nodes-base.removeDuplicates',
+					typeVersion: 1.1,
+				} as INode,
+				[{ json: { id: 1 } }, { json: { id: '1' } }, { json: { id: 2 } }, { json: { id: null } }],
+				['id'],
+				false,
+			),
+		).toThrow("'id' isn't always the same type");
+	});
+});

packages/nodes-base/nodes/Transform/RemoveDuplicates/utils.ts

@@ -3,7 +3,12 @@ import isEqual from 'lodash/isEqual';
 import isObject from 'lodash/isObject';
 import merge from 'lodash/merge';
 import reduce from 'lodash/reduce';
-import type { IDataObject, INode, INodeExecutionData } from 'n8n-workflow';
+import {
+	NodeOperationError,
+	type IDataObject,
+	type INode,
+	type INodeExecutionData,
+} from 'n8n-workflow';
 
 export const compareItems = (
 	obj: INodeExecutionData,
@@ -34,3 +39,42 @@ export const flattenKeys = (obj: IDataObject, path: string[] = []): IDataObject
 		? { [path.join('.')]: obj }
 		: reduce(obj, (cum, next, key) => merge(cum, flattenKeys(next as IDataObject, [...path, key])), {}); //prettier-ignore
 };
+
+export const validateInputData = (
+	node: INode,
+	items: INodeExecutionData[],
+	keysToCompare: string[],
+	disableDotNotation: boolean,
+) => {
+	for (const key of keysToCompare) {
+		let type: any = undefined;
+		for (const [i, item] of items.entries()) {
+			if (key === '') {
+				throw new NodeOperationError(node, 'Name of field to compare is blank');
+			}
+			const value = !disableDotNotation ? get(item.json, key) : item.json[key];
+			if (value === null && node.typeVersion > 1) continue;
+
+			if (value === undefined && disableDotNotation && key.includes('.')) {
+				throw new NodeOperationError(node, `'${key}' field is missing from some input items`, {
+					description:
+						"If you're trying to use a nested field, make sure you turn off 'disable dot notation' in the node options",
+				});
+			} else if (value === undefined) {
+				throw new NodeOperationError(node, `'${key}' field is missing from some input items`);
+			}
+			if (type !== undefined && value !== undefined && type !== typeof value) {
+				const description =
+					'The type of this field varies between items' +
+					(node.typeVersion > 1
+						? `, in item [${i - 1}] it's a ${type} and in item [${i}] it's a ${typeof value} `
+						: '');
+				throw new NodeOperationError(node, `'${key}' isn't always the same type`, {
+					description,
+				});
+			} else {
+				type = typeof value;
+			}
+		}
+	}
+};