fix(Item Lists Node): Don't check same type in remove duplicates operation (#7678)

Github issue / Community forum post (link here to close automatically):

---------

Co-authored-by: Michael Kret <michael.k@radency.com>

packages/nodes-base/nodes/ItemLists/ItemLists.node.ts

@@ -14,7 +14,7 @@ export class ItemLists extends VersionedNodeType {
 			group: ['input'],
 			subtitle: '={{$parameter["operation"] + ": " + $parameter["resource"]}}',
 			description: 'Helper for working with lists of items and transforming arrays',
-			defaultVersion: 3,
+			defaultVersion: 3.1,
 		};
 
 		const nodeVersions: IVersionedNodeType['nodeVersions'] = {
@@ -23,6 +23,7 @@ export class ItemLists extends VersionedNodeType {
 			2.1: new ItemListsV2(baseDescription),
 			2.2: new ItemListsV2(baseDescription),
 			3: new ItemListsV3(baseDescription),
+			3.1: new ItemListsV3(baseDescription),
 		};
 
 		super(nodeVersions, baseDescription);

packages/nodes-base/nodes/ItemLists/V3/actions/itemList/removeDuplicates.operation.ts

@@ -6,7 +6,7 @@ import isEqual from 'lodash/isEqual';
 import lt from 'lodash/lt';
 import pick from 'lodash/pick';
 
-import { compareItems, flattenKeys, prepareFieldsArray } from '../../helpers/utils';
+import { compareItems, flattenKeys, prepareFieldsArray, typeToNumber } from '../../helpers/utils';
 import { disableDotNotationBoolean } from '../common.descriptions';
 import { updateDisplayOptions } from '@utils/utilities';
 
@@ -105,6 +105,7 @@ export async function execute(
 		false,
 	) as boolean;
 	const removeOtherFields = this.getNodeParameter('options.removeOtherFields', 0, false) as boolean;
+	const nodeVersion = this.getNode().typeVersion;
 
 	let keys = disableDotNotation
 		? Object.keys(items[0].json)
@@ -163,24 +164,28 @@ export async function execute(
 				pairedItem: { item: index },
 			}) as INodeExecutionData,
 	);
+
 	//sort items using the compare keys
 	newItems.sort((a, b) => {
 		let result = 0;
 
 		for (const key of keys) {
-			let equal;
+			const a_value = disableDotNotation ? a.json[key] : get(a.json, key);
-			if (!disableDotNotation) {
+			const b_value = disableDotNotation ? b.json[key] : get(b.json, key);
-				equal = isEqual(get(a.json, key), get(b.json, key));
+
-			} else {
+			if (nodeVersion >= 3.1) {
-				equal = isEqual(a.json[key], b.json[key]);
+				const a_value_tnum = typeToNumber(a_value);
+				const b_value_tnum = typeToNumber(b_value);
+				if (a_value_tnum !== b_value_tnum) {
+					result = a_value_tnum - b_value_tnum;
+					break;
 				}
+			}
+
+			const equal = isEqual(a_value, b_value);
+
 			if (!equal) {
-				let lessThan;
+				const lessThan = lt(a_value, b_value);
-				if (!disableDotNotation) {
-					lessThan = lt(get(a.json, key), get(b.json, key));
-				} else {
-					lessThan = lt(a.json[key], b.json[key]);
-				}
 				result = lessThan ? -1 : 1;
 				break;
 			}
@@ -210,7 +215,7 @@ export async function execute(
 					`'${key}' field is missing from some input items`,
 				);
 			}
-			if (type !== undefined && value !== undefined && type !== typeof value) {
+			if (nodeVersion < 3.1 && type !== undefined && value !== undefined && type !== typeof value) {
 				throw new NodeOperationError(this.getNode(), `'${key}' isn't always the same type`, {
 					description: 'The type of this field varies between items',
 				});

packages/nodes-base/nodes/ItemLists/V3/actions/versionDescription.ts

@@ -10,7 +10,7 @@ export const versionDescription: INodeTypeDescription = {
 	group: ['input'],
 	subtitle: '={{$parameter["operation"] + ": " + $parameter["resource"]}}',
 	description: 'Helper for working with lists of items and transforming arrays',
-	version: 3,
+	version: [3, 3.1],
 	defaults: {
 		name: 'Item Lists',
 	},

packages/nodes-base/nodes/ItemLists/V3/helpers/utils.ts

@@ -5,6 +5,7 @@ import type {
 	IBinaryData,
 	INode,
 	INodeExecutionData,
+	GenericValue,
 } from 'n8n-workflow';
 import { NodeOperationError } from 'n8n-workflow';
 
@@ -147,3 +148,22 @@ export function addBinariesToItem(
 
 	return newItem;
 }
+
+export function typeToNumber(value: GenericValue): number {
+	if (typeof value === 'object') {
+		if (Array.isArray(value)) return 9;
+		if (value === null) return 10;
+		if (value instanceof Date) return 11;
+	}
+	const types = {
+		_string: 1,
+		_number: 2,
+		_bigint: 3,
+		_boolean: 4,
+		_symbol: 5,
+		_undefined: 6,
+		_object: 7,
+		_function: 8,
+	};
+	return types[`_${typeof value}`];
+}

packages/nodes-base/nodes/ItemLists/test/node/workflow.removeDuplicates_3_1.json

@@ -0,0 +1,292 @@
+{
+	"name": "My workflow 63",
+	"nodes": [
+		{
+			"parameters": {},
+			"id": "5f174c07-00e5-49fc-854f-b1571d35c5a3",
+			"name": "When clicking \"Execute Workflow\"",
+			"type": "n8n-nodes-base.manualTrigger",
+			"typeVersion": 1,
+			"position": [
+				960,
+				520
+			]
+		},
+		{
+			"parameters": {
+				"jsCode": "return [\n  { mixed: \"1\", match: \"foo\" },\n  { mixed: 1, match: \"foo\" },\n  { mixed: true, match: \"foo\" },\n  { mixed: false, match: \"foo\" },\n  { mixed: {}, match: \"foo\" },\n  { mixed: [], match: \"foo\" },\n  //duplicates\n  { mixed: \"1\", match: \"foo\" },\n  { mixed: 1, match: \"foo\" },\n  { mixed: true, match: \"foo\" },\n  { mixed: false, match: \"foo\" },\n  { mixed: {}, match: \"foo\" },\n  { mixed: [], match: \"foo\" },\n];"
+			},
+			"id": "2aa52759-f7a5-4a60-bf2e-9c1e3d2821dc",
+			"name": "Code",
+			"type": "n8n-nodes-base.code",
+			"typeVersion": 2,
+			"position": [
+				1160,
+				520
+			]
+		},
+		{
+			"parameters": {
+				"operation": "removeDuplicates"
+			},
+			"id": "90acc956-989f-4008-a3df-fe0162762b24",
+			"name": "Remove duplicates",
+			"type": "n8n-nodes-base.itemLists",
+			"typeVersion": 3.1,
+			"position": [
+				1440,
+				160
+			]
+		},
+		{
+			"parameters": {
+				"operation": "removeDuplicates",
+				"compare": "selectedFields",
+				"fieldsToCompare": "mixed",
+				"options": {}
+			},
+			"id": "6c39a6bb-b042-4d4c-828c-52699f178828",
+			"name": "Remove duplicates by mixed",
+			"type": "n8n-nodes-base.itemLists",
+			"typeVersion": 3.1,
+			"position": [
+				1440,
+				340
+			]
+		},
+		{
+			"parameters": {
+				"operation": "removeDuplicates",
+				"compare": "selectedFields",
+				"fieldsToCompare": "match",
+				"options": {}
+			},
+			"id": "d3783be2-6705-44d3-b481-72e7a9dba458",
+			"name": "Remove duplicates by match",
+			"type": "n8n-nodes-base.itemLists",
+			"typeVersion": 3.1,
+			"position": [
+				1440,
+				520
+			]
+		},
+		{
+			"parameters": {
+				"operation": "removeDuplicates",
+				"compare": "allFieldsExcept",
+				"fieldsToExclude": "mixed",
+				"options": {}
+			},
+			"id": "e634e39d-0ce6-477f-97cc-69eec2dd981b",
+			"name": "Remove duplicates except by mixed",
+			"type": "n8n-nodes-base.itemLists",
+			"typeVersion": 3.1,
+			"position": [
+				1440,
+				720
+			]
+		},
+		{
+			"parameters": {
+				"operation": "removeDuplicates",
+				"compare": "allFieldsExcept",
+				"fieldsToExclude": "match",
+				"options": {}
+			},
+			"id": "0a1bbc9a-3505-413c-ae86-c60dc60b5909",
+			"name": "Remove duplicates except by match",
+			"type": "n8n-nodes-base.itemLists",
+			"typeVersion": 3.1,
+			"position": [
+				1440,
+				940
+			]
+		}
+	],
+	"pinData": {
+		"Remove duplicates": [
+			{
+				"json": {
+					"mixed": "1",
+					"match": "foo"
+				}
+			},
+			{
+				"json": {
+					"mixed": 1,
+					"match": "foo"
+				}
+			},
+			{
+				"json": {
+					"mixed": true,
+					"match": "foo"
+				}
+			},
+			{
+				"json": {
+					"mixed": false,
+					"match": "foo"
+				}
+			},
+			{
+				"json": {
+					"mixed": {},
+					"match": "foo"
+				}
+			},
+			{
+				"json": {
+					"mixed": [],
+					"match": "foo"
+				}
+			}
+		],
+		"Remove duplicates by mixed": [
+			{
+				"json": {
+					"mixed": "1",
+					"match": "foo"
+				}
+			},
+			{
+				"json": {
+					"mixed": 1,
+					"match": "foo"
+				}
+			},
+			{
+				"json": {
+					"mixed": true,
+					"match": "foo"
+				}
+			},
+			{
+				"json": {
+					"mixed": false,
+					"match": "foo"
+				}
+			},
+			{
+				"json": {
+					"mixed": {},
+					"match": "foo"
+				}
+			},
+			{
+				"json": {
+					"mixed": [],
+					"match": "foo"
+				}
+			}
+		],
+		"Remove duplicates by match": [
+			{
+				"json": {
+					"mixed": "1",
+					"match": "foo"
+				}
+			}
+		],
+		"Remove duplicates except by mixed": [
+			{
+				"json": {
+					"mixed": "1",
+					"match": "foo"
+				}
+			}
+		],
+		"Remove duplicates except by match": [
+			{
+				"json": {
+					"mixed": "1",
+					"match": "foo"
+				}
+			},
+			{
+				"json": {
+					"mixed": 1,
+					"match": "foo"
+				}
+			},
+			{
+				"json": {
+					"mixed": true,
+					"match": "foo"
+				}
+			},
+			{
+				"json": {
+					"mixed": false,
+					"match": "foo"
+				}
+			},
+			{
+				"json": {
+					"mixed": {},
+					"match": "foo"
+				}
+			},
+			{
+				"json": {
+					"mixed": [],
+					"match": "foo"
+				}
+			}
+		]
+	},
+	"connections": {
+		"When clicking \"Execute Workflow\"": {
+			"main": [
+				[
+					{
+						"node": "Code",
+						"type": "main",
+						"index": 0
+					}
+				]
+			]
+		},
+		"Code": {
+			"main": [
+				[
+					{
+						"node": "Remove duplicates",
+						"type": "main",
+						"index": 0
+					},
+					{
+						"node": "Remove duplicates by mixed",
+						"type": "main",
+						"index": 0
+					},
+					{
+						"node": "Remove duplicates by match",
+						"type": "main",
+						"index": 0
+					},
+					{
+						"node": "Remove duplicates except by mixed",
+						"type": "main",
+						"index": 0
+					},
+					{
+						"node": "Remove duplicates except by match",
+						"type": "main",
+						"index": 0
+					}
+				]
+			]
+		}
+	},
+	"active": false,
+	"settings": {
+		"executionOrder": "v1"
+	},
+	"versionId": "c657bc6f-02bc-4e1b-b49a-d1dca8b13256",
+	"id": "cHSnZsTtYIJj3gL2",
+	"meta": {
+		"instanceId": "104a4d08d8897b8bdeb38aaca515021075e0bd8544c983c2bb8c86e6a8e6081c"
+	},
+	"tags": []
+}