From 4f307646f3a5691331c7c610c62f562921a005f8 Mon Sep 17 00:00:00 2001 From: Marcus <56945030+maspio@users.noreply.github.com> Date: Tue, 21 Nov 2023 13:10:59 +0100 Subject: [PATCH] fix(Item Lists Node): Don't check same type in remove duplicates operation (#7678) Github issue / Community forum post (link here to close automatically): --------- Co-authored-by: Michael Kret --- .../nodes/ItemLists/ItemLists.node.ts | 3 +- .../itemList/removeDuplicates.operation.ts | 33 +- .../V3/actions/versionDescription.ts | 2 +- .../nodes/ItemLists/V3/helpers/utils.ts | 20 ++ .../node/workflow.removeDuplicates_3_1.json | 292 ++++++++++++++++++ 5 files changed, 334 insertions(+), 16 deletions(-) create mode 100644 packages/nodes-base/nodes/ItemLists/test/node/workflow.removeDuplicates_3_1.json diff --git a/packages/nodes-base/nodes/ItemLists/ItemLists.node.ts b/packages/nodes-base/nodes/ItemLists/ItemLists.node.ts index d454c14e92..5118151eef 100644 --- a/packages/nodes-base/nodes/ItemLists/ItemLists.node.ts +++ b/packages/nodes-base/nodes/ItemLists/ItemLists.node.ts @@ -14,7 +14,7 @@ export class ItemLists extends VersionedNodeType { group: ['input'], subtitle: '={{$parameter["operation"] + ": " + $parameter["resource"]}}', description: 'Helper for working with lists of items and transforming arrays', - defaultVersion: 3, + defaultVersion: 3.1, }; const nodeVersions: IVersionedNodeType['nodeVersions'] = { @@ -23,6 +23,7 @@ export class ItemLists extends VersionedNodeType { 2.1: new ItemListsV2(baseDescription), 2.2: new ItemListsV2(baseDescription), 3: new ItemListsV3(baseDescription), + 3.1: new ItemListsV3(baseDescription), }; super(nodeVersions, baseDescription); diff --git a/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/removeDuplicates.operation.ts b/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/removeDuplicates.operation.ts index e97be1359a..8b8c7e6948 100644 --- a/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/removeDuplicates.operation.ts +++ b/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/removeDuplicates.operation.ts @@ -6,7 +6,7 @@ import isEqual from 'lodash/isEqual'; import lt from 'lodash/lt'; import pick from 'lodash/pick'; -import { compareItems, flattenKeys, prepareFieldsArray } from '../../helpers/utils'; +import { compareItems, flattenKeys, prepareFieldsArray, typeToNumber } from '../../helpers/utils'; import { disableDotNotationBoolean } from '../common.descriptions'; import { updateDisplayOptions } from '@utils/utilities'; @@ -105,6 +105,7 @@ export async function execute( false, ) as boolean; const removeOtherFields = this.getNodeParameter('options.removeOtherFields', 0, false) as boolean; + const nodeVersion = this.getNode().typeVersion; let keys = disableDotNotation ? Object.keys(items[0].json) @@ -163,24 +164,28 @@ export async function execute( pairedItem: { item: index }, }) as INodeExecutionData, ); + //sort items using the compare keys newItems.sort((a, b) => { let result = 0; for (const key of keys) { - let equal; - if (!disableDotNotation) { - equal = isEqual(get(a.json, key), get(b.json, key)); - } else { - equal = isEqual(a.json[key], b.json[key]); - } - if (!equal) { - let lessThan; - if (!disableDotNotation) { - lessThan = lt(get(a.json, key), get(b.json, key)); - } else { - lessThan = lt(a.json[key], b.json[key]); + const a_value = disableDotNotation ? a.json[key] : get(a.json, key); + const b_value = disableDotNotation ? b.json[key] : get(b.json, key); + + if (nodeVersion >= 3.1) { + const a_value_tnum = typeToNumber(a_value); + const b_value_tnum = typeToNumber(b_value); + if (a_value_tnum !== b_value_tnum) { + result = a_value_tnum - b_value_tnum; + break; } + } + + const equal = isEqual(a_value, b_value); + + if (!equal) { + const lessThan = lt(a_value, b_value); result = lessThan ? -1 : 1; break; } @@ -210,7 +215,7 @@ export async function execute( `'${key}' field is missing from some input items`, ); } - if (type !== undefined && value !== undefined && type !== typeof value) { + if (nodeVersion < 3.1 && type !== undefined && value !== undefined && type !== typeof value) { throw new NodeOperationError(this.getNode(), `'${key}' isn't always the same type`, { description: 'The type of this field varies between items', }); diff --git a/packages/nodes-base/nodes/ItemLists/V3/actions/versionDescription.ts b/packages/nodes-base/nodes/ItemLists/V3/actions/versionDescription.ts index 728f646769..8c992a308e 100644 --- a/packages/nodes-base/nodes/ItemLists/V3/actions/versionDescription.ts +++ b/packages/nodes-base/nodes/ItemLists/V3/actions/versionDescription.ts @@ -10,7 +10,7 @@ export const versionDescription: INodeTypeDescription = { group: ['input'], subtitle: '={{$parameter["operation"] + ": " + $parameter["resource"]}}', description: 'Helper for working with lists of items and transforming arrays', - version: 3, + version: [3, 3.1], defaults: { name: 'Item Lists', }, diff --git a/packages/nodes-base/nodes/ItemLists/V3/helpers/utils.ts b/packages/nodes-base/nodes/ItemLists/V3/helpers/utils.ts index 8f2337ec4d..7a22c6e55b 100644 --- a/packages/nodes-base/nodes/ItemLists/V3/helpers/utils.ts +++ b/packages/nodes-base/nodes/ItemLists/V3/helpers/utils.ts @@ -5,6 +5,7 @@ import type { IBinaryData, INode, INodeExecutionData, + GenericValue, } from 'n8n-workflow'; import { NodeOperationError } from 'n8n-workflow'; @@ -147,3 +148,22 @@ export function addBinariesToItem( return newItem; } + +export function typeToNumber(value: GenericValue): number { + if (typeof value === 'object') { + if (Array.isArray(value)) return 9; + if (value === null) return 10; + if (value instanceof Date) return 11; + } + const types = { + _string: 1, + _number: 2, + _bigint: 3, + _boolean: 4, + _symbol: 5, + _undefined: 6, + _object: 7, + _function: 8, + }; + return types[`_${typeof value}`]; +} diff --git a/packages/nodes-base/nodes/ItemLists/test/node/workflow.removeDuplicates_3_1.json b/packages/nodes-base/nodes/ItemLists/test/node/workflow.removeDuplicates_3_1.json new file mode 100644 index 0000000000..4a37795065 --- /dev/null +++ b/packages/nodes-base/nodes/ItemLists/test/node/workflow.removeDuplicates_3_1.json @@ -0,0 +1,292 @@ +{ + "name": "My workflow 63", + "nodes": [ + { + "parameters": {}, + "id": "5f174c07-00e5-49fc-854f-b1571d35c5a3", + "name": "When clicking \"Execute Workflow\"", + "type": "n8n-nodes-base.manualTrigger", + "typeVersion": 1, + "position": [ + 960, + 520 + ] + }, + { + "parameters": { + "jsCode": "return [\n { mixed: \"1\", match: \"foo\" },\n { mixed: 1, match: \"foo\" },\n { mixed: true, match: \"foo\" },\n { mixed: false, match: \"foo\" },\n { mixed: {}, match: \"foo\" },\n { mixed: [], match: \"foo\" },\n //duplicates\n { mixed: \"1\", match: \"foo\" },\n { mixed: 1, match: \"foo\" },\n { mixed: true, match: \"foo\" },\n { mixed: false, match: \"foo\" },\n { mixed: {}, match: \"foo\" },\n { mixed: [], match: \"foo\" },\n];" + }, + "id": "2aa52759-f7a5-4a60-bf2e-9c1e3d2821dc", + "name": "Code", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 1160, + 520 + ] + }, + { + "parameters": { + "operation": "removeDuplicates" + }, + "id": "90acc956-989f-4008-a3df-fe0162762b24", + "name": "Remove duplicates", + "type": "n8n-nodes-base.itemLists", + "typeVersion": 3.1, + "position": [ + 1440, + 160 + ] + }, + { + "parameters": { + "operation": "removeDuplicates", + "compare": "selectedFields", + "fieldsToCompare": "mixed", + "options": {} + }, + "id": "6c39a6bb-b042-4d4c-828c-52699f178828", + "name": "Remove duplicates by mixed", + "type": "n8n-nodes-base.itemLists", + "typeVersion": 3.1, + "position": [ + 1440, + 340 + ] + }, + { + "parameters": { + "operation": "removeDuplicates", + "compare": "selectedFields", + "fieldsToCompare": "match", + "options": {} + }, + "id": "d3783be2-6705-44d3-b481-72e7a9dba458", + "name": "Remove duplicates by match", + "type": "n8n-nodes-base.itemLists", + "typeVersion": 3.1, + "position": [ + 1440, + 520 + ] + }, + { + "parameters": { + "operation": "removeDuplicates", + "compare": "allFieldsExcept", + "fieldsToExclude": "mixed", + "options": {} + }, + "id": "e634e39d-0ce6-477f-97cc-69eec2dd981b", + "name": "Remove duplicates except by mixed", + "type": "n8n-nodes-base.itemLists", + "typeVersion": 3.1, + "position": [ + 1440, + 720 + ] + }, + { + "parameters": { + "operation": "removeDuplicates", + "compare": "allFieldsExcept", + "fieldsToExclude": "match", + "options": {} + }, + "id": "0a1bbc9a-3505-413c-ae86-c60dc60b5909", + "name": "Remove duplicates except by match", + "type": "n8n-nodes-base.itemLists", + "typeVersion": 3.1, + "position": [ + 1440, + 940 + ] + } + ], + "pinData": { + "Remove duplicates": [ + { + "json": { + "mixed": "1", + "match": "foo" + } + }, + { + "json": { + "mixed": 1, + "match": "foo" + } + }, + { + "json": { + "mixed": true, + "match": "foo" + } + }, + { + "json": { + "mixed": false, + "match": "foo" + } + }, + { + "json": { + "mixed": {}, + "match": "foo" + } + }, + { + "json": { + "mixed": [], + "match": "foo" + } + } + ], + "Remove duplicates by mixed": [ + { + "json": { + "mixed": "1", + "match": "foo" + } + }, + { + "json": { + "mixed": 1, + "match": "foo" + } + }, + { + "json": { + "mixed": true, + "match": "foo" + } + }, + { + "json": { + "mixed": false, + "match": "foo" + } + }, + { + "json": { + "mixed": {}, + "match": "foo" + } + }, + { + "json": { + "mixed": [], + "match": "foo" + } + } + ], + "Remove duplicates by match": [ + { + "json": { + "mixed": "1", + "match": "foo" + } + } + ], + "Remove duplicates except by mixed": [ + { + "json": { + "mixed": "1", + "match": "foo" + } + } + ], + "Remove duplicates except by match": [ + { + "json": { + "mixed": "1", + "match": "foo" + } + }, + { + "json": { + "mixed": 1, + "match": "foo" + } + }, + { + "json": { + "mixed": true, + "match": "foo" + } + }, + { + "json": { + "mixed": false, + "match": "foo" + } + }, + { + "json": { + "mixed": {}, + "match": "foo" + } + }, + { + "json": { + "mixed": [], + "match": "foo" + } + } + ] + }, + "connections": { + "When clicking \"Execute Workflow\"": { + "main": [ + [ + { + "node": "Code", + "type": "main", + "index": 0 + } + ] + ] + }, + "Code": { + "main": [ + [ + { + "node": "Remove duplicates", + "type": "main", + "index": 0 + }, + { + "node": "Remove duplicates by mixed", + "type": "main", + "index": 0 + }, + { + "node": "Remove duplicates by match", + "type": "main", + "index": 0 + }, + { + "node": "Remove duplicates except by mixed", + "type": "main", + "index": 0 + }, + { + "node": "Remove duplicates except by match", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "active": false, + "settings": { + "executionOrder": "v1" + }, + "versionId": "c657bc6f-02bc-4e1b-b49a-d1dca8b13256", + "id": "cHSnZsTtYIJj3gL2", + "meta": { + "instanceId": "104a4d08d8897b8bdeb38aaca515021075e0bd8544c983c2bb8c86e6a8e6081c" + }, + "tags": [] +} \ No newline at end of file