refactor(core): Extract duplicate utility functions and add unit tests (no-changelog) (#9814)

packages/nodes-base/jest.config.js

@@ -2,5 +2,9 @@
 module.exports = {
 	...require('../../jest.config'),
 	collectCoverageFrom: ['credentials/**/*.ts', 'nodes/**/*.ts', 'utils/**/*.ts'],
-	setupFilesAfterEnv: ['jest-expect-message', '<rootDir>/test/setup.ts'],
+	setupFilesAfterEnv: [
+		'jest-expect-message',
+		'n8n-workflow/test/setup.ts',
+		'<rootDir>/test/setup.ts',
+	],
 };

packages/nodes-base/nodes/Html/Html.node.ts

@@ -12,7 +12,7 @@ import get from 'lodash/get';
 import { placeholder } from './placeholder';
 import { getValue } from './utils';
 import type { IValueData } from './types';
-import { getResolvables, sanitazeDataPathKey } from '@utils/utilities';
+import { getResolvables, sanitizeDataPathKey } from '@utils/utilities';
 
 export const capitalizeHeader = (header: string, capitalize?: boolean) => {
 	if (!capitalize) return header;
@@ -516,7 +516,7 @@ export class Html implements INodeType {
 					let htmlArray: string[] | string = [];
 					if (sourceData === 'json') {
 						if (nodeVersion === 1) {
-							const key = sanitazeDataPathKey(item.json, dataPropertyName);
+							const key = sanitizeDataPathKey(item.json, dataPropertyName);
 							if (item.json[key] === undefined) {
 								throw new NodeOperationError(
 									this.getNode(),

packages/nodes-base/nodes/ItemLists/V1/ItemListsV1.node.ts

@@ -1,62 +1,22 @@
 import type {
 	IDataObject,
 	IExecuteFunctions,
-	INode,
 	INodeExecutionData,
 	INodeType,
 	INodeTypeBaseDescription,
 	INodeTypeDescription,
 } from 'n8n-workflow';
-import { NodeOperationError, randomInt } from 'n8n-workflow';
+import { NodeOperationError } from 'n8n-workflow';
 
 import get from 'lodash/get';
 import isEmpty from 'lodash/isEmpty';
 import isEqual from 'lodash/isEqual';
-import isObject from 'lodash/isObject';
 import lt from 'lodash/lt';
-import merge from 'lodash/merge';
 import pick from 'lodash/pick';
-import reduce from 'lodash/reduce';
 import set from 'lodash/set';
 import unset from 'lodash/unset';
 
-const compareItems = (
+import { flattenKeys, shuffleArray, compareItems } from '@utils/utilities';
-	obj: INodeExecutionData,
-	obj2: INodeExecutionData,
-	keys: string[],
-	disableDotNotation: boolean,
-	_node: INode,
-) => {
-	let result = true;
-	for (const key of keys) {
-		if (!disableDotNotation) {
-			if (!isEqual(get(obj.json, key), get(obj2.json, key))) {
-				result = false;
-				break;
-			}
-		} else {
-			if (!isEqual(obj.json[key], obj2.json[key])) {
-				result = false;
-				break;
-			}
-		}
-	}
-	return result;
-};
-
-const flattenKeys = (obj: IDataObject, path: string[] = []): IDataObject => {
-	return !isObject(obj)
-		? { [path.join('.')]: obj }
-		: reduce(obj, (cum, next, key) => merge(cum, flattenKeys(next as IDataObject, [...path, key])), {}); //prettier-ignore
-};
-
-const shuffleArray = (array: any[]) => {
-	for (let i = array.length - 1; i > 0; i--) {
-		const j = randomInt(i + 1);
-		[array[i], array[j]] = [array[j], array[i]];
-	}
-};
-
 import { sortByCode } from '../V3/helpers/utils';
 import * as summarize from './summarize.operation';
 
@@ -1226,7 +1186,7 @@ return 0;`,
 				const removedIndexes: number[] = [];
 				let temp = newItems[0];
 				for (let index = 1; index < newItems.length; index++) {
-					if (compareItems(newItems[index], temp, keys, disableDotNotation, this.getNode())) {
+					if (compareItems(newItems[index], temp, keys, disableDotNotation)) {
 						removedIndexes.push(newItems[index].json.__INDEX as unknown as number);
 					} else {
 						temp = newItems[index];

packages/nodes-base/nodes/ItemLists/V2/ItemListsV2.node.ts

@@ -1,63 +1,23 @@
 import type {
 	IDataObject,
 	IExecuteFunctions,
-	INode,
 	INodeExecutionData,
 	INodeType,
 	INodeTypeBaseDescription,
 	INodeTypeDescription,
 	IPairedItemData,
 } from 'n8n-workflow';
-import { NodeOperationError, deepCopy, randomInt } from 'n8n-workflow';
+import { NodeOperationError, deepCopy } from 'n8n-workflow';
 
 import get from 'lodash/get';
 import isEmpty from 'lodash/isEmpty';
 import isEqual from 'lodash/isEqual';
-import isObject from 'lodash/isObject';
 import lt from 'lodash/lt';
-import merge from 'lodash/merge';
 import pick from 'lodash/pick';
-import reduce from 'lodash/reduce';
 import set from 'lodash/set';
 import unset from 'lodash/unset';
 
-const compareItems = (
+import { flattenKeys, shuffleArray, compareItems } from '@utils/utilities';
-	obj: INodeExecutionData,
-	obj2: INodeExecutionData,
-	keys: string[],
-	disableDotNotation: boolean,
-	_node: INode,
-) => {
-	let result = true;
-	for (const key of keys) {
-		if (!disableDotNotation) {
-			if (!isEqual(get(obj.json, key), get(obj2.json, key))) {
-				result = false;
-				break;
-			}
-		} else {
-			if (!isEqual(obj.json[key], obj2.json[key])) {
-				result = false;
-				break;
-			}
-		}
-	}
-	return result;
-};
-
-const flattenKeys = (obj: IDataObject, path: string[] = []): IDataObject => {
-	return !isObject(obj)
-		? { [path.join('.')]: obj }
-		: reduce(obj, (cum, next, key) => merge(cum, flattenKeys(next as IDataObject, [...path, key])), {}); //prettier-ignore
-};
-
-const shuffleArray = (array: any[]) => {
-	for (let i = array.length - 1; i > 0; i--) {
-		const j = randomInt(i + 1);
-		[array[i], array[j]] = [array[j], array[i]];
-	}
-};
-
 import { sortByCode } from '../V3/helpers/utils';
 import * as summarize from './summarize.operation';
 
@@ -1273,7 +1233,7 @@ return 0;`,
 				const removedIndexes: number[] = [];
 				let temp = newItems[0];
 				for (let index = 1; index < newItems.length; index++) {
-					if (compareItems(newItems[index], temp, keys, disableDotNotation, this.getNode())) {
+					if (compareItems(newItems[index], temp, keys, disableDotNotation)) {
 						removedIndexes.push(newItems[index].json.__INDEX as unknown as number);
 					} else {
 						temp = newItems[index];

packages/nodes-base/nodes/ItemLists/V3/actions/itemList/removeDuplicates.operation.ts

@@ -6,9 +6,9 @@ import isEqual from 'lodash/isEqual';
 import lt from 'lodash/lt';
 import pick from 'lodash/pick';
 
-import { compareItems, flattenKeys, prepareFieldsArray, typeToNumber } from '../../helpers/utils';
+import { compareItems, flattenKeys, updateDisplayOptions } from '@utils/utilities';
+import { prepareFieldsArray, typeToNumber } from '../../helpers/utils';
 import { disableDotNotationBoolean } from '../common.descriptions';
-import { updateDisplayOptions } from '@utils/utilities';
 
 const properties: INodeProperties[] = [
 	{
@@ -229,7 +229,7 @@ export async function execute(
 	const removedIndexes: number[] = [];
 	let temp = newItems[0];
 	for (let index = 1; index < newItems.length; index++) {
-		if (compareItems(newItems[index], temp, keys, disableDotNotation, this.getNode())) {
+		if (compareItems(newItems[index], temp, keys, disableDotNotation)) {
 			removedIndexes.push(newItems[index].json.__INDEX as unknown as number);
 		} else {
 			temp = newItems[index];

packages/nodes-base/nodes/ItemLists/V3/actions/itemList/sort.operation.ts

@@ -11,9 +11,9 @@ import get from 'lodash/get';
 import isEqual from 'lodash/isEqual';
 import lt from 'lodash/lt';
 
-import { shuffleArray, sortByCode } from '../../helpers/utils';
+import { sortByCode } from '../../helpers/utils';
 import { disableDotNotationBoolean } from '../common.descriptions';
-import { updateDisplayOptions } from '@utils/utilities';
+import { shuffleArray, updateDisplayOptions } from '@utils/utilities';
 
 const properties: INodeProperties[] = [
 	{

packages/nodes-base/nodes/ItemLists/V3/helpers/utils.ts

@@ -1,56 +1,11 @@
 import { NodeVM } from '@n8n/vm2';
 import type {
-	IDataObject,
 	IExecuteFunctions,
 	IBinaryData,
-	INode,
 	INodeExecutionData,
 	GenericValue,
 } from 'n8n-workflow';
-import { ApplicationError, NodeOperationError, randomInt } from 'n8n-workflow';
+import { ApplicationError, NodeOperationError } from 'n8n-workflow';
-
-import get from 'lodash/get';
-import isEqual from 'lodash/isEqual';
-import isObject from 'lodash/isObject';
-import merge from 'lodash/merge';
-import reduce from 'lodash/reduce';
-
-export const compareItems = (
-	obj: INodeExecutionData,
-	obj2: INodeExecutionData,
-	keys: string[],
-	disableDotNotation: boolean,
-	_node: INode,
-) => {
-	let result = true;
-	for (const key of keys) {
-		if (!disableDotNotation) {
-			if (!isEqual(get(obj.json, key), get(obj2.json, key))) {
-				result = false;
-				break;
-			}
-		} else {
-			if (!isEqual(obj.json[key], obj2.json[key])) {
-				result = false;
-				break;
-			}
-		}
-	}
-	return result;
-};
-
-export const flattenKeys = (obj: IDataObject, path: string[] = []): IDataObject => {
-	return !isObject(obj)
-		? { [path.join('.')]: obj }
-		: reduce(obj, (cum, next, key) => merge(cum, flattenKeys(next as IDataObject, [...path, key])), {}); //prettier-ignore
-};
-
-export const shuffleArray = (array: any[]) => {
-	for (let i = array.length - 1; i > 0; i--) {
-		const j = randomInt(i + 1);
-		[array[i], array[j]] = [array[j], array[i]];
-	}
-};
 
 export const prepareFieldsArray = (fields: string | string[], fieldName = 'Fields') => {
 	if (typeof fields === 'string') {

packages/nodes-base/nodes/Set/v2/helpers/utils.ts

@@ -18,7 +18,7 @@ import get from 'lodash/get';
 import set from 'lodash/set';
 import unset from 'lodash/unset';
 
-import { getResolvables, sanitazeDataPathKey } from '../../../../utils/utilities';
+import { getResolvables, sanitizeDataPathKey } from '../../../../utils/utilities';
 import type { SetNodeOptions } from './interfaces';
 import { INCLUDE } from './interfaces';
 
@@ -38,13 +38,13 @@ const configureFieldHelper = (dotNotation?: boolean) => {
 	} else {
 		return {
 			set: (item: IDataObject, key: string, value: IDataObject) => {
-				item[sanitazeDataPathKey(item, key)] = value;
+				item[sanitizeDataPathKey(item, key)] = value;
 			},
 			get: (item: IDataObject, key: string) => {
-				return item[sanitazeDataPathKey(item, key)];
+				return item[sanitizeDataPathKey(item, key)];
 			},
 			unset: (item: IDataObject, key: string) => {
-				delete item[sanitazeDataPathKey(item, key)];
+				delete item[sanitizeDataPathKey(item, key)];
 			},
 		};
 	}

packages/nodes-base/nodes/Transform/RemoveDuplicates/RemoveDuplicates.node.ts

@@ -10,7 +10,8 @@ import {
 	type INodeTypeDescription,
 } from 'n8n-workflow';
 import { prepareFieldsArray } from '../utils/utils';
-import { compareItems, flattenKeys, validateInputData } from './utils';
+import { validateInputData } from './utils';
+import { compareItems, flattenKeys } from '@utils/utilities';
 
 export class RemoveDuplicates implements INodeType {
 	description: INodeTypeDescription = {
@@ -211,7 +212,7 @@ export class RemoveDuplicates implements INodeType {
 		const removedIndexes: number[] = [];
 		let temp = newItems[0];
 		for (let index = 1; index < newItems.length; index++) {
-			if (compareItems(newItems[index], temp, keys, disableDotNotation, this.getNode())) {
+			if (compareItems(newItems[index], temp, keys, disableDotNotation)) {
 				removedIndexes.push(newItems[index].json.__INDEX as unknown as number);
 			} else {
 				temp = newItems[index];

packages/nodes-base/nodes/Transform/RemoveDuplicates/utils.ts

@@ -1,44 +1,5 @@
 import get from 'lodash/get';
-import isEqual from 'lodash/isEqual';
+import { NodeOperationError, type INode, type INodeExecutionData } from 'n8n-workflow';
-import isObject from 'lodash/isObject';
-import merge from 'lodash/merge';
-import reduce from 'lodash/reduce';
-import {
-	NodeOperationError,
-	type IDataObject,
-	type INode,
-	type INodeExecutionData,
-} from 'n8n-workflow';
-
-export const compareItems = (
-	obj: INodeExecutionData,
-	obj2: INodeExecutionData,
-	keys: string[],
-	disableDotNotation: boolean,
-	_node: INode,
-) => {
-	let result = true;
-	for (const key of keys) {
-		if (!disableDotNotation) {
-			if (!isEqual(get(obj.json, key), get(obj2.json, key))) {
-				result = false;
-				break;
-			}
-		} else {
-			if (!isEqual(obj.json[key], obj2.json[key])) {
-				result = false;
-				break;
-			}
-		}
-	}
-	return result;
-};
-
-export const flattenKeys = (obj: IDataObject, path: string[] = []): IDataObject => {
-	return !isObject(obj)
-		? { [path.join('.')]: obj }
-		: reduce(obj, (cum, next, key) => merge(cum, flattenKeys(next as IDataObject, [...path, key])), {}); //prettier-ignore
-};
 
 export const validateInputData = (
 	node: INode,

packages/nodes-base/nodes/Transform/Sort/Sort.node.ts

@@ -9,7 +9,8 @@ import {
 	type INodeType,
 	type INodeTypeDescription,
 } from 'n8n-workflow';
-import { shuffleArray, sortByCode } from './utils';
+import { sortByCode } from './utils';
+import { shuffleArray } from '@utils/utilities';
 
 export class Sort implements INodeType {
 	description: INodeTypeDescription = {

packages/nodes-base/nodes/Transform/Sort/utils.ts

@@ -1,13 +1,6 @@
 import { NodeVM } from '@n8n/vm2';
 import type { IExecuteFunctions, INodeExecutionData } from 'n8n-workflow';
-import { NodeOperationError, randomInt } from 'n8n-workflow';
+import { NodeOperationError } from 'n8n-workflow';
-
-export const shuffleArray = (array: any[]) => {
-	for (let i = array.length - 1; i > 0; i--) {
-		const j = randomInt(i + 1);
-		[array[i], array[j]] = [array[j], array[i]];
-	}
-};
 
 const returnRegExp = /\breturn\b/g;
 export function sortByCode(

packages/nodes-base/test/utils/utilities.test.ts

@@ -1,4 +1,12 @@
-import { fuzzyCompare, getResolvables, keysToLowercase, wrapData } from '@utils/utilities';
+import {
+	compareItems,
+	flattenKeys,
+	fuzzyCompare,
+	getResolvables,
+	keysToLowercase,
+	shuffleArray,
+	wrapData,
+} from '@utils/utilities';
 
 //most test cases for fuzzyCompare are done in Compare Datasets node tests
 describe('Test fuzzyCompare', () => {
@@ -137,3 +145,110 @@ describe('Test getResolvables', () => {
 		]);
 	});
 });
+
+describe('shuffleArray', () => {
+	it('should shuffle array', () => {
+		const array = [1, 2, 3, 4, 5];
+		const toShuffle = [...array];
+		shuffleArray(toShuffle);
+		expect(toShuffle).not.toEqual(array);
+		expect(toShuffle).toHaveLength(array.length);
+		expect(toShuffle).toEqual(expect.arrayContaining(array));
+	});
+});
+
+describe('flattenKeys', () => {
+	const name = 'Lisa';
+	const city1 = 'Berlin';
+	const city2 = 'Schoenwald';
+	const withNestedObject = {
+		name,
+		address: { city: city1 },
+	};
+
+	const withNestedArrays = {
+		name,
+		addresses: [{ city: city1 }, { city: city2 }],
+	};
+
+	it('should handle empty object', () => {
+		const flattenedObj = flattenKeys({});
+		expect(flattenedObj).toEqual({});
+	});
+
+	it('should flatten object with nested object', () => {
+		const flattenedObj = flattenKeys(withNestedObject);
+		expect(flattenedObj).toEqual({
+			name,
+			'address.city': city1,
+		});
+	});
+
+	it('should handle object with nested arrays', () => {
+		const flattenedObj = flattenKeys(withNestedArrays);
+		expect(flattenedObj).toEqual({
+			name,
+			'addresses.0.city': city1,
+			'addresses.1.city': city2,
+		});
+	});
+
+	it('should flatten object with nested object and specified prefix', () => {
+		const flattenedObj = flattenKeys(withNestedObject, ['test']);
+		expect(flattenedObj).toEqual({
+			'test.name': name,
+			'test.address.city': city1,
+		});
+	});
+
+	it('should handle object with nested arrays and specified prefix', () => {
+		const flattenedObj = flattenKeys(withNestedArrays, ['test']);
+		expect(flattenedObj).toEqual({
+			'test.name': name,
+			'test.addresses.0.city': city1,
+			'test.addresses.1.city': city2,
+		});
+	});
+});
+
+describe('compareItems', () => {
+	it('should return true if all values of specified keys are equal', () => {
+		const obj1 = { json: { a: 1, b: 2, c: 3 } };
+		const obj2 = { json: { a: 1, b: 2, c: 3 } };
+		const keys = ['a', 'b', 'c'];
+		const result = compareItems(obj1, obj2, keys);
+		expect(result).toBe(true);
+	});
+
+	it('should return false if any values of specified keys are not equal', () => {
+		const obj1 = { json: { a: 1, b: 2, c: 3 } };
+		const obj2 = { json: { a: 1, b: 2, c: 4 } };
+		const keys = ['a', 'b', 'c'];
+		const result = compareItems(obj1, obj2, keys);
+		expect(result).toBe(false);
+	});
+
+	it('should return true if all values of specified keys are equal using dot notation', () => {
+		const obj1 = { json: { a: { b: { c: 1 } } } };
+		const obj2 = { json: { a: { b: { c: 1 } } } };
+		const keys = ['a.b.c'];
+		const result = compareItems(obj1, obj2, keys);
+		expect(result).toBe(true);
+	});
+
+	it('should return false if any values of specified keys are not equal using dot notation', () => {
+		const obj1 = { json: { a: { b: { c: 1 } } } };
+		const obj2 = { json: { a: { b: { c: 2 } } } };
+		const keys = ['a.b.c'];
+		const result = compareItems(obj1, obj2, keys);
+		expect(result).toBe(false);
+	});
+
+	it('should return true if all values of specified keys are equal using bracket notation', () => {
+		const obj1 = { json: { 'a.b': { 'c.d': 1 } } };
+		const obj2 = { json: { 'a.b': { 'c.d': 1 } } };
+		const keys = ['a.b.c.d'];
+		const result = compareItems(obj1, obj2, keys, true);
+		expect(result).toBe(true);
+	});
+});

packages/nodes-base/utils/utilities.ts

@@ -6,9 +6,9 @@ import type {
 	IPairedItemData,
 } from 'n8n-workflow';
 
-import { ApplicationError, jsonParse } from 'n8n-workflow';
+import { ApplicationError, jsonParse, randomInt } from 'n8n-workflow';
 
-import { isEqual, isNull, merge } from 'lodash';
+import { isEqual, isNull, merge, isObject, reduce, get } from 'lodash';
 
 /**
  * Creates an array of elements split into groups the length of `size`.
@@ -41,6 +41,32 @@ export function chunk<T>(array: T[], size = 1) {
 	return result as T[][];
 }
 
+/**
+ * Shuffles an array in place using the Fisher-Yates shuffle algorithm
+ * @param {Array} array The array to shuffle.
+ */
+export const shuffleArray = <T>(array: T[]): void => {
+	for (let i = array.length - 1; i > 0; i--) {
+		const j = randomInt(i + 1);
+		[array[i], array[j]] = [array[j], array[i]];
+	}
+};
+
+/**
+ * Flattens an object with deep data
+ * @param {IDataObject} data The object to flatten
+ * @param {string[]} prefix The prefix to add to each key in the returned flat object
+ */
+export const flattenKeys = (obj: IDataObject, prefix: string[] = []): IDataObject => {
+	return !isObject(obj)
+		? { [prefix.join('.')]: obj }
+		: reduce(
+				obj,
+				(cum, next, key) => merge(cum, flattenKeys(next as IDataObject, [...prefix, key])),
+				{},
+			);
+};
+
 /**
  * Takes a multidimensional array and converts it to a one-dimensional array.
  *
@@ -70,6 +96,38 @@ export function flatten<T>(nestedArray: T[][]) {
 	return result as any;
 }
 
+/**
+ * Compares the values of specified keys in two objects.
+ *
+ * @param {T} obj1 - The first object to compare.
+ * @param {T} obj2 - The second object to compare.
+ * @param {string[]} keys - An array of keys to compare.
+ * @param {boolean} disableDotNotation - Whether to use dot notation to access nested properties.
+ * @returns {boolean} - Whether the values of the specified keys are equal in both objects.
+ */
+export const compareItems = <T extends { json: Record<string, unknown> }>(
+	obj1: T,
+	obj2: T,
+	keys: string[],
+	disableDotNotation: boolean = false,
+): boolean => {
+	let result = true;
+	for (const key of keys) {
+		if (!disableDotNotation) {
+			if (!isEqual(get(obj1.json, key), get(obj2.json, key))) {
+				result = false;
+				break;
+			}
+		} else {
+			if (!isEqual(obj1.json[key], obj2.json[key])) {
+				result = false;
+				break;
+			}
+		}
+	}
+	return result;
+};
+
 export function updateDisplayOptions(
 	displayOptions: IDisplayOptions,
 	properties: INodeProperties[],
@@ -330,7 +388,7 @@ export function preparePairedItemDataArray(
 	return [pairedItem];
 }
 
-export const sanitazeDataPathKey = (item: IDataObject, key: string) => {
+export const sanitizeDataPathKey = (item: IDataObject, key: string) => {
 	if (item[key] !== undefined) {
 		return key;
 	}