mudhorn/n8n
1
0
Fork 0
mirror of https://github.com/n8n-io/n8n.git synced 2024-11-09 22:24:05 -08:00
Code Issues Actions Packages Projects Releases Wiki Activity

refactor(core): Extract duplicate utility functions and add unit tests (no-changelog) (#9814)

Browse source
This commit is contained in:
कारतोफ्फेलस्क्रिप्ट™ 2024-06-20 12:09:23 +02:00 committed by GitHub
parent 283d1ca583
commit e4463c62b4
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 207 additions and 199 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
packages/nodes-base/jest.config.js
View file

@ -2,5 +2,9 @@
module.exports = {
...require('../../jest.config'),
collectCoverageFrom: ['credentials/**/*.ts', 'nodes/**/*.ts', 'utils/**/*.ts'],
setupFilesAfterEnv: ['jest-expect-message', '<rootDir>/test/setup.ts'],
setupFilesAfterEnv: [
'jest-expect-message',
'n8n-workflow/test/setup.ts',
'<rootDir>/test/setup.ts',
],
};

4
packages/nodes-base/nodes/Html/Html.node.ts
View file

@ -12,7 +12,7 @@ import get from 'lodash/get';
import { placeholder } from './placeholder';
import { getValue } from './utils';
import type { IValueData } from './types';
import { getResolvables, sanitazeDataPathKey } from '@utils/utilities';
import { getResolvables, sanitizeDataPathKey } from '@utils/utilities';
export const capitalizeHeader = (header: string, capitalize?: boolean) => {
if (!capitalize) return header;
@ -516,7 +516,7 @@ export class Html implements INodeType {
let htmlArray: string[] | string = [];
if (sourceData === 'json') {
if (nodeVersion === 1) {
const key = sanitazeDataPathKey(item.json, dataPropertyName);
const key = sanitizeDataPathKey(item.json, dataPropertyName);
if (item.json[key] === undefined) {
throw new NodeOperationError(
this.getNode(),

46
packages/nodes-base/nodes/ItemLists/V1/ItemListsV1.node.ts
View file

@ -1,62 +1,22 @@
import type {
IDataObject,
IExecuteFunctions,
INode,
INodeExecutionData,
INodeType,
INodeTypeBaseDescription,
INodeTypeDescription,
} from 'n8n-workflow';
import { NodeOperationError, randomInt } from 'n8n-workflow';
import { NodeOperationError } from 'n8n-workflow';
import get from 'lodash/get';
import isEmpty from 'lodash/isEmpty';
import isEqual from 'lodash/isEqual';
import isObject from 'lodash/isObject';
import lt from 'lodash/lt';
import merge from 'lodash/merge';
import pick from 'lodash/pick';
import reduce from 'lodash/reduce';
import set from 'lodash/set';
import unset from 'lodash/unset';
const compareItems = (
obj: INodeExecutionData,
obj2: INodeExecutionData,
keys: string[],
disableDotNotation: boolean,
_node: INode,
) => {
let result = true;
for (const key of keys) {
if (!disableDotNotation) {
if (!isEqual(get(obj.json, key), get(obj2.json, key))) {
result = false;
break;
}
} else {
if (!isEqual(obj.json[key], obj2.json[key])) {
result = false;
break;
}
}
}
return result;
};
const flattenKeys = (obj: IDataObject, path: string[] = []): IDataObject => {
return !isObject(obj)
? { [path.join('.')]: obj }
: reduce(obj, (cum, next, key) => merge(cum, flattenKeys(next as IDataObject, [...path, key])), {}); //prettier-ignore
};
const shuffleArray = (array: any[]) => {
for (let i = array.length - 1; i > 0; i--) {
const j = randomInt(i + 1);
[array[i], array[j]] = [array[j], array[i]];
}
};
import { flattenKeys, shuffleArray, compareItems } from '@utils/utilities';
import { sortByCode } from '../V3/helpers/utils';
import * as summarize from './summarize.operation';
@ -1226,7 +1186,7 @@ return 0;`,
const removedIndexes: number[] = [];
let temp = newItems[0];
for (let index = 1; index < newItems.length; index++) {
if (compareItems(newItems[index], temp, keys, disableDotNotation, this.getNode())) {
if (compareItems(newItems[index], temp, keys, disableDotNotation)) {
removedIndexes.push(newItems[index].json.__INDEX as unknown as number);
} else {
temp = newItems[index];

46
packages/nodes-base/nodes/ItemLists/V2/ItemListsV2.node.ts
View file

@ -1,63 +1,23 @@
import type {
IDataObject,
IExecuteFunctions,
INode,
INodeExecutionData,
INodeType,
INodeTypeBaseDescription,
INodeTypeDescription,
IPairedItemData,
} from 'n8n-workflow';
import { NodeOperationError, deepCopy, randomInt } from 'n8n-workflow';
import { NodeOperationError, deepCopy } from 'n8n-workflow';
import get from 'lodash/get';
import isEmpty from 'lodash/isEmpty';
import isEqual from 'lodash/isEqual';
import isObject from 'lodash/isObject';
import lt from 'lodash/lt';
import merge from 'lodash/merge';
import pick from 'lodash/pick';
import reduce from 'lodash/reduce';
import set from 'lodash/set';
import unset from 'lodash/unset';
const compareItems = (
obj: INodeExecutionData,
obj2: INodeExecutionData,
keys: string[],
disableDotNotation: boolean,
_node: INode,
) => {
let result = true;
for (const key of keys) {
if (!disableDotNotation) {
if (!isEqual(get(obj.json, key), get(obj2.json, key))) {
result = false;
break;
}
} else {
if (!isEqual(obj.json[key], obj2.json[key])) {
result = false;
break;
}
}
}
return result;
};
const flattenKeys = (obj: IDataObject, path: string[] = []): IDataObject => {
return !isObject(obj)
? { [path.join('.')]: obj }
: reduce(obj, (cum, next, key) => merge(cum, flattenKeys(next as IDataObject, [...path, key])), {}); //prettier-ignore
};
const shuffleArray = (array: any[]) => {
for (let i = array.length - 1; i > 0; i--) {
const j = randomInt(i + 1);
[array[i], array[j]] = [array[j], array[i]];
}
};
import { flattenKeys, shuffleArray, compareItems } from '@utils/utilities';
import { sortByCode } from '../V3/helpers/utils';
import * as summarize from './summarize.operation';
@ -1273,7 +1233,7 @@ return 0;`,
const removedIndexes: number[] = [];
let temp = newItems[0];
for (let index = 1; index < newItems.length; index++) {
if (compareItems(newItems[index], temp, keys, disableDotNotation, this.getNode())) {
if (compareItems(newItems[index], temp, keys, disableDotNotation)) {
removedIndexes.push(newItems[index].json.__INDEX as unknown as number);
} else {
temp = newItems[index];

6
packages/nodes-base/nodes/ItemLists/V3/actions/itemList/removeDuplicates.operation.ts
View file

@ -6,9 +6,9 @@ import isEqual from 'lodash/isEqual';
import lt from 'lodash/lt';
import pick from 'lodash/pick';
import { compareItems, flattenKeys, prepareFieldsArray, typeToNumber } from '../../helpers/utils';
import { compareItems, flattenKeys, updateDisplayOptions } from '@utils/utilities';
import { prepareFieldsArray, typeToNumber } from '../../helpers/utils';
import { disableDotNotationBoolean } from '../common.descriptions';
import { updateDisplayOptions } from '@utils/utilities';
const properties: INodeProperties[] = [
{
@ -229,7 +229,7 @@ export async function execute(
const removedIndexes: number[] = [];
let temp = newItems[0];
for (let index = 1; index < newItems.length; index++) {
if (compareItems(newItems[index], temp, keys, disableDotNotation, this.getNode())) {
if (compareItems(newItems[index], temp, keys, disableDotNotation)) {
removedIndexes.push(newItems[index].json.__INDEX as unknown as number);
} else {
temp = newItems[index];

4
packages/nodes-base/nodes/ItemLists/V3/actions/itemList/sort.operation.ts
View file

@ -11,9 +11,9 @@ import get from 'lodash/get';
import isEqual from 'lodash/isEqual';
import lt from 'lodash/lt';
import { shuffleArray, sortByCode } from '../../helpers/utils';
import { sortByCode } from '../../helpers/utils';
import { disableDotNotationBoolean } from '../common.descriptions';
import { updateDisplayOptions } from '@utils/utilities';
import { shuffleArray, updateDisplayOptions } from '@utils/utilities';
const properties: INodeProperties[] = [
{

47
packages/nodes-base/nodes/ItemLists/V3/helpers/utils.ts
View file

@ -1,56 +1,11 @@
import { NodeVM } from '@n8n/vm2';
import type {
IDataObject,
IExecuteFunctions,
IBinaryData,
INode,
INodeExecutionData,
GenericValue,
} from 'n8n-workflow';
import { ApplicationError, NodeOperationError, randomInt } from 'n8n-workflow';
import get from 'lodash/get';
import isEqual from 'lodash/isEqual';
import isObject from 'lodash/isObject';
import merge from 'lodash/merge';
import reduce from 'lodash/reduce';
export const compareItems = (
obj: INodeExecutionData,
obj2: INodeExecutionData,
keys: string[],
disableDotNotation: boolean,
_node: INode,
) => {
let result = true;
for (const key of keys) {
if (!disableDotNotation) {
if (!isEqual(get(obj.json, key), get(obj2.json, key))) {
result = false;
break;
}
} else {
if (!isEqual(obj.json[key], obj2.json[key])) {
result = false;
break;
}
}
}
return result;
};
export const flattenKeys = (obj: IDataObject, path: string[] = []): IDataObject => {
return !isObject(obj)
? { [path.join('.')]: obj }
: reduce(obj, (cum, next, key) => merge(cum, flattenKeys(next as IDataObject, [...path, key])), {}); //prettier-ignore
};
export const shuffleArray = (array: any[]) => {
for (let i = array.length - 1; i > 0; i--) {
const j = randomInt(i + 1);
[array[i], array[j]] = [array[j], array[i]];
}
};
import { ApplicationError, NodeOperationError } from 'n8n-workflow';
export const prepareFieldsArray = (fields: string | string[], fieldName = 'Fields') => {
if (typeof fields === 'string') {

8
packages/nodes-base/nodes/Set/v2/helpers/utils.ts
View file

@ -18,7 +18,7 @@ import get from 'lodash/get';
import set from 'lodash/set';
import unset from 'lodash/unset';
import { getResolvables, sanitazeDataPathKey } from '../../../../utils/utilities';
import { getResolvables, sanitizeDataPathKey } from '../../../../utils/utilities';
import type { SetNodeOptions } from './interfaces';
import { INCLUDE } from './interfaces';
@ -38,13 +38,13 @@ const configureFieldHelper = (dotNotation?: boolean) => {
} else {
return {
set: (item: IDataObject, key: string, value: IDataObject) => {
item[sanitazeDataPathKey(item, key)] = value;
item[sanitizeDataPathKey(item, key)] = value;
},
get: (item: IDataObject, key: string) => {
return item[sanitazeDataPathKey(item, key)];
return item[sanitizeDataPathKey(item, key)];
},
unset: (item: IDataObject, key: string) => {
delete item[sanitazeDataPathKey(item, key)];
delete item[sanitizeDataPathKey(item, key)];
},
};
}

5
packages/nodes-base/nodes/Transform/RemoveDuplicates/RemoveDuplicates.node.ts
View file

@ -10,7 +10,8 @@ import {
type INodeTypeDescription,
} from 'n8n-workflow';
import { prepareFieldsArray } from '../utils/utils';
import { compareItems, flattenKeys, validateInputData } from './utils';
import { validateInputData } from './utils';
import { compareItems, flattenKeys } from '@utils/utilities';
export class RemoveDuplicates implements INodeType {
description: INodeTypeDescription = {
@ -211,7 +212,7 @@ export class RemoveDuplicates implements INodeType {
const removedIndexes: number[] = [];
let temp = newItems[0];
for (let index = 1; index < newItems.length; index++) {
if (compareItems(newItems[index], temp, keys, disableDotNotation, this.getNode())) {
if (compareItems(newItems[index], temp, keys, disableDotNotation)) {
removedIndexes.push(newItems[index].json.__INDEX as unknown as number);
} else {
temp = newItems[index];

41
packages/nodes-base/nodes/Transform/RemoveDuplicates/utils.ts
View file

@ -1,44 +1,5 @@
import get from 'lodash/get';
import isEqual from 'lodash/isEqual';
import isObject from 'lodash/isObject';
import merge from 'lodash/merge';
import reduce from 'lodash/reduce';
import {
NodeOperationError,
type IDataObject,
type INode,
type INodeExecutionData,
} from 'n8n-workflow';
export const compareItems = (
obj: INodeExecutionData,
obj2: INodeExecutionData,
keys: string[],
disableDotNotation: boolean,
_node: INode,
) => {
let result = true;
for (const key of keys) {
if (!disableDotNotation) {
if (!isEqual(get(obj.json, key), get(obj2.json, key))) {
result = false;
break;
}
} else {
if (!isEqual(obj.json[key], obj2.json[key])) {
result = false;
break;
}
}
}
return result;
};
export const flattenKeys = (obj: IDataObject, path: string[] = []): IDataObject => {
return !isObject(obj)
? { [path.join('.')]: obj }
: reduce(obj, (cum, next, key) => merge(cum, flattenKeys(next as IDataObject, [...path, key])), {}); //prettier-ignore
};
import { NodeOperationError, type INode, type INodeExecutionData } from 'n8n-workflow';
export const validateInputData = (
node: INode,

3
packages/nodes-base/nodes/Transform/Sort/Sort.node.ts
View file

@ -9,7 +9,8 @@ import {
type INodeType,
type INodeTypeDescription,
} from 'n8n-workflow';
import { shuffleArray, sortByCode } from './utils';
import { sortByCode } from './utils';
import { shuffleArray } from '@utils/utilities';
export class Sort implements INodeType {
description: INodeTypeDescription = {

9
packages/nodes-base/nodes/Transform/Sort/utils.ts
View file

@ -1,13 +1,6 @@
import { NodeVM } from '@n8n/vm2';
import type { IExecuteFunctions, INodeExecutionData } from 'n8n-workflow';
import { NodeOperationError, randomInt } from 'n8n-workflow';
export const shuffleArray = (array: any[]) => {
for (let i = array.length - 1; i > 0; i--) {
const j = randomInt(i + 1);
[array[i], array[j]] = [array[j], array[i]];
}
};
import { NodeOperationError } from 'n8n-workflow';
const returnRegExp = /\breturn\b/g;
export function sortByCode(

117
packages/nodes-base/test/utils/utilities.test.ts
View file

@ -1,4 +1,12 @@
import { fuzzyCompare, getResolvables, keysToLowercase, wrapData } from '@utils/utilities';
import {
compareItems,
flattenKeys,
fuzzyCompare,
getResolvables,
keysToLowercase,
shuffleArray,
wrapData,
} from '@utils/utilities';
//most test cases for fuzzyCompare are done in Compare Datasets node tests
describe('Test fuzzyCompare', () => {
@ -137,3 +145,110 @@ describe('Test getResolvables', () => {
]);
});
});
describe('shuffleArray', () => {
it('should shuffle array', () => {
const array = [1, 2, 3, 4, 5];
const toShuffle = [...array];
shuffleArray(toShuffle);
expect(toShuffle).not.toEqual(array);
expect(toShuffle).toHaveLength(array.length);
expect(toShuffle).toEqual(expect.arrayContaining(array));
});
});
describe('flattenKeys', () => {
const name = 'Lisa';
const city1 = 'Berlin';
const city2 = 'Schoenwald';
const withNestedObject = {
name,
address: { city: city1 },
};
const withNestedArrays = {
name,
addresses: [{ city: city1 }, { city: city2 }],
};
it('should handle empty object', () => {
const flattenedObj = flattenKeys({});
expect(flattenedObj).toEqual({});
});
it('should flatten object with nested object', () => {
const flattenedObj = flattenKeys(withNestedObject);
expect(flattenedObj).toEqual({
name,
'address.city': city1,
});
});
it('should handle object with nested arrays', () => {
const flattenedObj = flattenKeys(withNestedArrays);
expect(flattenedObj).toEqual({
name,
'addresses.0.city': city1,
'addresses.1.city': city2,
});
});
it('should flatten object with nested object and specified prefix', () => {
const flattenedObj = flattenKeys(withNestedObject, ['test']);
expect(flattenedObj).toEqual({
'test.name': name,
'test.address.city': city1,
});
});
it('should handle object with nested arrays and specified prefix', () => {
const flattenedObj = flattenKeys(withNestedArrays, ['test']);
expect(flattenedObj).toEqual({
'test.name': name,
'test.addresses.0.city': city1,
'test.addresses.1.city': city2,
});
});
});
describe('compareItems', () => {
it('should return true if all values of specified keys are equal', () => {
const obj1 = { json: { a: 1, b: 2, c: 3 } };
const obj2 = { json: { a: 1, b: 2, c: 3 } };
const keys = ['a', 'b', 'c'];
const result = compareItems(obj1, obj2, keys);
expect(result).toBe(true);
});
it('should return false if any values of specified keys are not equal', () => {
const obj1 = { json: { a: 1, b: 2, c: 3 } };
const obj2 = { json: { a: 1, b: 2, c: 4 } };
const keys = ['a', 'b', 'c'];
const result = compareItems(obj1, obj2, keys);
expect(result).toBe(false);
});
it('should return true if all values of specified keys are equal using dot notation', () => {
const obj1 = { json: { a: { b: { c: 1 } } } };
const obj2 = { json: { a: { b: { c: 1 } } } };
const keys = ['a.b.c'];
const result = compareItems(obj1, obj2, keys);
expect(result).toBe(true);
});
it('should return false if any values of specified keys are not equal using dot notation', () => {
const obj1 = { json: { a: { b: { c: 1 } } } };
const obj2 = { json: { a: { b: { c: 2 } } } };
const keys = ['a.b.c'];
const result = compareItems(obj1, obj2, keys);
expect(result).toBe(false);
});
it('should return true if all values of specified keys are equal using bracket notation', () => {
const obj1 = { json: { 'a.b': { 'c.d': 1 } } };
const obj2 = { json: { 'a.b': { 'c.d': 1 } } };
const keys = ['a.b.c.d'];
const result = compareItems(obj1, obj2, keys, true);
expect(result).toBe(true);
});
});

64
packages/nodes-base/utils/utilities.ts
View file

@ -6,9 +6,9 @@ import type {
IPairedItemData,
} from 'n8n-workflow';
import { ApplicationError, jsonParse } from 'n8n-workflow';
import { ApplicationError, jsonParse, randomInt } from 'n8n-workflow';
import { isEqual, isNull, merge } from 'lodash';
import { isEqual, isNull, merge, isObject, reduce, get } from 'lodash';
/**
* Creates an array of elements split into groups the length of `size`.
@ -41,6 +41,32 @@ export function chunk<T>(array: T[], size = 1) {
return result as T[][];
}
/**
* Shuffles an array in place using the Fisher-Yates shuffle algorithm
* @param {Array} array The array to shuffle.
*/
export const shuffleArray = <T>(array: T[]): void => {
for (let i = array.length - 1; i > 0; i--) {
const j = randomInt(i + 1);
[array[i], array[j]] = [array[j], array[i]];
}
};
/**
* Flattens an object with deep data
* @param {IDataObject} data The object to flatten
* @param {string[]} prefix The prefix to add to each key in the returned flat object
*/
export const flattenKeys = (obj: IDataObject, prefix: string[] = []): IDataObject => {
return !isObject(obj)
? { [prefix.join('.')]: obj }
: reduce(
obj,
(cum, next, key) => merge(cum, flattenKeys(next as IDataObject, [...prefix, key])),
{},
);
};
/**
* Takes a multidimensional array and converts it to a one-dimensional array.
*
@ -70,6 +96,38 @@ export function flatten<T>(nestedArray: T[][]) {
return result as any;
}
/**
* Compares the values of specified keys in two objects.
*
* @param {T} obj1 - The first object to compare.
* @param {T} obj2 - The second object to compare.
* @param {string[]} keys - An array of keys to compare.
* @param {boolean} disableDotNotation - Whether to use dot notation to access nested properties.
* @returns {boolean} - Whether the values of the specified keys are equal in both objects.
*/
export const compareItems = <T extends { json: Record<string, unknown> }>(
obj1: T,
obj2: T,
keys: string[],
disableDotNotation: boolean = false,
): boolean => {
let result = true;
for (const key of keys) {
if (!disableDotNotation) {
if (!isEqual(get(obj1.json, key), get(obj2.json, key))) {
result = false;
break;
}
} else {
if (!isEqual(obj1.json[key], obj2.json[key])) {
result = false;
break;
}
}
}
return result;
};
export function updateDisplayOptions(
displayOptions: IDisplayOptions,
properties: INodeProperties[],
@ -330,7 +388,7 @@ export function preparePairedItemDataArray(
return [pairedItem];
}
export const sanitazeDataPathKey = (item: IDataObject, key: string) => {
export const sanitizeDataPathKey = (item: IDataObject, key: string) => {
if (item[key] !== undefined) {
return key;
}