mudhorn/n8n
1
0
Fork 0
mirror of https://github.com/n8n-io/n8n.git synced 2024-11-10 06:34:05 -08:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix(Remove Duplicates Node): Tolerate null fields (#9642)

Browse source
This commit is contained in:
Michael Kret 2024-06-06 16:52:56 +03:00 committed by GitHub
parent ed963011c9
commit a684681ea1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 105 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
packages/nodes-base/nodes/Transform/RemoveDuplicates/RemoveDuplicates.node.ts
View file

@ -10,7 +10,7 @@ import {
type INodeTypeDescription,
} from 'n8n-workflow';
import { prepareFieldsArray } from '../utils/utils';
import { compareItems, flattenKeys } from './utils';
import { compareItems, flattenKeys, validateInputData } from './utils';
export class RemoveDuplicates implements INodeType {
description: INodeTypeDescription = {
@ -19,7 +19,7 @@ export class RemoveDuplicates implements INodeType {
icon: 'file:removeDuplicates.svg',
group: ['transform'],
subtitle: '',
version: 1,
version: [1, 1.1],
description: 'Delete items with matching field values',
defaults: {
name: 'Remove Duplicates',
@ -205,37 +205,7 @@ export class RemoveDuplicates implements INodeType {
return result;
});
for (const key of keys) {
let type: any = undefined;
for (const item of newItems) {
if (key === '') {
throw new NodeOperationError(this.getNode(), 'Name of field to compare is blank');
}
const value = !disableDotNotation ? get(item.json, key) : item.json[key];
if (value === undefined && disableDotNotation && key.includes('.')) {
throw new NodeOperationError(
this.getNode(),
`'${key}' field is missing from some input items`,
{
description:
"If you're trying to use a nested field, make sure you turn off 'disable dot notation' in the node options",
},
);
} else if (value === undefined) {
throw new NodeOperationError(
this.getNode(),
`'${key}' field is missing from some input items`,
);
}
if (type !== undefined && value !== undefined && type !== typeof value) {
throw new NodeOperationError(this.getNode(), `'${key}' isn't always the same type`, {
description: 'The type of this field varies between items',
});
} else {
type = typeof value;
}
}
}
validateInputData(this.getNode(), newItems, keys, disableDotNotation);
// collect the original indexes of items to be removed
const removedIndexes: number[] = [];

57
packages/nodes-base/nodes/Transform/RemoveDuplicates/test/RemoveDuplicates.test.ts
View file

@ -1,5 +1,62 @@
import type { INode } from 'n8n-workflow';
import { validateInputData } from '../utils';
import { testWorkflows, getWorkflowFilenames } from '@test/nodes/Helpers';
const workflows = getWorkflowFilenames(__dirname);
describe('Test Remove Duplicates Node', () => testWorkflows(workflows));
describe('Test Remove Duplicates Node, validateInputData util', () => {
test('Should throw error for version 1', () => {
expect(() =>
validateInputData(
{
name: 'Remove Duplicates',
type: 'n8n-nodes-base.removeDuplicates',
typeVersion: 1,
} as INode,
[
{ json: { country: 'uk' } },
{ json: { country: 'us' } },
{ json: { country: 'uk' } },
{ json: { country: null } },
],
['country'],
false,
),
).toThrow("'country' isn't always the same type");
});
test('Should ignore null values and not throw error for version grater than 1', () => {
expect(() =>
validateInputData(
{
name: 'Remove Duplicates',
type: 'n8n-nodes-base.removeDuplicates',
typeVersion: 1.1,
} as INode,
[
{ json: { country: 'uk' } },
{ json: { country: 'us' } },
{ json: { country: 'uk' } },
{ json: { country: null } },
],
['country'],
false,
),
).not.toThrow();
});
test('Should throw error for different types, version grater than 1', () => {
expect(() =>
validateInputData(
{
name: 'Remove Duplicates',
type: 'n8n-nodes-base.removeDuplicates',
typeVersion: 1.1,
} as INode,
[{ json: { id: 1 } }, { json: { id: '1' } }, { json: { id: 2 } }, { json: { id: null } }],
['id'],
false,
),
).toThrow("'id' isn't always the same type");
});
});

46
packages/nodes-base/nodes/Transform/RemoveDuplicates/utils.ts
View file

@ -3,7 +3,12 @@ import isEqual from 'lodash/isEqual';
import isObject from 'lodash/isObject';
import merge from 'lodash/merge';
import reduce from 'lodash/reduce';
import type { IDataObject, INode, INodeExecutionData } from 'n8n-workflow';
import {
NodeOperationError,
type IDataObject,
type INode,
type INodeExecutionData,
} from 'n8n-workflow';
export const compareItems = (
obj: INodeExecutionData,
@ -34,3 +39,42 @@ export const flattenKeys = (obj: IDataObject, path: string[] = []): IDataObject
? { [path.join('.')]: obj }
: reduce(obj, (cum, next, key) => merge(cum, flattenKeys(next as IDataObject, [...path, key])), {}); //prettier-ignore
};
export const validateInputData = (
node: INode,
items: INodeExecutionData[],
keysToCompare: string[],
disableDotNotation: boolean,
) => {
for (const key of keysToCompare) {
let type: any = undefined;
for (const [i, item] of items.entries()) {
if (key === '') {
throw new NodeOperationError(node, 'Name of field to compare is blank');
}
const value = !disableDotNotation ? get(item.json, key) : item.json[key];
if (value === null && node.typeVersion > 1) continue;
if (value === undefined && disableDotNotation && key.includes('.')) {
throw new NodeOperationError(node, `'${key}' field is missing from some input items`, {
description:
"If you're trying to use a nested field, make sure you turn off 'disable dot notation' in the node options",
});
} else if (value === undefined) {
throw new NodeOperationError(node, `'${key}' field is missing from some input items`);
}
if (type !== undefined && value !== undefined && type !== typeof value) {
const description =
'The type of this field varies between items' +
(node.typeVersion > 1
? `, in item [${i - 1}] it's a ${type} and in item [${i}] it's a ${typeof value} `
: '');
throw new NodeOperationError(node, `'${key}' isn't always the same type`, {
description,
});
} else {
type = typeof value;
}
}
}
};