n8n/packages/nodes-base/nodes/Transform/RemoveDuplicates/utils.ts

import { isEqual, lt, pick } from 'lodash';
import get from 'lodash/get';
import { NodeOperationError } from 'n8n-workflow';
import type { IExecuteFunctions, INode, INodeExecutionData } from 'n8n-workflow';

import { compareItems, flattenKeys } from '@utils/utilities';

import { prepareFieldsArray } from '../utils/utils';

export const validateInputData = (
	node: INode,
	items: INodeExecutionData[],
	keysToCompare: string[],
	disableDotNotation: boolean,
) => {
	for (const key of keysToCompare) {
		let type: any = undefined;
		for (const [i, item] of items.entries()) {
			if (key === '') {
				throw new NodeOperationError(node, 'Name of field to compare is blank');
			}
			const value = !disableDotNotation ? get(item.json, key) : item.json[key];
			if (value === null && node.typeVersion > 1) continue;

			if (value === undefined && disableDotNotation && key.includes('.')) {
				throw new NodeOperationError(node, `'${key}' field is missing from some input items`, {
					description:
						"If you're trying to use a nested field, make sure you turn off 'disable dot notation' in the node options",
				});
			} else if (value === undefined) {
				throw new NodeOperationError(node, `'${key}' field is missing from some input items`);
			}
			if (type !== undefined && value !== undefined && type !== typeof value) {
				const description =
					'The type of this field varies between items' +
					(node.typeVersion > 1
						? `, in item [${i - 1}] it's a ${type} and in item [${i}] it's a ${typeof value} `
						: '');
				throw new NodeOperationError(node, `'${key}' isn't always the same type`, {
					description,
				});
			} else {
				type = typeof value;
			}
		}
	}
};

export function removeDuplicateInputItems(context: IExecuteFunctions, items: INodeExecutionData[]) {
	const compare = context.getNodeParameter('compare', 0) as string;
	const disableDotNotation = context.getNodeParameter(
		'options.disableDotNotation',
		0,
		false,
	) as boolean;
	const removeOtherFields = context.getNodeParameter(
		'options.removeOtherFields',
		0,
		false,
	) as boolean;

	let keys = disableDotNotation
		? Object.keys(items[0].json)
		: Object.keys(flattenKeys(items[0].json));

	for (const item of items) {
		const itemKeys = disableDotNotation
			? Object.keys(item.json)
			: Object.keys(flattenKeys(item.json));
		for (const key of itemKeys) {
			if (!keys.includes(key)) {
				keys.push(key);
			}
		}
	}

	if (compare === 'allFieldsExcept') {
		const fieldsToExclude = prepareFieldsArray(
			context.getNodeParameter('fieldsToExclude', 0, '') as string,
			'Fields To Exclude',
		);

		if (!fieldsToExclude.length) {
			throw new NodeOperationError(
				context.getNode(),
				'No fields specified. Please add a field to exclude from comparison',
			);
		}
		if (!disableDotNotation) {
			keys = Object.keys(flattenKeys(items[0].json));
		}
		keys = keys.filter((key) => !fieldsToExclude.includes(key));
	}
	if (compare === 'selectedFields') {
		const fieldsToCompare = prepareFieldsArray(
			context.getNodeParameter('fieldsToCompare', 0, '') as string,
			'Fields To Compare',
		);
		if (!fieldsToCompare.length) {
			throw new NodeOperationError(
				context.getNode(),
				'No fields specified. Please add a field to compare on',
			);
		}
		if (!disableDotNotation) {
			keys = Object.keys(flattenKeys(items[0].json));
		}
		keys = fieldsToCompare.map((key) => key.trim());
	}

	// This solution is O(nlogn)
	// add original index to the items
	const newItems = items.map(
		(item, index) =>
			({
				json: { ...item.json, __INDEX: index },
				pairedItem: { item: index },
			}) as INodeExecutionData,
	);
	//sort items using the compare keys
	newItems.sort((a, b) => {
		let result = 0;

		for (const key of keys) {
			let equal;
			if (!disableDotNotation) {
				equal = isEqual(get(a.json, key), get(b.json, key));
			} else {
				equal = isEqual(a.json[key], b.json[key]);
			}
			if (!equal) {
				let lessThan;
				if (!disableDotNotation) {
					lessThan = lt(get(a.json, key), get(b.json, key));
				} else {
					lessThan = lt(a.json[key], b.json[key]);
				}
				result = lessThan ? -1 : 1;
				break;
			}
		}
		return result;
	});

	validateInputData(context.getNode(), newItems, keys, disableDotNotation);

	// collect the original indexes of items to be removed
	const removedIndexes: number[] = [];
	let temp = newItems[0];
	for (let index = 1; index < newItems.length; index++) {
		if (compareItems(newItems[index], temp, keys, disableDotNotation)) {
			removedIndexes.push(newItems[index].json.__INDEX as unknown as number);
		} else {
			temp = newItems[index];
		}
	}
	let updatedItems: INodeExecutionData[] = items.filter(
		(_, index) => !removedIndexes.includes(index),
	);

	if (removeOtherFields) {
		updatedItems = updatedItems.map((item, index) => ({
			json: pick(item.json, ...keys),
			pairedItem: { item: index },
		}));
	}
	return [updatedItems];
}