n8n/packages/nodes-base/nodes/ItemLists/V2/summarize.operation.ts

import type {
	GenericValue,
	IDataObject,
	IExecuteFunctions,
	INodeExecutionData,
	INodeProperties,
} from 'n8n-workflow';
import { NodeOperationError } from 'n8n-workflow';

import get from 'lodash.get';

type AggregationType =
	| 'append'
	| 'average'
	| 'concatenate'
	| 'count'
	| 'countUnique'
	| 'max'
	| 'min'
	| 'sum';

type Aggregation = {
	aggregation: AggregationType;
	field: string;
	includeEmpty?: boolean;
	separateBy?: string;
	customSeparator?: string;
};

type Aggregations = Aggregation[];

// eslint-disable-next-line no-restricted-syntax
const AggregationDisplayNames = {
	append: 'appended_',
	average: 'average_',
	concatenate: 'concatenated_',
	count: 'count_',
	countUnique: 'unique_count_',
	max: 'max_',
	min: 'min_',
	sum: 'sum_',
};

const NUMERICAL_AGGREGATIONS = ['average', 'max', 'min', 'sum'];

type SummarizeOptions = {
	disableDotNotation?: boolean;
	outputFormat?: 'separateItems' | 'singleItem';
	skipEmptySplitFields?: boolean;
};

type ValueGetterFn = (
	item: IDataObject,
	field: string,
) => IDataObject | IDataObject[] | GenericValue | GenericValue[];

export const description: INodeProperties[] = [
	{
		displayName: 'Fields to Summarize',
		name: 'fieldsToSummarize',
		type: 'fixedCollection',
		placeholder: 'Add Field',
		default: { values: [{ aggregation: 'count', field: '' }] },
		typeOptions: {
			multipleValues: true,
		},
		options: [
			{
				displayName: '',
				name: 'values',
				values: [
					{
						displayName: 'Aggregation',
						name: 'aggregation',
						type: 'options',
						options: [
							{
								name: 'Append',
								value: 'append',
							},
							{
								name: 'Average',
								value: 'average',
							},
							{
								name: 'Concatenate',
								value: 'concatenate',
							},
							{
								name: 'Count',
								value: 'count',
							},
							{
								name: 'Count Unique',
								value: 'countUnique',
							},
							{
								name: 'Max',
								value: 'max',
							},
							{
								name: 'Min',
								value: 'min',
							},
							{
								name: 'Sum',
								value: 'sum',
							},
						],
						default: 'count',
						description: 'How to combine the values of the field you want to summarize',
					},
					//field repeated to have different descriptions for different aggregations --------------------------------
					{
						displayName: 'Field',
						name: 'field',
						type: 'string',
						default: '',
						description: 'The name of an input field that you want to summarize',
						placeholder: 'e.g. cost',
						hint: ' Enter the field name as text',
						displayOptions: {
							hide: {
								aggregation: [...NUMERICAL_AGGREGATIONS, 'countUnique', 'count'],
							},
						},
						requiresDataPath: 'single',
					},
					{
						displayName: 'Field',
						name: 'field',
						type: 'string',
						default: '',
						description:
							'The name of an input field that you want to summarize. The field should contain numerical values; null, undefined, empty strings would be ignored.',
						placeholder: 'e.g. cost',
						hint: ' Enter the field name as text',
						displayOptions: {
							show: {
								aggregation: NUMERICAL_AGGREGATIONS,
							},
						},
						requiresDataPath: 'single',
					},
					{
						displayName: 'Field',
						name: 'field',
						type: 'string',
						default: '',
						description:
							'The name of an input field that you want to summarize; null, undefined, empty strings would be ignored',
						placeholder: 'e.g. cost',
						hint: ' Enter the field name as text',
						displayOptions: {
							show: {
								aggregation: ['countUnique', 'count'],
							},
						},
						requiresDataPath: 'single',
					},
					// ----------------------------------------------------------------------------------------------------------
					{
						displayName: 'Include Empty Values',
						name: 'includeEmpty',
						type: 'boolean',
						default: false,
						displayOptions: {
							show: {
								aggregation: ['append', 'concatenate'],
							},
						},
					},
					{
						displayName: 'Separator',
						name: 'separateBy',
						type: 'options',
						default: ',',
						// eslint-disable-next-line n8n-nodes-base/node-param-options-type-unsorted-items
						options: [
							{
								name: 'Comma',
								value: ',',
							},
							{
								name: 'Comma and Space',
								value: ', ',
							},
							{
								name: 'New Line',
								value: '\n',
							},
							{
								name: 'None',
								value: '',
							},
							{
								name: 'Space',
								value: ' ',
							},
							{
								name: 'Other',
								value: 'other',
							},
						],
						hint: 'What to insert between values',
						displayOptions: {
							show: {
								aggregation: ['concatenate'],
							},
						},
					},
					{
						displayName: 'Custom Separator',
						name: 'customSeparator',
						type: 'string',
						default: '',
						displayOptions: {
							show: {
								aggregation: ['concatenate'],
								separateBy: ['other'],
							},
						},
					},
				],
			},
		],
		displayOptions: {
			show: {
				resource: ['itemList'],
				operation: ['summarize'],
			},
		},
	},
	// fieldsToSplitBy repeated to have different displayName for singleItem and separateItems -----------------------------
	{
		displayName: 'Fields to Split By',
		name: 'fieldsToSplitBy',
		type: 'string',
		placeholder: 'e.g. country, city',
		default: '',
		description: 'The name of the input fields that you want to split the summary by',
		hint: 'Enter the name of the fields as text (separated by commas)',
		displayOptions: {
			show: {
				resource: ['itemList'],
				operation: ['summarize'],
			},
			hide: {
				'/options.outputFormat': ['singleItem'],
			},
		},
		requiresDataPath: 'multiple',
	},
	{
		displayName: 'Fields to Group By',
		name: 'fieldsToSplitBy',
		type: 'string',
		placeholder: 'e.g. country, city',
		default: '',
		description: 'The name of the input fields that you want to split the summary by',
		hint: 'Enter the name of the fields as text (separated by commas)',
		displayOptions: {
			show: {
				resource: ['itemList'],
				operation: ['summarize'],
				'/options.outputFormat': ['singleItem'],
			},
		},
		requiresDataPath: 'multiple',
	},
	// ----------------------------------------------------------------------------------------------------------
	{
		displayName: 'Options',
		name: 'options',
		type: 'collection',
		placeholder: 'Add Option',
		default: {},
		displayOptions: {
			show: {
				resource: ['itemList'],
				operation: ['summarize'],
			},
		},
		options: [
			{
				displayName: 'Disable Dot Notation',
				name: 'disableDotNotation',
				type: 'boolean',
				default: false,
				description:
					'Whether to disallow referencing child fields using `parent.child` in the field name',
			},
			{
				displayName: 'Output Format',
				name: 'outputFormat',
				type: 'options',
				default: 'separateItems',
				options: [
					{
						name: 'Each Split in a Separate Item',
						value: 'separateItems',
					},
					{
						name: 'All Splits in a Single Item',
						value: 'singleItem',
					},
				],
			},
			{
				// eslint-disable-next-line n8n-nodes-base/node-param-display-name-miscased
				displayName: 'Ignore items without valid fields to group by',
				name: 'skipEmptySplitFields',
				type: 'boolean',
				default: false,
			},
		],
	},
];

function isEmpty<T>(value: T) {
	return value === undefined || value === null || value === '';
}

function parseReturnData(returnData: IDataObject) {
	const regexBrackets = /[\]\["]/g;
	const regexSpaces = /[ .]/g;
	for (const key of Object.keys(returnData)) {
		if (key.match(regexBrackets)) {
			const newKey = key.replace(regexBrackets, '');
			returnData[newKey] = returnData[key];
			delete returnData[key];
		}
	}
	for (const key of Object.keys(returnData)) {
		if (key.match(regexSpaces)) {
			const newKey = key.replace(regexSpaces, '_');
			returnData[newKey] = returnData[key];
			delete returnData[key];
		}
	}
}

function parseFieldName(fieldName: string[]) {
	const regexBrackets = /[\]\["]/g;
	const regexSpaces = /[ .]/g;
	fieldName = fieldName.map((field) => {
		field = field.replace(regexBrackets, '');
		field = field.replace(regexSpaces, '_');
		return field;
	});
	return fieldName;
}

const fieldValueGetter = (disableDotNotation?: boolean) => {
	if (disableDotNotation) {
		return (item: IDataObject, field: string) => item[field];
	} else {
		return (item: IDataObject, field: string) => get(item, field);
	}
};

function checkIfFieldExists(
	this: IExecuteFunctions,
	items: IDataObject[],
	aggregations: Aggregations,
	getValue: ValueGetterFn,
) {
	for (const aggregation of aggregations) {
		if (aggregation.field === '') {
			continue;
		}
		const exist = items.some((item) => getValue(item, aggregation.field) !== undefined);
		if (!exist) {
			throw new NodeOperationError(
				this.getNode(),
				`The field '${aggregation.field}' does not exist in any items`,
			);
		}
	}
}

function aggregate(items: IDataObject[], entry: Aggregation, getValue: ValueGetterFn) {
	const { aggregation, field } = entry;
	let data = [...items];

	if (NUMERICAL_AGGREGATIONS.includes(aggregation)) {
		data = data.filter(
			(item) => typeof getValue(item, field) === 'number' && !isEmpty(getValue(item, field)),
		);
	}

	switch (aggregation) {
		//combine operations
		case 'append':
			if (!entry.includeEmpty) {
				data = data.filter((item) => !isEmpty(getValue(item, field)));
			}
			return data.map((item) => getValue(item, field));
		case 'concatenate':
			const separateBy = entry.separateBy === 'other' ? entry.customSeparator : entry.separateBy;
			if (!entry.includeEmpty) {
				data = data.filter((item) => !isEmpty(getValue(item, field)));
			}
			return data
				.map((item) => {
					let value = getValue(item, field);
					if (typeof value === 'object') {
						value = JSON.stringify(value);
					}
					if (typeof value === 'undefined') {
						value = 'undefined';
					}

					return value;
				})
				.join(separateBy);

		//numerical operations
		case 'average':
			return (
				data.reduce((acc, item) => {
					return acc + (getValue(item, field) as number);
				}, 0) / data.length
			);
		case 'sum':
			return data.reduce((acc, item) => {
				return acc + (getValue(item, field) as number);
			}, 0);
		case 'min':
			return Math.min(
				...(data.map((item) => {
					return getValue(item, field);
				}) as number[]),
			);
		case 'max':
			return Math.max(
				...(data.map((item) => {
					return getValue(item, field);
				}) as number[]),
			);

		//count operations
		case 'countUnique':
			return new Set(data.map((item) => getValue(item, field)).filter((item) => !isEmpty(item)))
				.size;
		default:
			//count by default
			return data.filter((item) => !isEmpty(getValue(item, field))).length;
	}
}

function aggregateData(
	data: IDataObject[],
	fieldsToSummarize: Aggregations,
	options: SummarizeOptions,
	getValue: ValueGetterFn,
) {
	const returnData = fieldsToSummarize.reduce((acc, aggregation) => {
		acc[`${AggregationDisplayNames[aggregation.aggregation]}${aggregation.field}`] = aggregate(
			data,
			aggregation,
			getValue,
		);
		return acc;
	}, {} as IDataObject);
	parseReturnData(returnData);
	if (options.outputFormat === 'singleItem') {
		parseReturnData(returnData);
		return returnData;
	} else {
		return { ...returnData, pairedItems: data.map((item) => item._itemIndex as number) };
	}
}

function splitData(
	splitKeys: string[],
	data: IDataObject[],
	fieldsToSummarize: Aggregations,
	options: SummarizeOptions,
	getValue: ValueGetterFn,
) {
	if (!splitKeys || splitKeys.length === 0) {
		return aggregateData(data, fieldsToSummarize, options, getValue);
	}

	const [firstSplitKey, ...restSplitKeys] = splitKeys;

	const groupedData = data.reduce((acc, item) => {
		let keyValuee = getValue(item, firstSplitKey) as string;

		if (typeof keyValuee === 'object') {
			keyValuee = JSON.stringify(keyValuee);
		}

		if (options.skipEmptySplitFields && typeof keyValuee !== 'number' && !keyValuee) {
			return acc;
		}

		if (acc[keyValuee] === undefined) {
			acc[keyValuee] = [item];
		} else {
			(acc[keyValuee] as IDataObject[]).push(item);
		}
		return acc;
	}, {} as IDataObject);

	return Object.keys(groupedData).reduce((acc, key) => {
		const value = groupedData[key] as IDataObject[];
		acc[key] = splitData(restSplitKeys, value, fieldsToSummarize, options, getValue);
		return acc;
	}, {} as IDataObject);
}

function aggregationToArray(
	aggregationResult: IDataObject,
	fieldsToSplitBy: string[],
	previousStage: IDataObject = {},
) {
	const returnData: IDataObject[] = [];
	fieldsToSplitBy = parseFieldName(fieldsToSplitBy);
	const splitFieldName = fieldsToSplitBy[0];
	const isNext = fieldsToSplitBy[1];

	if (isNext === undefined) {
		for (const fieldName of Object.keys(aggregationResult)) {
			returnData.push({
				...previousStage,
				[splitFieldName]: fieldName,
				...(aggregationResult[fieldName] as IDataObject),
			});
		}
		return returnData;
	} else {
		for (const key of Object.keys(aggregationResult)) {
			returnData.push(
				...aggregationToArray(aggregationResult[key] as IDataObject, fieldsToSplitBy.slice(1), {
					...previousStage,
					[splitFieldName]: key,
				}),
			);
		}
		return returnData;
	}
}

export async function execute(
	this: IExecuteFunctions,
	items: INodeExecutionData[],
): Promise<INodeExecutionData[][]> {
	const newItems = items.map(({ json }, i) => ({ ...json, _itemIndex: i }));

	const options = this.getNodeParameter('options', 0, {}) as SummarizeOptions;

	const fieldsToSplitBy = (this.getNodeParameter('fieldsToSplitBy', 0, '') as string)
		.split(',')
		.map((field) => field.trim())
		.filter((field) => field);

	const fieldsToSummarize = this.getNodeParameter(
		'fieldsToSummarize.values',
		0,
		[],
	) as Aggregations;

	if (fieldsToSummarize.filter((aggregation) => aggregation.field !== '').length === 0) {
		throw new NodeOperationError(
			this.getNode(),
			"You need to add at least one aggregation to 'Fields to Summarize' with non empty 'Field'",
		);
	}

	const getValue = fieldValueGetter(options.disableDotNotation);

	const nodeVersion = this.getNode().typeVersion;

	if (nodeVersion < 2.1) {
		checkIfFieldExists.call(this, newItems, fieldsToSummarize, getValue);
	}

	const aggregationResult = splitData(
		fieldsToSplitBy,
		newItems,
		fieldsToSummarize,
		options,
		getValue,
	);

	if (options.outputFormat === 'singleItem') {
		const executionData: INodeExecutionData = {
			json: aggregationResult,
			pairedItem: newItems.map((_v, index) => ({
				item: index,
			})),
		};
		return this.prepareOutputData([executionData]);
	} else {
		if (!fieldsToSplitBy.length) {
			const { pairedItems, ...json } = aggregationResult;
			const executionData: INodeExecutionData = {
				json,
				pairedItem: ((pairedItems as number[]) || []).map((index: number) => ({
					item: index,
				})),
			};
			return this.prepareOutputData([executionData]);
		}
		const returnData = aggregationToArray(aggregationResult, fieldsToSplitBy);
		const executionData = returnData.map((item) => {
			const { pairedItems, ...json } = item;
			return {
				json,
				pairedItem: ((pairedItems as number[]) || []).map((index: number) => ({
					item: index,
				})),
			};
		});
		return this.prepareOutputData(executionData);
	}
}