feat(Item Lists Node): Table tranformation

packages/nodes-base/nodes/ItemLists/ItemLists.node.json

@@ -32,7 +32,15 @@
 		"Map",
 		"Format",
 		"Nested",
-		"Iterate"
+		"Iterate",
+		"Summarise",
+		"Summarize",
+		"Group",
+		"Pivot",
+		"Sum",
+		"Count",
+		"Min",
+		"Max"
 	],
 	"subcategories": {
 		"Core Nodes": ["Helpers", "Data Transformation"]

packages/nodes-base/nodes/ItemLists/ItemLists.node.ts

@@ -49,6 +49,8 @@ const shuffleArray = (array: any[]) => {
 	}
 };
 
+import * as summarize from './summarize.operation';
+
 export class ItemLists implements INodeType {
 	description: INodeTypeDescription = {
 		displayName: 'Item Lists',
@@ -84,10 +86,10 @@ export class ItemLists implements INodeType {
 				noDataExpression: true,
 				options: [
 					{
-						name: 'Aggregate Items',
+						name: 'Concatenate Items',
 						value: 'aggregateItems',
-						description: 'Combine fields into a single new item',
-						action: 'Combine fields into a single new item',
+						description: 'Combine fields into a list in a single new item',
+						action: 'Combine fields into a list in a single new item',
 					},
 					{
 						name: 'Limit',
@@ -113,11 +115,16 @@ export class ItemLists implements INodeType {
 						description: 'Turn a list inside item(s) into separate items',
 						action: 'Turn a list inside item(s) into separate items',
 					},
+					{
+						name: 'Summarize',
+						value: 'summarize',
+						description: 'Aggregate items together (pivot table)',
+						action: 'Aggregate items together (pivot table)',
+					},
 				],
 				default: 'splitOutItems',
 			},
 			// Split out items - Fields
-
 			{
 				displayName: 'Field To Split Out',
 				name: 'fieldToSplitOut',
@@ -765,6 +772,8 @@ return 0;`,
 					},
 				],
 			},
+			// Remove duplicates - Fields
+			...summarize.description,
 		],
 	};
 
@@ -1388,6 +1397,8 @@ return 0;`,
 					newItems = items.slice(items.length - maxItems, items.length);
 				}
 				return this.prepareOutputData(newItems);
+			} else if (operation === 'summarize') {
+				return summarize.execute.call(this, items);
 			} else {
 				throw new NodeOperationError(this.getNode(), `Operation '${operation}' is not recognized`);
 			}

packages/nodes-base/nodes/ItemLists/summarize.operation.ts

@@ -0,0 +1,577 @@
+import {
+	GenericValue,
+	IDataObject,
+	IExecuteFunctions,
+	INodeExecutionData,
+	INodeProperties,
+	NodeOperationError,
+} from 'n8n-workflow';
+
+import { get } from 'lodash';
+
+type AggregationType =
+	| 'append'
+	| 'average'
+	| 'concatenate'
+	| 'count'
+	| 'countUnique'
+	| 'max'
+	| 'min'
+	| 'sum';
+
+type Aggregation = {
+	aggregation: AggregationType;
+	field: string;
+	includeEmpty?: boolean;
+	separateBy?: string;
+	customSeparator?: string;
+};
+
+type Aggregations = Aggregation[];
+
+enum AggregationDisplayNames {
+	append = 'appended_',
+	average = 'average_',
+	concatenate = 'concatenated_',
+	count = 'count_',
+	countUnique = 'unique_count_',
+	max = 'max_',
+	min = 'min_',
+	sum = 'sum_',
+}
+
+const NUMERICAL_AGGREGATIONS = ['average', 'max', 'min', 'sum'];
+
+type SummarizeOptions = {
+	disableDotNotation?: boolean;
+	outputFormat?: 'separateItems' | 'singleItem';
+	skipEmptySplitFields?: boolean;
+};
+
+type ValueGetterFn = (
+	item: IDataObject,
+	field: string,
+) => IDataObject | IDataObject[] | GenericValue | GenericValue[];
+
+export const description: INodeProperties[] = [
+	{
+		displayName: 'Fields to Summarize',
+		name: 'fieldsToSummarize',
+		type: 'fixedCollection',
+		placeholder: 'Add Field',
+		default: { values: [{ aggregation: 'count', field: '' }] },
+		typeOptions: {
+			multipleValues: true,
+		},
+		options: [
+			{
+				displayName: '',
+				name: 'values',
+				values: [
+					{
+						displayName: 'Aggregation',
+						name: 'aggregation',
+						type: 'options',
+						options: [
+							{
+								name: 'Append',
+								value: 'append',
+							},
+							{
+								name: 'Average',
+								value: 'average',
+							},
+							{
+								name: 'Concatenate',
+								value: 'concatenate',
+							},
+							{
+								name: 'Count',
+								value: 'count',
+							},
+							{
+								name: 'Count Unique',
+								value: 'countUnique',
+							},
+							{
+								name: 'Max',
+								value: 'max',
+							},
+							{
+								name: 'Min',
+								value: 'min',
+							},
+							{
+								name: 'Sum',
+								value: 'sum',
+							},
+						],
+						default: 'count',
+						description: 'How to combine the values of the field you want to summarize',
+					},
+					//field repeated to have different descriptions for different aggregations --------------------------------
+					{
+						displayName: 'Field',
+						name: 'field',
+						type: 'string',
+						default: '',
+						description: 'The name of an input field that you want to summarize',
+						placeholder: 'e.g. cost',
+						hint: ' Enter the field name as text',
+						displayOptions: {
+							hide: {
+								aggregation: [...NUMERICAL_AGGREGATIONS, 'countUnique', 'count'],
+							},
+						},
+					},
+					{
+						displayName: 'Field',
+						name: 'field',
+						type: 'string',
+						default: '',
+						description:
+							'The name of an input field that you want to summarize. The field should contain numerical values; null, undefined, empty strings would be ignored.',
+						placeholder: 'e.g. cost',
+						hint: ' Enter the field name as text',
+						displayOptions: {
+							show: {
+								aggregation: NUMERICAL_AGGREGATIONS,
+							},
+						},
+					},
+					{
+						displayName: 'Field',
+						name: 'field',
+						type: 'string',
+						default: '',
+						description:
+							'The name of an input field that you want to summarize; null, undefined, empty strings would be ignored',
+						placeholder: 'e.g. cost',
+						hint: ' Enter the field name as text',
+						displayOptions: {
+							show: {
+								aggregation: ['countUnique', 'count'],
+							},
+						},
+					},
+					// ----------------------------------------------------------------------------------------------------------
+					{
+						displayName: 'Include Empty Values',
+						name: 'includeEmpty',
+						type: 'boolean',
+						default: false,
+						displayOptions: {
+							show: {
+								aggregation: ['append', 'concatenate'],
+							},
+						},
+					},
+					{
+						displayName: 'Separator',
+						name: 'separateBy',
+						type: 'options',
+						default: ',',
+						// eslint-disable-next-line n8n-nodes-base/node-param-options-type-unsorted-items
+						options: [
+							{
+								name: 'Comma',
+								value: ',',
+							},
+							{
+								name: 'Comma and Space',
+								value: ', ',
+							},
+							{
+								name: 'New Line',
+								value: '\n',
+							},
+							{
+								name: 'None',
+								value: '',
+							},
+							{
+								name: 'Space',
+								value: ' ',
+							},
+							{
+								name: 'Other',
+								value: 'other',
+							},
+						],
+						hint: 'What to insert between values',
+						displayOptions: {
+							show: {
+								aggregation: ['concatenate'],
+							},
+						},
+					},
+					{
+						displayName: 'Custom Separator',
+						name: 'customSeparator',
+						type: 'string',
+						default: '',
+						displayOptions: {
+							show: {
+								aggregation: ['concatenate'],
+								separateBy: ['other'],
+							},
+						},
+					},
+				],
+			},
+		],
+		displayOptions: {
+			show: {
+				resource: ['itemList'],
+				operation: ['summarize'],
+			},
+		},
+	},
+	// fieldsToSplitBy repeated to have different displayName for singleItem and separateItems -----------------------------
+	{
+		displayName: 'Fields to Split By',
+		name: 'fieldsToSplitBy',
+		type: 'string',
+		placeholder: 'e.g. country, city',
+		default: '',
+		description: 'The name of the input fields that you want to split the summary by',
+		hint: 'Enter the name of the fields as text (separated by commas)',
+		displayOptions: {
+			show: {
+				resource: ['itemList'],
+				operation: ['summarize'],
+			},
+			hide: {
+				'/options.outputFormat': ['singleItem'],
+			},
+		},
+	},
+	{
+		displayName: 'Fields to Group By',
+		name: 'fieldsToSplitBy',
+		type: 'string',
+		placeholder: 'e.g. country, city',
+		default: '',
+		description: 'The name of the input fields that you want to split the summary by',
+		hint: 'Enter the name of the fields as text (separated by commas)',
+		displayOptions: {
+			show: {
+				resource: ['itemList'],
+				operation: ['summarize'],
+				'/options.outputFormat': ['singleItem'],
+			},
+		},
+	},
+	// ----------------------------------------------------------------------------------------------------------
+	{
+		displayName: 'Options',
+		name: 'options',
+		type: 'collection',
+		placeholder: 'Add Option',
+		default: {},
+		displayOptions: {
+			show: {
+				resource: ['itemList'],
+				operation: ['summarize'],
+			},
+		},
+		options: [
+			{
+				displayName: 'Disable Dot Notation',
+				name: 'disableDotNotation',
+				type: 'boolean',
+				default: false,
+				description:
+					'Whether to disallow referencing child fields using `parent.child` in the field name',
+			},
+			{
+				displayName: 'Output Format',
+				name: 'outputFormat',
+				type: 'options',
+				default: 'separateItems',
+				options: [
+					{
+						name: 'Each Split in a Separate Item',
+						value: 'separateItems',
+					},
+					{
+						name: 'All Splits in a Single Item',
+						value: 'singleItem',
+					},
+				],
+			},
+			{
+				// eslint-disable-next-line n8n-nodes-base/node-param-display-name-miscased
+				displayName: 'Ignore items without valid fields to group by',
+				name: 'skipEmptySplitFields',
+				type: 'boolean',
+				default: false,
+			},
+		],
+	},
+];
+
+function isEmpty<T>(value: T) {
+	return value === undefined || value === null || value === '';
+}
+
+const fieldValueGetter = (disableDotNotation?: boolean) => {
+	if (disableDotNotation) {
+		return (item: IDataObject, field: string) => item[field];
+	} else {
+		return (item: IDataObject, field: string) => get(item, field);
+	}
+};
+
+function checkIfFieldExists(
+	this: IExecuteFunctions,
+	items: IDataObject[],
+	aggregations: Aggregations,
+	getValue: ValueGetterFn,
+) {
+	for (const aggregation of aggregations) {
+		if (aggregation.field === '') {
+			continue;
+		}
+		const exist = items.some((item) => getValue(item, aggregation.field) !== undefined);
+		if (!exist) {
+			throw new NodeOperationError(
+				this.getNode(),
+				`The field '${aggregation.field}' does not exist in any items`,
+			);
+		}
+	}
+}
+
+function aggregate(items: IDataObject[], entry: Aggregation, getValue: ValueGetterFn) {
+	const { aggregation, field } = entry;
+	let data = [...items];
+
+	if (NUMERICAL_AGGREGATIONS.includes(aggregation)) {
+		data = data.filter(
+			(item) => typeof getValue(item, field) === 'number' && !isEmpty(getValue(item, field)),
+		);
+	}
+
+	switch (aggregation) {
+		//combine operations
+		case 'append':
+			if (!entry.includeEmpty) {
+				data = data.filter((item) => !isEmpty(getValue(item, field)));
+			}
+			return data.map((item) => getValue(item, field));
+		case 'concatenate':
+			const separateBy = entry.separateBy === 'other' ? entry.customSeparator : entry.separateBy;
+			if (!entry.includeEmpty) {
+				data = data.filter((item) => !isEmpty(getValue(item, field)));
+			}
+			return data
+				.map((item) => {
+					let value = getValue(item, field);
+					if (typeof value === 'object') {
+						value = JSON.stringify(value);
+					}
+					if (typeof value === 'undefined') {
+						value = 'undefined';
+					}
+
+					return value;
+				})
+				.join(separateBy);
+
+		//numerical operations
+		case 'average':
+			return (
+				data.reduce((acc, item) => {
+					return acc + (getValue(item, field) as number);
+				}, 0) / data.length
+			);
+		case 'sum':
+			return data.reduce((acc, item) => {
+				return acc + (getValue(item, field) as number);
+			}, 0);
+		case 'min':
+			return Math.min(
+				...(data.map((item) => {
+					return getValue(item, field);
+				}) as number[]),
+			);
+		case 'max':
+			return Math.max(
+				...(data.map((item) => {
+					return getValue(item, field);
+				}) as number[]),
+			);
+
+		//count operations
+		case 'countUnique':
+			return new Set(data.map((item) => getValue(item, field)).filter((item) => !isEmpty(item)))
+				.size;
+		default:
+			//count by default
+			return data.filter((item) => !isEmpty(getValue(item, field))).length;
+	}
+}
+
+function aggregateData(
+	data: IDataObject[],
+	fieldsToSummarize: Aggregations,
+	options: SummarizeOptions,
+	getValue: ValueGetterFn,
+) {
+	const returnData = fieldsToSummarize.reduce((acc, aggregation) => {
+		acc[`${AggregationDisplayNames[aggregation.aggregation]}${aggregation.field}`] = aggregate(
+			data,
+			aggregation,
+			getValue,
+		);
+		return acc;
+	}, {} as IDataObject);
+	if (options.outputFormat === 'singleItem') {
+		return returnData;
+	} else {
+		return { ...returnData, pairedItems: data.map((item) => item._itemIndex as number) };
+	}
+}
+
+function splitData(
+	splitKeys: string[],
+	data: IDataObject[],
+	fieldsToSummarize: Aggregations,
+	options: SummarizeOptions,
+	getValue: ValueGetterFn,
+) {
+	if (!splitKeys || splitKeys.length === 0) {
+		return aggregateData(data, fieldsToSummarize, options, getValue);
+	}
+
+	const [firstSplitKey, ...restSplitKeys] = splitKeys;
+
+	const groupedData = data.reduce((acc, item) => {
+		let keyValuee = getValue(item, firstSplitKey) as string;
+
+		if (typeof keyValuee === 'object') {
+			keyValuee = JSON.stringify(keyValuee);
+		}
+
+		if (options.skipEmptySplitFields && typeof keyValuee !== 'number' && !keyValuee) {
+			return acc;
+		}
+
+		if (acc[keyValuee] === undefined) {
+			acc[keyValuee] = [item];
+		} else {
+			(acc[keyValuee] as IDataObject[]).push(item);
+		}
+		return acc;
+	}, {} as IDataObject);
+
+	return Object.keys(groupedData).reduce((acc, key) => {
+		const value = groupedData[key] as IDataObject[];
+		acc[key] = splitData(restSplitKeys, value, fieldsToSummarize, options, getValue);
+		return acc;
+	}, {} as IDataObject);
+}
+
+function aggregationToArray(
+	aggregationResult: IDataObject,
+	fieldsToSplitBy: string[],
+	previousStage: IDataObject = {},
+) {
+	const returnData: IDataObject[] = [];
+
+	const splitFieldName = fieldsToSplitBy[0];
+	const isNext = fieldsToSplitBy[1];
+
+	if (isNext === undefined) {
+		for (const fieldName of Object.keys(aggregationResult)) {
+			returnData.push({
+				...previousStage,
+				[splitFieldName]: fieldName,
+				...(aggregationResult[fieldName] as IDataObject),
+			});
+		}
+		return returnData;
+	} else {
+		for (const key of Object.keys(aggregationResult)) {
+			returnData.push(
+				...aggregationToArray(aggregationResult[key] as IDataObject, fieldsToSplitBy.slice(1), {
+					...previousStage,
+					[splitFieldName]: key,
+				}),
+			);
+		}
+		return returnData;
+	}
+}
+
+export async function execute(
+	this: IExecuteFunctions,
+	items: INodeExecutionData[],
+): Promise<INodeExecutionData[][]> {
+	const newItems = items.map(({ json }, i) => ({ ...json, _itemIndex: i }));
+
+	const options = this.getNodeParameter('options', 0, {}) as SummarizeOptions;
+
+	const fieldsToSplitBy = (this.getNodeParameter('fieldsToSplitBy', 0, '') as string)
+		.split(',')
+		.map((field) => field.trim())
+		.filter((field) => field);
+
+	const fieldsToSummarize = this.getNodeParameter(
+		'fieldsToSummarize.values',
+		0,
+		[],
+	) as Aggregations;
+
+	if (fieldsToSummarize.filter((aggregation) => aggregation.field !== '').length === 0) {
+		throw new NodeOperationError(
+			this.getNode(),
+			"You need to add at least one aggregation to 'Fields to Summarize' with non empty 'Field'",
+		);
+	}
+
+	const getValue = fieldValueGetter(options.disableDotNotation);
+
+	checkIfFieldExists.call(this, newItems, fieldsToSummarize, getValue);
+
+	const aggregationResult = splitData(
+		fieldsToSplitBy,
+		newItems,
+		fieldsToSummarize,
+		options,
+		getValue,
+	);
+
+	if (options.outputFormat === 'singleItem') {
+		const executionData: INodeExecutionData = {
+			json: aggregationResult,
+			pairedItem: newItems.map((_v, index) => ({
+				item: index,
+			})),
+		};
+		return this.prepareOutputData([executionData]);
+	} else {
+		if (!fieldsToSplitBy.length) {
+			const { pairedItems, ...json } = aggregationResult;
+			const executionData: INodeExecutionData = {
+				json,
+				pairedItem: ((pairedItems as number[]) || []).map((index: number) => ({
+					item: index,
+				})),
+			};
+			return this.prepareOutputData([executionData]);
+		}
+		const returnData = aggregationToArray(aggregationResult, fieldsToSplitBy);
+		const executionData = returnData.map((item) => {
+			const { pairedItems, ...json } = item;
+			return {
+				json,
+				pairedItem: ((pairedItems as number[]) || []).map((index: number) => ({
+					item: index,
+				})),
+			};
+		});
+		return this.prepareOutputData(executionData);
+	}
+}