mirror of
https://github.com/n8n-io/n8n.git
synced 2024-12-24 20:24:05 -08:00
feat(Item Lists Node): Table tranformation
This commit is contained in:
Michael Kret
GitHub
parent
b7aa45536b
commit
5426690791
|
|
@ -32,7 +32,15 @@
|
|||
"Map",
|
||||
"Format",
|
||||
"Nested",
|
||||
"Iterate"
|
||||
"Iterate",
|
||||
"Summarise",
|
||||
"Summarize",
|
||||
"Group",
|
||||
"Pivot",
|
||||
"Sum",
|
||||
"Count",
|
||||
"Min",
|
||||
"Max"
|
||||
],
|
||||
"subcategories": {
|
||||
"Core Nodes": ["Helpers", "Data Transformation"]
|
||||
|
|
|
|||
|
|
@ -49,6 +49,8 @@ const shuffleArray = (array: any[]) => {
|
|||
}
|
||||
};
|
||||
|
||||
import * as summarize from './summarize.operation';
|
||||
|
||||
export class ItemLists implements INodeType {
|
||||
description: INodeTypeDescription = {
|
||||
displayName: 'Item Lists',
|
||||
|
|
@ -84,10 +86,10 @@ export class ItemLists implements INodeType {
|
|||
noDataExpression: true,
|
||||
options: [
|
||||
{
|
||||
name: 'Aggregate Items',
|
||||
name: 'Concatenate Items',
|
||||
value: 'aggregateItems',
|
||||
description: 'Combine fields into a single new item',
|
||||
action: 'Combine fields into a single new item',
|
||||
description: 'Combine fields into a list in a single new item',
|
||||
action: 'Combine fields into a list in a single new item',
|
||||
},
|
||||
{
|
||||
name: 'Limit',
|
||||
|
|
@ -113,11 +115,16 @@ export class ItemLists implements INodeType {
|
|||
description: 'Turn a list inside item(s) into separate items',
|
||||
action: 'Turn a list inside item(s) into separate items',
|
||||
},
|
||||
{
|
||||
name: 'Summarize',
|
||||
value: 'summarize',
|
||||
description: 'Aggregate items together (pivot table)',
|
||||
action: 'Aggregate items together (pivot table)',
|
||||
},
|
||||
],
|
||||
default: 'splitOutItems',
|
||||
},
|
||||
// Split out items - Fields
|
||||
|
||||
{
|
||||
displayName: 'Field To Split Out',
|
||||
name: 'fieldToSplitOut',
|
||||
|
|
@ -765,6 +772,8 @@ return 0;`,
|
|||
},
|
||||
],
|
||||
},
|
||||
// Remove duplicates - Fields
|
||||
...summarize.description,
|
||||
],
|
||||
};
|
||||
|
||||
|
|
@ -1388,6 +1397,8 @@ return 0;`,
|
|||
newItems = items.slice(items.length - maxItems, items.length);
|
||||
}
|
||||
return this.prepareOutputData(newItems);
|
||||
} else if (operation === 'summarize') {
|
||||
return summarize.execute.call(this, items);
|
||||
} else {
|
||||
throw new NodeOperationError(this.getNode(), `Operation '${operation}' is not recognized`);
|
||||
}
|
||||
|
|
|
|||
577
packages/nodes-base/nodes/ItemLists/summarize.operation.ts
Normal file
577
packages/nodes-base/nodes/ItemLists/summarize.operation.ts
Normal file
|
|
@ -0,0 +1,577 @@
|
|||
import {
|
||||
GenericValue,
|
||||
IDataObject,
|
||||
IExecuteFunctions,
|
||||
INodeExecutionData,
|
||||
INodeProperties,
|
||||
NodeOperationError,
|
||||
} from 'n8n-workflow';
|
||||
|
||||
import { get } from 'lodash';
|
||||
|
||||
type AggregationType =
|
||||
| 'append'
|
||||
| 'average'
|
||||
| 'concatenate'
|
||||
| 'count'
|
||||
| 'countUnique'
|
||||
| 'max'
|
||||
| 'min'
|
||||
| 'sum';
|
||||
|
||||
type Aggregation = {
|
||||
aggregation: AggregationType;
|
||||
field: string;
|
||||
includeEmpty?: boolean;
|
||||
separateBy?: string;
|
||||
customSeparator?: string;
|
||||
};
|
||||
|
||||
type Aggregations = Aggregation[];
|
||||
|
||||
enum AggregationDisplayNames {
|
||||
append = 'appended_',
|
||||
average = 'average_',
|
||||
concatenate = 'concatenated_',
|
||||
count = 'count_',
|
||||
countUnique = 'unique_count_',
|
||||
max = 'max_',
|
||||
min = 'min_',
|
||||
sum = 'sum_',
|
||||
}
|
||||
|
||||
const NUMERICAL_AGGREGATIONS = ['average', 'max', 'min', 'sum'];
|
||||
|
||||
type SummarizeOptions = {
|
||||
disableDotNotation?: boolean;
|
||||
outputFormat?: 'separateItems' | 'singleItem';
|
||||
skipEmptySplitFields?: boolean;
|
||||
};
|
||||
|
||||
type ValueGetterFn = (
|
||||
item: IDataObject,
|
||||
field: string,
|
||||
) => IDataObject | IDataObject[] | GenericValue | GenericValue[];
|
||||
|
||||
export const description: INodeProperties[] = [
|
||||
{
|
||||
displayName: 'Fields to Summarize',
|
||||
name: 'fieldsToSummarize',
|
||||
type: 'fixedCollection',
|
||||
placeholder: 'Add Field',
|
||||
default: { values: [{ aggregation: 'count', field: '' }] },
|
||||
typeOptions: {
|
||||
multipleValues: true,
|
||||
},
|
||||
options: [
|
||||
{
|
||||
displayName: '',
|
||||
name: 'values',
|
||||
values: [
|
||||
{
|
||||
displayName: 'Aggregation',
|
||||
name: 'aggregation',
|
||||
type: 'options',
|
||||
options: [
|
||||
{
|
||||
name: 'Append',
|
||||
value: 'append',
|
||||
},
|
||||
{
|
||||
name: 'Average',
|
||||
value: 'average',
|
||||
},
|
||||
{
|
||||
name: 'Concatenate',
|
||||
value: 'concatenate',
|
||||
},
|
||||
{
|
||||
name: 'Count',
|
||||
value: 'count',
|
||||
},
|
||||
{
|
||||
name: 'Count Unique',
|
||||
value: 'countUnique',
|
||||
},
|
||||
{
|
||||
name: 'Max',
|
||||
value: 'max',
|
||||
},
|
||||
{
|
||||
name: 'Min',
|
||||
value: 'min',
|
||||
},
|
||||
{
|
||||
name: 'Sum',
|
||||
value: 'sum',
|
||||
},
|
||||
],
|
||||
default: 'count',
|
||||
description: 'How to combine the values of the field you want to summarize',
|
||||
},
|
||||
//field repeated to have different descriptions for different aggregations --------------------------------
|
||||
{
|
||||
displayName: 'Field',
|
||||
name: 'field',
|
||||
type: 'string',
|
||||
default: '',
|
||||
description: 'The name of an input field that you want to summarize',
|
||||
placeholder: 'e.g. cost',
|
||||
hint: ' Enter the field name as text',
|
||||
displayOptions: {
|
||||
hide: {
|
||||
aggregation: [...NUMERICAL_AGGREGATIONS, 'countUnique', 'count'],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
displayName: 'Field',
|
||||
name: 'field',
|
||||
type: 'string',
|
||||
default: '',
|
||||
description:
|
||||
'The name of an input field that you want to summarize. The field should contain numerical values; null, undefined, empty strings would be ignored.',
|
||||
placeholder: 'e.g. cost',
|
||||
hint: ' Enter the field name as text',
|
||||
displayOptions: {
|
||||
show: {
|
||||
aggregation: NUMERICAL_AGGREGATIONS,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
displayName: 'Field',
|
||||
name: 'field',
|
||||
type: 'string',
|
||||
default: '',
|
||||
description:
|
||||
'The name of an input field that you want to summarize; null, undefined, empty strings would be ignored',
|
||||
placeholder: 'e.g. cost',
|
||||
hint: ' Enter the field name as text',
|
||||
displayOptions: {
|
||||
show: {
|
||||
aggregation: ['countUnique', 'count'],
|
||||
},
|
||||
},
|
||||
},
|
||||
// ----------------------------------------------------------------------------------------------------------
|
||||
{
|
||||
displayName: 'Include Empty Values',
|
||||
name: 'includeEmpty',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
displayOptions: {
|
||||
show: {
|
||||
aggregation: ['append', 'concatenate'],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
displayName: 'Separator',
|
||||
name: 'separateBy',
|
||||
type: 'options',
|
||||
default: ',',
|
||||
// eslint-disable-next-line n8n-nodes-base/node-param-options-type-unsorted-items
|
||||
options: [
|
||||
{
|
||||
name: 'Comma',
|
||||
value: ',',
|
||||
},
|
||||
{
|
||||
name: 'Comma and Space',
|
||||
value: ', ',
|
||||
},
|
||||
{
|
||||
name: 'New Line',
|
||||
value: '\n',
|
||||
},
|
||||
{
|
||||
name: 'None',
|
||||
value: '',
|
||||
},
|
||||
{
|
||||
name: 'Space',
|
||||
value: ' ',
|
||||
},
|
||||
{
|
||||
name: 'Other',
|
||||
value: 'other',
|
||||
},
|
||||
],
|
||||
hint: 'What to insert between values',
|
||||
displayOptions: {
|
||||
show: {
|
||||
aggregation: ['concatenate'],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
displayName: 'Custom Separator',
|
||||
name: 'customSeparator',
|
||||
type: 'string',
|
||||
default: '',
|
||||
displayOptions: {
|
||||
show: {
|
||||
aggregation: ['concatenate'],
|
||||
separateBy: ['other'],
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
displayOptions: {
|
||||
show: {
|
||||
resource: ['itemList'],
|
||||
operation: ['summarize'],
|
||||
},
|
||||
},
|
||||
},
|
||||
// fieldsToSplitBy repeated to have different displayName for singleItem and separateItems -----------------------------
|
||||
{
|
||||
displayName: 'Fields to Split By',
|
||||
name: 'fieldsToSplitBy',
|
||||
type: 'string',
|
||||
placeholder: 'e.g. country, city',
|
||||
default: '',
|
||||
description: 'The name of the input fields that you want to split the summary by',
|
||||
hint: 'Enter the name of the fields as text (separated by commas)',
|
||||
displayOptions: {
|
||||
show: {
|
||||
resource: ['itemList'],
|
||||
operation: ['summarize'],
|
||||
},
|
||||
hide: {
|
||||
'/options.outputFormat': ['singleItem'],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
displayName: 'Fields to Group By',
|
||||
name: 'fieldsToSplitBy',
|
||||
type: 'string',
|
||||
placeholder: 'e.g. country, city',
|
||||
default: '',
|
||||
description: 'The name of the input fields that you want to split the summary by',
|
||||
hint: 'Enter the name of the fields as text (separated by commas)',
|
||||
displayOptions: {
|
||||
show: {
|
||||
resource: ['itemList'],
|
||||
operation: ['summarize'],
|
||||
'/options.outputFormat': ['singleItem'],
|
||||
},
|
||||
},
|
||||
},
|
||||
// ----------------------------------------------------------------------------------------------------------
|
||||
{
|
||||
displayName: 'Options',
|
||||
name: 'options',
|
||||
type: 'collection',
|
||||
placeholder: 'Add Option',
|
||||
default: {},
|
||||
displayOptions: {
|
||||
show: {
|
||||
resource: ['itemList'],
|
||||
operation: ['summarize'],
|
||||
},
|
||||
},
|
||||
options: [
|
||||
{
|
||||
displayName: 'Disable Dot Notation',
|
||||
name: 'disableDotNotation',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
description:
|
||||
'Whether to disallow referencing child fields using `parent.child` in the field name',
|
||||
},
|
||||
{
|
||||
displayName: 'Output Format',
|
||||
name: 'outputFormat',
|
||||
type: 'options',
|
||||
default: 'separateItems',
|
||||
options: [
|
||||
{
|
||||
name: 'Each Split in a Separate Item',
|
||||
value: 'separateItems',
|
||||
},
|
||||
{
|
||||
name: 'All Splits in a Single Item',
|
||||
value: 'singleItem',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
// eslint-disable-next-line n8n-nodes-base/node-param-display-name-miscased
|
||||
displayName: 'Ignore items without valid fields to group by',
|
||||
name: 'skipEmptySplitFields',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
function isEmpty<T>(value: T) {
|
||||
return value === undefined || value === null || value === '';
|
||||
}
|
||||
|
||||
const fieldValueGetter = (disableDotNotation?: boolean) => {
|
||||
if (disableDotNotation) {
|
||||
return (item: IDataObject, field: string) => item[field];
|
||||
} else {
|
||||
return (item: IDataObject, field: string) => get(item, field);
|
||||
}
|
||||
};
|
||||
|
||||
function checkIfFieldExists(
|
||||
this: IExecuteFunctions,
|
||||
items: IDataObject[],
|
||||
aggregations: Aggregations,
|
||||
getValue: ValueGetterFn,
|
||||
) {
|
||||
for (const aggregation of aggregations) {
|
||||
if (aggregation.field === '') {
|
||||
continue;
|
||||
}
|
||||
const exist = items.some((item) => getValue(item, aggregation.field) !== undefined);
|
||||
if (!exist) {
|
||||
throw new NodeOperationError(
|
||||
this.getNode(),
|
||||
`The field '${aggregation.field}' does not exist in any items`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function aggregate(items: IDataObject[], entry: Aggregation, getValue: ValueGetterFn) {
|
||||
const { aggregation, field } = entry;
|
||||
let data = [...items];
|
||||
|
||||
if (NUMERICAL_AGGREGATIONS.includes(aggregation)) {
|
||||
data = data.filter(
|
||||
(item) => typeof getValue(item, field) === 'number' && !isEmpty(getValue(item, field)),
|
||||
);
|
||||
}
|
||||
|
||||
switch (aggregation) {
|
||||
//combine operations
|
||||
case 'append':
|
||||
if (!entry.includeEmpty) {
|
||||
data = data.filter((item) => !isEmpty(getValue(item, field)));
|
||||
}
|
||||
return data.map((item) => getValue(item, field));
|
||||
case 'concatenate':
|
||||
const separateBy = entry.separateBy === 'other' ? entry.customSeparator : entry.separateBy;
|
||||
if (!entry.includeEmpty) {
|
||||
data = data.filter((item) => !isEmpty(getValue(item, field)));
|
||||
}
|
||||
return data
|
||||
.map((item) => {
|
||||
let value = getValue(item, field);
|
||||
if (typeof value === 'object') {
|
||||
value = JSON.stringify(value);
|
||||
}
|
||||
if (typeof value === 'undefined') {
|
||||
value = 'undefined';
|
||||
}
|
||||
|
||||
return value;
|
||||
})
|
||||
.join(separateBy);
|
||||
|
||||
//numerical operations
|
||||
case 'average':
|
||||
return (
|
||||
data.reduce((acc, item) => {
|
||||
return acc + (getValue(item, field) as number);
|
||||
}, 0) / data.length
|
||||
);
|
||||
case 'sum':
|
||||
return data.reduce((acc, item) => {
|
||||
return acc + (getValue(item, field) as number);
|
||||
}, 0);
|
||||
case 'min':
|
||||
return Math.min(
|
||||
...(data.map((item) => {
|
||||
return getValue(item, field);
|
||||
}) as number[]),
|
||||
);
|
||||
case 'max':
|
||||
return Math.max(
|
||||
...(data.map((item) => {
|
||||
return getValue(item, field);
|
||||
}) as number[]),
|
||||
);
|
||||
|
||||
//count operations
|
||||
case 'countUnique':
|
||||
return new Set(data.map((item) => getValue(item, field)).filter((item) => !isEmpty(item)))
|
||||
.size;
|
||||
default:
|
||||
//count by default
|
||||
return data.filter((item) => !isEmpty(getValue(item, field))).length;
|
||||
}
|
||||
}
|
||||
|
||||
function aggregateData(
|
||||
data: IDataObject[],
|
||||
fieldsToSummarize: Aggregations,
|
||||
options: SummarizeOptions,
|
||||
getValue: ValueGetterFn,
|
||||
) {
|
||||
const returnData = fieldsToSummarize.reduce((acc, aggregation) => {
|
||||
acc[`${AggregationDisplayNames[aggregation.aggregation]}${aggregation.field}`] = aggregate(
|
||||
data,
|
||||
aggregation,
|
||||
getValue,
|
||||
);
|
||||
return acc;
|
||||
}, {} as IDataObject);
|
||||
if (options.outputFormat === 'singleItem') {
|
||||
return returnData;
|
||||
} else {
|
||||
return { ...returnData, pairedItems: data.map((item) => item._itemIndex as number) };
|
||||
}
|
||||
}
|
||||
|
||||
function splitData(
|
||||
splitKeys: string[],
|
||||
data: IDataObject[],
|
||||
fieldsToSummarize: Aggregations,
|
||||
options: SummarizeOptions,
|
||||
getValue: ValueGetterFn,
|
||||
) {
|
||||
if (!splitKeys || splitKeys.length === 0) {
|
||||
return aggregateData(data, fieldsToSummarize, options, getValue);
|
||||
}
|
||||
|
||||
const [firstSplitKey, ...restSplitKeys] = splitKeys;
|
||||
|
||||
const groupedData = data.reduce((acc, item) => {
|
||||
let keyValuee = getValue(item, firstSplitKey) as string;
|
||||
|
||||
if (typeof keyValuee === 'object') {
|
||||
keyValuee = JSON.stringify(keyValuee);
|
||||
}
|
||||
|
||||
if (options.skipEmptySplitFields && typeof keyValuee !== 'number' && !keyValuee) {
|
||||
return acc;
|
||||
}
|
||||
|
||||
if (acc[keyValuee] === undefined) {
|
||||
acc[keyValuee] = [item];
|
||||
} else {
|
||||
(acc[keyValuee] as IDataObject[]).push(item);
|
||||
}
|
||||
return acc;
|
||||
}, {} as IDataObject);
|
||||
|
||||
return Object.keys(groupedData).reduce((acc, key) => {
|
||||
const value = groupedData[key] as IDataObject[];
|
||||
acc[key] = splitData(restSplitKeys, value, fieldsToSummarize, options, getValue);
|
||||
return acc;
|
||||
}, {} as IDataObject);
|
||||
}
|
||||
|
||||
function aggregationToArray(
|
||||
aggregationResult: IDataObject,
|
||||
fieldsToSplitBy: string[],
|
||||
previousStage: IDataObject = {},
|
||||
) {
|
||||
const returnData: IDataObject[] = [];
|
||||
|
||||
const splitFieldName = fieldsToSplitBy[0];
|
||||
const isNext = fieldsToSplitBy[1];
|
||||
|
||||
if (isNext === undefined) {
|
||||
for (const fieldName of Object.keys(aggregationResult)) {
|
||||
returnData.push({
|
||||
...previousStage,
|
||||
[splitFieldName]: fieldName,
|
||||
...(aggregationResult[fieldName] as IDataObject),
|
||||
});
|
||||
}
|
||||
return returnData;
|
||||
} else {
|
||||
for (const key of Object.keys(aggregationResult)) {
|
||||
returnData.push(
|
||||
...aggregationToArray(aggregationResult[key] as IDataObject, fieldsToSplitBy.slice(1), {
|
||||
...previousStage,
|
||||
[splitFieldName]: key,
|
||||
}),
|
||||
);
|
||||
}
|
||||
return returnData;
|
||||
}
|
||||
}
|
||||
|
||||
export async function execute(
|
||||
this: IExecuteFunctions,
|
||||
items: INodeExecutionData[],
|
||||
): Promise<INodeExecutionData[][]> {
|
||||
const newItems = items.map(({ json }, i) => ({ ...json, _itemIndex: i }));
|
||||
|
||||
const options = this.getNodeParameter('options', 0, {}) as SummarizeOptions;
|
||||
|
||||
const fieldsToSplitBy = (this.getNodeParameter('fieldsToSplitBy', 0, '') as string)
|
||||
.split(',')
|
||||
.map((field) => field.trim())
|
||||
.filter((field) => field);
|
||||
|
||||
const fieldsToSummarize = this.getNodeParameter(
|
||||
'fieldsToSummarize.values',
|
||||
0,
|
||||
[],
|
||||
) as Aggregations;
|
||||
|
||||
if (fieldsToSummarize.filter((aggregation) => aggregation.field !== '').length === 0) {
|
||||
throw new NodeOperationError(
|
||||
this.getNode(),
|
||||
"You need to add at least one aggregation to 'Fields to Summarize' with non empty 'Field'",
|
||||
);
|
||||
}
|
||||
|
||||
const getValue = fieldValueGetter(options.disableDotNotation);
|
||||
|
||||
checkIfFieldExists.call(this, newItems, fieldsToSummarize, getValue);
|
||||
|
||||
const aggregationResult = splitData(
|
||||
fieldsToSplitBy,
|
||||
newItems,
|
||||
fieldsToSummarize,
|
||||
options,
|
||||
getValue,
|
||||
);
|
||||
|
||||
if (options.outputFormat === 'singleItem') {
|
||||
const executionData: INodeExecutionData = {
|
||||
json: aggregationResult,
|
||||
pairedItem: newItems.map((_v, index) => ({
|
||||
item: index,
|
||||
})),
|
||||
};
|
||||
return this.prepareOutputData([executionData]);
|
||||
} else {
|
||||
if (!fieldsToSplitBy.length) {
|
||||
const { pairedItems, ...json } = aggregationResult;
|
||||
const executionData: INodeExecutionData = {
|
||||
json,
|
||||
pairedItem: ((pairedItems as number[]) || []).map((index: number) => ({
|
||||
item: index,
|
||||
})),
|
||||
};
|
||||
return this.prepareOutputData([executionData]);
|
||||
}
|
||||
const returnData = aggregationToArray(aggregationResult, fieldsToSplitBy);
|
||||
const executionData = returnData.map((item) => {
|
||||
const { pairedItems, ...json } = item;
|
||||
return {
|
||||
json,
|
||||
pairedItem: ((pairedItems as number[]) || []).map((index: number) => ({
|
||||
item: index,
|
||||
})),
|
||||
};
|
||||
});
|
||||
return this.prepareOutputData(executionData);
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue