diff --git a/packages/nodes-base/nodes/ItemLists/ItemLists.node.ts b/packages/nodes-base/nodes/ItemLists/ItemLists.node.ts index dade91b64f..d454c14e92 100644 --- a/packages/nodes-base/nodes/ItemLists/ItemLists.node.ts +++ b/packages/nodes-base/nodes/ItemLists/ItemLists.node.ts @@ -2,8 +2,8 @@ import type { INodeTypeBaseDescription, IVersionedNodeType } from 'n8n-workflow' import { VersionedNodeType } from 'n8n-workflow'; import { ItemListsV1 } from './V1/ItemListsV1.node'; - import { ItemListsV2 } from './V2/ItemListsV2.node'; +import { ItemListsV3 } from './V3/ItemListsV3.node'; export class ItemLists extends VersionedNodeType { constructor() { @@ -14,7 +14,7 @@ export class ItemLists extends VersionedNodeType { group: ['input'], subtitle: '={{$parameter["operation"] + ": " + $parameter["resource"]}}', description: 'Helper for working with lists of items and transforming arrays', - defaultVersion: 2.2, + defaultVersion: 3, }; const nodeVersions: IVersionedNodeType['nodeVersions'] = { @@ -22,6 +22,7 @@ export class ItemLists extends VersionedNodeType { 2: new ItemListsV2(baseDescription), 2.1: new ItemListsV2(baseDescription), 2.2: new ItemListsV2(baseDescription), + 3: new ItemListsV3(baseDescription), }; super(nodeVersions, baseDescription); diff --git a/packages/nodes-base/nodes/ItemLists/V3/ItemListsV3.node.ts b/packages/nodes-base/nodes/ItemLists/V3/ItemListsV3.node.ts new file mode 100644 index 0000000000..84c8826619 --- /dev/null +++ b/packages/nodes-base/nodes/ItemLists/V3/ItemListsV3.node.ts @@ -0,0 +1,24 @@ +import type { + IExecuteFunctions, + INodeType, + INodeTypeBaseDescription, + INodeTypeDescription, +} from 'n8n-workflow'; + +import { versionDescription } from './actions/versionDescription'; +import { router } from './actions/router'; + +export class ItemListsV3 implements INodeType { + description: INodeTypeDescription; + + constructor(baseDescription: INodeTypeBaseDescription) { + this.description = { + ...baseDescription, + ...versionDescription, + }; + } + + async execute(this: IExecuteFunctions) { + return router.call(this); + } +} diff --git a/packages/nodes-base/nodes/ItemLists/V3/actions/common.descriptions.ts b/packages/nodes-base/nodes/ItemLists/V3/actions/common.descriptions.ts new file mode 100644 index 0000000000..d25b10e68b --- /dev/null +++ b/packages/nodes-base/nodes/ItemLists/V3/actions/common.descriptions.ts @@ -0,0 +1,10 @@ +import type { INodeProperties } from 'n8n-workflow'; + +export const disableDotNotationBoolean: INodeProperties = { + displayName: 'Disable Dot Notation', + name: 'disableDotNotation', + type: 'boolean', + default: false, + description: + 'Whether to disallow referencing child fields using `parent.child` in the field name', +}; diff --git a/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/concatenateItems.operation.ts b/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/concatenateItems.operation.ts new file mode 100644 index 0000000000..99a4279471 --- /dev/null +++ b/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/concatenateItems.operation.ts @@ -0,0 +1,349 @@ +import type { IExecuteFunctions } from 'n8n-core'; +import type { IDataObject, INodeExecutionData, INodeProperties } from 'n8n-workflow'; +import { NodeOperationError } from 'n8n-workflow'; + +import { updateDisplayOptions } from '@utils/utilities'; + +import get from 'lodash/get'; +import isEmpty from 'lodash/isEmpty'; +import set from 'lodash/set'; + +import { prepareFieldsArray } from '../../helpers/utils'; +import { disableDotNotationBoolean } from '../common.descriptions'; + +const properties: INodeProperties[] = [ + { + displayName: 'Aggregate', + name: 'aggregate', + type: 'options', + default: 'aggregateIndividualFields', + options: [ + { + name: 'Individual Fields', + value: 'aggregateIndividualFields', + }, + { + name: 'All Item Data (Into a Single List)', + value: 'aggregateAllItemData', + }, + ], + }, + { + displayName: 'Fields To Aggregate', + name: 'fieldsToAggregate', + type: 'fixedCollection', + typeOptions: { + multipleValues: true, + }, + placeholder: 'Add Field To Aggregate', + default: { fieldToAggregate: [{ fieldToAggregate: '', renameField: false }] }, + displayOptions: { + show: { + aggregate: ['aggregateIndividualFields'], + }, + }, + options: [ + { + displayName: '', + name: 'fieldToAggregate', + values: [ + { + displayName: 'Input Field Name', + name: 'fieldToAggregate', + type: 'string', + default: '', + description: 'The name of a field in the input items to aggregate together', + // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id + placeholder: 'e.g. id', + hint: ' Enter the field name as text', + requiresDataPath: 'single', + }, + { + displayName: 'Rename Field', + name: 'renameField', + type: 'boolean', + default: false, + description: 'Whether to give the field a different name in the output', + }, + { + displayName: 'Output Field Name', + name: 'outputFieldName', + displayOptions: { + show: { + renameField: [true], + }, + }, + type: 'string', + default: '', + description: + 'The name of the field to put the aggregated data in. Leave blank to use the input field name.', + requiresDataPath: 'single', + }, + ], + }, + ], + }, + { + displayName: 'Put Output in Field', + name: 'destinationFieldName', + type: 'string', + displayOptions: { + show: { + aggregate: ['aggregateAllItemData'], + }, + }, + default: 'data', + description: 'The name of the output field to put the data in', + }, + { + displayName: 'Include', + name: 'include', + type: 'options', + default: 'allFields', + options: [ + { + name: 'All Fields', + value: 'allFields', + }, + { + name: 'Specified Fields', + value: 'specifiedFields', + }, + { + name: 'All Fields Except', + value: 'allFieldsExcept', + }, + ], + displayOptions: { + show: { + aggregate: ['aggregateAllItemData'], + }, + }, + }, + { + displayName: 'Fields To Exclude', + name: 'fieldsToExclude', + type: 'string', + placeholder: 'e.g. email, name', + default: '', + requiresDataPath: 'multiple', + displayOptions: { + show: { + aggregate: ['aggregateAllItemData'], + include: ['allFieldsExcept'], + }, + }, + }, + { + displayName: 'Fields To Include', + name: 'fieldsToInclude', + type: 'string', + placeholder: 'e.g. email, name', + default: '', + requiresDataPath: 'multiple', + displayOptions: { + show: { + aggregate: ['aggregateAllItemData'], + include: ['specifiedFields'], + }, + }, + }, + { + displayName: 'Options', + name: 'options', + type: 'collection', + placeholder: 'Add Field', + default: {}, + displayOptions: { + hide: { + aggregate: ['aggregateAllItemData'], + }, + }, + options: [ + disableDotNotationBoolean, + { + displayName: 'Merge Lists', + name: 'mergeLists', + type: 'boolean', + default: false, + description: + 'Whether to merge the output into a single flat list (rather than a list of lists), if the field to aggregate is a list', + }, + { + displayName: 'Keep Missing And Null Values', + name: 'keepMissing', + type: 'boolean', + default: false, + description: + 'Whether to add a null entry to the aggregated list when there is a missing or null value', + }, + ], + }, +]; + +const displayOptions = { + show: { + resource: ['itemList'], + operation: ['concatenateItems'], + }, +}; + +export const description = updateDisplayOptions(displayOptions, properties); + +export async function execute( + this: IExecuteFunctions, + items: INodeExecutionData[], +): Promise { + const returnData: INodeExecutionData[] = []; + + const aggregate = this.getNodeParameter('aggregate', 0, '') as string; + + if (aggregate === 'aggregateIndividualFields') { + const disableDotNotation = this.getNodeParameter( + 'options.disableDotNotation', + 0, + false, + ) as boolean; + const mergeLists = this.getNodeParameter('options.mergeLists', 0, false) as boolean; + const fieldsToAggregate = this.getNodeParameter( + 'fieldsToAggregate.fieldToAggregate', + 0, + [], + ) as [{ fieldToAggregate: string; renameField: boolean; outputFieldName: string }]; + const keepMissing = this.getNodeParameter('options.keepMissing', 0, false) as boolean; + + if (!fieldsToAggregate.length) { + throw new NodeOperationError(this.getNode(), 'No fields specified', { + description: 'Please add a field to aggregate', + }); + } + + const newItem: INodeExecutionData = { + json: {}, + pairedItem: Array.from({ length: items.length }, (_, i) => i).map((index) => { + return { + item: index, + }; + }), + }; + + const values: { [key: string]: any } = {}; + const outputFields: string[] = []; + + for (const { fieldToAggregate, outputFieldName, renameField } of fieldsToAggregate) { + const field = renameField ? outputFieldName : fieldToAggregate; + + if (outputFields.includes(field)) { + throw new NodeOperationError( + this.getNode(), + `The '${field}' output field is used more than once`, + { description: 'Please make sure each output field name is unique' }, + ); + } else { + outputFields.push(field); + } + + const getFieldToAggregate = () => + !disableDotNotation && fieldToAggregate.includes('.') + ? fieldToAggregate.split('.').pop() + : fieldToAggregate; + + const _outputFieldName = outputFieldName + ? outputFieldName + : (getFieldToAggregate() as string); + + if (fieldToAggregate !== '') { + values[_outputFieldName] = []; + for (let i = 0; i < items.length; i++) { + if (!disableDotNotation) { + let value = get(items[i].json, fieldToAggregate); + + if (!keepMissing) { + if (Array.isArray(value)) { + value = value.filter((entry) => entry !== null); + } else if (value === null || value === undefined) { + continue; + } + } + + if (Array.isArray(value) && mergeLists) { + values[_outputFieldName].push(...value); + } else { + values[_outputFieldName].push(value); + } + } else { + let value = items[i].json[fieldToAggregate]; + + if (!keepMissing) { + if (Array.isArray(value)) { + value = value.filter((entry) => entry !== null); + } else if (value === null || value === undefined) { + continue; + } + } + + if (Array.isArray(value) && mergeLists) { + values[_outputFieldName].push(...value); + } else { + values[_outputFieldName].push(value); + } + } + } + } + } + + for (const key of Object.keys(values)) { + if (!disableDotNotation) { + set(newItem.json, key, values[key]); + } else { + newItem.json[key] = values[key]; + } + } + + returnData.push(newItem); + } else { + let newItems: IDataObject[] = items.map((item) => item.json); + const destinationFieldName = this.getNodeParameter('destinationFieldName', 0) as string; + + const fieldsToExclude = prepareFieldsArray( + this.getNodeParameter('fieldsToExclude', 0, '') as string, + 'Fields To Exclude', + ); + + const fieldsToInclude = prepareFieldsArray( + this.getNodeParameter('fieldsToInclude', 0, '') as string, + 'Fields To Include', + ); + + if (fieldsToExclude.length || fieldsToInclude.length) { + newItems = newItems.reduce((acc, item) => { + const newItem: IDataObject = {}; + let outputFields = Object.keys(item); + + if (fieldsToExclude.length) { + outputFields = outputFields.filter((key) => !fieldsToExclude.includes(key)); + } + if (fieldsToInclude.length) { + outputFields = outputFields.filter((key) => + fieldsToInclude.length ? fieldsToInclude.includes(key) : true, + ); + } + + outputFields.forEach((key) => { + newItem[key] = item[key]; + }); + + if (isEmpty(newItem)) { + return acc; + } + return acc.concat([newItem]); + }, [] as IDataObject[]); + } + + const output: INodeExecutionData = { json: { [destinationFieldName]: newItems } }; + + returnData.push(output); + } + + return returnData; +} diff --git a/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/index.ts b/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/index.ts new file mode 100644 index 0000000000..6cb20a66e3 --- /dev/null +++ b/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/index.ts @@ -0,0 +1,70 @@ +import type { INodeProperties } from 'n8n-workflow'; + +import * as concatenateItems from './concatenateItems.operation'; +import * as limit from './limit.operation'; +import * as removeDuplicates from './removeDuplicates.operation'; +import * as sort from './sort.operation'; +import * as splitOutItems from './splitOutItems.operation'; +import * as summarize from './summarize.operation'; + +export { concatenateItems, limit, removeDuplicates, sort, splitOutItems, summarize }; + +export const description: INodeProperties[] = [ + { + displayName: 'Operation', + name: 'operation', + type: 'options', + noDataExpression: true, + displayOptions: { + show: { + resource: ['itemList'], + }, + }, + options: [ + { + name: 'Concatenate Items', + value: 'concatenateItems', + description: 'Combine fields into a list in a single new item', + action: 'Concatenate Items', + }, + { + name: 'Limit', + value: 'limit', + description: 'Remove items if there are too many', + action: 'Limit', + }, + { + name: 'Remove Duplicates', + value: 'removeDuplicates', + description: 'Remove extra items that are similar', + action: 'Remove Duplicates', + }, + { + name: 'Sort', + value: 'sort', + description: 'Change the item order', + action: 'Sort', + }, + { + name: 'Split Out Items', + value: 'splitOutItems', + description: + "Turn a list or values of object's properties inside item(s) into separate items", + action: 'Split Out Items', + }, + { + name: 'Summarize', + value: 'summarize', + description: 'Aggregate items together (pivot table)', + action: 'Summarize', + }, + ], + default: 'splitOutItems', + }, + ...concatenateItems.description, + ...limit.description, + ...removeDuplicates.description, + ...sort.description, + ...splitOutItems.description, + ...summarize.description, +]; diff --git a/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/limit.operation.ts b/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/limit.operation.ts new file mode 100644 index 0000000000..8fbbbf3c6e --- /dev/null +++ b/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/limit.operation.ts @@ -0,0 +1,62 @@ +import type { IExecuteFunctions } from 'n8n-core'; +import type { INodeExecutionData, INodeProperties } from 'n8n-workflow'; +import { updateDisplayOptions } from '@utils/utilities'; + +const properties: INodeProperties[] = [ + { + displayName: 'Max Items', + name: 'maxItems', + type: 'number', + typeOptions: { + minValue: 1, + }, + default: 1, + description: 'If there are more items than this number, some are removed', + }, + { + displayName: 'Keep', + name: 'keep', + type: 'options', + options: [ + { + name: 'First Items', + value: 'firstItems', + }, + { + name: 'Last Items', + value: 'lastItems', + }, + ], + default: 'firstItems', + description: 'When removing items, whether to keep the ones at the start or the ending', + }, +]; + +const displayOptions = { + show: { + resource: ['itemList'], + operation: ['limit'], + }, +}; + +export const description = updateDisplayOptions(displayOptions, properties); + +export async function execute( + this: IExecuteFunctions, + items: INodeExecutionData[], +): Promise { + let returnData = items; + const maxItems = this.getNodeParameter('maxItems', 0) as number; + const keep = this.getNodeParameter('keep', 0) as string; + + if (maxItems > items.length) { + return returnData; + } + + if (keep === 'firstItems') { + returnData = items.slice(0, maxItems); + } else { + returnData = items.slice(items.length - maxItems, items.length); + } + return returnData; +} diff --git a/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/removeDuplicates.operation.ts b/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/removeDuplicates.operation.ts new file mode 100644 index 0000000000..c13e0f6b75 --- /dev/null +++ b/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/removeDuplicates.operation.ts @@ -0,0 +1,246 @@ +import type { IExecuteFunctions } from 'n8n-core'; +import type { INodeExecutionData, INodeProperties } from 'n8n-workflow'; +import { NodeOperationError } from 'n8n-workflow'; + +import { updateDisplayOptions } from '@utils/utilities'; + +import get from 'lodash/get'; +import isEqual from 'lodash/isEqual'; +import lt from 'lodash/lt'; +import pick from 'lodash/pick'; + +import { compareItems, flattenKeys, prepareFieldsArray } from '../../helpers/utils'; +import { disableDotNotationBoolean } from '../common.descriptions'; + +const properties: INodeProperties[] = [ + { + displayName: 'Compare', + name: 'compare', + type: 'options', + options: [ + { + name: 'All Fields', + value: 'allFields', + }, + { + name: 'All Fields Except', + value: 'allFieldsExcept', + }, + { + name: 'Selected Fields', + value: 'selectedFields', + }, + ], + default: 'allFields', + description: 'The fields of the input items to compare to see if they are the same', + }, + { + displayName: 'Fields To Exclude', + name: 'fieldsToExclude', + type: 'string', + placeholder: 'e.g. email, name', + requiresDataPath: 'multiple', + description: 'Fields in the input to exclude from the comparison', + default: '', + displayOptions: { + show: { + compare: ['allFieldsExcept'], + }, + }, + }, + { + displayName: 'Fields To Compare', + name: 'fieldsToCompare', + type: 'string', + placeholder: 'e.g. email, name', + requiresDataPath: 'multiple', + description: 'Fields in the input to add to the comparison', + default: '', + displayOptions: { + show: { + compare: ['selectedFields'], + }, + }, + }, + { + displayName: 'Options', + name: 'options', + type: 'collection', + placeholder: 'Add Field', + default: {}, + displayOptions: { + show: { + compare: ['allFieldsExcept', 'selectedFields'], + }, + }, + options: [ + disableDotNotationBoolean, + { + displayName: 'Remove Other Fields', + name: 'removeOtherFields', + type: 'boolean', + default: false, + description: + 'Whether to remove any fields that are not being compared. If disabled, will keep the values from the first of the duplicates.', + }, + ], + }, +]; + +const displayOptions = { + show: { + resource: ['itemList'], + operation: ['removeDuplicates'], + }, +}; + +export const description = updateDisplayOptions(displayOptions, properties); + +export async function execute( + this: IExecuteFunctions, + items: INodeExecutionData[], +): Promise { + const compare = this.getNodeParameter('compare', 0) as string; + const disableDotNotation = this.getNodeParameter( + 'options.disableDotNotation', + 0, + false, + ) as boolean; + const removeOtherFields = this.getNodeParameter('options.removeOtherFields', 0, false) as boolean; + + let keys = disableDotNotation + ? Object.keys(items[0].json) + : Object.keys(flattenKeys(items[0].json)); + + for (const item of items) { + for (const key of disableDotNotation + ? Object.keys(item.json) + : Object.keys(flattenKeys(item.json))) { + if (!keys.includes(key)) { + keys.push(key); + } + } + } + + if (compare === 'allFieldsExcept') { + const fieldsToExclude = prepareFieldsArray( + this.getNodeParameter('fieldsToExclude', 0, '') as string, + 'Fields To Exclude', + ); + + if (!fieldsToExclude.length) { + throw new NodeOperationError( + this.getNode(), + 'No fields specified. Please add a field to exclude from comparison', + ); + } + if (!disableDotNotation) { + keys = Object.keys(flattenKeys(items[0].json)); + } + keys = keys.filter((key) => !fieldsToExclude.includes(key)); + } + if (compare === 'selectedFields') { + const fieldsToCompare = prepareFieldsArray( + this.getNodeParameter('fieldsToCompare', 0, '') as string, + 'Fields To Compare', + ); + if (!fieldsToCompare.length) { + throw new NodeOperationError( + this.getNode(), + 'No fields specified. Please add a field to compare on', + ); + } + if (!disableDotNotation) { + keys = Object.keys(flattenKeys(items[0].json)); + } + keys = fieldsToCompare.map((key) => key.trim()); + } + + // This solution is O(nlogn) + // add original index to the items + const newItems = items.map( + (item, index) => + ({ + json: { ...item.json, __INDEX: index }, + pairedItem: { item: index }, + } as INodeExecutionData), + ); + //sort items using the compare keys + newItems.sort((a, b) => { + let result = 0; + + for (const key of keys) { + let equal; + if (!disableDotNotation) { + equal = isEqual(get(a.json, key), get(b.json, key)); + } else { + equal = isEqual(a.json[key], b.json[key]); + } + if (!equal) { + let lessThan; + if (!disableDotNotation) { + lessThan = lt(get(a.json, key), get(b.json, key)); + } else { + lessThan = lt(a.json[key], b.json[key]); + } + result = lessThan ? -1 : 1; + break; + } + } + return result; + }); + + for (const key of keys) { + let type: any = undefined; + for (const item of newItems) { + if (key === '') { + throw new NodeOperationError(this.getNode(), 'Name of field to compare is blank'); + } + const value = !disableDotNotation ? get(item.json, key) : item.json[key]; + if (value === undefined && disableDotNotation && key.includes('.')) { + throw new NodeOperationError( + this.getNode(), + `'${key}' field is missing from some input items`, + { + description: + "If you're trying to use a nested field, make sure you turn off 'disable dot notation' in the node options", + }, + ); + } else if (value === undefined) { + throw new NodeOperationError( + this.getNode(), + `'${key}' field is missing from some input items`, + ); + } + if (type !== undefined && value !== undefined && type !== typeof value) { + throw new NodeOperationError(this.getNode(), `'${key}' isn't always the same type`, { + description: 'The type of this field varies between items', + }); + } else { + type = typeof value; + } + } + } + + // collect the original indexes of items to be removed + const removedIndexes: number[] = []; + let temp = newItems[0]; + for (let index = 1; index < newItems.length; index++) { + if (compareItems(newItems[index], temp, keys, disableDotNotation, this.getNode())) { + removedIndexes.push(newItems[index].json.__INDEX as unknown as number); + } else { + temp = newItems[index]; + } + } + + let returnData = items.filter((_, index) => !removedIndexes.includes(index)); + + if (removeOtherFields) { + returnData = returnData.map((item, index) => ({ + json: pick(item.json, ...keys), + pairedItem: { item: index }, + })); + } + + return returnData; +} diff --git a/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/sort.operation.ts b/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/sort.operation.ts new file mode 100644 index 0000000000..18d2392dc2 --- /dev/null +++ b/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/sort.operation.ts @@ -0,0 +1,303 @@ +import type { IExecuteFunctions } from 'n8n-core'; +import type { IDataObject, INodeExecutionData, INodeProperties } from 'n8n-workflow'; +import { NodeOperationError } from 'n8n-workflow'; +import { updateDisplayOptions } from '@utils/utilities'; + +import type { NodeVMOptions } from 'vm2'; +import { NodeVM } from 'vm2'; + +import get from 'lodash/get'; + +import isEqual from 'lodash/isEqual'; +import lt from 'lodash/lt'; + +import { shuffleArray } from '../../helpers/utils'; +import { disableDotNotationBoolean } from '../common.descriptions'; + +const properties: INodeProperties[] = [ + { + displayName: 'Type', + name: 'type', + type: 'options', + options: [ + { + name: 'Simple', + value: 'simple', + }, + { + name: 'Random', + value: 'random', + }, + { + name: 'Code', + value: 'code', + }, + ], + default: 'simple', + description: 'The fields of the input items to compare to see if they are the same', + }, + { + displayName: 'Fields To Sort By', + name: 'sortFieldsUi', + type: 'fixedCollection', + typeOptions: { + multipleValues: true, + }, + placeholder: 'Add Field To Sort By', + options: [ + { + displayName: '', + name: 'sortField', + values: [ + { + displayName: 'Field Name', + name: 'fieldName', + type: 'string', + required: true, + default: '', + description: 'The field to sort by', + // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id + placeholder: 'e.g. id', + hint: ' Enter the field name as text', + requiresDataPath: 'single', + }, + { + displayName: 'Order', + name: 'order', + type: 'options', + options: [ + { + name: 'Ascending', + value: 'ascending', + }, + { + name: 'Descending', + value: 'descending', + }, + ], + default: 'ascending', + description: 'The order to sort by', + }, + ], + }, + ], + default: {}, + description: 'The fields of the input items to compare to see if they are the same', + displayOptions: { + show: { + type: ['simple'], + }, + }, + }, + { + displayName: 'Code', + name: 'code', + type: 'string', + typeOptions: { + alwaysOpenEditWindow: true, + editor: 'code', + rows: 10, + }, + default: `// The two items to compare are in the variables a and b +// Access the fields in a.json and b.json +// Return -1 if a should go before b +// Return 1 if b should go before a +// Return 0 if there's no difference + +fieldName = 'myField'; + +if (a.json[fieldName] < b.json[fieldName]) { +return -1; +} +if (a.json[fieldName] > b.json[fieldName]) { +return 1; +} +return 0;`, + description: 'Javascript code to determine the order of any two items', + displayOptions: { + show: { + type: ['code'], + }, + }, + }, + { + displayName: 'Options', + name: 'options', + type: 'collection', + placeholder: 'Add Field', + default: {}, + displayOptions: { + show: { + type: ['simple'], + }, + }, + options: [disableDotNotationBoolean], + }, +]; + +const displayOptions = { + show: { + resource: ['itemList'], + operation: ['sort'], + }, +}; + +export const description = updateDisplayOptions(displayOptions, properties); + +export async function execute( + this: IExecuteFunctions, + items: INodeExecutionData[], +): Promise { + let returnData = [...items]; + const type = this.getNodeParameter('type', 0) as string; + const disableDotNotation = this.getNodeParameter( + 'options.disableDotNotation', + 0, + false, + ) as boolean; + + if (type === 'random') { + shuffleArray(returnData); + return returnData; + } + + if (type === 'simple') { + const sortFieldsUi = this.getNodeParameter('sortFieldsUi', 0) as IDataObject; + const sortFields = sortFieldsUi.sortField as Array<{ + fieldName: string; + order: 'ascending' | 'descending'; + }>; + + if (!sortFields?.length) { + throw new NodeOperationError( + this.getNode(), + 'No sorting specified. Please add a field to sort by', + ); + } + + for (const { fieldName } of sortFields) { + let found = false; + for (const item of items) { + if (!disableDotNotation) { + if (get(item.json, fieldName) !== undefined) { + found = true; + } + } else if (item.json.hasOwnProperty(fieldName)) { + found = true; + } + } + if (!found && disableDotNotation && fieldName.includes('.')) { + throw new NodeOperationError( + this.getNode(), + `Couldn't find the field '${fieldName}' in the input data`, + { + description: + "If you're trying to use a nested field, make sure you turn off 'disable dot notation' in the node options", + }, + ); + } else if (!found) { + throw new NodeOperationError( + this.getNode(), + `Couldn't find the field '${fieldName}' in the input data`, + ); + } + } + + const sortFieldsWithDirection = sortFields.map((field) => ({ + name: field.fieldName, + dir: field.order === 'ascending' ? 1 : -1, + })); + + returnData.sort((a, b) => { + let result = 0; + for (const field of sortFieldsWithDirection) { + let equal; + if (!disableDotNotation) { + const _a = + typeof get(a.json, field.name) === 'string' + ? (get(a.json, field.name) as string).toLowerCase() + : get(a.json, field.name); + const _b = + typeof get(b.json, field.name) === 'string' + ? (get(b.json, field.name) as string).toLowerCase() + : get(b.json, field.name); + equal = isEqual(_a, _b); + } else { + const _a = + typeof a.json[field.name] === 'string' + ? (a.json[field.name] as string).toLowerCase() + : a.json[field.name]; + const _b = + typeof b.json[field.name] === 'string' + ? (b.json[field.name] as string).toLowerCase() + : b.json[field.name]; + equal = isEqual(_a, _b); + } + + if (!equal) { + let lessThan; + if (!disableDotNotation) { + const _a = + typeof get(a.json, field.name) === 'string' + ? (get(a.json, field.name) as string).toLowerCase() + : get(a.json, field.name); + const _b = + typeof get(b.json, field.name) === 'string' + ? (get(b.json, field.name) as string).toLowerCase() + : get(b.json, field.name); + lessThan = lt(_a, _b); + } else { + const _a = + typeof a.json[field.name] === 'string' + ? (a.json[field.name] as string).toLowerCase() + : a.json[field.name]; + const _b = + typeof b.json[field.name] === 'string' + ? (b.json[field.name] as string).toLowerCase() + : b.json[field.name]; + lessThan = lt(_a, _b); + } + if (lessThan) { + result = -1 * field.dir; + } else { + result = 1 * field.dir; + } + break; + } + } + return result; + }); + } else { + const code = this.getNodeParameter('code', 0) as string; + const regexCheck = /\breturn\b/g.exec(code); + + if (regexCheck?.length) { + const sandbox = { + newItems: returnData, + }; + const mode = this.getMode(); + const options = { + console: mode === 'manual' ? 'redirect' : 'inherit', + sandbox, + }; + const vm = new NodeVM(options as unknown as NodeVMOptions); + + returnData = await vm.run( + ` + module.exports = async function() { + newItems.sort( (a,b) => { + ${code} + }) + return newItems; + }()`, + __dirname, + ); + } else { + throw new NodeOperationError( + this.getNode(), + "Sort code doesn't return. Please add a 'return' statement to your code", + ); + } + } + return returnData; +} diff --git a/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/splitOutItems.operation.ts b/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/splitOutItems.operation.ts new file mode 100644 index 0000000000..cc4ee758a6 --- /dev/null +++ b/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/splitOutItems.operation.ts @@ -0,0 +1,218 @@ +import type { IExecuteFunctions } from 'n8n-core'; +import type { IDataObject, INodeExecutionData, INodeProperties } from 'n8n-workflow'; +import { deepCopy, NodeOperationError } from 'n8n-workflow'; + +import { updateDisplayOptions } from '@utils/utilities'; + +import get from 'lodash/get'; +import unset from 'lodash/unset'; +import { disableDotNotationBoolean } from '../common.descriptions'; +import { prepareFieldsArray } from '../../helpers/utils'; + +const properties: INodeProperties[] = [ + { + displayName: 'Fields To Split Out', + name: 'fieldToSplitOut', + type: 'string', + default: '', + required: true, + description: 'The name of the input fields to break out into separate items', + requiresDataPath: 'multiple', + }, + { + displayName: 'Include', + name: 'include', + type: 'options', + options: [ + { + name: 'No Other Fields', + value: 'noOtherFields', + }, + { + name: 'All Other Fields', + value: 'allOtherFields', + }, + { + name: 'Selected Other Fields', + value: 'selectedOtherFields', + }, + ], + default: 'noOtherFields', + description: 'Whether to copy any other fields into the new items', + }, + { + displayName: 'Fields To Include', + name: 'fieldsToInclude', + type: 'string', + placeholder: 'e.g. email, name', + requiresDataPath: 'multiple', + description: 'Fields in the input items to aggregate together', + default: '', + displayOptions: { + show: { + include: ['selectedOtherFields'], + }, + }, + }, + { + displayName: 'Options', + name: 'options', + type: 'collection', + placeholder: 'Add Field', + default: {}, + options: [ + disableDotNotationBoolean, + { + displayName: 'Destination Field Name', + name: 'destinationFieldName', + type: 'string', + requiresDataPath: 'multiple', + default: '', + description: 'The field in the output under which to put the split field contents', + }, + ], + }, +]; + +const displayOptions = { + show: { + resource: ['itemList'], + operation: ['splitOutItems'], + }, +}; + +export const description = updateDisplayOptions(displayOptions, properties); + +export async function execute( + this: IExecuteFunctions, + items: INodeExecutionData[], +): Promise { + const returnData: INodeExecutionData[] = []; + + for (let i = 0; i < items.length; i++) { + const fieldsToSplitOut = (this.getNodeParameter('fieldToSplitOut', i) as string) + .split(',') + .map((field) => field.trim()); + const disableDotNotation = this.getNodeParameter( + 'options.disableDotNotation', + 0, + false, + ) as boolean; + + const destinationFields = ( + this.getNodeParameter('options.destinationFieldName', i, '') as string + ) + .split(',') + .filter((field) => field.trim() !== '') + .map((field) => field.trim()); + + if (destinationFields.length && destinationFields.length !== fieldsToSplitOut.length) { + throw new NodeOperationError( + this.getNode(), + 'If multiple fields to split out are given, the same number of destination fields must be given', + ); + } + + const include = this.getNodeParameter('include', i) as + | 'selectedOtherFields' + | 'allOtherFields' + | 'noOtherFields'; + + const multiSplit = fieldsToSplitOut.length > 1; + + const item = { ...items[i].json }; + const splited: IDataObject[] = []; + for (const [entryIndex, fieldToSplitOut] of fieldsToSplitOut.entries()) { + const destinationFieldName = destinationFields[entryIndex] || ''; + + let arrayToSplit; + if (!disableDotNotation) { + arrayToSplit = get(item, fieldToSplitOut); + } else { + arrayToSplit = item[fieldToSplitOut]; + } + + if (arrayToSplit === undefined) { + arrayToSplit = []; + } + + if (typeof arrayToSplit !== 'object' || arrayToSplit === null) { + arrayToSplit = [arrayToSplit]; + } + + if (!Array.isArray(arrayToSplit)) { + arrayToSplit = Object.values(arrayToSplit); + } + + for (const [elementIndex, element] of arrayToSplit.entries()) { + if (splited[elementIndex] === undefined) { + splited[elementIndex] = {}; + } + + const fieldName = destinationFieldName || fieldToSplitOut; + + if (typeof element === 'object' && element !== null && include === 'noOtherFields') { + if (destinationFieldName === '' && !multiSplit) { + splited[elementIndex] = { ...splited[elementIndex], ...element }; + } else { + splited[elementIndex][fieldName] = element; + } + } else { + splited[elementIndex][fieldName] = element; + } + } + } + + for (const splitEntry of splited) { + let newItem: IDataObject = {}; + + if (include === 'noOtherFields') { + newItem = splitEntry; + } + + if (include === 'allOtherFields') { + const itemCopy = deepCopy(item); + for (const fieldToSplitOut of fieldsToSplitOut) { + if (!disableDotNotation) { + unset(itemCopy, fieldToSplitOut); + } else { + delete itemCopy[fieldToSplitOut]; + } + } + newItem = { ...itemCopy, ...splitEntry }; + } + + if (include === 'selectedOtherFields') { + const fieldsToInclude = prepareFieldsArray( + this.getNodeParameter('fieldsToInclude', i, '') as string, + 'Fields To Include', + ); + + if (!fieldsToInclude.length) { + throw new NodeOperationError(this.getNode(), 'No fields specified', { + description: 'Please add a field to include', + }); + } + + for (const field of fieldsToInclude) { + if (!disableDotNotation) { + splitEntry[field] = get(item, field); + } else { + splitEntry[field] = item[field]; + } + } + + newItem = splitEntry; + } + + returnData.push({ + json: newItem, + pairedItem: { + item: i, + }, + }); + } + } + + return returnData; +} diff --git a/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/summarize.operation.ts b/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/summarize.operation.ts new file mode 100644 index 0000000000..4ff21ac137 --- /dev/null +++ b/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/summarize.operation.ts @@ -0,0 +1,617 @@ +import type { + GenericValue, + IDataObject, + IExecuteFunctions, + INodeExecutionData, + INodeProperties, +} from 'n8n-workflow'; +import { NodeOperationError } from 'n8n-workflow'; + +import { updateDisplayOptions } from '@utils/utilities'; + +import get from 'lodash/get'; +import { disableDotNotationBoolean } from '../common.descriptions'; + +type AggregationType = + | 'append' + | 'average' + | 'concatenate' + | 'count' + | 'countUnique' + | 'max' + | 'min' + | 'sum'; + +type Aggregation = { + aggregation: AggregationType; + field: string; + includeEmpty?: boolean; + separateBy?: string; + customSeparator?: string; +}; + +type Aggregations = Aggregation[]; + +// eslint-disable-next-line no-restricted-syntax +const AggregationDisplayNames = { + append: 'appended_', + average: 'average_', + concatenate: 'concatenated_', + count: 'count_', + countUnique: 'unique_count_', + max: 'max_', + min: 'min_', + sum: 'sum_', +}; + +const NUMERICAL_AGGREGATIONS = ['average', 'sum']; + +type SummarizeOptions = { + disableDotNotation?: boolean; + outputFormat?: 'separateItems' | 'singleItem'; + skipEmptySplitFields?: boolean; +}; + +type ValueGetterFn = ( + item: IDataObject, + field: string, +) => IDataObject | IDataObject[] | GenericValue | GenericValue[]; + +export const properties: INodeProperties[] = [ + { + displayName: 'Fields to Summarize', + name: 'fieldsToSummarize', + type: 'fixedCollection', + placeholder: 'Add Field', + default: { values: [{ aggregation: 'count', field: '' }] }, + typeOptions: { + multipleValues: true, + }, + options: [ + { + displayName: '', + name: 'values', + values: [ + { + displayName: 'Aggregation', + name: 'aggregation', + type: 'options', + options: [ + { + name: 'Append', + value: 'append', + }, + { + name: 'Average', + value: 'average', + }, + { + name: 'Concatenate', + value: 'concatenate', + }, + { + name: 'Count', + value: 'count', + }, + { + name: 'Count Unique', + value: 'countUnique', + }, + { + name: 'Max', + value: 'max', + }, + { + name: 'Min', + value: 'min', + }, + { + name: 'Sum', + value: 'sum', + }, + ], + default: 'count', + description: 'How to combine the values of the field you want to summarize', + }, + //field repeated to have different descriptions for different aggregations -------------------------------- + { + displayName: 'Field', + name: 'field', + type: 'string', + default: '', + description: 'The name of an input field that you want to summarize', + placeholder: 'e.g. cost', + hint: ' Enter the field name as text', + displayOptions: { + hide: { + aggregation: [...NUMERICAL_AGGREGATIONS, 'countUnique', 'count', 'max', 'min'], + }, + }, + requiresDataPath: 'single', + }, + { + displayName: 'Field', + name: 'field', + type: 'string', + default: '', + description: + 'The name of an input field that you want to summarize. The field should contain numerical values; null, undefined, empty strings would be ignored.', + placeholder: 'e.g. cost', + hint: ' Enter the field name as text', + displayOptions: { + show: { + aggregation: NUMERICAL_AGGREGATIONS, + }, + }, + requiresDataPath: 'single', + }, + { + displayName: 'Field', + name: 'field', + type: 'string', + default: '', + description: + 'The name of an input field that you want to summarize; null, undefined, empty strings would be ignored', + placeholder: 'e.g. cost', + hint: ' Enter the field name as text', + displayOptions: { + show: { + aggregation: ['countUnique', 'count', 'max', 'min'], + }, + }, + requiresDataPath: 'single', + }, + // ---------------------------------------------------------------------------------------------------------- + { + displayName: 'Include Empty Values', + name: 'includeEmpty', + type: 'boolean', + default: false, + displayOptions: { + show: { + aggregation: ['append', 'concatenate'], + }, + }, + }, + { + displayName: 'Separator', + name: 'separateBy', + type: 'options', + default: ',', + // eslint-disable-next-line n8n-nodes-base/node-param-options-type-unsorted-items + options: [ + { + name: 'Comma', + value: ',', + }, + { + name: 'Comma and Space', + value: ', ', + }, + { + name: 'New Line', + value: '\n', + }, + { + name: 'None', + value: '', + }, + { + name: 'Space', + value: ' ', + }, + { + name: 'Other', + value: 'other', + }, + ], + hint: 'What to insert between values', + displayOptions: { + show: { + aggregation: ['concatenate'], + }, + }, + }, + { + displayName: 'Custom Separator', + name: 'customSeparator', + type: 'string', + default: '', + displayOptions: { + show: { + aggregation: ['concatenate'], + separateBy: ['other'], + }, + }, + }, + ], + }, + ], + }, + // fieldsToSplitBy repeated to have different displayName for singleItem and separateItems ----------------------------- + { + displayName: 'Fields to Split By', + name: 'fieldsToSplitBy', + type: 'string', + placeholder: 'e.g. country, city', + default: '', + description: 'The name of the input fields that you want to split the summary by', + hint: 'Enter the name of the fields as text (separated by commas)', + displayOptions: { + hide: { + '/options.outputFormat': ['singleItem'], + }, + }, + requiresDataPath: 'multiple', + }, + { + displayName: 'Fields to Group By', + name: 'fieldsToSplitBy', + type: 'string', + placeholder: 'e.g. country, city', + default: '', + description: 'The name of the input fields that you want to split the summary by', + hint: 'Enter the name of the fields as text (separated by commas)', + displayOptions: { + show: { + '/options.outputFormat': ['singleItem'], + }, + }, + requiresDataPath: 'multiple', + }, + // ---------------------------------------------------------------------------------------------------------- + { + displayName: 'Options', + name: 'options', + type: 'collection', + placeholder: 'Add Option', + default: {}, + options: [ + disableDotNotationBoolean, + { + displayName: 'Output Format', + name: 'outputFormat', + type: 'options', + default: 'separateItems', + options: [ + { + name: 'Each Split in a Separate Item', + value: 'separateItems', + }, + { + name: 'All Splits in a Single Item', + value: 'singleItem', + }, + ], + }, + { + // eslint-disable-next-line n8n-nodes-base/node-param-display-name-miscased + displayName: 'Ignore items without valid fields to group by', + name: 'skipEmptySplitFields', + type: 'boolean', + default: false, + }, + ], + }, +]; + +const displayOptions = { + show: { + resource: ['itemList'], + operation: ['summarize'], + }, +}; + +export const description = updateDisplayOptions(displayOptions, properties); + +function isEmpty(value: T) { + return value === undefined || value === null || value === ''; +} + +function parseReturnData(returnData: IDataObject) { + const regexBrackets = /[\]\["]/g; + const regexSpaces = /[ .]/g; + for (const key of Object.keys(returnData)) { + if (key.match(regexBrackets)) { + const newKey = key.replace(regexBrackets, ''); + returnData[newKey] = returnData[key]; + delete returnData[key]; + } + } + for (const key of Object.keys(returnData)) { + if (key.match(regexSpaces)) { + const newKey = key.replace(regexSpaces, '_'); + returnData[newKey] = returnData[key]; + delete returnData[key]; + } + } +} + +function parseFieldName(fieldName: string[]) { + const regexBrackets = /[\]\["]/g; + const regexSpaces = /[ .]/g; + fieldName = fieldName.map((field) => { + field = field.replace(regexBrackets, ''); + field = field.replace(regexSpaces, '_'); + return field; + }); + return fieldName; +} + +const fieldValueGetter = (disableDotNotation?: boolean) => { + if (disableDotNotation) { + return (item: IDataObject, field: string) => item[field]; + } else { + return (item: IDataObject, field: string) => get(item, field); + } +}; + +function checkIfFieldExists( + this: IExecuteFunctions, + items: IDataObject[], + aggregations: Aggregations, + getValue: ValueGetterFn, +) { + for (const aggregation of aggregations) { + if (aggregation.field === '') { + continue; + } + const exist = items.some((item) => getValue(item, aggregation.field) !== undefined); + if (!exist) { + throw new NodeOperationError( + this.getNode(), + `The field '${aggregation.field}' does not exist in any items`, + ); + } + } +} + +function aggregate(items: IDataObject[], entry: Aggregation, getValue: ValueGetterFn) { + const { aggregation, field } = entry; + let data = [...items]; + + if (NUMERICAL_AGGREGATIONS.includes(aggregation)) { + data = data.filter( + (item) => typeof getValue(item, field) === 'number' && !isEmpty(getValue(item, field)), + ); + } + + switch (aggregation) { + //combine operations + case 'append': + if (!entry.includeEmpty) { + data = data.filter((item) => !isEmpty(getValue(item, field))); + } + return data.map((item) => getValue(item, field)); + case 'concatenate': + const separateBy = entry.separateBy === 'other' ? entry.customSeparator : entry.separateBy; + if (!entry.includeEmpty) { + data = data.filter((item) => !isEmpty(getValue(item, field))); + } + return data + .map((item) => { + let value = getValue(item, field); + if (typeof value === 'object') { + value = JSON.stringify(value); + } + if (typeof value === 'undefined') { + value = 'undefined'; + } + + return value; + }) + .join(separateBy); + + //numerical operations + case 'average': + return ( + data.reduce((acc, item) => { + return acc + (getValue(item, field) as number); + }, 0) / data.length + ); + case 'sum': + return data.reduce((acc, item) => { + return acc + (getValue(item, field) as number); + }, 0); + //comparison operations + case 'min': + let min; + for (const item of data) { + const value = getValue(item, field); + if (value !== undefined && value !== null && value !== '') { + if (min === undefined || value < min) { + min = value; + } + } + } + return min !== undefined ? min : null; + case 'max': + let max; + for (const item of data) { + const value = getValue(item, field); + if (value !== undefined && value !== null && value !== '') { + if (max === undefined || value > max) { + max = value; + } + } + } + return max !== undefined ? max : null; + + //count operations + case 'countUnique': + return new Set(data.map((item) => getValue(item, field)).filter((item) => !isEmpty(item))) + .size; + default: + //count by default + return data.filter((item) => !isEmpty(getValue(item, field))).length; + } +} + +function aggregateData( + data: IDataObject[], + fieldsToSummarize: Aggregations, + options: SummarizeOptions, + getValue: ValueGetterFn, +) { + const returnData = fieldsToSummarize.reduce((acc, aggregation) => { + acc[`${AggregationDisplayNames[aggregation.aggregation]}${aggregation.field}`] = aggregate( + data, + aggregation, + getValue, + ); + return acc; + }, {} as IDataObject); + parseReturnData(returnData); + if (options.outputFormat === 'singleItem') { + parseReturnData(returnData); + return returnData; + } else { + return { ...returnData, pairedItems: data.map((item) => item._itemIndex as number) }; + } +} + +function splitData( + splitKeys: string[], + data: IDataObject[], + fieldsToSummarize: Aggregations, + options: SummarizeOptions, + getValue: ValueGetterFn, +) { + if (!splitKeys || splitKeys.length === 0) { + return aggregateData(data, fieldsToSummarize, options, getValue); + } + + const [firstSplitKey, ...restSplitKeys] = splitKeys; + + const groupedData = data.reduce((acc, item) => { + let keyValuee = getValue(item, firstSplitKey) as string; + + if (typeof keyValuee === 'object') { + keyValuee = JSON.stringify(keyValuee); + } + + if (options.skipEmptySplitFields && typeof keyValuee !== 'number' && !keyValuee) { + return acc; + } + + if (acc[keyValuee] === undefined) { + acc[keyValuee] = [item]; + } else { + (acc[keyValuee] as IDataObject[]).push(item); + } + return acc; + }, {} as IDataObject); + + return Object.keys(groupedData).reduce((acc, key) => { + const value = groupedData[key] as IDataObject[]; + acc[key] = splitData(restSplitKeys, value, fieldsToSummarize, options, getValue); + return acc; + }, {} as IDataObject); +} + +function aggregationToArray( + aggregationResult: IDataObject, + fieldsToSplitBy: string[], + previousStage: IDataObject = {}, +) { + const returnData: IDataObject[] = []; + fieldsToSplitBy = parseFieldName(fieldsToSplitBy); + const splitFieldName = fieldsToSplitBy[0]; + const isNext = fieldsToSplitBy[1]; + + if (isNext === undefined) { + for (const fieldName of Object.keys(aggregationResult)) { + returnData.push({ + ...previousStage, + [splitFieldName]: fieldName, + ...(aggregationResult[fieldName] as IDataObject), + }); + } + return returnData; + } else { + for (const key of Object.keys(aggregationResult)) { + returnData.push( + ...aggregationToArray(aggregationResult[key] as IDataObject, fieldsToSplitBy.slice(1), { + ...previousStage, + [splitFieldName]: key, + }), + ); + } + return returnData; + } +} + +export async function execute( + this: IExecuteFunctions, + items: INodeExecutionData[], +): Promise { + const newItems = items.map(({ json }, i) => ({ ...json, _itemIndex: i })); + + const options = this.getNodeParameter('options', 0, {}) as SummarizeOptions; + + const fieldsToSplitBy = (this.getNodeParameter('fieldsToSplitBy', 0, '') as string) + .split(',') + .map((field) => field.trim()) + .filter((field) => field); + + const fieldsToSummarize = this.getNodeParameter( + 'fieldsToSummarize.values', + 0, + [], + ) as Aggregations; + + if (fieldsToSummarize.filter((aggregation) => aggregation.field !== '').length === 0) { + throw new NodeOperationError( + this.getNode(), + "You need to add at least one aggregation to 'Fields to Summarize' with non empty 'Field'", + ); + } + + const getValue = fieldValueGetter(options.disableDotNotation); + + const nodeVersion = this.getNode().typeVersion; + + if (nodeVersion < 2.1) { + checkIfFieldExists.call(this, newItems, fieldsToSummarize, getValue); + } + + const aggregationResult = splitData( + fieldsToSplitBy, + newItems, + fieldsToSummarize, + options, + getValue, + ); + + if (options.outputFormat === 'singleItem') { + const executionData: INodeExecutionData = { + json: aggregationResult, + pairedItem: newItems.map((_v, index) => ({ + item: index, + })), + }; + return [executionData]; + } else { + if (!fieldsToSplitBy.length) { + const { pairedItems, ...json } = aggregationResult; + const executionData: INodeExecutionData = { + json, + pairedItem: ((pairedItems as number[]) || []).map((index: number) => ({ + item: index, + })), + }; + return [executionData]; + } + const returnData = aggregationToArray(aggregationResult, fieldsToSplitBy); + const executionData = returnData.map((item) => { + const { pairedItems, ...json } = item; + return { + json, + pairedItem: ((pairedItems as number[]) || []).map((index: number) => ({ + item: index, + })), + }; + }); + return executionData; + } +} diff --git a/packages/nodes-base/nodes/ItemLists/V3/actions/node.type.ts b/packages/nodes-base/nodes/ItemLists/V3/actions/node.type.ts new file mode 100644 index 0000000000..86750f30ea --- /dev/null +++ b/packages/nodes-base/nodes/ItemLists/V3/actions/node.type.ts @@ -0,0 +1,13 @@ +import type { AllEntities } from 'n8n-workflow'; + +type NodeMap = { + itemList: + | 'concatenateItems' + | 'limit' + | 'removeDuplicates' + | 'sort' + | 'splitOutItems' + | 'summarize'; +}; + +export type ItemListsType = AllEntities; diff --git a/packages/nodes-base/nodes/ItemLists/V3/actions/router.ts b/packages/nodes-base/nodes/ItemLists/V3/actions/router.ts new file mode 100644 index 0000000000..a5f24e6b86 --- /dev/null +++ b/packages/nodes-base/nodes/ItemLists/V3/actions/router.ts @@ -0,0 +1,35 @@ +import type { IExecuteFunctions, INodeExecutionData } from 'n8n-workflow'; +import { NodeOperationError } from 'n8n-workflow'; +import type { ItemListsType } from './node.type'; + +import * as itemList from './itemList'; + +export async function router(this: IExecuteFunctions): Promise { + let returnData: INodeExecutionData[] = []; + + const items = this.getInputData(); + const resource = this.getNodeParameter('resource', 0); + const operation = this.getNodeParameter('operation', 0); + + const itemListsNodeData = { + resource, + operation, + } as ItemListsType; + + try { + switch (itemListsNodeData.resource) { + case 'itemList': + returnData = await itemList[itemListsNodeData.operation].execute.call(this, items); + break; + default: + throw new NodeOperationError( + this.getNode(), + `The operation "${operation}" is not supported!`, + ); + } + } catch (error) { + throw error; + } + + return this.prepareOutputData(returnData); +} diff --git a/packages/nodes-base/nodes/ItemLists/V3/actions/versionDescription.ts b/packages/nodes-base/nodes/ItemLists/V3/actions/versionDescription.ts new file mode 100644 index 0000000000..728f646769 --- /dev/null +++ b/packages/nodes-base/nodes/ItemLists/V3/actions/versionDescription.ts @@ -0,0 +1,35 @@ +/* eslint-disable n8n-nodes-base/node-filename-against-convention */ +import type { INodeTypeDescription } from 'n8n-workflow'; + +import * as itemList from './itemList'; + +export const versionDescription: INodeTypeDescription = { + displayName: 'Item Lists', + name: 'itemLists', + icon: 'file:itemLists.svg', + group: ['input'], + subtitle: '={{$parameter["operation"] + ": " + $parameter["resource"]}}', + description: 'Helper for working with lists of items and transforming arrays', + version: 3, + defaults: { + name: 'Item Lists', + }, + inputs: ['main'], + outputs: ['main'], + credentials: [], + properties: [ + { + displayName: 'Resource', + name: 'resource', + type: 'hidden', + options: [ + { + name: 'Item List', + value: 'itemList', + }, + ], + default: 'itemList', + }, + ...itemList.description, + ], +}; diff --git a/packages/nodes-base/nodes/ItemLists/V3/helpers/utils.ts b/packages/nodes-base/nodes/ItemLists/V3/helpers/utils.ts new file mode 100644 index 0000000000..60600eb8d6 --- /dev/null +++ b/packages/nodes-base/nodes/ItemLists/V3/helpers/utils.ts @@ -0,0 +1,59 @@ +import type { IDataObject, INode, INodeExecutionData } from 'n8n-workflow'; + +import get from 'lodash/get'; +import isEqual from 'lodash/isEqual'; +import isObject from 'lodash/isObject'; +import merge from 'lodash/merge'; +import reduce from 'lodash/reduce'; + +export const compareItems = ( + obj: INodeExecutionData, + obj2: INodeExecutionData, + keys: string[], + disableDotNotation: boolean, + _node: INode, +) => { + let result = true; + for (const key of keys) { + if (!disableDotNotation) { + if (!isEqual(get(obj.json, key), get(obj2.json, key))) { + result = false; + break; + } + } else { + if (!isEqual(obj.json[key], obj2.json[key])) { + result = false; + break; + } + } + } + return result; +}; + +export const flattenKeys = (obj: IDataObject, path: string[] = []): IDataObject => { + return !isObject(obj) + ? { [path.join('.')]: obj } + : reduce(obj, (cum, next, key) => merge(cum, flattenKeys(next as IDataObject, [...path, key])), {}); //prettier-ignore +}; + +export const shuffleArray = (array: any[]) => { + for (let i = array.length - 1; i > 0; i--) { + const j = Math.floor(Math.random() * (i + 1)); + [array[i], array[j]] = [array[j], array[i]]; + } +}; + +export const prepareFieldsArray = (fields: string | string[], fieldName = 'Fields') => { + if (typeof fields === 'string') { + return fields + .split(',') + .map((entry) => entry.trim()) + .filter((entry) => entry !== ''); + } + if (Array.isArray(fields)) { + return fields; + } + throw new Error( + `The \'${fieldName}\' parameter must be a string of fields separated by commas or an array of strings.`, + ); +}; diff --git a/packages/nodes-base/nodes/ItemLists/test/node/workflow.update_3.json b/packages/nodes-base/nodes/ItemLists/test/node/workflow.update_3.json new file mode 100644 index 0000000000..3bbd2517f8 --- /dev/null +++ b/packages/nodes-base/nodes/ItemLists/test/node/workflow.update_3.json @@ -0,0 +1,606 @@ +{ + "name": "itemList refactor", + "nodes": [ + { + "parameters": {}, + "id": "e7ecaa9c-e35d-4095-a85b-85b83f807c2a", + "name": "When clicking \"Execute Workflow\"", + "type": "n8n-nodes-base.manualTrigger", + "typeVersion": 1, + "position": [420, 400] + }, + { + "parameters": { + "operation": "getAllPeople", + "returnAll": true + }, + "id": "7d925077-afaa-46d5-ba2f-0c19d93afecc", + "name": "Customer Datastore (n8n training)", + "type": "n8n-nodes-base.n8nTrainingCustomerDatastore", + "typeVersion": 1, + "position": [640, 400] + }, + { + "parameters": { + "operation": "concatenateItems", + "fieldsToAggregate": { + "fieldToAggregate": [ + { + "fieldToAggregate": "email" + }, + { + "fieldToAggregate": "notes" + } + ] + }, + "options": {} + }, + "id": "f80182d8-54f6-4a26-82b3-27d67e4ca39b", + "name": "Item Lists1", + "type": "n8n-nodes-base.itemLists", + "typeVersion": 3, + "position": [1120, -120] + }, + { + "parameters": { + "operation": "concatenateItems", + "aggregate": "aggregateAllItemData", + "destinationFieldName": "data2" + }, + "id": "23eefe2c-6394-4b53-a791-852dcd671ea0", + "name": "Item Lists", + "type": "n8n-nodes-base.itemLists", + "typeVersion": 3, + "position": [1120, 40] + }, + { + "parameters": { + "operation": "limit", + "maxItems": 2 + }, + "id": "676dc72f-9766-43c0-aab7-2f8a93ed46e5", + "name": "Item Lists2", + "type": "n8n-nodes-base.itemLists", + "typeVersion": 3, + "position": [1120, 200] + }, + { + "parameters": { + "operation": "limit", + "keep": "lastItems" + }, + "id": "9615299b-5acc-4459-a7d3-2cb23c3224ab", + "name": "Item Lists3", + "type": "n8n-nodes-base.itemLists", + "typeVersion": 3, + "position": [1120, 360] + }, + { + "parameters": { + "operation": "sort", + "sortFieldsUi": { + "sortField": [ + { + "fieldName": "country" + } + ] + }, + "options": {} + }, + "id": "fd2f190a-b161-48ff-93e1-0f30efce051a", + "name": "Item Lists4", + "type": "n8n-nodes-base.itemLists", + "typeVersion": 3, + "position": [1120, 540] + }, + { + "parameters": { + "operation": "limit", + "maxItems": 4 + }, + "id": "14759521-76ad-46d8-8add-1d7361788fe1", + "name": "Item Lists5", + "type": "n8n-nodes-base.itemLists", + "typeVersion": 3, + "position": [1360, 540] + }, + { + "parameters": { + "operation": "removeDuplicates", + "compare": "selectedFields", + "fieldsToCompare": "country", + "options": {} + }, + "id": "c0eba4d0-f975-4987-8bb2-853e8a98665e", + "name": "Item Lists6", + "type": "n8n-nodes-base.itemLists", + "typeVersion": 3, + "position": [1560, 540] + }, + { + "parameters": { + "operation": "concatenateItems", + "aggregate": "aggregateAllItemData", + "include": "specifiedFields", + "fieldsToInclude": "country, notes, name, created" + }, + "id": "b5962f40-a891-4b02-8fec-c2d76f85375f", + "name": "Item Lists7", + "type": "n8n-nodes-base.itemLists", + "typeVersion": 3, + "position": [1120, 740] + }, + { + "parameters": { + "fieldToSplitOut": "data", + "include": "allOtherFields", + "options": { + "destinationFieldName": "newData" + } + }, + "id": "15c6fc86-7e38-4d76-836a-8f8426ca05e3", + "name": "Item Lists8", + "type": "n8n-nodes-base.itemLists", + "typeVersion": 3, + "position": [1380, 740] + }, + { + "parameters": { + "operation": "summarize", + "fieldsToSummarize": { + "values": [ + { + "aggregation": "append", + "field": "newData.notes" + }, + { + "aggregation": "max", + "field": "newData.created" + }, + { + "aggregation": "min", + "field": "newData.created" + } + ] + }, + "options": {} + }, + "id": "8c51ae57-487e-472a-b268-6e8ad347edbb", + "name": "Item Lists9", + "type": "n8n-nodes-base.itemLists", + "typeVersion": 3, + "position": [1560, 940] + }, + { + "parameters": {}, + "id": "e859b082-284c-4bb3-96b6-39a86152d8f6", + "name": "No Operation, do nothing", + "type": "n8n-nodes-base.noOp", + "typeVersion": 1, + "position": [1760, -120] + }, + { + "parameters": {}, + "id": "9da56c21-739d-4f2f-adf7-953cf0550d97", + "name": "No Operation, do nothing1", + "type": "n8n-nodes-base.noOp", + "typeVersion": 1, + "position": [1760, 40] + }, + { + "parameters": {}, + "id": "f85ac031-24de-4701-bb3d-76c684924002", + "name": "No Operation, do nothing2", + "type": "n8n-nodes-base.noOp", + "typeVersion": 1, + "position": [1760, 200] + }, + { + "parameters": {}, + "id": "e7faff14-55d6-4d78-983d-027fd56bcd5a", + "name": "No Operation, do nothing3", + "type": "n8n-nodes-base.noOp", + "typeVersion": 1, + "position": [1760, 360] + }, + { + "parameters": {}, + "id": "dc8b7bbc-b1a8-4ba8-b214-d73341cb9f85", + "name": "No Operation, do nothing4", + "type": "n8n-nodes-base.noOp", + "typeVersion": 1, + "position": [1760, 740] + }, + { + "parameters": {}, + "id": "09bdeca1-6a9e-4668-b9d1-14ed6139e047", + "name": "No Operation, do nothing5", + "type": "n8n-nodes-base.noOp", + "typeVersion": 1, + "position": [1760, 540] + }, + { + "parameters": {}, + "id": "7d508889-d94e-4818-abc8-b669a0fe64ea", + "name": "No Operation, do nothing6", + "type": "n8n-nodes-base.noOp", + "typeVersion": 1, + "position": [1760, 940] + } + ], + "pinData": { + "No Operation, do nothing": [ + { + "json": { + "email": [ + "gatsby@west-egg.com", + "jab@macondo.co", + "info@in-and-out-of-weeks.org", + "captain@heartofgold.com", + "edmund@narnia.gov" + ], + "notes": [ + "Keeps asking about a green light??", + "Lots of people named after him. Very confusing", + "Keeps rolling his terrible eyes", + "Felt like I was talking to more than one person", + "Passionate sailor" + ] + } + } + ], + "No Operation, do nothing1": [ + { + "json": { + "data2": [ + { + "id": "23423532", + "name": "Jay Gatsby", + "email": "gatsby@west-egg.com", + "notes": "Keeps asking about a green light??", + "country": "US", + "created": "1925-04-10" + }, + { + "id": "23423533", + "name": "José Arcadio Buendía", + "email": "jab@macondo.co", + "notes": "Lots of people named after him. Very confusing", + "country": "CO", + "created": "1967-05-05" + }, + { + "id": "23423534", + "name": "Max Sendak", + "email": "info@in-and-out-of-weeks.org", + "notes": "Keeps rolling his terrible eyes", + "country": "US", + "created": "1963-04-09" + }, + { + "id": "23423535", + "name": "Zaphod Beeblebrox", + "email": "captain@heartofgold.com", + "notes": "Felt like I was talking to more than one person", + "country": null, + "created": "1979-10-12" + }, + { + "id": "23423536", + "name": "Edmund Pevensie", + "email": "edmund@narnia.gov", + "notes": "Passionate sailor", + "country": "UK", + "created": "1950-10-16" + } + ] + } + } + ], + "No Operation, do nothing2": [ + { + "json": { + "id": "23423532", + "name": "Jay Gatsby", + "email": "gatsby@west-egg.com", + "notes": "Keeps asking about a green light??", + "country": "US", + "created": "1925-04-10" + } + }, + { + "json": { + "id": "23423533", + "name": "José Arcadio Buendía", + "email": "jab@macondo.co", + "notes": "Lots of people named after him. Very confusing", + "country": "CO", + "created": "1967-05-05" + } + } + ], + "No Operation, do nothing3": [ + { + "json": { + "id": "23423536", + "name": "Edmund Pevensie", + "email": "edmund@narnia.gov", + "notes": "Passionate sailor", + "country": "UK", + "created": "1950-10-16" + } + } + ], + "No Operation, do nothing5": [ + { + "json": { + "id": "23423533", + "name": "José Arcadio Buendía", + "email": "jab@macondo.co", + "notes": "Lots of people named after him. Very confusing", + "country": "CO", + "created": "1967-05-05" + } + }, + { + "json": { + "id": "23423536", + "name": "Edmund Pevensie", + "email": "edmund@narnia.gov", + "notes": "Passionate sailor", + "country": "UK", + "created": "1950-10-16" + } + }, + { + "json": { + "id": "23423532", + "name": "Jay Gatsby", + "email": "gatsby@west-egg.com", + "notes": "Keeps asking about a green light??", + "country": "US", + "created": "1925-04-10" + } + } + ], + "No Operation, do nothing4": [ + { + "json": { + "newData": { + "name": "Jay Gatsby", + "notes": "Keeps asking about a green light??", + "country": "US", + "created": "1925-04-10" + } + } + }, + { + "json": { + "newData": { + "name": "José Arcadio Buendía", + "notes": "Lots of people named after him. Very confusing", + "country": "CO", + "created": "1967-05-05" + } + } + }, + { + "json": { + "newData": { + "name": "Max Sendak", + "notes": "Keeps rolling his terrible eyes", + "country": "US", + "created": "1963-04-09" + } + } + }, + { + "json": { + "newData": { + "name": "Zaphod Beeblebrox", + "notes": "Felt like I was talking to more than one person", + "country": null, + "created": "1979-10-12" + } + } + }, + { + "json": { + "newData": { + "name": "Edmund Pevensie", + "notes": "Passionate sailor", + "country": "UK", + "created": "1950-10-16" + } + } + } + ], + "No Operation, do nothing6": [ + { + "json": { + "appended_newData_notes": [ + "Keeps asking about a green light??", + "Lots of people named after him. Very confusing", + "Keeps rolling his terrible eyes", + "Felt like I was talking to more than one person", + "Passionate sailor" + ], + "max_newData_created": "1979-10-12", + "min_newData_created": "1925-04-10" + } + } + ] + }, + "connections": { + "When clicking \"Execute Workflow\"": { + "main": [ + [ + { + "node": "Customer Datastore (n8n training)", + "type": "main", + "index": 0 + } + ] + ] + }, + "Customer Datastore (n8n training)": { + "main": [ + [ + { + "node": "Item Lists1", + "type": "main", + "index": 0 + }, + { + "node": "Item Lists", + "type": "main", + "index": 0 + }, + { + "node": "Item Lists2", + "type": "main", + "index": 0 + }, + { + "node": "Item Lists3", + "type": "main", + "index": 0 + }, + { + "node": "Item Lists4", + "type": "main", + "index": 0 + }, + { + "node": "Item Lists7", + "type": "main", + "index": 0 + } + ] + ] + }, + "Item Lists1": { + "main": [ + [ + { + "node": "No Operation, do nothing", + "type": "main", + "index": 0 + } + ] + ] + }, + "Item Lists4": { + "main": [ + [ + { + "node": "Item Lists5", + "type": "main", + "index": 0 + } + ] + ] + }, + "Item Lists5": { + "main": [ + [ + { + "node": "Item Lists6", + "type": "main", + "index": 0 + } + ] + ] + }, + "Item Lists7": { + "main": [ + [ + { + "node": "Item Lists8", + "type": "main", + "index": 0 + } + ] + ] + }, + "Item Lists8": { + "main": [ + [ + { + "node": "Item Lists9", + "type": "main", + "index": 0 + }, + { + "node": "No Operation, do nothing4", + "type": "main", + "index": 0 + } + ] + ] + }, + "Item Lists": { + "main": [ + [ + { + "node": "No Operation, do nothing1", + "type": "main", + "index": 0 + } + ] + ] + }, + "Item Lists2": { + "main": [ + [ + { + "node": "No Operation, do nothing2", + "type": "main", + "index": 0 + } + ] + ] + }, + "Item Lists3": { + "main": [ + [ + { + "node": "No Operation, do nothing3", + "type": "main", + "index": 0 + } + ] + ] + }, + "Item Lists6": { + "main": [ + [ + { + "node": "No Operation, do nothing5", + "type": "main", + "index": 0 + } + ] + ] + }, + "Item Lists9": { + "main": [ + [ + { + "node": "No Operation, do nothing6", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "active": false, + "settings": {}, + "versionId": "ce3e0124-aa56-497c-a2e1-24158837c7f9", + "id": "m7QDuxo599dkZ0Ex", + "meta": { + "instanceId": "e34acda144ba98351e38adb4db781751ca8cd64a8248aef8b65608fc9a49008c" + }, + "tags": [] +}