From aa2beaa80076fc06360828e466a3dc05e7caddbe Mon Sep 17 00:00:00 2001 From: agobrech <45268029+agobrech@users.noreply.github.com> Date: Tue, 28 Feb 2023 18:00:39 +0100 Subject: [PATCH] fix(Item Lists Node): Tweak item list summarize field naming (#5572) * Remove brackets and double quotes from fieldname * Fix bug with duplicate field * Parse field names from splitbyfield * Fix error with field name remove console.logs * Add versioning to itemlist * Fix naming * Remove comment --- .../nodes/ItemLists/ItemLists.node.ts | 1441 +---------------- .../nodes/ItemLists/V1/ItemListsV1.node.ts | 1428 ++++++++++++++++ .../ItemLists/{ => V1}/summarize.operation.ts | 0 .../nodes/ItemLists/V2/ItemListsV2.node.ts | 1428 ++++++++++++++++ .../nodes/ItemLists/V2/summarize.operation.ts | 614 +++++++ 5 files changed, 3490 insertions(+), 1421 deletions(-) create mode 100644 packages/nodes-base/nodes/ItemLists/V1/ItemListsV1.node.ts rename packages/nodes-base/nodes/ItemLists/{ => V1}/summarize.operation.ts (100%) create mode 100644 packages/nodes-base/nodes/ItemLists/V2/ItemListsV2.node.ts create mode 100644 packages/nodes-base/nodes/ItemLists/V2/summarize.operation.ts diff --git a/packages/nodes-base/nodes/ItemLists/ItemLists.node.ts b/packages/nodes-base/nodes/ItemLists/ItemLists.node.ts index 11011680f1..25a6405280 100644 --- a/packages/nodes-base/nodes/ItemLists/ItemLists.node.ts +++ b/packages/nodes-base/nodes/ItemLists/ItemLists.node.ts @@ -1,1428 +1,27 @@ -import type { NodeVMOptions } from 'vm2'; -import { NodeVM } from 'vm2'; -import type { IExecuteFunctions } from 'n8n-core'; +import type { INodeTypeBaseDescription, IVersionedNodeType } from 'n8n-workflow'; +import { VersionedNodeType } from 'n8n-workflow'; -import type { - IDataObject, - INode, - INodeExecutionData, - INodeType, - INodeTypeDescription, -} from 'n8n-workflow'; -import { NodeOperationError } from 'n8n-workflow'; +import { ItemListsV1 } from './V1/ItemListsV1.node'; -import get from 'lodash.get'; -import isEmpty from 'lodash.isempty'; -import isEqual from 'lodash.isequal'; -import isObject from 'lodash.isobject'; -import lt from 'lodash.lt'; -import merge from 'lodash.merge'; -import pick from 'lodash.pick'; -import reduce from 'lodash.reduce'; -import set from 'lodash.set'; -import unset from 'lodash.unset'; +import { ItemListsV2 } from './V2/ItemListsV2.node'; -const compareItems = ( - obj: INodeExecutionData, - obj2: INodeExecutionData, - keys: string[], - disableDotNotation: boolean, - _node: INode, -) => { - let result = true; - for (const key of keys) { - if (!disableDotNotation) { - if (!isEqual(get(obj.json, key), get(obj2.json, key))) { - result = false; - break; - } - } else { - if (!isEqual(obj.json[key], obj2.json[key])) { - result = false; - break; - } - } - } - return result; -}; +export class ItemLists extends VersionedNodeType { + constructor() { + const baseDescription: INodeTypeBaseDescription = { + displayName: 'Item Lists', + name: 'itemLists', + icon: 'file:itemLists.svg', + group: ['input'], + subtitle: '={{$parameter["operation"] + ": " + $parameter["resource"]}}', + description: 'Helper for working with lists of items and transforming arrays', + defaultVersion: 2, + }; -const flattenKeys = (obj: IDataObject, path: string[] = []): IDataObject => { - return !isObject(obj) - ? { [path.join('.')]: obj } - : reduce(obj, (cum, next, key) => merge(cum, flattenKeys(next as IDataObject, [...path, key])), {}); //prettier-ignore -}; + const nodeVersions: IVersionedNodeType['nodeVersions'] = { + 1: new ItemListsV1(baseDescription), + 2: new ItemListsV2(baseDescription), + }; -const shuffleArray = (array: any[]) => { - for (let i = array.length - 1; i > 0; i--) { - const j = Math.floor(Math.random() * (i + 1)); - [array[i], array[j]] = [array[j], array[i]]; - } -}; - -import * as summarize from './summarize.operation'; - -export class ItemLists implements INodeType { - description: INodeTypeDescription = { - displayName: 'Item Lists', - name: 'itemLists', - icon: 'file:itemLists.svg', - group: ['input'], - version: 1, - subtitle: '={{$parameter["operation"] + ": " + $parameter["resource"]}}', - description: 'Helper for working with lists of items and transforming arrays', - defaults: { - name: 'Item Lists', - }, - inputs: ['main'], - outputs: ['main'], - credentials: [], - properties: [ - { - displayName: 'Resource', - name: 'resource', - type: 'hidden', - options: [ - { - name: 'Item List', - value: 'itemList', - }, - ], - default: 'itemList', - }, - { - displayName: 'Operation', - name: 'operation', - type: 'options', - noDataExpression: true, - options: [ - { - name: 'Concatenate Items', - value: 'aggregateItems', - description: 'Combine fields into a list in a single new item', - action: 'Combine fields into a list in a single new item', - }, - { - name: 'Limit', - value: 'limit', - description: 'Remove items if there are too many', - action: 'Remove items if there are too many', - }, - { - name: 'Remove Duplicates', - value: 'removeDuplicates', - description: 'Remove extra items that are similar', - action: 'Remove extra items that are similar', - }, - { - name: 'Sort', - value: 'sort', - description: 'Change the item order', - action: 'Change the item order', - }, - { - name: 'Split Out Items', - value: 'splitOutItems', - description: 'Turn a list inside item(s) into separate items', - action: 'Turn a list inside item(s) into separate items', - }, - { - name: 'Summarize', - value: 'summarize', - description: 'Aggregate items together (pivot table)', - action: 'Aggregate items together (pivot table)', - }, - ], - default: 'splitOutItems', - }, - // Split out items - Fields - { - displayName: 'Field To Split Out', - name: 'fieldToSplitOut', - type: 'string', - default: '', - required: true, - displayOptions: { - show: { - resource: ['itemList'], - operation: ['splitOutItems'], - }, - }, - description: 'The name of the input field to break out into separate items', - requiresDataPath: 'single', - }, - { - displayName: 'Include', - name: 'include', - type: 'options', - options: [ - { - name: 'No Other Fields', - value: 'noOtherFields', - }, - { - name: 'All Other Fields', - value: 'allOtherFields', - }, - { - name: 'Selected Other Fields', - value: 'selectedOtherFields', - }, - ], - default: 'noOtherFields', - description: 'Whether to copy any other fields into the new items', - displayOptions: { - show: { - resource: ['itemList'], - operation: ['splitOutItems'], - }, - }, - }, - { - displayName: 'Fields To Include', - name: 'fieldsToInclude', - type: 'fixedCollection', - typeOptions: { - multipleValues: true, - }, - placeholder: 'Add Field To Include', - default: {}, - displayOptions: { - show: { - resource: ['itemList'], - operation: ['splitOutItems'], - include: ['selectedOtherFields'], - }, - }, - options: [ - { - displayName: '', - name: 'fields', - values: [ - { - displayName: 'Field Name', - name: 'fieldName', - type: 'string', - default: '', - description: 'A field in the input items to aggregate together', - // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id - placeholder: 'e.g. id', - hint: ' Enter the field name as text', - requiresDataPath: 'single', - }, - ], - }, - ], - }, - // Aggregate Items - { - displayName: 'Aggregate', - name: 'aggregate', - type: 'options', - default: 'aggregateIndividualFields', - options: [ - { - name: 'Individual Fields', - value: 'aggregateIndividualFields', - }, - { - name: 'All Item Data (Into a Single List)', - value: 'aggregateAllItemData', - }, - ], - displayOptions: { - show: { - resource: ['itemList'], - operation: ['aggregateItems'], - }, - }, - }, - // Aggregate Individual Fields - { - displayName: 'Fields To Aggregate', - name: 'fieldsToAggregate', - type: 'fixedCollection', - typeOptions: { - multipleValues: true, - }, - placeholder: 'Add Field To Aggregate', - default: { fieldToAggregate: [{ fieldToAggregate: '', renameField: false }] }, - displayOptions: { - show: { - resource: ['itemList'], - operation: ['aggregateItems'], - aggregate: ['aggregateIndividualFields'], - }, - }, - options: [ - { - displayName: '', - name: 'fieldToAggregate', - values: [ - { - displayName: 'Input Field Name', - name: 'fieldToAggregate', - type: 'string', - default: '', - description: 'The name of a field in the input items to aggregate together', - // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id - placeholder: 'e.g. id', - hint: ' Enter the field name as text', - requiresDataPath: 'single', - }, - { - displayName: 'Rename Field', - name: 'renameField', - type: 'boolean', - default: false, - description: 'Whether to give the field a different name in the output', - }, - { - displayName: 'Output Field Name', - name: 'outputFieldName', - displayOptions: { - show: { - renameField: [true], - }, - }, - type: 'string', - default: '', - description: - 'The name of the field to put the aggregated data in. Leave blank to use the input field name.', - requiresDataPath: 'single', - }, - ], - }, - ], - }, - // Aggregate All Item Data - { - displayName: 'Put Output in Field', - name: 'destinationFieldName', - type: 'string', - displayOptions: { - show: { - resource: ['itemList'], - operation: ['aggregateItems'], - aggregate: ['aggregateAllItemData'], - }, - }, - default: 'data', - description: 'The name of the output field to put the data in', - }, - { - displayName: 'Include', - name: 'include', - type: 'options', - default: 'allFields', - options: [ - { - name: 'All Fields', - value: 'allFields', - }, - { - name: 'Specified Fields', - value: 'specifiedFields', - }, - { - name: 'All Fields Except', - value: 'allFieldsExcept', - }, - ], - displayOptions: { - show: { - resource: ['itemList'], - operation: ['aggregateItems'], - aggregate: ['aggregateAllItemData'], - }, - }, - }, - { - displayName: 'Fields To Exclude', - name: 'fieldsToExclude', - type: 'fixedCollection', - typeOptions: { - multipleValues: true, - }, - placeholder: 'Add Field To Exclude', - default: {}, - options: [ - { - displayName: '', - name: 'fields', - values: [ - { - displayName: 'Field Name', - name: 'fieldName', - type: 'string', - default: '', - description: 'A field in the input to exclude from the object in output array', - // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id - placeholder: 'e.g. id', - hint: ' Enter the field name as text', - requiresDataPath: 'single', - }, - ], - }, - ], - displayOptions: { - show: { - resource: ['itemList'], - operation: ['aggregateItems'], - aggregate: ['aggregateAllItemData'], - include: ['allFieldsExcept'], - }, - }, - }, - { - displayName: 'Fields To Include', - name: 'fieldsToInclude', - type: 'fixedCollection', - typeOptions: { - multipleValues: true, - }, - placeholder: 'Add Field To Include', - default: {}, - options: [ - { - displayName: '', - name: 'fields', - values: [ - { - displayName: 'Field Name', - name: 'fieldName', - type: 'string', - default: '', - description: 'Specify fields that will be included in output array', - // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id - placeholder: 'e.g. id', - hint: ' Enter the field name as text', - requiresDataPath: 'single', - }, - ], - }, - ], - displayOptions: { - show: { - resource: ['itemList'], - operation: ['aggregateItems'], - aggregate: ['aggregateAllItemData'], - include: ['specifiedFields'], - }, - }, - }, - // Remove duplicates - Fields - { - displayName: 'Compare', - name: 'compare', - type: 'options', - options: [ - { - name: 'All Fields', - value: 'allFields', - }, - { - name: 'All Fields Except', - value: 'allFieldsExcept', - }, - { - name: 'Selected Fields', - value: 'selectedFields', - }, - ], - default: 'allFields', - description: 'The fields of the input items to compare to see if they are the same', - displayOptions: { - show: { - resource: ['itemList'], - operation: ['removeDuplicates'], - }, - }, - }, - { - displayName: 'Fields To Exclude', - name: 'fieldsToExclude', - type: 'fixedCollection', - typeOptions: { - multipleValues: true, - }, - placeholder: 'Add Field To Exclude', - default: {}, - displayOptions: { - show: { - resource: ['itemList'], - operation: ['removeDuplicates'], - compare: ['allFieldsExcept'], - }, - }, - options: [ - { - displayName: '', - name: 'fields', - values: [ - { - displayName: 'Field Name', - name: 'fieldName', - type: 'string', - default: '', - description: 'A field in the input to exclude from the comparison', - // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id - placeholder: 'e.g. id', - hint: ' Enter the field name as text', - requiresDataPath: 'single', - }, - ], - }, - ], - }, - { - displayName: 'Fields To Compare', - name: 'fieldsToCompare', - type: 'fixedCollection', - typeOptions: { - multipleValues: true, - }, - placeholder: 'Add Field To Compare', - default: {}, - displayOptions: { - show: { - resource: ['itemList'], - operation: ['removeDuplicates'], - compare: ['selectedFields'], - }, - }, - options: [ - { - displayName: '', - name: 'fields', - values: [ - { - displayName: 'Field Name', - name: 'fieldName', - type: 'string', - default: '', - description: 'A field in the input to add to the comparison', - // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id - placeholder: 'e.g. id', - hint: ' Enter the field name as text', - requiresDataPath: 'single', - }, - ], - }, - ], - }, - // Sort - Fields - { - displayName: 'Type', - name: 'type', - type: 'options', - options: [ - { - name: 'Simple', - value: 'simple', - }, - { - name: 'Random', - value: 'random', - }, - { - name: 'Code', - value: 'code', - }, - ], - default: 'simple', - description: 'The fields of the input items to compare to see if they are the same', - displayOptions: { - show: { - resource: ['itemList'], - operation: ['sort'], - }, - }, - }, - { - displayName: 'Fields To Sort By', - name: 'sortFieldsUi', - type: 'fixedCollection', - typeOptions: { - multipleValues: true, - }, - placeholder: 'Add Field To Sort By', - options: [ - { - displayName: '', - name: 'sortField', - values: [ - { - displayName: 'Field Name', - name: 'fieldName', - type: 'string', - required: true, - default: '', - description: 'The field to sort by', - // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id - placeholder: 'e.g. id', - hint: ' Enter the field name as text', - requiresDataPath: 'single', - }, - { - displayName: 'Order', - name: 'order', - type: 'options', - options: [ - { - name: 'Ascending', - value: 'ascending', - }, - { - name: 'Descending', - value: 'descending', - }, - ], - default: 'ascending', - description: 'The order to sort by', - }, - ], - }, - ], - default: {}, - description: 'The fields of the input items to compare to see if they are the same', - displayOptions: { - show: { - resource: ['itemList'], - operation: ['sort'], - type: ['simple'], - }, - }, - }, - { - displayName: 'Code', - name: 'code', - type: 'string', - typeOptions: { - alwaysOpenEditWindow: true, - editor: 'code', - rows: 10, - }, - default: `// The two items to compare are in the variables a and b -// Access the fields in a.json and b.json -// Return -1 if a should go before b -// Return 1 if b should go before a -// Return 0 if there's no difference - -fieldName = 'myField'; - -if (a.json[fieldName] < b.json[fieldName]) { - return -1; -} -if (a.json[fieldName] > b.json[fieldName]) { - return 1; -} -return 0;`, - description: 'Javascript code to determine the order of any two items', - displayOptions: { - show: { - resource: ['itemList'], - operation: ['sort'], - type: ['code'], - }, - }, - }, - // Limit - Fields - { - displayName: 'Max Items', - name: 'maxItems', - type: 'number', - typeOptions: { - minValue: 1, - }, - default: 1, - description: 'If there are more items than this number, some are removed', - displayOptions: { - show: { - resource: ['itemList'], - operation: ['limit'], - }, - }, - }, - { - displayName: 'Keep', - name: 'keep', - type: 'options', - options: [ - { - name: 'First Items', - value: 'firstItems', - }, - { - name: 'Last Items', - value: 'lastItems', - }, - ], - default: 'firstItems', - description: 'When removing items, whether to keep the ones at the start or the ending', - displayOptions: { - show: { - resource: ['itemList'], - operation: ['limit'], - }, - }, - }, - { - displayName: 'Options', - name: 'options', - type: 'collection', - placeholder: 'Add Field', - default: {}, - displayOptions: { - show: { - resource: ['itemList'], - operation: ['removeDuplicates'], - compare: ['allFieldsExcept', 'selectedFields'], - }, - }, - options: [ - { - displayName: 'Remove Other Fields', - name: 'removeOtherFields', - type: 'boolean', - default: false, - description: - 'Whether to remove any fields that are not being compared. If disabled, will keep the values from the first of the duplicates.', - }, - { - displayName: 'Disable Dot Notation', - name: 'disableDotNotation', - type: 'boolean', - default: false, - description: - 'Whether to disallow referencing child fields using `parent.child` in the field name', - }, - ], - }, - { - displayName: 'Options', - name: 'options', - type: 'collection', - placeholder: 'Add Field', - default: {}, - displayOptions: { - show: { - resource: ['itemList'], - operation: ['sort'], - type: ['simple'], - }, - }, - options: [ - { - displayName: 'Disable Dot Notation', - name: 'disableDotNotation', - type: 'boolean', - default: false, - description: - 'Whether to disallow referencing child fields using `parent.child` in the field name', - }, - ], - }, - { - displayName: 'Options', - name: 'options', - type: 'collection', - placeholder: 'Add Field', - default: {}, - displayOptions: { - show: { - resource: ['itemList'], - operation: ['splitOutItems', 'aggregateItems'], - }, - hide: { - aggregate: ['aggregateAllItemData'], - }, - }, - options: [ - { - displayName: 'Disable Dot Notation', - name: 'disableDotNotation', - type: 'boolean', - displayOptions: { - show: { - '/operation': ['splitOutItems', 'aggregateItems'], - }, - }, - default: false, - description: - 'Whether to disallow referencing child fields using `parent.child` in the field name', - }, - { - displayName: 'Destination Field Name', - name: 'destinationFieldName', - type: 'string', - displayOptions: { - show: { - '/operation': ['splitOutItems'], - }, - }, - default: '', - description: 'The field in the output under which to put the split field contents', - }, - { - displayName: 'Merge Lists', - name: 'mergeLists', - type: 'boolean', - displayOptions: { - show: { - '/operation': ['aggregateItems'], - }, - }, - default: false, - description: - 'Whether to merge the output into a single flat list (rather than a list of lists), if the field to aggregate is a list', - }, - { - displayName: 'Keep Missing And Null Values', - name: 'keepMissing', - type: 'boolean', - displayOptions: { - show: { - '/operation': ['aggregateItems'], - }, - }, - default: false, - description: - 'Whether to add a null entry to the aggregated list when there is a missing or null value', - }, - ], - }, - // Remove duplicates - Fields - ...summarize.description, - ], - }; - - async execute(this: IExecuteFunctions): Promise { - const items = this.getInputData(); - const length = items.length; - const returnData: INodeExecutionData[] = []; - const resource = this.getNodeParameter('resource', 0); - const operation = this.getNodeParameter('operation', 0); - if (resource === 'itemList') { - if (operation === 'splitOutItems') { - for (let i = 0; i < length; i++) { - const fieldToSplitOut = this.getNodeParameter('fieldToSplitOut', i) as string; - const disableDotNotation = this.getNodeParameter( - 'options.disableDotNotation', - 0, - false, - ) as boolean; - const destinationFieldName = this.getNodeParameter( - 'options.destinationFieldName', - i, - '', - ) as string; - const include = this.getNodeParameter('include', i) as string; - - let arrayToSplit; - if (!disableDotNotation) { - arrayToSplit = get(items[i].json, fieldToSplitOut); - } else { - arrayToSplit = items[i].json[fieldToSplitOut]; - } - - if (arrayToSplit === undefined) { - if (fieldToSplitOut.includes('.') && disableDotNotation) { - throw new NodeOperationError( - this.getNode(), - `Couldn't find the field '${fieldToSplitOut}' in the input data`, - { - description: - "If you're trying to use a nested field, make sure you turn off 'disable dot notation' in the node options", - }, - ); - } else { - throw new NodeOperationError( - this.getNode(), - `Couldn't find the field '${fieldToSplitOut}' in the input data`, - { itemIndex: i }, - ); - } - } - - if (!Array.isArray(arrayToSplit)) { - throw new NodeOperationError( - this.getNode(), - `The provided field '${fieldToSplitOut}' is not an array`, - { itemIndex: i }, - ); - } else { - for (const element of arrayToSplit) { - let newItem = {}; - - if (include === 'selectedOtherFields') { - const fieldsToInclude = ( - this.getNodeParameter('fieldsToInclude.fields', i, []) as [{ fieldName: string }] - ).map((field) => field.fieldName); - - if (!fieldsToInclude.length) { - throw new NodeOperationError(this.getNode(), 'No fields specified', { - description: 'Please add a field to include', - }); - } - - newItem = { - ...fieldsToInclude.reduce((prev, field) => { - if (field === fieldToSplitOut) { - return prev; - } - let value; - if (!disableDotNotation) { - value = get(items[i].json, field); - } else { - value = items[i].json[field]; - } - prev = { ...prev, [field]: value }; - return prev; - }, {}), - }; - } else if (include === 'allOtherFields') { - const keys = Object.keys(items[i].json); - - newItem = { - ...keys.reduce((prev, field) => { - let value; - if (!disableDotNotation) { - value = get(items[i].json, field); - } else { - value = items[i].json[field]; - } - prev = { ...prev, [field]: value }; - return prev; - }, {}), - }; - - unset(newItem, fieldToSplitOut); - } - - if ( - typeof element === 'object' && - include === 'noOtherFields' && - destinationFieldName === '' - ) { - newItem = { ...newItem, ...element }; - } else { - newItem = { - ...newItem, - [destinationFieldName || fieldToSplitOut]: element, - }; - } - - returnData.push({ - json: newItem, - pairedItem: { - item: i, - }, - }); - } - } - } - - return this.prepareOutputData(returnData); - } else if (operation === 'aggregateItems') { - const aggregate = this.getNodeParameter('aggregate', 0, '') as string; - - if (aggregate === 'aggregateIndividualFields') { - const disableDotNotation = this.getNodeParameter( - 'options.disableDotNotation', - 0, - false, - ) as boolean; - const mergeLists = this.getNodeParameter('options.mergeLists', 0, false) as boolean; - const fieldsToAggregate = this.getNodeParameter( - 'fieldsToAggregate.fieldToAggregate', - 0, - [], - ) as [{ fieldToAggregate: string; renameField: boolean; outputFieldName: string }]; - const keepMissing = this.getNodeParameter('options.keepMissing', 0, false) as boolean; - - if (!fieldsToAggregate.length) { - throw new NodeOperationError(this.getNode(), 'No fields specified', { - description: 'Please add a field to aggregate', - }); - } - for (const { fieldToAggregate } of fieldsToAggregate) { - let found = false; - for (const item of items) { - if (fieldToAggregate === '') { - throw new NodeOperationError(this.getNode(), 'Field to aggregate is blank', { - description: 'Please add a field to aggregate', - }); - } - if (!disableDotNotation) { - if (get(item.json, fieldToAggregate) !== undefined) { - found = true; - } - } else if (item.json.hasOwnProperty(fieldToAggregate)) { - found = true; - } - } - if (!found && disableDotNotation && fieldToAggregate.includes('.')) { - throw new NodeOperationError( - this.getNode(), - `Couldn't find the field '${fieldToAggregate}' in the input data`, - { - description: - "If you're trying to use a nested field, make sure you turn off 'disable dot notation' in the node options", - }, - ); - } else if (!found && !keepMissing) { - throw new NodeOperationError( - this.getNode(), - `Couldn't find the field '${fieldToAggregate}' in the input data`, - ); - } - } - - const newItem: INodeExecutionData = { - json: {}, - pairedItem: Array.from({ length }, (_, i) => i).map((index) => { - return { - item: index, - }; - }), - }; - - const values: { [key: string]: any } = {}; - const outputFields: string[] = []; - - for (const { fieldToAggregate, outputFieldName, renameField } of fieldsToAggregate) { - const field = renameField ? outputFieldName : fieldToAggregate; - - if (outputFields.includes(field)) { - throw new NodeOperationError( - this.getNode(), - `The '${field}' output field is used more than once`, - { description: 'Please make sure each output field name is unique' }, - ); - } else { - outputFields.push(field); - } - - const getFieldToAggregate = () => - !disableDotNotation && fieldToAggregate.includes('.') - ? fieldToAggregate.split('.').pop() - : fieldToAggregate; - - const _outputFieldName = outputFieldName - ? outputFieldName - : (getFieldToAggregate() as string); - - if (fieldToAggregate !== '') { - values[_outputFieldName] = []; - for (let i = 0; i < length; i++) { - if (!disableDotNotation) { - let value = get(items[i].json, fieldToAggregate); - - if (!keepMissing) { - if (Array.isArray(value)) { - value = value.filter((entry) => entry !== null); - } else if (value === null || value === undefined) { - continue; - } - } - - if (Array.isArray(value) && mergeLists) { - values[_outputFieldName].push(...value); - } else { - values[_outputFieldName].push(value); - } - } else { - let value = items[i].json[fieldToAggregate]; - - if (!keepMissing) { - if (Array.isArray(value)) { - value = value.filter((entry) => entry !== null); - } else if (value === null || value === undefined) { - continue; - } - } - - if (Array.isArray(value) && mergeLists) { - values[_outputFieldName].push(...value); - } else { - values[_outputFieldName].push(value); - } - } - } - } - } - - for (const key of Object.keys(values)) { - if (!disableDotNotation) { - set(newItem.json, key, values[key]); - } else { - newItem.json[key] = values[key]; - } - } - - returnData.push(newItem); - - return this.prepareOutputData(returnData); - } else { - let newItems: IDataObject[] = items.map((item) => item.json); - const destinationFieldName = this.getNodeParameter('destinationFieldName', 0) as string; - const fieldsToExclude = ( - this.getNodeParameter('fieldsToExclude.fields', 0, []) as IDataObject[] - ).map((entry) => entry.fieldName); - const fieldsToInclude = ( - this.getNodeParameter('fieldsToInclude.fields', 0, []) as IDataObject[] - ).map((entry) => entry.fieldName); - - if (fieldsToExclude.length || fieldsToInclude.length) { - newItems = newItems.reduce((acc, item) => { - const newItem: IDataObject = {}; - let outputFields = Object.keys(item); - - if (fieldsToExclude.length) { - outputFields = outputFields.filter((key) => !fieldsToExclude.includes(key)); - } - if (fieldsToInclude.length) { - outputFields = outputFields.filter((key) => - fieldsToInclude.length ? fieldsToInclude.includes(key) : true, - ); - } - - outputFields.forEach((key) => { - newItem[key] = item[key]; - }); - - if (isEmpty(newItem)) { - return acc; - } - return acc.concat([newItem]); - }, [] as IDataObject[]); - } - - const output: INodeExecutionData = { json: { [destinationFieldName]: newItems } }; - - return this.prepareOutputData([output]); - } - } else if (operation === 'removeDuplicates') { - const compare = this.getNodeParameter('compare', 0) as string; - const disableDotNotation = this.getNodeParameter( - 'options.disableDotNotation', - 0, - false, - ) as boolean; - const removeOtherFields = this.getNodeParameter( - 'options.removeOtherFields', - 0, - false, - ) as boolean; - - let keys = disableDotNotation - ? Object.keys(items[0].json) - : Object.keys(flattenKeys(items[0].json)); - - for (const item of items) { - for (const key of disableDotNotation - ? Object.keys(item.json) - : Object.keys(flattenKeys(item.json))) { - if (!keys.includes(key)) { - keys.push(key); - } - } - } - - if (compare === 'allFieldsExcept') { - const fieldsToExclude = ( - this.getNodeParameter('fieldsToExclude.fields', 0, []) as [{ fieldName: string }] - ).map((field) => field.fieldName); - if (!fieldsToExclude.length) { - throw new NodeOperationError( - this.getNode(), - 'No fields specified. Please add a field to exclude from comparison', - ); - } - if (!disableDotNotation) { - keys = Object.keys(flattenKeys(items[0].json)); - } - keys = keys.filter((key) => !fieldsToExclude.includes(key)); - } - if (compare === 'selectedFields') { - const fieldsToCompare = ( - this.getNodeParameter('fieldsToCompare.fields', 0, []) as [{ fieldName: string }] - ).map((field) => field.fieldName); - if (!fieldsToCompare.length) { - throw new NodeOperationError( - this.getNode(), - 'No fields specified. Please add a field to compare on', - ); - } - if (!disableDotNotation) { - keys = Object.keys(flattenKeys(items[0].json)); - } - keys = fieldsToCompare.map((key) => key.trim()); - } - - // This solution is O(nlogn) - // add original index to the items - const newItems = items.map( - (item, index) => - ({ - json: { ...item.json, __INDEX: index }, - pairedItem: { item: index }, - } as INodeExecutionData), - ); - //sort items using the compare keys - newItems.sort((a, b) => { - let result = 0; - - for (const key of keys) { - let equal; - if (!disableDotNotation) { - equal = isEqual(get(a.json, key), get(b.json, key)); - } else { - equal = isEqual(a.json[key], b.json[key]); - } - if (!equal) { - let lessThan; - if (!disableDotNotation) { - lessThan = lt(get(a.json, key), get(b.json, key)); - } else { - lessThan = lt(a.json[key], b.json[key]); - } - result = lessThan ? -1 : 1; - break; - } - } - return result; - }); - - for (const key of keys) { - let type: any = undefined; - for (const item of newItems) { - if (key === '') { - throw new NodeOperationError(this.getNode(), 'Name of field to compare is blank'); - } - const value = !disableDotNotation ? get(item.json, key) : item.json[key]; - if (value === undefined && disableDotNotation && key.includes('.')) { - throw new NodeOperationError( - this.getNode(), - `'${key}' field is missing from some input items`, - { - description: - "If you're trying to use a nested field, make sure you turn off 'disable dot notation' in the node options", - }, - ); - } else if (value === undefined) { - throw new NodeOperationError( - this.getNode(), - `'${key}' field is missing from some input items`, - ); - } - if (type !== undefined && value !== undefined && type !== typeof value) { - throw new NodeOperationError(this.getNode(), `'${key}' isn't always the same type`, { - description: 'The type of this field varies between items', - }); - } else { - type = typeof value; - } - } - } - - // collect the original indexes of items to be removed - const removedIndexes: number[] = []; - let temp = newItems[0]; - for (let index = 1; index < newItems.length; index++) { - if (compareItems(newItems[index], temp, keys, disableDotNotation, this.getNode())) { - removedIndexes.push(newItems[index].json.__INDEX as unknown as number); - } else { - temp = newItems[index]; - } - } - - let data = items.filter((_, index) => !removedIndexes.includes(index)); - - if (removeOtherFields) { - data = data.map((item, index) => ({ - json: pick(item.json, ...keys), - pairedItem: { item: index }, - })); - } - - // return the filtered items - return this.prepareOutputData(data); - } else if (operation === 'sort') { - let newItems = [...items]; - const type = this.getNodeParameter('type', 0) as string; - const disableDotNotation = this.getNodeParameter( - 'options.disableDotNotation', - 0, - false, - ) as boolean; - - if (type === 'random') { - shuffleArray(newItems); - return this.prepareOutputData(newItems); - } - - if (type === 'simple') { - const sortFieldsUi = this.getNodeParameter('sortFieldsUi', 0) as IDataObject; - const sortFields = sortFieldsUi.sortField as Array<{ - fieldName: string; - order: 'ascending' | 'descending'; - }>; - - if (!sortFields?.length) { - throw new NodeOperationError( - this.getNode(), - 'No sorting specified. Please add a field to sort by', - ); - } - - for (const { fieldName } of sortFields) { - let found = false; - for (const item of items) { - if (!disableDotNotation) { - if (get(item.json, fieldName) !== undefined) { - found = true; - } - } else if (item.json.hasOwnProperty(fieldName)) { - found = true; - } - } - if (!found && disableDotNotation && fieldName.includes('.')) { - throw new NodeOperationError( - this.getNode(), - `Couldn't find the field '${fieldName}' in the input data`, - { - description: - "If you're trying to use a nested field, make sure you turn off 'disable dot notation' in the node options", - }, - ); - } else if (!found) { - throw new NodeOperationError( - this.getNode(), - `Couldn't find the field '${fieldName}' in the input data`, - ); - } - } - - const sortFieldsWithDirection = sortFields.map((field) => ({ - name: field.fieldName, - dir: field.order === 'ascending' ? 1 : -1, - })); - - newItems.sort((a, b) => { - let result = 0; - for (const field of sortFieldsWithDirection) { - let equal; - if (!disableDotNotation) { - const _a = - typeof get(a.json, field.name) === 'string' - ? (get(a.json, field.name) as string).toLowerCase() - : get(a.json, field.name); - const _b = - typeof get(b.json, field.name) === 'string' - ? (get(b.json, field.name) as string).toLowerCase() - : get(b.json, field.name); - equal = isEqual(_a, _b); - } else { - const _a = - typeof a.json[field.name] === 'string' - ? (a.json[field.name] as string).toLowerCase() - : a.json[field.name]; - const _b = - typeof b.json[field.name] === 'string' - ? (b.json[field.name] as string).toLowerCase() - : b.json[field.name]; - equal = isEqual(_a, _b); - } - - if (!equal) { - let lessThan; - if (!disableDotNotation) { - const _a = - typeof get(a.json, field.name) === 'string' - ? (get(a.json, field.name) as string).toLowerCase() - : get(a.json, field.name); - const _b = - typeof get(b.json, field.name) === 'string' - ? (get(b.json, field.name) as string).toLowerCase() - : get(b.json, field.name); - lessThan = lt(_a, _b); - } else { - const _a = - typeof a.json[field.name] === 'string' - ? (a.json[field.name] as string).toLowerCase() - : a.json[field.name]; - const _b = - typeof b.json[field.name] === 'string' - ? (b.json[field.name] as string).toLowerCase() - : b.json[field.name]; - lessThan = lt(_a, _b); - } - if (lessThan) { - result = -1 * field.dir; - } else { - result = 1 * field.dir; - } - break; - } - } - return result; - }); - } else { - const code = this.getNodeParameter('code', 0) as string; - const regexCheck = /\breturn\b/g.exec(code); - - if (regexCheck?.length) { - const sandbox = { - newItems, - }; - const mode = this.getMode(); - const options = { - console: mode === 'manual' ? 'redirect' : 'inherit', - sandbox, - }; - const vm = new NodeVM(options as unknown as NodeVMOptions); - - newItems = await vm.run( - ` - module.exports = async function() { - newItems.sort( (a,b) => { - ${code} - }) - return newItems; - }()`, - __dirname, - ); - } else { - throw new NodeOperationError( - this.getNode(), - "Sort code doesn't return. Please add a 'return' statement to your code", - ); - } - } - return this.prepareOutputData(newItems); - } else if (operation === 'limit') { - let newItems = items; - const maxItems = this.getNodeParameter('maxItems', 0) as number; - const keep = this.getNodeParameter('keep', 0) as string; - - if (maxItems > items.length) { - return this.prepareOutputData(newItems); - } - - if (keep === 'firstItems') { - newItems = items.slice(0, maxItems); - } else { - newItems = items.slice(items.length - maxItems, items.length); - } - return this.prepareOutputData(newItems); - } else if (operation === 'summarize') { - return summarize.execute.call(this, items); - } else { - throw new NodeOperationError(this.getNode(), `Operation '${operation}' is not recognized`); - } - } else { - throw new NodeOperationError(this.getNode(), `Resource '${resource}' is not recognized`); - } + super(nodeVersions, baseDescription); } } diff --git a/packages/nodes-base/nodes/ItemLists/V1/ItemListsV1.node.ts b/packages/nodes-base/nodes/ItemLists/V1/ItemListsV1.node.ts new file mode 100644 index 0000000000..7e230c94d1 --- /dev/null +++ b/packages/nodes-base/nodes/ItemLists/V1/ItemListsV1.node.ts @@ -0,0 +1,1428 @@ +import type { NodeVMOptions } from 'vm2'; +import { NodeVM } from 'vm2'; +import type { IExecuteFunctions } from 'n8n-core'; + +import type { + IDataObject, + INode, + INodeExecutionData, + INodeType, + INodeTypeBaseDescription, + INodeTypeDescription, +} from 'n8n-workflow'; +import { NodeOperationError } from 'n8n-workflow'; + +import get from 'lodash.get'; +import isEmpty from 'lodash.isempty'; +import isEqual from 'lodash.isequal'; +import isObject from 'lodash.isobject'; +import lt from 'lodash.lt'; +import merge from 'lodash.merge'; +import pick from 'lodash.pick'; +import reduce from 'lodash.reduce'; +import set from 'lodash.set'; +import unset from 'lodash.unset'; + +const compareItems = ( + obj: INodeExecutionData, + obj2: INodeExecutionData, + keys: string[], + disableDotNotation: boolean, + _node: INode, +) => { + let result = true; + for (const key of keys) { + if (!disableDotNotation) { + if (!isEqual(get(obj.json, key), get(obj2.json, key))) { + result = false; + break; + } + } else { + if (!isEqual(obj.json[key], obj2.json[key])) { + result = false; + break; + } + } + } + return result; +}; + +const flattenKeys = (obj: IDataObject, path: string[] = []): IDataObject => { + return !isObject(obj) + ? { [path.join('.')]: obj } + : reduce(obj, (cum, next, key) => merge(cum, flattenKeys(next as IDataObject, [...path, key])), {}); //prettier-ignore +}; + +const shuffleArray = (array: any[]) => { + for (let i = array.length - 1; i > 0; i--) { + const j = Math.floor(Math.random() * (i + 1)); + [array[i], array[j]] = [array[j], array[i]]; + } +}; + +import * as summarize from './summarize.operation'; + +export class ItemListsV1 implements INodeType { + description: INodeTypeDescription; + + constructor(baseDescription: INodeTypeBaseDescription) { + this.description = { + ...baseDescription, + version: 1, + defaults: { + name: 'Item Lists', + }, + inputs: ['main'], + outputs: ['main'], + credentials: [], + properties: [ + { + displayName: 'Resource', + name: 'resource', + type: 'hidden', + options: [ + { + name: 'Item List', + value: 'itemList', + }, + ], + default: 'itemList', + }, + { + displayName: 'Operation', + name: 'operation', + type: 'options', + noDataExpression: true, + options: [ + { + name: 'Concatenate Items', + value: 'aggregateItems', + description: 'Combine fields into a list in a single new item', + action: 'Combine fields into a list in a single new item', + }, + { + name: 'Limit', + value: 'limit', + description: 'Remove items if there are too many', + action: 'Remove items if there are too many', + }, + { + name: 'Remove Duplicates', + value: 'removeDuplicates', + description: 'Remove extra items that are similar', + action: 'Remove extra items that are similar', + }, + { + name: 'Sort', + value: 'sort', + description: 'Change the item order', + action: 'Change the item order', + }, + { + name: 'Split Out Items', + value: 'splitOutItems', + description: 'Turn a list inside item(s) into separate items', + action: 'Turn a list inside item(s) into separate items', + }, + { + name: 'Summarize', + value: 'summarize', + description: 'Aggregate items together (pivot table)', + action: 'Aggregate items together (pivot table)', + }, + ], + default: 'splitOutItems', + }, + // Split out items - Fields + { + displayName: 'Field To Split Out', + name: 'fieldToSplitOut', + type: 'string', + default: '', + required: true, + displayOptions: { + show: { + resource: ['itemList'], + operation: ['splitOutItems'], + }, + }, + description: 'The name of the input field to break out into separate items', + requiresDataPath: 'single', + }, + { + displayName: 'Include', + name: 'include', + type: 'options', + options: [ + { + name: 'No Other Fields', + value: 'noOtherFields', + }, + { + name: 'All Other Fields', + value: 'allOtherFields', + }, + { + name: 'Selected Other Fields', + value: 'selectedOtherFields', + }, + ], + default: 'noOtherFields', + description: 'Whether to copy any other fields into the new items', + displayOptions: { + show: { + resource: ['itemList'], + operation: ['splitOutItems'], + }, + }, + }, + { + displayName: 'Fields To Include', + name: 'fieldsToInclude', + type: 'fixedCollection', + typeOptions: { + multipleValues: true, + }, + placeholder: 'Add Field To Include', + default: {}, + displayOptions: { + show: { + resource: ['itemList'], + operation: ['splitOutItems'], + include: ['selectedOtherFields'], + }, + }, + options: [ + { + displayName: '', + name: 'fields', + values: [ + { + displayName: 'Field Name', + name: 'fieldName', + type: 'string', + default: '', + description: 'A field in the input items to aggregate together', + // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id + placeholder: 'e.g. id', + hint: ' Enter the field name as text', + requiresDataPath: 'single', + }, + ], + }, + ], + }, + // Aggregate Items + { + displayName: 'Aggregate', + name: 'aggregate', + type: 'options', + default: 'aggregateIndividualFields', + options: [ + { + name: 'Individual Fields', + value: 'aggregateIndividualFields', + }, + { + name: 'All Item Data (Into a Single List)', + value: 'aggregateAllItemData', + }, + ], + displayOptions: { + show: { + resource: ['itemList'], + operation: ['aggregateItems'], + }, + }, + }, + // Aggregate Individual Fields + { + displayName: 'Fields To Aggregate', + name: 'fieldsToAggregate', + type: 'fixedCollection', + typeOptions: { + multipleValues: true, + }, + placeholder: 'Add Field To Aggregate', + default: { fieldToAggregate: [{ fieldToAggregate: '', renameField: false }] }, + displayOptions: { + show: { + resource: ['itemList'], + operation: ['aggregateItems'], + aggregate: ['aggregateIndividualFields'], + }, + }, + options: [ + { + displayName: '', + name: 'fieldToAggregate', + values: [ + { + displayName: 'Input Field Name', + name: 'fieldToAggregate', + type: 'string', + default: '', + description: 'The name of a field in the input items to aggregate together', + // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id + placeholder: 'e.g. id', + hint: ' Enter the field name as text', + requiresDataPath: 'single', + }, + { + displayName: 'Rename Field', + name: 'renameField', + type: 'boolean', + default: false, + description: 'Whether to give the field a different name in the output', + }, + { + displayName: 'Output Field Name', + name: 'outputFieldName', + displayOptions: { + show: { + renameField: [true], + }, + }, + type: 'string', + default: '', + description: + 'The name of the field to put the aggregated data in. Leave blank to use the input field name.', + requiresDataPath: 'single', + }, + ], + }, + ], + }, + // Aggregate All Item Data + { + displayName: 'Put Output in Field', + name: 'destinationFieldName', + type: 'string', + displayOptions: { + show: { + resource: ['itemList'], + operation: ['aggregateItems'], + aggregate: ['aggregateAllItemData'], + }, + }, + default: 'data', + description: 'The name of the output field to put the data in', + }, + { + displayName: 'Include', + name: 'include', + type: 'options', + default: 'allFields', + options: [ + { + name: 'All Fields', + value: 'allFields', + }, + { + name: 'Specified Fields', + value: 'specifiedFields', + }, + { + name: 'All Fields Except', + value: 'allFieldsExcept', + }, + ], + displayOptions: { + show: { + resource: ['itemList'], + operation: ['aggregateItems'], + aggregate: ['aggregateAllItemData'], + }, + }, + }, + { + displayName: 'Fields To Exclude', + name: 'fieldsToExclude', + type: 'fixedCollection', + typeOptions: { + multipleValues: true, + }, + placeholder: 'Add Field To Exclude', + default: {}, + options: [ + { + displayName: '', + name: 'fields', + values: [ + { + displayName: 'Field Name', + name: 'fieldName', + type: 'string', + default: '', + description: 'A field in the input to exclude from the object in output array', + // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id + placeholder: 'e.g. id', + hint: ' Enter the field name as text', + requiresDataPath: 'single', + }, + ], + }, + ], + displayOptions: { + show: { + resource: ['itemList'], + operation: ['aggregateItems'], + aggregate: ['aggregateAllItemData'], + include: ['allFieldsExcept'], + }, + }, + }, + { + displayName: 'Fields To Include', + name: 'fieldsToInclude', + type: 'fixedCollection', + typeOptions: { + multipleValues: true, + }, + placeholder: 'Add Field To Include', + default: {}, + options: [ + { + displayName: '', + name: 'fields', + values: [ + { + displayName: 'Field Name', + name: 'fieldName', + type: 'string', + default: '', + description: 'Specify fields that will be included in output array', + // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id + placeholder: 'e.g. id', + hint: ' Enter the field name as text', + requiresDataPath: 'single', + }, + ], + }, + ], + displayOptions: { + show: { + resource: ['itemList'], + operation: ['aggregateItems'], + aggregate: ['aggregateAllItemData'], + include: ['specifiedFields'], + }, + }, + }, + // Remove duplicates - Fields + { + displayName: 'Compare', + name: 'compare', + type: 'options', + options: [ + { + name: 'All Fields', + value: 'allFields', + }, + { + name: 'All Fields Except', + value: 'allFieldsExcept', + }, + { + name: 'Selected Fields', + value: 'selectedFields', + }, + ], + default: 'allFields', + description: 'The fields of the input items to compare to see if they are the same', + displayOptions: { + show: { + resource: ['itemList'], + operation: ['removeDuplicates'], + }, + }, + }, + { + displayName: 'Fields To Exclude', + name: 'fieldsToExclude', + type: 'fixedCollection', + typeOptions: { + multipleValues: true, + }, + placeholder: 'Add Field To Exclude', + default: {}, + displayOptions: { + show: { + resource: ['itemList'], + operation: ['removeDuplicates'], + compare: ['allFieldsExcept'], + }, + }, + options: [ + { + displayName: '', + name: 'fields', + values: [ + { + displayName: 'Field Name', + name: 'fieldName', + type: 'string', + default: '', + description: 'A field in the input to exclude from the comparison', + // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id + placeholder: 'e.g. id', + hint: ' Enter the field name as text', + requiresDataPath: 'single', + }, + ], + }, + ], + }, + { + displayName: 'Fields To Compare', + name: 'fieldsToCompare', + type: 'fixedCollection', + typeOptions: { + multipleValues: true, + }, + placeholder: 'Add Field To Compare', + default: {}, + displayOptions: { + show: { + resource: ['itemList'], + operation: ['removeDuplicates'], + compare: ['selectedFields'], + }, + }, + options: [ + { + displayName: '', + name: 'fields', + values: [ + { + displayName: 'Field Name', + name: 'fieldName', + type: 'string', + default: '', + description: 'A field in the input to add to the comparison', + // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id + placeholder: 'e.g. id', + hint: ' Enter the field name as text', + requiresDataPath: 'single', + }, + ], + }, + ], + }, + // Sort - Fields + { + displayName: 'Type', + name: 'type', + type: 'options', + options: [ + { + name: 'Simple', + value: 'simple', + }, + { + name: 'Random', + value: 'random', + }, + { + name: 'Code', + value: 'code', + }, + ], + default: 'simple', + description: 'The fields of the input items to compare to see if they are the same', + displayOptions: { + show: { + resource: ['itemList'], + operation: ['sort'], + }, + }, + }, + { + displayName: 'Fields To Sort By', + name: 'sortFieldsUi', + type: 'fixedCollection', + typeOptions: { + multipleValues: true, + }, + placeholder: 'Add Field To Sort By', + options: [ + { + displayName: '', + name: 'sortField', + values: [ + { + displayName: 'Field Name', + name: 'fieldName', + type: 'string', + required: true, + default: '', + description: 'The field to sort by', + // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id + placeholder: 'e.g. id', + hint: ' Enter the field name as text', + requiresDataPath: 'single', + }, + { + displayName: 'Order', + name: 'order', + type: 'options', + options: [ + { + name: 'Ascending', + value: 'ascending', + }, + { + name: 'Descending', + value: 'descending', + }, + ], + default: 'ascending', + description: 'The order to sort by', + }, + ], + }, + ], + default: {}, + description: 'The fields of the input items to compare to see if they are the same', + displayOptions: { + show: { + resource: ['itemList'], + operation: ['sort'], + type: ['simple'], + }, + }, + }, + { + displayName: 'Code', + name: 'code', + type: 'string', + typeOptions: { + alwaysOpenEditWindow: true, + editor: 'code', + rows: 10, + }, + default: `// The two items to compare are in the variables a and b +// Access the fields in a.json and b.json +// Return -1 if a should go before b +// Return 1 if b should go before a +// Return 0 if there's no difference + +fieldName = 'myField'; + +if (a.json[fieldName] < b.json[fieldName]) { + return -1; +} +if (a.json[fieldName] > b.json[fieldName]) { + return 1; +} +return 0;`, + description: 'Javascript code to determine the order of any two items', + displayOptions: { + show: { + resource: ['itemList'], + operation: ['sort'], + type: ['code'], + }, + }, + }, + // Limit - Fields + { + displayName: 'Max Items', + name: 'maxItems', + type: 'number', + typeOptions: { + minValue: 1, + }, + default: 1, + description: 'If there are more items than this number, some are removed', + displayOptions: { + show: { + resource: ['itemList'], + operation: ['limit'], + }, + }, + }, + { + displayName: 'Keep', + name: 'keep', + type: 'options', + options: [ + { + name: 'First Items', + value: 'firstItems', + }, + { + name: 'Last Items', + value: 'lastItems', + }, + ], + default: 'firstItems', + description: 'When removing items, whether to keep the ones at the start or the ending', + displayOptions: { + show: { + resource: ['itemList'], + operation: ['limit'], + }, + }, + }, + { + displayName: 'Options', + name: 'options', + type: 'collection', + placeholder: 'Add Field', + default: {}, + displayOptions: { + show: { + resource: ['itemList'], + operation: ['removeDuplicates'], + compare: ['allFieldsExcept', 'selectedFields'], + }, + }, + options: [ + { + displayName: 'Remove Other Fields', + name: 'removeOtherFields', + type: 'boolean', + default: false, + description: + 'Whether to remove any fields that are not being compared. If disabled, will keep the values from the first of the duplicates.', + }, + { + displayName: 'Disable Dot Notation', + name: 'disableDotNotation', + type: 'boolean', + default: false, + description: + 'Whether to disallow referencing child fields using `parent.child` in the field name', + }, + ], + }, + { + displayName: 'Options', + name: 'options', + type: 'collection', + placeholder: 'Add Field', + default: {}, + displayOptions: { + show: { + resource: ['itemList'], + operation: ['sort'], + type: ['simple'], + }, + }, + options: [ + { + displayName: 'Disable Dot Notation', + name: 'disableDotNotation', + type: 'boolean', + default: false, + description: + 'Whether to disallow referencing child fields using `parent.child` in the field name', + }, + ], + }, + { + displayName: 'Options', + name: 'options', + type: 'collection', + placeholder: 'Add Field', + default: {}, + displayOptions: { + show: { + resource: ['itemList'], + operation: ['splitOutItems', 'aggregateItems'], + }, + hide: { + aggregate: ['aggregateAllItemData'], + }, + }, + options: [ + { + displayName: 'Disable Dot Notation', + name: 'disableDotNotation', + type: 'boolean', + displayOptions: { + show: { + '/operation': ['splitOutItems', 'aggregateItems'], + }, + }, + default: false, + description: + 'Whether to disallow referencing child fields using `parent.child` in the field name', + }, + { + displayName: 'Destination Field Name', + name: 'destinationFieldName', + type: 'string', + displayOptions: { + show: { + '/operation': ['splitOutItems'], + }, + }, + default: '', + description: 'The field in the output under which to put the split field contents', + }, + { + displayName: 'Merge Lists', + name: 'mergeLists', + type: 'boolean', + displayOptions: { + show: { + '/operation': ['aggregateItems'], + }, + }, + default: false, + description: + 'Whether to merge the output into a single flat list (rather than a list of lists), if the field to aggregate is a list', + }, + { + displayName: 'Keep Missing And Null Values', + name: 'keepMissing', + type: 'boolean', + displayOptions: { + show: { + '/operation': ['aggregateItems'], + }, + }, + default: false, + description: + 'Whether to add a null entry to the aggregated list when there is a missing or null value', + }, + ], + }, + // Remove duplicates - Fields + ...summarize.description, + ], + }; + } + + async execute(this: IExecuteFunctions): Promise { + const items = this.getInputData(); + const length = items.length; + const returnData: INodeExecutionData[] = []; + const resource = this.getNodeParameter('resource', 0); + const operation = this.getNodeParameter('operation', 0); + if (resource === 'itemList') { + if (operation === 'splitOutItems') { + for (let i = 0; i < length; i++) { + const fieldToSplitOut = this.getNodeParameter('fieldToSplitOut', i) as string; + const disableDotNotation = this.getNodeParameter( + 'options.disableDotNotation', + 0, + false, + ) as boolean; + const destinationFieldName = this.getNodeParameter( + 'options.destinationFieldName', + i, + '', + ) as string; + const include = this.getNodeParameter('include', i) as string; + + let arrayToSplit; + if (!disableDotNotation) { + arrayToSplit = get(items[i].json, fieldToSplitOut); + } else { + arrayToSplit = items[i].json[fieldToSplitOut]; + } + + if (arrayToSplit === undefined) { + if (fieldToSplitOut.includes('.') && disableDotNotation) { + throw new NodeOperationError( + this.getNode(), + `Couldn't find the field '${fieldToSplitOut}' in the input data`, + { + description: + "If you're trying to use a nested field, make sure you turn off 'disable dot notation' in the node options", + }, + ); + } else { + throw new NodeOperationError( + this.getNode(), + `Couldn't find the field '${fieldToSplitOut}' in the input data`, + { itemIndex: i }, + ); + } + } + + if (!Array.isArray(arrayToSplit)) { + throw new NodeOperationError( + this.getNode(), + `The provided field '${fieldToSplitOut}' is not an array`, + { itemIndex: i }, + ); + } else { + for (const element of arrayToSplit) { + let newItem = {}; + + if (include === 'selectedOtherFields') { + const fieldsToInclude = ( + this.getNodeParameter('fieldsToInclude.fields', i, []) as [{ fieldName: string }] + ).map((field) => field.fieldName); + + if (!fieldsToInclude.length) { + throw new NodeOperationError(this.getNode(), 'No fields specified', { + description: 'Please add a field to include', + }); + } + + newItem = { + ...fieldsToInclude.reduce((prev, field) => { + if (field === fieldToSplitOut) { + return prev; + } + let value; + if (!disableDotNotation) { + value = get(items[i].json, field); + } else { + value = items[i].json[field]; + } + prev = { ...prev, [field]: value }; + return prev; + }, {}), + }; + } else if (include === 'allOtherFields') { + const keys = Object.keys(items[i].json); + + newItem = { + ...keys.reduce((prev, field) => { + let value; + if (!disableDotNotation) { + value = get(items[i].json, field); + } else { + value = items[i].json[field]; + } + prev = { ...prev, [field]: value }; + return prev; + }, {}), + }; + + unset(newItem, fieldToSplitOut); + } + + if ( + typeof element === 'object' && + include === 'noOtherFields' && + destinationFieldName === '' + ) { + newItem = { ...newItem, ...element }; + } else { + newItem = { + ...newItem, + [destinationFieldName || fieldToSplitOut]: element, + }; + } + + returnData.push({ + json: newItem, + pairedItem: { + item: i, + }, + }); + } + } + } + + return this.prepareOutputData(returnData); + } else if (operation === 'aggregateItems') { + const aggregate = this.getNodeParameter('aggregate', 0, '') as string; + + if (aggregate === 'aggregateIndividualFields') { + const disableDotNotation = this.getNodeParameter( + 'options.disableDotNotation', + 0, + false, + ) as boolean; + const mergeLists = this.getNodeParameter('options.mergeLists', 0, false) as boolean; + const fieldsToAggregate = this.getNodeParameter( + 'fieldsToAggregate.fieldToAggregate', + 0, + [], + ) as [{ fieldToAggregate: string; renameField: boolean; outputFieldName: string }]; + const keepMissing = this.getNodeParameter('options.keepMissing', 0, false) as boolean; + + if (!fieldsToAggregate.length) { + throw new NodeOperationError(this.getNode(), 'No fields specified', { + description: 'Please add a field to aggregate', + }); + } + for (const { fieldToAggregate } of fieldsToAggregate) { + let found = false; + for (const item of items) { + if (fieldToAggregate === '') { + throw new NodeOperationError(this.getNode(), 'Field to aggregate is blank', { + description: 'Please add a field to aggregate', + }); + } + if (!disableDotNotation) { + if (get(item.json, fieldToAggregate) !== undefined) { + found = true; + } + } else if (item.json.hasOwnProperty(fieldToAggregate)) { + found = true; + } + } + if (!found && disableDotNotation && fieldToAggregate.includes('.')) { + throw new NodeOperationError( + this.getNode(), + `Couldn't find the field '${fieldToAggregate}' in the input data`, + { + description: + "If you're trying to use a nested field, make sure you turn off 'disable dot notation' in the node options", + }, + ); + } else if (!found && !keepMissing) { + throw new NodeOperationError( + this.getNode(), + `Couldn't find the field '${fieldToAggregate}' in the input data`, + ); + } + } + + const newItem: INodeExecutionData = { + json: {}, + pairedItem: Array.from({ length }, (_, i) => i).map((index) => { + return { + item: index, + }; + }), + }; + + const values: { [key: string]: any } = {}; + const outputFields: string[] = []; + + for (const { fieldToAggregate, outputFieldName, renameField } of fieldsToAggregate) { + const field = renameField ? outputFieldName : fieldToAggregate; + + if (outputFields.includes(field)) { + throw new NodeOperationError( + this.getNode(), + `The '${field}' output field is used more than once`, + { description: 'Please make sure each output field name is unique' }, + ); + } else { + outputFields.push(field); + } + + const getFieldToAggregate = () => + !disableDotNotation && fieldToAggregate.includes('.') + ? fieldToAggregate.split('.').pop() + : fieldToAggregate; + + const _outputFieldName = outputFieldName + ? outputFieldName + : (getFieldToAggregate() as string); + + if (fieldToAggregate !== '') { + values[_outputFieldName] = []; + for (let i = 0; i < length; i++) { + if (!disableDotNotation) { + let value = get(items[i].json, fieldToAggregate); + + if (!keepMissing) { + if (Array.isArray(value)) { + value = value.filter((entry) => entry !== null); + } else if (value === null || value === undefined) { + continue; + } + } + + if (Array.isArray(value) && mergeLists) { + values[_outputFieldName].push(...value); + } else { + values[_outputFieldName].push(value); + } + } else { + let value = items[i].json[fieldToAggregate]; + + if (!keepMissing) { + if (Array.isArray(value)) { + value = value.filter((entry) => entry !== null); + } else if (value === null || value === undefined) { + continue; + } + } + + if (Array.isArray(value) && mergeLists) { + values[_outputFieldName].push(...value); + } else { + values[_outputFieldName].push(value); + } + } + } + } + } + + for (const key of Object.keys(values)) { + if (!disableDotNotation) { + set(newItem.json, key, values[key]); + } else { + newItem.json[key] = values[key]; + } + } + + returnData.push(newItem); + + return this.prepareOutputData(returnData); + } else { + let newItems: IDataObject[] = items.map((item) => item.json); + const destinationFieldName = this.getNodeParameter('destinationFieldName', 0) as string; + const fieldsToExclude = ( + this.getNodeParameter('fieldsToExclude.fields', 0, []) as IDataObject[] + ).map((entry) => entry.fieldName); + const fieldsToInclude = ( + this.getNodeParameter('fieldsToInclude.fields', 0, []) as IDataObject[] + ).map((entry) => entry.fieldName); + + if (fieldsToExclude.length || fieldsToInclude.length) { + newItems = newItems.reduce((acc, item) => { + const newItem: IDataObject = {}; + let outputFields = Object.keys(item); + + if (fieldsToExclude.length) { + outputFields = outputFields.filter((key) => !fieldsToExclude.includes(key)); + } + if (fieldsToInclude.length) { + outputFields = outputFields.filter((key) => + fieldsToInclude.length ? fieldsToInclude.includes(key) : true, + ); + } + + outputFields.forEach((key) => { + newItem[key] = item[key]; + }); + + if (isEmpty(newItem)) { + return acc; + } + return acc.concat([newItem]); + }, [] as IDataObject[]); + } + + const output: INodeExecutionData = { json: { [destinationFieldName]: newItems } }; + + return this.prepareOutputData([output]); + } + } else if (operation === 'removeDuplicates') { + const compare = this.getNodeParameter('compare', 0) as string; + const disableDotNotation = this.getNodeParameter( + 'options.disableDotNotation', + 0, + false, + ) as boolean; + const removeOtherFields = this.getNodeParameter( + 'options.removeOtherFields', + 0, + false, + ) as boolean; + + let keys = disableDotNotation + ? Object.keys(items[0].json) + : Object.keys(flattenKeys(items[0].json)); + + for (const item of items) { + for (const key of disableDotNotation + ? Object.keys(item.json) + : Object.keys(flattenKeys(item.json))) { + if (!keys.includes(key)) { + keys.push(key); + } + } + } + + if (compare === 'allFieldsExcept') { + const fieldsToExclude = ( + this.getNodeParameter('fieldsToExclude.fields', 0, []) as [{ fieldName: string }] + ).map((field) => field.fieldName); + if (!fieldsToExclude.length) { + throw new NodeOperationError( + this.getNode(), + 'No fields specified. Please add a field to exclude from comparison', + ); + } + if (!disableDotNotation) { + keys = Object.keys(flattenKeys(items[0].json)); + } + keys = keys.filter((key) => !fieldsToExclude.includes(key)); + } + if (compare === 'selectedFields') { + const fieldsToCompare = ( + this.getNodeParameter('fieldsToCompare.fields', 0, []) as [{ fieldName: string }] + ).map((field) => field.fieldName); + if (!fieldsToCompare.length) { + throw new NodeOperationError( + this.getNode(), + 'No fields specified. Please add a field to compare on', + ); + } + if (!disableDotNotation) { + keys = Object.keys(flattenKeys(items[0].json)); + } + keys = fieldsToCompare.map((key) => key.trim()); + } + + // This solution is O(nlogn) + // add original index to the items + const newItems = items.map( + (item, index) => + ({ + json: { ...item.json, __INDEX: index }, + pairedItem: { item: index }, + } as INodeExecutionData), + ); + //sort items using the compare keys + newItems.sort((a, b) => { + let result = 0; + + for (const key of keys) { + let equal; + if (!disableDotNotation) { + equal = isEqual(get(a.json, key), get(b.json, key)); + } else { + equal = isEqual(a.json[key], b.json[key]); + } + if (!equal) { + let lessThan; + if (!disableDotNotation) { + lessThan = lt(get(a.json, key), get(b.json, key)); + } else { + lessThan = lt(a.json[key], b.json[key]); + } + result = lessThan ? -1 : 1; + break; + } + } + return result; + }); + + for (const key of keys) { + let type: any = undefined; + for (const item of newItems) { + if (key === '') { + throw new NodeOperationError(this.getNode(), 'Name of field to compare is blank'); + } + const value = !disableDotNotation ? get(item.json, key) : item.json[key]; + if (value === undefined && disableDotNotation && key.includes('.')) { + throw new NodeOperationError( + this.getNode(), + `'${key}' field is missing from some input items`, + { + description: + "If you're trying to use a nested field, make sure you turn off 'disable dot notation' in the node options", + }, + ); + } else if (value === undefined) { + throw new NodeOperationError( + this.getNode(), + `'${key}' field is missing from some input items`, + ); + } + if (type !== undefined && value !== undefined && type !== typeof value) { + throw new NodeOperationError(this.getNode(), `'${key}' isn't always the same type`, { + description: 'The type of this field varies between items', + }); + } else { + type = typeof value; + } + } + } + + // collect the original indexes of items to be removed + const removedIndexes: number[] = []; + let temp = newItems[0]; + for (let index = 1; index < newItems.length; index++) { + if (compareItems(newItems[index], temp, keys, disableDotNotation, this.getNode())) { + removedIndexes.push(newItems[index].json.__INDEX as unknown as number); + } else { + temp = newItems[index]; + } + } + + let data = items.filter((_, index) => !removedIndexes.includes(index)); + + if (removeOtherFields) { + data = data.map((item, index) => ({ + json: pick(item.json, ...keys), + pairedItem: { item: index }, + })); + } + + // return the filtered items + return this.prepareOutputData(data); + } else if (operation === 'sort') { + let newItems = [...items]; + const type = this.getNodeParameter('type', 0) as string; + const disableDotNotation = this.getNodeParameter( + 'options.disableDotNotation', + 0, + false, + ) as boolean; + + if (type === 'random') { + shuffleArray(newItems); + return this.prepareOutputData(newItems); + } + + if (type === 'simple') { + const sortFieldsUi = this.getNodeParameter('sortFieldsUi', 0) as IDataObject; + const sortFields = sortFieldsUi.sortField as Array<{ + fieldName: string; + order: 'ascending' | 'descending'; + }>; + + if (!sortFields?.length) { + throw new NodeOperationError( + this.getNode(), + 'No sorting specified. Please add a field to sort by', + ); + } + + for (const { fieldName } of sortFields) { + let found = false; + for (const item of items) { + if (!disableDotNotation) { + if (get(item.json, fieldName) !== undefined) { + found = true; + } + } else if (item.json.hasOwnProperty(fieldName)) { + found = true; + } + } + if (!found && disableDotNotation && fieldName.includes('.')) { + throw new NodeOperationError( + this.getNode(), + `Couldn't find the field '${fieldName}' in the input data`, + { + description: + "If you're trying to use a nested field, make sure you turn off 'disable dot notation' in the node options", + }, + ); + } else if (!found) { + throw new NodeOperationError( + this.getNode(), + `Couldn't find the field '${fieldName}' in the input data`, + ); + } + } + + const sortFieldsWithDirection = sortFields.map((field) => ({ + name: field.fieldName, + dir: field.order === 'ascending' ? 1 : -1, + })); + + newItems.sort((a, b) => { + let result = 0; + for (const field of sortFieldsWithDirection) { + let equal; + if (!disableDotNotation) { + const _a = + typeof get(a.json, field.name) === 'string' + ? (get(a.json, field.name) as string).toLowerCase() + : get(a.json, field.name); + const _b = + typeof get(b.json, field.name) === 'string' + ? (get(b.json, field.name) as string).toLowerCase() + : get(b.json, field.name); + equal = isEqual(_a, _b); + } else { + const _a = + typeof a.json[field.name] === 'string' + ? (a.json[field.name] as string).toLowerCase() + : a.json[field.name]; + const _b = + typeof b.json[field.name] === 'string' + ? (b.json[field.name] as string).toLowerCase() + : b.json[field.name]; + equal = isEqual(_a, _b); + } + + if (!equal) { + let lessThan; + if (!disableDotNotation) { + const _a = + typeof get(a.json, field.name) === 'string' + ? (get(a.json, field.name) as string).toLowerCase() + : get(a.json, field.name); + const _b = + typeof get(b.json, field.name) === 'string' + ? (get(b.json, field.name) as string).toLowerCase() + : get(b.json, field.name); + lessThan = lt(_a, _b); + } else { + const _a = + typeof a.json[field.name] === 'string' + ? (a.json[field.name] as string).toLowerCase() + : a.json[field.name]; + const _b = + typeof b.json[field.name] === 'string' + ? (b.json[field.name] as string).toLowerCase() + : b.json[field.name]; + lessThan = lt(_a, _b); + } + if (lessThan) { + result = -1 * field.dir; + } else { + result = 1 * field.dir; + } + break; + } + } + return result; + }); + } else { + const code = this.getNodeParameter('code', 0) as string; + const regexCheck = /\breturn\b/g.exec(code); + + if (regexCheck?.length) { + const sandbox = { + newItems, + }; + const mode = this.getMode(); + const options = { + console: mode === 'manual' ? 'redirect' : 'inherit', + sandbox, + }; + const vm = new NodeVM(options as unknown as NodeVMOptions); + + newItems = await vm.run( + ` + module.exports = async function() { + newItems.sort( (a,b) => { + ${code} + }) + return newItems; + }()`, + __dirname, + ); + } else { + throw new NodeOperationError( + this.getNode(), + "Sort code doesn't return. Please add a 'return' statement to your code", + ); + } + } + return this.prepareOutputData(newItems); + } else if (operation === 'limit') { + let newItems = items; + const maxItems = this.getNodeParameter('maxItems', 0) as number; + const keep = this.getNodeParameter('keep', 0) as string; + + if (maxItems > items.length) { + return this.prepareOutputData(newItems); + } + + if (keep === 'firstItems') { + newItems = items.slice(0, maxItems); + } else { + newItems = items.slice(items.length - maxItems, items.length); + } + return this.prepareOutputData(newItems); + } else if (operation === 'summarize') { + return summarize.execute.call(this, items); + } else { + throw new NodeOperationError(this.getNode(), `Operation '${operation}' is not recognized`); + } + } else { + throw new NodeOperationError(this.getNode(), `Resource '${resource}' is not recognized`); + } + } +} diff --git a/packages/nodes-base/nodes/ItemLists/summarize.operation.ts b/packages/nodes-base/nodes/ItemLists/V1/summarize.operation.ts similarity index 100% rename from packages/nodes-base/nodes/ItemLists/summarize.operation.ts rename to packages/nodes-base/nodes/ItemLists/V1/summarize.operation.ts diff --git a/packages/nodes-base/nodes/ItemLists/V2/ItemListsV2.node.ts b/packages/nodes-base/nodes/ItemLists/V2/ItemListsV2.node.ts new file mode 100644 index 0000000000..04cf2f1808 --- /dev/null +++ b/packages/nodes-base/nodes/ItemLists/V2/ItemListsV2.node.ts @@ -0,0 +1,1428 @@ +import type { NodeVMOptions } from 'vm2'; +import { NodeVM } from 'vm2'; +import type { IExecuteFunctions } from 'n8n-core'; + +import type { + IDataObject, + INode, + INodeExecutionData, + INodeType, + INodeTypeBaseDescription, + INodeTypeDescription, +} from 'n8n-workflow'; +import { NodeOperationError } from 'n8n-workflow'; + +import get from 'lodash.get'; +import isEmpty from 'lodash.isempty'; +import isEqual from 'lodash.isequal'; +import isObject from 'lodash.isobject'; +import lt from 'lodash.lt'; +import merge from 'lodash.merge'; +import pick from 'lodash.pick'; +import reduce from 'lodash.reduce'; +import set from 'lodash.set'; +import unset from 'lodash.unset'; + +const compareItems = ( + obj: INodeExecutionData, + obj2: INodeExecutionData, + keys: string[], + disableDotNotation: boolean, + _node: INode, +) => { + let result = true; + for (const key of keys) { + if (!disableDotNotation) { + if (!isEqual(get(obj.json, key), get(obj2.json, key))) { + result = false; + break; + } + } else { + if (!isEqual(obj.json[key], obj2.json[key])) { + result = false; + break; + } + } + } + return result; +}; + +const flattenKeys = (obj: IDataObject, path: string[] = []): IDataObject => { + return !isObject(obj) + ? { [path.join('.')]: obj } + : reduce(obj, (cum, next, key) => merge(cum, flattenKeys(next as IDataObject, [...path, key])), {}); //prettier-ignore +}; + +const shuffleArray = (array: any[]) => { + for (let i = array.length - 1; i > 0; i--) { + const j = Math.floor(Math.random() * (i + 1)); + [array[i], array[j]] = [array[j], array[i]]; + } +}; + +import * as summarize from './summarize.operation'; + +export class ItemListsV2 implements INodeType { + description: INodeTypeDescription; + + constructor(baseDescription: INodeTypeBaseDescription) { + this.description = { + ...baseDescription, + version: 2, + defaults: { + name: 'Item Lists', + }, + inputs: ['main'], + outputs: ['main'], + credentials: [], + properties: [ + { + displayName: 'Resource', + name: 'resource', + type: 'hidden', + options: [ + { + name: 'Item List', + value: 'itemList', + }, + ], + default: 'itemList', + }, + { + displayName: 'Operation', + name: 'operation', + type: 'options', + noDataExpression: true, + options: [ + { + name: 'Concatenate Items', + value: 'aggregateItems', + description: 'Combine fields into a list in a single new item', + action: 'Combine fields into a list in a single new item', + }, + { + name: 'Limit', + value: 'limit', + description: 'Remove items if there are too many', + action: 'Remove items if there are too many', + }, + { + name: 'Remove Duplicates', + value: 'removeDuplicates', + description: 'Remove extra items that are similar', + action: 'Remove extra items that are similar', + }, + { + name: 'Sort', + value: 'sort', + description: 'Change the item order', + action: 'Change the item order', + }, + { + name: 'Split Out Items', + value: 'splitOutItems', + description: 'Turn a list inside item(s) into separate items', + action: 'Turn a list inside item(s) into separate items', + }, + { + name: 'Summarize', + value: 'summarize', + description: 'Aggregate items together (pivot table)', + action: 'Aggregate items together (pivot table)', + }, + ], + default: 'splitOutItems', + }, + // Split out items - Fields + { + displayName: 'Field To Split Out', + name: 'fieldToSplitOut', + type: 'string', + default: '', + required: true, + displayOptions: { + show: { + resource: ['itemList'], + operation: ['splitOutItems'], + }, + }, + description: 'The name of the input field to break out into separate items', + requiresDataPath: 'single', + }, + { + displayName: 'Include', + name: 'include', + type: 'options', + options: [ + { + name: 'No Other Fields', + value: 'noOtherFields', + }, + { + name: 'All Other Fields', + value: 'allOtherFields', + }, + { + name: 'Selected Other Fields', + value: 'selectedOtherFields', + }, + ], + default: 'noOtherFields', + description: 'Whether to copy any other fields into the new items', + displayOptions: { + show: { + resource: ['itemList'], + operation: ['splitOutItems'], + }, + }, + }, + { + displayName: 'Fields To Include', + name: 'fieldsToInclude', + type: 'fixedCollection', + typeOptions: { + multipleValues: true, + }, + placeholder: 'Add Field To Include', + default: {}, + displayOptions: { + show: { + resource: ['itemList'], + operation: ['splitOutItems'], + include: ['selectedOtherFields'], + }, + }, + options: [ + { + displayName: '', + name: 'fields', + values: [ + { + displayName: 'Field Name', + name: 'fieldName', + type: 'string', + default: '', + description: 'A field in the input items to aggregate together', + // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id + placeholder: 'e.g. id', + hint: ' Enter the field name as text', + requiresDataPath: 'single', + }, + ], + }, + ], + }, + // Aggregate Items + { + displayName: 'Aggregate', + name: 'aggregate', + type: 'options', + default: 'aggregateIndividualFields', + options: [ + { + name: 'Individual Fields', + value: 'aggregateIndividualFields', + }, + { + name: 'All Item Data (Into a Single List)', + value: 'aggregateAllItemData', + }, + ], + displayOptions: { + show: { + resource: ['itemList'], + operation: ['aggregateItems'], + }, + }, + }, + // Aggregate Individual Fields + { + displayName: 'Fields To Aggregate', + name: 'fieldsToAggregate', + type: 'fixedCollection', + typeOptions: { + multipleValues: true, + }, + placeholder: 'Add Field To Aggregate', + default: { fieldToAggregate: [{ fieldToAggregate: '', renameField: false }] }, + displayOptions: { + show: { + resource: ['itemList'], + operation: ['aggregateItems'], + aggregate: ['aggregateIndividualFields'], + }, + }, + options: [ + { + displayName: '', + name: 'fieldToAggregate', + values: [ + { + displayName: 'Input Field Name', + name: 'fieldToAggregate', + type: 'string', + default: '', + description: 'The name of a field in the input items to aggregate together', + // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id + placeholder: 'e.g. id', + hint: ' Enter the field name as text', + requiresDataPath: 'single', + }, + { + displayName: 'Rename Field', + name: 'renameField', + type: 'boolean', + default: false, + description: 'Whether to give the field a different name in the output', + }, + { + displayName: 'Output Field Name', + name: 'outputFieldName', + displayOptions: { + show: { + renameField: [true], + }, + }, + type: 'string', + default: '', + description: + 'The name of the field to put the aggregated data in. Leave blank to use the input field name.', + requiresDataPath: 'single', + }, + ], + }, + ], + }, + // Aggregate All Item Data + { + displayName: 'Put Output in Field', + name: 'destinationFieldName', + type: 'string', + displayOptions: { + show: { + resource: ['itemList'], + operation: ['aggregateItems'], + aggregate: ['aggregateAllItemData'], + }, + }, + default: 'data', + description: 'The name of the output field to put the data in', + }, + { + displayName: 'Include', + name: 'include', + type: 'options', + default: 'allFields', + options: [ + { + name: 'All Fields', + value: 'allFields', + }, + { + name: 'Specified Fields', + value: 'specifiedFields', + }, + { + name: 'All Fields Except', + value: 'allFieldsExcept', + }, + ], + displayOptions: { + show: { + resource: ['itemList'], + operation: ['aggregateItems'], + aggregate: ['aggregateAllItemData'], + }, + }, + }, + { + displayName: 'Fields To Exclude', + name: 'fieldsToExclude', + type: 'fixedCollection', + typeOptions: { + multipleValues: true, + }, + placeholder: 'Add Field To Exclude', + default: {}, + options: [ + { + displayName: '', + name: 'fields', + values: [ + { + displayName: 'Field Name', + name: 'fieldName', + type: 'string', + default: '', + description: 'A field in the input to exclude from the object in output array', + // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id + placeholder: 'e.g. id', + hint: ' Enter the field name as text', + requiresDataPath: 'single', + }, + ], + }, + ], + displayOptions: { + show: { + resource: ['itemList'], + operation: ['aggregateItems'], + aggregate: ['aggregateAllItemData'], + include: ['allFieldsExcept'], + }, + }, + }, + { + displayName: 'Fields To Include', + name: 'fieldsToInclude', + type: 'fixedCollection', + typeOptions: { + multipleValues: true, + }, + placeholder: 'Add Field To Include', + default: {}, + options: [ + { + displayName: '', + name: 'fields', + values: [ + { + displayName: 'Field Name', + name: 'fieldName', + type: 'string', + default: '', + description: 'Specify fields that will be included in output array', + // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id + placeholder: 'e.g. id', + hint: ' Enter the field name as text', + requiresDataPath: 'single', + }, + ], + }, + ], + displayOptions: { + show: { + resource: ['itemList'], + operation: ['aggregateItems'], + aggregate: ['aggregateAllItemData'], + include: ['specifiedFields'], + }, + }, + }, + // Remove duplicates - Fields + { + displayName: 'Compare', + name: 'compare', + type: 'options', + options: [ + { + name: 'All Fields', + value: 'allFields', + }, + { + name: 'All Fields Except', + value: 'allFieldsExcept', + }, + { + name: 'Selected Fields', + value: 'selectedFields', + }, + ], + default: 'allFields', + description: 'The fields of the input items to compare to see if they are the same', + displayOptions: { + show: { + resource: ['itemList'], + operation: ['removeDuplicates'], + }, + }, + }, + { + displayName: 'Fields To Exclude', + name: 'fieldsToExclude', + type: 'fixedCollection', + typeOptions: { + multipleValues: true, + }, + placeholder: 'Add Field To Exclude', + default: {}, + displayOptions: { + show: { + resource: ['itemList'], + operation: ['removeDuplicates'], + compare: ['allFieldsExcept'], + }, + }, + options: [ + { + displayName: '', + name: 'fields', + values: [ + { + displayName: 'Field Name', + name: 'fieldName', + type: 'string', + default: '', + description: 'A field in the input to exclude from the comparison', + // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id + placeholder: 'e.g. id', + hint: ' Enter the field name as text', + requiresDataPath: 'single', + }, + ], + }, + ], + }, + { + displayName: 'Fields To Compare', + name: 'fieldsToCompare', + type: 'fixedCollection', + typeOptions: { + multipleValues: true, + }, + placeholder: 'Add Field To Compare', + default: {}, + displayOptions: { + show: { + resource: ['itemList'], + operation: ['removeDuplicates'], + compare: ['selectedFields'], + }, + }, + options: [ + { + displayName: '', + name: 'fields', + values: [ + { + displayName: 'Field Name', + name: 'fieldName', + type: 'string', + default: '', + description: 'A field in the input to add to the comparison', + // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id + placeholder: 'e.g. id', + hint: ' Enter the field name as text', + requiresDataPath: 'single', + }, + ], + }, + ], + }, + // Sort - Fields + { + displayName: 'Type', + name: 'type', + type: 'options', + options: [ + { + name: 'Simple', + value: 'simple', + }, + { + name: 'Random', + value: 'random', + }, + { + name: 'Code', + value: 'code', + }, + ], + default: 'simple', + description: 'The fields of the input items to compare to see if they are the same', + displayOptions: { + show: { + resource: ['itemList'], + operation: ['sort'], + }, + }, + }, + { + displayName: 'Fields To Sort By', + name: 'sortFieldsUi', + type: 'fixedCollection', + typeOptions: { + multipleValues: true, + }, + placeholder: 'Add Field To Sort By', + options: [ + { + displayName: '', + name: 'sortField', + values: [ + { + displayName: 'Field Name', + name: 'fieldName', + type: 'string', + required: true, + default: '', + description: 'The field to sort by', + // eslint-disable-next-line n8n-nodes-base/node-param-placeholder-miscased-id + placeholder: 'e.g. id', + hint: ' Enter the field name as text', + requiresDataPath: 'single', + }, + { + displayName: 'Order', + name: 'order', + type: 'options', + options: [ + { + name: 'Ascending', + value: 'ascending', + }, + { + name: 'Descending', + value: 'descending', + }, + ], + default: 'ascending', + description: 'The order to sort by', + }, + ], + }, + ], + default: {}, + description: 'The fields of the input items to compare to see if they are the same', + displayOptions: { + show: { + resource: ['itemList'], + operation: ['sort'], + type: ['simple'], + }, + }, + }, + { + displayName: 'Code', + name: 'code', + type: 'string', + typeOptions: { + alwaysOpenEditWindow: true, + editor: 'code', + rows: 10, + }, + default: `// The two items to compare are in the variables a and b +// Access the fields in a.json and b.json +// Return -1 if a should go before b +// Return 1 if b should go before a +// Return 0 if there's no difference + +fieldName = 'myField'; + +if (a.json[fieldName] < b.json[fieldName]) { + return -1; +} +if (a.json[fieldName] > b.json[fieldName]) { + return 1; +} +return 0;`, + description: 'Javascript code to determine the order of any two items', + displayOptions: { + show: { + resource: ['itemList'], + operation: ['sort'], + type: ['code'], + }, + }, + }, + // Limit - Fields + { + displayName: 'Max Items', + name: 'maxItems', + type: 'number', + typeOptions: { + minValue: 1, + }, + default: 1, + description: 'If there are more items than this number, some are removed', + displayOptions: { + show: { + resource: ['itemList'], + operation: ['limit'], + }, + }, + }, + { + displayName: 'Keep', + name: 'keep', + type: 'options', + options: [ + { + name: 'First Items', + value: 'firstItems', + }, + { + name: 'Last Items', + value: 'lastItems', + }, + ], + default: 'firstItems', + description: 'When removing items, whether to keep the ones at the start or the ending', + displayOptions: { + show: { + resource: ['itemList'], + operation: ['limit'], + }, + }, + }, + { + displayName: 'Options', + name: 'options', + type: 'collection', + placeholder: 'Add Field', + default: {}, + displayOptions: { + show: { + resource: ['itemList'], + operation: ['removeDuplicates'], + compare: ['allFieldsExcept', 'selectedFields'], + }, + }, + options: [ + { + displayName: 'Remove Other Fields', + name: 'removeOtherFields', + type: 'boolean', + default: false, + description: + 'Whether to remove any fields that are not being compared. If disabled, will keep the values from the first of the duplicates.', + }, + { + displayName: 'Disable Dot Notation', + name: 'disableDotNotation', + type: 'boolean', + default: false, + description: + 'Whether to disallow referencing child fields using `parent.child` in the field name', + }, + ], + }, + { + displayName: 'Options', + name: 'options', + type: 'collection', + placeholder: 'Add Field', + default: {}, + displayOptions: { + show: { + resource: ['itemList'], + operation: ['sort'], + type: ['simple'], + }, + }, + options: [ + { + displayName: 'Disable Dot Notation', + name: 'disableDotNotation', + type: 'boolean', + default: false, + description: + 'Whether to disallow referencing child fields using `parent.child` in the field name', + }, + ], + }, + { + displayName: 'Options', + name: 'options', + type: 'collection', + placeholder: 'Add Field', + default: {}, + displayOptions: { + show: { + resource: ['itemList'], + operation: ['splitOutItems', 'aggregateItems'], + }, + hide: { + aggregate: ['aggregateAllItemData'], + }, + }, + options: [ + { + displayName: 'Disable Dot Notation', + name: 'disableDotNotation', + type: 'boolean', + displayOptions: { + show: { + '/operation': ['splitOutItems', 'aggregateItems'], + }, + }, + default: false, + description: + 'Whether to disallow referencing child fields using `parent.child` in the field name', + }, + { + displayName: 'Destination Field Name', + name: 'destinationFieldName', + type: 'string', + displayOptions: { + show: { + '/operation': ['splitOutItems'], + }, + }, + default: '', + description: 'The field in the output under which to put the split field contents', + }, + { + displayName: 'Merge Lists', + name: 'mergeLists', + type: 'boolean', + displayOptions: { + show: { + '/operation': ['aggregateItems'], + }, + }, + default: false, + description: + 'Whether to merge the output into a single flat list (rather than a list of lists), if the field to aggregate is a list', + }, + { + displayName: 'Keep Missing And Null Values', + name: 'keepMissing', + type: 'boolean', + displayOptions: { + show: { + '/operation': ['aggregateItems'], + }, + }, + default: false, + description: + 'Whether to add a null entry to the aggregated list when there is a missing or null value', + }, + ], + }, + // Remove duplicates - Fields + ...summarize.description, + ], + }; + } + + async execute(this: IExecuteFunctions): Promise { + const items = this.getInputData(); + const length = items.length; + const returnData: INodeExecutionData[] = []; + const resource = this.getNodeParameter('resource', 0); + const operation = this.getNodeParameter('operation', 0); + if (resource === 'itemList') { + if (operation === 'splitOutItems') { + for (let i = 0; i < length; i++) { + const fieldToSplitOut = this.getNodeParameter('fieldToSplitOut', i) as string; + const disableDotNotation = this.getNodeParameter( + 'options.disableDotNotation', + 0, + false, + ) as boolean; + const destinationFieldName = this.getNodeParameter( + 'options.destinationFieldName', + i, + '', + ) as string; + const include = this.getNodeParameter('include', i) as string; + + let arrayToSplit; + if (!disableDotNotation) { + arrayToSplit = get(items[i].json, fieldToSplitOut); + } else { + arrayToSplit = items[i].json[fieldToSplitOut]; + } + + if (arrayToSplit === undefined) { + if (fieldToSplitOut.includes('.') && disableDotNotation) { + throw new NodeOperationError( + this.getNode(), + `Couldn't find the field '${fieldToSplitOut}' in the input data`, + { + description: + "If you're trying to use a nested field, make sure you turn off 'disable dot notation' in the node options", + }, + ); + } else { + throw new NodeOperationError( + this.getNode(), + `Couldn't find the field '${fieldToSplitOut}' in the input data`, + { itemIndex: i }, + ); + } + } + + if (!Array.isArray(arrayToSplit)) { + throw new NodeOperationError( + this.getNode(), + `The provided field '${fieldToSplitOut}' is not an array`, + { itemIndex: i }, + ); + } else { + for (const element of arrayToSplit) { + let newItem = {}; + + if (include === 'selectedOtherFields') { + const fieldsToInclude = ( + this.getNodeParameter('fieldsToInclude.fields', i, []) as [{ fieldName: string }] + ).map((field) => field.fieldName); + + if (!fieldsToInclude.length) { + throw new NodeOperationError(this.getNode(), 'No fields specified', { + description: 'Please add a field to include', + }); + } + + newItem = { + ...fieldsToInclude.reduce((prev, field) => { + if (field === fieldToSplitOut) { + return prev; + } + let value; + if (!disableDotNotation) { + value = get(items[i].json, field); + } else { + value = items[i].json[field]; + } + prev = { ...prev, [field]: value }; + return prev; + }, {}), + }; + } else if (include === 'allOtherFields') { + const keys = Object.keys(items[i].json); + + newItem = { + ...keys.reduce((prev, field) => { + let value; + if (!disableDotNotation) { + value = get(items[i].json, field); + } else { + value = items[i].json[field]; + } + prev = { ...prev, [field]: value }; + return prev; + }, {}), + }; + + unset(newItem, fieldToSplitOut); + } + + if ( + typeof element === 'object' && + include === 'noOtherFields' && + destinationFieldName === '' + ) { + newItem = { ...newItem, ...element }; + } else { + newItem = { + ...newItem, + [destinationFieldName || fieldToSplitOut]: element, + }; + } + + returnData.push({ + json: newItem, + pairedItem: { + item: i, + }, + }); + } + } + } + + return this.prepareOutputData(returnData); + } else if (operation === 'aggregateItems') { + const aggregate = this.getNodeParameter('aggregate', 0, '') as string; + + if (aggregate === 'aggregateIndividualFields') { + const disableDotNotation = this.getNodeParameter( + 'options.disableDotNotation', + 0, + false, + ) as boolean; + const mergeLists = this.getNodeParameter('options.mergeLists', 0, false) as boolean; + const fieldsToAggregate = this.getNodeParameter( + 'fieldsToAggregate.fieldToAggregate', + 0, + [], + ) as [{ fieldToAggregate: string; renameField: boolean; outputFieldName: string }]; + const keepMissing = this.getNodeParameter('options.keepMissing', 0, false) as boolean; + + if (!fieldsToAggregate.length) { + throw new NodeOperationError(this.getNode(), 'No fields specified', { + description: 'Please add a field to aggregate', + }); + } + for (const { fieldToAggregate } of fieldsToAggregate) { + let found = false; + for (const item of items) { + if (fieldToAggregate === '') { + throw new NodeOperationError(this.getNode(), 'Field to aggregate is blank', { + description: 'Please add a field to aggregate', + }); + } + if (!disableDotNotation) { + if (get(item.json, fieldToAggregate) !== undefined) { + found = true; + } + } else if (item.json.hasOwnProperty(fieldToAggregate)) { + found = true; + } + } + if (!found && disableDotNotation && fieldToAggregate.includes('.')) { + throw new NodeOperationError( + this.getNode(), + `Couldn't find the field '${fieldToAggregate}' in the input data`, + { + description: + "If you're trying to use a nested field, make sure you turn off 'disable dot notation' in the node options", + }, + ); + } else if (!found && !keepMissing) { + throw new NodeOperationError( + this.getNode(), + `Couldn't find the field '${fieldToAggregate}' in the input data`, + ); + } + } + + const newItem: INodeExecutionData = { + json: {}, + pairedItem: Array.from({ length }, (_, i) => i).map((index) => { + return { + item: index, + }; + }), + }; + + const values: { [key: string]: any } = {}; + const outputFields: string[] = []; + + for (const { fieldToAggregate, outputFieldName, renameField } of fieldsToAggregate) { + const field = renameField ? outputFieldName : fieldToAggregate; + + if (outputFields.includes(field)) { + throw new NodeOperationError( + this.getNode(), + `The '${field}' output field is used more than once`, + { description: 'Please make sure each output field name is unique' }, + ); + } else { + outputFields.push(field); + } + + const getFieldToAggregate = () => + !disableDotNotation && fieldToAggregate.includes('.') + ? fieldToAggregate.split('.').pop() + : fieldToAggregate; + + const _outputFieldName = outputFieldName + ? outputFieldName + : (getFieldToAggregate() as string); + + if (fieldToAggregate !== '') { + values[_outputFieldName] = []; + for (let i = 0; i < length; i++) { + if (!disableDotNotation) { + let value = get(items[i].json, fieldToAggregate); + + if (!keepMissing) { + if (Array.isArray(value)) { + value = value.filter((entry) => entry !== null); + } else if (value === null || value === undefined) { + continue; + } + } + + if (Array.isArray(value) && mergeLists) { + values[_outputFieldName].push(...value); + } else { + values[_outputFieldName].push(value); + } + } else { + let value = items[i].json[fieldToAggregate]; + + if (!keepMissing) { + if (Array.isArray(value)) { + value = value.filter((entry) => entry !== null); + } else if (value === null || value === undefined) { + continue; + } + } + + if (Array.isArray(value) && mergeLists) { + values[_outputFieldName].push(...value); + } else { + values[_outputFieldName].push(value); + } + } + } + } + } + + for (const key of Object.keys(values)) { + if (!disableDotNotation) { + set(newItem.json, key, values[key]); + } else { + newItem.json[key] = values[key]; + } + } + + returnData.push(newItem); + + return this.prepareOutputData(returnData); + } else { + let newItems: IDataObject[] = items.map((item) => item.json); + const destinationFieldName = this.getNodeParameter('destinationFieldName', 0) as string; + const fieldsToExclude = ( + this.getNodeParameter('fieldsToExclude.fields', 0, []) as IDataObject[] + ).map((entry) => entry.fieldName); + const fieldsToInclude = ( + this.getNodeParameter('fieldsToInclude.fields', 0, []) as IDataObject[] + ).map((entry) => entry.fieldName); + + if (fieldsToExclude.length || fieldsToInclude.length) { + newItems = newItems.reduce((acc, item) => { + const newItem: IDataObject = {}; + let outputFields = Object.keys(item); + + if (fieldsToExclude.length) { + outputFields = outputFields.filter((key) => !fieldsToExclude.includes(key)); + } + if (fieldsToInclude.length) { + outputFields = outputFields.filter((key) => + fieldsToInclude.length ? fieldsToInclude.includes(key) : true, + ); + } + + outputFields.forEach((key) => { + newItem[key] = item[key]; + }); + + if (isEmpty(newItem)) { + return acc; + } + return acc.concat([newItem]); + }, [] as IDataObject[]); + } + + const output: INodeExecutionData = { json: { [destinationFieldName]: newItems } }; + + return this.prepareOutputData([output]); + } + } else if (operation === 'removeDuplicates') { + const compare = this.getNodeParameter('compare', 0) as string; + const disableDotNotation = this.getNodeParameter( + 'options.disableDotNotation', + 0, + false, + ) as boolean; + const removeOtherFields = this.getNodeParameter( + 'options.removeOtherFields', + 0, + false, + ) as boolean; + + let keys = disableDotNotation + ? Object.keys(items[0].json) + : Object.keys(flattenKeys(items[0].json)); + + for (const item of items) { + for (const key of disableDotNotation + ? Object.keys(item.json) + : Object.keys(flattenKeys(item.json))) { + if (!keys.includes(key)) { + keys.push(key); + } + } + } + + if (compare === 'allFieldsExcept') { + const fieldsToExclude = ( + this.getNodeParameter('fieldsToExclude.fields', 0, []) as [{ fieldName: string }] + ).map((field) => field.fieldName); + if (!fieldsToExclude.length) { + throw new NodeOperationError( + this.getNode(), + 'No fields specified. Please add a field to exclude from comparison', + ); + } + if (!disableDotNotation) { + keys = Object.keys(flattenKeys(items[0].json)); + } + keys = keys.filter((key) => !fieldsToExclude.includes(key)); + } + if (compare === 'selectedFields') { + const fieldsToCompare = ( + this.getNodeParameter('fieldsToCompare.fields', 0, []) as [{ fieldName: string }] + ).map((field) => field.fieldName); + if (!fieldsToCompare.length) { + throw new NodeOperationError( + this.getNode(), + 'No fields specified. Please add a field to compare on', + ); + } + if (!disableDotNotation) { + keys = Object.keys(flattenKeys(items[0].json)); + } + keys = fieldsToCompare.map((key) => key.trim()); + } + + // This solution is O(nlogn) + // add original index to the items + const newItems = items.map( + (item, index) => + ({ + json: { ...item.json, __INDEX: index }, + pairedItem: { item: index }, + } as INodeExecutionData), + ); + //sort items using the compare keys + newItems.sort((a, b) => { + let result = 0; + + for (const key of keys) { + let equal; + if (!disableDotNotation) { + equal = isEqual(get(a.json, key), get(b.json, key)); + } else { + equal = isEqual(a.json[key], b.json[key]); + } + if (!equal) { + let lessThan; + if (!disableDotNotation) { + lessThan = lt(get(a.json, key), get(b.json, key)); + } else { + lessThan = lt(a.json[key], b.json[key]); + } + result = lessThan ? -1 : 1; + break; + } + } + return result; + }); + + for (const key of keys) { + let type: any = undefined; + for (const item of newItems) { + if (key === '') { + throw new NodeOperationError(this.getNode(), 'Name of field to compare is blank'); + } + const value = !disableDotNotation ? get(item.json, key) : item.json[key]; + if (value === undefined && disableDotNotation && key.includes('.')) { + throw new NodeOperationError( + this.getNode(), + `'${key}' field is missing from some input items`, + { + description: + "If you're trying to use a nested field, make sure you turn off 'disable dot notation' in the node options", + }, + ); + } else if (value === undefined) { + throw new NodeOperationError( + this.getNode(), + `'${key}' field is missing from some input items`, + ); + } + if (type !== undefined && value !== undefined && type !== typeof value) { + throw new NodeOperationError(this.getNode(), `'${key}' isn't always the same type`, { + description: 'The type of this field varies between items', + }); + } else { + type = typeof value; + } + } + } + + // collect the original indexes of items to be removed + const removedIndexes: number[] = []; + let temp = newItems[0]; + for (let index = 1; index < newItems.length; index++) { + if (compareItems(newItems[index], temp, keys, disableDotNotation, this.getNode())) { + removedIndexes.push(newItems[index].json.__INDEX as unknown as number); + } else { + temp = newItems[index]; + } + } + + let data = items.filter((_, index) => !removedIndexes.includes(index)); + + if (removeOtherFields) { + data = data.map((item, index) => ({ + json: pick(item.json, ...keys), + pairedItem: { item: index }, + })); + } + + // return the filtered items + return this.prepareOutputData(data); + } else if (operation === 'sort') { + let newItems = [...items]; + const type = this.getNodeParameter('type', 0) as string; + const disableDotNotation = this.getNodeParameter( + 'options.disableDotNotation', + 0, + false, + ) as boolean; + + if (type === 'random') { + shuffleArray(newItems); + return this.prepareOutputData(newItems); + } + + if (type === 'simple') { + const sortFieldsUi = this.getNodeParameter('sortFieldsUi', 0) as IDataObject; + const sortFields = sortFieldsUi.sortField as Array<{ + fieldName: string; + order: 'ascending' | 'descending'; + }>; + + if (!sortFields?.length) { + throw new NodeOperationError( + this.getNode(), + 'No sorting specified. Please add a field to sort by', + ); + } + + for (const { fieldName } of sortFields) { + let found = false; + for (const item of items) { + if (!disableDotNotation) { + if (get(item.json, fieldName) !== undefined) { + found = true; + } + } else if (item.json.hasOwnProperty(fieldName)) { + found = true; + } + } + if (!found && disableDotNotation && fieldName.includes('.')) { + throw new NodeOperationError( + this.getNode(), + `Couldn't find the field '${fieldName}' in the input data`, + { + description: + "If you're trying to use a nested field, make sure you turn off 'disable dot notation' in the node options", + }, + ); + } else if (!found) { + throw new NodeOperationError( + this.getNode(), + `Couldn't find the field '${fieldName}' in the input data`, + ); + } + } + + const sortFieldsWithDirection = sortFields.map((field) => ({ + name: field.fieldName, + dir: field.order === 'ascending' ? 1 : -1, + })); + + newItems.sort((a, b) => { + let result = 0; + for (const field of sortFieldsWithDirection) { + let equal; + if (!disableDotNotation) { + const _a = + typeof get(a.json, field.name) === 'string' + ? (get(a.json, field.name) as string).toLowerCase() + : get(a.json, field.name); + const _b = + typeof get(b.json, field.name) === 'string' + ? (get(b.json, field.name) as string).toLowerCase() + : get(b.json, field.name); + equal = isEqual(_a, _b); + } else { + const _a = + typeof a.json[field.name] === 'string' + ? (a.json[field.name] as string).toLowerCase() + : a.json[field.name]; + const _b = + typeof b.json[field.name] === 'string' + ? (b.json[field.name] as string).toLowerCase() + : b.json[field.name]; + equal = isEqual(_a, _b); + } + + if (!equal) { + let lessThan; + if (!disableDotNotation) { + const _a = + typeof get(a.json, field.name) === 'string' + ? (get(a.json, field.name) as string).toLowerCase() + : get(a.json, field.name); + const _b = + typeof get(b.json, field.name) === 'string' + ? (get(b.json, field.name) as string).toLowerCase() + : get(b.json, field.name); + lessThan = lt(_a, _b); + } else { + const _a = + typeof a.json[field.name] === 'string' + ? (a.json[field.name] as string).toLowerCase() + : a.json[field.name]; + const _b = + typeof b.json[field.name] === 'string' + ? (b.json[field.name] as string).toLowerCase() + : b.json[field.name]; + lessThan = lt(_a, _b); + } + if (lessThan) { + result = -1 * field.dir; + } else { + result = 1 * field.dir; + } + break; + } + } + return result; + }); + } else { + const code = this.getNodeParameter('code', 0) as string; + const regexCheck = /\breturn\b/g.exec(code); + + if (regexCheck?.length) { + const sandbox = { + newItems, + }; + const mode = this.getMode(); + const options = { + console: mode === 'manual' ? 'redirect' : 'inherit', + sandbox, + }; + const vm = new NodeVM(options as unknown as NodeVMOptions); + + newItems = await vm.run( + ` + module.exports = async function() { + newItems.sort( (a,b) => { + ${code} + }) + return newItems; + }()`, + __dirname, + ); + } else { + throw new NodeOperationError( + this.getNode(), + "Sort code doesn't return. Please add a 'return' statement to your code", + ); + } + } + return this.prepareOutputData(newItems); + } else if (operation === 'limit') { + let newItems = items; + const maxItems = this.getNodeParameter('maxItems', 0) as number; + const keep = this.getNodeParameter('keep', 0) as string; + + if (maxItems > items.length) { + return this.prepareOutputData(newItems); + } + + if (keep === 'firstItems') { + newItems = items.slice(0, maxItems); + } else { + newItems = items.slice(items.length - maxItems, items.length); + } + return this.prepareOutputData(newItems); + } else if (operation === 'summarize') { + return summarize.execute.call(this, items); + } else { + throw new NodeOperationError(this.getNode(), `Operation '${operation}' is not recognized`); + } + } else { + throw new NodeOperationError(this.getNode(), `Resource '${resource}' is not recognized`); + } + } +} diff --git a/packages/nodes-base/nodes/ItemLists/V2/summarize.operation.ts b/packages/nodes-base/nodes/ItemLists/V2/summarize.operation.ts new file mode 100644 index 0000000000..c3accbd330 --- /dev/null +++ b/packages/nodes-base/nodes/ItemLists/V2/summarize.operation.ts @@ -0,0 +1,614 @@ +import type { + GenericValue, + IDataObject, + IExecuteFunctions, + INodeExecutionData, + INodeProperties, +} from 'n8n-workflow'; +import { NodeOperationError } from 'n8n-workflow'; + +import get from 'lodash.get'; + +type AggregationType = + | 'append' + | 'average' + | 'concatenate' + | 'count' + | 'countUnique' + | 'max' + | 'min' + | 'sum'; + +type Aggregation = { + aggregation: AggregationType; + field: string; + includeEmpty?: boolean; + separateBy?: string; + customSeparator?: string; +}; + +type Aggregations = Aggregation[]; + +enum AggregationDisplayNames { + append = 'appended_', + average = 'average_', + concatenate = 'concatenated_', + count = 'count_', + countUnique = 'unique_count_', + max = 'max_', + min = 'min_', + sum = 'sum_', +} + +const NUMERICAL_AGGREGATIONS = ['average', 'max', 'min', 'sum']; + +type SummarizeOptions = { + disableDotNotation?: boolean; + outputFormat?: 'separateItems' | 'singleItem'; + skipEmptySplitFields?: boolean; +}; + +type ValueGetterFn = ( + item: IDataObject, + field: string, +) => IDataObject | IDataObject[] | GenericValue | GenericValue[]; + +export const description: INodeProperties[] = [ + { + displayName: 'Fields to Summarize', + name: 'fieldsToSummarize', + type: 'fixedCollection', + placeholder: 'Add Field', + default: { values: [{ aggregation: 'count', field: '' }] }, + typeOptions: { + multipleValues: true, + }, + options: [ + { + displayName: '', + name: 'values', + values: [ + { + displayName: 'Aggregation', + name: 'aggregation', + type: 'options', + options: [ + { + name: 'Append', + value: 'append', + }, + { + name: 'Average', + value: 'average', + }, + { + name: 'Concatenate', + value: 'concatenate', + }, + { + name: 'Count', + value: 'count', + }, + { + name: 'Count Unique', + value: 'countUnique', + }, + { + name: 'Max', + value: 'max', + }, + { + name: 'Min', + value: 'min', + }, + { + name: 'Sum', + value: 'sum', + }, + ], + default: 'count', + description: 'How to combine the values of the field you want to summarize', + }, + //field repeated to have different descriptions for different aggregations -------------------------------- + { + displayName: 'Field', + name: 'field', + type: 'string', + default: '', + description: 'The name of an input field that you want to summarize', + placeholder: 'e.g. cost', + hint: ' Enter the field name as text', + displayOptions: { + hide: { + aggregation: [...NUMERICAL_AGGREGATIONS, 'countUnique', 'count'], + }, + }, + requiresDataPath: 'single', + }, + { + displayName: 'Field', + name: 'field', + type: 'string', + default: '', + description: + 'The name of an input field that you want to summarize. The field should contain numerical values; null, undefined, empty strings would be ignored.', + placeholder: 'e.g. cost', + hint: ' Enter the field name as text', + displayOptions: { + show: { + aggregation: NUMERICAL_AGGREGATIONS, + }, + }, + requiresDataPath: 'single', + }, + { + displayName: 'Field', + name: 'field', + type: 'string', + default: '', + description: + 'The name of an input field that you want to summarize; null, undefined, empty strings would be ignored', + placeholder: 'e.g. cost', + hint: ' Enter the field name as text', + displayOptions: { + show: { + aggregation: ['countUnique', 'count'], + }, + }, + requiresDataPath: 'single', + }, + // ---------------------------------------------------------------------------------------------------------- + { + displayName: 'Include Empty Values', + name: 'includeEmpty', + type: 'boolean', + default: false, + displayOptions: { + show: { + aggregation: ['append', 'concatenate'], + }, + }, + }, + { + displayName: 'Separator', + name: 'separateBy', + type: 'options', + default: ',', + // eslint-disable-next-line n8n-nodes-base/node-param-options-type-unsorted-items + options: [ + { + name: 'Comma', + value: ',', + }, + { + name: 'Comma and Space', + value: ', ', + }, + { + name: 'New Line', + value: '\n', + }, + { + name: 'None', + value: '', + }, + { + name: 'Space', + value: ' ', + }, + { + name: 'Other', + value: 'other', + }, + ], + hint: 'What to insert between values', + displayOptions: { + show: { + aggregation: ['concatenate'], + }, + }, + }, + { + displayName: 'Custom Separator', + name: 'customSeparator', + type: 'string', + default: '', + displayOptions: { + show: { + aggregation: ['concatenate'], + separateBy: ['other'], + }, + }, + }, + ], + }, + ], + displayOptions: { + show: { + resource: ['itemList'], + operation: ['summarize'], + }, + }, + }, + // fieldsToSplitBy repeated to have different displayName for singleItem and separateItems ----------------------------- + { + displayName: 'Fields to Split By', + name: 'fieldsToSplitBy', + type: 'string', + placeholder: 'e.g. country, city', + default: '', + description: 'The name of the input fields that you want to split the summary by', + hint: 'Enter the name of the fields as text (separated by commas)', + displayOptions: { + show: { + resource: ['itemList'], + operation: ['summarize'], + }, + hide: { + '/options.outputFormat': ['singleItem'], + }, + }, + requiresDataPath: 'multiple', + }, + { + displayName: 'Fields to Group By', + name: 'fieldsToSplitBy', + type: 'string', + placeholder: 'e.g. country, city', + default: '', + description: 'The name of the input fields that you want to split the summary by', + hint: 'Enter the name of the fields as text (separated by commas)', + displayOptions: { + show: { + resource: ['itemList'], + operation: ['summarize'], + '/options.outputFormat': ['singleItem'], + }, + }, + requiresDataPath: 'multiple', + }, + // ---------------------------------------------------------------------------------------------------------- + { + displayName: 'Options', + name: 'options', + type: 'collection', + placeholder: 'Add Option', + default: {}, + displayOptions: { + show: { + resource: ['itemList'], + operation: ['summarize'], + }, + }, + options: [ + { + displayName: 'Disable Dot Notation', + name: 'disableDotNotation', + type: 'boolean', + default: false, + description: + 'Whether to disallow referencing child fields using `parent.child` in the field name', + }, + { + displayName: 'Output Format', + name: 'outputFormat', + type: 'options', + default: 'separateItems', + options: [ + { + name: 'Each Split in a Separate Item', + value: 'separateItems', + }, + { + name: 'All Splits in a Single Item', + value: 'singleItem', + }, + ], + }, + { + // eslint-disable-next-line n8n-nodes-base/node-param-display-name-miscased + displayName: 'Ignore items without valid fields to group by', + name: 'skipEmptySplitFields', + type: 'boolean', + default: false, + }, + ], + }, +]; + +function isEmpty(value: T) { + return value === undefined || value === null || value === ''; +} + +function parseReturnData(returnData: IDataObject) { + const regexBrackets = /[\]\["]/g; + const regexSpaces = /[ .]/g; + for (const key of Object.keys(returnData)) { + if (key.match(regexBrackets)) { + const newKey = key.replace(regexBrackets, ''); + returnData[newKey] = returnData[key]; + delete returnData[key]; + } + } + for (const key of Object.keys(returnData)) { + if (key.match(regexSpaces)) { + const newKey = key.replace(regexSpaces, '_'); + returnData[newKey] = returnData[key]; + delete returnData[key]; + } + } +} + +function parseFieldName(fieldName: string[]) { + const regexBrackets = /[\]\["]/g; + const regexSpaces = /[ .]/g; + fieldName = fieldName.map((field) => { + field = field.replace(regexBrackets, ''); + field = field.replace(regexSpaces, '_'); + return field; + }); + return fieldName; +} + +const fieldValueGetter = (disableDotNotation?: boolean) => { + if (disableDotNotation) { + return (item: IDataObject, field: string) => item[field]; + } else { + return (item: IDataObject, field: string) => get(item, field); + } +}; + +function checkIfFieldExists( + this: IExecuteFunctions, + items: IDataObject[], + aggregations: Aggregations, + getValue: ValueGetterFn, +) { + for (const aggregation of aggregations) { + if (aggregation.field === '') { + continue; + } + const exist = items.some((item) => getValue(item, aggregation.field) !== undefined); + if (!exist) { + throw new NodeOperationError( + this.getNode(), + `The field '${aggregation.field}' does not exist in any items`, + ); + } + } +} + +function aggregate(items: IDataObject[], entry: Aggregation, getValue: ValueGetterFn) { + const { aggregation, field } = entry; + let data = [...items]; + + if (NUMERICAL_AGGREGATIONS.includes(aggregation)) { + data = data.filter( + (item) => typeof getValue(item, field) === 'number' && !isEmpty(getValue(item, field)), + ); + } + + switch (aggregation) { + //combine operations + case 'append': + if (!entry.includeEmpty) { + data = data.filter((item) => !isEmpty(getValue(item, field))); + } + return data.map((item) => getValue(item, field)); + case 'concatenate': + const separateBy = entry.separateBy === 'other' ? entry.customSeparator : entry.separateBy; + if (!entry.includeEmpty) { + data = data.filter((item) => !isEmpty(getValue(item, field))); + } + return data + .map((item) => { + let value = getValue(item, field); + if (typeof value === 'object') { + value = JSON.stringify(value); + } + if (typeof value === 'undefined') { + value = 'undefined'; + } + + return value; + }) + .join(separateBy); + + //numerical operations + case 'average': + return ( + data.reduce((acc, item) => { + return acc + (getValue(item, field) as number); + }, 0) / data.length + ); + case 'sum': + return data.reduce((acc, item) => { + return acc + (getValue(item, field) as number); + }, 0); + case 'min': + return Math.min( + ...(data.map((item) => { + return getValue(item, field); + }) as number[]), + ); + case 'max': + return Math.max( + ...(data.map((item) => { + return getValue(item, field); + }) as number[]), + ); + + //count operations + case 'countUnique': + return new Set(data.map((item) => getValue(item, field)).filter((item) => !isEmpty(item))) + .size; + default: + //count by default + return data.filter((item) => !isEmpty(getValue(item, field))).length; + } +} + +function aggregateData( + data: IDataObject[], + fieldsToSummarize: Aggregations, + options: SummarizeOptions, + getValue: ValueGetterFn, +) { + const returnData = fieldsToSummarize.reduce((acc, aggregation) => { + acc[`${AggregationDisplayNames[aggregation.aggregation]}${aggregation.field}`] = aggregate( + data, + aggregation, + getValue, + ); + return acc; + }, {} as IDataObject); + parseReturnData(returnData); + if (options.outputFormat === 'singleItem') { + parseReturnData(returnData); + return returnData; + } else { + return { ...returnData, pairedItems: data.map((item) => item._itemIndex as number) }; + } +} + +function splitData( + splitKeys: string[], + data: IDataObject[], + fieldsToSummarize: Aggregations, + options: SummarizeOptions, + getValue: ValueGetterFn, +) { + if (!splitKeys || splitKeys.length === 0) { + return aggregateData(data, fieldsToSummarize, options, getValue); + } + + const [firstSplitKey, ...restSplitKeys] = splitKeys; + + const groupedData = data.reduce((acc, item) => { + let keyValuee = getValue(item, firstSplitKey) as string; + + if (typeof keyValuee === 'object') { + keyValuee = JSON.stringify(keyValuee); + } + + if (options.skipEmptySplitFields && typeof keyValuee !== 'number' && !keyValuee) { + return acc; + } + + if (acc[keyValuee] === undefined) { + acc[keyValuee] = [item]; + } else { + (acc[keyValuee] as IDataObject[]).push(item); + } + return acc; + }, {} as IDataObject); + + return Object.keys(groupedData).reduce((acc, key) => { + const value = groupedData[key] as IDataObject[]; + acc[key] = splitData(restSplitKeys, value, fieldsToSummarize, options, getValue); + return acc; + }, {} as IDataObject); +} + +function aggregationToArray( + aggregationResult: IDataObject, + fieldsToSplitBy: string[], + previousStage: IDataObject = {}, +) { + const returnData: IDataObject[] = []; + fieldsToSplitBy = parseFieldName(fieldsToSplitBy); + const splitFieldName = fieldsToSplitBy[0]; + const isNext = fieldsToSplitBy[1]; + + if (isNext === undefined) { + for (const fieldName of Object.keys(aggregationResult)) { + returnData.push({ + ...previousStage, + [splitFieldName]: fieldName, + ...(aggregationResult[fieldName] as IDataObject), + }); + } + return returnData; + } else { + for (const key of Object.keys(aggregationResult)) { + returnData.push( + ...aggregationToArray(aggregationResult[key] as IDataObject, fieldsToSplitBy.slice(1), { + ...previousStage, + [splitFieldName]: key, + }), + ); + } + return returnData; + } +} + +export async function execute( + this: IExecuteFunctions, + items: INodeExecutionData[], +): Promise { + const newItems = items.map(({ json }, i) => ({ ...json, _itemIndex: i })); + + const options = this.getNodeParameter('options', 0, {}) as SummarizeOptions; + + const fieldsToSplitBy = (this.getNodeParameter('fieldsToSplitBy', 0, '') as string) + .split(',') + .map((field) => field.trim()) + .filter((field) => field); + + const fieldsToSummarize = this.getNodeParameter( + 'fieldsToSummarize.values', + 0, + [], + ) as Aggregations; + + if (fieldsToSummarize.filter((aggregation) => aggregation.field !== '').length === 0) { + throw new NodeOperationError( + this.getNode(), + "You need to add at least one aggregation to 'Fields to Summarize' with non empty 'Field'", + ); + } + + const getValue = fieldValueGetter(options.disableDotNotation); + + checkIfFieldExists.call(this, newItems, fieldsToSummarize, getValue); + + const aggregationResult = splitData( + fieldsToSplitBy, + newItems, + fieldsToSummarize, + options, + getValue, + ); + + if (options.outputFormat === 'singleItem') { + const executionData: INodeExecutionData = { + json: aggregationResult, + pairedItem: newItems.map((_v, index) => ({ + item: index, + })), + }; + return this.prepareOutputData([executionData]); + } else { + if (!fieldsToSplitBy.length) { + const { pairedItems, ...json } = aggregationResult; + const executionData: INodeExecutionData = { + json, + pairedItem: ((pairedItems as number[]) || []).map((index: number) => ({ + item: index, + })), + }; + return this.prepareOutputData([executionData]); + } + const returnData = aggregationToArray(aggregationResult, fieldsToSplitBy); + const executionData = returnData.map((item) => { + const { pairedItems, ...json } = item; + return { + json, + pairedItem: ((pairedItems as number[]) || []).map((index: number) => ({ + item: index, + })), + }; + }); + return this.prepareOutputData(executionData); + } +}