From 553b14a13c7c9056447ef0b18c9427f26221b44d Mon Sep 17 00:00:00 2001 From: Michael Kret <88898367+michael-radency@users.noreply.github.com> Date: Wed, 27 Jul 2022 17:19:50 +0300 Subject: [PATCH] feat(Item List Node): Add operation for creating array from input items (#3149) * :hammer: create array operation * :hammer: removed semicolumn * :hammer: updated UI * :zap: display option fix * :zap: aggregate operation description update, default aggregate item --- .../nodes/ItemLists/ItemLists.node.ts | 383 ++++++++++++++---- 1 file changed, 296 insertions(+), 87 deletions(-) diff --git a/packages/nodes-base/nodes/ItemLists/ItemLists.node.ts b/packages/nodes-base/nodes/ItemLists/ItemLists.node.ts index f73e38f128..c097fb4d4c 100644 --- a/packages/nodes-base/nodes/ItemLists/ItemLists.node.ts +++ b/packages/nodes-base/nodes/ItemLists/ItemLists.node.ts @@ -13,6 +13,7 @@ import { import { get, + isEmpty, isEqual, isObject, lt, @@ -64,8 +65,8 @@ export class ItemLists implements INodeType { { name: 'Aggregate Items', value: 'aggregateItems', - description: 'Merge fields into a single new item', - action: 'Merge fields into a single new item', + description: 'Combine fields into a single new item', + action: 'Combine fields into a single new item', }, { name: 'Limit', @@ -183,6 +184,34 @@ export class ItemLists implements INodeType { }, ], }, + // Aggregate Items + { + displayName: 'Aggregate', + name: 'aggregate', + type: 'options', + default: 'aggregateIndividualFields', + options: [ + { + name: 'Individual Fields', + value: 'aggregateIndividualFields', + }, + { + name: 'All Item Data (Into a Single List)', + value: 'aggregateAllItemData', + }, + ], + displayOptions: { + show: { + resource: [ + 'itemList', + ], + operation: [ + 'aggregateItems', + ], + }, + }, + }, + // Aggregate Individual Fields { displayName: 'Fields To Aggregate', name: 'fieldsToAggregate', @@ -191,7 +220,7 @@ export class ItemLists implements INodeType { multipleValues: true, }, placeholder: 'Add Field To Aggregate', - default: {}, + default: {fieldToAggregate: [{fieldToAggregate: '', renameField: false}]}, displayOptions: { show: { resource: [ @@ -200,6 +229,9 @@ export class ItemLists implements INodeType { operation: [ 'aggregateItems', ], + aggregate: [ + 'aggregateIndividualFields', + ], }, }, options: [ @@ -239,7 +271,142 @@ export class ItemLists implements INodeType { }, ], }, - + // Aggregate All Item Data + { + displayName: 'Put Output in Field', + name: 'destinationFieldName', + type: 'string', + displayOptions: { + show: { + resource: [ + 'itemList', + ], + 'operation': [ + 'aggregateItems', + ], + aggregate: [ + 'aggregateAllItemData', + ], + }, + }, + default: 'data', + description: 'The name of the output field to put the data in', + }, + { + displayName: 'Include', + name: 'include', + type: 'options', + default: 'allFields', + options: [ + { + name: 'All Fields', + value: 'allFields', + }, + { + name: 'Specified Fields', + value: 'specifiedFields', + }, + { + name: 'All Fields Except', + value: 'allFieldsExcept', + }, + ], + displayOptions: { + show: { + resource: [ + 'itemList', + ], + operation: [ + 'aggregateItems', + ], + aggregate: [ + 'aggregateAllItemData', + ], + }, + }, + }, + { + displayName: 'Fields To Exclude', + name: 'fieldsToExclude', + type: 'fixedCollection', + typeOptions: { + multipleValues: true, + }, + placeholder: 'Add Field To Exclude', + default: {}, + options: [ + { + displayName: '', + name: 'fields', + values: [ + { + displayName: 'Field Name', + name: 'fieldName', + type: 'string', + default: '', + description: 'A field in the input to exclude from the object in output array', + }, + ], + }, + ], + displayOptions: { + show: { + resource: [ + 'itemList', + ], + operation: [ + 'aggregateItems', + ], + aggregate: [ + 'aggregateAllItemData', + ], + include: [ + 'allFieldsExcept', + ], + }, + }, + }, + { + displayName: 'Fields To Include', + name: 'fieldsToInclude', + type: 'fixedCollection', + typeOptions: { + multipleValues: true, + }, + placeholder: 'Add Field To Include', + default: {}, + options: [ + { + displayName: '', + name: 'fields', + values: [ + { + displayName: 'Field Name', + name: 'fieldName', + type: 'string', + default: '', + description: 'Specify fields that will be included in output array', + }, + ], + }, + ], + displayOptions: { + show: { + resource: [ + 'itemList', + ], + operation: [ + 'aggregateItems', + ], + aggregate: [ + 'aggregateAllItemData', + ], + include: [ + 'specifiedFields', + ], + }, + }, + }, // Remove duplicates - Fields { displayName: 'Compare', @@ -606,6 +773,11 @@ return 0;`, 'aggregateItems', ], }, + hide: { + 'aggregate': [ + 'aggregateAllItemData', + ], + }, }, options: [ { @@ -770,118 +942,155 @@ return 0;`, return this.prepareOutputData(returnData); } else if (operation === 'aggregateItems') { + const aggregate = this.getNodeParameter('aggregate', 0, '') as string; - const disableDotNotation = this.getNodeParameter('options.disableDotNotation', 0, false) as boolean; - const mergeLists = this.getNodeParameter('options.mergeLists', 0, false) as boolean; - const fieldsToAggregate = this.getNodeParameter('fieldsToAggregate.fieldToAggregate', 0, []) as [{ fieldToAggregate: string, renameField: boolean, outputFieldName: string }]; - const keepMissing = this.getNodeParameter('options.keepMissing', 0, false) as boolean; + if ( aggregate === 'aggregateIndividualFields') { + const disableDotNotation = this.getNodeParameter('options.disableDotNotation', 0, false) as boolean; + const mergeLists = this.getNodeParameter('options.mergeLists', 0, false) as boolean; + const fieldsToAggregate = this.getNodeParameter('fieldsToAggregate.fieldToAggregate', 0, []) as [{ fieldToAggregate: string, renameField: boolean, outputFieldName: string }]; + const keepMissing = this.getNodeParameter('options.keepMissing', 0, false) as boolean; - if (!fieldsToAggregate.length) { - throw new NodeOperationError(this.getNode(), 'No fields specified', { description: 'Please add a field to aggregate' }); - } - for (const { fieldToAggregate } of fieldsToAggregate) { - let found = false; - for (const item of items) { - if (fieldToAggregate === '') { - throw new NodeOperationError(this.getNode(), 'Field to aggregate is blank', { description: 'Please add a field to aggregate' }); - } - if (disableDotNotation === false) { - if (get(item.json, fieldToAggregate) !== undefined) { + if (!fieldsToAggregate.length) { + throw new NodeOperationError(this.getNode(), 'No fields specified', { description: 'Please add a field to aggregate' }); + } + for (const { fieldToAggregate } of fieldsToAggregate) { + let found = false; + for (const item of items) { + if (fieldToAggregate === '') { + throw new NodeOperationError(this.getNode(), 'Field to aggregate is blank', { description: 'Please add a field to aggregate' }); + } + if (disableDotNotation === false) { + if (get(item.json, fieldToAggregate) !== undefined) { + found = true; + } + } else if (item.json.hasOwnProperty(fieldToAggregate)) { found = true; } - } else if (item.json.hasOwnProperty(fieldToAggregate)) { - found = true; + } + if (found === false && disableDotNotation && fieldToAggregate.includes('.')) { + throw new NodeOperationError(this.getNode(), `Couldn't find the field '${fieldToAggregate}' in the input data`, { description: `If you're trying to use a nested field, make sure you turn off 'disable dot notation' in the node options` }); + } else if (found === false && keepMissing === false) { + throw new NodeOperationError(this.getNode(), `Couldn't find the field '${fieldToAggregate}' in the input data`); } } - if (found === false && disableDotNotation && fieldToAggregate.includes('.')) { - throw new NodeOperationError(this.getNode(), `Couldn't find the field '${fieldToAggregate}' in the input data`, { description: `If you're trying to use a nested field, make sure you turn off 'disable dot notation' in the node options` }); - } else if (found === false && keepMissing === false) { - throw new NodeOperationError(this.getNode(), `Couldn't find the field '${fieldToAggregate}' in the input data`); - } - } - let newItem: INodeExecutionData; - newItem = { - json: {}, - pairedItem: Array.from({length}, (_, i) => i).map(index => { - return { - item: index, - }; - }), - }; + let newItem: INodeExecutionData; + newItem = { + json: {}, + pairedItem: Array.from({length}, (_, i) => i).map(index => { + return { + item: index, + }; + }), + }; - // tslint:disable-next-line: no-any - const values: { [key: string]: any } = {}; - const outputFields: string[] = []; + // tslint:disable-next-line: no-any + const values: { [key: string]: any } = {}; + const outputFields: string[] = []; - for (const { fieldToAggregate, outputFieldName, renameField } of fieldsToAggregate) { + for (const { fieldToAggregate, outputFieldName, renameField } of fieldsToAggregate) { - const field = (renameField) ? outputFieldName : fieldToAggregate; + const field = (renameField) ? outputFieldName : fieldToAggregate; - if (outputFields.includes(field)) { - throw new NodeOperationError(this.getNode(), `The '${field}' output field is used more than once`, { description: `Please make sure each output field name is unique` }); - } else { - outputFields.push(field); - } + if (outputFields.includes(field)) { + throw new NodeOperationError(this.getNode(), `The '${field}' output field is used more than once`, { description: `Please make sure each output field name is unique` }); + } else { + outputFields.push(field); + } - const getFieldToAggregate = () => ((disableDotNotation === false && fieldToAggregate.includes('.')) ? fieldToAggregate.split('.').pop() : fieldToAggregate); + const getFieldToAggregate = () => ((disableDotNotation === false && fieldToAggregate.includes('.')) ? fieldToAggregate.split('.').pop() : fieldToAggregate); - const _outputFieldName = (outputFieldName) ? (outputFieldName) : getFieldToAggregate() as string; + const _outputFieldName = (outputFieldName) ? (outputFieldName) : getFieldToAggregate() as string; - if (fieldToAggregate !== '') { - values[_outputFieldName] = []; - for (let i = 0; i < length; i++) { - if (disableDotNotation === false) { - let value = get(items[i].json, fieldToAggregate); + if (fieldToAggregate !== '') { + values[_outputFieldName] = []; + for (let i = 0; i < length; i++) { + if (disableDotNotation === false) { + let value = get(items[i].json, fieldToAggregate); - if (!keepMissing) { - if (Array.isArray(value)) { - value = value.filter(value => value !== null); - } else if (value === null || value === undefined) { - continue; + if (!keepMissing) { + if (Array.isArray(value)) { + value = value.filter(value => value !== null); + } else if (value === null || value === undefined) { + continue; + } } - } - if (Array.isArray(value) && mergeLists) { - values[_outputFieldName].push(...value); - } else { - values[_outputFieldName].push(value); - } - - } else { - let value = items[i].json[fieldToAggregate]; - - if (!keepMissing) { - if (Array.isArray(value)) { - value = value.filter(value => value !== null); - } else if (value === null || value === undefined) { - continue; + if (Array.isArray(value) && mergeLists) { + values[_outputFieldName].push(...value); + } else { + values[_outputFieldName].push(value); } - } - if (Array.isArray(value) && mergeLists) { - values[_outputFieldName].push(...value); } else { - values[_outputFieldName].push(value); + let value = items[i].json[fieldToAggregate]; + + if (!keepMissing) { + if (Array.isArray(value)) { + value = value.filter(value => value !== null); + } else if (value === null || value === undefined) { + continue; + } + } + + if (Array.isArray(value) && mergeLists) { + values[_outputFieldName].push(...value); + } else { + values[_outputFieldName].push(value); + } } } } } - } - for (const key of Object.keys(values)) { - if (disableDotNotation === false) { - set(newItem.json, key, values[key]); - } else { - newItem.json[key] = values[key]; + for (const key of Object.keys(values)) { + if (disableDotNotation === false) { + set(newItem.json, key, values[key]); + } else { + newItem.json[key] = values[key]; + } } + + returnData.push(newItem); + + return this.prepareOutputData(returnData); + + } else { + let newItems: IDataObject[] = items.map(item => item.json); + const destinationFieldName = this.getNodeParameter('destinationFieldName', 0) as string; + const fieldsToExclude = (this.getNodeParameter('fieldsToExclude.fields', 0, []) as IDataObject[]).map(entry => entry.fieldName); + const fieldsToInclude = (this.getNodeParameter('fieldsToInclude.fields', 0, []) as IDataObject[]).map(entry => entry.fieldName); + + + if (fieldsToExclude.length || fieldsToInclude.length) { + newItems = newItems.reduce((acc, item) => { + const newItem:IDataObject = {}; + let outputFields = Object.keys(item); + + if (fieldsToExclude.length) { + outputFields = outputFields.filter(key => !fieldsToExclude.includes(key)); + } + if (fieldsToInclude.length) { + outputFields = outputFields.filter(key => fieldsToInclude.length ? fieldsToInclude.includes(key) : true); + } + + outputFields.forEach( key => { + newItem[key] = item[key]; + }); + + if (isEmpty(newItem)) { + return acc; + } + return acc.concat([newItem]); + }, [] as IDataObject[]); + } + + const output: INodeExecutionData = { json: {[destinationFieldName]: newItems} }; + + return this.prepareOutputData([output]); } - returnData.push(newItem); - - return this.prepareOutputData(returnData); - } else if (operation === 'removeDuplicates') { const compare = this.getNodeParameter('compare', 0) as string;