diff --git a/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/concatenateItems.operation.ts b/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/concatenateItems.operation.ts index c70947df59..9af543ece2 100644 --- a/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/concatenateItems.operation.ts +++ b/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/concatenateItems.operation.ts @@ -13,7 +13,7 @@ import get from 'lodash/get'; import isEmpty from 'lodash/isEmpty'; import set from 'lodash/set'; -import { prepareFieldsArray } from '../../helpers/utils'; +import { addBinariesToItem, prepareFieldsArray } from '../../helpers/utils'; import { disableDotNotationBoolean } from '../common.descriptions'; const properties: INodeProperties[] = [ @@ -159,13 +159,15 @@ const properties: INodeProperties[] = [ type: 'collection', placeholder: 'Add Field', default: {}, - displayOptions: { - hide: { - aggregate: ['aggregateAllItemData'], - }, - }, options: [ - disableDotNotationBoolean, + { + ...disableDotNotationBoolean, + displayOptions: { + hide: { + '/aggregate': ['aggregateAllItemData'], + }, + }, + }, { displayName: 'Merge Lists', name: 'mergeLists', @@ -173,6 +175,31 @@ const properties: INodeProperties[] = [ default: false, description: 'Whether to merge the output into a single flat list (rather than a list of lists), if the field to aggregate is a list', + displayOptions: { + hide: { + '/aggregate': ['aggregateAllItemData'], + }, + }, + }, + { + displayName: 'Include Binaries', + name: 'includeBinaries', + type: 'boolean', + default: false, + description: 'Whether to include the binary data in the new item', + }, + { + displayName: 'Keep Only Unique Binaries', + name: 'keepOnlyUnique', + type: 'boolean', + default: false, + description: + 'Whether to keep only unique binaries by comparing mime types, file types, file sizes and file extensions', + displayOptions: { + show: { + includeBinaries: [true], + }, + }, }, { displayName: 'Keep Missing And Null Values', @@ -181,6 +208,11 @@ const properties: INodeProperties[] = [ default: false, description: 'Whether to add a null entry to the aggregated list when there is a missing or null value', + displayOptions: { + hide: { + '/aggregate': ['aggregateAllItemData'], + }, + }, }, ], }, @@ -199,7 +231,7 @@ export async function execute( this: IExecuteFunctions, items: INodeExecutionData[], ): Promise { - const returnData: INodeExecutionData[] = []; + let returnData: INodeExecutionData = { json: {}, pairedItem: [] }; const aggregate = this.getNodeParameter('aggregate', 0, '') as string; @@ -305,7 +337,7 @@ export async function execute( } } - returnData.push(newItem); + returnData = newItem; } else { let newItems: IDataObject[] = items.map((item) => item.json); let pairedItem: IPairedItemData[] = []; @@ -353,8 +385,23 @@ export async function execute( } const output: INodeExecutionData = { json: { [destinationFieldName]: newItems }, pairedItem }; - returnData.push(output); + + returnData = output; } - return returnData; + const includeBinaries = this.getNodeParameter('options.includeBinaries', 0, false) as boolean; + + if (includeBinaries) { + const pairedItems = (returnData.pairedItem || []) as IPairedItemData[]; + + const aggregatedItems = pairedItems.map((item) => { + return items[item.item]; + }); + + const keepOnlyUnique = this.getNodeParameter('options.keepOnlyUnique', 0, false) as boolean; + + addBinariesToItem(returnData, aggregatedItems, keepOnlyUnique); + } + + return [returnData]; } diff --git a/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/splitOutItems.operation.ts b/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/splitOutItems.operation.ts index 68bd5dbfbe..35b162ef87 100644 --- a/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/splitOutItems.operation.ts +++ b/packages/nodes-base/nodes/ItemLists/V3/actions/itemList/splitOutItems.operation.ts @@ -1,4 +1,5 @@ import type { + IBinaryData, IDataObject, IExecuteFunctions, INodeExecutionData, @@ -20,7 +21,9 @@ const properties: INodeProperties[] = [ type: 'string', default: '', required: true, - description: 'The name of the input fields to break out into separate items', + placeholder: 'Drag fields from the left or type their names', + description: + 'The name of the input fields to break out into separate items. Separate multiple field names by commas. For binary data, use $binary.', requiresDataPath: 'multiple', }, { @@ -74,6 +77,13 @@ const properties: INodeProperties[] = [ default: '', description: 'The field in the output under which to put the split field contents', }, + { + displayName: 'Include Binary', + name: 'includeBinary', + type: 'boolean', + default: false, + description: 'Whether to include the binary data in the new items', + }, ], }, ]; @@ -96,16 +106,13 @@ export async function execute( for (let i = 0; i < items.length; i++) { const fieldsToSplitOut = (this.getNodeParameter('fieldToSplitOut', i) as string) .split(',') - .map((field) => field.trim()); - const disableDotNotation = this.getNodeParameter( - 'options.disableDotNotation', - 0, - false, - ) as boolean; + .map((field) => field.trim().replace(/^\$json\./, '')); - const destinationFields = ( - this.getNodeParameter('options.destinationFieldName', i, '') as string - ) + const options = this.getNodeParameter('options', i, {}); + + const disableDotNotation = options.disableDotNotation as boolean; + + const destinationFields = ((options.destinationFieldName as string) || '') .split(',') .filter((field) => field.trim() !== '') .map((field) => field.trim()); @@ -125,54 +132,71 @@ export async function execute( const multiSplit = fieldsToSplitOut.length > 1; const item = { ...items[i].json }; - const splited: IDataObject[] = []; + const splited: INodeExecutionData[] = []; for (const [entryIndex, fieldToSplitOut] of fieldsToSplitOut.entries()) { const destinationFieldName = destinationFields[entryIndex] || ''; - let arrayToSplit; - if (!disableDotNotation) { - arrayToSplit = get(item, fieldToSplitOut); + let entityToSplit: IDataObject[] = []; + + if (fieldToSplitOut === '$binary') { + entityToSplit = Object.entries(items[i].binary || {}).map(([key, value]) => ({ + [key]: value, + })); } else { - arrayToSplit = item[fieldToSplitOut]; + if (!disableDotNotation) { + entityToSplit = get(item, fieldToSplitOut) as IDataObject[]; + } else { + entityToSplit = item[fieldToSplitOut] as IDataObject[]; + } + + if (entityToSplit === undefined) { + entityToSplit = []; + } + + if (typeof entityToSplit !== 'object' || entityToSplit === null) { + entityToSplit = [entityToSplit]; + } + + if (!Array.isArray(entityToSplit)) { + entityToSplit = Object.values(entityToSplit); + } } - if (arrayToSplit === undefined) { - arrayToSplit = []; - } - - if (typeof arrayToSplit !== 'object' || arrayToSplit === null) { - arrayToSplit = [arrayToSplit]; - } - - if (!Array.isArray(arrayToSplit)) { - arrayToSplit = Object.values(arrayToSplit); - } - - for (const [elementIndex, element] of arrayToSplit.entries()) { + for (const [elementIndex, element] of entityToSplit.entries()) { if (splited[elementIndex] === undefined) { - splited[elementIndex] = {}; + splited[elementIndex] = { json: {}, pairedItem: { item: i } }; } const fieldName = destinationFieldName || fieldToSplitOut; + if (fieldToSplitOut === '$binary') { + if (splited[elementIndex].binary === undefined) { + splited[elementIndex].binary = {}; + } + splited[elementIndex].binary![Object.keys(element)[0]] = Object.values( + element, + )[0] as IBinaryData; + + continue; + } + if (typeof element === 'object' && element !== null && include === 'noOtherFields') { if (destinationFieldName === '' && !multiSplit) { - splited[elementIndex] = { ...splited[elementIndex], ...element }; + splited[elementIndex] = { + json: { ...splited[elementIndex].json, ...element }, + pairedItem: { item: i }, + }; } else { - splited[elementIndex][fieldName] = element; + splited[elementIndex].json[fieldName] = element; } } else { - splited[elementIndex][fieldName] = element; + splited[elementIndex].json[fieldName] = element; } } } for (const splitEntry of splited) { - let newItem: IDataObject = {}; - - if (include === 'noOtherFields') { - newItem = splitEntry; - } + let newItem: INodeExecutionData = splitEntry; if (include === 'allOtherFields') { const itemCopy = deepCopy(item); @@ -183,7 +207,7 @@ export async function execute( delete itemCopy[fieldToSplitOut]; } } - newItem = { ...itemCopy, ...splitEntry }; + newItem.json = { ...itemCopy, ...splitEntry.json }; } if (include === 'selectedOtherFields') { @@ -200,21 +224,24 @@ export async function execute( for (const field of fieldsToInclude) { if (!disableDotNotation) { - splitEntry[field] = get(item, field); + splitEntry.json[field] = get(item, field); } else { - splitEntry[field] = item[field]; + splitEntry.json[field] = item[field]; } } newItem = splitEntry; } - returnData.push({ - json: newItem, - pairedItem: { - item: i, - }, - }); + const includeBinary = options.includeBinary as boolean; + + if (includeBinary) { + if (items[i].binary && !newItem.binary) { + newItem.binary = items[i].binary; + } + } + + returnData.push(newItem); } } diff --git a/packages/nodes-base/nodes/ItemLists/V3/helpers/utils.ts b/packages/nodes-base/nodes/ItemLists/V3/helpers/utils.ts index 94ba6a2db8..8f2337ec4d 100644 --- a/packages/nodes-base/nodes/ItemLists/V3/helpers/utils.ts +++ b/packages/nodes-base/nodes/ItemLists/V3/helpers/utils.ts @@ -1,11 +1,12 @@ import { NodeVM } from '@n8n/vm2'; -import { - NodeOperationError, - type IDataObject, - type IExecuteFunctions, - type INode, - type INodeExecutionData, +import type { + IDataObject, + IExecuteFunctions, + IBinaryData, + INode, + INodeExecutionData, } from 'n8n-workflow'; +import { NodeOperationError } from 'n8n-workflow'; import get from 'lodash/get'; import isEqual from 'lodash/isEqual'; @@ -87,3 +88,62 @@ export function sortByCode( return vm.run(`module.exports = items.sort((a, b) => { ${code} })`); } + +type PartialBinaryData = Omit; +const isBinaryUniqueSetup = () => { + const binaries: PartialBinaryData[] = []; + return (binary: IBinaryData) => { + for (const existingBinary of binaries) { + if ( + existingBinary.mimeType === binary.mimeType && + existingBinary.fileType === binary.fileType && + existingBinary.fileSize === binary.fileSize && + existingBinary.fileExtension === binary.fileExtension + ) { + return false; + } + } + + binaries.push({ + mimeType: binary.mimeType, + fileType: binary.fileType, + fileSize: binary.fileSize, + fileExtension: binary.fileExtension, + }); + + return true; + }; +}; + +export function addBinariesToItem( + newItem: INodeExecutionData, + items: INodeExecutionData[], + uniqueOnly?: boolean, +) { + const isBinaryUnique = uniqueOnly ? isBinaryUniqueSetup() : undefined; + + for (const item of items) { + if (item.binary === undefined) continue; + + for (const key of Object.keys(item.binary)) { + if (!newItem.binary) newItem.binary = {}; + let binaryKey = key; + const binary = item.binary[key]; + + if (isBinaryUnique && !isBinaryUnique(binary)) { + continue; + } + + // If the binary key already exists add a suffix to it + let i = 1; + while (newItem.binary[binaryKey] !== undefined) { + binaryKey = `${key}_${i}`; + i++; + } + + newItem.binary[binaryKey] = binary; + } + } + + return newItem; +}