feat(Item Lists Node): Split merge binary data (#7297)

Github issue / Community forum post (link here to close automatically):

---------

Co-authored-by: Marcus <marcus@n8n.io>

packages/nodes-base/nodes/ItemLists/V3/actions/itemList/concatenateItems.operation.ts

@@ -13,7 +13,7 @@ import get from 'lodash/get';
 import isEmpty from 'lodash/isEmpty';
 import set from 'lodash/set';
 
-import { prepareFieldsArray } from '../../helpers/utils';
+import { addBinariesToItem, prepareFieldsArray } from '../../helpers/utils';
 import { disableDotNotationBoolean } from '../common.descriptions';
 
 const properties: INodeProperties[] = [
@@ -159,13 +159,15 @@ const properties: INodeProperties[] = [
 		type: 'collection',
 		placeholder: 'Add Field',
 		default: {},
+		options: [
+			{
+				...disableDotNotationBoolean,
 				displayOptions: {
 					hide: {
-				aggregate: ['aggregateAllItemData'],
+						'/aggregate': ['aggregateAllItemData'],
+					},
 				},
 			},
-		options: [
-			disableDotNotationBoolean,
 			{
 				displayName: 'Merge Lists',
 				name: 'mergeLists',
@@ -173,6 +175,31 @@ const properties: INodeProperties[] = [
 				default: false,
 				description:
 					'Whether to merge the output into a single flat list (rather than a list of lists), if the field to aggregate is a list',
+				displayOptions: {
+					hide: {
+						'/aggregate': ['aggregateAllItemData'],
+					},
+				},
+			},
+			{
+				displayName: 'Include Binaries',
+				name: 'includeBinaries',
+				type: 'boolean',
+				default: false,
+				description: 'Whether to include the binary data in the new item',
+			},
+			{
+				displayName: 'Keep Only Unique Binaries',
+				name: 'keepOnlyUnique',
+				type: 'boolean',
+				default: false,
+				description:
+					'Whether to keep only unique binaries by comparing mime types, file types, file sizes and file extensions',
+				displayOptions: {
+					show: {
+						includeBinaries: [true],
+					},
+				},
 			},
 			{
 				displayName: 'Keep Missing And Null Values',
@@ -181,6 +208,11 @@ const properties: INodeProperties[] = [
 				default: false,
 				description:
 					'Whether to add a null entry to the aggregated list when there is a missing or null value',
+				displayOptions: {
+					hide: {
+						'/aggregate': ['aggregateAllItemData'],
+					},
+				},
 			},
 		],
 	},
@@ -199,7 +231,7 @@ export async function execute(
 	this: IExecuteFunctions,
 	items: INodeExecutionData[],
 ): Promise<INodeExecutionData[]> {
-	const returnData: INodeExecutionData[] = [];
+	let returnData: INodeExecutionData = { json: {}, pairedItem: [] };
 
 	const aggregate = this.getNodeParameter('aggregate', 0, '') as string;
 
@@ -305,7 +337,7 @@ export async function execute(
 			}
 		}
 
-		returnData.push(newItem);
+		returnData = newItem;
 	} else {
 		let newItems: IDataObject[] = items.map((item) => item.json);
 		let pairedItem: IPairedItemData[] = [];
@@ -353,8 +385,23 @@ export async function execute(
 		}
 
 		const output: INodeExecutionData = { json: { [destinationFieldName]: newItems }, pairedItem };
-		returnData.push(output);
+
+		returnData = output;
 	}
 
-	return returnData;
+	const includeBinaries = this.getNodeParameter('options.includeBinaries', 0, false) as boolean;
+
+	if (includeBinaries) {
+		const pairedItems = (returnData.pairedItem || []) as IPairedItemData[];
+
+		const aggregatedItems = pairedItems.map((item) => {
+			return items[item.item];
+		});
+
+		const keepOnlyUnique = this.getNodeParameter('options.keepOnlyUnique', 0, false) as boolean;
+
+		addBinariesToItem(returnData, aggregatedItems, keepOnlyUnique);
+	}
+
+	return [returnData];
 }

packages/nodes-base/nodes/ItemLists/V3/actions/itemList/splitOutItems.operation.ts

@@ -1,4 +1,5 @@
 import type {
+	IBinaryData,
 	IDataObject,
 	IExecuteFunctions,
 	INodeExecutionData,
@@ -20,7 +21,9 @@ const properties: INodeProperties[] = [
 		type: 'string',
 		default: '',
 		required: true,
-		description: 'The name of the input fields to break out into separate items',
+		placeholder: 'Drag fields from the left or type their names',
+		description:
+			'The name of the input fields to break out into separate items. Separate multiple field names by commas. For binary data, use $binary.',
 		requiresDataPath: 'multiple',
 	},
 	{
@@ -74,6 +77,13 @@ const properties: INodeProperties[] = [
 				default: '',
 				description: 'The field in the output under which to put the split field contents',
 			},
+			{
+				displayName: 'Include Binary',
+				name: 'includeBinary',
+				type: 'boolean',
+				default: false,
+				description: 'Whether to include the binary data in the new items',
+			},
 		],
 	},
 ];
@@ -96,16 +106,13 @@ export async function execute(
 	for (let i = 0; i < items.length; i++) {
 		const fieldsToSplitOut = (this.getNodeParameter('fieldToSplitOut', i) as string)
 			.split(',')
-			.map((field) => field.trim());
-		const disableDotNotation = this.getNodeParameter(
-			'options.disableDotNotation',
-			0,
-			false,
-		) as boolean;
+			.map((field) => field.trim().replace(/^\$json\./, ''));
 
-		const destinationFields = (
-			this.getNodeParameter('options.destinationFieldName', i, '') as string
-		)
+		const options = this.getNodeParameter('options', i, {});
+
+		const disableDotNotation = options.disableDotNotation as boolean;
+
+		const destinationFields = ((options.destinationFieldName as string) || '')
 			.split(',')
 			.filter((field) => field.trim() !== '')
 			.map((field) => field.trim());
@@ -125,54 +132,71 @@ export async function execute(
 		const multiSplit = fieldsToSplitOut.length > 1;
 
 		const item = { ...items[i].json };
-		const splited: IDataObject[] = [];
+		const splited: INodeExecutionData[] = [];
 		for (const [entryIndex, fieldToSplitOut] of fieldsToSplitOut.entries()) {
 			const destinationFieldName = destinationFields[entryIndex] || '';
 
-			let arrayToSplit;
-			if (!disableDotNotation) {
-				arrayToSplit = get(item, fieldToSplitOut);
+			let entityToSplit: IDataObject[] = [];
+
+			if (fieldToSplitOut === '$binary') {
+				entityToSplit = Object.entries(items[i].binary || {}).map(([key, value]) => ({
+					[key]: value,
+				}));
 			} else {
-				arrayToSplit = item[fieldToSplitOut];
+				if (!disableDotNotation) {
+					entityToSplit = get(item, fieldToSplitOut) as IDataObject[];
+				} else {
+					entityToSplit = item[fieldToSplitOut] as IDataObject[];
 				}
 
-			if (arrayToSplit === undefined) {
-				arrayToSplit = [];
+				if (entityToSplit === undefined) {
+					entityToSplit = [];
 				}
 
-			if (typeof arrayToSplit !== 'object' || arrayToSplit === null) {
-				arrayToSplit = [arrayToSplit];
+				if (typeof entityToSplit !== 'object' || entityToSplit === null) {
+					entityToSplit = [entityToSplit];
 				}
 
-			if (!Array.isArray(arrayToSplit)) {
-				arrayToSplit = Object.values(arrayToSplit);
+				if (!Array.isArray(entityToSplit)) {
+					entityToSplit = Object.values(entityToSplit);
+				}
 			}
 
-			for (const [elementIndex, element] of arrayToSplit.entries()) {
+			for (const [elementIndex, element] of entityToSplit.entries()) {
 				if (splited[elementIndex] === undefined) {
-					splited[elementIndex] = {};
+					splited[elementIndex] = { json: {}, pairedItem: { item: i } };
 				}
 
 				const fieldName = destinationFieldName || fieldToSplitOut;
 
+				if (fieldToSplitOut === '$binary') {
+					if (splited[elementIndex].binary === undefined) {
+						splited[elementIndex].binary = {};
+					}
+					splited[elementIndex].binary![Object.keys(element)[0]] = Object.values(
+						element,
+					)[0] as IBinaryData;
+
+					continue;
+				}
+
 				if (typeof element === 'object' && element !== null && include === 'noOtherFields') {
 					if (destinationFieldName === '' && !multiSplit) {
-						splited[elementIndex] = { ...splited[elementIndex], ...element };
+						splited[elementIndex] = {
+							json: { ...splited[elementIndex].json, ...element },
+							pairedItem: { item: i },
+						};
 					} else {
-						splited[elementIndex][fieldName] = element;
+						splited[elementIndex].json[fieldName] = element;
 					}
 				} else {
-					splited[elementIndex][fieldName] = element;
+					splited[elementIndex].json[fieldName] = element;
 				}
 			}
 		}
 
 		for (const splitEntry of splited) {
-			let newItem: IDataObject = {};
-
-			if (include === 'noOtherFields') {
-				newItem = splitEntry;
-			}
+			let newItem: INodeExecutionData = splitEntry;
 
 			if (include === 'allOtherFields') {
 				const itemCopy = deepCopy(item);
@@ -183,7 +207,7 @@ export async function execute(
 						delete itemCopy[fieldToSplitOut];
 					}
 				}
-				newItem = { ...itemCopy, ...splitEntry };
+				newItem.json = { ...itemCopy, ...splitEntry.json };
 			}
 
 			if (include === 'selectedOtherFields') {
@@ -200,21 +224,24 @@ export async function execute(
 
 				for (const field of fieldsToInclude) {
 					if (!disableDotNotation) {
-						splitEntry[field] = get(item, field);
+						splitEntry.json[field] = get(item, field);
 					} else {
-						splitEntry[field] = item[field];
+						splitEntry.json[field] = item[field];
 					}
 				}
 
 				newItem = splitEntry;
 			}
 
-			returnData.push({
-				json: newItem,
-				pairedItem: {
-					item: i,
-				},
-			});
+			const includeBinary = options.includeBinary as boolean;
+
+			if (includeBinary) {
+				if (items[i].binary && !newItem.binary) {
+					newItem.binary = items[i].binary;
+				}
+			}
+
+			returnData.push(newItem);
 		}
 	}
 

packages/nodes-base/nodes/ItemLists/V3/helpers/utils.ts

@@ -1,11 +1,12 @@
 import { NodeVM } from '@n8n/vm2';
-import {
-	NodeOperationError,
-	type IDataObject,
-	type IExecuteFunctions,
-	type INode,
-	type INodeExecutionData,
+import type {
+	IDataObject,
+	IExecuteFunctions,
+	IBinaryData,
+	INode,
+	INodeExecutionData,
 } from 'n8n-workflow';
+import { NodeOperationError } from 'n8n-workflow';
 
 import get from 'lodash/get';
 import isEqual from 'lodash/isEqual';
@@ -87,3 +88,62 @@ export function sortByCode(
 
 	return vm.run(`module.exports = items.sort((a, b) => { ${code} })`);
 }
+
+type PartialBinaryData = Omit<IBinaryData, 'data'>;
+const isBinaryUniqueSetup = () => {
+	const binaries: PartialBinaryData[] = [];
+	return (binary: IBinaryData) => {
+		for (const existingBinary of binaries) {
+			if (
+				existingBinary.mimeType === binary.mimeType &&
+				existingBinary.fileType === binary.fileType &&
+				existingBinary.fileSize === binary.fileSize &&
+				existingBinary.fileExtension === binary.fileExtension
+			) {
+				return false;
+			}
+		}
+
+		binaries.push({
+			mimeType: binary.mimeType,
+			fileType: binary.fileType,
+			fileSize: binary.fileSize,
+			fileExtension: binary.fileExtension,
+		});
+
+		return true;
+	};
+};
+
+export function addBinariesToItem(
+	newItem: INodeExecutionData,
+	items: INodeExecutionData[],
+	uniqueOnly?: boolean,
+) {
+	const isBinaryUnique = uniqueOnly ? isBinaryUniqueSetup() : undefined;
+
+	for (const item of items) {
+		if (item.binary === undefined) continue;
+
+		for (const key of Object.keys(item.binary)) {
+			if (!newItem.binary) newItem.binary = {};
+			let binaryKey = key;
+			const binary = item.binary[key];
+
+			if (isBinaryUnique && !isBinaryUnique(binary)) {
+				continue;
+			}
+
+			// If the binary key already exists add a suffix to it
+			let i = 1;
+			while (newItem.binary[binaryKey] !== undefined) {
+				binaryKey = `${key}_${i}`;
+				i++;
+			}
+
+			newItem.binary[binaryKey] = binary;
+		}
+	}
+
+	return newItem;
+}