fix(core): Stop copying icons to cache (#5419)

Fixes 

- https://github.com/n8n-io/n8n/issues/4973
- https://github.com/n8n-io/n8n/issues/5274
- https://community.n8n.io/t/starting-n8n-fails-with-ebusy-error/21243
- https://community.n8n.io/t/problem-executing-workflow-ebusy-resource-busy-or-locked-copyfile/21280

Replaces

- https://github.com/n8n-io/n8n/pull/5052
- https://github.com/n8n-io/n8n/pull/5401

packages/cli/src/LoadNodesAndCredentials.ts

@@ -18,13 +18,7 @@ import type {
 import { LoggerProxy, ErrorReporterProxy as ErrorReporter } from 'n8n-workflow';
 
 import { createWriteStream } from 'fs';
-import {
+import { access as fsAccess, mkdir, readdir as fsReaddir, stat as fsStat } from 'fs/promises';
-	access as fsAccess,
-	copyFile,
-	mkdir,
-	readdir as fsReaddir,
-	stat as fsStat,
-} from 'fs/promises';
 import path from 'path';
 import config from '@/config';
 import type { InstalledPackages } from '@db/entities/InstalledPackages';
@@ -50,6 +44,8 @@ export class LoadNodesAndCredentialsClass implements INodesAndCredentials {
 
 	types: Types = { nodes: [], credentials: [] };
 
+	loaders: Record<string, DirectoryLoader> = {};
+
 	excludeNodes = config.getEnv('nodes.exclude');
 
 	includeNodes = config.getEnv('nodes.include');
@@ -73,6 +69,7 @@ export class LoadNodesAndCredentialsClass implements INodesAndCredentials {
 		await this.loadNodesFromBasePackages();
 		await this.loadNodesFromDownloadedPackages();
 		await this.loadNodesFromCustomDirectories();
+		await this.postProcessLoaders();
 		this.injectCustomApiCallOptions();
 	}
 
@@ -117,7 +114,7 @@ export class LoadNodesAndCredentialsClass implements INodesAndCredentials {
 		await writeStaticJSON('credentials', this.types.credentials);
 	}
 
-	async loadNodesFromBasePackages() {
+	private async loadNodesFromBasePackages() {
 		const nodeModulesPath = await this.getNodeModulesPath();
 		const nodePackagePaths = await this.getN8nNodePackages(nodeModulesPath);
 
@@ -126,7 +123,7 @@ export class LoadNodesAndCredentialsClass implements INodesAndCredentials {
 		}
 	}
 
-	async loadNodesFromDownloadedPackages(): Promise<void> {
+	private async loadNodesFromDownloadedPackages(): Promise<void> {
 		const nodePackages = [];
 		try {
 			// Read downloaded nodes and credentials
@@ -160,24 +157,23 @@ export class LoadNodesAndCredentialsClass implements INodesAndCredentials {
 		return customDirectories;
 	}
 
-	async loadNodesFromCustomDirectories(): Promise<void> {
+	private async loadNodesFromCustomDirectories(): Promise<void> {
 		for (const directory of this.getCustomDirectories()) {
 			await this.runDirectoryLoader(CustomDirectoryLoader, directory);
 		}
 	}
 
 	/**
-	 * Returns all the names of the packages which could
+	 * Returns all the names of the packages which could contain n8n nodes
-	 * contain n8n nodes
-	 *
 	 */
-	async getN8nNodePackages(baseModulesPath: string): Promise<string[]> {
+	private async getN8nNodePackages(baseModulesPath: string): Promise<string[]> {
 		const getN8nNodePackagesRecursive = async (relativePath: string): Promise<string[]> => {
 			const results: string[] = [];
 			const nodeModulesPath = `${baseModulesPath}/${relativePath}`;
-			for (const file of await fsReaddir(nodeModulesPath)) {
+			const nodeModules = await fsReaddir(nodeModulesPath);
-				const isN8nNodesPackage = file.indexOf('n8n-nodes-') === 0;
+			for (const nodeModule of nodeModules) {
-				const isNpmScopedPackage = file.indexOf('@') === 0;
+				const isN8nNodesPackage = nodeModule.indexOf('n8n-nodes-') === 0;
+				const isNpmScopedPackage = nodeModule.indexOf('@') === 0;
 				if (!isN8nNodesPackage && !isNpmScopedPackage) {
 					continue;
 				}
@@ -185,10 +181,10 @@ export class LoadNodesAndCredentialsClass implements INodesAndCredentials {
 					continue;
 				}
 				if (isN8nNodesPackage) {
-					results.push(`${baseModulesPath}/${relativePath}${file}`);
+					results.push(`${baseModulesPath}/${relativePath}${nodeModule}`);
 				}
 				if (isNpmScopedPackage) {
-					results.push(...(await getN8nNodePackagesRecursive(`${relativePath}${file}/`)));
+					results.push(...(await getN8nNodePackagesRecursive(`${relativePath}${nodeModule}/`)));
 				}
 			}
 			return results;
@@ -392,64 +388,52 @@ export class LoadNodesAndCredentialsClass implements INodesAndCredentials {
 	) {
 		const loader = new constructor(dir, this.excludeNodes, this.includeNodes);
 		await loader.loadAll();
-
+		this.loaders[dir] = loader;
-		// list of node & credential types that will be sent to the frontend
-		const { types } = loader;
-		this.types.nodes = this.types.nodes.concat(types.nodes);
-		this.types.credentials = this.types.credentials.concat(types.credentials);
-
-		// Copy over all icons and set `iconUrl` for the frontend
-		const iconPromises = Object.entries(types).flatMap(([typeName, typesArr]) =>
-			typesArr.map((type) => {
-				if (!type.icon?.startsWith('file:')) return;
-				const icon = type.icon.substring(5);
-				const iconUrl = `icons/${typeName}/${type.name}${path.extname(icon)}`;
-				delete type.icon;
-				type.iconUrl = iconUrl;
-				const source = path.join(dir, icon);
-				const destination = path.join(GENERATED_STATIC_DIR, iconUrl);
-				return mkdir(path.dirname(destination), { recursive: true }).then(async () =>
-					copyFile(source, destination),
-				);
-			}),
-		);
-
-		await Promise.all(iconPromises);
-
-		// Nodes and credentials that have been loaded immediately
-		for (const nodeTypeName in loader.nodeTypes) {
-			this.loaded.nodes[nodeTypeName] = loader.nodeTypes[nodeTypeName];
-		}
-
-		for (const credentialTypeName in loader.credentialTypes) {
-			this.loaded.credentials[credentialTypeName] = loader.credentialTypes[credentialTypeName];
-		}
-
-		// Nodes and credentials that will be lazy loaded
-		if (loader instanceof PackageDirectoryLoader) {
-			const { packageName, known } = loader;
-
-			for (const type in known.nodes) {
-				const { className, sourcePath } = known.nodes[type];
-				this.known.nodes[type] = {
-					className,
-					sourcePath: path.join(dir, sourcePath),
-				};
-			}
-
-			for (const type in known.credentials) {
-				const { className, sourcePath, nodesToTestWith } = known.credentials[type];
-				this.known.credentials[type] = {
-					className,
-					sourcePath: path.join(dir, sourcePath),
-					nodesToTestWith: nodesToTestWith?.map((nodeName) => `${packageName}.${nodeName}`),
-				};
-			}
-		}
-
 		return loader;
 	}
 
+	async postProcessLoaders() {
+		this.types.nodes = [];
+		this.types.credentials = [];
+		for (const [dir, loader] of Object.entries(this.loaders)) {
+			// list of node & credential types that will be sent to the frontend
+			const { types } = loader;
+			this.types.nodes = this.types.nodes.concat(types.nodes);
+			this.types.credentials = this.types.credentials.concat(types.credentials);
+
+			// Nodes and credentials that have been loaded immediately
+			for (const nodeTypeName in loader.nodeTypes) {
+				this.loaded.nodes[nodeTypeName] = loader.nodeTypes[nodeTypeName];
+			}
+
+			for (const credentialTypeName in loader.credentialTypes) {
+				this.loaded.credentials[credentialTypeName] = loader.credentialTypes[credentialTypeName];
+			}
+
+			// Nodes and credentials that will be lazy loaded
+			if (loader instanceof PackageDirectoryLoader) {
+				const { packageName, known } = loader;
+
+				for (const type in known.nodes) {
+					const { className, sourcePath } = known.nodes[type];
+					this.known.nodes[type] = {
+						className,
+						sourcePath: path.join(dir, sourcePath),
+					};
+				}
+
+				for (const type in known.credentials) {
+					const { className, sourcePath, nodesToTestWith } = known.credentials[type];
+					this.known.credentials[type] = {
+						className,
+						sourcePath: path.join(dir, sourcePath),
+						nodesToTestWith: nodesToTestWith?.map((nodeName) => `${packageName}.${nodeName}`),
+					};
+				}
+			}
+		}
+	}
+
 	private async getNodeModulesPath(): Promise<string> {
 		// Get the path to the node-modules folder to be later able
 		// to load the credentials and nodes

packages/cli/src/NodeTypes.ts

@@ -17,7 +17,7 @@ class NodeTypesClass implements INodeTypes {
 		// eslint-disable-next-line no-restricted-syntax
 		for (const nodeTypeData of Object.values(this.loadedNodes)) {
 			const nodeType = NodeHelpers.getVersionedNodeType(nodeTypeData.type);
-			this.applySpecialNodeParameters(nodeType);
+			NodeHelpers.applySpecialNodeParameters(nodeType);
 		}
 	}
 
@@ -57,20 +57,13 @@ class NodeTypesClass implements INodeTypes {
 		if (type in knownNodes) {
 			const { className, sourcePath } = knownNodes[type];
 			const loaded: INodeType = loadClassInIsolation(sourcePath, className);
-			this.applySpecialNodeParameters(loaded);
+			NodeHelpers.applySpecialNodeParameters(loaded);
 			loadedNodes[type] = { sourcePath, type: loaded };
 			return loadedNodes[type];
 		}
 		throw new Error(`${RESPONSE_ERROR_MESSAGES.NO_NODE}: ${type}`);
 	}
 
-	private applySpecialNodeParameters(nodeType: INodeType) {
-		const applyParameters = NodeHelpers.getSpecialNodeParameters(nodeType);
-		if (applyParameters.length) {
-			nodeType.description.properties.unshift(...applyParameters);
-		}
-	}
-
 	private get loadedNodes() {
 		return this.nodesAndCredentials.loaded.nodes;
 	}

packages/cli/src/Server.ts

@@ -122,7 +122,6 @@ import {
 import { getInstance as getMailerInstance } from '@/UserManagement/email';
 import * as Db from '@/Db';
 import type {
-	DatabaseType,
 	ICredentialsDb,
 	ICredentialsOverwrite,
 	IDiagnosticInfo,
@@ -139,10 +138,10 @@ import {
 } from '@/CredentialsHelper';
 import { CredentialsOverwrites } from '@/CredentialsOverwrites';
 import { CredentialTypes } from '@/CredentialTypes';
-import * as GenericHelpers from '@/GenericHelpers';
 import { NodeTypes } from '@/NodeTypes';
 import * as Push from '@/Push';
 import { LoadNodesAndCredentials } from '@/LoadNodesAndCredentials';
+import type { LoadNodesAndCredentialsClass } from '@/LoadNodesAndCredentials';
 import * as ResponseHelper from '@/ResponseHelper';
 import type { WaitTrackerClass } from '@/WaitTracker';
 import { WaitTracker } from '@/WaitTracker';
@@ -176,6 +175,8 @@ class Server extends AbstractServer {
 
 	presetCredentialsLoaded: boolean;
 
+	loadNodesAndCredentials: LoadNodesAndCredentialsClass;
+
 	nodeTypes: INodeTypes;
 
 	credentialTypes: ICredentialTypes;
@@ -185,6 +186,7 @@ class Server extends AbstractServer {
 
 		this.nodeTypes = NodeTypes();
 		this.credentialTypes = CredentialTypes();
+		this.loadNodesAndCredentials = LoadNodesAndCredentials();
 
 		this.activeExecutionsInstance = ActiveExecutions.getInstance();
 		this.waitTracker = WaitTracker();
@@ -1265,7 +1267,7 @@ class Server extends AbstractServer {
 
 						CredentialsOverwrites().setData(body);
 
-						await LoadNodesAndCredentials().generateTypesForFrontend();
+						await this.loadNodesAndCredentials.generateTypesForFrontend();
 
 						this.presetCredentialsLoaded = true;
 
@@ -1277,17 +1279,31 @@ class Server extends AbstractServer {
 			);
 		}
 
-		const staticOptions: ServeStaticOptions = {
-			cacheControl: false,
-			setHeaders: (res: express.Response, path: string) => {
-				const isIndex = path === pathJoin(GENERATED_STATIC_DIR, 'index.html');
-				const cacheControl = isIndex
-					? 'no-cache, no-store, must-revalidate'
-					: 'max-age=86400, immutable';
-				res.header('Cache-Control', cacheControl);
-			},
-		};
 		if (!config.getEnv('endpoints.disableUi')) {
+			const staticOptions: ServeStaticOptions = {
+				cacheControl: false,
+				setHeaders: (res: express.Response, path: string) => {
+					const isIndex = path === pathJoin(GENERATED_STATIC_DIR, 'index.html');
+					const cacheControl = isIndex
+						? 'no-cache, no-store, must-revalidate'
+						: 'max-age=86400, immutable';
+					res.header('Cache-Control', cacheControl);
+				},
+			};
+
+			for (const [dir, loader] of Object.entries(this.loadNodesAndCredentials.loaders)) {
+				const pathPrefix = `/icons/${loader.packageName}`;
+				this.app.use(`${pathPrefix}/*/*.(svg|png)`, async (req, res) => {
+					const filePath = pathResolve(dir, req.originalUrl.substring(pathPrefix.length + 1));
+					try {
+						await fsAccess(filePath);
+						res.sendFile(filePath);
+					} catch {
+						res.sendStatus(404);
+					}
+				});
+			}
+
 			this.app.use(
 				'/',
 				express.static(GENERATED_STATIC_DIR),

packages/core/bin/generate-ui-types

@@ -18,10 +18,7 @@ LoggerProxy.init({
 	const nodeTypes = Object.values(loader.nodeTypes)
 		.map((data) => {
 			const nodeType = NodeHelpers.getVersionedNodeType(data.type);
-			const applyParameters = NodeHelpers.getSpecialNodeParameters(nodeType);
+			NodeHelpers.applySpecialNodeParameters(nodeType);
-			if (applyParameters.length) {
-				nodeType.description.properties.unshift(...applyParameters);
-			}
 			return data.type;
 		})
 		.flatMap((nodeData) => {

packages/core/src/DirectoryLoader.ts

@@ -48,19 +48,21 @@ export abstract class DirectoryLoader {
 		protected readonly includeNodes: string[] = [],
 	) {}
 
+	abstract packageName: string;
 	abstract loadAll(): Promise<void>;
 
 	protected resolvePath(file: string) {
 		return path.resolve(this.directory, file);
 	}
 
-	protected loadNodeFromFile(packageName: string, nodeName: string, filePath: string) {
+	protected loadNodeFromFile(nodeName: string, filePath: string) {
 		let tempNode: INodeType | IVersionedNodeType;
 		let nodeVersion = 1;
+		const isCustom = this.packageName === 'CUSTOM';
 
 		try {
 			tempNode = loadClassInIsolation(filePath, nodeName);
-			this.addCodex({ node: tempNode, filePath, isCustom: packageName === 'CUSTOM' });
+			this.addCodex({ node: tempNode, filePath, isCustom });
 		} catch (error) {
 			Logger.error(
 				`Error loading node "${nodeName}" from: "${filePath}" - ${(error as Error).message}`,
@@ -68,7 +70,7 @@ export abstract class DirectoryLoader {
 			throw error;
 		}
 
-		const fullNodeName = `${packageName}.${tempNode.description.name}`;
+		const fullNodeName = `${this.packageName}.${tempNode.description.name}`;
 
 		if (this.includeNodes.length && !this.includeNodes.includes(fullNodeName)) {
 			return;
@@ -88,12 +90,12 @@ export abstract class DirectoryLoader {
 			}
 
 			const currentVersionNode = tempNode.nodeVersions[tempNode.currentVersion];
-			this.addCodex({ node: currentVersionNode, filePath, isCustom: packageName === 'CUSTOM' });
+			this.addCodex({ node: currentVersionNode, filePath, isCustom });
 			nodeVersion = tempNode.currentVersion;
 
 			if (currentVersionNode.hasOwnProperty('executeSingle')) {
 				Logger.warn(
-					`"executeSingle" will get deprecated soon. Please update the code of node "${packageName}.${nodeName}" to use "execute" instead!`,
+					`"executeSingle" will get deprecated soon. Please update the code of node "${this.packageName}.${nodeName}" to use "execute" instead!`,
 					{ filePath },
 				);
 			}
@@ -236,7 +238,8 @@ export abstract class DirectoryLoader {
 		if (obj.icon?.startsWith('file:')) {
 			const iconPath = path.join(path.dirname(filePath), obj.icon.substring(5));
 			const relativePath = path.relative(this.directory, iconPath);
-			obj.icon = `file:${relativePath}`;
+			obj.iconUrl = `icons/${this.packageName}/${relativePath}`;
+			delete obj.icon;
 		}
 	}
 }
@@ -246,6 +249,8 @@ export abstract class DirectoryLoader {
  * e.g. `~/.n8n/custom`
  */
 export class CustomDirectoryLoader extends DirectoryLoader {
+	packageName = 'CUSTOM';
+
 	override async loadAll() {
 		const filePaths = await glob('**/*.@(node|credentials).js', {
 			cwd: this.directory,
@@ -256,7 +261,7 @@ export class CustomDirectoryLoader extends DirectoryLoader {
 			const [fileName, type] = path.parse(filePath).name.split('.');
 
 			if (type === 'node') {
-				this.loadNodeFromFile('CUSTOM', fileName, filePath);
+				this.loadNodeFromFile(fileName, filePath);
 			} else if (type === 'credentials') {
 				this.loadCredentialFromFile(fileName, filePath);
 			}
@@ -300,7 +305,7 @@ export class PackageDirectoryLoader extends DirectoryLoader {
 				const filePath = this.resolvePath(node);
 				const [nodeName] = path.parse(node).name.split('.');
 
-				this.loadNodeFromFile(this.packageName, nodeName, filePath);
+				this.loadNodeFromFile(nodeName, filePath);
 			}
 		}
 

packages/workflow/src/NodeHelpers.ts

@@ -230,31 +230,29 @@ export const cronNodeOptions: INodePropertyCollection[] = [
 	},
 ];
 
-/**
+const specialNodeParameters: INodeProperties[] = [
- * Gets special parameters which should be added to nodeTypes depending
+	{
- * on their type or configuration
+		displayName: 'Poll Times',
- *
+		name: 'pollTimes',
- */
+		type: 'fixedCollection',
-export function getSpecialNodeParameters(nodeType: INodeType): INodeProperties[] {
+		typeOptions: {
-	if (nodeType.description.polling === true) {
+			multipleValues: true,
-		return [
+			multipleValueButtonText: 'Add Poll Time',
-			{
+		},
-				displayName: 'Poll Times',
+		default: { item: [{ mode: 'everyMinute' }] },
-				name: 'pollTimes',
+		description: 'Time at which polling should occur',
-				type: 'fixedCollection',
+		placeholder: 'Add Poll Time',
-				typeOptions: {
+		options: cronNodeOptions,
-					multipleValues: true,
+	},
-					multipleValueButtonText: 'Add Poll Time',
+];
-				},
-				default: { item: [{ mode: 'everyMinute' }] },
-				description: 'Time at which polling should occur',
-				placeholder: 'Add Poll Time',
-				options: cronNodeOptions,
-			},
-		];
-	}
 
-	return [];
+/**
+ * Apply special parameters which should be added to nodeTypes depending on their type or configuration
+ */
+export function applySpecialNodeParameters(nodeType: INodeType): void {
+	if (nodeType.description.polling === true) {
+		nodeType.description.properties.unshift(...specialNodeParameters);
+	}
 }
 
 /**