fix(core): Improve community nodes loading (#5608)

packages/cli/src/CredentialsHelper.ts

@@ -451,7 +451,6 @@ export class CredentialsHelper extends ICredentialsHelper {
 		type: string,
 		data: ICredentialDataDecryptedObject,
 	): Promise<void> {
-		// eslint-disable-next-line @typescript-eslint/await-thenable
 		const credentials = await this.getCredentials(nodeCredentials, type);
 
 		if (!Db.isInitialized) {

packages/cli/src/LoadNodesAndCredentials.ts

@@ -1,4 +1,5 @@
 import uniq from 'lodash.uniq';
+import glob from 'fast-glob';
 import type { DirectoryLoader, Types } from 'n8n-core';
 import {
 	CUSTOM_EXTENSION_ENV,
@@ -18,18 +19,18 @@ import type {
 import { LoggerProxy, ErrorReporterProxy as ErrorReporter } from 'n8n-workflow';
 
 import { createWriteStream } from 'fs';
-import { access as fsAccess, mkdir, readdir as fsReaddir, stat as fsStat } from 'fs/promises';
+import { mkdir } from 'fs/promises';
 import path from 'path';
 import config from '@/config';
 import type { InstalledPackages } from '@db/entities/InstalledPackages';
 import { executeCommand } from '@/CommunityNodes/helpers';
 import {
-	CLI_DIR,
 	GENERATED_STATIC_DIR,
 	RESPONSE_ERROR_MESSAGES,
 	CUSTOM_API_CALL_KEY,
 	CUSTOM_API_CALL_NAME,
 	inTest,
+	CLI_DIR,
 } from '@/constants';
 import { CredentialsOverwrites } from '@/CredentialsOverwrites';
 import { Service } from 'typedi';
@@ -52,6 +53,8 @@ export class LoadNodesAndCredentials implements INodesAndCredentials {
 
 	logger: ILogger;
 
+	private downloadFolder: string;
+
 	async init() {
 		// Make sure the imported modules can resolve dependencies fine.
 		const delimiter = process.platform === 'win32' ? ';' : ':';
@@ -61,8 +64,13 @@ export class LoadNodesAndCredentials implements INodesAndCredentials {
 		// eslint-disable-next-line @typescript-eslint/no-unsafe-call
 		if (!inTest) module.constructor._initPaths();
 
-		await this.loadNodesFromBasePackages();
+		this.downloadFolder = UserSettings.getUserN8nFolderDownloadedNodesPath();
-		await this.loadNodesFromDownloadedPackages();
+
+		// Load nodes from `n8n-nodes-base` and any other `n8n-nodes-*` package in the main `node_modules`
+		await this.loadNodesFromNodeModules(CLI_DIR);
+		// Load nodes from installed community packages
+		await this.loadNodesFromNodeModules(this.downloadFolder);
+
 		await this.loadNodesFromCustomDirectories();
 		await this.postProcessLoaders();
 		this.injectCustomApiCallOptions();
@@ -109,32 +117,20 @@ export class LoadNodesAndCredentials implements INodesAndCredentials {
 		await writeStaticJSON('credentials', this.types.credentials);
 	}
 
-	private async loadNodesFromBasePackages() {
+	private async loadNodesFromNodeModules(scanDir: string): Promise<void> {
-		const nodeModulesPath = await this.getNodeModulesPath();
+		const nodeModulesDir = path.join(scanDir, 'node_modules');
-		const nodePackagePaths = await this.getN8nNodePackages(nodeModulesPath);
+		const globOptions = { cwd: nodeModulesDir, onlyDirectories: true };
+		const installedPackagePaths = [
+			...(await glob('n8n-nodes-*', { ...globOptions, deep: 1 })),
+			...(await glob('@*/n8n-nodes-*', { ...globOptions, deep: 2 })),
+		];
 
-		for (const packagePath of nodePackagePaths) {
+		for (const packagePath of installedPackagePaths) {
-			await this.runDirectoryLoader(LazyPackageDirectoryLoader, packagePath);
-		}
-	}
-
-	private async loadNodesFromDownloadedPackages(): Promise<void> {
-		const nodePackages = [];
-		try {
-			// Read downloaded nodes and credentials
-			const downloadedNodesDir = UserSettings.getUserN8nFolderDownloadedNodesPath();
-			const downloadedNodesDirModules = path.join(downloadedNodesDir, 'node_modules');
-			await fsAccess(downloadedNodesDirModules);
-			const downloadedPackages = await this.getN8nNodePackages(downloadedNodesDirModules);
-			nodePackages.push(...downloadedPackages);
-		} catch (error) {
-			// Folder does not exist so ignore and return
-			return;
-		}
-
-		for (const packagePath of nodePackages) {
 			try {
-				await this.runDirectoryLoader(PackageDirectoryLoader, packagePath);
+				await this.runDirectoryLoader(
+					LazyPackageDirectoryLoader,
+					path.join(nodeModulesDir, packagePath),
+				);
 			} catch (error) {
 				ErrorReporter.error(error);
 			}
@@ -158,49 +154,45 @@ export class LoadNodesAndCredentials implements INodesAndCredentials {
 		}
 	}
 
-	/**
+	private async installOrUpdateNpmModule(
-	 * Returns all the names of the packages which could contain n8n nodes
+		packageName: string,
-	 */
+		options: { version?: string } | { installedPackage: InstalledPackages },
-	private async getN8nNodePackages(baseModulesPath: string): Promise<string[]> {
+	) {
-		const getN8nNodePackagesRecursive = async (relativePath: string): Promise<string[]> => {
+		const isUpdate = 'installedPackage' in options;
-			const results: string[] = [];
+		const command = isUpdate
-			const nodeModulesPath = `${baseModulesPath}/${relativePath}`;
+			? `npm update ${packageName}`
-			const nodeModules = await fsReaddir(nodeModulesPath);
+			: `npm install ${packageName}${options.version ? `@${options.version}` : ''}`;
-			for (const nodeModule of nodeModules) {
+
-				const isN8nNodesPackage = nodeModule.indexOf('n8n-nodes-') === 0;
+		try {
-				const isNpmScopedPackage = nodeModule.indexOf('@') === 0;
+			await executeCommand(command);
-				if (!isN8nNodesPackage && !isNpmScopedPackage) {
+		} catch (error) {
-					continue;
+			if (error instanceof Error && error.message === RESPONSE_ERROR_MESSAGES.PACKAGE_NOT_FOUND) {
-				}
+				throw new Error(`The npm package "${packageName}" could not be found.`);
-				if (!(await fsStat(nodeModulesPath)).isDirectory()) {
-					continue;
-				}
-				if (isN8nNodesPackage) {
-					results.push(`${baseModulesPath}/${relativePath}${nodeModule}`);
-				}
-				if (isNpmScopedPackage) {
-					results.push(...(await getN8nNodePackagesRecursive(`${relativePath}${nodeModule}/`)));
-				}
 			}
-			return results;
+			throw error;
-		};
+		}
-		return getN8nNodePackagesRecursive('');
-	}
 
-	async loadNpmModule(packageName: string, version?: string): Promise<InstalledPackages> {
+		const finalNodeUnpackedPath = path.join(this.downloadFolder, 'node_modules', packageName);
-		const downloadFolder = UserSettings.getUserN8nFolderDownloadedNodesPath();
-		const command = `npm install ${packageName}${version ? `@${version}` : ''}`;
 
-		await executeCommand(command);
+		let loader: PackageDirectoryLoader;
-
+		try {
-		const finalNodeUnpackedPath = path.join(downloadFolder, 'node_modules', packageName);
+			loader = await this.runDirectoryLoader(PackageDirectoryLoader, finalNodeUnpackedPath);
-
+		} catch (error) {
-		const loader = await this.runDirectoryLoader(PackageDirectoryLoader, finalNodeUnpackedPath);
+			// Remove this package since loading it failed
+			const removeCommand = `npm remove ${packageName}`;
+			try {
+				await executeCommand(removeCommand);
+			} catch {}
+			throw new Error(RESPONSE_ERROR_MESSAGES.PACKAGE_LOADING_FAILED, { cause: error });
+		}
 
 		if (loader.loadedNodes.length > 0) {
 			// Save info to DB
 			try {
-				const { persistInstalledPackageData } = await import('@/CommunityNodes/packageModel');
+				const { persistInstalledPackageData, removePackageFromDatabase } = await import(
+					'@/CommunityNodes/packageModel'
+				);
+				if (isUpdate) await removePackageFromDatabase(options.installedPackage);
 				const installedPackage = await persistInstalledPackageData(loader);
 				await this.postProcessLoaders();
 				await this.generateTypesForFrontend();
@@ -223,6 +215,10 @@ export class LoadNodesAndCredentials implements INodesAndCredentials {
 		}
 	}
 
+	async installNpmModule(packageName: string, version?: string): Promise<InstalledPackages> {
+		return this.installOrUpdateNpmModule(packageName, { version });
+	}
+
 	async removeNpmModule(packageName: string, installedPackage: InstalledPackages): Promise<void> {
 		const command = `npm remove ${packageName}`;
 
@@ -244,49 +240,7 @@ export class LoadNodesAndCredentials implements INodesAndCredentials {
 		packageName: string,
 		installedPackage: InstalledPackages,
 	): Promise<InstalledPackages> {
-		const downloadFolder = UserSettings.getUserN8nFolderDownloadedNodesPath();
+		return this.installOrUpdateNpmModule(packageName, { installedPackage });
-
-		const command = `npm i ${packageName}@latest`;
-
-		try {
-			await executeCommand(command);
-		} catch (error) {
-			if (error instanceof Error && error.message === RESPONSE_ERROR_MESSAGES.PACKAGE_NOT_FOUND) {
-				throw new Error(`The npm package "${packageName}" could not be found.`);
-			}
-			throw error;
-		}
-
-		const finalNodeUnpackedPath = path.join(downloadFolder, 'node_modules', packageName);
-
-		const loader = await this.runDirectoryLoader(PackageDirectoryLoader, finalNodeUnpackedPath);
-
-		if (loader.loadedNodes.length > 0) {
-			// Save info to DB
-			try {
-				const { persistInstalledPackageData, removePackageFromDatabase } = await import(
-					'@/CommunityNodes/packageModel'
-				);
-				await removePackageFromDatabase(installedPackage);
-				const newlyInstalledPackage = await persistInstalledPackageData(loader);
-				await this.postProcessLoaders();
-				await this.generateTypesForFrontend();
-				return newlyInstalledPackage;
-			} catch (error) {
-				LoggerProxy.error('Failed to save installed packages and nodes', {
-					error: error as Error,
-					packageName,
-				});
-				throw error;
-			}
-		} else {
-			// Remove this package since it contains no loadable nodes
-			const removeCommand = `npm remove ${packageName}`;
-			try {
-				await executeCommand(removeCommand);
-			} catch {}
-			throw new Error(RESPONSE_ERROR_MESSAGES.PACKAGE_DOES_NOT_CONTAIN_NODES);
-		}
 	}
 
 	/**
@@ -399,27 +353,4 @@ export class LoadNodesAndCredentials implements INodesAndCredentials {
 			}
 		}
 	}
-
-	private async getNodeModulesPath(): Promise<string> {
-		// Get the path to the node-modules folder to be later able
-		// to load the credentials and nodes
-		const checkPaths = [
-			// In case "n8n" package is in same node_modules folder.
-			path.join(CLI_DIR, '..', 'n8n-workflow'),
-			// In case "n8n" package is the root and the packages are
-			// in the "node_modules" folder underneath it.
-			path.join(CLI_DIR, 'node_modules', 'n8n-workflow'),
-			// In case "n8n" package is installed using npm/yarn workspaces
-			// the node_modules folder is in the root of the workspace.
-			path.join(CLI_DIR, '..', '..', 'node_modules', 'n8n-workflow'),
-		];
-		for (const checkPath of checkPaths) {
-			try {
-				await fsAccess(checkPath);
-				// Folder exists, so use it.
-				return path.dirname(checkPath);
-			} catch {} // Folder does not exist so get next one
-		}
-		throw new Error('Could not find "node_modules" folder!');
-	}
 }

packages/cli/src/commands/start.ts

@@ -267,11 +267,10 @@ export class Start extends BaseCommand {
 					// Optimistic approach - stop if any installation fails
 					// eslint-disable-next-line no-restricted-syntax
 					for (const missingPackage of missingPackages) {
-						// eslint-disable-next-line no-await-in-loop
+						await this.loadNodesAndCredentials.installNpmModule(
-						void (await this.loadNodesAndCredentials.loadNpmModule(
 							missingPackage.packageName,
 							missingPackage.version,
-						));
+						);
 						missingPackages.delete(missingPackage);
 					}
 					LoggerProxy.info('Packages reinstalled successfully. Resuming regular initialization.');

packages/cli/src/constants.ts

@@ -18,7 +18,7 @@ export const CUSTOM_API_CALL_KEY = '__CUSTOM_API_CALL__';
 
 export const CLI_DIR = resolve(__dirname, '..');
 export const TEMPLATES_DIR = join(CLI_DIR, 'templates');
-export const NODES_BASE_DIR = join(CLI_DIR, '..', 'nodes-base');
+export const NODES_BASE_DIR = dirname(require.resolve('n8n-nodes-base'));
 export const GENERATED_STATIC_DIR = join(UserSettings.getUserHome(), '.cache/n8n/public');
 export const EDITOR_UI_DIST_DIR = join(dirname(require.resolve('n8n-editor-ui')), 'dist');
 
@@ -45,6 +45,7 @@ export const RESPONSE_ERROR_MESSAGES = {
 	PACKAGE_NOT_FOUND: 'Package not found in npm',
 	PACKAGE_VERSION_NOT_FOUND: 'The specified package version was not found',
 	PACKAGE_DOES_NOT_CONTAIN_NODES: 'The specified package does not contain any nodes',
+	PACKAGE_LOADING_FAILED: 'The specified package could not be loaded',
 	DISK_IS_FULL: 'There appears to be insufficient disk space',
 };
 

packages/cli/src/controllers/nodes.controller.ts

@@ -109,7 +109,7 @@ export class NodesController {
 
 		let installedPackage: InstalledPackages;
 		try {
-			installedPackage = await this.loadNodesAndCredentials.loadNpmModule(
+			installedPackage = await this.loadNodesAndCredentials.installNpmModule(
 				parsed.packageName,
 				parsed.version,
 			);
@@ -125,7 +125,10 @@ export class NodesController {
 				failure_reason: errorMessage,
 			});
 
-			const message = [`Error loading package "${name}"`, errorMessage].join(':');
+			let message = [`Error loading package "${name}" `, errorMessage].join(':');
+			if (error instanceof Error && error.cause instanceof Error) {
+				message += `\nCause: ${error.cause.message}`;
+			}
 
 			const clientError = error instanceof Error ? isClientError(error) : false;
 			throw new (clientError ? BadRequestError : InternalServerError)(message);

packages/cli/test/integration/nodes.api.test.ts

@@ -191,7 +191,7 @@ describe('POST /nodes', () => {
 		mocked(hasPackageLoaded).mockReturnValueOnce(false);
 		mocked(checkNpmPackageStatus).mockResolvedValueOnce({ status: 'OK' });
 
-		mockLoadNodesAndCredentials.loadNpmModule.mockImplementationOnce(mockedEmptyPackage);
+		mockLoadNodesAndCredentials.installNpmModule.mockImplementationOnce(mockedEmptyPackage);
 
 		const { statusCode } = await authOwnerShellAgent.post('/nodes').send({
 			name: utils.installedPackagePayload().packageName,

packages/nodes-base/.eslintrc.js

@@ -8,6 +8,8 @@ module.exports = {
 
 	...sharedOptions(__dirname),
 
+	ignorePatterns: ['index.js'],
+
 	rules: {
 		'@typescript-eslint/consistent-type-imports': 'error',
 

packages/nodes-base/package.json

@@ -8,6 +8,7 @@
     "name": "Jan Oberhauser",
     "email": "jan@n8n.io"
   },
+  "main": "index.js",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/n8n-io/n8n.git"