n8n/packages/cli/src/load-nodes-and-credentials.ts

import { GlobalConfig } from '@n8n/config';
import glob from 'fast-glob';
import fsPromises from 'fs/promises';
import type { Class, DirectoryLoader, Types } from 'n8n-core';
import {
	CUSTOM_EXTENSION_ENV,
	ErrorReporter,
	InstanceSettings,
	CustomDirectoryLoader,
	PackageDirectoryLoader,
	LazyPackageDirectoryLoader,
	UnrecognizedCredentialTypeError,
	UnrecognizedNodeTypeError,
} from 'n8n-core';
import type {
	KnownNodesAndCredentials,
	INodeTypeBaseDescription,
	INodeTypeDescription,
	INodeTypeData,
	ICredentialTypeData,
	LoadedClass,
	ICredentialType,
	INodeType,
	IVersionedNodeType,
	INodeProperties,
} from 'n8n-workflow';
import { ApplicationError, NodeConnectionType } from 'n8n-workflow';
import path from 'path';
import picocolors from 'picocolors';
import { Container, Service } from 'typedi';

import {
	CUSTOM_API_CALL_KEY,
	CUSTOM_API_CALL_NAME,
	inTest,
	CLI_DIR,
	inE2ETests,
} from '@/constants';
import { Logger } from '@/logging/logger.service';
import { isContainedWithin } from '@/utils/path-util';

interface LoadedNodesAndCredentials {
	nodes: INodeTypeData;
	credentials: ICredentialTypeData;
}

@Service()
export class LoadNodesAndCredentials {
	private known: KnownNodesAndCredentials = { nodes: {}, credentials: {} };

	// This contains the actually loaded objects, and their source paths
	loaded: LoadedNodesAndCredentials = { nodes: {}, credentials: {} };

	// For nodes, this only contains the descriptions, loaded from either the
	// actual file, or the lazy loaded json
	types: Types = { nodes: [], credentials: [] };

	loaders: Record<string, DirectoryLoader> = {};

	excludeNodes = this.globalConfig.nodes.exclude;

	includeNodes = this.globalConfig.nodes.include;

	private postProcessors: Array<() => Promise<void>> = [];

	constructor(
		private readonly logger: Logger,
		private readonly errorReporter: ErrorReporter,
		private readonly instanceSettings: InstanceSettings,
		private readonly globalConfig: GlobalConfig,
	) {}

	async init() {
		if (inTest) throw new ApplicationError('Not available in tests');

		// Make sure the imported modules can resolve dependencies fine.
		const delimiter = process.platform === 'win32' ? ';' : ':';
		process.env.NODE_PATH = module.paths.join(delimiter);

		// @ts-ignore
		// eslint-disable-next-line @typescript-eslint/no-unsafe-call
		module.constructor._initPaths();

		if (!inE2ETests) {
			this.excludeNodes = this.excludeNodes ?? [];
			this.excludeNodes.push('n8n-nodes-base.e2eTest');
		}

		// Load nodes from `n8n-nodes-base`
		const basePathsToScan = [
			// In case "n8n" package is in same node_modules folder.
			path.join(CLI_DIR, '..'),
			// In case "n8n" package is the root and the packages are
			// in the "node_modules" folder underneath it.
			path.join(CLI_DIR, 'node_modules'),
		];

		for (const nodeModulesDir of basePathsToScan) {
			await this.loadNodesFromNodeModules(nodeModulesDir, 'n8n-nodes-base');
			await this.loadNodesFromNodeModules(nodeModulesDir, '@n8n/n8n-nodes-langchain');
		}

		// Load nodes from any other `n8n-nodes-*` packages in the download directory
		// This includes the community nodes
		await this.loadNodesFromNodeModules(
			path.join(this.instanceSettings.nodesDownloadDir, 'node_modules'),
		);

		await this.loadNodesFromCustomDirectories();
		await this.postProcessLoaders();
	}

	addPostProcessor(fn: () => Promise<void>) {
		this.postProcessors.push(fn);
	}

	isKnownNode(type: string) {
		return type in this.known.nodes;
	}

	get loadedCredentials() {
		return this.loaded.credentials;
	}

	get loadedNodes() {
		return this.loaded.nodes;
	}

	get knownCredentials() {
		return this.known.credentials;
	}

	get knownNodes() {
		return this.known.nodes;
	}

	private async loadNodesFromNodeModules(
		nodeModulesDir: string,
		packageName?: string,
	): Promise<void> {
		const globOptions = {
			cwd: nodeModulesDir,
			onlyDirectories: true,
			deep: 1,
		};
		const installedPackagePaths = packageName
			? await glob(packageName, globOptions)
			: [
					...(await glob('n8n-nodes-*', globOptions)),
					...(await glob('@*/n8n-nodes-*', { ...globOptions, deep: 2 })),
				];

		for (const packagePath of installedPackagePaths) {
			try {
				await this.runDirectoryLoader(
					LazyPackageDirectoryLoader,
					path.join(nodeModulesDir, packagePath),
				);
			} catch (error) {
				this.logger.error((error as Error).message);
				this.errorReporter.error(error);
			}
		}
	}

	resolveIcon(packageName: string, url: string): string | undefined {
		const loader = this.loaders[packageName];
		if (!loader) {
			return undefined;
		}
		const pathPrefix = `/icons/${packageName}/`;
		const filePath = path.resolve(loader.directory, url.substring(pathPrefix.length));

		return isContainedWithin(loader.directory, filePath) ? filePath : undefined;
	}

	getCustomDirectories(): string[] {
		const customDirectories = [this.instanceSettings.customExtensionDir];

		if (process.env[CUSTOM_EXTENSION_ENV] !== undefined) {
			const customExtensionFolders = process.env[CUSTOM_EXTENSION_ENV].split(';');
			customDirectories.push(...customExtensionFolders);
		}

		return customDirectories;
	}

	private async loadNodesFromCustomDirectories(): Promise<void> {
		for (const directory of this.getCustomDirectories()) {
			await this.runDirectoryLoader(CustomDirectoryLoader, directory);
		}
	}

	async loadPackage(packageName: string) {
		const finalNodeUnpackedPath = path.join(
			this.instanceSettings.nodesDownloadDir,
			'node_modules',
			packageName,
		);
		return await this.runDirectoryLoader(PackageDirectoryLoader, finalNodeUnpackedPath);
	}

	async unloadPackage(packageName: string) {
		if (packageName in this.loaders) {
			this.loaders[packageName].reset();
			delete this.loaders[packageName];
		}
	}

	/**
	 * Whether any of the node's credential types may be used to
	 * make a request from a node other than itself.
	 */
	private supportsProxyAuth(description: INodeTypeDescription) {
		if (!description.credentials) return false;

		return description.credentials.some(({ name }) => {
			const credType = this.types.credentials.find((t) => t.name === name);
			if (!credType) {
				this.logger.warn(
					`Failed to load Custom API options for the node "${description.name}": Unknown credential name "${name}"`,
				);
				return false;
			}
			if (credType.authenticate !== undefined) return true;

			return (
				Array.isArray(credType.extends) &&
				credType.extends.some((parentType) =>
					['oAuth2Api', 'googleOAuth2Api', 'oAuth1Api'].includes(parentType),
				)
			);
		});
	}

	/**
	 * Inject a `Custom API Call` option into `resource` and `operation`
	 * parameters in a latest-version node that supports proxy auth.
	 */
	private injectCustomApiCallOptions() {
		this.types.nodes.forEach((node: INodeTypeDescription) => {
			const isLatestVersion =
				node.defaultVersion === undefined || node.defaultVersion === node.version;

			if (isLatestVersion) {
				if (!this.supportsProxyAuth(node)) return;

				node.properties.forEach((p) => {
					if (
						['resource', 'operation'].includes(p.name) &&
						Array.isArray(p.options) &&
						p.options[p.options.length - 1].name !== CUSTOM_API_CALL_NAME
					) {
						p.options.push({
							name: CUSTOM_API_CALL_NAME,
							value: CUSTOM_API_CALL_KEY,
						});
					}
				});
			}
		});
	}

	/**
	 * Run a loader of source files of nodes and credentials in a directory.
	 */
	private async runDirectoryLoader<T extends DirectoryLoader>(
		constructor: Class<T, ConstructorParameters<typeof DirectoryLoader>>,
		dir: string,
	) {
		const loader = new constructor(dir, this.excludeNodes, this.includeNodes);
		if (loader instanceof PackageDirectoryLoader && loader.packageName in this.loaders) {
			throw new ApplicationError(
				picocolors.red(
					`nodes package ${loader.packageName} is already loaded.\n Please delete this second copy at path ${dir}`,
				),
			);
		}
		await loader.loadAll();
		this.loaders[loader.packageName] = loader;
		return loader;
	}

	/**
	 * This creates all AI Agent tools by duplicating the node descriptions for
	 * all nodes that are marked as `usableAsTool`. It basically modifies the
	 * description. The actual wrapping happens in the langchain code for getting
	 * the connected tools.
	 */
	createAiTools() {
		const usableNodes: Array<INodeTypeBaseDescription | INodeTypeDescription> =
			this.types.nodes.filter((nodetype) => nodetype.usableAsTool === true);

		for (const usableNode of usableNodes) {
			const description: INodeTypeBaseDescription | INodeTypeDescription =
				structuredClone(usableNode);
			const wrapped = this.convertNodeToAiTool({ description }).description;

			this.types.nodes.push(wrapped);
			this.known.nodes[wrapped.name] = structuredClone(this.known.nodes[usableNode.name]);

			const credentialNames = Object.entries(this.known.credentials)
				.filter(([_, credential]) => credential?.supportedNodes?.includes(usableNode.name))
				.map(([credentialName]) => credentialName);

			credentialNames.forEach((name) =>
				this.known.credentials[name]?.supportedNodes?.push(wrapped.name),
			);
		}
	}

	async postProcessLoaders() {
		this.known = { nodes: {}, credentials: {} };
		this.loaded = { nodes: {}, credentials: {} };
		this.types = { nodes: [], credentials: [] };

		for (const loader of Object.values(this.loaders)) {
			// list of node & credential types that will be sent to the frontend
			const { known, types, directory, packageName } = loader;
			this.types.nodes = this.types.nodes.concat(
				types.nodes.map(({ name, ...rest }) => ({
					...rest,
					name: `${packageName}.${name}`,
				})),
			);
			this.types.credentials = this.types.credentials.concat(types.credentials);

			// Nodes and credentials that have been loaded immediately
			for (const nodeTypeName in loader.nodeTypes) {
				this.loaded.nodes[`${packageName}.${nodeTypeName}`] = loader.nodeTypes[nodeTypeName];
			}

			for (const credentialTypeName in loader.credentialTypes) {
				this.loaded.credentials[credentialTypeName] = loader.credentialTypes[credentialTypeName];
			}

			for (const type in known.nodes) {
				const { className, sourcePath } = known.nodes[type];
				this.known.nodes[`${packageName}.${type}`] = {
					className,
					sourcePath: path.join(directory, sourcePath),
				};
			}

			for (const type in known.credentials) {
				const {
					className,
					sourcePath,
					supportedNodes,
					extends: extendsArr,
				} = known.credentials[type];
				this.known.credentials[type] = {
					className,
					sourcePath: path.join(directory, sourcePath),
					supportedNodes:
						loader instanceof PackageDirectoryLoader
							? supportedNodes?.map((nodeName) => `${loader.packageName}.${nodeName}`)
							: undefined,
					extends: extendsArr,
				};
			}
		}

		this.createAiTools();

		this.injectCustomApiCallOptions();

		for (const postProcessor of this.postProcessors) {
			await postProcessor();
		}
	}

	getNode(fullNodeType: string): LoadedClass<INodeType | IVersionedNodeType> {
		const [packageName, nodeType] = fullNodeType.split('.');
		const { loaders } = this;
		const loader = loaders[packageName];
		if (!loader) {
			throw new UnrecognizedNodeTypeError(packageName, nodeType);
		}
		return loader.getNode(nodeType);
	}

	getCredential(credentialType: string): LoadedClass<ICredentialType> {
		const { loadedCredentials } = this;

		for (const loader of Object.values(this.loaders)) {
			if (credentialType in loader.known.credentials) {
				const loaded = loader.getCredential(credentialType);
				loadedCredentials[credentialType] = loaded;
			}
		}

		if (credentialType in loadedCredentials) {
			return loadedCredentials[credentialType];
		}

		throw new UnrecognizedCredentialTypeError(credentialType);
	}

	/**
	 * Modifies the description of the passed in object, such that it can be used
	 * as an AI Agent Tool.
	 * Returns the modified item (not copied)
	 */
	convertNodeToAiTool<
		T extends object & { description: INodeTypeDescription | INodeTypeBaseDescription },
	>(item: T): T {
		// quick helper function for type-guard down below
		function isFullDescription(obj: unknown): obj is INodeTypeDescription {
			return typeof obj === 'object' && obj !== null && 'properties' in obj;
		}

		if (isFullDescription(item.description)) {
			item.description.name += 'Tool';
			item.description.inputs = [];
			item.description.outputs = [NodeConnectionType.AiTool];
			item.description.displayName += ' Tool';
			delete item.description.usableAsTool;

			const hasResource = item.description.properties.some((prop) => prop.name === 'resource');
			const hasOperation = item.description.properties.some((prop) => prop.name === 'operation');

			if (!item.description.properties.map((prop) => prop.name).includes('toolDescription')) {
				const descriptionType: INodeProperties = {
					displayName: 'Tool Description',
					name: 'descriptionType',
					type: 'options',
					noDataExpression: true,
					options: [
						{
							name: 'Set Automatically',
							value: 'auto',
							description: 'Automatically set based on resource and operation',
						},
						{
							name: 'Set Manually',
							value: 'manual',
							description: 'Manually set the description',
						},
					],
					default: 'auto',
				};

				const descProp: INodeProperties = {
					displayName: 'Description',
					name: 'toolDescription',
					type: 'string',
					default: item.description.description,
					required: true,
					typeOptions: { rows: 2 },
					description:
						'Explain to the LLM what this tool does, a good, specific description would allow LLMs to produce expected results much more often',
					placeholder: `e.g. ${item.description.description}`,
				};

				const noticeProp: INodeProperties = {
					displayName:
						"Use the expression {{ $fromAI('placeholder_name') }} for any data to be filled by the model",
					name: 'notice',
					type: 'notice',
					default: '',
				};

				item.description.properties.unshift(descProp);

				// If node has resource or operation we can determine pre-populate tool description based on it
				// so we add the descriptionType property as the first property
				if (hasResource || hasOperation) {
					item.description.properties.unshift(descriptionType);

					descProp.displayOptions = {
						show: {
							descriptionType: ['manual'],
						},
					};
				}

				item.description.properties.unshift(noticeProp);
			}
		}

		const resources = item.description.codex?.resources ?? {};

		item.description.codex = {
			categories: ['AI'],
			subcategories: {
				AI: ['Tools'],
				Tools: ['Other Tools'],
			},
			resources,
		};
		return item;
	}

	async setupHotReload() {
		const { default: debounce } = await import('lodash/debounce');
		// eslint-disable-next-line import/no-extraneous-dependencies
		const { watch } = await import('chokidar');

		const { Push } = await import('@/push');
		const push = Container.get(Push);

		Object.values(this.loaders).forEach(async (loader) => {
			try {
				await fsPromises.access(loader.directory);
			} catch {
				// If directory doesn't exist, there is nothing to watch
				return;
			}

			const realModulePath = path.join(await fsPromises.realpath(loader.directory), path.sep);
			const reloader = debounce(async () => {
				const modulesToUnload = Object.keys(require.cache).filter((filePath) =>
					filePath.startsWith(realModulePath),
				);
				modulesToUnload.forEach((filePath) => {
					delete require.cache[filePath];
				});

				loader.reset();
				await loader.loadAll();
				await this.postProcessLoaders();
				push.broadcast('nodeDescriptionUpdated', {});
			}, 100);

			const toWatch = loader.isLazyLoaded
				? ['**/nodes.json', '**/credentials.json']
				: ['**/*.js', '**/*.json'];
			const files = await glob(toWatch, {
				cwd: realModulePath,
				ignore: ['node_modules/**'],
			});
			const watcher = watch(files, {
				cwd: realModulePath,
				ignoreInitial: true,
			});
			watcher.on('add', reloader).on('change', reloader).on('unlink', reloader);
		});
	}
}