import { GlobalConfig } from '@n8n/config'; import glob from 'fast-glob'; import fsPromises from 'fs/promises'; import type { Class, DirectoryLoader, Types } from 'n8n-core'; import { CUSTOM_EXTENSION_ENV, InstanceSettings, CustomDirectoryLoader, PackageDirectoryLoader, LazyPackageDirectoryLoader, } from 'n8n-core'; import type { KnownNodesAndCredentials, INodeTypeBaseDescription, INodeTypeDescription, INodeTypeData, ICredentialTypeData, } from 'n8n-workflow'; import { NodeHelpers, ApplicationError, ErrorReporterProxy as ErrorReporter } from 'n8n-workflow'; import path from 'path'; import picocolors from 'picocolors'; import { Container, Service } from 'typedi'; import { CUSTOM_API_CALL_KEY, CUSTOM_API_CALL_NAME, inTest, CLI_DIR, inE2ETests, } from '@/constants'; import { Logger } from '@/logging/logger.service'; import { isContainedWithin } from '@/utils/path-util'; interface LoadedNodesAndCredentials { nodes: INodeTypeData; credentials: ICredentialTypeData; } @Service() export class LoadNodesAndCredentials { private known: KnownNodesAndCredentials = { nodes: {}, credentials: {} }; // This contains the actually loaded objects, and their source paths loaded: LoadedNodesAndCredentials = { nodes: {}, credentials: {} }; // For nodes, this only contains the descriptions, loaded from either the // actual file, or the lazy loaded json types: Types = { nodes: [], credentials: [] }; loaders: Record = {}; excludeNodes = this.globalConfig.nodes.exclude; includeNodes = this.globalConfig.nodes.include; private postProcessors: Array<() => Promise> = []; constructor( private readonly logger: Logger, private readonly instanceSettings: InstanceSettings, private readonly globalConfig: GlobalConfig, ) {} async init() { if (inTest) throw new ApplicationError('Not available in tests'); // Make sure the imported modules can resolve dependencies fine. const delimiter = process.platform === 'win32' ? ';' : ':'; process.env.NODE_PATH = module.paths.join(delimiter); // @ts-ignore // eslint-disable-next-line @typescript-eslint/no-unsafe-call module.constructor._initPaths(); if (!inE2ETests) { this.excludeNodes = this.excludeNodes ?? []; this.excludeNodes.push('n8n-nodes-base.e2eTest'); } // Load nodes from `n8n-nodes-base` const basePathsToScan = [ // In case "n8n" package is in same node_modules folder. path.join(CLI_DIR, '..'), // In case "n8n" package is the root and the packages are // in the "node_modules" folder underneath it. path.join(CLI_DIR, 'node_modules'), ]; for (const nodeModulesDir of basePathsToScan) { await this.loadNodesFromNodeModules(nodeModulesDir, 'n8n-nodes-base'); await this.loadNodesFromNodeModules(nodeModulesDir, '@n8n/n8n-nodes-langchain'); } // Load nodes from any other `n8n-nodes-*` packages in the download directory // This includes the community nodes await this.loadNodesFromNodeModules( path.join(this.instanceSettings.nodesDownloadDir, 'node_modules'), ); await this.loadNodesFromCustomDirectories(); await this.postProcessLoaders(); } addPostProcessor(fn: () => Promise) { this.postProcessors.push(fn); } isKnownNode(type: string) { return type in this.known.nodes; } get loadedCredentials() { return this.loaded.credentials; } get loadedNodes() { return this.loaded.nodes; } get knownCredentials() { return this.known.credentials; } get knownNodes() { return this.known.nodes; } private async loadNodesFromNodeModules( nodeModulesDir: string, packageName?: string, ): Promise { const globOptions = { cwd: nodeModulesDir, onlyDirectories: true, deep: 1, }; const installedPackagePaths = packageName ? await glob(packageName, globOptions) : [ ...(await glob('n8n-nodes-*', globOptions)), ...(await glob('@*/n8n-nodes-*', { ...globOptions, deep: 2 })), ]; for (const packagePath of installedPackagePaths) { try { await this.runDirectoryLoader( LazyPackageDirectoryLoader, path.join(nodeModulesDir, packagePath), ); } catch (error) { this.logger.error((error as Error).message); ErrorReporter.error(error); } } } resolveIcon(packageName: string, url: string): string | undefined { const loader = this.loaders[packageName]; if (!loader) { return undefined; } const pathPrefix = `/icons/${packageName}/`; const filePath = path.resolve(loader.directory, url.substring(pathPrefix.length)); return isContainedWithin(loader.directory, filePath) ? filePath : undefined; } getCustomDirectories(): string[] { const customDirectories = [this.instanceSettings.customExtensionDir]; if (process.env[CUSTOM_EXTENSION_ENV] !== undefined) { const customExtensionFolders = process.env[CUSTOM_EXTENSION_ENV].split(';'); customDirectories.push(...customExtensionFolders); } return customDirectories; } private async loadNodesFromCustomDirectories(): Promise { for (const directory of this.getCustomDirectories()) { await this.runDirectoryLoader(CustomDirectoryLoader, directory); } } async loadPackage(packageName: string) { const finalNodeUnpackedPath = path.join( this.instanceSettings.nodesDownloadDir, 'node_modules', packageName, ); return await this.runDirectoryLoader(PackageDirectoryLoader, finalNodeUnpackedPath); } async unloadPackage(packageName: string) { if (packageName in this.loaders) { this.loaders[packageName].reset(); delete this.loaders[packageName]; } } /** * Whether any of the node's credential types may be used to * make a request from a node other than itself. */ private supportsProxyAuth(description: INodeTypeDescription) { if (!description.credentials) return false; return description.credentials.some(({ name }) => { const credType = this.types.credentials.find((t) => t.name === name); if (!credType) { this.logger.warn( `Failed to load Custom API options for the node "${description.name}": Unknown credential name "${name}"`, ); return false; } if (credType.authenticate !== undefined) return true; return ( Array.isArray(credType.extends) && credType.extends.some((parentType) => ['oAuth2Api', 'googleOAuth2Api', 'oAuth1Api'].includes(parentType), ) ); }); } /** * Inject a `Custom API Call` option into `resource` and `operation` * parameters in a latest-version node that supports proxy auth. */ private injectCustomApiCallOptions() { this.types.nodes.forEach((node: INodeTypeDescription) => { const isLatestVersion = node.defaultVersion === undefined || node.defaultVersion === node.version; if (isLatestVersion) { if (!this.supportsProxyAuth(node)) return; node.properties.forEach((p) => { if ( ['resource', 'operation'].includes(p.name) && Array.isArray(p.options) && p.options[p.options.length - 1].name !== CUSTOM_API_CALL_NAME ) { p.options.push({ name: CUSTOM_API_CALL_NAME, value: CUSTOM_API_CALL_KEY, }); } }); } }); } /** * Run a loader of source files of nodes and credentials in a directory. */ private async runDirectoryLoader( constructor: Class>, dir: string, ) { const loader = new constructor(dir, this.excludeNodes, this.includeNodes); if (loader instanceof PackageDirectoryLoader && loader.packageName in this.loaders) { throw new ApplicationError( picocolors.red( `nodes package ${loader.packageName} is already loaded.\n Please delete this second copy at path ${dir}`, ), ); } await loader.loadAll(); this.loaders[loader.packageName] = loader; return loader; } /** * This creates all AI Agent tools by duplicating the node descriptions for * all nodes that are marked as `usableAsTool`. It basically modifies the * description. The actual wrapping happens in the langchain code for getting * the connected tools. */ createAiTools() { const usableNodes: Array = this.types.nodes.filter((nodetype) => nodetype.usableAsTool === true); for (const usableNode of usableNodes) { const description: INodeTypeBaseDescription | INodeTypeDescription = structuredClone(usableNode); const wrapped = NodeHelpers.convertNodeToAiTool({ description }).description; this.types.nodes.push(wrapped); this.known.nodes[wrapped.name] = structuredClone(this.known.nodes[usableNode.name]); const credentialNames = Object.entries(this.known.credentials) .filter(([_, credential]) => credential?.supportedNodes?.includes(usableNode.name)) .map(([credentialName]) => credentialName); credentialNames.forEach((name) => this.known.credentials[name]?.supportedNodes?.push(wrapped.name), ); } } async postProcessLoaders() { this.known = { nodes: {}, credentials: {} }; this.loaded = { nodes: {}, credentials: {} }; this.types = { nodes: [], credentials: [] }; for (const loader of Object.values(this.loaders)) { // list of node & credential types that will be sent to the frontend const { known, types, directory } = loader; this.types.nodes = this.types.nodes.concat(types.nodes); this.types.credentials = this.types.credentials.concat(types.credentials); // Nodes and credentials that have been loaded immediately for (const nodeTypeName in loader.nodeTypes) { this.loaded.nodes[nodeTypeName] = loader.nodeTypes[nodeTypeName]; } for (const credentialTypeName in loader.credentialTypes) { this.loaded.credentials[credentialTypeName] = loader.credentialTypes[credentialTypeName]; } for (const type in known.nodes) { const { className, sourcePath } = known.nodes[type]; this.known.nodes[type] = { className, sourcePath: path.join(directory, sourcePath), }; } for (const type in known.credentials) { const { className, sourcePath, supportedNodes, extends: extendsArr, } = known.credentials[type]; this.known.credentials[type] = { className, sourcePath: path.join(directory, sourcePath), supportedNodes: loader instanceof PackageDirectoryLoader ? supportedNodes?.map((nodeName) => `${loader.packageName}.${nodeName}`) : undefined, extends: extendsArr, }; } } this.createAiTools(); this.injectCustomApiCallOptions(); for (const postProcessor of this.postProcessors) { await postProcessor(); } } async setupHotReload() { const { default: debounce } = await import('lodash/debounce'); // eslint-disable-next-line import/no-extraneous-dependencies const { watch } = await import('chokidar'); const { Push } = await import('@/push'); const push = Container.get(Push); Object.values(this.loaders).forEach(async (loader) => { try { await fsPromises.access(loader.directory); } catch { // If directory doesn't exist, there is nothing to watch return; } const realModulePath = path.join(await fsPromises.realpath(loader.directory), path.sep); const reloader = debounce(async () => { const modulesToUnload = Object.keys(require.cache).filter((filePath) => filePath.startsWith(realModulePath), ); modulesToUnload.forEach((filePath) => { delete require.cache[filePath]; }); loader.reset(); await loader.loadAll(); await this.postProcessLoaders(); push.broadcast('nodeDescriptionUpdated', {}); }, 100); const toWatch = loader.isLazyLoaded ? ['**/nodes.json', '**/credentials.json'] : ['**/*.js', '**/*.json']; watch(toWatch, { cwd: realModulePath }).on('change', reloader); }); } }