From 7553908348a3b3a878b6e1a05e6ee2472ca2183c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E0=A4=95=E0=A4=BE=E0=A4=B0=E0=A4=A4=E0=A5=8B=E0=A4=AB?= =?UTF-8?q?=E0=A5=8D=E0=A4=AB=E0=A5=87=E0=A4=B2=E0=A4=B8=E0=A5=8D=E0=A4=95?= =?UTF-8?q?=E0=A5=8D=E0=A4=B0=E0=A4=BF=E0=A4=AA=E0=A5=8D=E0=A4=9F=E2=84=A2?= Date: Fri, 8 Nov 2024 16:55:12 +0100 Subject: [PATCH] feat(core): Hot reload nodes on all servers --- packages/cli/src/commands/base-command.ts | 69 ++++++++++++++++++- packages/cli/src/commands/start.ts | 2 + packages/cli/src/commands/webhook.ts | 2 + packages/cli/src/commands/worker.ts | 2 + .../cli/src/load-nodes-and-credentials.ts | 49 +------------ packages/cli/src/server.ts | 4 -- 6 files changed, 74 insertions(+), 54 deletions(-) diff --git a/packages/cli/src/commands/base-command.ts b/packages/cli/src/commands/base-command.ts index 33b7a28bf3..858e75ab33 100644 --- a/packages/cli/src/commands/base-command.ts +++ b/packages/cli/src/commands/base-command.ts @@ -1,6 +1,8 @@ import 'reflect-metadata'; import { GlobalConfig } from '@n8n/config'; import { Command, Errors } from '@oclif/core'; +import glob from 'fast-glob'; +import { access as fsAccess, realpath as fsRealPath } from 'fs/promises'; import { BinaryDataService, InstanceSettings, @@ -8,7 +10,13 @@ import { DataDeduplicationService, ErrorReporter, } from 'n8n-core'; -import { ApplicationError, ensureError, sleep } from 'n8n-workflow'; +import { + ApplicationError, + ensureError, + sleep, +} from 'n8n-workflow'; +import path from 'path'; +import picocolors from 'picocolors'; import { Container } from 'typedi'; import type { AbstractServer } from '@/abstract-server'; @@ -40,6 +48,8 @@ export abstract class BaseCommand extends Command { protected nodeTypes: NodeTypes; + protected loadNodesAndCredentials: LoadNodesAndCredentials; + protected instanceSettings: InstanceSettings = Container.get(InstanceSettings); protected server?: AbstractServer; @@ -67,7 +77,8 @@ export abstract class BaseCommand extends Command { process.once('SIGINT', this.onTerminationSignal('SIGINT')); this.nodeTypes = Container.get(NodeTypes); - await Container.get(LoadNodesAndCredentials).init(); + this.loadNodesAndCredentials = Container.get(LoadNodesAndCredentials); + await this.loadNodesAndCredentials.init(); await Db.init().catch( async (error: Error) => await this.exitWithCrash('There was an error initializing DB', error), @@ -317,4 +328,58 @@ export abstract class BaseCommand extends Command { clearTimeout(forceShutdownTimer); }; } + + protected async setupHotReload() { + if (!inDevelopment || process.env.N8N_DEV_RELOAD !== 'true') return; + + const { default: debounce } = await import('lodash/debounce'); + // eslint-disable-next-line import/no-extraneous-dependencies + const { watch } = await import('chokidar'); + + const { Push } = await import('@/push'); + const push = Container.get(Push); + + Object.values(this.loadNodesAndCredentials.loaders).forEach(async (loader) => { + try { + await fsAccess(loader.directory); + } catch { + // If directory doesn't exist, there is nothing to watch + return; + } + + const realModulePath = path.join(await fsRealPath(loader.directory), path.sep); + const reloader = debounce(async (fileName: string) => { + console.info( + picocolors.green('тно Reloading'), + picocolors.bold(fileName), + 'in', + loader.packageName, + ); + const modulesToUnload = Object.keys(require.cache).filter((filePath) => + filePath.startsWith(realModulePath), + ); + modulesToUnload.forEach((filePath) => { + delete require.cache[filePath]; + }); + + loader.reset(); + await loader.loadAll(); + await this.loadNodesAndCredentials.postProcessLoaders(); + push.broadcast('nodeDescriptionUpdated', {}); + }, 100); + + const toWatch = loader.isLazyLoaded + ? ['**/nodes.json', '**/credentials.json'] + : ['**/*.js', '**/*.json']; + const files = await glob(toWatch, { + cwd: realModulePath, + ignore: ['node_modules/**'], + }); + const watcher = watch(files, { + cwd: realModulePath, + ignoreInitial: true, + }); + watcher.on('add', reloader).on('change', reloader).on('unlink', reloader); + }); + } } diff --git a/packages/cli/src/commands/start.ts b/packages/cli/src/commands/start.ts index 63ec3d9240..5fcd4042c1 100644 --- a/packages/cli/src/commands/start.ts +++ b/packages/cli/src/commands/start.ts @@ -359,6 +359,8 @@ export class Start extends BaseCommand { } }); } + + void this.setupHotReload(); } async catch(error: Error) { diff --git a/packages/cli/src/commands/webhook.ts b/packages/cli/src/commands/webhook.ts index 77ec770aa0..ab42239aa8 100644 --- a/packages/cli/src/commands/webhook.ts +++ b/packages/cli/src/commands/webhook.ts @@ -94,6 +94,8 @@ export class Webhook extends BaseCommand { await this.server.start(); this.logger.info('Webhook listener waiting for requests.'); + void this.setupHotReload(); + // Make sure that the process does not close await new Promise(() => {}); } diff --git a/packages/cli/src/commands/worker.ts b/packages/cli/src/commands/worker.ts index 64c5a34dae..859d80cd47 100644 --- a/packages/cli/src/commands/worker.ts +++ b/packages/cli/src/commands/worker.ts @@ -192,6 +192,8 @@ export class Worker extends BaseCommand { }); } + void this.setupHotReload(); + // Make sure that the process does not close if (!inTest) await new Promise(() => {}); } diff --git a/packages/cli/src/load-nodes-and-credentials.ts b/packages/cli/src/load-nodes-and-credentials.ts index 5a46d6c70d..0ef56b92dc 100644 --- a/packages/cli/src/load-nodes-and-credentials.ts +++ b/packages/cli/src/load-nodes-and-credentials.ts @@ -1,6 +1,5 @@ import { GlobalConfig } from '@n8n/config'; import glob from 'fast-glob'; -import fsPromises from 'fs/promises'; import type { Class, DirectoryLoader, Types } from 'n8n-core'; import { CUSTOM_EXTENSION_ENV, @@ -26,7 +25,7 @@ import type { import { NodeHelpers, ApplicationError } from 'n8n-workflow'; import path from 'path'; import picocolors from 'picocolors'; -import { Container, Service } from 'typedi'; +import { Service } from 'typedi'; import { CUSTOM_API_CALL_KEY, @@ -395,50 +394,4 @@ export class LoadNodesAndCredentials { throw new UnrecognizedCredentialTypeError(credentialType); } - - async setupHotReload() { - const { default: debounce } = await import('lodash/debounce'); - // eslint-disable-next-line import/no-extraneous-dependencies - const { watch } = await import('chokidar'); - - const { Push } = await import('@/push'); - const push = Container.get(Push); - - Object.values(this.loaders).forEach(async (loader) => { - try { - await fsPromises.access(loader.directory); - } catch { - // If directory doesn't exist, there is nothing to watch - return; - } - - const realModulePath = path.join(await fsPromises.realpath(loader.directory), path.sep); - const reloader = debounce(async () => { - const modulesToUnload = Object.keys(require.cache).filter((filePath) => - filePath.startsWith(realModulePath), - ); - modulesToUnload.forEach((filePath) => { - delete require.cache[filePath]; - }); - - loader.reset(); - await loader.loadAll(); - await this.postProcessLoaders(); - push.broadcast('nodeDescriptionUpdated', {}); - }, 100); - - const toWatch = loader.isLazyLoaded - ? ['**/nodes.json', '**/credentials.json'] - : ['**/*.js', '**/*.json']; - const files = await glob(toWatch, { - cwd: realModulePath, - ignore: ['node_modules/**'], - }); - const watcher = watch(files, { - cwd: realModulePath, - ignoreInitial: true, - }); - watcher.on('add', reloader).on('change', reloader).on('unlink', reloader); - }); - } } diff --git a/packages/cli/src/server.ts b/packages/cli/src/server.ts index 74a1311444..f24e97f391 100644 --- a/packages/cli/src/server.ts +++ b/packages/cli/src/server.ts @@ -101,10 +101,6 @@ export class Server extends AbstractServer { await super.start(); this.logger.debug(`Server ID: ${this.instanceSettings.hostId}`); - if (inDevelopment && process.env.N8N_DEV_RELOAD === 'true') { - void this.loadNodesAndCredentials.setupHotReload(); - } - this.eventService.emit('server-started'); }