From d7d620bf296eb341d74b6edc94621d1b6d4f04f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E0=A4=95=E0=A4=BE=E0=A4=B0=E0=A4=A4=E0=A5=8B=E0=A4=AB?= =?UTF-8?q?=E0=A5=8D=E0=A4=AB=E0=A5=87=E0=A4=B2=E0=A4=B8=E0=A5=8D=E0=A4=95?= =?UTF-8?q?=E0=A5=8D=E0=A4=B0=E0=A4=BF=E0=A4=AA=E0=A5=8D=E0=A4=9F=E2=84=A2?= Date: Fri, 8 Nov 2024 16:55:12 +0100 Subject: [PATCH] feat(core): Hot reload nodes on all servers --- packages/cli/src/commands/base-command.ts | 63 ++++++++++++++++++- packages/cli/src/commands/start.ts | 2 + packages/cli/src/commands/webhook.ts | 2 + packages/cli/src/commands/worker.ts | 2 + .../cli/src/load-nodes-and-credentials.ts | 49 +-------------- packages/cli/src/server.ts | 4 -- 6 files changed, 69 insertions(+), 53 deletions(-) diff --git a/packages/cli/src/commands/base-command.ts b/packages/cli/src/commands/base-command.ts index 214d7f4ce7..2eb818b736 100644 --- a/packages/cli/src/commands/base-command.ts +++ b/packages/cli/src/commands/base-command.ts @@ -1,6 +1,8 @@ import 'reflect-metadata'; import { GlobalConfig } from '@n8n/config'; import { Command, Errors } from '@oclif/core'; +import glob from 'fast-glob'; +import { access as fsAccess, realpath as fsRealPath } from 'fs/promises'; import { BinaryDataService, InstanceSettings, @@ -13,6 +15,8 @@ import { ErrorReporterProxy as ErrorReporter, sleep, } from 'n8n-workflow'; +import path from 'path'; +import picocolors from 'picocolors'; import { Container } from 'typedi'; import type { AbstractServer } from '@/abstract-server'; @@ -42,6 +46,8 @@ export abstract class BaseCommand extends Command { protected nodeTypes: NodeTypes; + protected loadNodesAndCredentials: LoadNodesAndCredentials; + protected instanceSettings: InstanceSettings = Container.get(InstanceSettings); protected server?: AbstractServer; @@ -69,7 +75,8 @@ export abstract class BaseCommand extends Command { process.once('SIGINT', this.onTerminationSignal('SIGINT')); this.nodeTypes = Container.get(NodeTypes); - await Container.get(LoadNodesAndCredentials).init(); + this.loadNodesAndCredentials = Container.get(LoadNodesAndCredentials); + await this.loadNodesAndCredentials.init(); await Db.init().catch( async (error: Error) => await this.exitWithCrash('There was an error initializing DB', error), @@ -338,4 +345,58 @@ export abstract class BaseCommand extends Command { clearTimeout(forceShutdownTimer); }; } + + protected async setupHotReload() { + if (!inDevelopment || process.env.N8N_DEV_RELOAD !== 'true') return; + + const { default: debounce } = await import('lodash/debounce'); + // eslint-disable-next-line import/no-extraneous-dependencies + const { watch } = await import('chokidar'); + + const { Push } = await import('@/push'); + const push = Container.get(Push); + + Object.values(this.loadNodesAndCredentials.loaders).forEach(async (loader) => { + try { + await fsAccess(loader.directory); + } catch { + // If directory doesn't exist, there is nothing to watch + return; + } + + const realModulePath = path.join(await fsRealPath(loader.directory), path.sep); + const reloader = debounce(async (fileName: string) => { + console.info( + picocolors.green('тно Reloading'), + picocolors.bold(fileName), + 'in', + loader.packageName, + ); + const modulesToUnload = Object.keys(require.cache).filter((filePath) => + filePath.startsWith(realModulePath), + ); + modulesToUnload.forEach((filePath) => { + delete require.cache[filePath]; + }); + + loader.reset(); + await loader.loadAll(); + await this.loadNodesAndCredentials.postProcessLoaders(); + push.broadcast('nodeDescriptionUpdated', {}); + }, 100); + + const toWatch = loader.isLazyLoaded + ? ['**/nodes.json', '**/credentials.json'] + : ['**/*.js', '**/*.json']; + const files = await glob(toWatch, { + cwd: realModulePath, + ignore: ['node_modules/**'], + }); + const watcher = watch(files, { + cwd: realModulePath, + ignoreInitial: true, + }); + watcher.on('add', reloader).on('change', reloader).on('unlink', reloader); + }); + } } diff --git a/packages/cli/src/commands/start.ts b/packages/cli/src/commands/start.ts index 42b5df13e6..ca3f7d43cd 100644 --- a/packages/cli/src/commands/start.ts +++ b/packages/cli/src/commands/start.ts @@ -356,6 +356,8 @@ export class Start extends BaseCommand { } }); } + + void this.setupHotReload(); } async catch(error: Error) { diff --git a/packages/cli/src/commands/webhook.ts b/packages/cli/src/commands/webhook.ts index 77ec770aa0..ab42239aa8 100644 --- a/packages/cli/src/commands/webhook.ts +++ b/packages/cli/src/commands/webhook.ts @@ -94,6 +94,8 @@ export class Webhook extends BaseCommand { await this.server.start(); this.logger.info('Webhook listener waiting for requests.'); + void this.setupHotReload(); + // Make sure that the process does not close await new Promise(() => {}); } diff --git a/packages/cli/src/commands/worker.ts b/packages/cli/src/commands/worker.ts index 0291a9e416..73be036cc2 100644 --- a/packages/cli/src/commands/worker.ts +++ b/packages/cli/src/commands/worker.ts @@ -186,6 +186,8 @@ export class Worker extends BaseCommand { }); } + void this.setupHotReload(); + // Make sure that the process does not close if (!inTest) await new Promise(() => {}); } diff --git a/packages/cli/src/load-nodes-and-credentials.ts b/packages/cli/src/load-nodes-and-credentials.ts index 22273fb894..c728203e5a 100644 --- a/packages/cli/src/load-nodes-and-credentials.ts +++ b/packages/cli/src/load-nodes-and-credentials.ts @@ -1,6 +1,5 @@ import { GlobalConfig } from '@n8n/config'; import glob from 'fast-glob'; -import fsPromises from 'fs/promises'; import type { Class, DirectoryLoader, Types } from 'n8n-core'; import { CUSTOM_EXTENSION_ENV, @@ -19,7 +18,7 @@ import type { import { NodeHelpers, ApplicationError, ErrorReporterProxy as ErrorReporter } from 'n8n-workflow'; import path from 'path'; import picocolors from 'picocolors'; -import { Container, Service } from 'typedi'; +import { Service } from 'typedi'; import { CUSTOM_API_CALL_KEY, @@ -355,50 +354,4 @@ export class LoadNodesAndCredentials { await postProcessor(); } } - - async setupHotReload() { - const { default: debounce } = await import('lodash/debounce'); - // eslint-disable-next-line import/no-extraneous-dependencies - const { watch } = await import('chokidar'); - - const { Push } = await import('@/push'); - const push = Container.get(Push); - - Object.values(this.loaders).forEach(async (loader) => { - try { - await fsPromises.access(loader.directory); - } catch { - // If directory doesn't exist, there is nothing to watch - return; - } - - const realModulePath = path.join(await fsPromises.realpath(loader.directory), path.sep); - const reloader = debounce(async () => { - const modulesToUnload = Object.keys(require.cache).filter((filePath) => - filePath.startsWith(realModulePath), - ); - modulesToUnload.forEach((filePath) => { - delete require.cache[filePath]; - }); - - loader.reset(); - await loader.loadAll(); - await this.postProcessLoaders(); - push.broadcast('nodeDescriptionUpdated', {}); - }, 100); - - const toWatch = loader.isLazyLoaded - ? ['**/nodes.json', '**/credentials.json'] - : ['**/*.js', '**/*.json']; - const files = await glob(toWatch, { - cwd: realModulePath, - ignore: ['node_modules/**'], - }); - const watcher = watch(files, { - cwd: realModulePath, - ignoreInitial: true, - }); - watcher.on('add', reloader).on('change', reloader).on('unlink', reloader); - }); - } } diff --git a/packages/cli/src/server.ts b/packages/cli/src/server.ts index 3cfd93054b..bf9122325e 100644 --- a/packages/cli/src/server.ts +++ b/packages/cli/src/server.ts @@ -100,10 +100,6 @@ export class Server extends AbstractServer { await super.start(); this.logger.debug(`Server ID: ${this.instanceSettings.hostId}`); - if (inDevelopment && process.env.N8N_DEV_RELOAD === 'true') { - void this.loadNodesAndCredentials.setupHotReload(); - } - this.eventService.emit('server-started'); }