feat: Reduce initial memory spike at server startup (no-changelog) (#4735)

* feat: Reduce initial memory spike at server startup (no-changelog)

This changes the frontend types generation to generate less garbage for the GC to collect.

* switch to stream pipelines for writing all the static files

and, move all static file generation before the server starts

packages/cli/package.json

@@ -85,6 +85,7 @@
     "@types/parseurl": "^1.3.1",
     "@types/passport-jwt": "^3.0.6",
     "@types/psl": "^1.1.0",
+    "@types/replacestream": "^4.0.1",
     "@types/send": "^0.17.1",
     "@types/shelljs": "^0.8.11",
     "@types/superagent": "4.1.13",
@@ -168,6 +169,7 @@
     "posthog-node": "^1.3.0",
     "prom-client": "^13.1.0",
     "psl": "^1.8.0",
+    "replacestream": "^4.0.3",
     "semver": "^7.3.8",
     "shelljs": "^0.8.5",
     "sqlite3": "^5.1.2",

packages/cli/src/LoadNodesAndCredentials.ts

@@ -15,13 +15,13 @@ import type {
 } from 'n8n-workflow';
 import { LoggerProxy, ErrorReporterProxy as ErrorReporter } from 'n8n-workflow';
 
+import { createWriteStream } from 'fs';
 import {
 	access as fsAccess,
 	copyFile,
 	mkdir,
 	readdir as fsReaddir,
 	stat as fsStat,
-	writeFile,
 } from 'fs/promises';
 import path from 'path';
 import config from '@/config';
@@ -76,10 +76,17 @@ export class LoadNodesAndCredentialsClass implements INodesAndCredentials {
 		// pre-render all the node and credential types as static json files
 		await mkdir(path.join(GENERATED_STATIC_DIR, 'types'), { recursive: true });
 
-		const writeStaticJSON = async (name: string, data: any[]) => {
+		const writeStaticJSON = async (name: string, data: object[]) => {
 			const filePath = path.join(GENERATED_STATIC_DIR, `types/${name}.json`);
-			const payload = `[\n${data.map((entry) => JSON.stringify(entry)).join(',\n')}\n]`;
+			const stream = createWriteStream(filePath, 'utf-8');
-			await writeFile(filePath, payload, { encoding: 'utf-8' });
+			stream.write('[\n');
+			data.forEach((entry, index) => {
+				stream.write(JSON.stringify(entry));
+				if (index !== data.length - 1) stream.write(',');
+				stream.write('\n');
+			});
+			stream.write(']\n');
+			stream.end();
 		};
 
 		await writeStaticJSON('nodes', this.types.nodes);

packages/cli/src/Server.ts

@@ -28,10 +28,10 @@
 /* eslint-disable no-await-in-loop */
 
 import { exec as callbackExec } from 'child_process';
-import { existsSync, readFileSync, writeFileSync } from 'fs';
+import { readFileSync } from 'fs';
-import { access as fsAccess, readFile, writeFile, mkdir } from 'fs/promises';
+import { access as fsAccess } from 'fs/promises';
 import os from 'os';
-import { dirname as pathDirname, join as pathJoin, resolve as pathResolve } from 'path';
+import { join as pathJoin, resolve as pathResolve } from 'path';
 import { createHmac } from 'crypto';
 import { promisify } from 'util';
 import cookieParser from 'cookie-parser';
@@ -78,7 +78,6 @@ import parseUrl from 'parseurl';
 import promClient, { Registry } from 'prom-client';
 import history from 'connect-history-api-fallback';
 import bodyParser from 'body-parser';
-import glob from 'fast-glob';
 
 import config from '@/config';
 import * as Queue from '@/Queue';
@@ -91,6 +90,7 @@ import { nodesController } from '@/api/nodes.api';
 import { workflowsController } from '@/workflows/workflows.controller';
 import {
 	AUTH_COOKIE_NAME,
+	EDITOR_UI_DIST_DIR,
 	GENERATED_STATIC_DIR,
 	NODES_BASE_DIR,
 	RESPONSE_ERROR_MESSAGES,
@@ -113,7 +113,6 @@ import { executionsController } from '@/api/executions.api';
 import { nodeTypesController } from '@/api/nodeTypes.api';
 import { tagsController } from '@/api/tags.api';
 import { loadPublicApiVersions } from '@/PublicApi';
-import * as telemetryScripts from '@/telemetry/scripts';
 import {
 	getInstanceBaseUrl,
 	isEmailSetUp,
@@ -1640,56 +1639,7 @@ class App {
 		}
 
 		if (!config.getEnv('endpoints.disableUi')) {
-			// Read the index file and replace the path placeholder
+			this.app.use('/', express.static(GENERATED_STATIC_DIR), express.static(EDITOR_UI_DIST_DIR));
-			const n8nPath = config.getEnv('path');
-			const basePathRegEx = /\/{{BASE_PATH}}\//g;
-			const hooksUrls = config.getEnv('externalFrontendHooksUrls');
-
-			let scriptsString = '';
-			if (hooksUrls) {
-				scriptsString = hooksUrls.split(';').reduce((acc, curr) => {
-					return `${acc}<script src="${curr}"></script>`;
-				}, '');
-			}
-
-			if (this.frontendSettings.telemetry.enabled) {
-				const phLoadingScript = telemetryScripts.createPostHogLoadingScript({
-					apiKey: config.getEnv('diagnostics.config.posthog.apiKey'),
-					apiHost: config.getEnv('diagnostics.config.posthog.apiHost'),
-					autocapture: false,
-					disableSessionRecording: config.getEnv(
-						'diagnostics.config.posthog.disableSessionRecording',
-					),
-					debug: config.getEnv('logs.level') === 'debug',
-				});
-
-				scriptsString += phLoadingScript;
-			}
-
-			const editorUiDistDir = pathJoin(pathDirname(require.resolve('n8n-editor-ui')), 'dist');
-
-			const closingTitleTag = '</title>';
-			const compileFile = async (fileName: string) => {
-				const filePath = pathJoin(editorUiDistDir, fileName);
-				if (/(index\.html)|.*\.(js|css)/.test(filePath) && existsSync(filePath)) {
-					const srcFile = await readFile(filePath, 'utf8');
-					let payload = srcFile
-						.replace(basePathRegEx, n8nPath)
-						.replace(/\/static\//g, n8nPath + 'static/');
-					if (filePath.endsWith('index.html')) {
-						payload = payload.replace(closingTitleTag, closingTitleTag + scriptsString);
-					}
-					const destFile = pathJoin(GENERATED_STATIC_DIR, fileName);
-					await mkdir(pathDirname(destFile), { recursive: true });
-					await writeFile(destFile, payload, 'utf-8');
-				}
-			};
-
-			await compileFile('index.html');
-			const files = await glob('**/*.{css,js}', { cwd: editorUiDistDir });
-			await Promise.all(files.map(compileFile));
-
-			this.app.use('/', express.static(GENERATED_STATIC_DIR), express.static(editorUiDistDir));
 
 			const startTime = new Date().toUTCString();
 			this.app.use('/index.html', (req, res, next) => {

packages/cli/src/commands/start.ts

@@ -6,10 +6,17 @@
 /* eslint-disable no-console */
 /* eslint-disable @typescript-eslint/no-unsafe-call */
 /* eslint-disable @typescript-eslint/no-unsafe-member-access */
+import path from 'path';
+import { mkdir } from 'fs/promises';
+import { createReadStream, createWriteStream, existsSync } from 'fs';
 import localtunnel from 'localtunnel';
 import { BinaryDataManager, TUNNEL_SUBDOMAIN_ENV, UserSettings } from 'n8n-core';
 import { Command, flags } from '@oclif/command';
 import Redis from 'ioredis';
+import stream from 'stream';
+import replaceStream from 'replacestream';
+import { promisify } from 'util';
+import glob from 'fast-glob';
 
 import { IDataObject, LoggerProxy, sleep } from 'n8n-workflow';
 import { createHash } from 'crypto';
@@ -34,9 +41,12 @@ import { getLogger } from '@/Logger';
 import { getAllInstalledPackages } from '@/CommunityNodes/packageModel';
 import { initErrorHandling } from '@/ErrorReporting';
 import * as CrashJournal from '@/CrashJournal';
+import { createPostHogLoadingScript } from '@/telemetry/scripts';
+import { EDITOR_UI_DIST_DIR, GENERATED_STATIC_DIR } from '@/constants';
 
 // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-var-requires
 const open = require('open');
+const pipeline = promisify(stream.pipeline);
 
 let activeWorkflowRunner: ActiveWorkflowRunner.ActiveWorkflowRunner | undefined;
 let processExitCode = 0;
@@ -152,6 +162,60 @@ export class Start extends Command {
 		exit();
 	}
 
+	static async generateStaticAssets() {
+		// Read the index file and replace the path placeholder
+		const n8nPath = config.getEnv('path');
+		const hooksUrls = config.getEnv('externalFrontendHooksUrls');
+
+		let scriptsString = '';
+		if (hooksUrls) {
+			scriptsString = hooksUrls.split(';').reduce((acc, curr) => {
+				return `${acc}<script src="${curr}"></script>`;
+			}, '');
+		}
+
+		if (config.getEnv('diagnostics.enabled')) {
+			const phLoadingScript = createPostHogLoadingScript({
+				apiKey: config.getEnv('diagnostics.config.posthog.apiKey'),
+				apiHost: config.getEnv('diagnostics.config.posthog.apiHost'),
+				autocapture: false,
+				disableSessionRecording: config.getEnv(
+					'diagnostics.config.posthog.disableSessionRecording',
+				),
+				debug: config.getEnv('logs.level') === 'debug',
+			});
+
+			scriptsString += phLoadingScript;
+		}
+
+		const closingTitleTag = '</title>';
+		const compileFile = async (fileName: string) => {
+			const filePath = path.join(EDITOR_UI_DIST_DIR, fileName);
+			if (/(index\.html)|.*\.(js|css)/.test(filePath) && existsSync(filePath)) {
+				const destFile = path.join(GENERATED_STATIC_DIR, fileName);
+				await mkdir(path.dirname(destFile), { recursive: true });
+				const streams = [
+					createReadStream(filePath, 'utf-8'),
+					replaceStream('/{{BASE_PATH}}/', n8nPath, { ignoreCase: false }),
+					replaceStream('/static/', n8nPath + 'static/', { ignoreCase: false }),
+				];
+				if (filePath.endsWith('index.html')) {
+					streams.push(
+						replaceStream(closingTitleTag, closingTitleTag + scriptsString, {
+							ignoreCase: false,
+						}),
+					);
+				}
+				streams.push(createWriteStream(destFile, 'utf-8'));
+				return pipeline(streams);
+			}
+		};
+
+		await compileFile('index.html');
+		const files = await glob('**/*.{css,js}', { cwd: EDITOR_UI_DIST_DIR });
+		await Promise.all(files.map(compileFile));
+	}
+
 	async run() {
 		// Make sure that n8n shuts down gracefully if possible
 		process.once('SIGTERM', Start.stopProcess);
@@ -200,6 +264,10 @@ export class Start extends Command {
 					);
 				}
 
+				if (!config.getEnv('endpoints.disableUi')) {
+					await Start.generateStaticAssets();
+				}
+
 				// Load all node and credential types
 				const loadNodesAndCredentials = LoadNodesAndCredentials();
 				await loadNodesAndCredentials.init();

packages/cli/src/constants.ts

@@ -1,13 +1,14 @@
 /* eslint-disable @typescript-eslint/no-unsafe-assignment */
 /* eslint-disable @typescript-eslint/no-unsafe-member-access */
 /* eslint-disable @typescript-eslint/naming-convention */
-import { resolve, join } from 'path';
+import { resolve, join, dirname } from 'path';
 import { RESPONSE_ERROR_MESSAGES as CORE_RESPONSE_ERROR_MESSAGES, UserSettings } from 'n8n-core';
 
 export const CLI_DIR = resolve(__dirname, '..');
 export const TEMPLATES_DIR = join(CLI_DIR, 'templates');
 export const NODES_BASE_DIR = join(CLI_DIR, '..', 'nodes-base');
 export const GENERATED_STATIC_DIR = join(UserSettings.getUserHome(), '.cache/n8n/public');
+export const EDITOR_UI_DIST_DIR = join(dirname(require.resolve('n8n-editor-ui')), 'dist');
 
 export const NODE_PACKAGE_PREFIX = 'n8n-nodes-';
 

pnpm-lock.yaml

@@ -125,6 +125,7 @@ importers:
       '@types/parseurl': ^1.3.1
       '@types/passport-jwt': ^3.0.6
       '@types/psl': ^1.1.0
+      '@types/replacestream': ^4.0.1
       '@types/send': ^0.17.1
       '@types/shelljs': ^0.8.11
       '@types/superagent': 4.1.13
@@ -193,6 +194,7 @@ importers:
       posthog-node: ^1.3.0
       prom-client: ^13.1.0
       psl: ^1.8.0
+      replacestream: ^4.0.3
       run-script-os: ^1.0.7
       semver: ^7.3.8
       shelljs: ^0.8.5
@@ -276,6 +278,7 @@ importers:
       posthog-node: 1.3.0
       prom-client: 13.2.0
       psl: 1.9.0
+      replacestream: 4.0.3
       semver: 7.3.8
       shelljs: 0.8.5
       sqlite3: 5.1.2
@@ -312,6 +315,7 @@ importers:
       '@types/parseurl': 1.3.1
       '@types/passport-jwt': 3.0.7
       '@types/psl': 1.1.0
+      '@types/replacestream': 4.0.1
       '@types/send': 0.17.1
       '@types/shelljs': 0.8.11
       '@types/superagent': 4.1.13
@@ -5990,6 +5994,10 @@ packages:
       '@types/node': 16.11.65
     dev: true
 
+  /@types/replacestream/4.0.1:
+    resolution: {integrity: sha512-3ecTmnzB90sgarVpIszCF1cX2cnxwqDovWb31jGrKfxAL0Knui1H7Reaz/zlT9zaE3u0un7L5cNy9fQPy0d2sg==}
+    dev: true
+
   /@types/request-promise-native/1.0.18:
     resolution: {integrity: sha512-tPnODeISFc/c1LjWyLuZUY+Z0uLB3+IMfNoQyDEi395+j6kTFTTRAqjENjoPJUid4vHRGEozoTrcTrfZM+AcbA==}
     dependencies:
@@ -18464,6 +18472,14 @@ packages:
       yargs: 17.6.0
     dev: false
 
+  /replacestream/4.0.3:
+    resolution: {integrity: sha512-AC0FiLS352pBBiZhd4VXB1Ab/lh0lEgpP+GGvZqbQh8a5cmXVoTe5EX/YeTFArnp4SRGTHh1qCHu9lGs1qG8sA==}
+    dependencies:
+      escape-string-regexp: 1.0.5
+      object-assign: 4.1.1
+      readable-stream: 2.3.7
+    dev: false
+
   /request-progress/3.0.0:
     resolution: {integrity: sha512-MnWzEHHaxHO2iWiQuHrUPBi/1WeBf5PkxQqNyNvLl9VAYSdXkP8tQ3pBSeCPD+yw0v0Aq1zosWLz0BdeXpWwZg==}
     dependencies: