mudhorn/n8n
1
0
Fork 0
mirror of https://github.com/n8n-io/n8n.git synced 2025-01-12 05:17:28 -08:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: Reduce initial memory spike at server startup (no-changelog) (#4735)

Browse source 
* feat: Reduce initial memory spike at server startup (no-changelog)

This changes the frontend types generation to generate less garbage for the GC to collect.

* switch to stream pipelines for writing all the static files

and, move all static file generation before the server starts
This commit is contained in:
कारतोफ्फेलस्क्रिप्ट™ 2022-11-28 17:41:44 +01:00 committed by GitHub
parent 540f6e0abd
commit aac207a947
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 104 additions and 60 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
packages/cli/package.json
View file

@ -85,6 +85,7 @@
"@types/parseurl": "^1.3.1",
"@types/passport-jwt": "^3.0.6",
"@types/psl": "^1.1.0",
"@types/replacestream": "^4.0.1",
"@types/send": "^0.17.1",
"@types/shelljs": "^0.8.11",
"@types/superagent": "4.1.13",
@ -168,6 +169,7 @@
"posthog-node": "^1.3.0",
"prom-client": "^13.1.0",
"psl": "^1.8.0",
"replacestream": "^4.0.3",
"semver": "^7.3.8",
"shelljs": "^0.8.5",
"sqlite3": "^5.1.2",

15
packages/cli/src/LoadNodesAndCredentials.ts
View file

@ -15,13 +15,13 @@ import type {
} from 'n8n-workflow';
import { LoggerProxy, ErrorReporterProxy as ErrorReporter } from 'n8n-workflow';
import { createWriteStream } from 'fs';
import {
access as fsAccess,
copyFile,
mkdir,
readdir as fsReaddir,
stat as fsStat,
writeFile,
} from 'fs/promises';
import path from 'path';
import config from '@/config';
@ -76,10 +76,17 @@ export class LoadNodesAndCredentialsClass implements INodesAndCredentials {
// pre-render all the node and credential types as static json files
await mkdir(path.join(GENERATED_STATIC_DIR, 'types'), { recursive: true });
const writeStaticJSON = async (name: string, data: any[]) => {
const writeStaticJSON = async (name: string, data: object[]) => {
const filePath = path.join(GENERATED_STATIC_DIR, `types/${name}.json`);
const payload = `[\n${data.map((entry) => JSON.stringify(entry)).join(',\n')}\n]`;
await writeFile(filePath, payload, { encoding: 'utf-8' });
const stream = createWriteStream(filePath, 'utf-8');
stream.write('[\n');
data.forEach((entry, index) => {
stream.write(JSON.stringify(entry));
if (index !== data.length - 1) stream.write(',');
stream.write('\n');
});
stream.write(']\n');
stream.end();
};
await writeStaticJSON('nodes', this.types.nodes);

60
packages/cli/src/Server.ts
View file

@ -28,10 +28,10 @@
/* eslint-disable no-await-in-loop */
import { exec as callbackExec } from 'child_process';
import { existsSync, readFileSync, writeFileSync } from 'fs';
import { access as fsAccess, readFile, writeFile, mkdir } from 'fs/promises';
import { readFileSync } from 'fs';
import { access as fsAccess } from 'fs/promises';
import os from 'os';
import { dirname as pathDirname, join as pathJoin, resolve as pathResolve } from 'path';
import { join as pathJoin, resolve as pathResolve } from 'path';
import { createHmac } from 'crypto';
import { promisify } from 'util';
import cookieParser from 'cookie-parser';
@ -78,7 +78,6 @@ import parseUrl from 'parseurl';
import promClient, { Registry } from 'prom-client';
import history from 'connect-history-api-fallback';
import bodyParser from 'body-parser';
import glob from 'fast-glob';
import config from '@/config';
import * as Queue from '@/Queue';
@ -91,6 +90,7 @@ import { nodesController } from '@/api/nodes.api';
import { workflowsController } from '@/workflows/workflows.controller';
import {
AUTH_COOKIE_NAME,
EDITOR_UI_DIST_DIR,
GENERATED_STATIC_DIR,
NODES_BASE_DIR,
RESPONSE_ERROR_MESSAGES,
@ -113,7 +113,6 @@ import { executionsController } from '@/api/executions.api';
import { nodeTypesController } from '@/api/nodeTypes.api';
import { tagsController } from '@/api/tags.api';
import { loadPublicApiVersions } from '@/PublicApi';
import * as telemetryScripts from '@/telemetry/scripts';
import {
getInstanceBaseUrl,
isEmailSetUp,
@ -1640,56 +1639,7 @@ class App {
}
if (!config.getEnv('endpoints.disableUi')) {
// Read the index file and replace the path placeholder
const n8nPath = config.getEnv('path');
const basePathRegEx = /\/{{BASE_PATH}}\//g;
const hooksUrls = config.getEnv('externalFrontendHooksUrls');
let scriptsString = '';
if (hooksUrls) {
scriptsString = hooksUrls.split(';').reduce((acc, curr) => {
return `${acc}<script src="${curr}"></script>`;
}, '');
}
if (this.frontendSettings.telemetry.enabled) {
const phLoadingScript = telemetryScripts.createPostHogLoadingScript({
apiKey: config.getEnv('diagnostics.config.posthog.apiKey'),
apiHost: config.getEnv('diagnostics.config.posthog.apiHost'),
autocapture: false,
disableSessionRecording: config.getEnv(
'diagnostics.config.posthog.disableSessionRecording',
),
debug: config.getEnv('logs.level') === 'debug',
});
scriptsString += phLoadingScript;
}
const editorUiDistDir = pathJoin(pathDirname(require.resolve('n8n-editor-ui')), 'dist');
const closingTitleTag = '</title>';
const compileFile = async (fileName: string) => {
const filePath = pathJoin(editorUiDistDir, fileName);
if (/(index\.html)|.*\.(js|css)/.test(filePath) && existsSync(filePath)) {
const srcFile = await readFile(filePath, 'utf8');
let payload = srcFile
.replace(basePathRegEx, n8nPath)
.replace(/\/static\//g, n8nPath + 'static/');
if (filePath.endsWith('index.html')) {
payload = payload.replace(closingTitleTag, closingTitleTag + scriptsString);
}
const destFile = pathJoin(GENERATED_STATIC_DIR, fileName);
await mkdir(pathDirname(destFile), { recursive: true });
await writeFile(destFile, payload, 'utf-8');
}
};
await compileFile('index.html');
const files = await glob('**/*.{css,js}', { cwd: editorUiDistDir });
await Promise.all(files.map(compileFile));
this.app.use('/', express.static(GENERATED_STATIC_DIR), express.static(editorUiDistDir));
this.app.use('/', express.static(GENERATED_STATIC_DIR), express.static(EDITOR_UI_DIST_DIR));
const startTime = new Date().toUTCString();
this.app.use('/index.html', (req, res, next) => {

68
packages/cli/src/commands/start.ts
View file

@ -6,10 +6,17 @@
/* eslint-disable no-console */
/* eslint-disable @typescript-eslint/no-unsafe-call */
/* eslint-disable @typescript-eslint/no-unsafe-member-access */
import path from 'path';
import { mkdir } from 'fs/promises';
import { createReadStream, createWriteStream, existsSync } from 'fs';
import localtunnel from 'localtunnel';
import { BinaryDataManager, TUNNEL_SUBDOMAIN_ENV, UserSettings } from 'n8n-core';
import { Command, flags } from '@oclif/command';
import Redis from 'ioredis';
import stream from 'stream';
import replaceStream from 'replacestream';
import { promisify } from 'util';
import glob from 'fast-glob';
import { IDataObject, LoggerProxy, sleep } from 'n8n-workflow';
import { createHash } from 'crypto';
@ -34,9 +41,12 @@ import { getLogger } from '@/Logger';
import { getAllInstalledPackages } from '@/CommunityNodes/packageModel';
import { initErrorHandling } from '@/ErrorReporting';
import * as CrashJournal from '@/CrashJournal';
import { createPostHogLoadingScript } from '@/telemetry/scripts';
import { EDITOR_UI_DIST_DIR, GENERATED_STATIC_DIR } from '@/constants';
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-var-requires
const open = require('open');
const pipeline = promisify(stream.pipeline);
let activeWorkflowRunner: ActiveWorkflowRunner.ActiveWorkflowRunner | undefined;
let processExitCode = 0;
@ -152,6 +162,60 @@ export class Start extends Command {
exit();
}
static async generateStaticAssets() {
// Read the index file and replace the path placeholder
const n8nPath = config.getEnv('path');
const hooksUrls = config.getEnv('externalFrontendHooksUrls');
let scriptsString = '';
if (hooksUrls) {
scriptsString = hooksUrls.split(';').reduce((acc, curr) => {
return `${acc}<script src="${curr}"></script>`;
}, '');
}
if (config.getEnv('diagnostics.enabled')) {
const phLoadingScript = createPostHogLoadingScript({
apiKey: config.getEnv('diagnostics.config.posthog.apiKey'),
apiHost: config.getEnv('diagnostics.config.posthog.apiHost'),
autocapture: false,
disableSessionRecording: config.getEnv(
'diagnostics.config.posthog.disableSessionRecording',
),
debug: config.getEnv('logs.level') === 'debug',
});
scriptsString += phLoadingScript;
}
const closingTitleTag = '</title>';
const compileFile = async (fileName: string) => {
const filePath = path.join(EDITOR_UI_DIST_DIR, fileName);
if (/(index\.html)|.*\.(js|css)/.test(filePath) && existsSync(filePath)) {
const destFile = path.join(GENERATED_STATIC_DIR, fileName);
await mkdir(path.dirname(destFile), { recursive: true });
const streams = [
createReadStream(filePath, 'utf-8'),
replaceStream('/{{BASE_PATH}}/', n8nPath, { ignoreCase: false }),
replaceStream('/static/', n8nPath + 'static/', { ignoreCase: false }),
];
if (filePath.endsWith('index.html')) {
streams.push(
replaceStream(closingTitleTag, closingTitleTag + scriptsString, {
ignoreCase: false,
}),
);
}
streams.push(createWriteStream(destFile, 'utf-8'));
return pipeline(streams);
}
};
await compileFile('index.html');
const files = await glob('**/*.{css,js}', { cwd: EDITOR_UI_DIST_DIR });
await Promise.all(files.map(compileFile));
}
async run() {
// Make sure that n8n shuts down gracefully if possible
process.once('SIGTERM', Start.stopProcess);
@ -200,6 +264,10 @@ export class Start extends Command {
);
}
if (!config.getEnv('endpoints.disableUi')) {
await Start.generateStaticAssets();
}
// Load all node and credential types
const loadNodesAndCredentials = LoadNodesAndCredentials();
await loadNodesAndCredentials.init();

3
packages/cli/src/constants.ts
View file

@ -1,13 +1,14 @@
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
/* eslint-disable @typescript-eslint/no-unsafe-member-access */
/* eslint-disable @typescript-eslint/naming-convention */
import { resolve, join } from 'path';
import { resolve, join, dirname } from 'path';
import { RESPONSE_ERROR_MESSAGES as CORE_RESPONSE_ERROR_MESSAGES, UserSettings } from 'n8n-core';
export const CLI_DIR = resolve(__dirname, '..');
export const TEMPLATES_DIR = join(CLI_DIR, 'templates');
export const NODES_BASE_DIR = join(CLI_DIR, '..', 'nodes-base');
export const GENERATED_STATIC_DIR = join(UserSettings.getUserHome(), '.cache/n8n/public');
export const EDITOR_UI_DIST_DIR = join(dirname(require.resolve('n8n-editor-ui')), 'dist');
export const NODE_PACKAGE_PREFIX = 'n8n-nodes-';

16
pnpm-lock.yaml
View file

@ -125,6 +125,7 @@ importers:
'@types/parseurl': ^1.3.1
'@types/passport-jwt': ^3.0.6
'@types/psl': ^1.1.0
'@types/replacestream': ^4.0.1
'@types/send': ^0.17.1
'@types/shelljs': ^0.8.11
'@types/superagent': 4.1.13
@ -193,6 +194,7 @@ importers:
posthog-node: ^1.3.0
prom-client: ^13.1.0
psl: ^1.8.0
replacestream: ^4.0.3
run-script-os: ^1.0.7
semver: ^7.3.8
shelljs: ^0.8.5
@ -276,6 +278,7 @@ importers:
posthog-node: 1.3.0
prom-client: 13.2.0
psl: 1.9.0
replacestream: 4.0.3
semver: 7.3.8
shelljs: 0.8.5
sqlite3: 5.1.2
@ -312,6 +315,7 @@ importers:
'@types/parseurl': 1.3.1
'@types/passport-jwt': 3.0.7
'@types/psl': 1.1.0
'@types/replacestream': 4.0.1
'@types/send': 0.17.1
'@types/shelljs': 0.8.11
'@types/superagent': 4.1.13
@ -5990,6 +5994,10 @@ packages:
'@types/node': 16.11.65
dev: true
/@types/replacestream/4.0.1:
resolution: {integrity: sha512-3ecTmnzB90sgarVpIszCF1cX2cnxwqDovWb31jGrKfxAL0Knui1H7Reaz/zlT9zaE3u0un7L5cNy9fQPy0d2sg==}
dev: true
/@types/request-promise-native/1.0.18:
resolution: {integrity: sha512-tPnODeISFc/c1LjWyLuZUY+Z0uLB3+IMfNoQyDEi395+j6kTFTTRAqjENjoPJUid4vHRGEozoTrcTrfZM+AcbA==}
dependencies:
@ -18464,6 +18472,14 @@ packages:
yargs: 17.6.0
dev: false
/replacestream/4.0.3:
resolution: {integrity: sha512-AC0FiLS352pBBiZhd4VXB1Ab/lh0lEgpP+GGvZqbQh8a5cmXVoTe5EX/YeTFArnp4SRGTHh1qCHu9lGs1qG8sA==}
dependencies:
escape-string-regexp: 1.0.5
object-assign: 4.1.1
readable-stream: 2.3.7
dev: false
/request-progress/3.0.0:
resolution: {integrity: sha512-MnWzEHHaxHO2iWiQuHrUPBi/1WeBf5PkxQqNyNvLl9VAYSdXkP8tQ3pBSeCPD+yw0v0Aq1zosWLz0BdeXpWwZg==}
dependencies: