feat: Add benchmarking of pooled sqlite (no-changelog) (#10550)

.github/workflows/benchmark-nightly.yml

@@ -60,10 +60,10 @@ jobs:
 
       - name: Run the benchmark with debug logging
         if: github.event.inputs.debug == 'true'
-        run: pnpm run-in-cloud sqlite --debug
+        run: pnpm run-in-cloud --debug
         working-directory: packages/@n8n/benchmark
 
       - name: Run the benchmark
         if: github.event.inputs.debug != 'true'
-        run: pnpm run-in-cloud sqlite
+        run: pnpm run-in-cloud
         working-directory: packages/@n8n/benchmark

packages/@n8n/benchmark/scripts/runInCloud.mjs

@@ -15,7 +15,7 @@
 // @ts-check
 import fs from 'fs';
 import minimist from 'minimist';
-import { $, sleep, which } from 'zx';
+import { sleep, which } from 'zx';
 import path from 'path';
 import { SshClient } from './sshClient.mjs';
 import { TerraformClient } from './terraformClient.mjs';
@@ -61,7 +61,6 @@ async function ensureDependencies() {
 }
 
 /**
- *
  * @param {Config} config
  * @param {BenchmarkEnv} benchmarkEnv
  */
@@ -86,7 +85,32 @@ async function runBenchmarksOnVm(config, benchmarkEnv) {
 	// Give some time for the VM to be ready
 	await sleep(1000);
 
-	console.log('Running benchmarks...');
+	if (config.n8nSetupToUse === 'all') {
+		const availableSetups = readAvailableN8nSetups();
+
+		for (const n8nSetup of availableSetups) {
+			await runBenchmarkForN8nSetup({
+				config,
+				sshClient,
+				scriptsDir,
+				n8nSetup,
+			});
+		}
+	} else {
+		await runBenchmarkForN8nSetup({
+			config,
+			sshClient,
+			scriptsDir,
+			n8nSetup: config.n8nSetupToUse,
+		});
+	}
+}
+
+/**
+ * @param {{ config: Config; sshClient: any; scriptsDir: string; n8nSetup: string; }} opts
+ */
+async function runBenchmarkForN8nSetup({ config, sshClient, scriptsDir, n8nSetup }) {
+	console.log(`Running benchmarks for ${n8nSetup}...`);
 	const runScriptPath = path.join(scriptsDir, 'runOnVm.mjs');
 
 	const flags = {
@@ -100,7 +124,7 @@ async function runBenchmarksOnVm(config, benchmarkEnv) {
 		.map(([key, value]) => `--${key}=${value}`)
 		.join(' ');
 
-	await sshClient.ssh(`npx zx ${runScriptPath} ${flagsString} ${config.n8nSetupToUse}`, {
+	await sshClient.ssh(`npx zx ${runScriptPath} ${flagsString} ${n8nSetup}`, {
 		// Test run should always log its output
 		verbose: true,
 	});
@@ -138,10 +162,15 @@ function readAvailableN8nSetups() {
  * @returns {Promise<Config>}
  */
 async function parseAndValidateConfig() {
-	const args = minimist(process.argv.slice(2), {
-		boolean: ['debug'],
+	const args = minimist(process.argv.slice(3), {
+		boolean: ['debug', 'help'],
 	});
 
+	if (args.help) {
+		printUsage();
+		process.exit(0);
+	}
+
 	const n8nSetupToUse = await getAndValidateN8nSetup(args);
 	const isVerbose = args.debug || false;
 	const n8nTag = args.n8nTag || process.env.N8N_DOCKER_TAG || 'latest';
@@ -163,10 +192,8 @@ async function parseAndValidateConfig() {
 async function getAndValidateN8nSetup(args) {
 	// Last parameter is the n8n setup to use
 	const n8nSetupToUse = args._[args._.length - 1];
-
-	if (!n8nSetupToUse) {
-		printUsage();
-		process.exit(1);
+	if (!n8nSetupToUse || n8nSetupToUse === 'all') {
+		return 'all';
 	}
 
 	const availableSetups = readAvailableN8nSetups();
@@ -182,19 +209,20 @@ async function getAndValidateN8nSetup(args) {
 function printUsage() {
 	const availableSetups = readAvailableN8nSetups();
 
-	console.log('Usage: zx scripts/runInCloud.mjs <n8n setup name>');
-	console.log('   eg: zx scripts/runInCloud.mjs sqlite');
+	console.log('Usage: zx scripts/runInCloud.mjs [n8n setup name]');
+	console.log('   eg: zx scripts/runInCloud.mjs');
 	console.log('');
 	console.log('Options:');
+	console.log(
+		`  [n8n setup name]     Against which n8n setup to run the benchmarks. One of: ${['all', ...availableSetups].join(', ')}. Default is all`,
+	);
 	console.log('  --debug              Enable verbose output');
 	console.log('  --n8nTag             Docker tag for n8n image. Default is latest');
 	console.log('  --benchmarkTag       Docker tag for benchmark cli image. Default is latest');
 	console.log(
-		'  --k6ApiToken         API token for k6 cloud. Default is read from K6_API_TOKEN env var. If omitted, k6 cloud will not be used.',
+		'  --k6ApiToken         API token for k6 cloud. Default is read from K6_API_TOKEN env var. If omitted, k6 cloud will not be used',
 	);
 	console.log('');
-	console.log('Available setups:');
-	console.log(`  ${availableSetups.join(', ')}`);
 }
 
 main().catch(console.error);

packages/@n8n/benchmark/scripts/runOnVm/bootstrap.sh

@@ -10,8 +10,8 @@ CURRENT_USER=$(whoami)
 # Mount the data disk
 if [ -d "/n8n" ]; then
 	echo "Data disk already mounted. Clearing it..."
-	rm -rf /n8n/*
-	rm -rf /n8n/.[!.]*
+	sudo rm -rf /n8n/*
+	sudo rm -rf /n8n/.[!.]*
 else
 	sudo mkdir -p /n8n
 	sudo parted /dev/sdc --script mklabel gpt mkpart xfspart xfs 0% 100%

packages/@n8n/benchmark/scripts/runOnVm/n8nSetups/sqlite-legacy/docker-compose.yml

@@ -0,0 +1,17 @@
+services:
+  n8n:
+    image: ghcr.io/n8n-io/n8n:${N8N_VERSION:-latest}
+    environment:
+      - N8N_DIAGNOSTICS_ENABLED=false
+      - N8N_USER_FOLDER=/n8n
+    ports:
+      - 5678:5678
+    volumes:
+      - /n8n:/n8n
+  benchmark:
+    image: ghcr.io/n8n-io/n8n-benchmark:${N8N_BENCHMARK_VERSION:-latest}
+    depends_on:
+      - n8n
+    environment:
+      - N8N_BASE_URL=http://n8n:5678
+      - K6_API_TOKEN=${K6_API_TOKEN}

packages/@n8n/benchmark/scripts/runOnVm/n8nSetups/sqlite/docker-compose.yml

@@ -4,6 +4,8 @@ services:
     environment:
       - N8N_DIAGNOSTICS_ENABLED=false
       - N8N_USER_FOLDER=/n8n
+      - DB_SQLITE_POOL_SIZE=3
+      - DB_SQLITE_ENABLE_WAL=true
     ports:
       - 5678:5678
     volumes:

packages/@n8n/benchmark/scripts/runOnVm/runOnVm.mjs

@@ -2,23 +2,19 @@
 /**
  * This script runs the benchmarks using a given docker compose setup
  */
+// @ts-check
+import path from 'path';
+import { $, argv, fs } from 'zx';
 
-import { $ } from 'zx';
-
-const [n8nSetupToUse] = argv._;
-
-if (!n8nSetupToUse) {
-	printUsage();
-	process.exit(1);
-}
-
-function printUsage() {
-	console.log('Usage: zx runOnVm.mjs <envName>');
-	console.log('   eg: zx runOnVm.mjs sqlite');
-}
+const paths = {
+	n8nSetupsDir: path.join(__dirname, 'n8nSetups'),
+};
 
 async function main() {
-	const composeFilePath = path.join(__dirname, 'n8nSetups', n8nSetupToUse);
+	const [n8nSetupToUse] = argv._;
+	validateN8nSetup(n8nSetupToUse);
+
+	const composeFilePath = path.join(paths.n8nSetupsDir, n8nSetupToUse);
 	const n8nTag = argv.n8nDockerTag || process.env.N8N_DOCKER_TAG || 'latest';
 	const benchmarkTag = argv.benchmarkDockerTag || process.env.BENCHMARK_DOCKER_TAG || 'latest';
 	const k6ApiToken = argv.k6ApiToken || process.env.K6_API_TOKEN || undefined;
@@ -30,6 +26,7 @@ async function main() {
 			N8N_VERSION: n8nTag,
 			BENCHMARK_VERSION: benchmarkTag,
 			K6_API_TOKEN: k6ApiToken,
+			N8N_BENCHMARK_SCENARIO_NAME_PREFIX: n8nSetupToUse,
 		},
 	});
 
@@ -52,4 +49,28 @@ async function dumpN8nInstanceLogs($$) {
 	await $$`docker-compose logs n8n`;
 }
 
+function printUsage() {
+	const availableSetups = getAllN8nSetups();
+	console.log('Usage: zx runOnVm.mjs <n8n setup to use>');
+	console.log(`   eg: zx runOnVm.mjs ${availableSetups[0]}`);
+	console.log('');
+	console.log('Available setups:');
+	console.log(availableSetups.join(', '));
+}
+
+/**
+ * @returns {string[]}
+ */
+function getAllN8nSetups() {
+	return fs.readdirSync(paths.n8nSetupsDir);
+}
+
+function validateN8nSetup(givenSetup) {
+	const availableSetups = getAllN8nSetups();
+	if (!availableSetups.includes(givenSetup)) {
+		printUsage();
+		process.exit(1);
+	}
+}
+
 main();

packages/@n8n/benchmark/src/commands/run.ts

@@ -34,6 +34,7 @@ export default class RunCommand extends Command {
 				email: config.get('n8n.user.email'),
 				password: config.get('n8n.user.password'),
 			},
+			config.get('scenarioNamePrefix'),
 		);
 
 		const allScenarios = scenarioLoader.loadAll(config.get('testScenariosPath'));

packages/@n8n/benchmark/src/config/config.ts

@@ -31,6 +31,12 @@ const configSchema = {
 			},
 		},
 	},
+	scenarioNamePrefix: {
+		doc: 'Prefix for the scenario name',
+		format: String,
+		default: 'Unnamed',
+		env: 'N8N_BENCHMARK_SCENARIO_NAME_PREFIX',
+	},
 	k6: {
 		executablePath: {
 			doc: 'The path to the k6 binary',

packages/@n8n/benchmark/src/testExecution/k6Executor.ts

@@ -9,6 +9,11 @@ export type K6ExecutorOpts = {
 	n8nApiBaseUrl: string;
 };
 
+export type K6RunOpts = {
+	/** Name of the scenario run. Used e.g. when the run is reported to k6 cloud */
+	scenarioRunName: string;
+};
+
 /**
  * Flag for the k6 CLI.
  * @example ['--duration', '1m']
@@ -36,8 +41,8 @@ export function handleSummary(data) {
 
 	constructor(private readonly opts: K6ExecutorOpts) {}
 
-	async executeTestScenario(scenario: Scenario) {
-		const augmentedTestScriptPath = this.augmentSummaryScript(scenario);
+	async executeTestScenario(scenario: Scenario, { scenarioRunName }: K6RunOpts) {
+		const augmentedTestScriptPath = this.augmentSummaryScript(scenario, scenarioRunName);
 		const runDirPath = path.dirname(augmentedTestScriptPath);
 
 		const flags: K6CliFlag[] = [['--quiet'], ['--duration', '1m'], ['--vus', '5']];
@@ -62,7 +67,7 @@ export function handleSummary(data) {
 			console.log((chunk as Buffer).toString());
 		}
 
-		this.loadEndOfTestSummary(runDirPath, scenario.name);
+		this.loadEndOfTestSummary(runDirPath, scenarioRunName);
 	}
 
 	/**
@@ -70,24 +75,24 @@ export function handleSummary(data) {
 	 *
 	 * @returns Absolute path to the augmented test script
 	 */
-	private augmentSummaryScript(scenario: Scenario) {
+	private augmentSummaryScript(scenario: Scenario, scenarioRunName: string) {
 		const fullTestScriptPath = path.join(scenario.scenarioDirPath, scenario.scriptPath);
 		const testScript = fs.readFileSync(fullTestScriptPath, 'utf8');
-		const summaryScript = this.handleSummaryScript.replace('{{scenarioName}}', scenario.name);
+		const summaryScript = this.handleSummaryScript.replace('{{scenarioName}}', scenarioRunName);
 
 		const augmentedTestScript = `${testScript}\n\n${summaryScript}`;
 
-		const tempFilePath = tmpfile(`${scenario.name}.ts`, augmentedTestScript);
+		const tempFilePath = tmpfile(`${scenarioRunName}.ts`, augmentedTestScript);
 
 		return tempFilePath;
 	}
 
-	private loadEndOfTestSummary(dir: string, scenarioName: string): K6EndOfTestSummary {
-		const summaryReportPath = path.join(dir, `${scenarioName}.summary.json`);
+	private loadEndOfTestSummary(dir: string, scenarioRunName: string): K6EndOfTestSummary {
+		const summaryReportPath = path.join(dir, `${scenarioRunName}.summary.json`);
 		const summaryReport = fs.readFileSync(summaryReportPath, 'utf8');
 
 		try {
-			return JSON.parse(summaryReport);
+			return JSON.parse(summaryReport) as K6EndOfTestSummary;
 		} catch (error) {
 			throw new Error(`Failed to parse the summary report at ${summaryReportPath}`);
 		}

packages/@n8n/benchmark/src/testExecution/scenarioRunner.ts

@@ -17,6 +17,7 @@ export class ScenarioRunner {
 			email: string;
 			password: string;
 		},
+		private readonly scenarioPrefix: string,
 	) {}
 
 	async runManyScenarios(scenarios: Scenario[]) {
@@ -38,13 +39,25 @@ export class ScenarioRunner {
 	}
 
 	private async runSingleTestScenario(testDataImporter: ScenarioDataImporter, scenario: Scenario) {
-		console.log('Running scenario:', scenario.name);
+		const scenarioRunName = this.formTestScenarioRunName(scenario);
+		console.log('Running scenario:', scenarioRunName);
 
 		console.log('Loading and importing data');
 		const testData = await this.dataLoader.loadDataForScenario(scenario);
 		await testDataImporter.importTestScenarioData(testData.workflows);
 
 		console.log('Executing scenario script');
-		await this.k6Executor.executeTestScenario(scenario);
+		await this.k6Executor.executeTestScenario(scenario, {
+			scenarioRunName,
+		});
+	}
+
+	/**
+	 * Forms a name for the scenario by combining prefix and scenario name.
+	 * The benchmarks are ran against different n8n setups, so we use the
+	 * prefix to differentiate between them.
+	 */
+	private formTestScenarioRunName(scenario: Scenario) {
+		return `${this.scenarioPrefix}-${scenario.name}`;
 	}
 }