feat(benchmark): New options for n8n benchmark (#10741)

.github/workflows/benchmark-nightly.yml

@@ -70,7 +70,13 @@ jobs:
         working-directory: packages/@n8n/benchmark
 
       - name: Run the benchmark
-        run: pnpm benchmark-in-cloud --n8nTag ${{ env.N8N_TAG }} --benchmarkTag ${{ env.N8N_BENCHMARK_TAG }} ${{ env.DEBUG }}
+        run: |
+          pnpm benchmark-in-cloud \
+            --vus 5 \
+            --duration 1m \
+            --n8nTag ${{ env.N8N_TAG }} \
+            --benchmarkTag ${{ env.N8N_BENCHMARK_TAG }} \
+            ${{ env.DEBUG }}
         working-directory: packages/@n8n/benchmark
 
         # We need to login again because the access token expires

packages/@n8n/benchmark/package.json

@@ -33,7 +33,6 @@
   "dependencies": {
     "@oclif/core": "4.0.7",
     "axios": "catalog:",
-    "convict": "6.2.4",
     "dotenv": "8.6.0",
     "zx": "^8.1.4"
   },

packages/@n8n/benchmark/scripts/run.mjs

@@ -36,6 +36,8 @@ async function main() {
 			n8nLicenseCert: config.n8nLicenseCert,
 			n8nTag: config.n8nTag,
 			n8nSetupsToUse,
+			vus: config.vus,
+			duration: config.duration,
 		});
 	} else {
 		await runLocally({
@@ -46,6 +48,8 @@ async function main() {
 			n8nTag: config.n8nTag,
 			runDir: config.runDir,
 			n8nSetupsToUse,
+			vus: config.vus,
+			duration: config.duration,
 		});
 	}
 }
@@ -66,6 +70,8 @@ function readAvailableN8nSetups() {
  * @property {string} [k6ApiToken]
  * @property {string} [n8nLicenseCert]
  * @property {string} [runDir]
+ * @property {string} [vus]
+ * @property {string} [duration]
  *
  * @returns {Promise<Config>}
  */
@@ -87,6 +93,8 @@ async function parseAndValidateConfig() {
 	const n8nLicenseCert = args.n8nLicenseCert || process.env.N8N_LICENSE_CERT || undefined;
 	const runDir = args.runDir || undefined;
 	const env = args.env || 'local';
+	const vus = args.vus;
+	const duration = args.duration;
 
 	if (!env) {
 		printUsage();
@@ -102,6 +110,8 @@ async function parseAndValidateConfig() {
 		k6ApiToken,
 		n8nLicenseCert,
 		runDir,
+		vus,
+		duration,
 	};
 }
 
@@ -141,6 +151,8 @@ function printUsage() {
 	console.log('  --debug              Enable verbose output');
 	console.log('  --n8nTag             Docker tag for n8n image. Default is latest');
 	console.log('  --benchmarkTag       Docker tag for benchmark cli image. Default is latest');
+	console.log('  --vus                How many concurrent requests to make');
+	console.log('  --duration           Test duration, e.g. 1m or 30s');
 	console.log(
 		'  --k6ApiToken         API token for k6 cloud. Default is read from K6_API_TOKEN env var. If omitted, k6 cloud will not be used',
 	);

packages/@n8n/benchmark/scripts/runForN8nSetup.mjs

@@ -6,6 +6,7 @@
 import path from 'path';
 import { $, argv, fs } from 'zx';
 import { DockerComposeClient } from './clients/dockerComposeClient.mjs';
+import { flagsObjectToCliArgs } from './utils/flags.mjs';
 
 const paths = {
 	n8nSetupsDir: path.join(__dirname, 'n8nSetups'),
@@ -27,6 +28,8 @@ async function main() {
 	const n8nLicenseCert = argv.n8nLicenseCert || process.env.N8N_LICENSE_CERT || undefined;
 	const n8nLicenseActivationKey = process.env.N8N_LICENSE_ACTIVATION_KEY || '';
 	const n8nLicenseTenantId = argv.n8nLicenseTenantId || process.env.N8N_LICENSE_TENANT_ID || '';
+	const vus = argv.vus;
+	const duration = argv.duration;
 
 	if (!fs.existsSync(baseRunDir)) {
 		console.error(
@@ -66,7 +69,21 @@ async function main() {
 	try {
 		await dockerComposeClient.$('up', '-d', '--remove-orphans', 'n8n');
 
-		await dockerComposeClient.$('run', 'benchmark', 'run', `--scenarioNamePrefix=${n8nSetupToUse}`);
+		const tags = Object.entries({
+			N8nVersion: n8nTag,
+			N8nSetup: n8nSetupToUse,
+		})
+			.map(([key, value]) => `${key}=${value}`)
+			.join(',');
+
+		const cliArgs = flagsObjectToCliArgs({
+			scenarioNamePrefix: n8nSetupToUse,
+			vus,
+			duration,
+			tags,
+		});
+
+		await dockerComposeClient.$('run', 'benchmark', 'run', ...cliArgs);
 	} catch (error) {
 		console.error('An error occurred while running the benchmarks:');
 		console.error(error.message);

packages/@n8n/benchmark/scripts/runInCloud.mjs

@@ -13,6 +13,7 @@ import { sleep, which, $, tmpdir } from 'zx';
 import path from 'path';
 import { SshClient } from './clients/sshClient.mjs';
 import { TerraformClient } from './clients/terraformClient.mjs';
+import { flagsObjectToCliArgs } from './utils/flags.mjs';
 
 /**
  * @typedef {Object} BenchmarkEnv
@@ -30,6 +31,8 @@ import { TerraformClient } from './clients/terraformClient.mjs';
  * @property {string} benchmarkTag
  * @property {string} [k6ApiToken]
  * @property {string} [n8nLicenseCert]
+ * @property {string} [vus]
+ * @property {string} [duration]
  *
  * @param {Config} config
  */
@@ -93,17 +96,16 @@ async function runBenchmarkForN8nSetup({ config, sshClient, scriptsDir, n8nSetup
 	console.log(`Running benchmarks for ${n8nSetup}...`);
 	const runScriptPath = path.join(scriptsDir, 'runForN8nSetup.mjs');
 
-	const flags = {
+	const cliArgs = flagsObjectToCliArgs({
 		n8nDockerTag: config.n8nTag,
 		benchmarkDockerTag: config.benchmarkTag,
 		k6ApiToken: config.k6ApiToken,
 		n8nLicenseCert: config.n8nLicenseCert,
-	};
+		vus: config.vus,
+		duration: config.duration,
+	});
 
-	const flagsString = Object.entries(flags)
-		.filter(([, value]) => value !== undefined)
-		.map(([key, value]) => `--${key}=${value}`)
-		.join(' ');
+	const flagsString = cliArgs.join(' ');
 
 	await sshClient.ssh(`npx zx ${runScriptPath} ${flagsString} ${n8nSetup}`, {
 		// Test run should always log its output

packages/@n8n/benchmark/scripts/runLocally.mjs

@@ -11,6 +11,7 @@
 // @ts-check
 import { $ } from 'zx';
 import path from 'path';
+import { flagsObjectToCliArgs } from './utils/flags.mjs';
 
 /**
  * @typedef {Object} BenchmarkEnv
@@ -30,19 +31,21 @@ const paths = {
  * @property {string} [runDir]
  * @property {string} [k6ApiToken]
  * @property {string} [n8nLicenseCert]
+ * @property {string} [vus]
+ * @property {string} [duration]
  *
  * @param {Config} config
  */
 export async function runLocally(config) {
 	const runScriptPath = path.join(paths.scriptsDir, 'runForN8nSetup.mjs');
 
-	const flags = Object.entries({
+	const cliArgs = flagsObjectToCliArgs({
 		n8nDockerTag: config.n8nTag,
 		benchmarkDockerTag: config.benchmarkTag,
 		runDir: config.runDir,
-	})
-		.filter(([, value]) => value !== undefined)
-		.map(([key, value]) => `--${key}=${value}`);
+		vus: config.vus,
+		duration: config.duration,
+	});
 
 	try {
 		for (const n8nSetup of config.n8nSetupsToUse) {
@@ -54,7 +57,7 @@ export async function runLocally(config) {
 					K6_API_TOKEN: config.k6ApiToken,
 					N8N_LICENSE_CERT: config.n8nLicenseCert,
 				},
-			})`npx ${runScriptPath} ${flags} ${n8nSetup}`;
+			})`npx ${runScriptPath} ${cliArgs} ${n8nSetup}`;
 		}
 	} catch (error) {
 		console.error('An error occurred while running the benchmarks:');

packages/@n8n/benchmark/scripts/utils/flags.mjs

@@ -0,0 +1,14 @@
+// @ts-check
+
+/**
+ * Converts an object of flags to an array of CLI arguments.
+ *
+ * @param {Record<string, string | undefined>} flags
+ *
+ * @returns {string[]}
+ */
+export function flagsObjectToCliArgs(flags) {
+	return Object.entries(flags)
+		.filter(([, value]) => value !== undefined)
+		.map(([key, value]) => `--${key}=${value}`);
+}

packages/@n8n/benchmark/src/commands/list.ts

@@ -1,15 +1,19 @@
 import { Command } from '@oclif/core';
 import { ScenarioLoader } from '@/scenario/scenarioLoader';
-import { loadConfig } from '@/config/config';
+import { testScenariosPath } from '@/config/commonFlags';
 
 export default class ListCommand extends Command {
 	static description = 'List all available scenarios';
 
+	static flags = {
+		testScenariosPath,
+	};
+
 	async run() {
-		const config = loadConfig();
+		const { flags } = await this.parse(ListCommand);
 		const scenarioLoader = new ScenarioLoader();
 
-		const allScenarios = scenarioLoader.loadAll(config.get('testScenariosPath'));
+		const allScenarios = scenarioLoader.loadAll(flags.testScenariosPath);
 
 		console.log('Available test scenarios:');
 		console.log('');

packages/@n8n/benchmark/src/commands/run.ts

@@ -1,59 +1,97 @@
 import { Command, Flags } from '@oclif/core';
-import { loadConfig } from '@/config/config';
 import { ScenarioLoader } from '@/scenario/scenarioLoader';
 import { ScenarioRunner } from '@/testExecution/scenarioRunner';
 import { N8nApiClient } from '@/n8nApiClient/n8nApiClient';
 import { ScenarioDataFileLoader } from '@/scenario/scenarioDataLoader';
+import type { K6Tag } from '@/testExecution/k6Executor';
 import { K6Executor } from '@/testExecution/k6Executor';
+import { testScenariosPath } from '@/config/commonFlags';
 
 export default class RunCommand extends Command {
 	static description = 'Run all (default) or specified test scenarios';
 
-	// TODO: Add support for filtering scenarios
 	static flags = {
-		scenarios: Flags.string({
-			char: 't',
-			description: 'Comma-separated list of test scenarios to run',
-			required: false,
-		}),
+		testScenariosPath,
 		scenarioNamePrefix: Flags.string({
-			description: 'Prefix for the scenario name. Defaults to Unnamed',
-			required: false,
+			description: 'Prefix for the scenario name',
+			default: 'Unnamed',
+		}),
+		n8nBaseUrl: Flags.string({
+			description: 'The base URL for the n8n instance',
+			default: 'http://localhost:5678',
+			env: 'N8N_BASE_URL',
+		}),
+		n8nUserEmail: Flags.string({
+			description: 'The email address of the n8n user',
+			default: 'benchmark-user@n8n.io',
+			env: 'N8N_USER_EMAIL',
+		}),
+		k6ExecutablePath: Flags.string({
+			doc: 'The path to the k6 binary',
+			default: 'k6',
+			env: 'K6_PATH',
+		}),
+		k6ApiToken: Flags.string({
+			doc: 'The API token for k6 cloud',
+			default: undefined,
+			env: 'K6_API_TOKEN',
+		}),
+		n8nUserPassword: Flags.string({
+			description: 'The password of the n8n user',
+			default: 'VerySecret!123',
+			env: 'N8N_USER_PASSWORD',
+		}),
+		tags: Flags.string({
+			char: 't',
+			description: 'Tags to attach to the run. Comma separated list of key=value pairs',
+		}),
+		vus: Flags.integer({
+			description: 'Number of concurrent requests to make',
+			default: 5,
+		}),
+		duration: Flags.string({
+			description: 'Duration of the test with a unit, e.g. 1m',
+			default: '1m',
 		}),
 	};
 
 	async run() {
-		const config = await this.loadConfigAndMergeWithFlags();
+		const { flags } = await this.parse(RunCommand);
+		const tags = await this.parseTags();
 		const scenarioLoader = new ScenarioLoader();
 
 		const scenarioRunner = new ScenarioRunner(
-			new N8nApiClient(config.get('n8n.baseUrl')),
+			new N8nApiClient(flags.n8nBaseUrl),
 			new ScenarioDataFileLoader(),
 			new K6Executor({
-				k6ExecutablePath: config.get('k6.executablePath'),
-				k6ApiToken: config.get('k6.apiToken'),
-				n8nApiBaseUrl: config.get('n8n.baseUrl'),
+				duration: flags.duration,
+				vus: flags.vus,
+				k6ExecutablePath: flags.k6ExecutablePath,
+				k6ApiToken: flags.k6ApiToken,
+				n8nApiBaseUrl: flags.n8nBaseUrl,
+				tags,
 			}),
 			{
-				email: config.get('n8n.user.email'),
-				password: config.get('n8n.user.password'),
+				email: flags.n8nUserEmail,
+				password: flags.n8nUserPassword,
 			},
-			config.get('scenarioNamePrefix'),
+			flags.scenarioNamePrefix,
 		);
 
-		const allScenarios = scenarioLoader.loadAll(config.get('testScenariosPath'));
+		const allScenarios = scenarioLoader.loadAll(flags.testScenariosPath);
 
 		await scenarioRunner.runManyScenarios(allScenarios);
 	}
 
-	private async loadConfigAndMergeWithFlags() {
-		const config = loadConfig();
+	private async parseTags(): Promise<K6Tag[]> {
 		const { flags } = await this.parse(RunCommand);
-
-		if (flags.scenarioNamePrefix) {
-			config.set('scenarioNamePrefix', flags.scenarioNamePrefix);
+		if (!flags.tags) {
+			return [];
 		}
 
-		return config;
+		return flags.tags.split(',').map((tag) => {
+			const [name, value] = tag.split('=');
+			return { name, value };
+		});
 	}
 }

packages/@n8n/benchmark/src/config/commonFlags.ts

@@ -0,0 +1,6 @@
+import { Flags } from '@oclif/core';
+
+export const testScenariosPath = Flags.string({
+	description: 'The path to the scenarios',
+	default: 'scenarios',
+});

packages/@n8n/benchmark/src/config/config.ts

@@ -1,64 +0,0 @@
-import convict from 'convict';
-import dotenv from 'dotenv';
-
-dotenv.config();
-
-const configSchema = {
-	testScenariosPath: {
-		doc: 'The path to the scenarios',
-		format: String,
-		default: 'scenarios',
-	},
-	n8n: {
-		baseUrl: {
-			doc: 'The base URL for the n8n instance',
-			format: String,
-			default: 'http://localhost:5678',
-			env: 'N8N_BASE_URL',
-		},
-		user: {
-			email: {
-				doc: 'The email address of the n8n user',
-				format: String,
-				default: 'benchmark-user@n8n.io',
-				env: 'N8N_USER_EMAIL',
-			},
-			password: {
-				doc: 'The password of the n8n user',
-				format: String,
-				default: 'VerySecret!123',
-				env: 'N8N_USER_PASSWORD',
-			},
-		},
-	},
-	scenarioNamePrefix: {
-		doc: 'Prefix for the scenario name',
-		format: String,
-		default: 'Unnamed',
-		env: 'N8N_BENCHMARK_SCENARIO_NAME_PREFIX',
-	},
-	k6: {
-		executablePath: {
-			doc: 'The path to the k6 binary',
-			format: String,
-			default: 'k6',
-			env: 'K6_PATH',
-		},
-		apiToken: {
-			doc: 'The API token for k6 cloud',
-			format: String,
-			default: undefined,
-			env: 'K6_API_TOKEN',
-		},
-	},
-};
-
-export type Config = ReturnType<typeof loadConfig>;
-
-export function loadConfig() {
-	const config = convict(configSchema);
-
-	config.validate({ allowed: 'strict' });
-
-	return config;
-}

packages/@n8n/benchmark/src/testExecution/k6Executor.ts

@@ -3,10 +3,20 @@ import path from 'path';
 import { $, which, tmpfile } from 'zx';
 import type { Scenario } from '@/types/scenario';
 
+export type K6Tag = {
+	name: string;
+	value: string;
+};
+
 export type K6ExecutorOpts = {
 	k6ExecutablePath: string;
+	/** How many concurrent requests to make */
+	vus: number;
+	/** Test duration, e.g. 1m or 30s */
+	duration: string;
 	k6ApiToken?: string;
 	n8nApiBaseUrl: string;
+	tags?: K6Tag[];
 };
 
 export type K6RunOpts = {
@@ -19,7 +29,7 @@ export type K6RunOpts = {
  * @example ['--duration', '1m']
  * @example ['--quiet']
  */
-type K6CliFlag = [string] | [string, string];
+type K6CliFlag = [string | number] | [string, string | number];
 
 /**
  * Executes test scenarios using k6
@@ -45,7 +55,11 @@ export function handleSummary(data) {
 		const augmentedTestScriptPath = this.augmentSummaryScript(scenario, scenarioRunName);
 		const runDirPath = path.dirname(augmentedTestScriptPath);
 
-		const flags: K6CliFlag[] = [['--quiet'], ['--duration', '3m'], ['--vus', '5']];
+		const flags: K6CliFlag[] = [
+			['--quiet'],
+			['--duration', this.opts.duration],
+			['--vus', this.opts.vus],
+		];
 
 		if (this.opts.k6ApiToken) {
 			flags.push(['--out', 'cloud']);

packages/@n8n/benchmark/src/testExecution/k6Summary.ts

@@ -243,7 +243,7 @@ interface RootGroup {
 	name: string;
 	path: string;
 	id: string;
-	groups: any[];
+	groups: unknown[];
 	checks: Check[];
 }
 

pnpm-lock.yaml

@@ -226,9 +226,6 @@ importers:
       axios:
         specifier: 'catalog:'
         version: 1.7.4(debug@4.3.6)
-      convict:
-        specifier: 6.2.4
-        version: 6.2.4
       dotenv:
         specifier: 8.6.0
         version: 8.6.0
@@ -21628,7 +21625,7 @@ snapshots:
 
   eslint-import-resolver-node@0.3.9:
     dependencies:
-      debug: 3.2.7(supports-color@5.5.0)
+      debug: 3.2.7(supports-color@8.1.1)
       is-core-module: 2.13.1
       resolve: 1.22.8
     transitivePeerDependencies:
@@ -21653,7 +21650,7 @@ snapshots:
 
   eslint-module-utils@2.8.0(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.5.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.5.2))(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0):
     dependencies:
-      debug: 3.2.7(supports-color@5.5.0)
+      debug: 3.2.7(supports-color@8.1.1)
     optionalDependencies:
       '@typescript-eslint/parser': 7.2.0(eslint@8.57.0)(typescript@5.5.2)
       eslint: 8.57.0
@@ -21673,7 +21670,7 @@ snapshots:
       array.prototype.findlastindex: 1.2.3
       array.prototype.flat: 1.3.2
       array.prototype.flatmap: 1.3.2
-      debug: 3.2.7(supports-color@5.5.0)
+      debug: 3.2.7(supports-color@8.1.1)
       doctrine: 2.1.0
       eslint: 8.57.0
       eslint-import-resolver-node: 0.3.9
@@ -22543,7 +22540,7 @@ snapshots:
       array-parallel: 0.1.3
       array-series: 0.1.5
       cross-spawn: 4.0.2
-      debug: 3.2.7(supports-color@5.5.0)
+      debug: 3.2.7(supports-color@8.1.1)
     transitivePeerDependencies:
       - supports-color
 
@@ -25564,7 +25561,7 @@ snapshots:
 
   pdf-parse@1.1.1:
     dependencies:
-      debug: 3.2.7(supports-color@5.5.0)
+      debug: 3.2.7(supports-color@8.1.1)
       node-ensure: 0.0.0
     transitivePeerDependencies:
       - supports-color
@@ -26450,7 +26447,7 @@ snapshots:
 
   rhea@1.0.24:
     dependencies:
-      debug: 3.2.7(supports-color@5.5.0)
+      debug: 3.2.7(supports-color@8.1.1)
     transitivePeerDependencies:
       - supports-color