feat: Report benchmark results (no-changelog) (#10534)

.github/workflows/benchmark-nightly.yml

@@ -23,6 +23,7 @@ env:
   ARM_CLIENT_ID: ${{ secrets.BENCHMARK_ARM_CLIENT_ID }}
   ARM_SUBSCRIPTION_ID: ${{ secrets.BENCHMARK_ARM_SUBSCRIPTION_ID }}
   ARM_TENANT_ID: ${{ secrets.BENCHMARK_ARM_TENANT_ID }}
+  K6_API_TOKEN: ${{ secrets.K6_API_TOKEN }}
 
 permissions:
   id-token: write

packages/@n8n/benchmark/.eslintrc.js

@@ -12,7 +12,7 @@ module.exports = {
 		project: './tsconfig.json',
 	},
 
-	ignorePatterns: ['scenarios/**'],
+	ignorePatterns: ['scenarios/**', 'scripts/**'],
 
 	rules: {
 		'n8n-local-rules/no-plain-errors': 'off',

packages/@n8n/benchmark/scripts/runInCloud.mjs

@@ -15,7 +15,7 @@
 // @ts-check
 import fs from 'fs';
 import minimist from 'minimist';
-import { $, sleep, tmpdir, which } from 'zx';
+import { $, sleep, which } from 'zx';
 import path from 'path';
 import { SshClient } from './sshClient.mjs';
 import { TerraformClient } from './terraformClient.mjs';
@@ -124,6 +124,7 @@ function readAvailableN8nSetups() {
  * @property {string} n8nSetupToUse
  * @property {string} n8nTag
  * @property {string} benchmarkTag
+ * @property {string} [k6ApiToken]
  *
  * @returns {Promise<Config>}
  */
@@ -136,12 +137,14 @@ async function parseAndValidateConfig() {
 	const isVerbose = args.debug || false;
 	const n8nTag = args.n8nTag || process.env.N8N_DOCKER_TAG || 'latest';
 	const benchmarkTag = args.benchmarkTag || process.env.BENCHMARK_DOCKER_TAG || 'latest';
+	const k6ApiToken = args.k6ApiToken || process.env.K6_API_TOKEN || undefined;
 
 	return {
 		isVerbose,
 		n8nSetupToUse,
 		n8nTag,
 		benchmarkTag,
+		k6ApiToken,
 	};
 }
 
@@ -177,6 +180,9 @@ function printUsage() {
 	console.log('  --debug              Enable verbose output');
 	console.log('  --n8nTag             Docker tag for n8n image. Default is latest');
 	console.log('  --benchmarkTag       Docker tag for benchmark cli image. Default is latest');
+	console.log(
+		'  --k6ApiToken         API token for k6 cloud. Default is read from K6_API_TOKEN env var. If omitted, k6 cloud will not be used.',
+	);
 	console.log('');
 	console.log('Available setups:');
 	console.log(`  ${availableSetups.join(', ')}`);

packages/@n8n/benchmark/scripts/runOnVm/n8nSetups/sqlite/docker-compose.yml

@@ -14,3 +14,4 @@ services:
       - n8n
     environment:
       - N8N_BASE_URL=http://n8n:5678
+      - K6_API_TOKEN=${K6_API_TOKEN}

packages/@n8n/benchmark/scripts/runOnVm/runOnVm.mjs

@@ -21,6 +21,7 @@ async function main() {
 	const composeFilePath = path.join(__dirname, 'n8nSetups', n8nSetupToUse);
 	const n8nTag = argv.n8nDockerTag || process.env.N8N_DOCKER_TAG || 'latest';
 	const benchmarkTag = argv.benchmarkDockerTag || process.env.BENCHMARK_DOCKER_TAG || 'latest';
+	const k6ApiToken = argv.k6ApiToken || process.env.K6_API_TOKEN || undefined;
 
 	const $$ = $({
 		cwd: composeFilePath,
@@ -28,6 +29,7 @@ async function main() {
 		env: {
 			N8N_VERSION: n8nTag,
 			BENCHMARK_VERSION: benchmarkTag,
+			K6_API_TOKEN: k6ApiToken,
 		},
 	});
 

packages/@n8n/benchmark/src/commands/run.ts

@@ -25,7 +25,11 @@ export default class RunCommand extends Command {
 		const scenarioRunner = new ScenarioRunner(
 			new N8nApiClient(config.get('n8n.baseUrl')),
 			new ScenarioDataFileLoader(),
-			new K6Executor(config.get('k6ExecutablePath'), config.get('n8n.baseUrl')),
+			new K6Executor({
+				k6ExecutablePath: config.get('k6.executablePath'),
+				k6ApiToken: config.get('k6.apiToken'),
+				n8nApiBaseUrl: config.get('n8n.baseUrl'),
+			}),
 			{
 				email: config.get('n8n.user.email'),
 				password: config.get('n8n.user.password'),

packages/@n8n/benchmark/src/config/config.ts

@@ -31,12 +31,20 @@ const configSchema = {
 			},
 		},
 	},
-	k6ExecutablePath: {
+	k6: {
+		executablePath: {
 			doc: 'The path to the k6 binary',
 			format: String,
 			default: 'k6',
 			env: 'K6_PATH',
 		},
+		apiToken: {
+			doc: 'The API token for k6 cloud',
+			format: String,
+			default: undefined,
+			env: 'K6_API_TOKEN',
+		},
+	},
 };
 
 export type Config = ReturnType<typeof loadConfig>;

packages/@n8n/benchmark/src/testExecution/k6Executor.ts

@@ -1,38 +1,103 @@
-import { $, which } from 'zx';
+import fs from 'fs';
+import path from 'path';
+import { $, which, tmpfile } from 'zx';
 import type { Scenario } from '@/types/scenario';
 
+export type K6ExecutorOpts = {
+	k6ExecutablePath: string;
+	k6ApiToken?: string;
+	n8nApiBaseUrl: string;
+};
+
+/**
+ * Flag for the k6 CLI.
+ * @example ['--duration', '1m']
+ * @example ['--quiet']
+ */
+type K6CliFlag = [string] | [string, string];
+
 /**
  * Executes test scenarios using k6
  */
 export class K6Executor {
-	constructor(
-		private readonly k6ExecutablePath: string,
-		private readonly n8nApiBaseUrl: string,
-	) {}
+	/**
+	 * This script is dynamically injected into the k6 test script to generate
+	 * a summary report of the test execution.
+	 */
+	private readonly handleSummaryScript = `
+import { textSummary } from 'https://jslib.k6.io/k6-summary/0.0.2/index.js';
+export function handleSummary(data) {
+	return {
+		stdout: textSummary(data),
+		'{{scenarioName}}.summary.json': JSON.stringify(data),
+	};
+}
+`;
+
+	constructor(private readonly opts: K6ExecutorOpts) {}
 
 	async executeTestScenario(scenario: Scenario) {
-		// For 1 min with 5 virtual users
-		const stage = '1m:5';
+		const augmentedTestScriptPath = this.augmentSummaryScript(scenario);
+		const runDirPath = path.dirname(augmentedTestScriptPath);
+
+		const flags: K6CliFlag[] = [['--quiet'], ['--duration', '1m'], ['--vus', '5']];
+
+		if (this.opts.k6ApiToken) {
+			flags.push(['--out', 'cloud']);
+		}
+
+		const flattedFlags = flags.flat(2);
 
 		const k6ExecutablePath = await this.resolveK6ExecutablePath();
 
 		const processPromise = $({
-			cwd: scenario.scenarioDirPath,
+			cwd: runDirPath,
 			env: {
-				API_BASE_URL: this.n8nApiBaseUrl,
+				API_BASE_URL: this.opts.n8nApiBaseUrl,
+				K6_CLOUD_TOKEN: this.opts.k6ApiToken,
 			},
-		})`${k6ExecutablePath} run --quiet --stage ${stage} ${scenario.scriptPath}`;
+		})`${k6ExecutablePath} run ${flattedFlags} ${augmentedTestScriptPath}`;
 
 		for await (const chunk of processPromise.stdout) {
 			console.log((chunk as Buffer).toString());
 		}
+
+		this.loadEndOfTestSummary(runDirPath, scenario.name);
+	}
+
+	/**
+	 * Augments the test script with a summary script
+	 *
+	 * @returns Absolute path to the augmented test script
+	 */
+	private augmentSummaryScript(scenario: Scenario) {
+		const fullTestScriptPath = path.join(scenario.scenarioDirPath, scenario.scriptPath);
+		const testScript = fs.readFileSync(fullTestScriptPath, 'utf8');
+		const summaryScript = this.handleSummaryScript.replace('{{scenarioName}}', scenario.name);
+
+		const augmentedTestScript = `${testScript}\n\n${summaryScript}`;
+
+		const tempFilePath = tmpfile(`${scenario.name}.ts`, augmentedTestScript);
+
+		return tempFilePath;
+	}
+
+	private loadEndOfTestSummary(dir: string, scenarioName: string): K6EndOfTestSummary {
+		const summaryReportPath = path.join(dir, `${scenarioName}.summary.json`);
+		const summaryReport = fs.readFileSync(summaryReportPath, 'utf8');
+
+		try {
+			return JSON.parse(summaryReport);
+		} catch (error) {
+			throw new Error(`Failed to parse the summary report at ${summaryReportPath}`);
+		}
 	}
 
 	/**
 	 * @returns Resolved path to the k6 executable
 	 */
 	private async resolveK6ExecutablePath(): Promise<string> {
-		const k6ExecutablePath = await which(this.k6ExecutablePath, { nothrow: true });
+		const k6ExecutablePath = await which(this.opts.k6ExecutablePath, { nothrow: true });
 		if (!k6ExecutablePath) {
 			throw new Error(
 				'Could not find k6 executable based on your `PATH`. Please ensure k6 is available in your system and add it to your `PATH` or specify the path to the k6 executable using the `K6_PATH` environment variable.',

packages/@n8n/benchmark/src/testExecution/k6Summary.ts

@@ -0,0 +1,255 @@
+/**
+Example JSON:
+
+{
+	"options": {
+		"summaryTrendStats": ["avg", "min", "med", "max", "p(90)", "p(95)"],
+		"summaryTimeUnit": "",
+		"noColor": false
+	},
+	"state": { "isStdOutTTY": false, "isStdErrTTY": false, "testRunDurationMs": 23.374 },
+	"metrics": {
+		"http_req_tls_handshaking": {
+			"type": "trend",
+			"contains": "time",
+			"values": { "avg": 0, "min": 0, "med": 0, "max": 0, "p(90)": 0, "p(95)": 0 }
+		},
+		"checks": {
+			"type": "rate",
+			"contains": "default",
+			"values": { "rate": 1, "passes": 1, "fails": 0 }
+		},
+		"http_req_sending": {
+			"type": "trend",
+			"contains": "time",
+			"values": {
+				"p(90)": 0.512,
+				"p(95)": 0.512,
+				"avg": 0.512,
+				"min": 0.512,
+				"med": 0.512,
+				"max": 0.512
+			}
+		},
+		"http_reqs": {
+			"contains": "default",
+			"values": { "count": 1, "rate": 42.78257893385813 },
+			"type": "counter"
+		},
+		"http_req_blocked": {
+			"contains": "time",
+			"values": {
+				"avg": 1.496,
+				"min": 1.496,
+				"med": 1.496,
+				"max": 1.496,
+				"p(90)": 1.496,
+				"p(95)": 1.496
+			},
+			"type": "trend"
+		},
+		"data_received": {
+			"type": "counter",
+			"contains": "data",
+			"values": { "count": 269, "rate": 11508.513733207838 }
+		},
+		"iterations": {
+			"type": "counter",
+			"contains": "default",
+			"values": { "count": 1, "rate": 42.78257893385813 }
+		},
+		"http_req_waiting": {
+			"type": "trend",
+			"contains": "time",
+			"values": {
+				"p(95)": 18.443,
+				"avg": 18.443,
+				"min": 18.443,
+				"med": 18.443,
+				"max": 18.443,
+				"p(90)": 18.443
+			}
+		},
+		"http_req_receiving": {
+			"type": "trend",
+			"contains": "time",
+			"values": {
+				"avg": 0.186,
+				"min": 0.186,
+				"med": 0.186,
+				"max": 0.186,
+				"p(90)": 0.186,
+				"p(95)": 0.186
+			}
+		},
+		"http_req_duration{expected_response:true}": {
+			"type": "trend",
+			"contains": "time",
+			"values": {
+				"max": 19.141,
+				"p(90)": 19.141,
+				"p(95)": 19.141,
+				"avg": 19.141,
+				"min": 19.141,
+				"med": 19.141
+			}
+		},
+		"iteration_duration": {
+			"type": "trend",
+			"contains": "time",
+			"values": {
+				"avg": 22.577833,
+				"min": 22.577833,
+				"med": 22.577833,
+				"max": 22.577833,
+				"p(90)": 22.577833,
+				"p(95)": 22.577833
+			}
+		},
+		"http_req_connecting": {
+			"type": "trend",
+			"contains": "time",
+			"values": {
+				"avg": 0.673,
+				"min": 0.673,
+				"med": 0.673,
+				"max": 0.673,
+				"p(90)": 0.673,
+				"p(95)": 0.673
+			}
+		},
+		"http_req_failed": {
+			"type": "rate",
+			"contains": "default",
+			"values": { "rate": 0, "passes": 0, "fails": 1 }
+		},
+		"http_req_duration": {
+			"type": "trend",
+			"contains": "time",
+			"values": {
+				"p(90)": 19.141,
+				"p(95)": 19.141,
+				"avg": 19.141,
+				"min": 19.141,
+				"med": 19.141,
+				"max": 19.141
+			}
+		},
+		"data_sent": {
+			"type": "counter",
+			"contains": "data",
+			"values": { "count": 102, "rate": 4363.82305125353 }
+		}
+	},
+	"root_group": {
+		"name": "",
+		"path": "",
+		"id": "d41d8cd98f00b204e9800998ecf8427e",
+		"groups": [],
+		"checks": [
+			{
+				"name": "is status 200",
+				"path": "::is status 200",
+				"id": "548d37ca5f33793206f7832e7cea54fb",
+				"passes": 1,
+				"fails": 0
+			}
+		]
+	}
+}
+ */
+
+type TrendStat = 'avg' | 'min' | 'med' | 'max' | 'p(90)' | 'p(95)';
+type MetricType = 'trend' | 'rate' | 'counter';
+type MetricContains = 'time' | 'default' | 'data';
+
+interface TrendValues {
+	avg: number;
+	min: number;
+	med: number;
+	max: number;
+	'p(90)': number;
+	'p(95)': number;
+}
+
+interface RateValues {
+	rate: number;
+	passes: number;
+	fails: number;
+}
+
+interface CounterValues {
+	count: number;
+	rate: number;
+}
+
+interface TrendMetric {
+	type: 'trend';
+	contains: 'time';
+	values: TrendValues;
+}
+
+interface RateMetric {
+	type: 'rate';
+	contains: 'default';
+	values: RateValues;
+}
+
+interface CounterMetric {
+	type: 'counter';
+	contains: MetricContains;
+	values: CounterValues;
+}
+
+interface Options {
+	summaryTrendStats: TrendStat[];
+	summaryTimeUnit: string;
+	noColor: boolean;
+}
+
+interface State {
+	isStdOutTTY: boolean;
+	isStdErrTTY: boolean;
+	testRunDurationMs: number;
+}
+
+interface Metrics {
+	http_req_tls_handshaking: TrendMetric;
+	checks: RateMetric;
+	http_req_sending: TrendMetric;
+	http_reqs: CounterMetric;
+	http_req_blocked: TrendMetric;
+	data_received: CounterMetric;
+	iterations: CounterMetric;
+	http_req_waiting: TrendMetric;
+	http_req_receiving: TrendMetric;
+	'http_req_duration{expected_response:true}': TrendMetric;
+	iteration_duration: TrendMetric;
+	http_req_connecting: TrendMetric;
+	http_req_failed: RateMetric;
+	http_req_duration: TrendMetric;
+	data_sent: CounterMetric;
+}
+
+interface Check {
+	name: string;
+	path: string;
+	id: string;
+	passes: number;
+	fails: number;
+}
+
+interface RootGroup {
+	name: string;
+	path: string;
+	id: string;
+	groups: any[];
+	checks: Check[];
+}
+
+interface K6EndOfTestSummary {
+	options: Options;
+	state: State;
+	metrics: Metrics;
+	root_group: RootGroup;
+}