feat(benchmark): Report benchmark results to a configurable webhook (#10754)

.github/workflows/benchmark-nightly.yml

@@ -23,7 +23,8 @@ env:
   ARM_CLIENT_ID: ${{ secrets.BENCHMARK_ARM_CLIENT_ID }}
   ARM_SUBSCRIPTION_ID: ${{ secrets.BENCHMARK_ARM_SUBSCRIPTION_ID }}
   ARM_TENANT_ID: ${{ secrets.BENCHMARK_ARM_TENANT_ID }}
-  K6_API_TOKEN: ${{ secrets.K6_API_TOKEN }}
+  BENCHMARK_RESULT_WEBHOOK_URL: ${{ secrets.BENCHMARK_RESULT_WEBHOOK_URL }}
+  BENCHMARK_RESULT_WEBHOOK_AUTH_HEADER: ${{ secrets.BENCHMARK_RESULT_WEBHOOK_AUTH_HEADER }}
   N8N_TAG: ${{ inputs.n8n_tag || 'nightly' }}
   N8N_BENCHMARK_TAG: ${{ inputs.benchmark_tag || 'latest' }}
   DEBUG: ${{ inputs.debug == 'true' && '--debug' || '' }}

packages/@n8n/benchmark/package.json

@@ -34,6 +34,7 @@
     "@oclif/core": "4.0.7",
     "axios": "catalog:",
     "dotenv": "8.6.0",
+    "nanoid": "catalog:",
     "zx": "^8.1.4"
   },
   "devDependencies": {

packages/@n8n/benchmark/scripts/n8nSetups/postgres/docker-compose.yml

@@ -55,3 +55,5 @@ services:
     environment:
       - N8N_BASE_URL=http://n8n:5678
       - K6_API_TOKEN=${K6_API_TOKEN}
+      - BENCHMARK_RESULT_WEBHOOK_URL=${BENCHMARK_RESULT_WEBHOOK_URL}
+      - BENCHMARK_RESULT_WEBHOOK_AUTH_HEADER=${BENCHMARK_RESULT_WEBHOOK_AUTH_HEADER}

packages/@n8n/benchmark/scripts/n8nSetups/queue/docker-compose.yml

@@ -138,3 +138,5 @@ services:
     environment:
       - N8N_BASE_URL=http://n8n:5678
       - K6_API_TOKEN=${K6_API_TOKEN}
+      - BENCHMARK_RESULT_WEBHOOK_URL=${BENCHMARK_RESULT_WEBHOOK_URL}
+      - BENCHMARK_RESULT_WEBHOOK_AUTH_HEADER=${BENCHMARK_RESULT_WEBHOOK_AUTH_HEADER}

packages/@n8n/benchmark/scripts/n8nSetups/scaling/docker-compose.yml

@@ -192,3 +192,5 @@ services:
     environment:
       - N8N_BASE_URL=http://n8n:80
       - K6_API_TOKEN=${K6_API_TOKEN}
+      - BENCHMARK_RESULT_WEBHOOK_URL=${BENCHMARK_RESULT_WEBHOOK_URL}
+      - BENCHMARK_RESULT_WEBHOOK_AUTH_HEADER=${BENCHMARK_RESULT_WEBHOOK_AUTH_HEADER}

packages/@n8n/benchmark/scripts/n8nSetups/sqlite-legacy/docker-compose.yml

@@ -33,3 +33,5 @@ services:
     environment:
       - N8N_BASE_URL=http://n8n:5678
       - K6_API_TOKEN=${K6_API_TOKEN}
+      - BENCHMARK_RESULT_WEBHOOK_URL=${BENCHMARK_RESULT_WEBHOOK_URL}
+      - BENCHMARK_RESULT_WEBHOOK_AUTH_HEADER=${BENCHMARK_RESULT_WEBHOOK_AUTH_HEADER}

packages/@n8n/benchmark/scripts/n8nSetups/sqlite/docker-compose.yml

@@ -35,3 +35,5 @@ services:
     environment:
       - N8N_BASE_URL=http://n8n:5678
       - K6_API_TOKEN=${K6_API_TOKEN}
+      - BENCHMARK_RESULT_WEBHOOK_URL=${BENCHMARK_RESULT_WEBHOOK_URL}
+      - BENCHMARK_RESULT_WEBHOOK_AUTH_HEADER=${BENCHMARK_RESULT_WEBHOOK_AUTH_HEADER}

packages/@n8n/benchmark/scripts/run.mjs

@@ -33,17 +33,21 @@ async function main() {
 			benchmarkTag: config.benchmarkTag,
 			isVerbose: config.isVerbose,
 			k6ApiToken: config.k6ApiToken,
+			resultWebhookUrl: config.resultWebhookUrl,
+			resultWebhookAuthHeader: config.resultWebhookAuthHeader,
 			n8nLicenseCert: config.n8nLicenseCert,
 			n8nTag: config.n8nTag,
 			n8nSetupsToUse,
 			vus: config.vus,
 			duration: config.duration,
 		});
-	} else {
+	} else if (config.env === 'local') {
 		await runLocally({
 			benchmarkTag: config.benchmarkTag,
 			isVerbose: config.isVerbose,
 			k6ApiToken: config.k6ApiToken,
+			resultWebhookUrl: config.resultWebhookUrl,
+			resultWebhookAuthHeader: config.resultWebhookAuthHeader,
 			n8nLicenseCert: config.n8nLicenseCert,
 			n8nTag: config.n8nTag,
 			runDir: config.runDir,
@@ -51,6 +55,10 @@ async function main() {
 			vus: config.vus,
 			duration: config.duration,
 		});
+	} else {
+		console.error('Invalid env:', config.env);
+		printUsage();
+		process.exit(1);
 	}
 }
 
@@ -68,6 +76,8 @@ function readAvailableN8nSetups() {
  * @property {string} n8nTag
  * @property {string} benchmarkTag
  * @property {string} [k6ApiToken]
+ * @property {string} [resultWebhookUrl]
+ * @property {string} [resultWebhookAuthHeader]
  * @property {string} [n8nLicenseCert]
  * @property {string} [runDir]
  * @property {string} [vus]
@@ -90,6 +100,10 @@ async function parseAndValidateConfig() {
 	const n8nTag = args.n8nTag || process.env.N8N_DOCKER_TAG || 'latest';
 	const benchmarkTag = args.benchmarkTag || process.env.BENCHMARK_DOCKER_TAG || 'latest';
 	const k6ApiToken = args.k6ApiToken || process.env.K6_API_TOKEN || undefined;
+	const resultWebhookUrl =
+		args.resultWebhookUrl || process.env.BENCHMARK_RESULT_WEBHOOK_URL || undefined;
+	const resultWebhookAuthHeader =
+		args.resultWebhookAuthHeader || process.env.BENCHMARK_RESULT_WEBHOOK_AUTH_HEADER || undefined;
 	const n8nLicenseCert = args.n8nLicenseCert || process.env.N8N_LICENSE_CERT || undefined;
 	const runDir = args.runDir || undefined;
 	const env = args.env || 'local';
@@ -108,6 +122,8 @@ async function parseAndValidateConfig() {
 		n8nTag,
 		benchmarkTag,
 		k6ApiToken,
+		resultWebhookUrl,
+		resultWebhookAuthHeader,
 		n8nLicenseCert,
 		runDir,
 		vus,

packages/@n8n/benchmark/scripts/runForN8nSetup.mjs

@@ -24,10 +24,15 @@ async function main() {
 	const n8nTag = argv.n8nDockerTag || process.env.N8N_DOCKER_TAG || 'latest';
 	const benchmarkTag = argv.benchmarkDockerTag || process.env.BENCHMARK_DOCKER_TAG || 'latest';
 	const k6ApiToken = argv.k6ApiToken || process.env.K6_API_TOKEN || undefined;
+	const resultWebhookUrl =
+		argv.resultWebhookUrl || process.env.BENCHMARK_RESULT_WEBHOOK_URL || undefined;
+	const resultWebhookAuthHeader =
+		argv.resultWebhookAuthHeader || process.env.BENCHMARK_RESULT_WEBHOOK_AUTH_HEADER || undefined;
 	const baseRunDir = argv.runDir || process.env.RUN_DIR || '/n8n';
 	const n8nLicenseCert = argv.n8nLicenseCert || process.env.N8N_LICENSE_CERT || undefined;
 	const n8nLicenseActivationKey = process.env.N8N_LICENSE_ACTIVATION_KEY || '';
 	const n8nLicenseTenantId = argv.n8nLicenseTenantId || process.env.N8N_LICENSE_TENANT_ID || '';
+	const envTag = argv.env || 'local';
 	const vus = argv.vus;
 	const duration = argv.duration;
 
@@ -54,6 +59,8 @@ async function main() {
 				N8N_ENCRYPTION_KEY,
 				BENCHMARK_VERSION: benchmarkTag,
 				K6_API_TOKEN: k6ApiToken,
+				BENCHMARK_RESULT_WEBHOOK_URL: resultWebhookUrl,
+				BENCHMARK_RESULT_WEBHOOK_AUTH_HEADER: resultWebhookAuthHeader,
 				RUN_DIR: runDir,
 				MOCK_API_DATA_PATH: paths.mockApiDataPath,
 			},
@@ -70,6 +77,7 @@ async function main() {
 		await dockerComposeClient.$('up', '-d', '--remove-orphans', 'n8n');
 
 		const tags = Object.entries({
+			Env: envTag,
 			N8nVersion: n8nTag,
 			N8nSetup: n8nSetupToUse,
 		})

packages/@n8n/benchmark/scripts/runInCloud.mjs

@@ -30,6 +30,8 @@ import { flagsObjectToCliArgs } from './utils/flags.mjs';
  * @property {string} n8nTag
  * @property {string} benchmarkTag
  * @property {string} [k6ApiToken]
+ * @property {string} [resultWebhookUrl]
+ * @property {string} [resultWebhookAuthHeader]
  * @property {string} [n8nLicenseCert]
  * @property {string} [vus]
  * @property {string} [duration]
@@ -100,9 +102,12 @@ async function runBenchmarkForN8nSetup({ config, sshClient, scriptsDir, n8nSetup
 		n8nDockerTag: config.n8nTag,
 		benchmarkDockerTag: config.benchmarkTag,
 		k6ApiToken: config.k6ApiToken,
+		resultWebhookUrl: config.resultWebhookUrl,
+		resultWebhookAuthHeader: config.resultWebhookAuthHeader,
 		n8nLicenseCert: config.n8nLicenseCert,
 		vus: config.vus,
 		duration: config.duration,
+		env: 'cloud',
 	});
 
 	const flagsString = cliArgs.join(' ');

packages/@n8n/benchmark/scripts/runLocally.mjs

@@ -30,6 +30,8 @@ const paths = {
  * @property {string} benchmarkTag
  * @property {string} [runDir]
  * @property {string} [k6ApiToken]
+ * @property {string} [resultWebhookUrl]
+ * @property {string} [resultWebhookAuthHeader]
  * @property {string} [n8nLicenseCert]
  * @property {string} [vus]
  * @property {string} [duration]
@@ -45,6 +47,7 @@ export async function runLocally(config) {
 		runDir: config.runDir,
 		vus: config.vus,
 		duration: config.duration,
+		env: 'local',
 	});
 
 	try {
@@ -55,6 +58,8 @@ export async function runLocally(config) {
 				env: {
 					...process.env,
 					K6_API_TOKEN: config.k6ApiToken,
+					BENCHMARK_RESULT_WEBHOOK_URL: config.resultWebhookUrl,
+					BENCHMARK_RESULT_WEBHOOK_AUTH_HEADER: config.resultWebhookAuthHeader,
 					N8N_LICENSE_CERT: config.n8nLicenseCert,
 				},
 			})`npx ${runScriptPath} ${cliArgs} ${n8nSetup}`;

packages/@n8n/benchmark/src/commands/run.ts

@@ -36,6 +36,16 @@ export default class RunCommand extends Command {
 			default: undefined,
 			env: 'K6_API_TOKEN',
 		}),
+		resultWebhookUrl: Flags.string({
+			doc: 'The URL where the benchmark results should be sent to',
+			default: undefined,
+			env: 'BENCHMARK_RESULT_WEBHOOK_URL',
+		}),
+		resultWebhookAuthHeader: Flags.string({
+			doc: 'The Authorization header value for the benchmark results webhook',
+			default: undefined,
+			env: 'BENCHMARK_RESULT_WEBHOOK_AUTH_HEADER',
+		}),
 		n8nUserPassword: Flags.string({
 			description: 'The password of the n8n user',
 			default: 'VerySecret!123',
@@ -70,6 +80,10 @@ export default class RunCommand extends Command {
 				k6ApiToken: flags.k6ApiToken,
 				n8nApiBaseUrl: flags.n8nBaseUrl,
 				tags,
+				resultsWebhook: {
+					url: flags.resultWebhookUrl,
+					authHeader: flags.resultWebhookAuthHeader,
+				},
 			}),
 			{
 				email: flags.n8nUserEmail,

packages/@n8n/benchmark/src/testExecution/k6Executor.ts

@@ -1,12 +1,10 @@
 import fs from 'fs';
 import path from 'path';
+import assert from 'node:assert/strict';
 import { $, which, tmpfile } from 'zx';
 import type { Scenario } from '@/types/scenario';
-
+import { buildTestReport, type K6Tag } from '@/testExecution/testReport';
-export type K6Tag = {
+export type { K6Tag };
-	name: string;
-	value: string;
-};
 
 export type K6ExecutorOpts = {
 	k6ExecutablePath: string;
@@ -17,6 +15,10 @@ export type K6ExecutorOpts = {
 	k6ApiToken?: string;
 	n8nApiBaseUrl: string;
 	tags?: K6Tag[];
+	resultsWebhook?: {
+		url: string;
+		authHeader: string;
+	};
 };
 
 export type K6RunOpts = {
@@ -61,7 +63,7 @@ export function handleSummary(data) {
 			['--vus', this.opts.vus],
 		];
 
-		if (this.opts.k6ApiToken) {
+		if (!this.opts.resultsWebhook && this.opts.k6ApiToken) {
 			flags.push(['--out', 'cloud']);
 		}
 
@@ -69,20 +71,46 @@ export function handleSummary(data) {
 
 		const k6ExecutablePath = await this.resolveK6ExecutablePath();
 
-		const processPromise = $({
+		await $({
 			cwd: runDirPath,
 			env: {
 				API_BASE_URL: this.opts.n8nApiBaseUrl,
 				K6_CLOUD_TOKEN: this.opts.k6ApiToken,
 				SCRIPT_FILE_PATH: augmentedTestScriptPath,
 			},
+			stdio: 'inherit',
 		})`${k6ExecutablePath} run ${flattedFlags} ${augmentedTestScriptPath}`;
 
-		for await (const chunk of processPromise.stdout) {
+		console.log('\n');
-			console.log((chunk as Buffer).toString());
+
+		if (this.opts.resultsWebhook) {
+			const endOfTestSummary = this.loadEndOfTestSummary(runDirPath, scenarioRunName);
+
+			const testReport = buildTestReport(scenario, endOfTestSummary, [
+				...(this.opts.tags ?? []),
+				{ name: 'Vus', value: this.opts.vus.toString() },
+				{ name: 'Duration', value: this.opts.duration.toString() },
+			]);
+
+			await this.sendTestReport(testReport);
+		}
 	}
 
-		this.loadEndOfTestSummary(runDirPath, scenarioRunName);
+	async sendTestReport(testReport: unknown) {
+		assert(this.opts.resultsWebhook);
+
+		const response = await fetch(this.opts.resultsWebhook.url, {
+			method: 'POST',
+			body: JSON.stringify(testReport),
+			headers: {
+				Authorization: this.opts.resultsWebhook.authHeader,
+				'Content-Type': 'application/json',
+			},
+		});
+
+		if (!response.ok) {
+			console.warn(`Failed to send test summary: ${response.status} ${await response.text()}`);
+		}
 	}
 
 	/**

packages/@n8n/benchmark/src/testExecution/k6Summary.ts

@@ -183,7 +183,7 @@ interface CounterValues {
 	rate: number;
 }
 
-interface TrendMetric {
+interface K6TrendMetric {
 	type: 'trend';
 	contains: 'time';
 	values: TrendValues;
@@ -195,7 +195,7 @@ interface RateMetric {
 	values: RateValues;
 }
 
-interface CounterMetric {
+interface K6CounterMetric {
 	type: 'counter';
 	contains: MetricContains;
 	values: CounterValues;
@@ -214,24 +214,24 @@ interface State {
 }
 
 interface Metrics {
-	http_req_tls_handshaking: TrendMetric;
+	http_req_tls_handshaking: K6TrendMetric;
 	checks: RateMetric;
-	http_req_sending: TrendMetric;
+	http_req_sending: K6TrendMetric;
-	http_reqs: CounterMetric;
+	http_reqs: K6CounterMetric;
-	http_req_blocked: TrendMetric;
+	http_req_blocked: K6TrendMetric;
-	data_received: CounterMetric;
+	data_received: K6CounterMetric;
-	iterations: CounterMetric;
+	iterations: K6CounterMetric;
-	http_req_waiting: TrendMetric;
+	http_req_waiting: K6TrendMetric;
-	http_req_receiving: TrendMetric;
+	http_req_receiving: K6TrendMetric;
-	'http_req_duration{expected_response:true}': TrendMetric;
+	'http_req_duration{expected_response:true}': K6TrendMetric;
-	iteration_duration: TrendMetric;
+	iteration_duration: K6TrendMetric;
-	http_req_connecting: TrendMetric;
+	http_req_connecting: K6TrendMetric;
 	http_req_failed: RateMetric;
-	http_req_duration: TrendMetric;
+	http_req_duration: K6TrendMetric;
-	data_sent: CounterMetric;
+	data_sent: K6CounterMetric;
 }
 
-interface Check {
+interface K6Check {
 	name: string;
 	path: string;
 	id: string;
@@ -244,7 +244,7 @@ interface RootGroup {
 	path: string;
 	id: string;
 	groups: unknown[];
-	checks: Check[];
+	checks: K6Check[];
 }
 
 interface K6EndOfTestSummary {

packages/@n8n/benchmark/src/testExecution/testReport.ts

@@ -0,0 +1,102 @@
+import { nanoid } from 'nanoid';
+import type { Scenario } from '@/types/scenario';
+
+export type K6Tag = {
+	name: string;
+	value: string;
+};
+
+export type Check = {
+	name: string;
+	passes: number;
+	fails: number;
+};
+
+export type CounterMetric = {
+	type: 'counter';
+	count: number;
+	rate: number;
+};
+
+export type TrendMetric = {
+	type: 'trend';
+	'p(95)': number;
+	avg: number;
+	min: number;
+	med: number;
+	max: number;
+	'p(90)': number;
+};
+
+export type TestReport = {
+	runId: string;
+	ts: string; // ISO8601
+	scenarioName: string;
+	tags: K6Tag[];
+	metrics: {
+		iterations: CounterMetric;
+		dataReceived: CounterMetric;
+		dataSent: CounterMetric;
+		httpRequests: CounterMetric;
+		httpRequestDuration: TrendMetric;
+		httpRequestSending: TrendMetric;
+		httpRequestReceiving: TrendMetric;
+		httpRequestWaiting: TrendMetric;
+	};
+	checks: Check[];
+};
+
+function k6CheckToCheck(check: K6Check): Check {
+	return {
+		name: check.name,
+		passes: check.passes,
+		fails: check.fails,
+	};
+}
+
+function k6CounterToCounter(counter: K6CounterMetric): CounterMetric {
+	return {
+		type: 'counter',
+		count: counter.values.count,
+		rate: counter.values.rate,
+	};
+}
+
+function k6TrendToTrend(trend: K6TrendMetric): TrendMetric {
+	return {
+		type: 'trend',
+		'p(90)': trend.values['p(90)'],
+		avg: trend.values.avg,
+		min: trend.values.min,
+		med: trend.values.med,
+		max: trend.values.max,
+		'p(95)': trend.values['p(95)'],
+	};
+}
+
+/**
+ * Converts the k6 test summary to a test report
+ */
+export function buildTestReport(
+	scenario: Scenario,
+	endOfTestSummary: K6EndOfTestSummary,
+	tags: K6Tag[],
+): TestReport {
+	return {
+		runId: nanoid(),
+		ts: new Date().toISOString(),
+		scenarioName: scenario.name,
+		tags,
+		checks: endOfTestSummary.root_group.checks.map(k6CheckToCheck),
+		metrics: {
+			dataReceived: k6CounterToCounter(endOfTestSummary.metrics.data_received),
+			dataSent: k6CounterToCounter(endOfTestSummary.metrics.data_sent),
+			httpRequests: k6CounterToCounter(endOfTestSummary.metrics.http_reqs),
+			httpRequestDuration: k6TrendToTrend(endOfTestSummary.metrics.http_req_duration),
+			httpRequestSending: k6TrendToTrend(endOfTestSummary.metrics.http_req_sending),
+			httpRequestReceiving: k6TrendToTrend(endOfTestSummary.metrics.http_req_receiving),
+			httpRequestWaiting: k6TrendToTrend(endOfTestSummary.metrics.http_req_waiting),
+			iterations: k6CounterToCounter(endOfTestSummary.metrics.iterations),
+		},
+	};
+}

pnpm-lock.yaml

@@ -229,6 +229,9 @@ importers:
       dotenv:
         specifier: 8.6.0
         version: 8.6.0
+      nanoid:
+        specifier: 'catalog:'
+        version: 3.3.6
       zx:
         specifier: ^8.1.4
         version: 8.1.4
@@ -21625,7 +21628,7 @@ snapshots:
 
   eslint-import-resolver-node@0.3.9:
     dependencies:
-      debug: 3.2.7(supports-color@8.1.1)
+      debug: 3.2.7(supports-color@5.5.0)
       is-core-module: 2.13.1
       resolve: 1.22.8
     transitivePeerDependencies:
@@ -21650,7 +21653,7 @@ snapshots:
 
   eslint-module-utils@2.8.0(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.5.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.5.2))(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0):
     dependencies:
-      debug: 3.2.7(supports-color@8.1.1)
+      debug: 3.2.7(supports-color@5.5.0)
     optionalDependencies:
       '@typescript-eslint/parser': 7.2.0(eslint@8.57.0)(typescript@5.5.2)
       eslint: 8.57.0
@@ -21670,7 +21673,7 @@ snapshots:
       array.prototype.findlastindex: 1.2.3
       array.prototype.flat: 1.3.2
       array.prototype.flatmap: 1.3.2
-      debug: 3.2.7(supports-color@8.1.1)
+      debug: 3.2.7(supports-color@5.5.0)
       doctrine: 2.1.0
       eslint: 8.57.0
       eslint-import-resolver-node: 0.3.9
@@ -22540,7 +22543,7 @@ snapshots:
       array-parallel: 0.1.3
       array-series: 0.1.5
       cross-spawn: 4.0.2
-      debug: 3.2.7(supports-color@8.1.1)
+      debug: 3.2.7(supports-color@5.5.0)
     transitivePeerDependencies:
       - supports-color
 
@@ -25561,7 +25564,7 @@ snapshots:
 
   pdf-parse@1.1.1:
     dependencies:
-      debug: 3.2.7(supports-color@8.1.1)
+      debug: 3.2.7(supports-color@5.5.0)
       node-ensure: 0.0.0
     transitivePeerDependencies:
       - supports-color
@@ -26447,7 +26450,7 @@ snapshots:
 
   rhea@1.0.24:
     dependencies:
-      debug: 3.2.7(supports-color@8.1.1)
+      debug: 3.2.7(supports-color@5.5.0)
     transitivePeerDependencies:
       - supports-color