feat: Add n8n-benchmark cli (no-changelog) (#10410)

.github/workflows/docker-images-benchmark.yml

@@ -0,0 +1,43 @@
+name: Benchmark Docker Image CI
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths:
+      - 'packages/benchmark/**'
+      - 'pnpm-lock.yaml'
+      - 'pnpm-workspace.yaml'
+      - '.github/workflows/docker-images-benchmark.yml'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4.1.1
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3.0.0
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3.0.0
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3.0.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build
+        uses: docker/build-push-action@v5.1.0
+        with:
+          context: .
+          file: ./packages/benchmark/Dockerfile
+          platforms: linux/amd64
+          provenance: false
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository_owner }}/n8n-benchmark:latest

packages/@n8n/benchmark/Dockerfile

@@ -0,0 +1,62 @@
+# syntax=docker/dockerfile:1
+FROM node:20.16.0 AS base
+
+# Install required dependencies
+RUN apt-get update && apt-get install -y gnupg2 curl
+
+# Add k6 GPG key and repository
+RUN mkdir -p /etc/apt/keyrings && \
+	curl -sS https://dl.k6.io/key.gpg | gpg --dearmor --yes -o /etc/apt/keyrings/k6.gpg && \
+	chmod a+x /etc/apt/keyrings/k6.gpg && \
+	echo "deb [signed-by=/etc/apt/keyrings/k6.gpg] https://dl.k6.io/deb stable main" | tee /etc/apt/sources.list.d/k6.list
+
+# Update and install k6
+RUN apt-get update && \
+	apt-get install -y k6 tini && \
+	apt-get clean && \
+	rm -rf /var/lib/apt/lists/*
+
+ENV PNPM_HOME="/pnpm"
+ENV PATH="$PNPM_HOME:$PATH"
+RUN corepack enable
+
+#
+# Builder
+FROM base AS builder
+
+WORKDIR /app
+
+COPY --chown=node:node ./pnpm-lock.yaml /app/pnpm-lock.yaml
+COPY --chown=node:node ./pnpm-workspace.yaml /app/pnpm-workspace.yaml
+COPY --chown=node:node ./package.json /app/package.json
+COPY --chown=node:node ./packages/@n8n/benchmark/package.json /app/packages/@n8n/benchmark/package.json
+COPY --chown=node:node ./patches /app/patches
+COPY --chown=node:node ./scripts /app/scripts
+
+RUN pnpm install --frozen-lockfile
+
+# TS config files
+COPY --chown=node:node ./tsconfig.json /app/tsconfig.json
+COPY --chown=node:node ./tsconfig.build.json /app/tsconfig.build.json
+COPY --chown=node:node ./tsconfig.backend.json /app/tsconfig.backend.json
+COPY --chown=node:node ./packages/@n8n/benchmark/tsconfig.json /app/packages/@n8n/benchmark/tsconfig.json
+COPY --chown=node:node ./packages/@n8n/benchmark/tsconfig.build.json /app/packages/@n8n/benchmark/tsconfig.build.json
+
+# Source files
+COPY --chown=node:node ./packages/@n8n/benchmark/src /app/packages/@n8n/benchmark/src
+COPY --chown=node:node ./packages/@n8n/benchmark/bin /app/packages/@n8n/benchmark/bin
+COPY --chown=node:node ./packages/@n8n/benchmark/scenarios /app/packages/@n8n/benchmark/scenarios
+
+WORKDIR /app/packages/@n8n/benchmark
+RUN pnpm build
+
+#
+# Runner
+FROM base AS runner
+
+COPY --from=builder /app /app
+
+WORKDIR /app/packages/@n8n/benchmark
+USER node
+
+ENTRYPOINT [ "/app/packages/@n8n/benchmark/bin/n8n-benchmark" ]

packages/@n8n/benchmark/README.md

@@ -0,0 +1,55 @@
+# n8n benchmarking tool
+
+Tool for executing benchmarks against an n8n instance.
+
+## Running locally with Docker
+
+Build the Docker image:
+
+```sh
+# Must be run in the repository root
+# k6 doesn't have an arm64 build available for linux, we need to build against amd64
+docker build --platform linux/amd64 -t n8n-benchmark -f packages/@n8n/benchmark/Dockerfile .
+```
+
+Run the image
+
+```sh
+docker run \
+  -e N8N_USER_EMAIL=user@n8n.io \
+  -e N8N_USER_PASSWORD=password \
+  # For macos, n8n running outside docker
+  -e N8N_BASE_URL=http://host.docker.internal:5678 \
+  n8n-benchmark
+```
+
+## Running locally without Docker
+
+Requirements:
+
+- [k6](https://grafana.com/docs/k6/latest/set-up/install-k6/)
+- Node.js v20 or higher
+
+```sh
+pnpm build
+
+# Run tests against http://localhost:5678 with specified email and password
+N8N_USER_EMAIL=user@n8n.io N8N_USER_PASSWORD=password ./bin/n8n-benchmark run
+
+# If you installed k6 using brew, you might have to specify it explicitly
+K6_PATH=/opt/homebrew/bin/k6 N8N_USER_EMAIL=user@n8n.io N8N_USER_PASSWORD=password ./bin/n8n-benchmark run
+```
+
+## Configuration
+
+The configuration options the cli accepts can be seen from [config.ts](./src/config/config.ts)
+
+## Benchmark scenarios
+
+A benchmark scenario defines one or multiple steps to execute and measure. It consists of:
+
+- Manifest file which describes and configures the scenario
+- Any test data that is imported before the scenario is run
+- A [`k6`](https://grafana.com/docs/k6/latest/using-k6/http-requests/) script which executes the steps and receives `API_BASE_URL` environment variable in runtime.
+
+Available scenarios are located in [`./scenarios`](./scenarios/).

packages/@n8n/benchmark/bin/n8n-benchmark

@@ -0,0 +1,13 @@
+#!/usr/bin/env node
+
+// Check if version should be displayed
+const versionFlags = ['-v', '-V', '--version'];
+if (versionFlags.includes(process.argv.slice(-1)[0])) {
+	console.log(require('../package').version);
+	process.exit(0);
+}
+
+(async () => {
+	const oclif = require('@oclif/core');
+	await oclif.execute({ dir: __dirname });
+})();

packages/@n8n/benchmark/package.json

@@ -0,0 +1,48 @@
+{
+  "name": "@n8n/n8n-benchmark",
+  "version": "1.0.0",
+  "description": "Cli for running benchmark tests for n8n",
+  "main": "dist/index",
+  "scripts": {
+    "build": "tsc -p tsconfig.build.json && tsc-alias -p tsconfig.build.json",
+    "start": "./bin/n8n-benchmark",
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "typecheck": "tsc --noEmit",
+    "watch": "concurrently \"tsc -w -p tsconfig.build.json\" \"tsc-alias -w -p tsconfig.build.json\""
+  },
+  "engines": {
+    "node": ">=20.10"
+  },
+  "keywords": [
+    "automate",
+    "automation",
+    "IaaS",
+    "iPaaS",
+    "n8n",
+    "workflow",
+    "benchmark",
+    "performance"
+  ],
+  "dependencies": {
+    "@oclif/core": "4.0.7",
+    "axios": "catalog:",
+    "convict": "6.2.4",
+    "dotenv": "8.6.0",
+    "zx": "^8.1.4"
+  },
+  "devDependencies": {
+    "@types/convict": "^6.1.1",
+    "@types/k6": "^0.52.0",
+    "@types/node": "^20.14.8",
+    "tsc-alias": "^1.8.7",
+    "typescript": "^5.5.2"
+  },
+  "bin": {
+    "n8n-benchmark": "./bin/n8n-benchmark"
+  },
+  "oclif": {
+    "bin": "n8n-benchmark",
+    "commands": "./dist/commands",
+    "topicSeparator": " "
+  }
+}

packages/@n8n/benchmark/scenarios/scenario.schema.json

@@ -0,0 +1,42 @@
+{
+	"definitions": {
+		"ScenarioData": {
+			"type": "object",
+			"properties": {
+				"workflowFiles": {
+					"type": "array",
+					"items": {
+						"type": "string"
+					}
+				}
+			},
+			"required": [],
+			"additionalProperties": false
+		}
+	},
+	"type": "object",
+	"properties": {
+		"$schema": {
+			"type": "string",
+			"description": "The JSON schema to validate this file"
+		},
+		"name": {
+			"type": "string",
+			"description": "The name of the scenario"
+		},
+		"description": {
+			"type": "string",
+			"description": "A longer description of the scenario"
+		},
+		"scriptPath": {
+			"type": "string",
+			"description": "Relative path to the k6 test script"
+		},
+		"scenarioData": {
+			"$ref": "#/definitions/ScenarioData",
+			"description": "Data to import before running the scenario"
+		}
+	},
+	"required": ["name", "description", "scriptPath", "scenarioData"],
+	"additionalProperties": false
+}

packages/@n8n/benchmark/scenarios/singleWebhook/singleWebhook.json

@@ -0,0 +1,25 @@
+{
+	"createdAt": "2024-08-06T12:19:51.268Z",
+	"updatedAt": "2024-08-06T12:20:45.000Z",
+	"name": "Single Webhook",
+	"active": true,
+	"nodes": [
+		{
+			"parameters": { "path": "single-webhook", "options": {} },
+			"id": "7587ab0e-cc15-424f-83c0-c887a0eb97fb",
+			"name": "Webhook",
+			"type": "n8n-nodes-base.webhook",
+			"typeVersion": 2,
+			"position": [760, 400],
+			"webhookId": "fa563fc2-c73f-4631-99a1-39c16f1f858f"
+		}
+	],
+	"connections": {},
+	"settings": { "executionOrder": "v1" },
+	"staticData": null,
+	"meta": { "templateCredsSetupCompleted": true, "responseMode": "lastNode", "options": {} },
+	"pinData": {},
+	"versionId": "840a38a1-ba37-433d-9f20-de73f5131a2b",
+	"triggerCount": 1,
+	"tags": []
+}

packages/@n8n/benchmark/scenarios/singleWebhook/singleWebhook.manifest.json

@@ -0,0 +1,7 @@
+{
+	"$schema": "../scenario.schema.json",
+	"name": "SingleWebhook",
+	"description": "A single webhook trigger that responds with a 200 status code",
+	"scenarioData": { "workflowFiles": ["singleWebhook.json"] },
+	"scriptPath": "singleWebhook.script.ts"
+}

packages/@n8n/benchmark/scenarios/singleWebhook/singleWebhook.script.ts

@@ -0,0 +1,11 @@
+import http from 'k6/http';
+import { check } from 'k6';
+
+const apiBaseUrl = __ENV.API_BASE_URL;
+
+export default function () {
+	const res = http.get(`${apiBaseUrl}/webhook/single-webhook`);
+	check(res, {
+		'is status 200': (r) => r.status === 200,
+	});
+}

packages/@n8n/benchmark/src/commands/list.ts

@@ -0,0 +1,21 @@
+import { Command } from '@oclif/core';
+import { ScenarioLoader } from '@/scenario/scenarioLoader';
+import { loadConfig } from '@/config/config';
+
+export default class ListCommand extends Command {
+	static description = 'List all available scenarios';
+
+	async run() {
+		const config = loadConfig();
+		const scenarioLoader = new ScenarioLoader();
+
+		const allScenarios = scenarioLoader.loadAll(config.get('testScenariosPath'));
+
+		console.log('Available test scenarios:');
+		console.log('');
+
+		for (const scenario of allScenarios) {
+			console.log('\t', scenario.name, ':', scenario.description);
+		}
+	}
+}

packages/@n8n/benchmark/src/commands/run.ts

@@ -0,0 +1,39 @@
+import { Command, Flags } from '@oclif/core';
+import { loadConfig } from '@/config/config';
+import { ScenarioLoader } from '@/scenario/scenarioLoader';
+import { ScenarioRunner } from '@/testExecution/scenarioRunner';
+import { N8nApiClient } from '@/n8nApiClient/n8nApiClient';
+import { ScenarioDataFileLoader } from '@/scenario/scenarioDataLoader';
+import { K6Executor } from '@/testExecution/k6Executor';
+
+export default class RunCommand extends Command {
+	static description = 'Run all (default) or specified test scenarios';
+
+	// TODO: Add support for filtering scenarios
+	static flags = {
+		scenarios: Flags.string({
+			char: 't',
+			description: 'Comma-separated list of test scenarios to run',
+			required: false,
+		}),
+	};
+
+	async run() {
+		const config = loadConfig();
+		const scenarioLoader = new ScenarioLoader();
+
+		const scenarioRunner = new ScenarioRunner(
+			new N8nApiClient(config.get('n8n.baseUrl')),
+			new ScenarioDataFileLoader(),
+			new K6Executor(config.get('k6ExecutablePath'), config.get('n8n.baseUrl')),
+			{
+				email: config.get('n8n.user.email'),
+				password: config.get('n8n.user.password'),
+			},
+		);
+
+		const allScenarios = scenarioLoader.loadAll(config.get('testScenariosPath'));
+
+		await scenarioRunner.runManyScenarios(allScenarios);
+	}
+}

packages/@n8n/benchmark/src/config/config.ts

@@ -0,0 +1,50 @@
+import convict from 'convict';
+import dotenv from 'dotenv';
+
+dotenv.config();
+
+const configSchema = {
+	testScenariosPath: {
+		doc: 'The path to the scenarios',
+		format: String,
+		default: 'scenarios',
+	},
+	n8n: {
+		baseUrl: {
+			doc: 'The base URL for the n8n instance',
+			format: String,
+			default: 'http://localhost:5678',
+			env: 'N8N_BASE_URL',
+		},
+		user: {
+			email: {
+				doc: 'The email address of the n8n user',
+				format: String,
+				default: 'benchmark-user@n8n.io',
+				env: 'N8N_USER_EMAIL',
+			},
+			password: {
+				doc: 'The password of the n8n user',
+				format: String,
+				default: 'VerySecret!123',
+				env: 'N8N_USER_PASSWORD',
+			},
+		},
+	},
+	k6ExecutablePath: {
+		doc: 'The path to the k6 binary',
+		format: String,
+		default: 'k6',
+		env: 'K6_PATH',
+	},
+};
+
+export type Config = ReturnType<typeof loadConfig>;
+
+export function loadConfig() {
+	const config = convict(configSchema);
+
+	config.validate({ allowed: 'strict' });
+
+	return config;
+}

packages/@n8n/benchmark/src/n8nApiClient/authenticatedN8nApiClient.ts

@@ -0,0 +1,67 @@
+import { strict as assert } from 'node:assert';
+import { N8nApiClient } from './n8nApiClient';
+import { AxiosRequestConfig } from 'axios';
+
+export class AuthenticatedN8nApiClient extends N8nApiClient {
+	constructor(
+		apiBaseUrl: string,
+		private readonly authCookie: string,
+	) {
+		super(apiBaseUrl);
+	}
+
+	static async createUsingUsernameAndPassword(
+		apiClient: N8nApiClient,
+		loginDetails: {
+			email: string;
+			password: string;
+		},
+	) {
+		const response = await apiClient.restApiRequest('/login', {
+			method: 'POST',
+			data: loginDetails,
+		});
+
+		const cookieHeader = response.headers['set-cookie'];
+		const authCookie = Array.isArray(cookieHeader) ? cookieHeader.join('; ') : cookieHeader;
+		assert(authCookie);
+
+		return new AuthenticatedN8nApiClient(apiClient.apiBaseUrl, authCookie);
+	}
+
+	async get<T>(endpoint: string) {
+		return await this.authenticatedRequest<T>(endpoint, {
+			method: 'GET',
+		});
+	}
+
+	async post<T>(endpoint: string, data: unknown) {
+		return await this.authenticatedRequest<T>(endpoint, {
+			method: 'POST',
+			data,
+		});
+	}
+
+	async patch<T>(endpoint: string, data: unknown) {
+		return await this.authenticatedRequest<T>(endpoint, {
+			method: 'PATCH',
+			data,
+		});
+	}
+
+	async delete<T>(endpoint: string) {
+		return await this.authenticatedRequest<T>(endpoint, {
+			method: 'DELETE',
+		});
+	}
+
+	protected async authenticatedRequest<T>(endpoint: string, init: Omit<AxiosRequestConfig, 'url'>) {
+		return await this.restApiRequest<T>(endpoint, {
+			...init,
+			headers: {
+				...init.headers,
+				cookie: this.authCookie,
+			},
+		});
+	}
+}

packages/@n8n/benchmark/src/n8nApiClient/n8nApiClient.ts

@@ -0,0 +1,78 @@
+import axios, { AxiosError, AxiosRequestConfig } from 'axios';
+
+export class N8nApiClient {
+	constructor(public readonly apiBaseUrl: string) {}
+
+	async waitForInstanceToBecomeOnline(): Promise<void> {
+		const HEALTH_ENDPOINT = 'healthz';
+		const START_TIME = Date.now();
+		const INTERVAL_MS = 1000;
+		const TIMEOUT_MS = 60_000;
+
+		while (Date.now() - START_TIME < TIMEOUT_MS) {
+			try {
+				const response = await axios.request({
+					url: `${this.apiBaseUrl}/${HEALTH_ENDPOINT}`,
+					method: 'GET',
+				});
+
+				if (response.status === 200 && response.data.status === 'ok') {
+					return;
+				}
+			} catch {}
+
+			console.log(`n8n instance not online yet, retrying in ${INTERVAL_MS / 1000} seconds...`);
+			await this.delay(INTERVAL_MS);
+		}
+
+		throw new Error(`n8n instance did not come online within ${TIMEOUT_MS / 1000} seconds`);
+	}
+
+	async setupOwnerIfNeeded(loginDetails: { email: string; password: string }) {
+		const response = await this.restApiRequest<{ message: string }>('/owner/setup', {
+			method: 'POST',
+			data: {
+				email: loginDetails.email,
+				password: loginDetails.password,
+				firstName: 'Test',
+				lastName: 'User',
+			},
+			// Don't throw on non-2xx responses
+			validateStatus: () => true,
+		});
+
+		const responsePayload = response.data;
+
+		if (response.status === 200) {
+			console.log('Owner setup successful');
+		} else if (response.status === 400) {
+			if (responsePayload.message === 'Instance owner already setup')
+				console.log('Owner already set up');
+		} else {
+			throw new Error(
+				`Owner setup failed with status ${response.status}: ${responsePayload.message}`,
+			);
+		}
+	}
+
+	async restApiRequest<T>(endpoint: string, init: Omit<AxiosRequestConfig, 'url'>) {
+		try {
+			return await axios.request<T>({
+				...init,
+				url: this.getRestEndpointUrl(endpoint),
+			});
+		} catch (e) {
+			const error = e as AxiosError;
+			console.error(`[ERROR] Request failed ${init.method} ${endpoint}`, error?.response?.data);
+			throw error;
+		}
+	}
+
+	protected getRestEndpointUrl(endpoint: string) {
+		return `${this.apiBaseUrl}/rest${endpoint}`;
+	}
+
+	private delay(ms: number): Promise<void> {
+		return new Promise((resolve) => setTimeout(resolve, ms));
+	}
+}

packages/@n8n/benchmark/src/n8nApiClient/n8nApiClient.types.ts

@@ -0,0 +1,8 @@
+/**
+ * n8n workflow. This is a simplified version of the actual workflow object.
+ */
+export type Workflow = {
+	id: string;
+	name: string;
+	tags?: string[];
+};

packages/@n8n/benchmark/src/n8nApiClient/workflowsApiClient.ts

@@ -0,0 +1,31 @@
+import { Workflow } from '@/n8nApiClient/n8nApiClient.types';
+import { AuthenticatedN8nApiClient } from './authenticatedN8nApiClient';
+
+export class WorkflowApiClient {
+	constructor(private readonly apiClient: AuthenticatedN8nApiClient) {}
+
+	async getAllWorkflows(): Promise<Workflow[]> {
+		const response = await this.apiClient.get<{ count: number; data: Workflow[] }>('/workflows');
+
+		return response.data.data;
+	}
+
+	async createWorkflow(workflow: unknown): Promise<Workflow> {
+		const response = await this.apiClient.post<{ data: Workflow }>('/workflows', workflow);
+
+		return response.data.data;
+	}
+
+	async activateWorkflow(workflow: Workflow): Promise<Workflow> {
+		const response = await this.apiClient.patch<{ data: Workflow }>(`/workflows/${workflow.id}`, {
+			...workflow,
+			active: true,
+		});
+
+		return response.data.data;
+	}
+
+	async deleteWorkflow(workflowId: Workflow['id']): Promise<void> {
+		await this.apiClient.delete(`/workflows/${workflowId}`);
+	}
+}

packages/@n8n/benchmark/src/scenario/scenarioDataLoader.ts

@@ -0,0 +1,35 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { Scenario } from '@/types/scenario';
+import { Workflow } from '@/n8nApiClient/n8nApiClient.types';
+
+/**
+ * Loads scenario data files from FS
+ */
+export class ScenarioDataFileLoader {
+	async loadDataForScenario(scenario: Scenario): Promise<{
+		workflows: Workflow[];
+	}> {
+		const workflows = await Promise.all(
+			scenario.scenarioData.workflowFiles?.map((workflowFilePath) =>
+				this.loadSingleWorkflowFromFile(path.join(scenario.scenarioDirPath, workflowFilePath)),
+			) ?? [],
+		);
+
+		return {
+			workflows,
+		};
+	}
+
+	private loadSingleWorkflowFromFile(workflowFilePath: string): Workflow {
+		const fileContent = fs.readFileSync(workflowFilePath, 'utf8');
+
+		try {
+			return JSON.parse(fileContent);
+		} catch (error) {
+			throw new Error(
+				`Failed to parse workflow file ${workflowFilePath}: ${error instanceof Error ? error.message : error}`,
+			);
+		}
+	}
+}

packages/@n8n/benchmark/src/scenario/scenarioLoader.ts

@@ -0,0 +1,67 @@
+import * as fs from 'node:fs';
+import * as path from 'path';
+import { createHash } from 'node:crypto';
+import type { Scenario, ScenarioManifest } from '@/types/scenario';
+
+export class ScenarioLoader {
+	/**
+	 * Loads all scenarios from the given path
+	 */
+	loadAll(pathToScenarios: string): Scenario[] {
+		pathToScenarios = path.resolve(pathToScenarios);
+		const scenarioFolders = fs
+			.readdirSync(pathToScenarios, { withFileTypes: true })
+			.filter((dirent) => dirent.isDirectory())
+			.map((dirent) => dirent.name);
+
+		const scenarios: Scenario[] = [];
+
+		for (const folder of scenarioFolders) {
+			const scenarioPath = path.join(pathToScenarios, folder);
+			const manifestFileName = `${folder}.manifest.json`;
+			const scenarioManifestPath = path.join(pathToScenarios, folder, manifestFileName);
+			if (!fs.existsSync(scenarioManifestPath)) {
+				console.warn(`Scenario at ${scenarioPath} is missing the ${manifestFileName} file`);
+				continue;
+			}
+
+			// Load the scenario manifest file
+			const [scenario, validationErrors] =
+				this.loadAndValidateScenarioManifest(scenarioManifestPath);
+			if (validationErrors) {
+				console.warn(
+					`Scenario at ${scenarioPath} has the following validation errors: ${validationErrors.join(', ')}`,
+				);
+				continue;
+			}
+
+			scenarios.push({
+				...scenario,
+				id: this.formScenarioId(scenarioPath),
+				scenarioDirPath: scenarioPath,
+			});
+		}
+
+		return scenarios;
+	}
+
+	private loadAndValidateScenarioManifest(
+		scenarioManifestPath: string,
+	): [ScenarioManifest, null] | [null, string[]] {
+		const scenario = JSON.parse(fs.readFileSync(scenarioManifestPath, 'utf8'));
+		const validationErrors: string[] = [];
+
+		if (!scenario.name) {
+			validationErrors.push(`Scenario at ${scenarioManifestPath} is missing a name`);
+		}
+		if (!scenario.description) {
+			validationErrors.push(`Scenario at ${scenarioManifestPath} is missing a description`);
+		}
+
+		return validationErrors.length === 0 ? [scenario, null] : [null, validationErrors];
+	}
+
+	private formScenarioId(scenarioPath: string): string {
+		return createHash('sha256').update(scenarioPath).digest('hex');
+	}
+}

packages/@n8n/benchmark/src/testExecution/k6Executor.ts

@@ -0,0 +1,28 @@
+import { $ } from 'zx';
+import { Scenario } from '@/types/scenario';
+
+/**
+ * Executes test scenarios using k6
+ */
+export class K6Executor {
+	constructor(
+		private readonly k6ExecutablePath: string,
+		private readonly n8nApiBaseUrl: string,
+	) {}
+
+	async executeTestScenario(scenario: Scenario) {
+		// For 1 min with 5 virtual users
+		const stage = '1m:5';
+
+		const processPromise = $({
+			cwd: scenario.scenarioDirPath,
+			env: {
+				API_BASE_URL: this.n8nApiBaseUrl,
+			},
+		})`${this.k6ExecutablePath} run --quiet --stage ${stage} ${scenario.scriptPath}`;
+
+		for await (const chunk of processPromise.stdout) {
+			console.log(chunk.toString());
+		}
+	}
+}

packages/@n8n/benchmark/src/testExecution/scenarioDataImporter.ts

@@ -0,0 +1,56 @@
+import { AuthenticatedN8nApiClient } from '@/n8nApiClient/authenticatedN8nApiClient';
+import { Workflow } from '@/n8nApiClient/n8nApiClient.types';
+import { WorkflowApiClient } from '@/n8nApiClient/workflowsApiClient';
+
+/**
+ * Imports scenario data into an n8n instance
+ */
+export class ScenarioDataImporter {
+	private readonly workflowApiClient: WorkflowApiClient;
+
+	constructor(n8nApiClient: AuthenticatedN8nApiClient) {
+		this.workflowApiClient = new WorkflowApiClient(n8nApiClient);
+	}
+
+	async importTestScenarioData(workflows: Workflow[]) {
+		const existingWorkflows = await this.workflowApiClient.getAllWorkflows();
+
+		for (const workflow of workflows) {
+			await this.importWorkflow({ existingWorkflows, workflow });
+		}
+	}
+
+	/**
+	 * Imports a single workflow into n8n removing any existing workflows with the same name
+	 */
+	private async importWorkflow(opts: { existingWorkflows: Workflow[]; workflow: Workflow }) {
+		const existingWorkflows = this.findExistingWorkflows(opts.existingWorkflows, opts.workflow);
+		if (existingWorkflows.length > 0) {
+			for (const toDelete of existingWorkflows) {
+				await this.workflowApiClient.deleteWorkflow(toDelete.id);
+			}
+		}
+
+		const createdWorkflow = await this.workflowApiClient.createWorkflow({
+			...opts.workflow,
+			name: this.getBenchmarkWorkflowName(opts.workflow),
+		});
+
+		return await this.workflowApiClient.activateWorkflow(createdWorkflow);
+	}
+
+	private findExistingWorkflows(
+		existingWorkflows: Workflow[],
+		workflowToImport: Workflow,
+	): Workflow[] {
+		const benchmarkWorkflowName = this.getBenchmarkWorkflowName(workflowToImport);
+
+		return existingWorkflows.filter(
+			(existingWorkflow) => existingWorkflow.name === benchmarkWorkflowName,
+		);
+	}
+
+	private getBenchmarkWorkflowName(workflow: Workflow) {
+		return `[BENCHMARK] ${workflow.name}`;
+	}
+}

packages/@n8n/benchmark/src/testExecution/scenarioRunner.ts

@@ -0,0 +1,50 @@
+import { Scenario } from '@/types/scenario';
+import { N8nApiClient } from '@/n8nApiClient/n8nApiClient';
+import { ScenarioDataFileLoader } from '@/scenario/scenarioDataLoader';
+import { K6Executor } from './k6Executor';
+import { ScenarioDataImporter } from '@/testExecution/scenarioDataImporter';
+import { AuthenticatedN8nApiClient } from '@/n8nApiClient/authenticatedN8nApiClient';
+
+/**
+ * Runs scenarios
+ */
+export class ScenarioRunner {
+	constructor(
+		private readonly n8nClient: N8nApiClient,
+		private readonly dataLoader: ScenarioDataFileLoader,
+		private readonly k6Executor: K6Executor,
+		private readonly ownerConfig: {
+			email: string;
+			password: string;
+		},
+	) {}
+
+	async runManyScenarios(scenarios: Scenario[]) {
+		console.log(`Waiting for n8n ${this.n8nClient.apiBaseUrl} to become online`);
+		await this.n8nClient.waitForInstanceToBecomeOnline();
+
+		console.log('Setting up owner');
+		await this.n8nClient.setupOwnerIfNeeded(this.ownerConfig);
+
+		const authenticatedN8nClient = await AuthenticatedN8nApiClient.createUsingUsernameAndPassword(
+			this.n8nClient,
+			this.ownerConfig,
+		);
+		const testDataImporter = new ScenarioDataImporter(authenticatedN8nClient);
+
+		for (const scenario of scenarios) {
+			await this.runSingleTestScenario(testDataImporter, scenario);
+		}
+	}
+
+	private async runSingleTestScenario(testDataImporter: ScenarioDataImporter, scenario: Scenario) {
+		console.log('Running scenario:', scenario.name);
+
+		console.log('Loading and importing data');
+		const testData = await this.dataLoader.loadDataForScenario(scenario);
+		await testDataImporter.importTestScenarioData(testData.workflows);
+
+		console.log('Executing scenario script');
+		await this.k6Executor.executeTestScenario(scenario);
+	}
+}

packages/@n8n/benchmark/src/types/scenario.ts

@@ -0,0 +1,27 @@
+export type ScenarioData = {
+	/** Relative paths to the workflow files */
+	workflowFiles?: string[];
+};
+
+/**
+ * Configuration that defines the benchmark scenario
+ */
+export type ScenarioManifest = {
+	/** The name of the scenario */
+	name: string;
+	/** A longer description of the scenario */
+	description: string;
+	/** Relative path to the k6 script */
+	scriptPath: string;
+	/** Data to import before running the scenario */
+	scenarioData: ScenarioData;
+};
+
+/**
+ * Scenario with additional metadata
+ */
+export type Scenario = ScenarioManifest & {
+	id: string;
+	/** Path to the directory containing the scenario */
+	scenarioDirPath: string;
+};

packages/@n8n/benchmark/tsconfig.build.json

@@ -0,0 +1,9 @@
+{
+	"extends": ["./tsconfig.json", "../../../tsconfig.build.json"],
+	"compilerOptions": {
+		"rootDir": "src",
+		"outDir": "dist",
+		"tsBuildInfoFile": "dist/build.tsbuildinfo"
+	},
+	"include": ["src/**/*.ts"]
+}

packages/@n8n/benchmark/tsconfig.json

@@ -0,0 +1,11 @@
+{
+	"extends": ["../../../tsconfig.json", "../../../tsconfig.backend.json"],
+	"compilerOptions": {
+		"rootDir": ".",
+		"baseUrl": "src",
+		"paths": {
+			"@/*": ["./*"]
+		}
+	},
+	"include": ["src/**/*.ts"]
+}