test: Add scaling n8n setup (multi-main) (#10644)

packages/@n8n/benchmark/scripts/n8nSetups/queue/docker-compose.yml

@@ -38,11 +38,12 @@ services:
     environment:
       - N8N_DIAGNOSTICS_ENABLED=false
       - N8N_USER_FOLDER=/n8n/worker1
-      - N8N_ENCRYPTION_KEY=very-secret-encryption-key
+      - N8N_ENCRYPTION_KEY=${N8N_ENCRYPTION_KEY}
       # Queue mode config
       - EXECUTIONS_MODE=queue
       - QUEUE_BULL_REDIS_HOST=redis
       - QUEUE_HEALTH_CHECK_ACTIVE=true
+      - N8N_CONCURRENCY_PRODUCTION_LIMIT=10
       # DB config
       - DB_TYPE=postgresdb
       - DB_POSTGRESDB_HOST=postgres
@@ -67,11 +68,12 @@ services:
     environment:
       - N8N_DIAGNOSTICS_ENABLED=false
       - N8N_USER_FOLDER=/n8n/worker2
-      - N8N_ENCRYPTION_KEY=very-secret-encryption-key
+      - N8N_ENCRYPTION_KEY=${N8N_ENCRYPTION_KEY}
       # Queue mode config
       - EXECUTIONS_MODE=queue
       - QUEUE_BULL_REDIS_HOST=redis
       - QUEUE_HEALTH_CHECK_ACTIVE=true
+      - N8N_CONCURRENCY_PRODUCTION_LIMIT=10
       # DB config
       - DB_TYPE=postgresdb
       - DB_POSTGRESDB_HOST=postgres
@@ -99,7 +101,7 @@ services:
     environment:
       - N8N_DIAGNOSTICS_ENABLED=false
       - N8N_USER_FOLDER=/n8n/main
-      - N8N_ENCRYPTION_KEY=very-secret-encryption-key
+      - N8N_ENCRYPTION_KEY=${N8N_ENCRYPTION_KEY}
       # Queue mode config
       - EXECUTIONS_MODE=queue
       - QUEUE_BULL_REDIS_HOST=redis

packages/@n8n/benchmark/scripts/n8nSetups/scaling/docker-compose.yml

@@ -0,0 +1,194 @@
+services:
+  mockapi:
+    image: wiremock/wiremock:3.9.1
+    ports:
+      - '8088:8080'
+    volumes:
+      - ${MOCK_API_DATA_PATH}/mappings:/home/wiremock/mappings
+
+  redis:
+    image: redis:6-alpine
+    restart: always
+    ports:
+      - 6379:6379
+    healthcheck:
+      test: ['CMD', 'redis-cli', 'ping']
+      interval: 1s
+      timeout: 3s
+
+  postgres:
+    image: postgres:16
+    restart: always
+    environment:
+      - POSTGRES_DB=n8n
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=password
+      - PGDATA=/var/lib/postgresql/data/pgdata
+    volumes:
+      - ${RUN_DIR}/postgres:/var/lib/postgresql/data
+    healthcheck:
+      test: ['CMD-SHELL', 'pg_isready -U postgres']
+      interval: 5s
+      timeout: 5s
+      retries: 10
+
+  n8n_worker1:
+    image: ghcr.io/n8n-io/n8n:${N8N_VERSION:-latest}
+    environment:
+      - N8N_DIAGNOSTICS_ENABLED=false
+      - N8N_USER_FOLDER=/n8n/worker1
+      - N8N_ENCRYPTION_KEY=${N8N_ENCRYPTION_KEY}
+      - N8N_LICENSE_CERT=${N8N_LICENSE_CERT}
+      - N8N_LICENSE_ACTIVATION_KEY=${N8N_LICENSE_ACTIVATION_KEY}
+      - N8N_LICENSE_TENANT_ID=${N8N_LICENSE_TENANT_ID}
+      # Scaling mode config
+      - EXECUTIONS_MODE=queue
+      - QUEUE_BULL_REDIS_HOST=redis
+      - QUEUE_HEALTH_CHECK_ACTIVE=true
+      - N8N_CONCURRENCY_PRODUCTION_LIMIT=10
+      # DB config
+      - DB_TYPE=postgresdb
+      - DB_POSTGRESDB_HOST=postgres
+      - DB_POSTGRESDB_PASSWORD=password
+    command: worker
+    volumes:
+      - ${RUN_DIR}/n8n-worker1:/n8n
+    depends_on:
+      postgres:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    healthcheck:
+      test: ['CMD-SHELL', 'wget --spider -q http://localhost:5678/healthz || exit 1']
+      interval: 5s
+      timeout: 5s
+      retries: 10
+
+  n8n_worker2:
+    image: ghcr.io/n8n-io/n8n:${N8N_VERSION:-latest}
+    environment:
+      - N8N_DIAGNOSTICS_ENABLED=false
+      - N8N_USER_FOLDER=/n8n/worker2
+      - N8N_ENCRYPTION_KEY=${N8N_ENCRYPTION_KEY}
+      - N8N_LICENSE_CERT=${N8N_LICENSE_CERT}
+      - N8N_LICENSE_ACTIVATION_KEY=${N8N_LICENSE_ACTIVATION_KEY}
+      - N8N_LICENSE_TENANT_ID=${N8N_LICENSE_TENANT_ID}
+      # Scaling mode config
+      - EXECUTIONS_MODE=queue
+      - QUEUE_BULL_REDIS_HOST=redis
+      - QUEUE_HEALTH_CHECK_ACTIVE=true
+      - N8N_CONCURRENCY_PRODUCTION_LIMIT=10
+      # DB config
+      - DB_TYPE=postgresdb
+      - DB_POSTGRESDB_HOST=postgres
+      - DB_POSTGRESDB_PASSWORD=password
+    command: worker
+    volumes:
+      - ${RUN_DIR}/n8n-worker2:/n8n
+    depends_on:
+      # We let the worker 1 start first so it can run the DB migrations
+      n8n_worker1:
+        condition: service_healthy
+      postgres:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    healthcheck:
+      test: ['CMD-SHELL', 'wget --spider -q http://localhost:5678/healthz || exit 1']
+      interval: 5s
+      timeout: 5s
+      retries: 10
+
+  n8n_main2:
+    image: ghcr.io/n8n-io/n8n:${N8N_VERSION:-latest}
+    environment:
+      - N8N_DIAGNOSTICS_ENABLED=false
+      - N8N_USER_FOLDER=/n8n
+      - N8N_ENCRYPTION_KEY=${N8N_ENCRYPTION_KEY}
+      - N8N_LICENSE_CERT=${N8N_LICENSE_CERT}
+      - N8N_LICENSE_ACTIVATION_KEY=${N8N_LICENSE_ACTIVATION_KEY}
+      - N8N_LICENSE_TENANT_ID=${N8N_LICENSE_TENANT_ID}
+      # Scaling mode config
+      - EXECUTIONS_MODE=queue
+      - QUEUE_BULL_REDIS_HOST=redis
+      - N8N_MULTI_MAIN_SETUP_ENABLED=true
+      # DB config
+      - DB_TYPE=postgresdb
+      - DB_POSTGRESDB_HOST=postgres
+      - DB_POSTGRESDB_PASSWORD=password
+    volumes:
+      - ${RUN_DIR}/n8n-main2:/n8n
+    depends_on:
+      n8n_worker1:
+        condition: service_healthy
+      n8n_worker2:
+        condition: service_healthy
+      postgres:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+      mockapi:
+        condition: service_started
+    healthcheck:
+      test: ['CMD-SHELL', 'wget --spider -q http://n8n_main2:5678/healthz || exit 1']
+      interval: 5s
+      timeout: 5s
+      retries: 10
+
+  n8n_main1:
+    image: ghcr.io/n8n-io/n8n:${N8N_VERSION:-latest}
+    environment:
+      - N8N_DIAGNOSTICS_ENABLED=false
+      - N8N_USER_FOLDER=/n8n
+      - N8N_ENCRYPTION_KEY=${N8N_ENCRYPTION_KEY}
+      - N8N_LICENSE_CERT=${N8N_LICENSE_CERT}
+      - N8N_LICENSE_ACTIVATION_KEY=${N8N_LICENSE_ACTIVATION_KEY}
+      - N8N_LICENSE_TENANT_ID=${N8N_LICENSE_TENANT_ID}
+      # Scaling mode config
+      - EXECUTIONS_MODE=queue
+      - QUEUE_BULL_REDIS_HOST=redis
+      - N8N_MULTI_MAIN_SETUP_ENABLED=true
+      # DB config
+      - DB_TYPE=postgresdb
+      - DB_POSTGRESDB_HOST=postgres
+      - DB_POSTGRESDB_PASSWORD=password
+
+    volumes:
+      - ${RUN_DIR}/n8n-main:/n8n
+    depends_on:
+      n8n_worker1:
+        condition: service_healthy
+      n8n_worker2:
+        condition: service_healthy
+      postgres:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+      mockapi:
+        condition: service_started
+    healthcheck:
+      test: ['CMD-SHELL', 'wget --spider -q http://n8n_main1:5678/healthz || exit 1']
+      interval: 5s
+      timeout: 5s
+      retries: 10
+
+  # Load balancer that acts as an entry point for n8n
+  n8n:
+    image: nginx:latest
+    ports:
+      - '5678:80'
+    volumes:
+      - ./nginx.conf:/etc/nginx/nginx.conf
+    depends_on:
+      n8n_main1:
+        condition: service_healthy
+      n8n_main2:
+        condition: service_healthy
+
+  benchmark:
+    image: ghcr.io/n8n-io/n8n-benchmark:${N8N_BENCHMARK_VERSION:-latest}
+    depends_on:
+      - n8n
+    environment:
+      - N8N_BASE_URL=http://n8n:80
+      - K6_API_TOKEN=${K6_API_TOKEN}

packages/@n8n/benchmark/scripts/n8nSetups/scaling/nginx.conf

@@ -0,0 +1,20 @@
+events {}
+
+http {
+    upstream backend {
+        server n8n_main1:5678;
+        server n8n_main2:5678;
+    }
+
+    server {
+        listen 80;
+
+        location / {
+            proxy_pass http://backend;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+        }
+    }
+}

packages/@n8n/benchmark/scripts/n8nSetups/scaling/setup.mjs

@@ -0,0 +1,16 @@
+#!/usr/bin/env zx
+
+import path from 'path';
+import { fs } from 'zx';
+
+/**
+ * Creates the needed directories for the queue setup so their
+ * permissions get set correctly.
+ */
+export function setup({ runDir }) {
+	const neededDirs = ['n8n-worker1', 'n8n-worker2', 'n8n-main1', 'n8n_main2', 'postgres'];
+
+	for (const dir of neededDirs) {
+		fs.ensureDirSync(path.join(runDir, dir));
+	}
+}

packages/@n8n/benchmark/scripts/run.mjs

@@ -33,6 +33,7 @@ async function main() {
 			benchmarkTag: config.benchmarkTag,
 			isVerbose: config.isVerbose,
 			k6ApiToken: config.k6ApiToken,
+			n8nLicenseCert: config.n8nLicenseCert,
 			n8nTag: config.n8nTag,
 			n8nSetupsToUse,
 		});
@@ -41,6 +42,7 @@ async function main() {
 			benchmarkTag: config.benchmarkTag,
 			isVerbose: config.isVerbose,
 			k6ApiToken: config.k6ApiToken,
+			n8nLicenseCert: config.n8nLicenseCert,
 			n8nTag: config.n8nTag,
 			runDir: config.runDir,
 			n8nSetupsToUse,
@@ -62,6 +64,7 @@ function readAvailableN8nSetups() {
  * @property {string} n8nTag
  * @property {string} benchmarkTag
  * @property {string} [k6ApiToken]
+ * @property {string} [n8nLicenseCert]
  * @property {string} [runDir]
  *
  * @returns {Promise<Config>}
@@ -81,6 +84,7 @@ async function parseAndValidateConfig() {
 	const n8nTag = args.n8nTag || process.env.N8N_DOCKER_TAG || 'latest';
 	const benchmarkTag = args.benchmarkTag || process.env.BENCHMARK_DOCKER_TAG || 'latest';
 	const k6ApiToken = args.k6ApiToken || process.env.K6_API_TOKEN || undefined;
+	const n8nLicenseCert = args.n8nLicenseCert || process.env.N8N_LICENSE_CERT || undefined;
 	const runDir = args.runDir || undefined;
 	const env = args.env || 'local';
 
@@ -96,6 +100,7 @@ async function parseAndValidateConfig() {
 		n8nTag,
 		benchmarkTag,
 		k6ApiToken,
+		n8nLicenseCert,
 		runDir,
 	};
 }

packages/@n8n/benchmark/scripts/runForN8nSetup.mjs

@@ -12,6 +12,8 @@ const paths = {
 	mockApiDataPath: path.join(__dirname, 'mockApi'),
 };
 
+const N8N_ENCRYPTION_KEY = 'very-secret-encryption-key';
+
 async function main() {
 	const [n8nSetupToUse] = argv._;
 	validateN8nSetup(n8nSetupToUse);
@@ -22,6 +24,9 @@ async function main() {
 	const benchmarkTag = argv.benchmarkDockerTag || process.env.BENCHMARK_DOCKER_TAG || 'latest';
 	const k6ApiToken = argv.k6ApiToken || process.env.K6_API_TOKEN || undefined;
 	const baseRunDir = argv.runDir || process.env.RUN_DIR || '/n8n';
+	const n8nLicenseCert = argv.n8nLicenseCert || process.env.N8N_LICENSE_CERT || undefined;
+	const n8nLicenseActivationKey = process.env.N8N_LICENSE_ACTIVATION_KEY || '';
+	const n8nLicenseTenantId = argv.n8nLicenseTenantId || process.env.N8N_LICENSE_TENANT_ID || '';
 
 	if (!fs.existsSync(baseRunDir)) {
 		console.error(
@@ -38,7 +43,12 @@ async function main() {
 			cwd: composeFilePath,
 			verbose: true,
 			env: {
+				PATH: process.env.PATH,
 				N8N_VERSION: n8nTag,
+				N8N_LICENSE_CERT: n8nLicenseCert,
+				N8N_LICENSE_ACTIVATION_KEY: n8nLicenseActivationKey,
+				N8N_LICENSE_TENANT_ID: n8nLicenseTenantId,
+				N8N_ENCRYPTION_KEY,
 				BENCHMARK_VERSION: benchmarkTag,
 				K6_API_TOKEN: k6ApiToken,
 				RUN_DIR: runDir,
@@ -59,17 +69,24 @@ async function main() {
 		await dockerComposeClient.$('run', 'benchmark', 'run', `--scenarioNamePrefix=${n8nSetupToUse}`);
 	} catch (error) {
 		console.error('An error occurred while running the benchmarks:');
-		console.error(error);
+		console.error(error.message);
 		console.error('');
-		await dumpN8nInstanceLogs(dockerComposeClient);
+		await printContainerStatus(dockerComposeClient);
+		console.error('');
+		await dumpLogs(dockerComposeClient);
 	} finally {
 		await dockerComposeClient.$('down');
 	}
 }
 
-async function dumpN8nInstanceLogs(dockerComposeClient) {
+async function printContainerStatus(dockerComposeClient) {
-	console.error('n8n instance logs:');
+	console.error('Container statuses:');
-	await dockerComposeClient.$('logs', 'n8n');
+	await dockerComposeClient.$('ps', '-a');
+}
+
+async function dumpLogs(dockerComposeClient) {
+	console.error('Container logs:');
+	await dockerComposeClient.$('logs');
 }
 
 function printUsage() {

packages/@n8n/benchmark/scripts/runInCloud.mjs

@@ -29,6 +29,7 @@ import { TerraformClient } from './clients/terraformClient.mjs';
  * @property {string} n8nTag
  * @property {string} benchmarkTag
  * @property {string} [k6ApiToken]
+ * @property {string} [n8nLicenseCert]
  *
  * @param {Config} config
  */
@@ -96,6 +97,7 @@ async function runBenchmarkForN8nSetup({ config, sshClient, scriptsDir, n8nSetup
 		n8nDockerTag: config.n8nTag,
 		benchmarkDockerTag: config.benchmarkTag,
 		k6ApiToken: config.k6ApiToken,
+		n8nLicenseCert: config.n8nLicenseCert,
 	};
 
 	const flagsString = Object.entries(flags)

packages/@n8n/benchmark/scripts/runLocally.mjs

@@ -29,6 +29,7 @@ const paths = {
  * @property {string} benchmarkTag
  * @property {string} [runDir]
  * @property {string} [k6ApiToken]
+ * @property {string} [n8nLicenseCert]
  *
  * @param {Config} config
  */
@@ -51,6 +52,7 @@ export async function runLocally(config) {
 				env: {
 					...process.env,
 					K6_API_TOKEN: config.k6ApiToken,
+					N8N_LICENSE_CERT: config.n8nLicenseCert,
 				},
 			})`npx ${runScriptPath} ${flags} ${n8nSetup}`;
 		}