fix: Fixes to cloud benchmarks (no-changelog) (#10634)

packages/@n8n/benchmark/infra/benchmark-env.tf

@@ -25,7 +25,7 @@ resource "azurerm_dedicated_host_group" "main" {
   automatic_placement_enabled = false
   zone                        = 1
 
-	tags = local.common_tags
+  tags = local.common_tags
 }
 
 resource "azurerm_dedicated_host" "hosts" {
@@ -35,7 +35,7 @@ resource "azurerm_dedicated_host" "hosts" {
   sku_name                = var.host_size_family
   platform_fault_domain   = 0
 
-	tags = local.common_tags
+  tags = local.common_tags
 }
 
 # VM

packages/@n8n/benchmark/infra/modules/benchmark-vm/output.tf

@@ -5,3 +5,7 @@ output "vm_name" {
 output "ip" {
   value = azurerm_public_ip.main.ip_address
 }
+
+output "ssh_username" {
+  value = azurerm_linux_virtual_machine.main.admin_username
+}

packages/@n8n/benchmark/infra/modules/benchmark-vm/vm.tf

@@ -118,9 +118,9 @@ resource "azurerm_linux_virtual_machine" "main" {
     version   = "latest"
   }
 
-	identity {
+  identity {
-		type = "SystemAssigned"
+    type = "SystemAssigned"
-	}
+  }
 
   tags = var.tags
 }

packages/@n8n/benchmark/infra/output.tf

@@ -1,3 +1,16 @@
 output "vm_name" {
   value = module.test_vm.vm_name
 }
+
+output "ip" {
+  value = module.test_vm.ip
+}
+
+output "ssh_username" {
+  value = module.test_vm.ssh_username
+}
+
+output "ssh_private_key" {
+  value     = tls_private_key.ssh_key.private_key_pem
+  sensitive = true
+}

packages/@n8n/benchmark/infra/vars.tf

@@ -29,6 +29,6 @@ locals {
     Id        = "N8nBenchmark"
     Terraform = "true"
     Owner     = "Catalysts"
-		CreatedAt = timestamp()
+    CreatedAt = timestamp()
   }
 }

packages/@n8n/benchmark/scripts/bootstrap.sh

@@ -34,11 +34,9 @@ else
 	sudo mkfs.xfs /dev/sdc1
 	sudo partprobe /dev/sdc1
 	sudo mount /dev/sdc1 /n8n
+	sudo chown -R "$CURRENT_USER":"$CURRENT_USER" /n8n
 fi
 
-# Allow the current user to write to the data disk
-sudo chmod a+rw /n8n
-
 # Include nodejs v20 repository
 curl -fsSL https://deb.nodesource.com/setup_20.x -o nodesource_setup.sh
 sudo -E bash nodesource_setup.sh

packages/@n8n/benchmark/scripts/clients/sshClient.mjs

@@ -4,12 +4,13 @@ import { $ } from 'zx';
 export class SshClient {
 	/**
 	 *
-	 * @param {{ vmName: string; resourceGroupName: string; verbose?: boolean }} param0
+	 * @param {{ privateKeyPath: string; ip: string; username: string; verbose?: boolean }} param0
 	 */
-	constructor({ vmName, resourceGroupName, verbose = false }) {
+	constructor({ privateKeyPath, ip, username, verbose = false }) {
-		this.vmName = vmName;
-		this.resourceGroupName = resourceGroupName;
 		this.verbose = verbose;
+		this.privateKeyPath = privateKeyPath;
+		this.ip = ip;
+		this.username = username;
 
 		this.$$ = $({
 			verbose,
@@ -23,6 +24,14 @@ export class SshClient {
 	async ssh(command, options = {}) {
 		const $$ = options?.verbose ? $({ verbose: true }) : this.$$;
 
-		await $$`az ssh vm -n ${this.vmName} -g ${this.resourceGroupName} --yes -- -o StrictHostKeyChecking=accept-new ${command}`;
+		const target = `${this.username}@${this.ip}`;
+
+		await $$`ssh -i ${this.privateKeyPath} -o StrictHostKeyChecking=accept-new ${target} ${command}`;
+	}
+
+	async scp(source, destination) {
+		const target = `${this.username}@${this.ip}:${destination}`;
+		await this
+			.$$`scp -i ${this.privateKeyPath} -o StrictHostKeyChecking=accept-new ${source} ${target}`;
 	}
 }

packages/@n8n/benchmark/scripts/clients/terraformClient.mjs

@@ -20,6 +20,9 @@ export class TerraformClient {
 	/**
 	 * @typedef {Object} BenchmarkEnv
 	 * @property {string} vmName
+	 * @property {string} ip
+	 * @property {string} sshUsername
+	 * @property {string} sshPrivateKeyPath
 	 *
 	 * @returns {Promise<BenchmarkEnv>}
 	 */
@@ -27,9 +30,14 @@ export class TerraformClient {
 		console.log('Provisioning cloud environment...');
 
 		await this.$$`terraform init`;
-		await this.$$`terraform apply -input=false -auto-approve`;
+		// await this.$$`terraform apply -input=false -auto-approve`;
+
+		const privateKeyName = await this.extractPrivateKey();
 
 		return {
+			ip: await this.getTerraformOutput('ip'),
+			sshUsername: await this.getTerraformOutput('ssh_username'),
+			sshPrivateKeyPath: path.join(paths.infraCodeDir, privateKeyName),
 			vmName: await this.getTerraformOutput('vm_name'),
 		};
 	}
@@ -42,11 +50,18 @@ export class TerraformClient {
 
 		console.log('Destroying cloud environment...');
 
-		await this.$$`terraform destroy -input=false -auto-approve`;
+		// await this.$$`terraform destroy -input=false -auto-approve`;
 	}
 
 	async getTerraformOutput(key) {
 		const output = await this.$$`terraform output -raw ${key}`;
 		return output.stdout.trim();
 	}
+
+	async extractPrivateKey() {
+		await this.$$`terraform output -raw ssh_private_key > privatekey.pem`;
+		await this.$$`chmod 600 privatekey.pem`;
+
+		return 'privatekey.pem';
+	}
 }

packages/@n8n/benchmark/scripts/n8nSetups/postgres/docker-compose.yml

@@ -2,12 +2,23 @@ services:
   postgres:
     image: postgres:16
     restart: always
+    user: ${RUN_USER_AND_GROUP}
     environment:
       - POSTGRES_DB=n8n
       - POSTGRES_USER=postgres
       - POSTGRES_PASSWORD=password
+      - PGDATA=/var/lib/postgresql/data/pgdata
+    volumes:
+      - ${RUN_DIR}/postgres:/var/lib/postgresql/data
+    healthcheck:
+      test: ['CMD-SHELL', 'pg_isready -U postgres']
+      interval: 5s
+      timeout: 5s
+      retries: 5
+
   n8n:
     image: ghcr.io/n8n-io/n8n:${N8N_VERSION:-latest}
+    user: ${RUN_USER_AND_GROUP}
     environment:
       - N8N_DIAGNOSTICS_ENABLED=false
       - N8N_USER_FOLDER=/n8n
@@ -17,13 +28,21 @@ services:
     ports:
       - 5678:5678
     volumes:
-      - ${RUN_DIR}:/n8n
+      - ${RUN_DIR}/n8n:/n8n
     depends_on:
-      - postgres
+      postgres:
+        condition: service_healthy
+    healthcheck:
+      test: ['CMD-SHELL', 'wget --spider -q http://localhost:5678/healthz || exit 1']
+      interval: 5s
+      timeout: 5s
+      retries: 10
+
   benchmark:
     image: ghcr.io/n8n-io/n8n-benchmark:${N8N_BENCHMARK_VERSION:-latest}
     depends_on:
-      - n8n
+      n8n:
+        condition: service_healthy
     environment:
       - N8N_BASE_URL=http://n8n:5678
       - K6_API_TOKEN=${K6_API_TOKEN}

packages/@n8n/benchmark/scripts/n8nSetups/postgres/setup.mjs

@@ -0,0 +1,16 @@
+#!/usr/bin/env zx
+
+import path from 'path';
+import { fs } from 'zx';
+
+/**
+ * Creates the needed directories for the queue setup so their
+ * permissions get set correctly.
+ */
+export function setup({ runDir }) {
+	const neededDirs = ['n8n', 'postgres'];
+
+	for (const dir of neededDirs) {
+		fs.ensureDirSync(path.join(runDir, dir));
+	}
+}

packages/@n8n/benchmark/scripts/n8nSetups/queue/docker-compose.yml

@@ -3,71 +3,127 @@ services:
     image: redis:6-alpine
     ports:
       - 6379:6379
+    healthcheck:
+      test: ['CMD', 'redis-cli', 'ping']
+      interval: 1s
+      timeout: 3s
+
   postgres:
     image: postgres:16
+    user: ${RUN_USER_AND_GROUP}
     restart: always
     environment:
       - POSTGRES_DB=n8n
       - POSTGRES_USER=postgres
       - POSTGRES_PASSWORD=password
+      - PGDATA=/var/lib/postgresql/data/pgdata
+    volumes:
+      - ${RUN_DIR}/postgres:/var/lib/postgresql/data
+    healthcheck:
+      test: ['CMD-SHELL', 'pg_isready -U postgres']
+      interval: 5s
+      timeout: 5s
+      retries: 10
+
   n8n_worker1:
     image: ghcr.io/n8n-io/n8n:${N8N_VERSION:-latest}
+    user: ${RUN_USER_AND_GROUP}
     environment:
       - N8N_DIAGNOSTICS_ENABLED=false
       - N8N_USER_FOLDER=/n8n/worker1
       - N8N_ENCRYPTION_KEY=very-secret-encryption-key
+      # Queue mode config
       - EXECUTIONS_MODE=queue
       - QUEUE_BULL_REDIS_HOST=redis
+      - QUEUE_HEALTH_CHECK_ACTIVE=true
+      # DB config
       - DB_TYPE=postgresdb
       - DB_POSTGRESDB_HOST=postgres
       - DB_POSTGRESDB_PASSWORD=password
     command: worker
     volumes:
-      - ${RUN_DIR}:/n8n
+      - ${RUN_DIR}/n8n-worker1:/n8n
     depends_on:
-      - postgres
+      postgres:
-      - redis
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    healthcheck:
+      test: ['CMD-SHELL', 'wget --spider -q http://localhost:5678/healthz || exit 1']
+      interval: 5s
+      timeout: 5s
+      retries: 10
+
   n8n_worker2:
     image: ghcr.io/n8n-io/n8n:${N8N_VERSION:-latest}
+    user: ${RUN_USER_AND_GROUP}
     environment:
       - N8N_DIAGNOSTICS_ENABLED=false
       - N8N_USER_FOLDER=/n8n/worker2
       - N8N_ENCRYPTION_KEY=very-secret-encryption-key
+      # Queue mode config
       - EXECUTIONS_MODE=queue
       - QUEUE_BULL_REDIS_HOST=redis
+      - QUEUE_HEALTH_CHECK_ACTIVE=true
+      # DB config
       - DB_TYPE=postgresdb
       - DB_POSTGRESDB_HOST=postgres
       - DB_POSTGRESDB_PASSWORD=password
     command: worker
     volumes:
-      - ${RUN_DIR}:/n8n
+      - ${RUN_DIR}/n8n-worker2:/n8n
     depends_on:
-      - postgres
+      # We let the worker 1 start first so it can run the DB migrations
-      - redis
+      n8n_worker1:
+        condition: service_healthy
+      postgres:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    healthcheck:
+      test: ['CMD-SHELL', 'wget --spider -q http://localhost:5678/healthz || exit 1']
+      interval: 5s
+      timeout: 5s
+      retries: 10
+
   n8n:
     image: ghcr.io/n8n-io/n8n:${N8N_VERSION:-latest}
+    user: ${RUN_USER_AND_GROUP}
     environment:
       - N8N_DIAGNOSTICS_ENABLED=false
       - N8N_USER_FOLDER=/n8n/main
       - N8N_ENCRYPTION_KEY=very-secret-encryption-key
+      # Queue mode config
       - EXECUTIONS_MODE=queue
       - QUEUE_BULL_REDIS_HOST=redis
+      # DB config
       - DB_TYPE=postgresdb
       - DB_POSTGRESDB_HOST=postgres
       - DB_POSTGRESDB_PASSWORD=password
     ports:
       - 5678:5678
     volumes:
-      - ${RUN_DIR}:/n8n
+      - ${RUN_DIR}/n8n-main:/n8n
     depends_on:
-      - postgres
+      n8n_worker1:
-      - redis
+        condition: service_healthy
-      - n8n_worker1
+      n8n_worker2:
-      - n8n_worker2
+        condition: service_healthy
+      postgres:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    healthcheck:
+      test: ['CMD-SHELL', 'wget --spider -q http://localhost:5678/healthz || exit 1']
+      interval: 5s
+      timeout: 5s
+      retries: 10
+
   benchmark:
     image: ghcr.io/n8n-io/n8n-benchmark:${N8N_BENCHMARK_VERSION:-latest}
     depends_on:
-      - n8n
+      n8n:
+        condition: service_healthy
     environment:
       - N8N_BASE_URL=http://n8n:5678
       - K6_API_TOKEN=${K6_API_TOKEN}

packages/@n8n/benchmark/scripts/n8nSetups/queue/setup.mjs

@@ -0,0 +1,16 @@
+#!/usr/bin/env zx
+
+import path from 'path';
+import { fs } from 'zx';
+
+/**
+ * Creates the needed directories for the queue setup so their
+ * permissions get set correctly.
+ */
+export function setup({ runDir }) {
+	const neededDirs = ['n8n-worker1', 'n8n-worker2', 'n8n-main', 'postgres'];
+
+	for (const dir of neededDirs) {
+		fs.ensureDirSync(path.join(runDir, dir));
+	}
+}

packages/@n8n/benchmark/scripts/n8nSetups/sqlite-legacy/docker-compose.yml

@@ -1,6 +1,7 @@
 services:
   n8n:
     image: ghcr.io/n8n-io/n8n:${N8N_VERSION:-latest}
+    user: ${RUN_USER_AND_GROUP}
     environment:
       - N8N_DIAGNOSTICS_ENABLED=false
       - N8N_USER_FOLDER=/n8n
@@ -8,10 +9,17 @@ services:
       - 5678:5678
     volumes:
       - ${RUN_DIR}:/n8n
+    healthcheck:
+      test: ['CMD-SHELL', 'wget --spider -q http://localhost:5678/healthz || exit 1']
+      interval: 5s
+      timeout: 5s
+      retries: 10
+
   benchmark:
     image: ghcr.io/n8n-io/n8n-benchmark:${N8N_BENCHMARK_VERSION:-latest}
     depends_on:
-      - n8n
+      n8n:
+        condition: service_healthy
     environment:
       - N8N_BASE_URL=http://n8n:5678
       - K6_API_TOKEN=${K6_API_TOKEN}

packages/@n8n/benchmark/scripts/n8nSetups/sqlite-legacy/setup.mjs

@@ -0,0 +1,16 @@
+#!/usr/bin/env zx
+
+import path from 'path';
+import { fs } from 'zx';
+
+/**
+ * Creates the needed directories for the queue setup so their
+ * permissions get set correctly.
+ */
+export function setup({ runDir }) {
+	const neededDirs = ['n8n'];
+
+	for (const dir of neededDirs) {
+		fs.ensureDirSync(path.join(runDir, dir));
+	}
+}

packages/@n8n/benchmark/scripts/n8nSetups/sqlite/docker-compose.yml

@@ -1,6 +1,7 @@
 services:
   n8n:
     image: ghcr.io/n8n-io/n8n:${N8N_VERSION:-latest}
+    user: ${RUN_USER_AND_GROUP}
     environment:
       - N8N_DIAGNOSTICS_ENABLED=false
       - N8N_USER_FOLDER=/n8n
@@ -10,10 +11,17 @@ services:
       - 5678:5678
     volumes:
       - ${RUN_DIR}:/n8n
+    healthcheck:
+      test: ['CMD-SHELL', 'wget --spider -q http://localhost:5678/healthz || exit 1']
+      interval: 5s
+      timeout: 5s
+      retries: 10
+
   benchmark:
     image: ghcr.io/n8n-io/n8n-benchmark:${N8N_BENCHMARK_VERSION:-latest}
     depends_on:
-      - n8n
+      n8n:
+        condition: service_healthy
     environment:
       - N8N_BASE_URL=http://n8n:5678
       - K6_API_TOKEN=${K6_API_TOKEN}

packages/@n8n/benchmark/scripts/n8nSetups/sqlite/setup.mjs

@@ -0,0 +1,16 @@
+#!/usr/bin/env zx
+
+import path from 'path';
+import { fs } from 'zx';
+
+/**
+ * Creates the needed directories for the queue setup so their
+ * permissions get set correctly.
+ */
+export function setup({ runDir }) {
+	const neededDirs = ['n8n'];
+
+	for (const dir of neededDirs) {
+		fs.ensureDirSync(path.join(runDir, dir));
+	}
+}

packages/@n8n/benchmark/scripts/run.mjs

@@ -148,4 +148,9 @@ function printUsage() {
 	console.log('');
 }
 
-main().catch(console.error);
+main().catch((error) => {
+	console.error('An error occurred while running the benchmarks:');
+	console.error(error);
+
+	process.exit(1);
+});

packages/@n8n/benchmark/scripts/runForN8nSetup.mjs

@@ -16,6 +16,7 @@ async function main() {
 	validateN8nSetup(n8nSetupToUse);
 
 	const composeFilePath = path.join(paths.n8nSetupsDir, n8nSetupToUse);
+	const setupScriptPath = path.join(paths.n8nSetupsDir, n8nSetupToUse, 'setup.mjs');
 	const n8nTag = argv.n8nDockerTag || process.env.N8N_DOCKER_TAG || 'latest';
 	const benchmarkTag = argv.benchmarkDockerTag || process.env.BENCHMARK_DOCKER_TAG || 'latest';
 	const k6ApiToken = argv.k6ApiToken || process.env.K6_API_TOKEN || undefined;
@@ -30,9 +31,13 @@ async function main() {
 
 	const runDir = path.join(baseRunDir, n8nSetupToUse);
 	fs.emptyDirSync(runDir);
-	// Make sure the n8n container user (node) has write permissions to the run directory
-	await $`chmod 777 ${runDir}`;
 
+	if (!process.getuid) {
+		console.error('Windows is not supported');
+		process.exit(1);
+	}
+
+	const currentUserId = process.getuid();
 	const dockerComposeClient = new DockerComposeClient({
 		$: $({
 			cwd: composeFilePath,
@@ -42,10 +47,17 @@ async function main() {
 				BENCHMARK_VERSION: benchmarkTag,
 				K6_API_TOKEN: k6ApiToken,
 				RUN_DIR: runDir,
+				RUN_USER_AND_GROUP: `${currentUserId}:${currentUserId}`,
 			},
 		}),
 	});
 
+	// Run the setup script if it exists
+	if (fs.existsSync(setupScriptPath)) {
+		const setupScript = await import(setupScriptPath);
+		await setupScript.setup({ runDir });
+	}
+
 	try {
 		await dockerComposeClient.$('up', '-d', '--remove-orphans', 'n8n');
 

packages/@n8n/benchmark/scripts/runInCloud.mjs

@@ -9,7 +9,7 @@
  * NOTE: Must be run in the root of the package.
  */
 // @ts-check
-import { sleep, which } from 'zx';
+import { sleep, which, $, tmpdir } from 'zx';
 import path from 'path';
 import { SshClient } from './clients/sshClient.mjs';
 import { TerraformClient } from './clients/terraformClient.mjs';
@@ -17,10 +17,11 @@ import { TerraformClient } from './clients/terraformClient.mjs';
 /**
  * @typedef {Object} BenchmarkEnv
  * @property {string} vmName
+ * @property {string} ip
+ * @property {string} sshUsername
+ * @property {string} sshPrivateKeyPath
  */
 
-const RESOURCE_GROUP_NAME = 'n8n-benchmarking';
-
 /**
  * @typedef {Object} Config
  * @property {boolean} isVerbose
@@ -63,14 +64,15 @@ async function runBenchmarksOnVm(config, benchmarkEnv) {
 	console.log(`Setting up the environment...`);
 
 	const sshClient = new SshClient({
-		vmName: benchmarkEnv.vmName,
+		ip: benchmarkEnv.ip,
-		resourceGroupName: RESOURCE_GROUP_NAME,
+		username: benchmarkEnv.sshUsername,
+		privateKeyPath: benchmarkEnv.sshPrivateKeyPath,
 		verbose: config.isVerbose,
 	});
 
 	await ensureVmIsReachable(sshClient);
 
-	const scriptsDir = await transferScriptsToVm(sshClient);
+	const scriptsDir = await transferScriptsToVm(sshClient, config);
 
 	// Bootstrap the environment with dependencies
 	console.log('Running bootstrap script...');
@@ -121,8 +123,22 @@ async function ensureVmIsReachable(sshClient) {
 /**
  * @returns Path where the scripts are located on the VM
  */
-async function transferScriptsToVm(sshClient) {
+async function transferScriptsToVm(sshClient, config) {
-	await sshClient.ssh('rm -rf ~/n8n && git clone --depth=1 https://github.com/n8n-io/n8n.git');
+	const cwd = process.cwd();
+	const scriptsDir = path.resolve(cwd, './scripts');
+	const tarFilename = 'scripts.tar.gz';
+	const scriptsTarPath = path.join(tmpdir('n8n-benchmark'), tarFilename);
 
-	return '~/n8n/packages/@n8n/benchmark/scripts';
+	const $$ = $({ verbose: config.isVerbose });
+
+	// Compress the scripts folder
+	await $$`tar -czf ${scriptsTarPath} ${scriptsDir} -C ${cwd} ./scripts`;
+
+	// Transfer the scripts to the VM
+	await sshClient.scp(scriptsTarPath, `~/${tarFilename}`);
+
+	// Extract the scripts on the VM
+	await sshClient.ssh(`tar -xzf ~/${tarFilename}`);
+
+	return '~/scripts';
 }