refactor(benchmark): Separate cloud env provisioning from running benchmarks (#10657)

.github/workflows/benchmark-nightly.yml

@@ -24,6 +24,9 @@ env:
   ARM_SUBSCRIPTION_ID: ${{ secrets.BENCHMARK_ARM_SUBSCRIPTION_ID }}
   ARM_TENANT_ID: ${{ secrets.BENCHMARK_ARM_TENANT_ID }}
   K6_API_TOKEN: ${{ secrets.K6_API_TOKEN }}
+  N8N_TAG: ${{ inputs.n8n_tag || 'nightly' }}
+  N8N_BENCHMARK_TAG: ${{ inputs.benchmark_tag || 'latest' }}
+  DEBUG: ${{ inputs.debug == 'true' && '--debug' || '' }}
 
 permissions:
   id-token: write
@@ -62,12 +65,23 @@ jobs:
         run: pnpm destroy-cloud-env
         working-directory: packages/@n8n/benchmark
 
-      - name: Run the benchmark with debug logging
-        if: github.event.inputs.debug == 'true'
-        run: pnpm benchmark-in-cloud --n8nTag ${{ inputs.n8n_tag || 'nightly' }} --benchmarkTag ${{ inputs.benchmark_tag || 'latest' }} --debug
+      - name: Provision the environment
+        run: pnpm provision-cloud-env ${{ env.DEBUG }}
         working-directory: packages/@n8n/benchmark
 
       - name: Run the benchmark
-        if: github.event.inputs.debug != 'true'
-        run: pnpm benchmark-in-cloud --n8nTag ${{ inputs.n8n_tag || 'nightly' }} --benchmarkTag ${{ inputs.benchmark_tag || 'latest' }}
+        run: pnpm benchmark-in-cloud --n8nTag ${{ env.N8N_TAG }} --benchmarkTag ${{ env.N8N_BENCHMARK_TAG }} ${{ env.DEBUG }}
+        working-directory: packages/@n8n/benchmark
+
+        # We need to login again because the access token expires
+      - name: Azure login
+        uses: azure/login@v2.1.1
+        with:
+          client-id: ${{ env.ARM_CLIENT_ID }}
+          tenant-id: ${{ env.ARM_TENANT_ID }}
+          subscription-id: ${{ env.ARM_SUBSCRIPTION_ID }}
+
+      - name: Destroy the environment
+        if: always()
+        run: pnpm destroy-cloud-env ${{ env.DEBUG }}
         working-directory: packages/@n8n/benchmark

packages/@n8n/benchmark/package.json

@@ -13,6 +13,7 @@
     "benchmark": "zx scripts/run.mjs",
     "benchmark-in-cloud": "pnpm benchmark --env cloud",
     "benchmark-locally": "pnpm benchmark --env local",
+    "provision-cloud-env": "zx scripts/provisionCloudEnv.mjs",
     "destroy-cloud-env": "zx scripts/destroyCloudEnv.mjs",
     "watch": "concurrently \"tsc -w -p tsconfig.build.json\" \"tsc-alias -w -p tsconfig.build.json\""
   },

packages/@n8n/benchmark/scripts/clients/terraformClient.mjs

@@ -17,6 +17,16 @@ export class TerraformClient {
 		});
 	}
 
+	/**
+	 * Provisions the environment
+	 */
+	async provisionEnvironment() {
+		console.log('Provisioning cloud environment...');
+
+		await this.$$`terraform init`;
+		await this.$$`terraform apply -input=false -auto-approve`;
+	}
+
 	/**
 	 * @typedef {Object} BenchmarkEnv
 	 * @property {string} vmName
@@ -26,12 +36,7 @@ export class TerraformClient {
 	 *
 	 * @returns {Promise<BenchmarkEnv>}
 	 */
-	async provisionEnvironment() {
-		console.log('Provisioning cloud environment...');
-
-		await this.$$`terraform init`;
-		await this.$$`terraform apply -input=false -auto-approve`;
-
+	async getTerraformOutputs() {
 		const privateKeyName = await this.extractPrivateKey();
 
 		return {
@@ -42,12 +47,11 @@ export class TerraformClient {
 		};
 	}
 
-	async destroyEnvironment() {
-		if (!fs.existsSync(paths.terraformStateFile)) {
-			console.log('No cloud environment to destroy. Skipping...');
-			return;
-		}
+	hasTerraformState() {
+		return fs.existsSync(paths.terraformStateFile);
+	}
 
+	async destroyEnvironment() {
 		console.log('Destroying cloud environment...');
 
 		await this.$$`terraform destroy -input=false -auto-approve`;

packages/@n8n/benchmark/scripts/destroyCloudEnv.mjs

@@ -1,52 +1,43 @@
 #!/usr/bin/env zx
 /**
- * Script that deletes all resources created by the benchmark environment
- * and that are older than 2 hours.
+ * Script that deletes all resources created by the benchmark environment.
  *
- * Even tho the environment is provisioned using terraform, the terraform
- * state is not persisted. Hence we can't use terraform to delete the resources.
- * We could store the state to a storage account, but then we wouldn't be able
- * to spin up new envs on-demand. Hence this design.
- *
- * Usage:
- * 	 zx scripts/deleteCloudEnv.mjs
+ * This scripts tries to delete resources created by Terraform. If Terraform
+ * state file is not found, it will try to delete resources using Azure CLI.
+ * The terraform state is not persisted, so we want to support both cases.
  */
 // @ts-check
-import { $ } from 'zx';
+import { $, minimist } from 'zx';
+import { TerraformClient } from './clients/terraformClient.mjs';
 
-const EXPIRE_TIME_IN_H = 2;
-const EXPIRE_TIME_IN_MS = EXPIRE_TIME_IN_H * 60 * 60 * 1000;
 const RESOURCE_GROUP_NAME = 'n8n-benchmarking';
 
+const args = minimist(process.argv.slice(3), {
+	boolean: ['debug'],
+});
+
+const isVerbose = !!args.debug;
+
 async function main() {
+	const terraformClient = new TerraformClient({ isVerbose });
+
+	if (terraformClient.hasTerraformState()) {
+		await terraformClient.destroyEnvironment();
+	} else {
+		await destroyUsingAz();
+	}
+}
+
+async function destroyUsingAz() {
 	const resourcesResult =
 		await $`az resource list --resource-group ${RESOURCE_GROUP_NAME} --query "[?tags.Id == 'N8nBenchmark'].{id:id, createdAt:tags.CreatedAt}" -o json`;
 
 	const resources = JSON.parse(resourcesResult.stdout);
 
-	const now = Date.now();
-
-	const resourcesToDelete = resources
-		.filter((resource) => {
-			if (resource.createdAt === undefined) {
-				return true;
-			}
-
-			const createdAt = new Date(resource.createdAt);
-			const resourceExpiredAt = createdAt.getTime() + EXPIRE_TIME_IN_MS;
-
-			return now > resourceExpiredAt;
-		})
-		.map((resource) => resource.id);
+	const resourcesToDelete = resources.map((resource) => resource.id);
 
 	if (resourcesToDelete.length === 0) {
-		if (resources.length === 0) {
-			console.log('No resources found in the resource group.');
-		} else {
-			console.log(
-				`Found ${resources.length} resources in the resource group, but none are older than ${EXPIRE_TIME_IN_H} hours.`,
-			);
-		}
+		console.log('No resources found in the resource group.');
 
 		return;
 	}
@@ -87,4 +78,9 @@ async function deleteById(id) {
 	}
 }
 
-main();
+main().catch((error) => {
+	console.error('An error occurred destroying cloud env:');
+	console.error(error);
+
+	process.exit(1);
+});

packages/@n8n/benchmark/scripts/provisionCloudEnv.mjs

@@ -0,0 +1,36 @@
+#!/usr/bin/env zx
+/**
+ * Provisions the cloud benchmark environment
+ *
+ * NOTE: Must be run in the root of the package.
+ */
+// @ts-check
+import { which, minimist } from 'zx';
+import { TerraformClient } from './clients/terraformClient.mjs';
+
+const args = minimist(process.argv.slice(3), {
+	boolean: ['debug'],
+});
+
+const isVerbose = !!args.debug;
+
+export async function provision() {
+	await ensureDependencies();
+
+	const terraformClient = new TerraformClient({
+		isVerbose,
+	});
+
+	await terraformClient.provisionEnvironment();
+}
+
+async function ensureDependencies() {
+	await which('terraform');
+}
+
+provision().catch((error) => {
+	console.error('An error occurred while provisioning cloud env:');
+	console.error(error);
+
+	process.exit(1);
+});

packages/@n8n/benchmark/scripts/run.mjs

@@ -1,12 +1,9 @@
 #!/usr/bin/env zx
 /**
  * Script to run benchmarks either on the cloud benchmark environment or locally.
+ * The cloud environment needs to be provisioned using Terraform before running the benchmarks.
  *
  * NOTE: Must be run in the root of the package.
- *
- * Usage:
- * 	 zx scripts/run.mjs
- *
  */
 // @ts-check
 import fs from 'fs';

packages/@n8n/benchmark/scripts/runInCloud.mjs

@@ -39,16 +39,9 @@ export async function runInCloud(config) {
 		isVerbose: config.isVerbose,
 	});
 
-	try {
-		const benchmarkEnv = await terraformClient.provisionEnvironment();
+	const benchmarkEnv = await terraformClient.getTerraformOutputs();
 
-		await runBenchmarksOnVm(config, benchmarkEnv);
-	} catch (error) {
-		console.error('An error occurred while running the benchmarks:');
-		console.error(error);
-	} finally {
-		await terraformClient.destroyEnvironment();
-	}
+	await runBenchmarksOnVm(config, benchmarkEnv);
 }
 
 async function ensureDependencies() {
@@ -117,7 +110,15 @@ async function runBenchmarkForN8nSetup({ config, sshClient, scriptsDir, n8nSetup
 }
 
 async function ensureVmIsReachable(sshClient) {
-	await sshClient.ssh('echo "VM is reachable"');
+	try {
+		await sshClient.ssh('echo "VM is reachable"');
+	} catch (error) {
+		console.error(`VM is not reachable: ${error.message}`);
+		console.error(
+			`Did you provision the cloud environment first with 'pnpm provision-cloud-env'? You can also run the benchmarks locally with 'pnpm run benchmark-locally'.`,
+		);
+		process.exit(1);
+	}
 }
 
 /**