diff --git a/packages/@n8n/benchmark/README.md b/packages/@n8n/benchmark/README.md index a16e03572a..d2b7784fa6 100644 --- a/packages/@n8n/benchmark/README.md +++ b/packages/@n8n/benchmark/README.md @@ -1,8 +1,38 @@ # n8n benchmarking tool -Tool for executing benchmarks against an n8n instance. +Tool for executing benchmarks against an n8n instance. The tool consists of these components: -## Running locally with Docker +## Directory structure + +```text +packages/@n8n/benchmark +├── scenarios Benchmark scenarios +├── src Source code for the n8n-benchmark cli +├── Dockerfile Dockerfile for the n8n-benchmark cli +├── scripts Orchestration scripts +``` + +## Running the entire benchmark suite + +The benchmark suite consists of [benchmark scenarios](#benchmark-scenarios) and different [n8n setups](#n8n-setups). + +### locally + +```sh +pnpm run-locally +``` + +### In the cloud + +```sh +pnpm run-in-cloud +``` + +## Running the `n8n-benchmark` cli + +The `n8n-benchmark` cli is a node.js program that runs one or more scenarios against a single n8n instance. + +### Locally with Docker Build the Docker image: @@ -23,7 +53,7 @@ docker run \ n8n-benchmark ``` -## Running locally without Docker +### Locally without Docker Requirements: @@ -35,23 +65,8 @@ pnpm build # Run tests against http://localhost:5678 with specified email and password N8N_USER_EMAIL=user@n8n.io N8N_USER_PASSWORD=password ./bin/n8n-benchmark run - -# If you installed k6 using brew, you might have to specify it explicitly -K6_PATH=/opt/homebrew/bin/k6 N8N_USER_EMAIL=user@n8n.io N8N_USER_PASSWORD=password ./bin/n8n-benchmark run ``` -## Running in the cloud - -There's a script to run the performance tests in a cloud environment. The script provisions a cloud environment, sets up n8n in the environment, runs the tests and destroys the environment. - -```sh -pnpm run-in-cloud -``` - -## Configuration - -The configuration options the cli accepts can be seen from [config.ts](./src/config/config.ts) - ## Benchmark scenarios A benchmark scenario defines one or multiple steps to execute and measure. It consists of: @@ -61,3 +76,7 @@ A benchmark scenario defines one or multiple steps to execute and measure. It co - A [`k6`](https://grafana.com/docs/k6/latest/using-k6/http-requests/) script which executes the steps and receives `API_BASE_URL` environment variable in runtime. Available scenarios are located in [`./scenarios`](./scenarios/). + +## n8n setups + +A n8n setup defines a single n8n runtime configuration using Docker compose. Different n8n setups are located in [`./scripts/n8nSetups`](./scripts/n8nSetups). diff --git a/packages/@n8n/benchmark/package.json b/packages/@n8n/benchmark/package.json index 1e73381427..2d9affef5d 100644 --- a/packages/@n8n/benchmark/package.json +++ b/packages/@n8n/benchmark/package.json @@ -10,7 +10,9 @@ "start": "./bin/n8n-benchmark", "test": "echo \"Error: no test specified\" && exit 1", "typecheck": "tsc --noEmit", - "run-in-cloud": "zx scripts/runInCloud.mjs", + "benchmark": "zx scripts/run.mjs", + "benchmark-in-cloud": "pnpm benchmark --env cloud", + "benchmark-locally": "pnpm benchmark --env local", "destroy-cloud-env": "zx scripts/destroyCloudEnv.mjs", "watch": "concurrently \"tsc -w -p tsconfig.build.json\" \"tsc-alias -w -p tsconfig.build.json\"" }, diff --git a/packages/@n8n/benchmark/scripts/runOnVm/bootstrap.sh b/packages/@n8n/benchmark/scripts/bootstrap.sh similarity index 100% rename from packages/@n8n/benchmark/scripts/runOnVm/bootstrap.sh rename to packages/@n8n/benchmark/scripts/bootstrap.sh diff --git a/packages/@n8n/benchmark/scripts/clients/dockerComposeClient.mjs b/packages/@n8n/benchmark/scripts/clients/dockerComposeClient.mjs new file mode 100644 index 0000000000..064893a572 --- /dev/null +++ b/packages/@n8n/benchmark/scripts/clients/dockerComposeClient.mjs @@ -0,0 +1,45 @@ +import { which } from 'zx'; + +export class DockerComposeClient { + /** + * + * @param {{ $: Shell; verbose?: boolean }} opts + */ + constructor({ $ }) { + this.$$ = $; + } + + async $(...args) { + await this.resolveExecutableIfNeeded(); + + if (this.isCompose) { + return await this.$$`docker-compose ${args}`; + } else { + return await this.$$`docker compose ${args}`; + } + } + + async resolveExecutableIfNeeded() { + if (this.isResolved) { + return; + } + + // The VM deployment doesn't have `docker compose` available, + // so try to resolve the `docker-compose` first + const compose = await which('docker-compose', { nothrow: true }); + if (compose) { + this.isResolved = true; + this.isCompose = true; + return; + } + + const docker = await which('docker', { nothrow: true }); + if (docker) { + this.isResolved = true; + this.isCompose = false; + return; + } + + throw new Error('Could not resolve docker-compose or docker'); + } +} diff --git a/packages/@n8n/benchmark/scripts/sshClient.mjs b/packages/@n8n/benchmark/scripts/clients/sshClient.mjs similarity index 100% rename from packages/@n8n/benchmark/scripts/sshClient.mjs rename to packages/@n8n/benchmark/scripts/clients/sshClient.mjs diff --git a/packages/@n8n/benchmark/scripts/terraformClient.mjs b/packages/@n8n/benchmark/scripts/clients/terraformClient.mjs similarity index 92% rename from packages/@n8n/benchmark/scripts/terraformClient.mjs rename to packages/@n8n/benchmark/scripts/clients/terraformClient.mjs index 1ba4fcedb4..bfbf914faa 100644 --- a/packages/@n8n/benchmark/scripts/terraformClient.mjs +++ b/packages/@n8n/benchmark/scripts/clients/terraformClient.mjs @@ -9,8 +9,7 @@ const paths = { }; export class TerraformClient { - constructor({ privateKeyPath, isVerbose = false }) { - this.privateKeyPath = privateKeyPath; + constructor({ isVerbose = false }) { this.isVerbose = isVerbose; this.$$ = $({ cwd: paths.infraCodeDir, diff --git a/packages/@n8n/benchmark/scripts/runOnVm/n8nSetups/sqlite-legacy/docker-compose.yml b/packages/@n8n/benchmark/scripts/n8nSetups/sqlite-legacy/docker-compose.yml similarity index 94% rename from packages/@n8n/benchmark/scripts/runOnVm/n8nSetups/sqlite-legacy/docker-compose.yml rename to packages/@n8n/benchmark/scripts/n8nSetups/sqlite-legacy/docker-compose.yml index 02b61961f1..4210339f8e 100644 --- a/packages/@n8n/benchmark/scripts/runOnVm/n8nSetups/sqlite-legacy/docker-compose.yml +++ b/packages/@n8n/benchmark/scripts/n8nSetups/sqlite-legacy/docker-compose.yml @@ -7,7 +7,7 @@ services: ports: - 5678:5678 volumes: - - /n8n:/n8n + - ${RUN_DIR}:/n8n benchmark: image: ghcr.io/n8n-io/n8n-benchmark:${N8N_BENCHMARK_VERSION:-latest} depends_on: diff --git a/packages/@n8n/benchmark/scripts/runOnVm/n8nSetups/sqlite/docker-compose.yml b/packages/@n8n/benchmark/scripts/n8nSetups/sqlite/docker-compose.yml similarity index 95% rename from packages/@n8n/benchmark/scripts/runOnVm/n8nSetups/sqlite/docker-compose.yml rename to packages/@n8n/benchmark/scripts/n8nSetups/sqlite/docker-compose.yml index 3d953b04d2..36ad256fd2 100644 --- a/packages/@n8n/benchmark/scripts/runOnVm/n8nSetups/sqlite/docker-compose.yml +++ b/packages/@n8n/benchmark/scripts/n8nSetups/sqlite/docker-compose.yml @@ -9,7 +9,7 @@ services: ports: - 5678:5678 volumes: - - /n8n:/n8n + - ${RUN_DIR}:/n8n benchmark: image: ghcr.io/n8n-io/n8n-benchmark:${N8N_BENCHMARK_VERSION:-latest} depends_on: diff --git a/packages/@n8n/benchmark/scripts/run.mjs b/packages/@n8n/benchmark/scripts/run.mjs new file mode 100755 index 0000000000..fe268843e7 --- /dev/null +++ b/packages/@n8n/benchmark/scripts/run.mjs @@ -0,0 +1,151 @@ +#!/usr/bin/env zx +/** + * Script to run benchmarks either on the cloud benchmark environment or locally. + * + * NOTE: Must be run in the root of the package. + * + * Usage: + * zx scripts/run.mjs + * + */ +// @ts-check +import fs from 'fs'; +import minimist from 'minimist'; +import path from 'path'; +import { runInCloud } from './runInCloud.mjs'; +import { runLocally } from './runLocally.mjs'; + +const paths = { + n8nSetupsDir: path.join(path.resolve('scripts'), 'n8nSetups'), +}; + +async function main() { + const config = await parseAndValidateConfig(); + + const n8nSetupsToUse = + config.n8nSetupToUse === 'all' ? readAvailableN8nSetups() : [config.n8nSetupToUse]; + + console.log('Using n8n tag', config.n8nTag); + console.log('Using benchmark cli tag', config.benchmarkTag); + console.log('Using environment', config.env); + console.log('Using n8n setups', n8nSetupsToUse.join(', ')); + console.log(''); + + if (config.env === 'cloud') { + await runInCloud({ + benchmarkTag: config.benchmarkTag, + isVerbose: config.isVerbose, + k6ApiToken: config.k6ApiToken, + n8nTag: config.n8nTag, + n8nSetupsToUse, + }); + } else { + await runLocally({ + benchmarkTag: config.benchmarkTag, + isVerbose: config.isVerbose, + k6ApiToken: config.k6ApiToken, + n8nTag: config.n8nTag, + runDir: config.runDir, + n8nSetupsToUse, + }); + } +} + +function readAvailableN8nSetups() { + const setups = fs.readdirSync(paths.n8nSetupsDir); + + return setups; +} + +/** + * @typedef {Object} Config + * @property {boolean} isVerbose + * @property {'cloud' | 'local'} env + * @property {string} n8nSetupToUse + * @property {string} n8nTag + * @property {string} benchmarkTag + * @property {string} [k6ApiToken] + * @property {string} [runDir] + * + * @returns {Promise} + */ +async function parseAndValidateConfig() { + const args = minimist(process.argv.slice(3), { + boolean: ['debug', 'help'], + }); + + if (args.help) { + printUsage(); + process.exit(0); + } + + const n8nSetupToUse = await getAndValidateN8nSetup(args); + const isVerbose = args.debug || false; + const n8nTag = args.n8nTag || process.env.N8N_DOCKER_TAG || 'latest'; + const benchmarkTag = args.benchmarkTag || process.env.BENCHMARK_DOCKER_TAG || 'latest'; + const k6ApiToken = args.k6ApiToken || process.env.K6_API_TOKEN || undefined; + const runDir = args.runDir || undefined; + const env = args.env || 'local'; + + if (!env) { + printUsage(); + process.exit(1); + } + + return { + isVerbose, + env, + n8nSetupToUse, + n8nTag, + benchmarkTag, + k6ApiToken, + runDir, + }; +} + +/** + * @param {minimist.ParsedArgs} args + */ +async function getAndValidateN8nSetup(args) { + // Last parameter is the n8n setup to use + const n8nSetupToUse = args._[args._.length - 1]; + if (!n8nSetupToUse || n8nSetupToUse === 'all') { + return 'all'; + } + + const availableSetups = readAvailableN8nSetups(); + + if (!availableSetups.includes(n8nSetupToUse)) { + printUsage(); + process.exit(1); + } + + return n8nSetupToUse; +} + +function printUsage() { + const availableSetups = readAvailableN8nSetups(); + + console.log(`Usage: zx scripts/${path.basename(__filename)} [n8n setup name]`); + console.log(` eg: zx scripts/${path.basename(__filename)}`); + console.log(''); + console.log('Options:'); + console.log( + ` [n8n setup name] Against which n8n setup to run the benchmarks. One of: ${['all', ...availableSetups].join(', ')}. Default is all`, + ); + console.log( + ' --env Env where to run the benchmarks. Either cloud or local. Default is local.', + ); + console.log(' --debug Enable verbose output'); + console.log(' --n8nTag Docker tag for n8n image. Default is latest'); + console.log(' --benchmarkTag Docker tag for benchmark cli image. Default is latest'); + console.log( + ' --k6ApiToken API token for k6 cloud. Default is read from K6_API_TOKEN env var. If omitted, k6 cloud will not be used', + ); + console.log( + ' --runDir Directory to share with the n8n container for storing data. Needed only for local runs.', + ); + console.log(''); +} + +main().catch(console.error); diff --git a/packages/@n8n/benchmark/scripts/runForN8nSetup.mjs b/packages/@n8n/benchmark/scripts/runForN8nSetup.mjs new file mode 100755 index 0000000000..1bd31a1b5e --- /dev/null +++ b/packages/@n8n/benchmark/scripts/runForN8nSetup.mjs @@ -0,0 +1,97 @@ +#!/usr/bin/env zx +/** + * This script runs the benchmarks for the given n8n setup. + */ +// @ts-check +import path from 'path'; +import { $, argv, fs } from 'zx'; +import { DockerComposeClient } from './clients/dockerComposeClient.mjs'; + +const paths = { + n8nSetupsDir: path.join(__dirname, 'n8nSetups'), +}; + +async function main() { + const [n8nSetupToUse] = argv._; + validateN8nSetup(n8nSetupToUse); + + const composeFilePath = path.join(paths.n8nSetupsDir, n8nSetupToUse); + const n8nTag = argv.n8nDockerTag || process.env.N8N_DOCKER_TAG || 'latest'; + const benchmarkTag = argv.benchmarkDockerTag || process.env.BENCHMARK_DOCKER_TAG || 'latest'; + const k6ApiToken = argv.k6ApiToken || process.env.K6_API_TOKEN || undefined; + const runDir = argv.runDir || process.env.RUN_DIR || '/n8n'; + + if (!fs.existsSync(runDir)) { + console.error( + `The run directory "${runDir}" does not exist. Please specify a valid directory using --runDir`, + ); + process.exit(1); + } + + const dockerComposeClient = new DockerComposeClient({ + $: $({ + cwd: composeFilePath, + verbose: true, + env: { + N8N_VERSION: n8nTag, + BENCHMARK_VERSION: benchmarkTag, + K6_API_TOKEN: k6ApiToken, + RUN_DIR: runDir, + }, + }), + }); + + try { + await dockerComposeClient.$('up', '-d', 'n8n'); + + await dockerComposeClient.$('run', 'benchmark', 'run', `--scenarioNamePrefix=${n8nSetupToUse}`); + } catch (error) { + console.error('An error occurred while running the benchmarks:'); + console.error(error); + console.error(''); + await dumpN8nInstanceLogs(dockerComposeClient); + } finally { + await dockerComposeClient.$('down'); + } +} + +async function dumpN8nInstanceLogs(dockerComposeClient) { + console.error('n8n instance logs:'); + await dockerComposeClient.$('logs', 'n8n'); +} + +function printUsage() { + const availableSetups = getAllN8nSetups(); + console.log('Usage: zx runForN8nSetup.mjs --runDir /path/for/n8n/data '); + console.log(` eg: zx runForN8nSetup.mjs --runDir /path/for/n8n/data ${availableSetups[0]}`); + console.log(''); + console.log('Flags:'); + console.log( + ' --runDir Directory to share with the n8n container for storing data. Default is /n8n', + ); + console.log(' --n8nDockerTag Docker tag for n8n image. Default is latest'); + console.log( + ' --benchmarkDockerTag Docker tag for benchmark cli image. Default is latest', + ); + console.log(' --k6ApiToken K6 API token to upload the results'); + console.log(''); + console.log('Available setups:'); + console.log(availableSetups.join(', ')); +} + +/** + * @returns {string[]} + */ +function getAllN8nSetups() { + return fs.readdirSync(paths.n8nSetupsDir); +} + +function validateN8nSetup(givenSetup) { + const availableSetups = getAllN8nSetups(); + if (!availableSetups.includes(givenSetup)) { + printUsage(); + process.exit(1); + } +} + +main(); diff --git a/packages/@n8n/benchmark/scripts/runInCloud.mjs b/packages/@n8n/benchmark/scripts/runInCloud.mjs index e1c71d37d2..dee3a30f06 100755 --- a/packages/@n8n/benchmark/scripts/runInCloud.mjs +++ b/packages/@n8n/benchmark/scripts/runInCloud.mjs @@ -7,18 +7,12 @@ * 3. Destroy the cloud environment. * * NOTE: Must be run in the root of the package. - * - * Usage: - * zx scripts/runBenchmarksOnCloud.mjs [--debug] - * */ // @ts-check -import fs from 'fs'; -import minimist from 'minimist'; import { sleep, which } from 'zx'; import path from 'path'; -import { SshClient } from './sshClient.mjs'; -import { TerraformClient } from './terraformClient.mjs'; +import { SshClient } from './clients/sshClient.mjs'; +import { TerraformClient } from './clients/terraformClient.mjs'; /** * @typedef {Object} BenchmarkEnv @@ -27,19 +21,20 @@ import { TerraformClient } from './terraformClient.mjs'; const RESOURCE_GROUP_NAME = 'n8n-benchmarking'; -const paths = { - n8nSetupsDir: path.join(path.resolve('scripts'), 'runOnVm', 'n8nSetups'), -}; - -async function main() { - const config = await parseAndValidateConfig(); +/** + * @typedef {Object} Config + * @property {boolean} isVerbose + * @property {string[]} n8nSetupsToUse + * @property {string} n8nTag + * @property {string} benchmarkTag + * @property {string} [k6ApiToken] + * + * @param {Config} config + */ +export async function runInCloud(config) { await ensureDependencies(); - console.log('Using n8n tag', config.n8nTag); - console.log('Using benchmark cli tag', config.benchmarkTag); - const terraformClient = new TerraformClient({ - privateKeyPath: paths.privateKeyPath, isVerbose: config.isVerbose, }); @@ -65,7 +60,7 @@ async function ensureDependencies() { * @param {BenchmarkEnv} benchmarkEnv */ async function runBenchmarksOnVm(config, benchmarkEnv) { - console.log(`Setting up the environment for ${config.n8nSetupToUse}...`); + console.log(`Setting up the environment...`); const sshClient = new SshClient({ vmName: benchmarkEnv.vmName, @@ -85,23 +80,12 @@ async function runBenchmarksOnVm(config, benchmarkEnv) { // Give some time for the VM to be ready await sleep(1000); - if (config.n8nSetupToUse === 'all') { - const availableSetups = readAvailableN8nSetups(); - - for (const n8nSetup of availableSetups) { - await runBenchmarkForN8nSetup({ - config, - sshClient, - scriptsDir, - n8nSetup, - }); - } - } else { + for (const n8nSetup of config.n8nSetupsToUse) { await runBenchmarkForN8nSetup({ config, sshClient, scriptsDir, - n8nSetup: config.n8nSetupToUse, + n8nSetup, }); } } @@ -111,7 +95,7 @@ async function runBenchmarksOnVm(config, benchmarkEnv) { */ async function runBenchmarkForN8nSetup({ config, sshClient, scriptsDir, n8nSetup }) { console.log(`Running benchmarks for ${n8nSetup}...`); - const runScriptPath = path.join(scriptsDir, 'runOnVm.mjs'); + const runScriptPath = path.join(scriptsDir, 'runForN8nSetup.mjs'); const flags = { n8nDockerTag: config.n8nTag, @@ -142,87 +126,5 @@ async function transferScriptsToVm(sshClient) { await sshClient.ssh('git clone --depth=1 https://github.com/n8n-io/n8n.git'); - return '~/n8n/packages/@n8n/benchmark/scripts/runOnVm'; + return '~/n8n/packages/@n8n/benchmark/scripts'; } - -function readAvailableN8nSetups() { - const setups = fs.readdirSync(paths.n8nSetupsDir); - - return setups; -} - -/** - * @typedef {Object} Config - * @property {boolean} isVerbose - * @property {string} n8nSetupToUse - * @property {string} n8nTag - * @property {string} benchmarkTag - * @property {string} [k6ApiToken] - * - * @returns {Promise} - */ -async function parseAndValidateConfig() { - const args = minimist(process.argv.slice(3), { - boolean: ['debug', 'help'], - }); - - if (args.help) { - printUsage(); - process.exit(0); - } - - const n8nSetupToUse = await getAndValidateN8nSetup(args); - const isVerbose = args.debug || false; - const n8nTag = args.n8nTag || process.env.N8N_DOCKER_TAG || 'latest'; - const benchmarkTag = args.benchmarkTag || process.env.BENCHMARK_DOCKER_TAG || 'latest'; - const k6ApiToken = args.k6ApiToken || process.env.K6_API_TOKEN || undefined; - - return { - isVerbose, - n8nSetupToUse, - n8nTag, - benchmarkTag, - k6ApiToken, - }; -} - -/** - * @param {minimist.ParsedArgs} args - */ -async function getAndValidateN8nSetup(args) { - // Last parameter is the n8n setup to use - const n8nSetupToUse = args._[args._.length - 1]; - if (!n8nSetupToUse || n8nSetupToUse === 'all') { - return 'all'; - } - - const availableSetups = readAvailableN8nSetups(); - - if (!availableSetups.includes(n8nSetupToUse)) { - printUsage(); - process.exit(1); - } - - return n8nSetupToUse; -} - -function printUsage() { - const availableSetups = readAvailableN8nSetups(); - - console.log('Usage: zx scripts/runInCloud.mjs [n8n setup name]'); - console.log(' eg: zx scripts/runInCloud.mjs'); - console.log(''); - console.log('Options:'); - console.log( - ` [n8n setup name] Against which n8n setup to run the benchmarks. One of: ${['all', ...availableSetups].join(', ')}. Default is all`, - ); - console.log(' --debug Enable verbose output'); - console.log(' --n8nTag Docker tag for n8n image. Default is latest'); - console.log(' --benchmarkTag Docker tag for benchmark cli image. Default is latest'); - console.log( - ' --k6ApiToken API token for k6 cloud. Default is read from K6_API_TOKEN env var. If omitted, k6 cloud will not be used', - ); - console.log(''); -} - -main().catch(console.error); diff --git a/packages/@n8n/benchmark/scripts/runLocally.mjs b/packages/@n8n/benchmark/scripts/runLocally.mjs new file mode 100755 index 0000000000..8fcdcacfb7 --- /dev/null +++ b/packages/@n8n/benchmark/scripts/runLocally.mjs @@ -0,0 +1,61 @@ +#!/usr/bin/env zx +/** + * Script to run benchmarks on the cloud benchmark environment. + * This script will: + * 1. Provision a benchmark environment using Terraform. + * 2. Run the benchmarks on the VM. + * 3. Destroy the cloud environment. + * + * NOTE: Must be run in the root of the package. + */ +// @ts-check +import { $ } from 'zx'; +import path from 'path'; + +/** + * @typedef {Object} BenchmarkEnv + * @property {string} vmName + */ + +const paths = { + scriptsDir: path.join(path.resolve('scripts')), +}; + +/** + * @typedef {Object} Config + * @property {boolean} isVerbose + * @property {string[]} n8nSetupsToUse + * @property {string} n8nTag + * @property {string} benchmarkTag + * @property {string} [runDir] + * @property {string} [k6ApiToken] + * + * @param {Config} config + */ +export async function runLocally(config) { + const runScriptPath = path.join(paths.scriptsDir, 'runForN8nSetup.mjs'); + + const flags = Object.entries({ + n8nDockerTag: config.n8nTag, + benchmarkDockerTag: config.benchmarkTag, + runDir: config.runDir, + }) + .filter(([, value]) => value !== undefined) + .map(([key, value]) => `--${key}=${value}`); + + try { + for (const n8nSetup of config.n8nSetupsToUse) { + console.log(`Running benchmarks for n8n setup: ${n8nSetup}`); + + await $({ + env: { + ...process.env, + K6_API_TOKEN: config.k6ApiToken, + }, + })`npx ${runScriptPath} ${flags} ${n8nSetup}`; + } + } catch (error) { + console.error('An error occurred while running the benchmarks:'); + console.error(error); + } +} diff --git a/packages/@n8n/benchmark/scripts/runOnVm/runOnVm.mjs b/packages/@n8n/benchmark/scripts/runOnVm/runOnVm.mjs deleted file mode 100755 index fd853203ac..0000000000 --- a/packages/@n8n/benchmark/scripts/runOnVm/runOnVm.mjs +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env zx -/** - * This script runs the benchmarks using a given docker compose setup - */ -// @ts-check -import path from 'path'; -import { $, argv, fs } from 'zx'; - -const paths = { - n8nSetupsDir: path.join(__dirname, 'n8nSetups'), -}; - -async function main() { - const [n8nSetupToUse] = argv._; - validateN8nSetup(n8nSetupToUse); - - const composeFilePath = path.join(paths.n8nSetupsDir, n8nSetupToUse); - const n8nTag = argv.n8nDockerTag || process.env.N8N_DOCKER_TAG || 'latest'; - const benchmarkTag = argv.benchmarkDockerTag || process.env.BENCHMARK_DOCKER_TAG || 'latest'; - const k6ApiToken = argv.k6ApiToken || process.env.K6_API_TOKEN || undefined; - - const $$ = $({ - cwd: composeFilePath, - verbose: true, - env: { - N8N_VERSION: n8nTag, - BENCHMARK_VERSION: benchmarkTag, - K6_API_TOKEN: k6ApiToken, - }, - }); - - try { - await $$`docker-compose up -d n8n`; - - await $$`docker-compose run benchmark run --scenarioNamePrefix=${n8nSetupToUse} `; - } catch (error) { - console.error('An error occurred while running the benchmarks:'); - console.error(error); - console.error(''); - await dumpN8nInstanceLogs($$); - } finally { - await $$`docker-compose down`; - } -} - -async function dumpN8nInstanceLogs($$) { - console.error('n8n instance logs:'); - await $$`docker-compose logs n8n`; -} - -function printUsage() { - const availableSetups = getAllN8nSetups(); - console.log('Usage: zx runOnVm.mjs '); - console.log(` eg: zx runOnVm.mjs ${availableSetups[0]}`); - console.log(''); - console.log('Available setups:'); - console.log(availableSetups.join(', ')); -} - -/** - * @returns {string[]} - */ -function getAllN8nSetups() { - return fs.readdirSync(paths.n8nSetupsDir); -} - -function validateN8nSetup(givenSetup) { - const availableSetups = getAllN8nSetups(); - if (!availableSetups.includes(givenSetup)) { - printUsage(); - process.exit(1); - } -} - -main();