feat: Add local orchestration of benchmarks (no-changelog) (#10589)

This commit is contained in:
Tomi Turtiainen 2024-08-30 08:46:55 +03:00 committed by GitHub
parent 47eb28d767
commit 1c5164c786
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 415 additions and 214 deletions

View file

@ -1,8 +1,38 @@
# n8n benchmarking tool
Tool for executing benchmarks against an n8n instance.
Tool for executing benchmarks against an n8n instance. The tool consists of these components:
## Running locally with Docker
## Directory structure
```text
packages/@n8n/benchmark
├── scenarios Benchmark scenarios
├── src Source code for the n8n-benchmark cli
├── Dockerfile Dockerfile for the n8n-benchmark cli
├── scripts Orchestration scripts
```
## Running the entire benchmark suite
The benchmark suite consists of [benchmark scenarios](#benchmark-scenarios) and different [n8n setups](#n8n-setups).
### locally
```sh
pnpm run-locally
```
### In the cloud
```sh
pnpm run-in-cloud
```
## Running the `n8n-benchmark` cli
The `n8n-benchmark` cli is a node.js program that runs one or more scenarios against a single n8n instance.
### Locally with Docker
Build the Docker image:
@ -23,7 +53,7 @@ docker run \
n8n-benchmark
```
## Running locally without Docker
### Locally without Docker
Requirements:
@ -35,23 +65,8 @@ pnpm build
# Run tests against http://localhost:5678 with specified email and password
N8N_USER_EMAIL=user@n8n.io N8N_USER_PASSWORD=password ./bin/n8n-benchmark run
# If you installed k6 using brew, you might have to specify it explicitly
K6_PATH=/opt/homebrew/bin/k6 N8N_USER_EMAIL=user@n8n.io N8N_USER_PASSWORD=password ./bin/n8n-benchmark run
```
## Running in the cloud
There's a script to run the performance tests in a cloud environment. The script provisions a cloud environment, sets up n8n in the environment, runs the tests and destroys the environment.
```sh
pnpm run-in-cloud
```
## Configuration
The configuration options the cli accepts can be seen from [config.ts](./src/config/config.ts)
## Benchmark scenarios
A benchmark scenario defines one or multiple steps to execute and measure. It consists of:
@ -61,3 +76,7 @@ A benchmark scenario defines one or multiple steps to execute and measure. It co
- A [`k6`](https://grafana.com/docs/k6/latest/using-k6/http-requests/) script which executes the steps and receives `API_BASE_URL` environment variable in runtime.
Available scenarios are located in [`./scenarios`](./scenarios/).
## n8n setups
A n8n setup defines a single n8n runtime configuration using Docker compose. Different n8n setups are located in [`./scripts/n8nSetups`](./scripts/n8nSetups).

View file

@ -10,7 +10,9 @@
"start": "./bin/n8n-benchmark",
"test": "echo \"Error: no test specified\" && exit 1",
"typecheck": "tsc --noEmit",
"run-in-cloud": "zx scripts/runInCloud.mjs",
"benchmark": "zx scripts/run.mjs",
"benchmark-in-cloud": "pnpm benchmark --env cloud",
"benchmark-locally": "pnpm benchmark --env local",
"destroy-cloud-env": "zx scripts/destroyCloudEnv.mjs",
"watch": "concurrently \"tsc -w -p tsconfig.build.json\" \"tsc-alias -w -p tsconfig.build.json\""
},

View file

@ -0,0 +1,45 @@
import { which } from 'zx';
export class DockerComposeClient {
/**
*
* @param {{ $: Shell; verbose?: boolean }} opts
*/
constructor({ $ }) {
this.$$ = $;
}
async $(...args) {
await this.resolveExecutableIfNeeded();
if (this.isCompose) {
return await this.$$`docker-compose ${args}`;
} else {
return await this.$$`docker compose ${args}`;
}
}
async resolveExecutableIfNeeded() {
if (this.isResolved) {
return;
}
// The VM deployment doesn't have `docker compose` available,
// so try to resolve the `docker-compose` first
const compose = await which('docker-compose', { nothrow: true });
if (compose) {
this.isResolved = true;
this.isCompose = true;
return;
}
const docker = await which('docker', { nothrow: true });
if (docker) {
this.isResolved = true;
this.isCompose = false;
return;
}
throw new Error('Could not resolve docker-compose or docker');
}
}

View file

@ -9,8 +9,7 @@ const paths = {
};
export class TerraformClient {
constructor({ privateKeyPath, isVerbose = false }) {
this.privateKeyPath = privateKeyPath;
constructor({ isVerbose = false }) {
this.isVerbose = isVerbose;
this.$$ = $({
cwd: paths.infraCodeDir,

View file

@ -7,7 +7,7 @@ services:
ports:
- 5678:5678
volumes:
- /n8n:/n8n
- ${RUN_DIR}:/n8n
benchmark:
image: ghcr.io/n8n-io/n8n-benchmark:${N8N_BENCHMARK_VERSION:-latest}
depends_on:

View file

@ -9,7 +9,7 @@ services:
ports:
- 5678:5678
volumes:
- /n8n:/n8n
- ${RUN_DIR}:/n8n
benchmark:
image: ghcr.io/n8n-io/n8n-benchmark:${N8N_BENCHMARK_VERSION:-latest}
depends_on:

View file

@ -0,0 +1,151 @@
#!/usr/bin/env zx
/**
* Script to run benchmarks either on the cloud benchmark environment or locally.
*
* NOTE: Must be run in the root of the package.
*
* Usage:
* zx scripts/run.mjs
*
*/
// @ts-check
import fs from 'fs';
import minimist from 'minimist';
import path from 'path';
import { runInCloud } from './runInCloud.mjs';
import { runLocally } from './runLocally.mjs';
const paths = {
n8nSetupsDir: path.join(path.resolve('scripts'), 'n8nSetups'),
};
async function main() {
const config = await parseAndValidateConfig();
const n8nSetupsToUse =
config.n8nSetupToUse === 'all' ? readAvailableN8nSetups() : [config.n8nSetupToUse];
console.log('Using n8n tag', config.n8nTag);
console.log('Using benchmark cli tag', config.benchmarkTag);
console.log('Using environment', config.env);
console.log('Using n8n setups', n8nSetupsToUse.join(', '));
console.log('');
if (config.env === 'cloud') {
await runInCloud({
benchmarkTag: config.benchmarkTag,
isVerbose: config.isVerbose,
k6ApiToken: config.k6ApiToken,
n8nTag: config.n8nTag,
n8nSetupsToUse,
});
} else {
await runLocally({
benchmarkTag: config.benchmarkTag,
isVerbose: config.isVerbose,
k6ApiToken: config.k6ApiToken,
n8nTag: config.n8nTag,
runDir: config.runDir,
n8nSetupsToUse,
});
}
}
function readAvailableN8nSetups() {
const setups = fs.readdirSync(paths.n8nSetupsDir);
return setups;
}
/**
* @typedef {Object} Config
* @property {boolean} isVerbose
* @property {'cloud' | 'local'} env
* @property {string} n8nSetupToUse
* @property {string} n8nTag
* @property {string} benchmarkTag
* @property {string} [k6ApiToken]
* @property {string} [runDir]
*
* @returns {Promise<Config>}
*/
async function parseAndValidateConfig() {
const args = minimist(process.argv.slice(3), {
boolean: ['debug', 'help'],
});
if (args.help) {
printUsage();
process.exit(0);
}
const n8nSetupToUse = await getAndValidateN8nSetup(args);
const isVerbose = args.debug || false;
const n8nTag = args.n8nTag || process.env.N8N_DOCKER_TAG || 'latest';
const benchmarkTag = args.benchmarkTag || process.env.BENCHMARK_DOCKER_TAG || 'latest';
const k6ApiToken = args.k6ApiToken || process.env.K6_API_TOKEN || undefined;
const runDir = args.runDir || undefined;
const env = args.env || 'local';
if (!env) {
printUsage();
process.exit(1);
}
return {
isVerbose,
env,
n8nSetupToUse,
n8nTag,
benchmarkTag,
k6ApiToken,
runDir,
};
}
/**
* @param {minimist.ParsedArgs} args
*/
async function getAndValidateN8nSetup(args) {
// Last parameter is the n8n setup to use
const n8nSetupToUse = args._[args._.length - 1];
if (!n8nSetupToUse || n8nSetupToUse === 'all') {
return 'all';
}
const availableSetups = readAvailableN8nSetups();
if (!availableSetups.includes(n8nSetupToUse)) {
printUsage();
process.exit(1);
}
return n8nSetupToUse;
}
function printUsage() {
const availableSetups = readAvailableN8nSetups();
console.log(`Usage: zx scripts/${path.basename(__filename)} [n8n setup name]`);
console.log(` eg: zx scripts/${path.basename(__filename)}`);
console.log('');
console.log('Options:');
console.log(
` [n8n setup name] Against which n8n setup to run the benchmarks. One of: ${['all', ...availableSetups].join(', ')}. Default is all`,
);
console.log(
' --env Env where to run the benchmarks. Either cloud or local. Default is local.',
);
console.log(' --debug Enable verbose output');
console.log(' --n8nTag Docker tag for n8n image. Default is latest');
console.log(' --benchmarkTag Docker tag for benchmark cli image. Default is latest');
console.log(
' --k6ApiToken API token for k6 cloud. Default is read from K6_API_TOKEN env var. If omitted, k6 cloud will not be used',
);
console.log(
' --runDir Directory to share with the n8n container for storing data. Needed only for local runs.',
);
console.log('');
}
main().catch(console.error);

View file

@ -0,0 +1,97 @@
#!/usr/bin/env zx
/**
* This script runs the benchmarks for the given n8n setup.
*/
// @ts-check
import path from 'path';
import { $, argv, fs } from 'zx';
import { DockerComposeClient } from './clients/dockerComposeClient.mjs';
const paths = {
n8nSetupsDir: path.join(__dirname, 'n8nSetups'),
};
async function main() {
const [n8nSetupToUse] = argv._;
validateN8nSetup(n8nSetupToUse);
const composeFilePath = path.join(paths.n8nSetupsDir, n8nSetupToUse);
const n8nTag = argv.n8nDockerTag || process.env.N8N_DOCKER_TAG || 'latest';
const benchmarkTag = argv.benchmarkDockerTag || process.env.BENCHMARK_DOCKER_TAG || 'latest';
const k6ApiToken = argv.k6ApiToken || process.env.K6_API_TOKEN || undefined;
const runDir = argv.runDir || process.env.RUN_DIR || '/n8n';
if (!fs.existsSync(runDir)) {
console.error(
`The run directory "${runDir}" does not exist. Please specify a valid directory using --runDir`,
);
process.exit(1);
}
const dockerComposeClient = new DockerComposeClient({
$: $({
cwd: composeFilePath,
verbose: true,
env: {
N8N_VERSION: n8nTag,
BENCHMARK_VERSION: benchmarkTag,
K6_API_TOKEN: k6ApiToken,
RUN_DIR: runDir,
},
}),
});
try {
await dockerComposeClient.$('up', '-d', 'n8n');
await dockerComposeClient.$('run', 'benchmark', 'run', `--scenarioNamePrefix=${n8nSetupToUse}`);
} catch (error) {
console.error('An error occurred while running the benchmarks:');
console.error(error);
console.error('');
await dumpN8nInstanceLogs(dockerComposeClient);
} finally {
await dockerComposeClient.$('down');
}
}
async function dumpN8nInstanceLogs(dockerComposeClient) {
console.error('n8n instance logs:');
await dockerComposeClient.$('logs', 'n8n');
}
function printUsage() {
const availableSetups = getAllN8nSetups();
console.log('Usage: zx runForN8nSetup.mjs --runDir /path/for/n8n/data <n8n setup to use>');
console.log(` eg: zx runForN8nSetup.mjs --runDir /path/for/n8n/data ${availableSetups[0]}`);
console.log('');
console.log('Flags:');
console.log(
' --runDir <path> Directory to share with the n8n container for storing data. Default is /n8n',
);
console.log(' --n8nDockerTag <tag> Docker tag for n8n image. Default is latest');
console.log(
' --benchmarkDockerTag <tag> Docker tag for benchmark cli image. Default is latest',
);
console.log(' --k6ApiToken <token> K6 API token to upload the results');
console.log('');
console.log('Available setups:');
console.log(availableSetups.join(', '));
}
/**
* @returns {string[]}
*/
function getAllN8nSetups() {
return fs.readdirSync(paths.n8nSetupsDir);
}
function validateN8nSetup(givenSetup) {
const availableSetups = getAllN8nSetups();
if (!availableSetups.includes(givenSetup)) {
printUsage();
process.exit(1);
}
}
main();

View file

@ -7,18 +7,12 @@
* 3. Destroy the cloud environment.
*
* NOTE: Must be run in the root of the package.
*
* Usage:
* zx scripts/runBenchmarksOnCloud.mjs [--debug] <n8n setup to use>
*
*/
// @ts-check
import fs from 'fs';
import minimist from 'minimist';
import { sleep, which } from 'zx';
import path from 'path';
import { SshClient } from './sshClient.mjs';
import { TerraformClient } from './terraformClient.mjs';
import { SshClient } from './clients/sshClient.mjs';
import { TerraformClient } from './clients/terraformClient.mjs';
/**
* @typedef {Object} BenchmarkEnv
@ -27,19 +21,20 @@ import { TerraformClient } from './terraformClient.mjs';
const RESOURCE_GROUP_NAME = 'n8n-benchmarking';
const paths = {
n8nSetupsDir: path.join(path.resolve('scripts'), 'runOnVm', 'n8nSetups'),
};
async function main() {
const config = await parseAndValidateConfig();
/**
* @typedef {Object} Config
* @property {boolean} isVerbose
* @property {string[]} n8nSetupsToUse
* @property {string} n8nTag
* @property {string} benchmarkTag
* @property {string} [k6ApiToken]
*
* @param {Config} config
*/
export async function runInCloud(config) {
await ensureDependencies();
console.log('Using n8n tag', config.n8nTag);
console.log('Using benchmark cli tag', config.benchmarkTag);
const terraformClient = new TerraformClient({
privateKeyPath: paths.privateKeyPath,
isVerbose: config.isVerbose,
});
@ -65,7 +60,7 @@ async function ensureDependencies() {
* @param {BenchmarkEnv} benchmarkEnv
*/
async function runBenchmarksOnVm(config, benchmarkEnv) {
console.log(`Setting up the environment for ${config.n8nSetupToUse}...`);
console.log(`Setting up the environment...`);
const sshClient = new SshClient({
vmName: benchmarkEnv.vmName,
@ -85,23 +80,12 @@ async function runBenchmarksOnVm(config, benchmarkEnv) {
// Give some time for the VM to be ready
await sleep(1000);
if (config.n8nSetupToUse === 'all') {
const availableSetups = readAvailableN8nSetups();
for (const n8nSetup of availableSetups) {
await runBenchmarkForN8nSetup({
config,
sshClient,
scriptsDir,
n8nSetup,
});
}
} else {
for (const n8nSetup of config.n8nSetupsToUse) {
await runBenchmarkForN8nSetup({
config,
sshClient,
scriptsDir,
n8nSetup: config.n8nSetupToUse,
n8nSetup,
});
}
}
@ -111,7 +95,7 @@ async function runBenchmarksOnVm(config, benchmarkEnv) {
*/
async function runBenchmarkForN8nSetup({ config, sshClient, scriptsDir, n8nSetup }) {
console.log(`Running benchmarks for ${n8nSetup}...`);
const runScriptPath = path.join(scriptsDir, 'runOnVm.mjs');
const runScriptPath = path.join(scriptsDir, 'runForN8nSetup.mjs');
const flags = {
n8nDockerTag: config.n8nTag,
@ -142,87 +126,5 @@ async function transferScriptsToVm(sshClient) {
await sshClient.ssh('git clone --depth=1 https://github.com/n8n-io/n8n.git');
return '~/n8n/packages/@n8n/benchmark/scripts/runOnVm';
return '~/n8n/packages/@n8n/benchmark/scripts';
}
function readAvailableN8nSetups() {
const setups = fs.readdirSync(paths.n8nSetupsDir);
return setups;
}
/**
* @typedef {Object} Config
* @property {boolean} isVerbose
* @property {string} n8nSetupToUse
* @property {string} n8nTag
* @property {string} benchmarkTag
* @property {string} [k6ApiToken]
*
* @returns {Promise<Config>}
*/
async function parseAndValidateConfig() {
const args = minimist(process.argv.slice(3), {
boolean: ['debug', 'help'],
});
if (args.help) {
printUsage();
process.exit(0);
}
const n8nSetupToUse = await getAndValidateN8nSetup(args);
const isVerbose = args.debug || false;
const n8nTag = args.n8nTag || process.env.N8N_DOCKER_TAG || 'latest';
const benchmarkTag = args.benchmarkTag || process.env.BENCHMARK_DOCKER_TAG || 'latest';
const k6ApiToken = args.k6ApiToken || process.env.K6_API_TOKEN || undefined;
return {
isVerbose,
n8nSetupToUse,
n8nTag,
benchmarkTag,
k6ApiToken,
};
}
/**
* @param {minimist.ParsedArgs} args
*/
async function getAndValidateN8nSetup(args) {
// Last parameter is the n8n setup to use
const n8nSetupToUse = args._[args._.length - 1];
if (!n8nSetupToUse || n8nSetupToUse === 'all') {
return 'all';
}
const availableSetups = readAvailableN8nSetups();
if (!availableSetups.includes(n8nSetupToUse)) {
printUsage();
process.exit(1);
}
return n8nSetupToUse;
}
function printUsage() {
const availableSetups = readAvailableN8nSetups();
console.log('Usage: zx scripts/runInCloud.mjs [n8n setup name]');
console.log(' eg: zx scripts/runInCloud.mjs');
console.log('');
console.log('Options:');
console.log(
` [n8n setup name] Against which n8n setup to run the benchmarks. One of: ${['all', ...availableSetups].join(', ')}. Default is all`,
);
console.log(' --debug Enable verbose output');
console.log(' --n8nTag Docker tag for n8n image. Default is latest');
console.log(' --benchmarkTag Docker tag for benchmark cli image. Default is latest');
console.log(
' --k6ApiToken API token for k6 cloud. Default is read from K6_API_TOKEN env var. If omitted, k6 cloud will not be used',
);
console.log('');
}
main().catch(console.error);

View file

@ -0,0 +1,61 @@
#!/usr/bin/env zx
/**
* Script to run benchmarks on the cloud benchmark environment.
* This script will:
* 1. Provision a benchmark environment using Terraform.
* 2. Run the benchmarks on the VM.
* 3. Destroy the cloud environment.
*
* NOTE: Must be run in the root of the package.
*/
// @ts-check
import { $ } from 'zx';
import path from 'path';
/**
* @typedef {Object} BenchmarkEnv
* @property {string} vmName
*/
const paths = {
scriptsDir: path.join(path.resolve('scripts')),
};
/**
* @typedef {Object} Config
* @property {boolean} isVerbose
* @property {string[]} n8nSetupsToUse
* @property {string} n8nTag
* @property {string} benchmarkTag
* @property {string} [runDir]
* @property {string} [k6ApiToken]
*
* @param {Config} config
*/
export async function runLocally(config) {
const runScriptPath = path.join(paths.scriptsDir, 'runForN8nSetup.mjs');
const flags = Object.entries({
n8nDockerTag: config.n8nTag,
benchmarkDockerTag: config.benchmarkTag,
runDir: config.runDir,
})
.filter(([, value]) => value !== undefined)
.map(([key, value]) => `--${key}=${value}`);
try {
for (const n8nSetup of config.n8nSetupsToUse) {
console.log(`Running benchmarks for n8n setup: ${n8nSetup}`);
await $({
env: {
...process.env,
K6_API_TOKEN: config.k6ApiToken,
},
})`npx ${runScriptPath} ${flags} ${n8nSetup}`;
}
} catch (error) {
console.error('An error occurred while running the benchmarks:');
console.error(error);
}
}

View file

@ -1,75 +0,0 @@
#!/usr/bin/env zx
/**
* This script runs the benchmarks using a given docker compose setup
*/
// @ts-check
import path from 'path';
import { $, argv, fs } from 'zx';
const paths = {
n8nSetupsDir: path.join(__dirname, 'n8nSetups'),
};
async function main() {
const [n8nSetupToUse] = argv._;
validateN8nSetup(n8nSetupToUse);
const composeFilePath = path.join(paths.n8nSetupsDir, n8nSetupToUse);
const n8nTag = argv.n8nDockerTag || process.env.N8N_DOCKER_TAG || 'latest';
const benchmarkTag = argv.benchmarkDockerTag || process.env.BENCHMARK_DOCKER_TAG || 'latest';
const k6ApiToken = argv.k6ApiToken || process.env.K6_API_TOKEN || undefined;
const $$ = $({
cwd: composeFilePath,
verbose: true,
env: {
N8N_VERSION: n8nTag,
BENCHMARK_VERSION: benchmarkTag,
K6_API_TOKEN: k6ApiToken,
},
});
try {
await $$`docker-compose up -d n8n`;
await $$`docker-compose run benchmark run --scenarioNamePrefix=${n8nSetupToUse} `;
} catch (error) {
console.error('An error occurred while running the benchmarks:');
console.error(error);
console.error('');
await dumpN8nInstanceLogs($$);
} finally {
await $$`docker-compose down`;
}
}
async function dumpN8nInstanceLogs($$) {
console.error('n8n instance logs:');
await $$`docker-compose logs n8n`;
}
function printUsage() {
const availableSetups = getAllN8nSetups();
console.log('Usage: zx runOnVm.mjs <n8n setup to use>');
console.log(` eg: zx runOnVm.mjs ${availableSetups[0]}`);
console.log('');
console.log('Available setups:');
console.log(availableSetups.join(', '));
}
/**
* @returns {string[]}
*/
function getAllN8nSetups() {
return fs.readdirSync(paths.n8nSetupsDir);
}
function validateN8nSetup(givenSetup) {
const availableSetups = getAllN8nSetups();
if (!availableSetups.includes(givenSetup)) {
printUsage();
process.exit(1);
}
}
main();