mirror of
https://github.com/n8n-io/n8n.git
synced 2025-01-11 12:57:29 -08:00
feat: Add local orchestration of benchmarks (no-changelog) (#10589)
This commit is contained in:
parent
47eb28d767
commit
1c5164c786
|
@ -1,8 +1,38 @@
|
||||||
# n8n benchmarking tool
|
# n8n benchmarking tool
|
||||||
|
|
||||||
Tool for executing benchmarks against an n8n instance.
|
Tool for executing benchmarks against an n8n instance. The tool consists of these components:
|
||||||
|
|
||||||
## Running locally with Docker
|
## Directory structure
|
||||||
|
|
||||||
|
```text
|
||||||
|
packages/@n8n/benchmark
|
||||||
|
├── scenarios Benchmark scenarios
|
||||||
|
├── src Source code for the n8n-benchmark cli
|
||||||
|
├── Dockerfile Dockerfile for the n8n-benchmark cli
|
||||||
|
├── scripts Orchestration scripts
|
||||||
|
```
|
||||||
|
|
||||||
|
## Running the entire benchmark suite
|
||||||
|
|
||||||
|
The benchmark suite consists of [benchmark scenarios](#benchmark-scenarios) and different [n8n setups](#n8n-setups).
|
||||||
|
|
||||||
|
### locally
|
||||||
|
|
||||||
|
```sh
|
||||||
|
pnpm run-locally
|
||||||
|
```
|
||||||
|
|
||||||
|
### In the cloud
|
||||||
|
|
||||||
|
```sh
|
||||||
|
pnpm run-in-cloud
|
||||||
|
```
|
||||||
|
|
||||||
|
## Running the `n8n-benchmark` cli
|
||||||
|
|
||||||
|
The `n8n-benchmark` cli is a node.js program that runs one or more scenarios against a single n8n instance.
|
||||||
|
|
||||||
|
### Locally with Docker
|
||||||
|
|
||||||
Build the Docker image:
|
Build the Docker image:
|
||||||
|
|
||||||
|
@ -23,7 +53,7 @@ docker run \
|
||||||
n8n-benchmark
|
n8n-benchmark
|
||||||
```
|
```
|
||||||
|
|
||||||
## Running locally without Docker
|
### Locally without Docker
|
||||||
|
|
||||||
Requirements:
|
Requirements:
|
||||||
|
|
||||||
|
@ -35,23 +65,8 @@ pnpm build
|
||||||
|
|
||||||
# Run tests against http://localhost:5678 with specified email and password
|
# Run tests against http://localhost:5678 with specified email and password
|
||||||
N8N_USER_EMAIL=user@n8n.io N8N_USER_PASSWORD=password ./bin/n8n-benchmark run
|
N8N_USER_EMAIL=user@n8n.io N8N_USER_PASSWORD=password ./bin/n8n-benchmark run
|
||||||
|
|
||||||
# If you installed k6 using brew, you might have to specify it explicitly
|
|
||||||
K6_PATH=/opt/homebrew/bin/k6 N8N_USER_EMAIL=user@n8n.io N8N_USER_PASSWORD=password ./bin/n8n-benchmark run
|
|
||||||
```
|
```
|
||||||
|
|
||||||
## Running in the cloud
|
|
||||||
|
|
||||||
There's a script to run the performance tests in a cloud environment. The script provisions a cloud environment, sets up n8n in the environment, runs the tests and destroys the environment.
|
|
||||||
|
|
||||||
```sh
|
|
||||||
pnpm run-in-cloud
|
|
||||||
```
|
|
||||||
|
|
||||||
## Configuration
|
|
||||||
|
|
||||||
The configuration options the cli accepts can be seen from [config.ts](./src/config/config.ts)
|
|
||||||
|
|
||||||
## Benchmark scenarios
|
## Benchmark scenarios
|
||||||
|
|
||||||
A benchmark scenario defines one or multiple steps to execute and measure. It consists of:
|
A benchmark scenario defines one or multiple steps to execute and measure. It consists of:
|
||||||
|
@ -61,3 +76,7 @@ A benchmark scenario defines one or multiple steps to execute and measure. It co
|
||||||
- A [`k6`](https://grafana.com/docs/k6/latest/using-k6/http-requests/) script which executes the steps and receives `API_BASE_URL` environment variable in runtime.
|
- A [`k6`](https://grafana.com/docs/k6/latest/using-k6/http-requests/) script which executes the steps and receives `API_BASE_URL` environment variable in runtime.
|
||||||
|
|
||||||
Available scenarios are located in [`./scenarios`](./scenarios/).
|
Available scenarios are located in [`./scenarios`](./scenarios/).
|
||||||
|
|
||||||
|
## n8n setups
|
||||||
|
|
||||||
|
A n8n setup defines a single n8n runtime configuration using Docker compose. Different n8n setups are located in [`./scripts/n8nSetups`](./scripts/n8nSetups).
|
||||||
|
|
|
@ -10,7 +10,9 @@
|
||||||
"start": "./bin/n8n-benchmark",
|
"start": "./bin/n8n-benchmark",
|
||||||
"test": "echo \"Error: no test specified\" && exit 1",
|
"test": "echo \"Error: no test specified\" && exit 1",
|
||||||
"typecheck": "tsc --noEmit",
|
"typecheck": "tsc --noEmit",
|
||||||
"run-in-cloud": "zx scripts/runInCloud.mjs",
|
"benchmark": "zx scripts/run.mjs",
|
||||||
|
"benchmark-in-cloud": "pnpm benchmark --env cloud",
|
||||||
|
"benchmark-locally": "pnpm benchmark --env local",
|
||||||
"destroy-cloud-env": "zx scripts/destroyCloudEnv.mjs",
|
"destroy-cloud-env": "zx scripts/destroyCloudEnv.mjs",
|
||||||
"watch": "concurrently \"tsc -w -p tsconfig.build.json\" \"tsc-alias -w -p tsconfig.build.json\""
|
"watch": "concurrently \"tsc -w -p tsconfig.build.json\" \"tsc-alias -w -p tsconfig.build.json\""
|
||||||
},
|
},
|
||||||
|
|
|
@ -0,0 +1,45 @@
|
||||||
|
import { which } from 'zx';
|
||||||
|
|
||||||
|
export class DockerComposeClient {
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
* @param {{ $: Shell; verbose?: boolean }} opts
|
||||||
|
*/
|
||||||
|
constructor({ $ }) {
|
||||||
|
this.$$ = $;
|
||||||
|
}
|
||||||
|
|
||||||
|
async $(...args) {
|
||||||
|
await this.resolveExecutableIfNeeded();
|
||||||
|
|
||||||
|
if (this.isCompose) {
|
||||||
|
return await this.$$`docker-compose ${args}`;
|
||||||
|
} else {
|
||||||
|
return await this.$$`docker compose ${args}`;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async resolveExecutableIfNeeded() {
|
||||||
|
if (this.isResolved) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// The VM deployment doesn't have `docker compose` available,
|
||||||
|
// so try to resolve the `docker-compose` first
|
||||||
|
const compose = await which('docker-compose', { nothrow: true });
|
||||||
|
if (compose) {
|
||||||
|
this.isResolved = true;
|
||||||
|
this.isCompose = true;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const docker = await which('docker', { nothrow: true });
|
||||||
|
if (docker) {
|
||||||
|
this.isResolved = true;
|
||||||
|
this.isCompose = false;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new Error('Could not resolve docker-compose or docker');
|
||||||
|
}
|
||||||
|
}
|
|
@ -9,8 +9,7 @@ const paths = {
|
||||||
};
|
};
|
||||||
|
|
||||||
export class TerraformClient {
|
export class TerraformClient {
|
||||||
constructor({ privateKeyPath, isVerbose = false }) {
|
constructor({ isVerbose = false }) {
|
||||||
this.privateKeyPath = privateKeyPath;
|
|
||||||
this.isVerbose = isVerbose;
|
this.isVerbose = isVerbose;
|
||||||
this.$$ = $({
|
this.$$ = $({
|
||||||
cwd: paths.infraCodeDir,
|
cwd: paths.infraCodeDir,
|
|
@ -7,7 +7,7 @@ services:
|
||||||
ports:
|
ports:
|
||||||
- 5678:5678
|
- 5678:5678
|
||||||
volumes:
|
volumes:
|
||||||
- /n8n:/n8n
|
- ${RUN_DIR}:/n8n
|
||||||
benchmark:
|
benchmark:
|
||||||
image: ghcr.io/n8n-io/n8n-benchmark:${N8N_BENCHMARK_VERSION:-latest}
|
image: ghcr.io/n8n-io/n8n-benchmark:${N8N_BENCHMARK_VERSION:-latest}
|
||||||
depends_on:
|
depends_on:
|
|
@ -9,7 +9,7 @@ services:
|
||||||
ports:
|
ports:
|
||||||
- 5678:5678
|
- 5678:5678
|
||||||
volumes:
|
volumes:
|
||||||
- /n8n:/n8n
|
- ${RUN_DIR}:/n8n
|
||||||
benchmark:
|
benchmark:
|
||||||
image: ghcr.io/n8n-io/n8n-benchmark:${N8N_BENCHMARK_VERSION:-latest}
|
image: ghcr.io/n8n-io/n8n-benchmark:${N8N_BENCHMARK_VERSION:-latest}
|
||||||
depends_on:
|
depends_on:
|
151
packages/@n8n/benchmark/scripts/run.mjs
Executable file
151
packages/@n8n/benchmark/scripts/run.mjs
Executable file
|
@ -0,0 +1,151 @@
|
||||||
|
#!/usr/bin/env zx
|
||||||
|
/**
|
||||||
|
* Script to run benchmarks either on the cloud benchmark environment or locally.
|
||||||
|
*
|
||||||
|
* NOTE: Must be run in the root of the package.
|
||||||
|
*
|
||||||
|
* Usage:
|
||||||
|
* zx scripts/run.mjs
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
// @ts-check
|
||||||
|
import fs from 'fs';
|
||||||
|
import minimist from 'minimist';
|
||||||
|
import path from 'path';
|
||||||
|
import { runInCloud } from './runInCloud.mjs';
|
||||||
|
import { runLocally } from './runLocally.mjs';
|
||||||
|
|
||||||
|
const paths = {
|
||||||
|
n8nSetupsDir: path.join(path.resolve('scripts'), 'n8nSetups'),
|
||||||
|
};
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
const config = await parseAndValidateConfig();
|
||||||
|
|
||||||
|
const n8nSetupsToUse =
|
||||||
|
config.n8nSetupToUse === 'all' ? readAvailableN8nSetups() : [config.n8nSetupToUse];
|
||||||
|
|
||||||
|
console.log('Using n8n tag', config.n8nTag);
|
||||||
|
console.log('Using benchmark cli tag', config.benchmarkTag);
|
||||||
|
console.log('Using environment', config.env);
|
||||||
|
console.log('Using n8n setups', n8nSetupsToUse.join(', '));
|
||||||
|
console.log('');
|
||||||
|
|
||||||
|
if (config.env === 'cloud') {
|
||||||
|
await runInCloud({
|
||||||
|
benchmarkTag: config.benchmarkTag,
|
||||||
|
isVerbose: config.isVerbose,
|
||||||
|
k6ApiToken: config.k6ApiToken,
|
||||||
|
n8nTag: config.n8nTag,
|
||||||
|
n8nSetupsToUse,
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
await runLocally({
|
||||||
|
benchmarkTag: config.benchmarkTag,
|
||||||
|
isVerbose: config.isVerbose,
|
||||||
|
k6ApiToken: config.k6ApiToken,
|
||||||
|
n8nTag: config.n8nTag,
|
||||||
|
runDir: config.runDir,
|
||||||
|
n8nSetupsToUse,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function readAvailableN8nSetups() {
|
||||||
|
const setups = fs.readdirSync(paths.n8nSetupsDir);
|
||||||
|
|
||||||
|
return setups;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @typedef {Object} Config
|
||||||
|
* @property {boolean} isVerbose
|
||||||
|
* @property {'cloud' | 'local'} env
|
||||||
|
* @property {string} n8nSetupToUse
|
||||||
|
* @property {string} n8nTag
|
||||||
|
* @property {string} benchmarkTag
|
||||||
|
* @property {string} [k6ApiToken]
|
||||||
|
* @property {string} [runDir]
|
||||||
|
*
|
||||||
|
* @returns {Promise<Config>}
|
||||||
|
*/
|
||||||
|
async function parseAndValidateConfig() {
|
||||||
|
const args = minimist(process.argv.slice(3), {
|
||||||
|
boolean: ['debug', 'help'],
|
||||||
|
});
|
||||||
|
|
||||||
|
if (args.help) {
|
||||||
|
printUsage();
|
||||||
|
process.exit(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
const n8nSetupToUse = await getAndValidateN8nSetup(args);
|
||||||
|
const isVerbose = args.debug || false;
|
||||||
|
const n8nTag = args.n8nTag || process.env.N8N_DOCKER_TAG || 'latest';
|
||||||
|
const benchmarkTag = args.benchmarkTag || process.env.BENCHMARK_DOCKER_TAG || 'latest';
|
||||||
|
const k6ApiToken = args.k6ApiToken || process.env.K6_API_TOKEN || undefined;
|
||||||
|
const runDir = args.runDir || undefined;
|
||||||
|
const env = args.env || 'local';
|
||||||
|
|
||||||
|
if (!env) {
|
||||||
|
printUsage();
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
isVerbose,
|
||||||
|
env,
|
||||||
|
n8nSetupToUse,
|
||||||
|
n8nTag,
|
||||||
|
benchmarkTag,
|
||||||
|
k6ApiToken,
|
||||||
|
runDir,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param {minimist.ParsedArgs} args
|
||||||
|
*/
|
||||||
|
async function getAndValidateN8nSetup(args) {
|
||||||
|
// Last parameter is the n8n setup to use
|
||||||
|
const n8nSetupToUse = args._[args._.length - 1];
|
||||||
|
if (!n8nSetupToUse || n8nSetupToUse === 'all') {
|
||||||
|
return 'all';
|
||||||
|
}
|
||||||
|
|
||||||
|
const availableSetups = readAvailableN8nSetups();
|
||||||
|
|
||||||
|
if (!availableSetups.includes(n8nSetupToUse)) {
|
||||||
|
printUsage();
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
return n8nSetupToUse;
|
||||||
|
}
|
||||||
|
|
||||||
|
function printUsage() {
|
||||||
|
const availableSetups = readAvailableN8nSetups();
|
||||||
|
|
||||||
|
console.log(`Usage: zx scripts/${path.basename(__filename)} [n8n setup name]`);
|
||||||
|
console.log(` eg: zx scripts/${path.basename(__filename)}`);
|
||||||
|
console.log('');
|
||||||
|
console.log('Options:');
|
||||||
|
console.log(
|
||||||
|
` [n8n setup name] Against which n8n setup to run the benchmarks. One of: ${['all', ...availableSetups].join(', ')}. Default is all`,
|
||||||
|
);
|
||||||
|
console.log(
|
||||||
|
' --env Env where to run the benchmarks. Either cloud or local. Default is local.',
|
||||||
|
);
|
||||||
|
console.log(' --debug Enable verbose output');
|
||||||
|
console.log(' --n8nTag Docker tag for n8n image. Default is latest');
|
||||||
|
console.log(' --benchmarkTag Docker tag for benchmark cli image. Default is latest');
|
||||||
|
console.log(
|
||||||
|
' --k6ApiToken API token for k6 cloud. Default is read from K6_API_TOKEN env var. If omitted, k6 cloud will not be used',
|
||||||
|
);
|
||||||
|
console.log(
|
||||||
|
' --runDir Directory to share with the n8n container for storing data. Needed only for local runs.',
|
||||||
|
);
|
||||||
|
console.log('');
|
||||||
|
}
|
||||||
|
|
||||||
|
main().catch(console.error);
|
97
packages/@n8n/benchmark/scripts/runForN8nSetup.mjs
Executable file
97
packages/@n8n/benchmark/scripts/runForN8nSetup.mjs
Executable file
|
@ -0,0 +1,97 @@
|
||||||
|
#!/usr/bin/env zx
|
||||||
|
/**
|
||||||
|
* This script runs the benchmarks for the given n8n setup.
|
||||||
|
*/
|
||||||
|
// @ts-check
|
||||||
|
import path from 'path';
|
||||||
|
import { $, argv, fs } from 'zx';
|
||||||
|
import { DockerComposeClient } from './clients/dockerComposeClient.mjs';
|
||||||
|
|
||||||
|
const paths = {
|
||||||
|
n8nSetupsDir: path.join(__dirname, 'n8nSetups'),
|
||||||
|
};
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
const [n8nSetupToUse] = argv._;
|
||||||
|
validateN8nSetup(n8nSetupToUse);
|
||||||
|
|
||||||
|
const composeFilePath = path.join(paths.n8nSetupsDir, n8nSetupToUse);
|
||||||
|
const n8nTag = argv.n8nDockerTag || process.env.N8N_DOCKER_TAG || 'latest';
|
||||||
|
const benchmarkTag = argv.benchmarkDockerTag || process.env.BENCHMARK_DOCKER_TAG || 'latest';
|
||||||
|
const k6ApiToken = argv.k6ApiToken || process.env.K6_API_TOKEN || undefined;
|
||||||
|
const runDir = argv.runDir || process.env.RUN_DIR || '/n8n';
|
||||||
|
|
||||||
|
if (!fs.existsSync(runDir)) {
|
||||||
|
console.error(
|
||||||
|
`The run directory "${runDir}" does not exist. Please specify a valid directory using --runDir`,
|
||||||
|
);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
const dockerComposeClient = new DockerComposeClient({
|
||||||
|
$: $({
|
||||||
|
cwd: composeFilePath,
|
||||||
|
verbose: true,
|
||||||
|
env: {
|
||||||
|
N8N_VERSION: n8nTag,
|
||||||
|
BENCHMARK_VERSION: benchmarkTag,
|
||||||
|
K6_API_TOKEN: k6ApiToken,
|
||||||
|
RUN_DIR: runDir,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
await dockerComposeClient.$('up', '-d', 'n8n');
|
||||||
|
|
||||||
|
await dockerComposeClient.$('run', 'benchmark', 'run', `--scenarioNamePrefix=${n8nSetupToUse}`);
|
||||||
|
} catch (error) {
|
||||||
|
console.error('An error occurred while running the benchmarks:');
|
||||||
|
console.error(error);
|
||||||
|
console.error('');
|
||||||
|
await dumpN8nInstanceLogs(dockerComposeClient);
|
||||||
|
} finally {
|
||||||
|
await dockerComposeClient.$('down');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function dumpN8nInstanceLogs(dockerComposeClient) {
|
||||||
|
console.error('n8n instance logs:');
|
||||||
|
await dockerComposeClient.$('logs', 'n8n');
|
||||||
|
}
|
||||||
|
|
||||||
|
function printUsage() {
|
||||||
|
const availableSetups = getAllN8nSetups();
|
||||||
|
console.log('Usage: zx runForN8nSetup.mjs --runDir /path/for/n8n/data <n8n setup to use>');
|
||||||
|
console.log(` eg: zx runForN8nSetup.mjs --runDir /path/for/n8n/data ${availableSetups[0]}`);
|
||||||
|
console.log('');
|
||||||
|
console.log('Flags:');
|
||||||
|
console.log(
|
||||||
|
' --runDir <path> Directory to share with the n8n container for storing data. Default is /n8n',
|
||||||
|
);
|
||||||
|
console.log(' --n8nDockerTag <tag> Docker tag for n8n image. Default is latest');
|
||||||
|
console.log(
|
||||||
|
' --benchmarkDockerTag <tag> Docker tag for benchmark cli image. Default is latest',
|
||||||
|
);
|
||||||
|
console.log(' --k6ApiToken <token> K6 API token to upload the results');
|
||||||
|
console.log('');
|
||||||
|
console.log('Available setups:');
|
||||||
|
console.log(availableSetups.join(', '));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @returns {string[]}
|
||||||
|
*/
|
||||||
|
function getAllN8nSetups() {
|
||||||
|
return fs.readdirSync(paths.n8nSetupsDir);
|
||||||
|
}
|
||||||
|
|
||||||
|
function validateN8nSetup(givenSetup) {
|
||||||
|
const availableSetups = getAllN8nSetups();
|
||||||
|
if (!availableSetups.includes(givenSetup)) {
|
||||||
|
printUsage();
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
main();
|
|
@ -7,18 +7,12 @@
|
||||||
* 3. Destroy the cloud environment.
|
* 3. Destroy the cloud environment.
|
||||||
*
|
*
|
||||||
* NOTE: Must be run in the root of the package.
|
* NOTE: Must be run in the root of the package.
|
||||||
*
|
|
||||||
* Usage:
|
|
||||||
* zx scripts/runBenchmarksOnCloud.mjs [--debug] <n8n setup to use>
|
|
||||||
*
|
|
||||||
*/
|
*/
|
||||||
// @ts-check
|
// @ts-check
|
||||||
import fs from 'fs';
|
|
||||||
import minimist from 'minimist';
|
|
||||||
import { sleep, which } from 'zx';
|
import { sleep, which } from 'zx';
|
||||||
import path from 'path';
|
import path from 'path';
|
||||||
import { SshClient } from './sshClient.mjs';
|
import { SshClient } from './clients/sshClient.mjs';
|
||||||
import { TerraformClient } from './terraformClient.mjs';
|
import { TerraformClient } from './clients/terraformClient.mjs';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @typedef {Object} BenchmarkEnv
|
* @typedef {Object} BenchmarkEnv
|
||||||
|
@ -27,19 +21,20 @@ import { TerraformClient } from './terraformClient.mjs';
|
||||||
|
|
||||||
const RESOURCE_GROUP_NAME = 'n8n-benchmarking';
|
const RESOURCE_GROUP_NAME = 'n8n-benchmarking';
|
||||||
|
|
||||||
const paths = {
|
/**
|
||||||
n8nSetupsDir: path.join(path.resolve('scripts'), 'runOnVm', 'n8nSetups'),
|
* @typedef {Object} Config
|
||||||
};
|
* @property {boolean} isVerbose
|
||||||
|
* @property {string[]} n8nSetupsToUse
|
||||||
async function main() {
|
* @property {string} n8nTag
|
||||||
const config = await parseAndValidateConfig();
|
* @property {string} benchmarkTag
|
||||||
|
* @property {string} [k6ApiToken]
|
||||||
|
*
|
||||||
|
* @param {Config} config
|
||||||
|
*/
|
||||||
|
export async function runInCloud(config) {
|
||||||
await ensureDependencies();
|
await ensureDependencies();
|
||||||
|
|
||||||
console.log('Using n8n tag', config.n8nTag);
|
|
||||||
console.log('Using benchmark cli tag', config.benchmarkTag);
|
|
||||||
|
|
||||||
const terraformClient = new TerraformClient({
|
const terraformClient = new TerraformClient({
|
||||||
privateKeyPath: paths.privateKeyPath,
|
|
||||||
isVerbose: config.isVerbose,
|
isVerbose: config.isVerbose,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
@ -65,7 +60,7 @@ async function ensureDependencies() {
|
||||||
* @param {BenchmarkEnv} benchmarkEnv
|
* @param {BenchmarkEnv} benchmarkEnv
|
||||||
*/
|
*/
|
||||||
async function runBenchmarksOnVm(config, benchmarkEnv) {
|
async function runBenchmarksOnVm(config, benchmarkEnv) {
|
||||||
console.log(`Setting up the environment for ${config.n8nSetupToUse}...`);
|
console.log(`Setting up the environment...`);
|
||||||
|
|
||||||
const sshClient = new SshClient({
|
const sshClient = new SshClient({
|
||||||
vmName: benchmarkEnv.vmName,
|
vmName: benchmarkEnv.vmName,
|
||||||
|
@ -85,23 +80,12 @@ async function runBenchmarksOnVm(config, benchmarkEnv) {
|
||||||
// Give some time for the VM to be ready
|
// Give some time for the VM to be ready
|
||||||
await sleep(1000);
|
await sleep(1000);
|
||||||
|
|
||||||
if (config.n8nSetupToUse === 'all') {
|
for (const n8nSetup of config.n8nSetupsToUse) {
|
||||||
const availableSetups = readAvailableN8nSetups();
|
|
||||||
|
|
||||||
for (const n8nSetup of availableSetups) {
|
|
||||||
await runBenchmarkForN8nSetup({
|
|
||||||
config,
|
|
||||||
sshClient,
|
|
||||||
scriptsDir,
|
|
||||||
n8nSetup,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
await runBenchmarkForN8nSetup({
|
await runBenchmarkForN8nSetup({
|
||||||
config,
|
config,
|
||||||
sshClient,
|
sshClient,
|
||||||
scriptsDir,
|
scriptsDir,
|
||||||
n8nSetup: config.n8nSetupToUse,
|
n8nSetup,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -111,7 +95,7 @@ async function runBenchmarksOnVm(config, benchmarkEnv) {
|
||||||
*/
|
*/
|
||||||
async function runBenchmarkForN8nSetup({ config, sshClient, scriptsDir, n8nSetup }) {
|
async function runBenchmarkForN8nSetup({ config, sshClient, scriptsDir, n8nSetup }) {
|
||||||
console.log(`Running benchmarks for ${n8nSetup}...`);
|
console.log(`Running benchmarks for ${n8nSetup}...`);
|
||||||
const runScriptPath = path.join(scriptsDir, 'runOnVm.mjs');
|
const runScriptPath = path.join(scriptsDir, 'runForN8nSetup.mjs');
|
||||||
|
|
||||||
const flags = {
|
const flags = {
|
||||||
n8nDockerTag: config.n8nTag,
|
n8nDockerTag: config.n8nTag,
|
||||||
|
@ -142,87 +126,5 @@ async function transferScriptsToVm(sshClient) {
|
||||||
|
|
||||||
await sshClient.ssh('git clone --depth=1 https://github.com/n8n-io/n8n.git');
|
await sshClient.ssh('git clone --depth=1 https://github.com/n8n-io/n8n.git');
|
||||||
|
|
||||||
return '~/n8n/packages/@n8n/benchmark/scripts/runOnVm';
|
return '~/n8n/packages/@n8n/benchmark/scripts';
|
||||||
}
|
}
|
||||||
|
|
||||||
function readAvailableN8nSetups() {
|
|
||||||
const setups = fs.readdirSync(paths.n8nSetupsDir);
|
|
||||||
|
|
||||||
return setups;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @typedef {Object} Config
|
|
||||||
* @property {boolean} isVerbose
|
|
||||||
* @property {string} n8nSetupToUse
|
|
||||||
* @property {string} n8nTag
|
|
||||||
* @property {string} benchmarkTag
|
|
||||||
* @property {string} [k6ApiToken]
|
|
||||||
*
|
|
||||||
* @returns {Promise<Config>}
|
|
||||||
*/
|
|
||||||
async function parseAndValidateConfig() {
|
|
||||||
const args = minimist(process.argv.slice(3), {
|
|
||||||
boolean: ['debug', 'help'],
|
|
||||||
});
|
|
||||||
|
|
||||||
if (args.help) {
|
|
||||||
printUsage();
|
|
||||||
process.exit(0);
|
|
||||||
}
|
|
||||||
|
|
||||||
const n8nSetupToUse = await getAndValidateN8nSetup(args);
|
|
||||||
const isVerbose = args.debug || false;
|
|
||||||
const n8nTag = args.n8nTag || process.env.N8N_DOCKER_TAG || 'latest';
|
|
||||||
const benchmarkTag = args.benchmarkTag || process.env.BENCHMARK_DOCKER_TAG || 'latest';
|
|
||||||
const k6ApiToken = args.k6ApiToken || process.env.K6_API_TOKEN || undefined;
|
|
||||||
|
|
||||||
return {
|
|
||||||
isVerbose,
|
|
||||||
n8nSetupToUse,
|
|
||||||
n8nTag,
|
|
||||||
benchmarkTag,
|
|
||||||
k6ApiToken,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param {minimist.ParsedArgs} args
|
|
||||||
*/
|
|
||||||
async function getAndValidateN8nSetup(args) {
|
|
||||||
// Last parameter is the n8n setup to use
|
|
||||||
const n8nSetupToUse = args._[args._.length - 1];
|
|
||||||
if (!n8nSetupToUse || n8nSetupToUse === 'all') {
|
|
||||||
return 'all';
|
|
||||||
}
|
|
||||||
|
|
||||||
const availableSetups = readAvailableN8nSetups();
|
|
||||||
|
|
||||||
if (!availableSetups.includes(n8nSetupToUse)) {
|
|
||||||
printUsage();
|
|
||||||
process.exit(1);
|
|
||||||
}
|
|
||||||
|
|
||||||
return n8nSetupToUse;
|
|
||||||
}
|
|
||||||
|
|
||||||
function printUsage() {
|
|
||||||
const availableSetups = readAvailableN8nSetups();
|
|
||||||
|
|
||||||
console.log('Usage: zx scripts/runInCloud.mjs [n8n setup name]');
|
|
||||||
console.log(' eg: zx scripts/runInCloud.mjs');
|
|
||||||
console.log('');
|
|
||||||
console.log('Options:');
|
|
||||||
console.log(
|
|
||||||
` [n8n setup name] Against which n8n setup to run the benchmarks. One of: ${['all', ...availableSetups].join(', ')}. Default is all`,
|
|
||||||
);
|
|
||||||
console.log(' --debug Enable verbose output');
|
|
||||||
console.log(' --n8nTag Docker tag for n8n image. Default is latest');
|
|
||||||
console.log(' --benchmarkTag Docker tag for benchmark cli image. Default is latest');
|
|
||||||
console.log(
|
|
||||||
' --k6ApiToken API token for k6 cloud. Default is read from K6_API_TOKEN env var. If omitted, k6 cloud will not be used',
|
|
||||||
);
|
|
||||||
console.log('');
|
|
||||||
}
|
|
||||||
|
|
||||||
main().catch(console.error);
|
|
||||||
|
|
61
packages/@n8n/benchmark/scripts/runLocally.mjs
Executable file
61
packages/@n8n/benchmark/scripts/runLocally.mjs
Executable file
|
@ -0,0 +1,61 @@
|
||||||
|
#!/usr/bin/env zx
|
||||||
|
/**
|
||||||
|
* Script to run benchmarks on the cloud benchmark environment.
|
||||||
|
* This script will:
|
||||||
|
* 1. Provision a benchmark environment using Terraform.
|
||||||
|
* 2. Run the benchmarks on the VM.
|
||||||
|
* 3. Destroy the cloud environment.
|
||||||
|
*
|
||||||
|
* NOTE: Must be run in the root of the package.
|
||||||
|
*/
|
||||||
|
// @ts-check
|
||||||
|
import { $ } from 'zx';
|
||||||
|
import path from 'path';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @typedef {Object} BenchmarkEnv
|
||||||
|
* @property {string} vmName
|
||||||
|
*/
|
||||||
|
|
||||||
|
const paths = {
|
||||||
|
scriptsDir: path.join(path.resolve('scripts')),
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @typedef {Object} Config
|
||||||
|
* @property {boolean} isVerbose
|
||||||
|
* @property {string[]} n8nSetupsToUse
|
||||||
|
* @property {string} n8nTag
|
||||||
|
* @property {string} benchmarkTag
|
||||||
|
* @property {string} [runDir]
|
||||||
|
* @property {string} [k6ApiToken]
|
||||||
|
*
|
||||||
|
* @param {Config} config
|
||||||
|
*/
|
||||||
|
export async function runLocally(config) {
|
||||||
|
const runScriptPath = path.join(paths.scriptsDir, 'runForN8nSetup.mjs');
|
||||||
|
|
||||||
|
const flags = Object.entries({
|
||||||
|
n8nDockerTag: config.n8nTag,
|
||||||
|
benchmarkDockerTag: config.benchmarkTag,
|
||||||
|
runDir: config.runDir,
|
||||||
|
})
|
||||||
|
.filter(([, value]) => value !== undefined)
|
||||||
|
.map(([key, value]) => `--${key}=${value}`);
|
||||||
|
|
||||||
|
try {
|
||||||
|
for (const n8nSetup of config.n8nSetupsToUse) {
|
||||||
|
console.log(`Running benchmarks for n8n setup: ${n8nSetup}`);
|
||||||
|
|
||||||
|
await $({
|
||||||
|
env: {
|
||||||
|
...process.env,
|
||||||
|
K6_API_TOKEN: config.k6ApiToken,
|
||||||
|
},
|
||||||
|
})`npx ${runScriptPath} ${flags} ${n8nSetup}`;
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.error('An error occurred while running the benchmarks:');
|
||||||
|
console.error(error);
|
||||||
|
}
|
||||||
|
}
|
|
@ -1,75 +0,0 @@
|
||||||
#!/usr/bin/env zx
|
|
||||||
/**
|
|
||||||
* This script runs the benchmarks using a given docker compose setup
|
|
||||||
*/
|
|
||||||
// @ts-check
|
|
||||||
import path from 'path';
|
|
||||||
import { $, argv, fs } from 'zx';
|
|
||||||
|
|
||||||
const paths = {
|
|
||||||
n8nSetupsDir: path.join(__dirname, 'n8nSetups'),
|
|
||||||
};
|
|
||||||
|
|
||||||
async function main() {
|
|
||||||
const [n8nSetupToUse] = argv._;
|
|
||||||
validateN8nSetup(n8nSetupToUse);
|
|
||||||
|
|
||||||
const composeFilePath = path.join(paths.n8nSetupsDir, n8nSetupToUse);
|
|
||||||
const n8nTag = argv.n8nDockerTag || process.env.N8N_DOCKER_TAG || 'latest';
|
|
||||||
const benchmarkTag = argv.benchmarkDockerTag || process.env.BENCHMARK_DOCKER_TAG || 'latest';
|
|
||||||
const k6ApiToken = argv.k6ApiToken || process.env.K6_API_TOKEN || undefined;
|
|
||||||
|
|
||||||
const $$ = $({
|
|
||||||
cwd: composeFilePath,
|
|
||||||
verbose: true,
|
|
||||||
env: {
|
|
||||||
N8N_VERSION: n8nTag,
|
|
||||||
BENCHMARK_VERSION: benchmarkTag,
|
|
||||||
K6_API_TOKEN: k6ApiToken,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
try {
|
|
||||||
await $$`docker-compose up -d n8n`;
|
|
||||||
|
|
||||||
await $$`docker-compose run benchmark run --scenarioNamePrefix=${n8nSetupToUse} `;
|
|
||||||
} catch (error) {
|
|
||||||
console.error('An error occurred while running the benchmarks:');
|
|
||||||
console.error(error);
|
|
||||||
console.error('');
|
|
||||||
await dumpN8nInstanceLogs($$);
|
|
||||||
} finally {
|
|
||||||
await $$`docker-compose down`;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function dumpN8nInstanceLogs($$) {
|
|
||||||
console.error('n8n instance logs:');
|
|
||||||
await $$`docker-compose logs n8n`;
|
|
||||||
}
|
|
||||||
|
|
||||||
function printUsage() {
|
|
||||||
const availableSetups = getAllN8nSetups();
|
|
||||||
console.log('Usage: zx runOnVm.mjs <n8n setup to use>');
|
|
||||||
console.log(` eg: zx runOnVm.mjs ${availableSetups[0]}`);
|
|
||||||
console.log('');
|
|
||||||
console.log('Available setups:');
|
|
||||||
console.log(availableSetups.join(', '));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @returns {string[]}
|
|
||||||
*/
|
|
||||||
function getAllN8nSetups() {
|
|
||||||
return fs.readdirSync(paths.n8nSetupsDir);
|
|
||||||
}
|
|
||||||
|
|
||||||
function validateN8nSetup(givenSetup) {
|
|
||||||
const availableSetups = getAllN8nSetups();
|
|
||||||
if (!availableSetups.includes(givenSetup)) {
|
|
||||||
printUsage();
|
|
||||||
process.exit(1);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
main();
|
|
Loading…
Reference in a new issue