mudhorn/n8n
1
0
Fork 0
mirror of https://github.com/n8n-io/n8n.git synced 2024-12-24 04:04:06 -08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

feat(benchmark): Report benchmark results to a configurable webhook (#10754)

Browse source
This commit is contained in:
Tomi Turtiainen 2024-09-10 17:41:33 +03:00 committed by GitHub
parent 8450ec5a5c
commit e56dabd63a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
16 changed files with 229 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.github/workflows/benchmark-nightly.yml vendored
View file

@ -23,7 +23,8 @@ env:
ARM_CLIENT_ID: ${{ secrets.BENCHMARK_ARM_CLIENT_ID }}
ARM_SUBSCRIPTION_ID: ${{ secrets.BENCHMARK_ARM_SUBSCRIPTION_ID }}
ARM_TENANT_ID: ${{ secrets.BENCHMARK_ARM_TENANT_ID }}
K6_API_TOKEN: ${{ secrets.K6_API_TOKEN }}
BENCHMARK_RESULT_WEBHOOK_URL: ${{ secrets.BENCHMARK_RESULT_WEBHOOK_URL }}
BENCHMARK_RESULT_WEBHOOK_AUTH_HEADER: ${{ secrets.BENCHMARK_RESULT_WEBHOOK_AUTH_HEADER }}
N8N_TAG: ${{ inputs.n8n_tag || 'nightly' }}
N8N_BENCHMARK_TAG: ${{ inputs.benchmark_tag || 'latest' }}
DEBUG: ${{ inputs.debug == 'true' && '--debug' || '' }}

1
packages/@n8n/benchmark/package.json
View file

@ -34,6 +34,7 @@
"@oclif/core": "4.0.7",
"axios": "catalog:",
"dotenv": "8.6.0",
"nanoid": "catalog:",
"zx": "^8.1.4"
},
"devDependencies": {

2
packages/@n8n/benchmark/scripts/n8nSetups/postgres/docker-compose.yml
View file

@ -55,3 +55,5 @@ services:
environment:
- N8N_BASE_URL=http://n8n:5678
- K6_API_TOKEN=${K6_API_TOKEN}
- BENCHMARK_RESULT_WEBHOOK_URL=${BENCHMARK_RESULT_WEBHOOK_URL}
- BENCHMARK_RESULT_WEBHOOK_AUTH_HEADER=${BENCHMARK_RESULT_WEBHOOK_AUTH_HEADER}

2
packages/@n8n/benchmark/scripts/n8nSetups/queue/docker-compose.yml
View file

@ -138,3 +138,5 @@ services:
environment:
- N8N_BASE_URL=http://n8n:5678
- K6_API_TOKEN=${K6_API_TOKEN}
- BENCHMARK_RESULT_WEBHOOK_URL=${BENCHMARK_RESULT_WEBHOOK_URL}
- BENCHMARK_RESULT_WEBHOOK_AUTH_HEADER=${BENCHMARK_RESULT_WEBHOOK_AUTH_HEADER}

2
packages/@n8n/benchmark/scripts/n8nSetups/scaling/docker-compose.yml
View file

@ -192,3 +192,5 @@ services:
environment:
- N8N_BASE_URL=http://n8n:80
- K6_API_TOKEN=${K6_API_TOKEN}
- BENCHMARK_RESULT_WEBHOOK_URL=${BENCHMARK_RESULT_WEBHOOK_URL}
- BENCHMARK_RESULT_WEBHOOK_AUTH_HEADER=${BENCHMARK_RESULT_WEBHOOK_AUTH_HEADER}

2
packages/@n8n/benchmark/scripts/n8nSetups/sqlite-legacy/docker-compose.yml
View file

@ -33,3 +33,5 @@ services:
environment:
- N8N_BASE_URL=http://n8n:5678
- K6_API_TOKEN=${K6_API_TOKEN}
- BENCHMARK_RESULT_WEBHOOK_URL=${BENCHMARK_RESULT_WEBHOOK_URL}
- BENCHMARK_RESULT_WEBHOOK_AUTH_HEADER=${BENCHMARK_RESULT_WEBHOOK_AUTH_HEADER}

2
packages/@n8n/benchmark/scripts/n8nSetups/sqlite/docker-compose.yml
View file

@ -35,3 +35,5 @@ services:
environment:
- N8N_BASE_URL=http://n8n:5678
- K6_API_TOKEN=${K6_API_TOKEN}
- BENCHMARK_RESULT_WEBHOOK_URL=${BENCHMARK_RESULT_WEBHOOK_URL}
- BENCHMARK_RESULT_WEBHOOK_AUTH_HEADER=${BENCHMARK_RESULT_WEBHOOK_AUTH_HEADER}

18
packages/@n8n/benchmark/scripts/run.mjs
View file

@ -33,17 +33,21 @@ async function main() {
benchmarkTag: config.benchmarkTag,
isVerbose: config.isVerbose,
k6ApiToken: config.k6ApiToken,
resultWebhookUrl: config.resultWebhookUrl,
resultWebhookAuthHeader: config.resultWebhookAuthHeader,
n8nLicenseCert: config.n8nLicenseCert,
n8nTag: config.n8nTag,
n8nSetupsToUse,
vus: config.vus,
duration: config.duration,
});
} else {
} else if (config.env === 'local') {
await runLocally({
benchmarkTag: config.benchmarkTag,
isVerbose: config.isVerbose,
k6ApiToken: config.k6ApiToken,
resultWebhookUrl: config.resultWebhookUrl,
resultWebhookAuthHeader: config.resultWebhookAuthHeader,
n8nLicenseCert: config.n8nLicenseCert,
n8nTag: config.n8nTag,
runDir: config.runDir,
@ -51,6 +55,10 @@ async function main() {
vus: config.vus,
duration: config.duration,
});
} else {
console.error('Invalid env:', config.env);
printUsage();
process.exit(1);
}
}
@ -68,6 +76,8 @@ function readAvailableN8nSetups() {
* @property {string} n8nTag
* @property {string} benchmarkTag
* @property {string} [k6ApiToken]
* @property {string} [resultWebhookUrl]
* @property {string} [resultWebhookAuthHeader]
* @property {string} [n8nLicenseCert]
* @property {string} [runDir]
* @property {string} [vus]
@ -90,6 +100,10 @@ async function parseAndValidateConfig() {
const n8nTag = args.n8nTag || process.env.N8N_DOCKER_TAG || 'latest';
const benchmarkTag = args.benchmarkTag || process.env.BENCHMARK_DOCKER_TAG || 'latest';
const k6ApiToken = args.k6ApiToken || process.env.K6_API_TOKEN || undefined;
const resultWebhookUrl =
args.resultWebhookUrl || process.env.BENCHMARK_RESULT_WEBHOOK_URL || undefined;
const resultWebhookAuthHeader =
args.resultWebhookAuthHeader || process.env.BENCHMARK_RESULT_WEBHOOK_AUTH_HEADER || undefined;
const n8nLicenseCert = args.n8nLicenseCert || process.env.N8N_LICENSE_CERT || undefined;
const runDir = args.runDir || undefined;
const env = args.env || 'local';
@ -108,6 +122,8 @@ async function parseAndValidateConfig() {
n8nTag,
benchmarkTag,
k6ApiToken,
resultWebhookUrl,
resultWebhookAuthHeader,
n8nLicenseCert,
runDir,
vus,

8
packages/@n8n/benchmark/scripts/runForN8nSetup.mjs
View file

@ -24,10 +24,15 @@ async function main() {
const n8nTag = argv.n8nDockerTag || process.env.N8N_DOCKER_TAG || 'latest';
const benchmarkTag = argv.benchmarkDockerTag || process.env.BENCHMARK_DOCKER_TAG || 'latest';
const k6ApiToken = argv.k6ApiToken || process.env.K6_API_TOKEN || undefined;
const resultWebhookUrl =
argv.resultWebhookUrl || process.env.BENCHMARK_RESULT_WEBHOOK_URL || undefined;
const resultWebhookAuthHeader =
argv.resultWebhookAuthHeader || process.env.BENCHMARK_RESULT_WEBHOOK_AUTH_HEADER || undefined;
const baseRunDir = argv.runDir || process.env.RUN_DIR || '/n8n';
const n8nLicenseCert = argv.n8nLicenseCert || process.env.N8N_LICENSE_CERT || undefined;
const n8nLicenseActivationKey = process.env.N8N_LICENSE_ACTIVATION_KEY || '';
const n8nLicenseTenantId = argv.n8nLicenseTenantId || process.env.N8N_LICENSE_TENANT_ID || '';
const envTag = argv.env || 'local';
const vus = argv.vus;
const duration = argv.duration;
@ -54,6 +59,8 @@ async function main() {
N8N_ENCRYPTION_KEY,
BENCHMARK_VERSION: benchmarkTag,
K6_API_TOKEN: k6ApiToken,
BENCHMARK_RESULT_WEBHOOK_URL: resultWebhookUrl,
BENCHMARK_RESULT_WEBHOOK_AUTH_HEADER: resultWebhookAuthHeader,
RUN_DIR: runDir,
MOCK_API_DATA_PATH: paths.mockApiDataPath,
},
@ -70,6 +77,7 @@ async function main() {
await dockerComposeClient.$('up', '-d', '--remove-orphans', 'n8n');
const tags = Object.entries({
Env: envTag,
N8nVersion: n8nTag,
N8nSetup: n8nSetupToUse,
})

5
packages/@n8n/benchmark/scripts/runInCloud.mjs
View file

@ -30,6 +30,8 @@ import { flagsObjectToCliArgs } from './utils/flags.mjs';
* @property {string} n8nTag
* @property {string} benchmarkTag
* @property {string} [k6ApiToken]
* @property {string} [resultWebhookUrl]
* @property {string} [resultWebhookAuthHeader]
* @property {string} [n8nLicenseCert]
* @property {string} [vus]
* @property {string} [duration]
@ -100,9 +102,12 @@ async function runBenchmarkForN8nSetup({ config, sshClient, scriptsDir, n8nSetup
n8nDockerTag: config.n8nTag,
benchmarkDockerTag: config.benchmarkTag,
k6ApiToken: config.k6ApiToken,
resultWebhookUrl: config.resultWebhookUrl,
resultWebhookAuthHeader: config.resultWebhookAuthHeader,
n8nLicenseCert: config.n8nLicenseCert,
vus: config.vus,
duration: config.duration,
env: 'cloud',
});
const flagsString = cliArgs.join(' ');

5
packages/@n8n/benchmark/scripts/runLocally.mjs
View file

@ -30,6 +30,8 @@ const paths = {
* @property {string} benchmarkTag
* @property {string} [runDir]
* @property {string} [k6ApiToken]
* @property {string} [resultWebhookUrl]
* @property {string} [resultWebhookAuthHeader]
* @property {string} [n8nLicenseCert]
* @property {string} [vus]
* @property {string} [duration]
@ -45,6 +47,7 @@ export async function runLocally(config) {
runDir: config.runDir,
vus: config.vus,
duration: config.duration,
env: 'local',
});
try {
@ -55,6 +58,8 @@ export async function runLocally(config) {
env: {
...process.env,
K6_API_TOKEN: config.k6ApiToken,
BENCHMARK_RESULT_WEBHOOK_URL: config.resultWebhookUrl,
BENCHMARK_RESULT_WEBHOOK_AUTH_HEADER: config.resultWebhookAuthHeader,
N8N_LICENSE_CERT: config.n8nLicenseCert,
},
})`npx ${runScriptPath} ${cliArgs} ${n8nSetup}`;

14
packages/@n8n/benchmark/src/commands/run.ts
View file

@ -36,6 +36,16 @@ export default class RunCommand extends Command {
default: undefined,
env: 'K6_API_TOKEN',
}),
resultWebhookUrl: Flags.string({
doc: 'The URL where the benchmark results should be sent to',
default: undefined,
env: 'BENCHMARK_RESULT_WEBHOOK_URL',
}),
resultWebhookAuthHeader: Flags.string({
doc: 'The Authorization header value for the benchmark results webhook',
default: undefined,
env: 'BENCHMARK_RESULT_WEBHOOK_AUTH_HEADER',
}),
n8nUserPassword: Flags.string({
description: 'The password of the n8n user',
default: 'VerySecret!123',
@ -70,6 +80,10 @@ export default class RunCommand extends Command {
k6ApiToken: flags.k6ApiToken,
n8nApiBaseUrl: flags.n8nBaseUrl,
tags,
resultsWebhook: {
url: flags.resultWebhookUrl,
authHeader: flags.resultWebhookAuthHeader,
},
}),
{
email: flags.n8nUserEmail,

50
packages/@n8n/benchmark/src/testExecution/k6Executor.ts
View file

@ -1,12 +1,10 @@
import fs from 'fs';
import path from 'path';
import assert from 'node:assert/strict';
import { $, which, tmpfile } from 'zx';
import type { Scenario } from '@/types/scenario';
export type K6Tag = {
name: string;
value: string;
};
import { buildTestReport, type K6Tag } from '@/testExecution/testReport';
export type { K6Tag };
export type K6ExecutorOpts = {
k6ExecutablePath: string;
@ -17,6 +15,10 @@ export type K6ExecutorOpts = {
k6ApiToken?: string;
n8nApiBaseUrl: string;
tags?: K6Tag[];
resultsWebhook?: {
url: string;
authHeader: string;
};
};
export type K6RunOpts = {
@ -61,7 +63,7 @@ export function handleSummary(data) {
['--vus', this.opts.vus],
];
if (this.opts.k6ApiToken) {
if (!this.opts.resultsWebhook && this.opts.k6ApiToken) {
flags.push(['--out', 'cloud']);
}
@ -69,20 +71,46 @@ export function handleSummary(data) {
const k6ExecutablePath = await this.resolveK6ExecutablePath();
const processPromise = $({
await $({
cwd: runDirPath,
env: {
API_BASE_URL: this.opts.n8nApiBaseUrl,
K6_CLOUD_TOKEN: this.opts.k6ApiToken,
SCRIPT_FILE_PATH: augmentedTestScriptPath,
},
stdio: 'inherit',
})`${k6ExecutablePath} run ${flattedFlags} ${augmentedTestScriptPath}`;
for await (const chunk of processPromise.stdout) {
console.log((chunk as Buffer).toString());
}
console.log('\n');
this.loadEndOfTestSummary(runDirPath, scenarioRunName);
if (this.opts.resultsWebhook) {
const endOfTestSummary = this.loadEndOfTestSummary(runDirPath, scenarioRunName);
const testReport = buildTestReport(scenario, endOfTestSummary, [
...(this.opts.tags ?? []),
{ name: 'Vus', value: this.opts.vus.toString() },
{ name: 'Duration', value: this.opts.duration.toString() },
]);
await this.sendTestReport(testReport);
}
}
async sendTestReport(testReport: unknown) {
assert(this.opts.resultsWebhook);
const response = await fetch(this.opts.resultsWebhook.url, {
method: 'POST',
body: JSON.stringify(testReport),
headers: {
Authorization: this.opts.resultsWebhook.authHeader,
'Content-Type': 'application/json',
},
});
if (!response.ok) {
console.warn(`Failed to send test summary: ${response.status} ${await response.text()}`);
}
}
/**

34
packages/@n8n/benchmark/src/testExecution/k6Summary.ts
View file

@ -183,7 +183,7 @@ interface CounterValues {
rate: number;
}
interface TrendMetric {
interface K6TrendMetric {
type: 'trend';
contains: 'time';
values: TrendValues;
@ -195,7 +195,7 @@ interface RateMetric {
values: RateValues;
}
interface CounterMetric {
interface K6CounterMetric {
type: 'counter';
contains: MetricContains;
values: CounterValues;
@ -214,24 +214,24 @@ interface State {
}
interface Metrics {
http_req_tls_handshaking: TrendMetric;
http_req_tls_handshaking: K6TrendMetric;
checks: RateMetric;
http_req_sending: TrendMetric;
http_reqs: CounterMetric;
http_req_blocked: TrendMetric;
data_received: CounterMetric;
iterations: CounterMetric;
http_req_waiting: TrendMetric;
http_req_receiving: TrendMetric;
'http_req_duration{expected_response:true}': TrendMetric;
iteration_duration: TrendMetric;
http_req_connecting: TrendMetric;
http_req_sending: K6TrendMetric;
http_reqs: K6CounterMetric;
http_req_blocked: K6TrendMetric;
data_received: K6CounterMetric;
iterations: K6CounterMetric;
http_req_waiting: K6TrendMetric;
http_req_receiving: K6TrendMetric;
'http_req_duration{expected_response:true}': K6TrendMetric;
iteration_duration: K6TrendMetric;
http_req_connecting: K6TrendMetric;
http_req_failed: RateMetric;
http_req_duration: TrendMetric;
data_sent: CounterMetric;
http_req_duration: K6TrendMetric;
data_sent: K6CounterMetric;
}
interface Check {
interface K6Check {
name: string;
path: string;
id: string;
@ -244,7 +244,7 @@ interface RootGroup {
path: string;
id: string;
groups: unknown[];
checks: Check[];
checks: K6Check[];
}
interface K6EndOfTestSummary {

102
packages/@n8n/benchmark/src/testExecution/testReport.ts Normal file
View file

@ -0,0 +1,102 @@
import { nanoid } from 'nanoid';
import type { Scenario } from '@/types/scenario';
export type K6Tag = {
name: string;
value: string;
};
export type Check = {
name: string;
passes: number;
fails: number;
};
export type CounterMetric = {
type: 'counter';
count: number;
rate: number;
};
export type TrendMetric = {
type: 'trend';
'p(95)': number;
avg: number;
min: number;
med: number;
max: number;
'p(90)': number;
};
export type TestReport = {
runId: string;
ts: string; // ISO8601
scenarioName: string;
tags: K6Tag[];
metrics: {
iterations: CounterMetric;
dataReceived: CounterMetric;
dataSent: CounterMetric;
httpRequests: CounterMetric;
httpRequestDuration: TrendMetric;
httpRequestSending: TrendMetric;
httpRequestReceiving: TrendMetric;
httpRequestWaiting: TrendMetric;
};
checks: Check[];
};
function k6CheckToCheck(check: K6Check): Check {
return {
name: check.name,
passes: check.passes,
fails: check.fails,
};
}
function k6CounterToCounter(counter: K6CounterMetric): CounterMetric {
return {
type: 'counter',
count: counter.values.count,
rate: counter.values.rate,
};
}
function k6TrendToTrend(trend: K6TrendMetric): TrendMetric {
return {
type: 'trend',
'p(90)': trend.values['p(90)'],
avg: trend.values.avg,
min: trend.values.min,
med: trend.values.med,
max: trend.values.max,
'p(95)': trend.values['p(95)'],
};
}
/**
* Converts the k6 test summary to a test report
*/
export function buildTestReport(
scenario: Scenario,
endOfTestSummary: K6EndOfTestSummary,
tags: K6Tag[],
): TestReport {
return {
runId: nanoid(),
ts: new Date().toISOString(),
scenarioName: scenario.name,
tags,
checks: endOfTestSummary.root_group.checks.map(k6CheckToCheck),
metrics: {
dataReceived: k6CounterToCounter(endOfTestSummary.metrics.data_received),
dataSent: k6CounterToCounter(endOfTestSummary.metrics.data_sent),
httpRequests: k6CounterToCounter(endOfTestSummary.metrics.http_reqs),
httpRequestDuration: k6TrendToTrend(endOfTestSummary.metrics.http_req_duration),
httpRequestSending: k6TrendToTrend(endOfTestSummary.metrics.http_req_sending),
httpRequestReceiving: k6TrendToTrend(endOfTestSummary.metrics.http_req_receiving),
httpRequestWaiting: k6TrendToTrend(endOfTestSummary.metrics.http_req_waiting),
iterations: k6CounterToCounter(endOfTestSummary.metrics.iterations),
},
};
}

15
pnpm-lock.yaml
View file

@ -229,6 +229,9 @@ importers:
dotenv:
specifier: 8.6.0
version: 8.6.0
nanoid:
specifier: 'catalog:'
version: 3.3.6
zx:
specifier: ^8.1.4
version: 8.1.4
@ -21625,7 +21628,7 @@ snapshots:
eslint-import-resolver-node@0.3.9:
dependencies:
debug: 3.2.7(supports-color@8.1.1)
debug: 3.2.7(supports-color@5.5.0)
is-core-module: 2.13.1
resolve: 1.22.8
transitivePeerDependencies:
@ -21650,7 +21653,7 @@ snapshots:
eslint-module-utils@2.8.0(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.5.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.5.2))(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0):
dependencies:
debug: 3.2.7(supports-color@8.1.1)
debug: 3.2.7(supports-color@5.5.0)
optionalDependencies:
'@typescript-eslint/parser': 7.2.0(eslint@8.57.0)(typescript@5.5.2)
eslint: 8.57.0
@ -21670,7 +21673,7 @@ snapshots:
array.prototype.findlastindex: 1.2.3
array.prototype.flat: 1.3.2
array.prototype.flatmap: 1.3.2
debug: 3.2.7(supports-color@8.1.1)
debug: 3.2.7(supports-color@5.5.0)
doctrine: 2.1.0
eslint: 8.57.0
eslint-import-resolver-node: 0.3.9
@ -22540,7 +22543,7 @@ snapshots:
array-parallel: 0.1.3
array-series: 0.1.5
cross-spawn: 4.0.2
debug: 3.2.7(supports-color@8.1.1)
debug: 3.2.7(supports-color@5.5.0)
transitivePeerDependencies:
- supports-color
@ -25561,7 +25564,7 @@ snapshots:
pdf-parse@1.1.1:
dependencies:
debug: 3.2.7(supports-color@8.1.1)
debug: 3.2.7(supports-color@5.5.0)
node-ensure: 0.0.0
transitivePeerDependencies:
- supports-color
@ -26447,7 +26450,7 @@ snapshots:
rhea@1.0.24:
dependencies:
debug: 3.2.7(supports-color@8.1.1)
debug: 3.2.7(supports-color@5.5.0)
transitivePeerDependencies:
- supports-color