feat: Report benchmark results (no-changelog) (#10534)

This commit is contained in:
Tomi Turtiainen 2024-08-23 16:59:19 +03:00 committed by GitHub
parent a6f4dbb0ab
commit 7194b1c3a1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 361 additions and 19 deletions

View file

@ -23,6 +23,7 @@ env:
ARM_CLIENT_ID: ${{ secrets.BENCHMARK_ARM_CLIENT_ID }}
ARM_SUBSCRIPTION_ID: ${{ secrets.BENCHMARK_ARM_SUBSCRIPTION_ID }}
ARM_TENANT_ID: ${{ secrets.BENCHMARK_ARM_TENANT_ID }}
K6_API_TOKEN: ${{ secrets.K6_API_TOKEN }}
permissions:
id-token: write

View file

@ -12,7 +12,7 @@ module.exports = {
project: './tsconfig.json',
},
ignorePatterns: ['scenarios/**'],
ignorePatterns: ['scenarios/**', 'scripts/**'],
rules: {
'n8n-local-rules/no-plain-errors': 'off',

View file

@ -15,7 +15,7 @@
// @ts-check
import fs from 'fs';
import minimist from 'minimist';
import { $, sleep, tmpdir, which } from 'zx';
import { $, sleep, which } from 'zx';
import path from 'path';
import { SshClient } from './sshClient.mjs';
import { TerraformClient } from './terraformClient.mjs';
@ -124,6 +124,7 @@ function readAvailableN8nSetups() {
* @property {string} n8nSetupToUse
* @property {string} n8nTag
* @property {string} benchmarkTag
* @property {string} [k6ApiToken]
*
* @returns {Promise<Config>}
*/
@ -136,12 +137,14 @@ async function parseAndValidateConfig() {
const isVerbose = args.debug || false;
const n8nTag = args.n8nTag || process.env.N8N_DOCKER_TAG || 'latest';
const benchmarkTag = args.benchmarkTag || process.env.BENCHMARK_DOCKER_TAG || 'latest';
const k6ApiToken = args.k6ApiToken || process.env.K6_API_TOKEN || undefined;
return {
isVerbose,
n8nSetupToUse,
n8nTag,
benchmarkTag,
k6ApiToken,
};
}
@ -177,6 +180,9 @@ function printUsage() {
console.log(' --debug Enable verbose output');
console.log(' --n8nTag Docker tag for n8n image. Default is latest');
console.log(' --benchmarkTag Docker tag for benchmark cli image. Default is latest');
console.log(
' --k6ApiToken API token for k6 cloud. Default is read from K6_API_TOKEN env var. If omitted, k6 cloud will not be used.',
);
console.log('');
console.log('Available setups:');
console.log(` ${availableSetups.join(', ')}`);

View file

@ -14,3 +14,4 @@ services:
- n8n
environment:
- N8N_BASE_URL=http://n8n:5678
- K6_API_TOKEN=${K6_API_TOKEN}

View file

@ -21,6 +21,7 @@ async function main() {
const composeFilePath = path.join(__dirname, 'n8nSetups', n8nSetupToUse);
const n8nTag = argv.n8nDockerTag || process.env.N8N_DOCKER_TAG || 'latest';
const benchmarkTag = argv.benchmarkDockerTag || process.env.BENCHMARK_DOCKER_TAG || 'latest';
const k6ApiToken = argv.k6ApiToken || process.env.K6_API_TOKEN || undefined;
const $$ = $({
cwd: composeFilePath,
@ -28,6 +29,7 @@ async function main() {
env: {
N8N_VERSION: n8nTag,
BENCHMARK_VERSION: benchmarkTag,
K6_API_TOKEN: k6ApiToken,
},
});

View file

@ -25,7 +25,11 @@ export default class RunCommand extends Command {
const scenarioRunner = new ScenarioRunner(
new N8nApiClient(config.get('n8n.baseUrl')),
new ScenarioDataFileLoader(),
new K6Executor(config.get('k6ExecutablePath'), config.get('n8n.baseUrl')),
new K6Executor({
k6ExecutablePath: config.get('k6.executablePath'),
k6ApiToken: config.get('k6.apiToken'),
n8nApiBaseUrl: config.get('n8n.baseUrl'),
}),
{
email: config.get('n8n.user.email'),
password: config.get('n8n.user.password'),

View file

@ -31,11 +31,19 @@ const configSchema = {
},
},
},
k6ExecutablePath: {
doc: 'The path to the k6 binary',
format: String,
default: 'k6',
env: 'K6_PATH',
k6: {
executablePath: {
doc: 'The path to the k6 binary',
format: String,
default: 'k6',
env: 'K6_PATH',
},
apiToken: {
doc: 'The API token for k6 cloud',
format: String,
default: undefined,
env: 'K6_API_TOKEN',
},
},
};

View file

@ -1,38 +1,103 @@
import { $, which } from 'zx';
import fs from 'fs';
import path from 'path';
import { $, which, tmpfile } from 'zx';
import type { Scenario } from '@/types/scenario';
export type K6ExecutorOpts = {
k6ExecutablePath: string;
k6ApiToken?: string;
n8nApiBaseUrl: string;
};
/**
* Flag for the k6 CLI.
* @example ['--duration', '1m']
* @example ['--quiet']
*/
type K6CliFlag = [string] | [string, string];
/**
* Executes test scenarios using k6
*/
export class K6Executor {
constructor(
private readonly k6ExecutablePath: string,
private readonly n8nApiBaseUrl: string,
) {}
/**
* This script is dynamically injected into the k6 test script to generate
* a summary report of the test execution.
*/
private readonly handleSummaryScript = `
import { textSummary } from 'https://jslib.k6.io/k6-summary/0.0.2/index.js';
export function handleSummary(data) {
return {
stdout: textSummary(data),
'{{scenarioName}}.summary.json': JSON.stringify(data),
};
}
`;
constructor(private readonly opts: K6ExecutorOpts) {}
async executeTestScenario(scenario: Scenario) {
// For 1 min with 5 virtual users
const stage = '1m:5';
const augmentedTestScriptPath = this.augmentSummaryScript(scenario);
const runDirPath = path.dirname(augmentedTestScriptPath);
const flags: K6CliFlag[] = [['--quiet'], ['--duration', '1m'], ['--vus', '5']];
if (this.opts.k6ApiToken) {
flags.push(['--out', 'cloud']);
}
const flattedFlags = flags.flat(2);
const k6ExecutablePath = await this.resolveK6ExecutablePath();
const processPromise = $({
cwd: scenario.scenarioDirPath,
cwd: runDirPath,
env: {
API_BASE_URL: this.n8nApiBaseUrl,
API_BASE_URL: this.opts.n8nApiBaseUrl,
K6_CLOUD_TOKEN: this.opts.k6ApiToken,
},
})`${k6ExecutablePath} run --quiet --stage ${stage} ${scenario.scriptPath}`;
})`${k6ExecutablePath} run ${flattedFlags} ${augmentedTestScriptPath}`;
for await (const chunk of processPromise.stdout) {
console.log((chunk as Buffer).toString());
}
this.loadEndOfTestSummary(runDirPath, scenario.name);
}
/**
* Augments the test script with a summary script
*
* @returns Absolute path to the augmented test script
*/
private augmentSummaryScript(scenario: Scenario) {
const fullTestScriptPath = path.join(scenario.scenarioDirPath, scenario.scriptPath);
const testScript = fs.readFileSync(fullTestScriptPath, 'utf8');
const summaryScript = this.handleSummaryScript.replace('{{scenarioName}}', scenario.name);
const augmentedTestScript = `${testScript}\n\n${summaryScript}`;
const tempFilePath = tmpfile(`${scenario.name}.ts`, augmentedTestScript);
return tempFilePath;
}
private loadEndOfTestSummary(dir: string, scenarioName: string): K6EndOfTestSummary {
const summaryReportPath = path.join(dir, `${scenarioName}.summary.json`);
const summaryReport = fs.readFileSync(summaryReportPath, 'utf8');
try {
return JSON.parse(summaryReport);
} catch (error) {
throw new Error(`Failed to parse the summary report at ${summaryReportPath}`);
}
}
/**
* @returns Resolved path to the k6 executable
*/
private async resolveK6ExecutablePath(): Promise<string> {
const k6ExecutablePath = await which(this.k6ExecutablePath, { nothrow: true });
const k6ExecutablePath = await which(this.opts.k6ExecutablePath, { nothrow: true });
if (!k6ExecutablePath) {
throw new Error(
'Could not find k6 executable based on your `PATH`. Please ensure k6 is available in your system and add it to your `PATH` or specify the path to the k6 executable using the `K6_PATH` environment variable.',

View file

@ -0,0 +1,255 @@
/**
Example JSON:
{
"options": {
"summaryTrendStats": ["avg", "min", "med", "max", "p(90)", "p(95)"],
"summaryTimeUnit": "",
"noColor": false
},
"state": { "isStdOutTTY": false, "isStdErrTTY": false, "testRunDurationMs": 23.374 },
"metrics": {
"http_req_tls_handshaking": {
"type": "trend",
"contains": "time",
"values": { "avg": 0, "min": 0, "med": 0, "max": 0, "p(90)": 0, "p(95)": 0 }
},
"checks": {
"type": "rate",
"contains": "default",
"values": { "rate": 1, "passes": 1, "fails": 0 }
},
"http_req_sending": {
"type": "trend",
"contains": "time",
"values": {
"p(90)": 0.512,
"p(95)": 0.512,
"avg": 0.512,
"min": 0.512,
"med": 0.512,
"max": 0.512
}
},
"http_reqs": {
"contains": "default",
"values": { "count": 1, "rate": 42.78257893385813 },
"type": "counter"
},
"http_req_blocked": {
"contains": "time",
"values": {
"avg": 1.496,
"min": 1.496,
"med": 1.496,
"max": 1.496,
"p(90)": 1.496,
"p(95)": 1.496
},
"type": "trend"
},
"data_received": {
"type": "counter",
"contains": "data",
"values": { "count": 269, "rate": 11508.513733207838 }
},
"iterations": {
"type": "counter",
"contains": "default",
"values": { "count": 1, "rate": 42.78257893385813 }
},
"http_req_waiting": {
"type": "trend",
"contains": "time",
"values": {
"p(95)": 18.443,
"avg": 18.443,
"min": 18.443,
"med": 18.443,
"max": 18.443,
"p(90)": 18.443
}
},
"http_req_receiving": {
"type": "trend",
"contains": "time",
"values": {
"avg": 0.186,
"min": 0.186,
"med": 0.186,
"max": 0.186,
"p(90)": 0.186,
"p(95)": 0.186
}
},
"http_req_duration{expected_response:true}": {
"type": "trend",
"contains": "time",
"values": {
"max": 19.141,
"p(90)": 19.141,
"p(95)": 19.141,
"avg": 19.141,
"min": 19.141,
"med": 19.141
}
},
"iteration_duration": {
"type": "trend",
"contains": "time",
"values": {
"avg": 22.577833,
"min": 22.577833,
"med": 22.577833,
"max": 22.577833,
"p(90)": 22.577833,
"p(95)": 22.577833
}
},
"http_req_connecting": {
"type": "trend",
"contains": "time",
"values": {
"avg": 0.673,
"min": 0.673,
"med": 0.673,
"max": 0.673,
"p(90)": 0.673,
"p(95)": 0.673
}
},
"http_req_failed": {
"type": "rate",
"contains": "default",
"values": { "rate": 0, "passes": 0, "fails": 1 }
},
"http_req_duration": {
"type": "trend",
"contains": "time",
"values": {
"p(90)": 19.141,
"p(95)": 19.141,
"avg": 19.141,
"min": 19.141,
"med": 19.141,
"max": 19.141
}
},
"data_sent": {
"type": "counter",
"contains": "data",
"values": { "count": 102, "rate": 4363.82305125353 }
}
},
"root_group": {
"name": "",
"path": "",
"id": "d41d8cd98f00b204e9800998ecf8427e",
"groups": [],
"checks": [
{
"name": "is status 200",
"path": "::is status 200",
"id": "548d37ca5f33793206f7832e7cea54fb",
"passes": 1,
"fails": 0
}
]
}
}
*/
type TrendStat = 'avg' | 'min' | 'med' | 'max' | 'p(90)' | 'p(95)';
type MetricType = 'trend' | 'rate' | 'counter';
type MetricContains = 'time' | 'default' | 'data';
interface TrendValues {
avg: number;
min: number;
med: number;
max: number;
'p(90)': number;
'p(95)': number;
}
interface RateValues {
rate: number;
passes: number;
fails: number;
}
interface CounterValues {
count: number;
rate: number;
}
interface TrendMetric {
type: 'trend';
contains: 'time';
values: TrendValues;
}
interface RateMetric {
type: 'rate';
contains: 'default';
values: RateValues;
}
interface CounterMetric {
type: 'counter';
contains: MetricContains;
values: CounterValues;
}
interface Options {
summaryTrendStats: TrendStat[];
summaryTimeUnit: string;
noColor: boolean;
}
interface State {
isStdOutTTY: boolean;
isStdErrTTY: boolean;
testRunDurationMs: number;
}
interface Metrics {
http_req_tls_handshaking: TrendMetric;
checks: RateMetric;
http_req_sending: TrendMetric;
http_reqs: CounterMetric;
http_req_blocked: TrendMetric;
data_received: CounterMetric;
iterations: CounterMetric;
http_req_waiting: TrendMetric;
http_req_receiving: TrendMetric;
'http_req_duration{expected_response:true}': TrendMetric;
iteration_duration: TrendMetric;
http_req_connecting: TrendMetric;
http_req_failed: RateMetric;
http_req_duration: TrendMetric;
data_sent: CounterMetric;
}
interface Check {
name: string;
path: string;
id: string;
passes: number;
fails: number;
}
interface RootGroup {
name: string;
path: string;
id: string;
groups: any[];
checks: Check[];
}
interface K6EndOfTestSummary {
options: Options;
state: State;
metrics: Metrics;
root_group: RootGroup;
}