diff --git a/.github/workflows/benchmark-nightly.yml b/.github/workflows/benchmark-nightly.yml index 146668491e..eb5aacbd0f 100644 --- a/.github/workflows/benchmark-nightly.yml +++ b/.github/workflows/benchmark-nightly.yml @@ -23,6 +23,7 @@ env: ARM_CLIENT_ID: ${{ secrets.BENCHMARK_ARM_CLIENT_ID }} ARM_SUBSCRIPTION_ID: ${{ secrets.BENCHMARK_ARM_SUBSCRIPTION_ID }} ARM_TENANT_ID: ${{ secrets.BENCHMARK_ARM_TENANT_ID }} + K6_API_TOKEN: ${{ secrets.K6_API_TOKEN }} permissions: id-token: write diff --git a/packages/@n8n/benchmark/.eslintrc.js b/packages/@n8n/benchmark/.eslintrc.js index 99c277f212..463a7d2f52 100644 --- a/packages/@n8n/benchmark/.eslintrc.js +++ b/packages/@n8n/benchmark/.eslintrc.js @@ -12,7 +12,7 @@ module.exports = { project: './tsconfig.json', }, - ignorePatterns: ['scenarios/**'], + ignorePatterns: ['scenarios/**', 'scripts/**'], rules: { 'n8n-local-rules/no-plain-errors': 'off', diff --git a/packages/@n8n/benchmark/scripts/runInCloud.mjs b/packages/@n8n/benchmark/scripts/runInCloud.mjs index ee5c6b48e8..c567ec0475 100755 --- a/packages/@n8n/benchmark/scripts/runInCloud.mjs +++ b/packages/@n8n/benchmark/scripts/runInCloud.mjs @@ -15,7 +15,7 @@ // @ts-check import fs from 'fs'; import minimist from 'minimist'; -import { $, sleep, tmpdir, which } from 'zx'; +import { $, sleep, which } from 'zx'; import path from 'path'; import { SshClient } from './sshClient.mjs'; import { TerraformClient } from './terraformClient.mjs'; @@ -124,6 +124,7 @@ function readAvailableN8nSetups() { * @property {string} n8nSetupToUse * @property {string} n8nTag * @property {string} benchmarkTag + * @property {string} [k6ApiToken] * * @returns {Promise} */ @@ -136,12 +137,14 @@ async function parseAndValidateConfig() { const isVerbose = args.debug || false; const n8nTag = args.n8nTag || process.env.N8N_DOCKER_TAG || 'latest'; const benchmarkTag = args.benchmarkTag || process.env.BENCHMARK_DOCKER_TAG || 'latest'; + const k6ApiToken = args.k6ApiToken || process.env.K6_API_TOKEN || undefined; return { isVerbose, n8nSetupToUse, n8nTag, benchmarkTag, + k6ApiToken, }; } @@ -177,6 +180,9 @@ function printUsage() { console.log(' --debug Enable verbose output'); console.log(' --n8nTag Docker tag for n8n image. Default is latest'); console.log(' --benchmarkTag Docker tag for benchmark cli image. Default is latest'); + console.log( + ' --k6ApiToken API token for k6 cloud. Default is read from K6_API_TOKEN env var. If omitted, k6 cloud will not be used.', + ); console.log(''); console.log('Available setups:'); console.log(` ${availableSetups.join(', ')}`); diff --git a/packages/@n8n/benchmark/scripts/runOnVm/n8nSetups/sqlite/docker-compose.yml b/packages/@n8n/benchmark/scripts/runOnVm/n8nSetups/sqlite/docker-compose.yml index c5c8b2d4f1..02b61961f1 100644 --- a/packages/@n8n/benchmark/scripts/runOnVm/n8nSetups/sqlite/docker-compose.yml +++ b/packages/@n8n/benchmark/scripts/runOnVm/n8nSetups/sqlite/docker-compose.yml @@ -14,3 +14,4 @@ services: - n8n environment: - N8N_BASE_URL=http://n8n:5678 + - K6_API_TOKEN=${K6_API_TOKEN} diff --git a/packages/@n8n/benchmark/scripts/runOnVm/runOnVm.mjs b/packages/@n8n/benchmark/scripts/runOnVm/runOnVm.mjs index 05908426c8..e3402e70c5 100755 --- a/packages/@n8n/benchmark/scripts/runOnVm/runOnVm.mjs +++ b/packages/@n8n/benchmark/scripts/runOnVm/runOnVm.mjs @@ -21,6 +21,7 @@ async function main() { const composeFilePath = path.join(__dirname, 'n8nSetups', n8nSetupToUse); const n8nTag = argv.n8nDockerTag || process.env.N8N_DOCKER_TAG || 'latest'; const benchmarkTag = argv.benchmarkDockerTag || process.env.BENCHMARK_DOCKER_TAG || 'latest'; + const k6ApiToken = argv.k6ApiToken || process.env.K6_API_TOKEN || undefined; const $$ = $({ cwd: composeFilePath, @@ -28,6 +29,7 @@ async function main() { env: { N8N_VERSION: n8nTag, BENCHMARK_VERSION: benchmarkTag, + K6_API_TOKEN: k6ApiToken, }, }); diff --git a/packages/@n8n/benchmark/src/commands/run.ts b/packages/@n8n/benchmark/src/commands/run.ts index d69b4a54d4..3d9a3bf803 100644 --- a/packages/@n8n/benchmark/src/commands/run.ts +++ b/packages/@n8n/benchmark/src/commands/run.ts @@ -25,7 +25,11 @@ export default class RunCommand extends Command { const scenarioRunner = new ScenarioRunner( new N8nApiClient(config.get('n8n.baseUrl')), new ScenarioDataFileLoader(), - new K6Executor(config.get('k6ExecutablePath'), config.get('n8n.baseUrl')), + new K6Executor({ + k6ExecutablePath: config.get('k6.executablePath'), + k6ApiToken: config.get('k6.apiToken'), + n8nApiBaseUrl: config.get('n8n.baseUrl'), + }), { email: config.get('n8n.user.email'), password: config.get('n8n.user.password'), diff --git a/packages/@n8n/benchmark/src/config/config.ts b/packages/@n8n/benchmark/src/config/config.ts index 896ecc9296..2fa8d9249b 100644 --- a/packages/@n8n/benchmark/src/config/config.ts +++ b/packages/@n8n/benchmark/src/config/config.ts @@ -31,11 +31,19 @@ const configSchema = { }, }, }, - k6ExecutablePath: { - doc: 'The path to the k6 binary', - format: String, - default: 'k6', - env: 'K6_PATH', + k6: { + executablePath: { + doc: 'The path to the k6 binary', + format: String, + default: 'k6', + env: 'K6_PATH', + }, + apiToken: { + doc: 'The API token for k6 cloud', + format: String, + default: undefined, + env: 'K6_API_TOKEN', + }, }, }; diff --git a/packages/@n8n/benchmark/src/testExecution/k6Executor.ts b/packages/@n8n/benchmark/src/testExecution/k6Executor.ts index 176f1c15bb..a9ab325b5f 100644 --- a/packages/@n8n/benchmark/src/testExecution/k6Executor.ts +++ b/packages/@n8n/benchmark/src/testExecution/k6Executor.ts @@ -1,38 +1,103 @@ -import { $, which } from 'zx'; +import fs from 'fs'; +import path from 'path'; +import { $, which, tmpfile } from 'zx'; import type { Scenario } from '@/types/scenario'; +export type K6ExecutorOpts = { + k6ExecutablePath: string; + k6ApiToken?: string; + n8nApiBaseUrl: string; +}; + +/** + * Flag for the k6 CLI. + * @example ['--duration', '1m'] + * @example ['--quiet'] + */ +type K6CliFlag = [string] | [string, string]; + /** * Executes test scenarios using k6 */ export class K6Executor { - constructor( - private readonly k6ExecutablePath: string, - private readonly n8nApiBaseUrl: string, - ) {} + /** + * This script is dynamically injected into the k6 test script to generate + * a summary report of the test execution. + */ + private readonly handleSummaryScript = ` +import { textSummary } from 'https://jslib.k6.io/k6-summary/0.0.2/index.js'; +export function handleSummary(data) { + return { + stdout: textSummary(data), + '{{scenarioName}}.summary.json': JSON.stringify(data), + }; +} +`; + + constructor(private readonly opts: K6ExecutorOpts) {} async executeTestScenario(scenario: Scenario) { - // For 1 min with 5 virtual users - const stage = '1m:5'; + const augmentedTestScriptPath = this.augmentSummaryScript(scenario); + const runDirPath = path.dirname(augmentedTestScriptPath); + + const flags: K6CliFlag[] = [['--quiet'], ['--duration', '1m'], ['--vus', '5']]; + + if (this.opts.k6ApiToken) { + flags.push(['--out', 'cloud']); + } + + const flattedFlags = flags.flat(2); const k6ExecutablePath = await this.resolveK6ExecutablePath(); const processPromise = $({ - cwd: scenario.scenarioDirPath, + cwd: runDirPath, env: { - API_BASE_URL: this.n8nApiBaseUrl, + API_BASE_URL: this.opts.n8nApiBaseUrl, + K6_CLOUD_TOKEN: this.opts.k6ApiToken, }, - })`${k6ExecutablePath} run --quiet --stage ${stage} ${scenario.scriptPath}`; + })`${k6ExecutablePath} run ${flattedFlags} ${augmentedTestScriptPath}`; for await (const chunk of processPromise.stdout) { console.log((chunk as Buffer).toString()); } + + this.loadEndOfTestSummary(runDirPath, scenario.name); + } + + /** + * Augments the test script with a summary script + * + * @returns Absolute path to the augmented test script + */ + private augmentSummaryScript(scenario: Scenario) { + const fullTestScriptPath = path.join(scenario.scenarioDirPath, scenario.scriptPath); + const testScript = fs.readFileSync(fullTestScriptPath, 'utf8'); + const summaryScript = this.handleSummaryScript.replace('{{scenarioName}}', scenario.name); + + const augmentedTestScript = `${testScript}\n\n${summaryScript}`; + + const tempFilePath = tmpfile(`${scenario.name}.ts`, augmentedTestScript); + + return tempFilePath; + } + + private loadEndOfTestSummary(dir: string, scenarioName: string): K6EndOfTestSummary { + const summaryReportPath = path.join(dir, `${scenarioName}.summary.json`); + const summaryReport = fs.readFileSync(summaryReportPath, 'utf8'); + + try { + return JSON.parse(summaryReport); + } catch (error) { + throw new Error(`Failed to parse the summary report at ${summaryReportPath}`); + } } /** * @returns Resolved path to the k6 executable */ private async resolveK6ExecutablePath(): Promise { - const k6ExecutablePath = await which(this.k6ExecutablePath, { nothrow: true }); + const k6ExecutablePath = await which(this.opts.k6ExecutablePath, { nothrow: true }); if (!k6ExecutablePath) { throw new Error( 'Could not find k6 executable based on your `PATH`. Please ensure k6 is available in your system and add it to your `PATH` or specify the path to the k6 executable using the `K6_PATH` environment variable.', diff --git a/packages/@n8n/benchmark/src/testExecution/k6Summary.ts b/packages/@n8n/benchmark/src/testExecution/k6Summary.ts new file mode 100644 index 0000000000..c40b0cae97 --- /dev/null +++ b/packages/@n8n/benchmark/src/testExecution/k6Summary.ts @@ -0,0 +1,255 @@ +/** +Example JSON: + +{ + "options": { + "summaryTrendStats": ["avg", "min", "med", "max", "p(90)", "p(95)"], + "summaryTimeUnit": "", + "noColor": false + }, + "state": { "isStdOutTTY": false, "isStdErrTTY": false, "testRunDurationMs": 23.374 }, + "metrics": { + "http_req_tls_handshaking": { + "type": "trend", + "contains": "time", + "values": { "avg": 0, "min": 0, "med": 0, "max": 0, "p(90)": 0, "p(95)": 0 } + }, + "checks": { + "type": "rate", + "contains": "default", + "values": { "rate": 1, "passes": 1, "fails": 0 } + }, + "http_req_sending": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 0.512, + "p(95)": 0.512, + "avg": 0.512, + "min": 0.512, + "med": 0.512, + "max": 0.512 + } + }, + "http_reqs": { + "contains": "default", + "values": { "count": 1, "rate": 42.78257893385813 }, + "type": "counter" + }, + "http_req_blocked": { + "contains": "time", + "values": { + "avg": 1.496, + "min": 1.496, + "med": 1.496, + "max": 1.496, + "p(90)": 1.496, + "p(95)": 1.496 + }, + "type": "trend" + }, + "data_received": { + "type": "counter", + "contains": "data", + "values": { "count": 269, "rate": 11508.513733207838 } + }, + "iterations": { + "type": "counter", + "contains": "default", + "values": { "count": 1, "rate": 42.78257893385813 } + }, + "http_req_waiting": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 18.443, + "avg": 18.443, + "min": 18.443, + "med": 18.443, + "max": 18.443, + "p(90)": 18.443 + } + }, + "http_req_receiving": { + "type": "trend", + "contains": "time", + "values": { + "avg": 0.186, + "min": 0.186, + "med": 0.186, + "max": 0.186, + "p(90)": 0.186, + "p(95)": 0.186 + } + }, + "http_req_duration{expected_response:true}": { + "type": "trend", + "contains": "time", + "values": { + "max": 19.141, + "p(90)": 19.141, + "p(95)": 19.141, + "avg": 19.141, + "min": 19.141, + "med": 19.141 + } + }, + "iteration_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 22.577833, + "min": 22.577833, + "med": 22.577833, + "max": 22.577833, + "p(90)": 22.577833, + "p(95)": 22.577833 + } + }, + "http_req_connecting": { + "type": "trend", + "contains": "time", + "values": { + "avg": 0.673, + "min": 0.673, + "med": 0.673, + "max": 0.673, + "p(90)": 0.673, + "p(95)": 0.673 + } + }, + "http_req_failed": { + "type": "rate", + "contains": "default", + "values": { "rate": 0, "passes": 0, "fails": 1 } + }, + "http_req_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 19.141, + "p(95)": 19.141, + "avg": 19.141, + "min": 19.141, + "med": 19.141, + "max": 19.141 + } + }, + "data_sent": { + "type": "counter", + "contains": "data", + "values": { "count": 102, "rate": 4363.82305125353 } + } + }, + "root_group": { + "name": "", + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e", + "groups": [], + "checks": [ + { + "name": "is status 200", + "path": "::is status 200", + "id": "548d37ca5f33793206f7832e7cea54fb", + "passes": 1, + "fails": 0 + } + ] + } +} + */ + +type TrendStat = 'avg' | 'min' | 'med' | 'max' | 'p(90)' | 'p(95)'; +type MetricType = 'trend' | 'rate' | 'counter'; +type MetricContains = 'time' | 'default' | 'data'; + +interface TrendValues { + avg: number; + min: number; + med: number; + max: number; + 'p(90)': number; + 'p(95)': number; +} + +interface RateValues { + rate: number; + passes: number; + fails: number; +} + +interface CounterValues { + count: number; + rate: number; +} + +interface TrendMetric { + type: 'trend'; + contains: 'time'; + values: TrendValues; +} + +interface RateMetric { + type: 'rate'; + contains: 'default'; + values: RateValues; +} + +interface CounterMetric { + type: 'counter'; + contains: MetricContains; + values: CounterValues; +} + +interface Options { + summaryTrendStats: TrendStat[]; + summaryTimeUnit: string; + noColor: boolean; +} + +interface State { + isStdOutTTY: boolean; + isStdErrTTY: boolean; + testRunDurationMs: number; +} + +interface Metrics { + http_req_tls_handshaking: TrendMetric; + checks: RateMetric; + http_req_sending: TrendMetric; + http_reqs: CounterMetric; + http_req_blocked: TrendMetric; + data_received: CounterMetric; + iterations: CounterMetric; + http_req_waiting: TrendMetric; + http_req_receiving: TrendMetric; + 'http_req_duration{expected_response:true}': TrendMetric; + iteration_duration: TrendMetric; + http_req_connecting: TrendMetric; + http_req_failed: RateMetric; + http_req_duration: TrendMetric; + data_sent: CounterMetric; +} + +interface Check { + name: string; + path: string; + id: string; + passes: number; + fails: number; +} + +interface RootGroup { + name: string; + path: string; + id: string; + groups: any[]; + checks: Check[]; +} + +interface K6EndOfTestSummary { + options: Options; + state: State; + metrics: Metrics; + root_group: RootGroup; +}