From 837866f9622069a8c3196bbfaa45000ab43f25df Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 17 Apr 2023 15:34:23 -0700 Subject: [PATCH] Consolidate logic for running randomized tests in scripts --- script/randomized-test-ci | 93 ++++++++++-------- script/randomized-test-minimize | 164 +++++++++++++++++++------------- 2 files changed, 149 insertions(+), 108 deletions(-) diff --git a/script/randomized-test-ci b/script/randomized-test-ci index 3241bbcca0..31fdea6aee 100755 --- a/script/randomized-test-ci +++ b/script/randomized-test-ci @@ -1,50 +1,63 @@ -#!/bin/bash +#!/usr/bin/env node --redirect-warnings=/dev/null -set -u +const fs = require('fs') +const {randomBytes} = require('crypto') +const {execFileSync} = require('child_process') +const {minimizeTestPlan, buildTests, runTests} = require('./randomized-test-minimize'); -: $ZED_SERVER_URL -: $ZED_CLIENT_SECRET_TOKEN +const {ZED_SERVER_URL, ZED_CLIENT_SECRET_TOKEN} = process.env +if (!ZED_SERVER_URL) throw new Error('Missing env var `ZED_SERVER_URL`') +if (!ZED_CLIENT_SECRET_TOKEN) throw new Error('Missing env var `ZED_CLIENT_SECRET_TOKEN`') -# Compile the tests first -mkdir -p target -cargo test --release --lib --package collab --no-run -if [[ $? != 0 ]]; then - echo "Build failed" - exit 1 -fi +main() -LOG_FILE=target/randomized-tests.log -MIN_PLAN=target/test-plan.min.json -export SAVE_PLAN=target/test-plan.json -export OPERATIONS=200 -export ITERATIONS=100000 -export SEED=$(od -A n -N 8 -t u8 /dev/urandom | xargs) +async function main() { + buildTests() -echo "Starting seed: ${SEED}" + const seed = randomU64(); + const commit = execFileSync( + 'git', + ['rev-parse', 'HEAD'], + {encoding: 'utf8'} + ).trim() -cargo test --release --lib --package collab random 2>&1 > $LOG_FILE -if [[ $? == 0 ]]; then - echo "Tests passed" - exit 0 -fi + console.log("commit:", commit) + console.log("starting seed:", seed) -failing_seed=$(script/randomized-test-minimize $SAVE_PLAN $MIN_PLAN) + const planPath = 'target/test-plan.json' + const minPlanPath = 'target/test-plan.min.json' + const failingSeed = runTests({ + SEED: seed, + SAVE_PLAN: planPath, + ITERATIONS: 50000, + OPERATIONS: 200, + }) -# If the tests failed, find the failing seed in the logs -commit=$(git rev-parse HEAD) -failing_plan=$(cat $MIN_PLAN) -request="{ - \"seed\": \"${failing_seed}\", - \"commit\": \"${commit}\", - \"token\": \"${ZED_CLIENT_SECRET_TOKEN}\", - \"plan\": ${failing_plan} -}" + if (!failingSeed) { + console.log("tests passed") + return + } -echo "Reporting test failure." -echo $request + console.log("found failure at seed", failingSeed) + const minimizedSeed = minimizeTestPlan(planPath, minPlanPath) + const minimizedPlan = JSON.parse(fs.readFileSync(minPlanPath, 'utf8')) -curl \ - -X POST \ - -H "Content-Type: application/json" \ - -d "${request}" \ - "${ZED_SERVER_URL}/api/randomized_test_failure" + const url = `${ZED_SERVER_URL}/api/randomized_test_failure` + const body = { + seed: minimizedSeed, + token: ZED_CLIENT_SECRET_TOKEN, + plan: minimizedPlan, + commit: commit, + } + await fetch(url, { + method: 'POST', + headers: {"Content-Type": "application/json"}, + body: JSON.stringify(body) + }) +} + +function randomU64() { + const bytes = randomBytes(8) + const hexString = bytes.reduce(((string, byte) => string + byte.toString(16)), '') + return BigInt('0x' + hexString).toString(10) +} diff --git a/script/randomized-test-minimize b/script/randomized-test-minimize index fe2686b69b..ce0b7203b4 100755 --- a/script/randomized-test-minimize +++ b/script/randomized-test-minimize @@ -4,85 +4,109 @@ const fs = require('fs') const path = require('path') const {spawnSync} = require('child_process') -if (process.argv.length < 4) { - process.stderr.write("usage: script/randomized-test-minimize [start-index]\n") - process.exit(1) -} - -const inputPlanPath = process.argv[2] -const outputPlanPath = process.argv[3] -const startIndex = parseInt(process.argv[4]) || 0 - -const tempPlanPath = inputPlanPath + '.try' - const FAILING_SEED_REGEX = /failing seed: (\d+)/ig +const CARGO_TEST_ARGS = [ + '--release', + '--lib', + '--package', 'collab', + 'random_collaboration', +] -fs.copyFileSync(inputPlanPath, outputPlanPath) -let testPlan = JSON.parse(fs.readFileSync(outputPlanPath, 'utf8')) - -process.stderr.write("minimizing failing test plan...\n") -for (let ix = startIndex; ix < testPlan.length; ix++) { - // Skip 'MutateClients' entries, since they themselves are not single operations. - if (testPlan[ix].MutateClients) { - continue +if (require.main === module) { + if (process.argv.length < 4) { + process.stderr.write("usage: script/randomized-test-minimize [start-index]\n") + process.exit(1) } - // Remove a row from the test plan - const newTestPlan = testPlan.slice() - newTestPlan.splice(ix, 1) - fs.writeFileSync(tempPlanPath, serializeTestPlan(newTestPlan), 'utf8'); + minimizeTestPlan( + process.argv[2], + process.argv[3], + parseInt(process.argv[4]) || 0 + ); +} - process.stderr.write(`${ix}/${testPlan.length}: ${JSON.stringify(testPlan[ix])}`) +function minimizeTestPlan( + inputPlanPath, + outputPlanPath, + startIndex = 0 +) { + const tempPlanPath = inputPlanPath + '.try' - const failingSeed = runTestsForPlan(tempPlanPath, 500) + fs.copyFileSync(inputPlanPath, outputPlanPath) + let testPlan = JSON.parse(fs.readFileSync(outputPlanPath, 'utf8')) - // If the test failed, keep the test plan with the removed row. Reload the test - // plan from the JSON file, since the test itself will remove any operations - // which are no longer valid before saving the test plan. - if (failingSeed != null) { - process.stderr.write(` - remove. failing seed: ${failingSeed}.\n`) - fs.copyFileSync(tempPlanPath, outputPlanPath) - testPlan = JSON.parse(fs.readFileSync(outputPlanPath, 'utf8')) - ix-- - } else { - process.stderr.write(` - keep.\n`) + process.stderr.write("minimizing failing test plan...\n") + for (let ix = startIndex; ix < testPlan.length; ix++) { + // Skip 'MutateClients' entries, since they themselves are not single operations. + if (testPlan[ix].MutateClients) { + continue + } + + // Remove a row from the test plan + const newTestPlan = testPlan.slice() + newTestPlan.splice(ix, 1) + fs.writeFileSync(tempPlanPath, serializeTestPlan(newTestPlan), 'utf8'); + + process.stderr.write(`${ix}/${testPlan.length}: ${JSON.stringify(testPlan[ix])}`) + const failingSeed = runTests({ + SEED: '0', + LOAD_PLAN: tempPlanPath, + SAVE_PLAN: tempPlanPath, + ITERATIONS: '500' + }) + + // If the test failed, keep the test plan with the removed row. Reload the test + // plan from the JSON file, since the test itself will remove any operations + // which are no longer valid before saving the test plan. + if (failingSeed != null) { + process.stderr.write(` - remove. failing seed: ${failingSeed}.\n`) + fs.copyFileSync(tempPlanPath, outputPlanPath) + testPlan = JSON.parse(fs.readFileSync(outputPlanPath, 'utf8')) + ix-- + } else { + process.stderr.write(` - keep.\n`) + } + } + + fs.unlinkSync(tempPlanPath) + + // Re-run the final minimized plan to get the correct failing seed. + // This is a workaround for the fact that the execution order can + // slightly change when replaying a test plan after it has been + // saved and loaded. + const failingSeed = runTests({ + SEED: '0', + ITERATIONS: '5000', + LOAD_PLAN: outputPlanPath, + }) + + process.stderr.write(`final test plan: ${outputPlanPath}\n`) + process.stderr.write(`final seed: ${failingSeed}\n`) + return failingSeed +} + +function buildTests() { + const {status} = spawnSync('cargo', ['test', '--no-run', ...CARGO_TEST_ARGS], { + stdio: 'inherit', + encoding: 'utf8', + env: { + ...process.env, + } + }); + if (status !== 0) { + throw new Error('build failed') } } -fs.unlinkSync(tempPlanPath) - -// Re-run the final minimized plan to get the correct failing seed. -// This is a workaround for the fact that the execution order can -// slightly change when replaying a test plan after it has been -// saved and loaded. -const failingSeed = runTestsForPlan(outputPlanPath, 5000) - -process.stderr.write(`final test plan: ${outputPlanPath}\n`) -process.stderr.write(`final seed: ${failingSeed}\n`) -console.log(failingSeed) - -function runTestsForPlan(path, iterations) { - const {status, stdout, stderr} = spawnSync( - 'cargo', - [ - 'test', - '--release', - '--lib', - '--package', 'collab', - 'random_collaboration' - ], - { - stdio: 'pipe', - encoding: 'utf8', - env: { - ...process.env, - 'SEED': 0, - 'LOAD_PLAN': path, - 'SAVE_PLAN': path, - 'ITERATIONS': String(iterations), - } +function runTests(env) { + const {status, stdout} = spawnSync('cargo', ['test', ...CARGO_TEST_ARGS], { + stdio: 'pipe', + encoding: 'utf8', + env: { + ...process.env, + ...env, } - ); + }); if (status !== 0) { FAILING_SEED_REGEX.lastIndex = 0 @@ -102,3 +126,7 @@ function runTestsForPlan(path, iterations) { function serializeTestPlan(plan) { return "[\n" + plan.map(row => JSON.stringify(row)).join(",\n") + "\n]\n" } + +exports.buildTests = buildTests +exports.runTests = runTests +exports.minimizeTestPlan = minimizeTestPlan