#!/usr/bin/env node --redirect-warnings=/dev/null const fs = require('fs') const path = require('path') const {spawnSync} = require('child_process') const FAILING_SEED_REGEX = /failing seed: (\d+)/ig const CARGO_TEST_ARGS = [ '--release', '--lib', '--package', 'collab', 'random_collaboration', ] if (require.main === module) { if (process.argv.length < 4) { process.stderr.write("usage: script/randomized-test-minimize [start-index]\n") process.exit(1) } minimizeTestPlan( process.argv[2], process.argv[3], parseInt(process.argv[4]) || 0 ); } function minimizeTestPlan( inputPlanPath, outputPlanPath, startIndex = 0 ) { const tempPlanPath = inputPlanPath + '.try' fs.copyFileSync(inputPlanPath, outputPlanPath) let testPlan = JSON.parse(fs.readFileSync(outputPlanPath, 'utf8')) process.stderr.write("minimizing failing test plan...\n") for (let ix = startIndex; ix < testPlan.length; ix++) { // Skip 'MutateClients' entries, since they themselves are not single operations. if (testPlan[ix].MutateClients) { continue } // Remove a row from the test plan const newTestPlan = testPlan.slice() newTestPlan.splice(ix, 1) fs.writeFileSync(tempPlanPath, serializeTestPlan(newTestPlan), 'utf8'); process.stderr.write(`${ix}/${testPlan.length}: ${JSON.stringify(testPlan[ix])}`) const failingSeed = runTests({ SEED: '0', LOAD_PLAN: tempPlanPath, SAVE_PLAN: tempPlanPath, ITERATIONS: '500' }) // If the test failed, keep the test plan with the removed row. Reload the test // plan from the JSON file, since the test itself will remove any operations // which are no longer valid before saving the test plan. if (failingSeed != null) { process.stderr.write(` - remove. failing seed: ${failingSeed}.\n`) fs.copyFileSync(tempPlanPath, outputPlanPath) testPlan = JSON.parse(fs.readFileSync(outputPlanPath, 'utf8')) ix-- } else { process.stderr.write(` - keep.\n`) } } fs.unlinkSync(tempPlanPath) // Re-run the final minimized plan to get the correct failing seed. // This is a workaround for the fact that the execution order can // slightly change when replaying a test plan after it has been // saved and loaded. const failingSeed = runTests({ SEED: '0', ITERATIONS: '5000', LOAD_PLAN: outputPlanPath, }) process.stderr.write(`final test plan: ${outputPlanPath}\n`) process.stderr.write(`final seed: ${failingSeed}\n`) return failingSeed } function buildTests() { const {status} = spawnSync('cargo', ['test', '--no-run', ...CARGO_TEST_ARGS], { stdio: 'inherit', encoding: 'utf8', env: { ...process.env, } }); if (status !== 0) { throw new Error('build failed') } } function runTests(env) { const {status, stdout} = spawnSync('cargo', ['test', ...CARGO_TEST_ARGS], { stdio: 'pipe', encoding: 'utf8', env: { ...process.env, ...env, } }); if (status !== 0) { FAILING_SEED_REGEX.lastIndex = 0 const match = FAILING_SEED_REGEX.exec(stdout) if (!match) { process.stderr.write("test failed, but no failing seed found:\n") process.stderr.write(stdout) process.stderr.write('\n') process.exit(1) } return match[1] } else { return null } } function serializeTestPlan(plan) { return "[\n" + plan.map(row => JSON.stringify(row)).join(",\n") + "\n]\n" } exports.buildTests = buildTests exports.runTests = runTests exports.minimizeTestPlan = minimizeTestPlan