Minimize randomized test failures before reporting issues

script/randomized-test-ci

@@ -14,6 +14,7 @@ if [[ $? != 0 ]]; then
 fi
 
 LOG_FILE=target/randomized-tests.log
+MIN_PLAN=target/test-plan.min.json
 export SAVE_PLAN=target/test-plan.json
 export OPERATIONS=200
 export ITERATIONS=100000
@@ -27,10 +28,11 @@ if [[ $? == 0 ]]; then
   exit 0
 fi
 
+failing_seed=$(script/randomized-test-minimize $SAVE_PLAN $MIN_PLAN)
+
 # If the tests failed, find the failing seed in the logs
 commit=$(git rev-parse HEAD)
-failing_seed=$(grep "failing seed" $LOG_FILE | tail -n1 | cut -d: -f2 | xargs)
-failing_plan=$(cat $SAVE_PLAN)
+failing_plan=$(cat $MIN_PLAN)
 request="{
   \"seed\": \"${failing_seed}\",
   \"commit\": \"${commit}\",

script/randomized-test-minimize

@@ -0,0 +1,104 @@
+#!/usr/bin/env node --redirect-warnings=/dev/null
+
+const fs = require('fs')
+const path = require('path')
+const {spawnSync} = require('child_process')
+
+if (process.argv.length < 4) {
+  process.stderr.write("usage: script/randomized-test-minimize <input-plan> <output-plan> [start-index]\n")
+  process.exit(1)
+}
+
+const inputPlanPath = process.argv[2]
+const outputPlanPath = process.argv[3]
+const startIndex = parseInt(process.argv[4]) || 0
+
+const tempPlanPath = inputPlanPath + '.try'
+
+const FAILING_SEED_REGEX = /failing seed: (\d+)/ig
+
+fs.copyFileSync(inputPlanPath, outputPlanPath)
+let testPlan = JSON.parse(fs.readFileSync(outputPlanPath, 'utf8'))
+
+process.stderr.write("minimizing failing test plan...\n")
+for (let ix = startIndex; ix < testPlan.length; ix++) {
+  // Skip 'MutateClients' entries, since they themselves are not single operations.
+  if (testPlan[ix].MutateClients) {
+    continue
+  }
+
+  // Remove a row from the test plan
+  const newTestPlan = testPlan.slice()
+  newTestPlan.splice(ix, 1)
+  fs.writeFileSync(tempPlanPath, serializeTestPlan(newTestPlan), 'utf8');
+
+  process.stderr.write(`${ix}/${testPlan.length}: ${JSON.stringify(testPlan[ix])}`)
+
+  const failingSeed = runTestsForPlan(tempPlanPath, 500)
+
+  // If the test failed, keep the test plan with the removed row. Reload the test
+  // plan from the JSON file, since the test itself will remove any operations
+  // which are no longer valid before saving the test plan.
+  if (failingSeed != null) {
+    process.stderr.write(` - remove. failing seed: ${failingSeed}.\n`)
+    fs.copyFileSync(tempPlanPath, outputPlanPath)
+    testPlan = JSON.parse(fs.readFileSync(outputPlanPath, 'utf8'))
+    ix--
+  } else {
+    process.stderr.write(` - keep.\n`)
+  }
+}
+
+fs.unlinkSync(tempPlanPath)
+
+// Re-run the final minimized plan to get the correct failing seed.
+// This is a workaround for the fact that the execution order can
+// slightly change when replaying a test plan after it has been
+// saved and loaded.
+const failingSeed = runTestsForPlan(outputPlanPath, 5000)
+
+process.stderr.write(`final test plan: ${outputPlanPath}\n`)
+process.stderr.write(`final seed: ${failingSeed}\n`)
+console.log(failingSeed)
+
+function runTestsForPlan(path, iterations) {
+  const {status, stdout, stderr} = spawnSync(
+    'cargo',
+    [
+      'test',
+      '--release',
+      '--lib',
+      '--package', 'collab',
+      'random_collaboration'
+    ],
+    {
+      stdio: 'pipe',
+      encoding: 'utf8',
+      env: {
+        ...process.env,
+        'SEED': 0,
+        'LOAD_PLAN': path,
+        'SAVE_PLAN': path,
+        'ITERATIONS': String(iterations),
+      }
+    }
+  );
+
+  if (status !== 0) {
+    FAILING_SEED_REGEX.lastIndex = 0
+    const match = FAILING_SEED_REGEX.exec(stdout)
+    if (!match) {
+      process.stderr.write("test failed, but no failing seed found:\n")
+      process.stderr.write(stdout)
+      process.stderr.write('\n')
+      process.exit(1)
+    }
+    return match[1]
+  } else {
+    return null
+  }
+}
+
+function serializeTestPlan(plan) {
+  return "[\n" + plan.map(row => JSON.stringify(row)).join(",\n") + "\n]\n"
+}