zed/script/randomized-test-minimize

#!/usr/bin/env node --redirect-warnings=/dev/null

const fs = require('fs')
const path = require('path')
const {spawnSync} = require('child_process')

if (process.argv.length < 4) {
  process.stderr.write("usage: script/randomized-test-minimize <input-plan> <output-plan> [start-index]\n")
  process.exit(1)
}

const inputPlanPath = process.argv[2]
const outputPlanPath = process.argv[3]
const startIndex = parseInt(process.argv[4]) || 0

const tempPlanPath = inputPlanPath + '.try'

const FAILING_SEED_REGEX = /failing seed: (\d+)/ig

fs.copyFileSync(inputPlanPath, outputPlanPath)
let testPlan = JSON.parse(fs.readFileSync(outputPlanPath, 'utf8'))

process.stderr.write("minimizing failing test plan...\n")
for (let ix = startIndex; ix < testPlan.length; ix++) {
  // Skip 'MutateClients' entries, since they themselves are not single operations.
  if (testPlan[ix].MutateClients) {
    continue
  }

  // Remove a row from the test plan
  const newTestPlan = testPlan.slice()
  newTestPlan.splice(ix, 1)
  fs.writeFileSync(tempPlanPath, serializeTestPlan(newTestPlan), 'utf8');

  process.stderr.write(`${ix}/${testPlan.length}: ${JSON.stringify(testPlan[ix])}`)

  const failingSeed = runTestsForPlan(tempPlanPath, 500)

  // If the test failed, keep the test plan with the removed row. Reload the test
  // plan from the JSON file, since the test itself will remove any operations
  // which are no longer valid before saving the test plan.
  if (failingSeed != null) {
    process.stderr.write(` - remove. failing seed: ${failingSeed}.\n`)
    fs.copyFileSync(tempPlanPath, outputPlanPath)
    testPlan = JSON.parse(fs.readFileSync(outputPlanPath, 'utf8'))
    ix--
  } else {
    process.stderr.write(` - keep.\n`)
  }
}

fs.unlinkSync(tempPlanPath)

// Re-run the final minimized plan to get the correct failing seed.
// This is a workaround for the fact that the execution order can
// slightly change when replaying a test plan after it has been
// saved and loaded.
const failingSeed = runTestsForPlan(outputPlanPath, 5000)

process.stderr.write(`final test plan: ${outputPlanPath}\n`)
process.stderr.write(`final seed: ${failingSeed}\n`)
console.log(failingSeed)

function runTestsForPlan(path, iterations) {
  const {status, stdout, stderr} = spawnSync(
    'cargo',
    [
      'test',
      '--release',
      '--lib',
      '--package', 'collab',
      'random_collaboration'
    ],
    {
      stdio: 'pipe',
      encoding: 'utf8',
      env: {
        ...process.env,
        'SEED': 0,
        'LOAD_PLAN': path,
        'SAVE_PLAN': path,
        'ITERATIONS': String(iterations),
      }
    }
  );

  if (status !== 0) {
    FAILING_SEED_REGEX.lastIndex = 0
    const match = FAILING_SEED_REGEX.exec(stdout)
    if (!match) {
      process.stderr.write("test failed, but no failing seed found:\n")
      process.stderr.write(stdout)
      process.stderr.write('\n')
      process.exit(1)
    }
    return match[1]
  } else {
    return null
  }
}

function serializeTestPlan(plan) {
  return "[\n" + plan.map(row => JSON.stringify(row)).join(",\n") + "\n]\n"
}