Consolidate logic for running randomized tests in scripts

script/randomized-test-ci

@@ -1,50 +1,63 @@
-#!/bin/bash
+#!/usr/bin/env node --redirect-warnings=/dev/null
 
-set -u
+const fs = require('fs')
+const {randomBytes} = require('crypto')
+const {execFileSync} = require('child_process')
+const {minimizeTestPlan, buildTests, runTests} = require('./randomized-test-minimize');
 
-: $ZED_SERVER_URL
-: $ZED_CLIENT_SECRET_TOKEN
+const {ZED_SERVER_URL, ZED_CLIENT_SECRET_TOKEN} = process.env
+if (!ZED_SERVER_URL) throw new Error('Missing env var `ZED_SERVER_URL`')
+if (!ZED_CLIENT_SECRET_TOKEN) throw new Error('Missing env var `ZED_CLIENT_SECRET_TOKEN`')
 
-# Compile the tests first
-mkdir -p target
-cargo test --release --lib --package collab --no-run
-if [[ $? != 0 ]]; then
-  echo "Build failed"
-  exit 1
-fi
+main()
 
-LOG_FILE=target/randomized-tests.log
-MIN_PLAN=target/test-plan.min.json
-export SAVE_PLAN=target/test-plan.json
-export OPERATIONS=200
-export ITERATIONS=100000
-export SEED=$(od -A n -N 8 -t u8 /dev/urandom | xargs)
+async function main() {
+  buildTests()
 
-echo "Starting seed: ${SEED}"
+  const seed = randomU64();
+  const commit = execFileSync(
+    'git',
+    ['rev-parse', 'HEAD'],
+    {encoding: 'utf8'}
+  ).trim()
 
-cargo test --release --lib --package collab random 2>&1 > $LOG_FILE
-if [[ $? == 0 ]]; then
-  echo "Tests passed"
-  exit 0
-fi
+  console.log("commit:", commit)
+  console.log("starting seed:", seed)
 
-failing_seed=$(script/randomized-test-minimize $SAVE_PLAN $MIN_PLAN)
+  const planPath = 'target/test-plan.json'
+  const minPlanPath = 'target/test-plan.min.json'
+  const failingSeed = runTests({
+    SEED: seed,
+    SAVE_PLAN: planPath,
+    ITERATIONS: 50000,
+    OPERATIONS: 200,
+  })
 
-# If the tests failed, find the failing seed in the logs
-commit=$(git rev-parse HEAD)
-failing_plan=$(cat $MIN_PLAN)
-request="{
-  \"seed\": \"${failing_seed}\",
-  \"commit\": \"${commit}\",
-  \"token\": \"${ZED_CLIENT_SECRET_TOKEN}\",
-  \"plan\": ${failing_plan}
-}"
+  if (!failingSeed) {
+    console.log("tests passed")
+    return
+  }
 
-echo "Reporting test failure."
-echo $request
+  console.log("found failure at seed", failingSeed)
+  const minimizedSeed = minimizeTestPlan(planPath, minPlanPath)
+  const minimizedPlan = JSON.parse(fs.readFileSync(minPlanPath, 'utf8'))
 
-curl \
-    -X POST \
-    -H "Content-Type: application/json" \
-    -d "${request}" \
-    "${ZED_SERVER_URL}/api/randomized_test_failure"
+  const url = `${ZED_SERVER_URL}/api/randomized_test_failure`
+  const body = {
+    seed: minimizedSeed,
+    token: ZED_CLIENT_SECRET_TOKEN,
+    plan: minimizedPlan,
+    commit: commit,
+  }
+  await fetch(url, {
+    method: 'POST',
+    headers: {"Content-Type": "application/json"},
+    body: JSON.stringify(body)
+  })
+}
+
+function randomU64() {
+  const bytes = randomBytes(8)
+  const hexString = bytes.reduce(((string, byte) => string + byte.toString(16)), '')
+  return BigInt('0x' + hexString).toString(10)
+}

script/randomized-test-minimize

@@ -4,19 +4,34 @@ const fs = require('fs')
 const path = require('path')
 const {spawnSync} = require('child_process')
 
+const FAILING_SEED_REGEX = /failing seed: (\d+)/ig
+const CARGO_TEST_ARGS = [
+  '--release',
+  '--lib',
+  '--package', 'collab',
+  'random_collaboration',
+]
+
+if (require.main === module) {
   if (process.argv.length < 4) {
     process.stderr.write("usage: script/randomized-test-minimize <input-plan> <output-plan> [start-index]\n")
     process.exit(1)
   }
 
-const inputPlanPath = process.argv[2]
-const outputPlanPath = process.argv[3]
-const startIndex = parseInt(process.argv[4]) || 0
+  minimizeTestPlan(
+    process.argv[2],
+    process.argv[3],
+    parseInt(process.argv[4]) || 0
+  );
+}
 
+function minimizeTestPlan(
+  inputPlanPath,
+  outputPlanPath,
+  startIndex = 0
+) {
   const tempPlanPath = inputPlanPath + '.try'
 
-const FAILING_SEED_REGEX = /failing seed: (\d+)/ig
-
   fs.copyFileSync(inputPlanPath, outputPlanPath)
   let testPlan = JSON.parse(fs.readFileSync(outputPlanPath, 'utf8'))
 
@@ -33,8 +48,12 @@ for (let ix = startIndex; ix < testPlan.length; ix++) {
     fs.writeFileSync(tempPlanPath, serializeTestPlan(newTestPlan), 'utf8');
 
     process.stderr.write(`${ix}/${testPlan.length}: ${JSON.stringify(testPlan[ix])}`)
-
-  const failingSeed = runTestsForPlan(tempPlanPath, 500)
+    const failingSeed = runTests({
+      SEED: '0',
+      LOAD_PLAN: tempPlanPath,
+      SAVE_PLAN: tempPlanPath,
+      ITERATIONS: '500'
+    })
 
     // If the test failed, keep the test plan with the removed row. Reload the test
     // plan from the JSON file, since the test itself will remove any operations
@@ -55,34 +74,39 @@ fs.unlinkSync(tempPlanPath)
   // This is a workaround for the fact that the execution order can
   // slightly change when replaying a test plan after it has been
   // saved and loaded.
-const failingSeed = runTestsForPlan(outputPlanPath, 5000)
+  const failingSeed = runTests({
+    SEED: '0',
+    ITERATIONS: '5000',
+    LOAD_PLAN: outputPlanPath,
+  })
 
   process.stderr.write(`final test plan: ${outputPlanPath}\n`)
   process.stderr.write(`final seed: ${failingSeed}\n`)
-console.log(failingSeed)
+  return failingSeed
+}
 
-function runTestsForPlan(path, iterations) {
-  const {status, stdout, stderr} = spawnSync(
-    'cargo',
-    [
-      'test',
-      '--release',
-      '--lib',
-      '--package', 'collab',
-      'random_collaboration'
-    ],
-    {
+function buildTests() {
+  const {status} = spawnSync('cargo', ['test', '--no-run', ...CARGO_TEST_ARGS], {
+    stdio: 'inherit',
+    encoding: 'utf8',
+    env: {
+      ...process.env,
+    }
+  });
+  if (status !== 0) {
+    throw new Error('build failed')
+  }
+}
+
+function runTests(env) {
+  const {status, stdout} = spawnSync('cargo', ['test', ...CARGO_TEST_ARGS], {
     stdio: 'pipe',
     encoding: 'utf8',
     env: {
       ...process.env,
-        'SEED': 0,
-        'LOAD_PLAN': path,
-        'SAVE_PLAN': path,
-        'ITERATIONS': String(iterations),
+      ...env,
     }
-    }
-  );
+  });
 
   if (status !== 0) {
     FAILING_SEED_REGEX.lastIndex = 0
@@ -102,3 +126,7 @@ function runTestsForPlan(path, iterations) {
 function serializeTestPlan(plan) {
   return "[\n" + plan.map(row => JSON.stringify(row)).join(",\n") + "\n]\n"
 }
+
+exports.buildTests = buildTests
+exports.runTests = runTests
+exports.minimizeTestPlan = minimizeTestPlan