From 253411bfd0f7db89acbf18a2a0cbe58107566112 Mon Sep 17 00:00:00 2001
From: Max Brunsfeld <maxbrunsfeld@gmail.com>
Date: Fri, 14 Apr 2023 11:41:52 -0700
Subject: [PATCH 1/6] Start work randomized test runner GH action workflow

---
 .github/workflows/randomized_tests.yml | 51 ++++++++++++++++++++++++++
 1 file changed, 51 insertions(+)
 create mode 100644 .github/workflows/randomized_tests.yml

diff --git a/.github/workflows/randomized_tests.yml b/.github/workflows/randomized_tests.yml
new file mode 100644
index 0000000000..b9f5bbcbd3
--- /dev/null
+++ b/.github/workflows/randomized_tests.yml
@@ -0,0 +1,51 @@
+name: Randomized Tests
+
+concurrency: randomized-tests
+
+on:
+  push:
+    branches:
+      - main
+      - randomized-tests-runner
+  schedule:
+     - cron: '*/15 * * * *'
+
+env:
+  CARGO_TERM_COLOR: always
+  CARGO_INCREMENTAL: 0
+  RUST_BACKTRACE: 1
+  OPERATIONS: 200
+  ITERATIONS: 10000
+
+jobs:
+  tests:
+    name: Run randomized tests
+    runs-on:
+      - self-hosted
+      - randomized-tests
+    steps:
+      - name: Install Rust
+        run: |
+          rustup set profile minimal
+          rustup update stable
+
+      - name: Install Node
+        uses: actions/setup-node@v2
+        with:
+          node-version: '16'
+
+      - name: Checkout repo
+        uses: actions/checkout@v2
+        with:
+          clean: false
+          submodules: 'recursive'
+
+      - name: Select seed
+        run: |
+          set -eu
+          seed=$(od -A n -N 8 -t u8 /dev/urandom | xargs)
+          echo "seed: ${seed}"
+          echo "SEED=${seed}" >> $GITHUB_ENV
+
+      - name: Run tests
+        run: cargo test --release --package collab random

From c329546570afb29decbf14e0e74270f52dd9b307 Mon Sep 17 00:00:00 2001
From: Max Brunsfeld <maxbrunsfeld@gmail.com>
Date: Fri, 14 Apr 2023 14:25:55 -0700
Subject: [PATCH 2/6] Extract randomized test CI process into a script

---
 .github/workflows/randomized_tests.yml | 13 ++-----------
 crates/gpui/src/test.rs                |  2 +-
 script/randomized-test-ci              | 27 ++++++++++++++++++++++++++
 3 files changed, 30 insertions(+), 12 deletions(-)
 create mode 100755 script/randomized-test-ci

diff --git a/.github/workflows/randomized_tests.yml b/.github/workflows/randomized_tests.yml
index b9f5bbcbd3..3b3d3b21cd 100644
--- a/.github/workflows/randomized_tests.yml
+++ b/.github/workflows/randomized_tests.yml
@@ -14,8 +14,6 @@ env:
   CARGO_TERM_COLOR: always
   CARGO_INCREMENTAL: 0
   RUST_BACKTRACE: 1
-  OPERATIONS: 200
-  ITERATIONS: 10000
 
 jobs:
   tests:
@@ -40,12 +38,5 @@ jobs:
           clean: false
           submodules: 'recursive'
 
-      - name: Select seed
-        run: |
-          set -eu
-          seed=$(od -A n -N 8 -t u8 /dev/urandom | xargs)
-          echo "seed: ${seed}"
-          echo "SEED=${seed}" >> $GITHUB_ENV
-
-      - name: Run tests
-        run: cargo test --release --package collab random
+      - name: Run randomized tests
+        run: script/randomized-test-ci
diff --git a/crates/gpui/src/test.rs b/crates/gpui/src/test.rs
index 372131b06c..cacb8a435a 100644
--- a/crates/gpui/src/test.rs
+++ b/crates/gpui/src/test.rs
@@ -76,7 +76,7 @@ pub fn run_test(
                 let seed = atomic_seed.load(SeqCst);
 
                 if is_randomized {
-                    dbg!(seed);
+                    eprintln!("seed = {seed}");
                 }
 
                 let deterministic = executor::Deterministic::new(seed);
diff --git a/script/randomized-test-ci b/script/randomized-test-ci
new file mode 100755
index 0000000000..e8837d42b2
--- /dev/null
+++ b/script/randomized-test-ci
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# Compile the tests first
+mkdir -p target
+cargo test --release --package collab --no-run
+if [[ $? != 0 ]]; then
+  echo "Build failed"
+  exit 1
+fi
+
+set -eu
+
+LOG_FILE=target/randomized-tests.log
+export SAVE_PLAN=target/test-plan.json
+export OPERATIONS=200
+export ITERATIONS=10000
+export SEED=$(od -A n -N 8 -t u8 /dev/urandom | xargs)
+
+cargo test --release --package collab random -- --nocapture 2> >(tee $LOG_FILE)
+if [[ $? == 0 ]]; then
+  echo "Tests passed"
+  exit 0
+fi
+
+# If the tests failed, find the failing seed in the logs
+failing_seed=$(grep "failing seed" $LOG_FILE | cut -d: -f2 | xargs)
+echo "Tests failed. seed: $failing_seed"

From 5c3da91e15b052d64f786c0c28d796d13e9f2689 Mon Sep 17 00:00:00 2001
From: Max Brunsfeld <maxbrunsfeld@gmail.com>
Date: Fri, 14 Apr 2023 15:36:55 -0700
Subject: [PATCH 3/6] Report randomized test failures to zed.dev, to create
 issues in linear

---
 .github/workflows/randomized_tests.yml |  4 ++-
 script/randomized-test-ci              | 35 ++++++++++++++++++++------
 2 files changed, 31 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/randomized_tests.yml b/.github/workflows/randomized_tests.yml
index 3b3d3b21cd..f180313ed3 100644
--- a/.github/workflows/randomized_tests.yml
+++ b/.github/workflows/randomized_tests.yml
@@ -8,12 +8,14 @@ on:
       - main
       - randomized-tests-runner
   schedule:
-     - cron: '*/15 * * * *'
+     - cron: '0 * * * *'
 
 env:
   CARGO_TERM_COLOR: always
   CARGO_INCREMENTAL: 0
   RUST_BACKTRACE: 1
+  ZED_SERVER_URL: https://zed.dev
+  ZED_CLIENT_SECRET_TOKEN: ${{ secrets.ZED_CLIENT_SECRET_TOKEN }}
 
 jobs:
   tests:
diff --git a/script/randomized-test-ci b/script/randomized-test-ci
index e8837d42b2..556bc34b6b 100755
--- a/script/randomized-test-ci
+++ b/script/randomized-test-ci
@@ -1,27 +1,48 @@
 #!/bin/bash
 
+set -u
+
+: $ZED_SERVER_URL
+: $ZED_CLIENT_SECRET_TOKEN
+
 # Compile the tests first
 mkdir -p target
-cargo test --release --package collab --no-run
+cargo test --release --lib --package collab --no-run
 if [[ $? != 0 ]]; then
   echo "Build failed"
   exit 1
 fi
 
-set -eu
-
 LOG_FILE=target/randomized-tests.log
 export SAVE_PLAN=target/test-plan.json
 export OPERATIONS=200
-export ITERATIONS=10000
+export ITERATIONS=100000
 export SEED=$(od -A n -N 8 -t u8 /dev/urandom | xargs)
 
-cargo test --release --package collab random -- --nocapture 2> >(tee $LOG_FILE)
+echo "Starting seed: ${SEED}"
+
+cargo test --release --lib --package collab random 2>&1 > $LOG_FILE
 if [[ $? == 0 ]]; then
   echo "Tests passed"
   exit 0
 fi
 
 # If the tests failed, find the failing seed in the logs
-failing_seed=$(grep "failing seed" $LOG_FILE | cut -d: -f2 | xargs)
-echo "Tests failed. seed: $failing_seed"
+commit=$(git rev-parse HEAD)
+failing_seed=$(grep "failing seed" $LOG_FILE | tail -n1 | cut -d: -f2 | xargs)
+failing_plan=$(cat $SAVE_PLAN)
+request="{
+  \"seed\": \"${failing_seed}\",
+  \"commit\": \"${commit}\",
+  \"token\": \"${ZED_CLIENT_SECRET_TOKEN}\",
+  \"plan\": ${failing_plan}
+}"
+
+echo "Reporting test failure."
+echo $request
+
+curl \
+    -X POST \
+    -H "Content-Type: application/json" \
+    -d "${request}" \
+    "${ZED_SERVER_URL}/api/randomized_test_failure"

From 3569c617848c92612a7d1803637530aa79a983a3 Mon Sep 17 00:00:00 2001
From: Max Brunsfeld <maxbrunsfeld@gmail.com>
Date: Fri, 14 Apr 2023 17:47:00 -0700
Subject: [PATCH 4/6] Minimize randomized test failures before reporting issues

---
 script/randomized-test-ci       |   6 +-
 script/randomized-test-minimize | 104 ++++++++++++++++++++++++++++++++
 2 files changed, 108 insertions(+), 2 deletions(-)
 create mode 100755 script/randomized-test-minimize

diff --git a/script/randomized-test-ci b/script/randomized-test-ci
index 556bc34b6b..3241bbcca0 100755
--- a/script/randomized-test-ci
+++ b/script/randomized-test-ci
@@ -14,6 +14,7 @@ if [[ $? != 0 ]]; then
 fi
 
 LOG_FILE=target/randomized-tests.log
+MIN_PLAN=target/test-plan.min.json
 export SAVE_PLAN=target/test-plan.json
 export OPERATIONS=200
 export ITERATIONS=100000
@@ -27,10 +28,11 @@ if [[ $? == 0 ]]; then
   exit 0
 fi
 
+failing_seed=$(script/randomized-test-minimize $SAVE_PLAN $MIN_PLAN)
+
 # If the tests failed, find the failing seed in the logs
 commit=$(git rev-parse HEAD)
-failing_seed=$(grep "failing seed" $LOG_FILE | tail -n1 | cut -d: -f2 | xargs)
-failing_plan=$(cat $SAVE_PLAN)
+failing_plan=$(cat $MIN_PLAN)
 request="{
   \"seed\": \"${failing_seed}\",
   \"commit\": \"${commit}\",
diff --git a/script/randomized-test-minimize b/script/randomized-test-minimize
new file mode 100755
index 0000000000..fe2686b69b
--- /dev/null
+++ b/script/randomized-test-minimize
@@ -0,0 +1,104 @@
+#!/usr/bin/env node --redirect-warnings=/dev/null
+
+const fs = require('fs')
+const path = require('path')
+const {spawnSync} = require('child_process')
+
+if (process.argv.length < 4) {
+  process.stderr.write("usage: script/randomized-test-minimize <input-plan> <output-plan> [start-index]\n")
+  process.exit(1)
+}
+
+const inputPlanPath = process.argv[2]
+const outputPlanPath = process.argv[3]
+const startIndex = parseInt(process.argv[4]) || 0
+
+const tempPlanPath = inputPlanPath + '.try'
+
+const FAILING_SEED_REGEX = /failing seed: (\d+)/ig
+
+fs.copyFileSync(inputPlanPath, outputPlanPath)
+let testPlan = JSON.parse(fs.readFileSync(outputPlanPath, 'utf8'))
+
+process.stderr.write("minimizing failing test plan...\n")
+for (let ix = startIndex; ix < testPlan.length; ix++) {
+  // Skip 'MutateClients' entries, since they themselves are not single operations.
+  if (testPlan[ix].MutateClients) {
+    continue
+  }
+
+  // Remove a row from the test plan
+  const newTestPlan = testPlan.slice()
+  newTestPlan.splice(ix, 1)
+  fs.writeFileSync(tempPlanPath, serializeTestPlan(newTestPlan), 'utf8');
+
+  process.stderr.write(`${ix}/${testPlan.length}: ${JSON.stringify(testPlan[ix])}`)
+
+  const failingSeed = runTestsForPlan(tempPlanPath, 500)
+
+  // If the test failed, keep the test plan with the removed row. Reload the test
+  // plan from the JSON file, since the test itself will remove any operations
+  // which are no longer valid before saving the test plan.
+  if (failingSeed != null) {
+    process.stderr.write(` - remove. failing seed: ${failingSeed}.\n`)
+    fs.copyFileSync(tempPlanPath, outputPlanPath)
+    testPlan = JSON.parse(fs.readFileSync(outputPlanPath, 'utf8'))
+    ix--
+  } else {
+    process.stderr.write(` - keep.\n`)
+  }
+}
+
+fs.unlinkSync(tempPlanPath)
+
+// Re-run the final minimized plan to get the correct failing seed.
+// This is a workaround for the fact that the execution order can
+// slightly change when replaying a test plan after it has been
+// saved and loaded.
+const failingSeed = runTestsForPlan(outputPlanPath, 5000)
+
+process.stderr.write(`final test plan: ${outputPlanPath}\n`)
+process.stderr.write(`final seed: ${failingSeed}\n`)
+console.log(failingSeed)
+
+function runTestsForPlan(path, iterations) {
+  const {status, stdout, stderr} = spawnSync(
+    'cargo',
+    [
+      'test',
+      '--release',
+      '--lib',
+      '--package', 'collab',
+      'random_collaboration'
+    ],
+    {
+      stdio: 'pipe',
+      encoding: 'utf8',
+      env: {
+        ...process.env,
+        'SEED': 0,
+        'LOAD_PLAN': path,
+        'SAVE_PLAN': path,
+        'ITERATIONS': String(iterations),
+      }
+    }
+  );
+
+  if (status !== 0) {
+    FAILING_SEED_REGEX.lastIndex = 0
+    const match = FAILING_SEED_REGEX.exec(stdout)
+    if (!match) {
+      process.stderr.write("test failed, but no failing seed found:\n")
+      process.stderr.write(stdout)
+      process.stderr.write('\n')
+      process.exit(1)
+    }
+    return match[1]
+  } else {
+    return null
+  }
+}
+
+function serializeTestPlan(plan) {
+  return "[\n" + plan.map(row => JSON.stringify(row)).join(",\n") + "\n]\n"
+}

From 837866f9622069a8c3196bbfaa45000ab43f25df Mon Sep 17 00:00:00 2001
From: Max Brunsfeld <maxbrunsfeld@gmail.com>
Date: Mon, 17 Apr 2023 15:34:23 -0700
Subject: [PATCH 5/6] Consolidate logic for running randomized tests in scripts

---
 script/randomized-test-ci       |  93 ++++++++++--------
 script/randomized-test-minimize | 164 +++++++++++++++++++-------------
 2 files changed, 149 insertions(+), 108 deletions(-)

diff --git a/script/randomized-test-ci b/script/randomized-test-ci
index 3241bbcca0..31fdea6aee 100755
--- a/script/randomized-test-ci
+++ b/script/randomized-test-ci
@@ -1,50 +1,63 @@
-#!/bin/bash
+#!/usr/bin/env node --redirect-warnings=/dev/null
 
-set -u
+const fs = require('fs')
+const {randomBytes} = require('crypto')
+const {execFileSync} = require('child_process')
+const {minimizeTestPlan, buildTests, runTests} = require('./randomized-test-minimize');
 
-: $ZED_SERVER_URL
-: $ZED_CLIENT_SECRET_TOKEN
+const {ZED_SERVER_URL, ZED_CLIENT_SECRET_TOKEN} = process.env
+if (!ZED_SERVER_URL) throw new Error('Missing env var `ZED_SERVER_URL`')
+if (!ZED_CLIENT_SECRET_TOKEN) throw new Error('Missing env var `ZED_CLIENT_SECRET_TOKEN`')
 
-# Compile the tests first
-mkdir -p target
-cargo test --release --lib --package collab --no-run
-if [[ $? != 0 ]]; then
-  echo "Build failed"
-  exit 1
-fi
+main()
 
-LOG_FILE=target/randomized-tests.log
-MIN_PLAN=target/test-plan.min.json
-export SAVE_PLAN=target/test-plan.json
-export OPERATIONS=200
-export ITERATIONS=100000
-export SEED=$(od -A n -N 8 -t u8 /dev/urandom | xargs)
+async function main() {
+  buildTests()
 
-echo "Starting seed: ${SEED}"
+  const seed = randomU64();
+  const commit = execFileSync(
+    'git',
+    ['rev-parse', 'HEAD'],
+    {encoding: 'utf8'}
+  ).trim()
 
-cargo test --release --lib --package collab random 2>&1 > $LOG_FILE
-if [[ $? == 0 ]]; then
-  echo "Tests passed"
-  exit 0
-fi
+  console.log("commit:", commit)
+  console.log("starting seed:", seed)
 
-failing_seed=$(script/randomized-test-minimize $SAVE_PLAN $MIN_PLAN)
+  const planPath = 'target/test-plan.json'
+  const minPlanPath = 'target/test-plan.min.json'
+  const failingSeed = runTests({
+    SEED: seed,
+    SAVE_PLAN: planPath,
+    ITERATIONS: 50000,
+    OPERATIONS: 200,
+  })
 
-# If the tests failed, find the failing seed in the logs
-commit=$(git rev-parse HEAD)
-failing_plan=$(cat $MIN_PLAN)
-request="{
-  \"seed\": \"${failing_seed}\",
-  \"commit\": \"${commit}\",
-  \"token\": \"${ZED_CLIENT_SECRET_TOKEN}\",
-  \"plan\": ${failing_plan}
-}"
+  if (!failingSeed) {
+    console.log("tests passed")
+    return
+  }
 
-echo "Reporting test failure."
-echo $request
+  console.log("found failure at seed", failingSeed)
+  const minimizedSeed = minimizeTestPlan(planPath, minPlanPath)
+  const minimizedPlan = JSON.parse(fs.readFileSync(minPlanPath, 'utf8'))
 
-curl \
-    -X POST \
-    -H "Content-Type: application/json" \
-    -d "${request}" \
-    "${ZED_SERVER_URL}/api/randomized_test_failure"
+  const url = `${ZED_SERVER_URL}/api/randomized_test_failure`
+  const body = {
+    seed: minimizedSeed,
+    token: ZED_CLIENT_SECRET_TOKEN,
+    plan: minimizedPlan,
+    commit: commit,
+  }
+  await fetch(url, {
+    method: 'POST',
+    headers: {"Content-Type": "application/json"},
+    body: JSON.stringify(body)
+  })
+}
+
+function randomU64() {
+  const bytes = randomBytes(8)
+  const hexString = bytes.reduce(((string, byte) => string + byte.toString(16)), '')
+  return BigInt('0x' + hexString).toString(10)
+}
diff --git a/script/randomized-test-minimize b/script/randomized-test-minimize
index fe2686b69b..ce0b7203b4 100755
--- a/script/randomized-test-minimize
+++ b/script/randomized-test-minimize
@@ -4,85 +4,109 @@ const fs = require('fs')
 const path = require('path')
 const {spawnSync} = require('child_process')
 
-if (process.argv.length < 4) {
-  process.stderr.write("usage: script/randomized-test-minimize <input-plan> <output-plan> [start-index]\n")
-  process.exit(1)
-}
-
-const inputPlanPath = process.argv[2]
-const outputPlanPath = process.argv[3]
-const startIndex = parseInt(process.argv[4]) || 0
-
-const tempPlanPath = inputPlanPath + '.try'
-
 const FAILING_SEED_REGEX = /failing seed: (\d+)/ig
+const CARGO_TEST_ARGS = [
+  '--release',
+  '--lib',
+  '--package', 'collab',
+  'random_collaboration',
+]
 
-fs.copyFileSync(inputPlanPath, outputPlanPath)
-let testPlan = JSON.parse(fs.readFileSync(outputPlanPath, 'utf8'))
-
-process.stderr.write("minimizing failing test plan...\n")
-for (let ix = startIndex; ix < testPlan.length; ix++) {
-  // Skip 'MutateClients' entries, since they themselves are not single operations.
-  if (testPlan[ix].MutateClients) {
-    continue
+if (require.main === module) {
+  if (process.argv.length < 4) {
+    process.stderr.write("usage: script/randomized-test-minimize <input-plan> <output-plan> [start-index]\n")
+    process.exit(1)
   }
 
-  // Remove a row from the test plan
-  const newTestPlan = testPlan.slice()
-  newTestPlan.splice(ix, 1)
-  fs.writeFileSync(tempPlanPath, serializeTestPlan(newTestPlan), 'utf8');
+  minimizeTestPlan(
+    process.argv[2],
+    process.argv[3],
+    parseInt(process.argv[4]) || 0
+  );
+}
 
-  process.stderr.write(`${ix}/${testPlan.length}: ${JSON.stringify(testPlan[ix])}`)
+function minimizeTestPlan(
+  inputPlanPath,
+  outputPlanPath,
+  startIndex = 0
+) {
+  const tempPlanPath = inputPlanPath + '.try'
 
-  const failingSeed = runTestsForPlan(tempPlanPath, 500)
+  fs.copyFileSync(inputPlanPath, outputPlanPath)
+  let testPlan = JSON.parse(fs.readFileSync(outputPlanPath, 'utf8'))
 
-  // If the test failed, keep the test plan with the removed row. Reload the test
-  // plan from the JSON file, since the test itself will remove any operations
-  // which are no longer valid before saving the test plan.
-  if (failingSeed != null) {
-    process.stderr.write(` - remove. failing seed: ${failingSeed}.\n`)
-    fs.copyFileSync(tempPlanPath, outputPlanPath)
-    testPlan = JSON.parse(fs.readFileSync(outputPlanPath, 'utf8'))
-    ix--
-  } else {
-    process.stderr.write(` - keep.\n`)
+  process.stderr.write("minimizing failing test plan...\n")
+  for (let ix = startIndex; ix < testPlan.length; ix++) {
+    // Skip 'MutateClients' entries, since they themselves are not single operations.
+    if (testPlan[ix].MutateClients) {
+      continue
+    }
+
+    // Remove a row from the test plan
+    const newTestPlan = testPlan.slice()
+    newTestPlan.splice(ix, 1)
+    fs.writeFileSync(tempPlanPath, serializeTestPlan(newTestPlan), 'utf8');
+
+    process.stderr.write(`${ix}/${testPlan.length}: ${JSON.stringify(testPlan[ix])}`)
+    const failingSeed = runTests({
+      SEED: '0',
+      LOAD_PLAN: tempPlanPath,
+      SAVE_PLAN: tempPlanPath,
+      ITERATIONS: '500'
+    })
+
+    // If the test failed, keep the test plan with the removed row. Reload the test
+    // plan from the JSON file, since the test itself will remove any operations
+    // which are no longer valid before saving the test plan.
+    if (failingSeed != null) {
+      process.stderr.write(` - remove. failing seed: ${failingSeed}.\n`)
+      fs.copyFileSync(tempPlanPath, outputPlanPath)
+      testPlan = JSON.parse(fs.readFileSync(outputPlanPath, 'utf8'))
+      ix--
+    } else {
+      process.stderr.write(` - keep.\n`)
+    }
+  }
+
+  fs.unlinkSync(tempPlanPath)
+
+  // Re-run the final minimized plan to get the correct failing seed.
+  // This is a workaround for the fact that the execution order can
+  // slightly change when replaying a test plan after it has been
+  // saved and loaded.
+  const failingSeed = runTests({
+    SEED: '0',
+    ITERATIONS: '5000',
+    LOAD_PLAN: outputPlanPath,
+  })
+
+  process.stderr.write(`final test plan: ${outputPlanPath}\n`)
+  process.stderr.write(`final seed: ${failingSeed}\n`)
+  return failingSeed
+}
+
+function buildTests() {
+  const {status} = spawnSync('cargo', ['test', '--no-run', ...CARGO_TEST_ARGS], {
+    stdio: 'inherit',
+    encoding: 'utf8',
+    env: {
+      ...process.env,
+    }
+  });
+  if (status !== 0) {
+    throw new Error('build failed')
   }
 }
 
-fs.unlinkSync(tempPlanPath)
-
-// Re-run the final minimized plan to get the correct failing seed.
-// This is a workaround for the fact that the execution order can
-// slightly change when replaying a test plan after it has been
-// saved and loaded.
-const failingSeed = runTestsForPlan(outputPlanPath, 5000)
-
-process.stderr.write(`final test plan: ${outputPlanPath}\n`)
-process.stderr.write(`final seed: ${failingSeed}\n`)
-console.log(failingSeed)
-
-function runTestsForPlan(path, iterations) {
-  const {status, stdout, stderr} = spawnSync(
-    'cargo',
-    [
-      'test',
-      '--release',
-      '--lib',
-      '--package', 'collab',
-      'random_collaboration'
-    ],
-    {
-      stdio: 'pipe',
-      encoding: 'utf8',
-      env: {
-        ...process.env,
-        'SEED': 0,
-        'LOAD_PLAN': path,
-        'SAVE_PLAN': path,
-        'ITERATIONS': String(iterations),
-      }
+function runTests(env) {
+  const {status, stdout} = spawnSync('cargo', ['test', ...CARGO_TEST_ARGS], {
+    stdio: 'pipe',
+    encoding: 'utf8',
+    env: {
+      ...process.env,
+      ...env,
     }
-  );
+  });
 
   if (status !== 0) {
     FAILING_SEED_REGEX.lastIndex = 0
@@ -102,3 +126,7 @@ function runTestsForPlan(path, iterations) {
 function serializeTestPlan(plan) {
   return "[\n" + plan.map(row => JSON.stringify(row)).join(",\n") + "\n]\n"
 }
+
+exports.buildTests = buildTests
+exports.runTests = runTests
+exports.minimizeTestPlan = minimizeTestPlan

From 485c56e3bd9c2a5e2c86756088366aa59d5f63e4 Mon Sep 17 00:00:00 2001
From: Max Brunsfeld <maxbrunsfeld@gmail.com>
Date: Mon, 17 Apr 2023 15:43:12 -0700
Subject: [PATCH 6/6] Don't run randomized tests on pushes to main

---
 .github/workflows/randomized_tests.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/randomized_tests.yml b/.github/workflows/randomized_tests.yml
index f180313ed3..c3579ad41e 100644
--- a/.github/workflows/randomized_tests.yml
+++ b/.github/workflows/randomized_tests.yml
@@ -5,7 +5,6 @@ concurrency: randomized-tests
 on:
   push:
     branches:
-      - main
       - randomized-tests-runner
   schedule:
      - cron: '0 * * * *'