server/benchmarks: configurable pause after each adhoc benchmark, bef…

…ore checking allocations

It seems like there might be some non-determinism exposed after #10206 we see a huge regression reported in live_bytes in huge_schema which apparently disappears, later reappearing in
https://github.com/hasura/graphql-engine-mono/pull/10226#issuecomment-1697897108

PR-URL: https://github.com/hasura/graphql-engine-mono/pull/10240
GitOrigin-RevId: 8e9533cb9fd7db6c002f00d1d9cb0331d574d6a2

server/benchmarks/bench.sh

@@ -371,6 +371,10 @@ function run_adhoc_operation_benchmarks() (
       if ! [[ "$iterations" =~ ^[0-9]+$ ]] ; then
           echo "Error: $script must define 'iterations'" >&2; exit 1
       fi
+      # shellcheck disable=SC2154
+      if ! [[ "$pause_after_seconds" =~ ^[0-9]+$ ]] ; then
+          echo "Error: $script must define 'pause_after_seconds'" >&2; exit 1
+      fi
 
       # TODO I was relying on being able to also get 'mutator_cpu_ns' to get a
       # stable metric of CPU usage (like counting instructions with 'perf').
@@ -409,6 +413,9 @@ function run_adhoc_operation_benchmarks() (
       done
       echo
 
+      # Allowing any asynchronous activity to settle:
+      sleep "$pause_after_seconds"
+
       allocated_bytes_after=$(curl "$HASURA_URL/dev/rts_stats" 2>/dev/null | jq '.allocated_bytes')
       mem_in_use_bytes_after=$(curl "$HASURA_URL/dev/rts_stats" 2>/dev/null | jq '.gc.gcdetails_mem_in_use_bytes')
       live_bytes_after=$(curl "$HASURA_URL/dev/rts_stats" 2>/dev/null | jq '.gc.gcdetails_live_bytes')

server/benchmarks/benchmark_sets/chinook/adhoc_operations/replace_metadata.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 export iterations=30
+export pause_after_seconds=10
 
 function adhoc_operation() {
   # This is the same replace_metadata command we run during setup (i.e. a noop)

server/benchmarks/benchmark_sets/deep_schema/adhoc_operations/replace_metadata.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 export iterations=200
+export pause_after_seconds=10
 
 function adhoc_operation() {
   scriptDir=$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")

server/benchmarks/benchmark_sets/huge_schema/adhoc_operations/allowlist.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 export iterations=4
+export pause_after_seconds=5
 
 # There are some incremental Metadata API methods that have no good
 # justification for taking so much time to complete. Allowlist management is one

server/benchmarks/benchmark_sets/huge_schema/adhoc_operations/replace_metadata.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 export iterations=4
+export pause_after_seconds=45
 
 function adhoc_operation() {
   # This is the same replace_metadata command we run during setup (i.e. a noop)

server/benchmarks/benchmark_sets/huge_schema/adhoc_operations/untrack_track_table.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 export iterations=4
+export pause_after_seconds=5
 
 function adhoc_operation() {
   # This is the same replace_metadata command we run during setup (i.e. a noop)