Verify benchmarks compile and execute in the gate (#3640)

Execution of `sbt runtime/bench` doesn't seem to be part of the gate. As such it can happen a change into the Enso language syntax, standard libraries, runtime & co. can break the benchmarks suite without being noticed. Integrating such PR causes unnecessary disruptions to others using the benchmarks.

Let's make sure verification of the benchmarks (e.g. that they compile and can execute without error) is part of the CI.

# Important Notes
Currently the gate shall fail. The fix is being prepared in parallel PR - #3639. When the two PRs are combined, the gate shall succeed again.

.github/workflows/benchmark.yml

@@ -6,7 +6,13 @@ name: Benchmark Engine
       - develop
   schedule:
     - cron: 0 5 * * 2-6
-  workflow_dispatch: {}
+  workflow_dispatch:
+    inputs:
+      just-check:
+        description: "If set, benchmarks will be only checked to run correctly, not to measure actual performance."
+        required: true
+        type: boolean
+        default: false
 jobs:
   benchmark-engine:
     name: Benchmark Engine
@@ -54,4 +60,5 @@ jobs:
         run: ls -lAR
         if: "failure() && runner.os != 'Windows'"
 env:
+  ENSO_BUILD_MINIMAL_RUN: "${{ inputs.just-check }}"
   ENSO_BUILD_SKIP_VERSION_CHECK: "true"

Cargo.lock

@@ -1750,7 +1750,7 @@ dependencies = [
 [[package]]
 name = "enso-build"
 version = "0.1.0"
-source = "git+https://github.com/enso-org/ci-build?branch=develop#94389f9c820f76009b05419d661636b2dc199f20"
+source = "git+https://github.com/enso-org/ci-build?branch=develop#d0f5ebd35b87ecb9cf2e424f65dab3e4183cd85a"
 dependencies = [
  "anyhow",
  "async-compression",
@@ -1824,7 +1824,7 @@ dependencies = [
 [[package]]
 name = "enso-build-cli"
 version = "0.1.0"
-source = "git+https://github.com/enso-org/ci-build?branch=develop#94389f9c820f76009b05419d661636b2dc199f20"
+source = "git+https://github.com/enso-org/ci-build?branch=develop#d0f5ebd35b87ecb9cf2e424f65dab3e4183cd85a"
 dependencies = [
  "anyhow",
  "byte-unit",
@@ -3721,10 +3721,10 @@ version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5617e92fc2f2501c3e2bc6ce547cad841adba2bae5b921c7e52510beca6d084c"
 dependencies = [
- "base64 0.10.1",
+ "base64 0.11.0",
  "bytes 1.1.0",
  "http",
- "httpdate 0.3.2",
+ "httpdate 1.0.2",
  "language-tags 0.3.2",
  "mime 0.3.16",
  "percent-encoding 2.1.0",
@@ -3734,7 +3734,7 @@ dependencies = [
 [[package]]
 name = "ide-ci"
 version = "0.1.0"
-source = "git+https://github.com/enso-org/ci-build?branch=develop#94389f9c820f76009b05419d661636b2dc199f20"
+source = "git+https://github.com/enso-org/ci-build?branch=develop#d0f5ebd35b87ecb9cf2e424f65dab3e4183cd85a"
 dependencies = [
  "anyhow",
  "async-compression",

build.sbt

@@ -994,10 +994,19 @@ lazy val `interpreter-dsl` = (project in file("lib/scala/interpreter-dsl"))
 // === Sub-Projects ===========================================================
 // ============================================================================
 
-val truffleRunOptions = Seq(
+val truffleRunOptions = if (java.lang.Boolean.getBoolean("bench.compileOnly")) {
-  "-Dpolyglot.engine.IterativePartialEscape=true",
+  Seq(
-  "-Dpolyglot.engine.BackgroundCompilation=false"
+    "-Dpolyglot.engine.IterativePartialEscape=true",
-)
+    "-Dpolyglot.engine.BackgroundCompilation=false",
+    "-Dbench.compileOnly=true"
+  )
+} else {
+  Seq(
+    "-Dpolyglot.engine.IterativePartialEscape=true",
+    "-Dpolyglot.engine.BackgroundCompilation=false"
+  )
+}
+
 
 val truffleRunOptionsSettings = Seq(
   fork := true,

engine/runtime/src/bench/java/org/enso/interpreter/bench/BenchmarksRunner.java

@@ -1,18 +1,18 @@
 package org.enso.interpreter.bench;
 
-import org.openjdk.jmh.results.RunResult;
-import org.openjdk.jmh.runner.BenchmarkList;
-import org.openjdk.jmh.runner.BenchmarkListEntry;
-import org.openjdk.jmh.runner.Runner;
-import org.openjdk.jmh.runner.RunnerException;
-import org.openjdk.jmh.runner.options.Options;
-import org.openjdk.jmh.runner.options.OptionsBuilder;
-
 import jakarta.xml.bind.JAXBException;
 import java.io.File;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.stream.Collectors;
+import org.openjdk.jmh.results.RunResult;
+import org.openjdk.jmh.runner.BenchmarkList;
+import org.openjdk.jmh.runner.BenchmarkListEntry;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.ChainedOptionsBuilder;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
 
 /** Runner class for the benchmarks. Discovers, runs and reports benchmark results. */
 public class BenchmarksRunner {
@@ -32,10 +32,17 @@ public class BenchmarksRunner {
    * @return a {@link BenchmarkItem} containing current run result and historical results.
    */
   public BenchmarkItem run(String label) throws RunnerException, JAXBException {
-    Options benchmarkOptions = new OptionsBuilder()
+    ChainedOptionsBuilder builder = new OptionsBuilder()
       .jvmArgsAppend("-Xss16M", "-Dpolyglot.engine.MultiTier=false")
-      .include("^" + label + "$")
+      .include("^" + label + "$");
-      .build();
+
+    if (Boolean.getBoolean("bench.compileOnly")) {
+      builder
+        .measurementIterations(1)
+        .warmupIterations(0);
+    }
+
+    Options benchmarkOptions = builder.build();
     RunResult benchmarksResult = new Runner(benchmarkOptions).runSingle();
 
     Report report;