Verify benchmarks compile and execute in the gate (#3640)

Execution of `sbt runtime/bench` doesn't seem to be part of the gate. As such it can happen a change into the Enso language syntax, standard libraries, runtime & co. can break the benchmarks suite without being noticed. Integrating such PR causes unnecessary disruptions to others using the benchmarks.

Let's make sure verification of the benchmarks (e.g. that they compile and can execute without error) is part of the CI.

# Important Notes
Currently the gate shall fail. The fix is being prepared in parallel PR - #3639. When the two PRs are combined, the gate shall succeed again.
This commit is contained in:
Jaroslav Tulach 2022-08-11 09:21:44 +02:00 committed by GitHub
parent 3dca738cf7
commit a7bc3c6c89
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 44 additions and 21 deletions

View File

@ -6,7 +6,13 @@ name: Benchmark Engine
- develop
schedule:
- cron: 0 5 * * 2-6
workflow_dispatch: {}
workflow_dispatch:
inputs:
just-check:
description: "If set, benchmarks will be only checked to run correctly, not to measure actual performance."
required: true
type: boolean
default: false
jobs:
benchmark-engine:
name: Benchmark Engine
@ -54,4 +60,5 @@ jobs:
run: ls -lAR
if: "failure() && runner.os != 'Windows'"
env:
ENSO_BUILD_MINIMAL_RUN: "${{ inputs.just-check }}"
ENSO_BUILD_SKIP_VERSION_CHECK: "true"

10
Cargo.lock generated
View File

@ -1750,7 +1750,7 @@ dependencies = [
[[package]]
name = "enso-build"
version = "0.1.0"
source = "git+https://github.com/enso-org/ci-build?branch=develop#94389f9c820f76009b05419d661636b2dc199f20"
source = "git+https://github.com/enso-org/ci-build?branch=develop#d0f5ebd35b87ecb9cf2e424f65dab3e4183cd85a"
dependencies = [
"anyhow",
"async-compression",
@ -1824,7 +1824,7 @@ dependencies = [
[[package]]
name = "enso-build-cli"
version = "0.1.0"
source = "git+https://github.com/enso-org/ci-build?branch=develop#94389f9c820f76009b05419d661636b2dc199f20"
source = "git+https://github.com/enso-org/ci-build?branch=develop#d0f5ebd35b87ecb9cf2e424f65dab3e4183cd85a"
dependencies = [
"anyhow",
"byte-unit",
@ -3721,10 +3721,10 @@ version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5617e92fc2f2501c3e2bc6ce547cad841adba2bae5b921c7e52510beca6d084c"
dependencies = [
"base64 0.10.1",
"base64 0.11.0",
"bytes 1.1.0",
"http",
"httpdate 0.3.2",
"httpdate 1.0.2",
"language-tags 0.3.2",
"mime 0.3.16",
"percent-encoding 2.1.0",
@ -3734,7 +3734,7 @@ dependencies = [
[[package]]
name = "ide-ci"
version = "0.1.0"
source = "git+https://github.com/enso-org/ci-build?branch=develop#94389f9c820f76009b05419d661636b2dc199f20"
source = "git+https://github.com/enso-org/ci-build?branch=develop#d0f5ebd35b87ecb9cf2e424f65dab3e4183cd85a"
dependencies = [
"anyhow",
"async-compression",

View File

@ -994,10 +994,19 @@ lazy val `interpreter-dsl` = (project in file("lib/scala/interpreter-dsl"))
// === Sub-Projects ===========================================================
// ============================================================================
val truffleRunOptions = Seq(
"-Dpolyglot.engine.IterativePartialEscape=true",
"-Dpolyglot.engine.BackgroundCompilation=false"
)
val truffleRunOptions = if (java.lang.Boolean.getBoolean("bench.compileOnly")) {
Seq(
"-Dpolyglot.engine.IterativePartialEscape=true",
"-Dpolyglot.engine.BackgroundCompilation=false",
"-Dbench.compileOnly=true"
)
} else {
Seq(
"-Dpolyglot.engine.IterativePartialEscape=true",
"-Dpolyglot.engine.BackgroundCompilation=false"
)
}
val truffleRunOptionsSettings = Seq(
fork := true,

View File

@ -1,18 +1,18 @@
package org.enso.interpreter.bench;
import org.openjdk.jmh.results.RunResult;
import org.openjdk.jmh.runner.BenchmarkList;
import org.openjdk.jmh.runner.BenchmarkListEntry;
import org.openjdk.jmh.runner.Runner;
import org.openjdk.jmh.runner.RunnerException;
import org.openjdk.jmh.runner.options.Options;
import org.openjdk.jmh.runner.options.OptionsBuilder;
import jakarta.xml.bind.JAXBException;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import org.openjdk.jmh.results.RunResult;
import org.openjdk.jmh.runner.BenchmarkList;
import org.openjdk.jmh.runner.BenchmarkListEntry;
import org.openjdk.jmh.runner.Runner;
import org.openjdk.jmh.runner.RunnerException;
import org.openjdk.jmh.runner.options.ChainedOptionsBuilder;
import org.openjdk.jmh.runner.options.Options;
import org.openjdk.jmh.runner.options.OptionsBuilder;
/** Runner class for the benchmarks. Discovers, runs and reports benchmark results. */
public class BenchmarksRunner {
@ -32,10 +32,17 @@ public class BenchmarksRunner {
* @return a {@link BenchmarkItem} containing current run result and historical results.
*/
public BenchmarkItem run(String label) throws RunnerException, JAXBException {
Options benchmarkOptions = new OptionsBuilder()
ChainedOptionsBuilder builder = new OptionsBuilder()
.jvmArgsAppend("-Xss16M", "-Dpolyglot.engine.MultiTier=false")
.include("^" + label + "$")
.build();
.include("^" + label + "$");
if (Boolean.getBoolean("bench.compileOnly")) {
builder
.measurementIterations(1)
.warmupIterations(0);
}
Options benchmarkOptions = builder.build();
RunResult benchmarksResult = new Runner(benchmarkOptions).runSingle();
Report report;