mirror of
https://github.com/enso-org/enso.git
synced 2024-09-11 13:15:52 +03:00
dry-run benchmarks exits when some benchmark fails (#9397)
If some benchmark fails in dry-run (compileOnly) mode, the whole process exits with non-zero return code. Also fixes failing engine compiler benchmarks.
# Important Notes
Manually added failure:
```diff
diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/ArrayProxyBenchmarks.java b/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/ArrayProxyBenchmarks.java
index c8d86cecc..f9f4d7cbc 100644
--- a/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/ArrayProxyBenchmarks.java
+++ b/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/ArrayProxyBenchmarks.java
@@ -95,7 +95,8 @@ public class ArrayProxyBenchmarks {
@Benchmark
public void sumOverComputingProxy(Blackhole matter) {
- performBenchmark(matter);
+ //performBenchmark(matter);
+ throw new AssertionError("My error");
}
@Benchmark
```
Run with `sbt "-Dbench.compileOnly=true runtime-benchmarks/benchOnly org.enso.interpreter.bench.benchmarks.semantic.ArrayProxyBenchmarks.sumOverComputingProxy"` fails with:
```
[info] Running benchmarks [org.enso.interpreter.bench.benchmarks.semantic.ArrayProxyBenchmarks.sumOverComputingProxy] in compileOnly mode
[info] # JMH version: 1.36
[info] # VM version: JDK 21.0.2, Java HotSpot(TM) 64-Bit Server VM, 21.0.2+13-LTS-jvmci-23.1-b30
[info] # VM invoker: /home/pavel/.sdkman/candidates/java/21.0.2-graal/bin/java
[info] # VM options: -XX:ThreadPriorityPolicy=1 -XX:+UnlockExperimentalVMOptions -XX:+EnableJVMCIProduct -XX:-UnlockExperimentalVMOptions -Dslf4j.provider=org.slf4j.nop.NOPServiceProvider -Dbench.compileOnly=true --module-path=/home/pavel/.cache/coursier/v1/https/repo1.maven.org/maven2/org/graalvm/sdk/nativeimage/23.1.2/nativeimage-23.1.2.jar:/home/pavel/.cache/coursier/v1/https/repo1.maven.org/maven2/org/graalvm/sdk/word/23.1.2/word-23.1.2.jar:/home/pavel/.cache/coursier/v1/https/repo1.maven.org/maven2/org/graalvm/sdk/jniutils/23.1.2/jniutils-23.1.2.jar:/home/pavel/.cache/coursier/v1/https/repo1.maven.org/maven2/org/graalvm/sdk/collections/23.1.2/collections-23.1.2.jar:/home/pavel/.cache/coursier/v1/https/repo1.maven.org/maven2/org/graalvm/polyglot/polyglot/23.1.2/polyglot-23.1.2.jar:/home/pavel/.cache/coursier/v1/https/repo1.maven.org/maven2/org/graalvm/truffle/truffle-api/23.1.2/truffle-api-23.1.2.jar:/home/pavel/.cache/coursier/v1/https/repo1.maven.org/maven2/org/graalvm/truffle/truffle-runtime/23.1.2/truffle-runtime-23.1.2.jar:/home/pavel/.cache/coursier/v1/https/repo1.maven.org/maven2/org/graalvm/truffle/truffle-compiler/23.1.2/truffle-compiler-23.1.2.jar:/home/pavel/.cache/coursier/v1/https/repo1.maven.org/maven2/org/graalvm/js/js-language/23.1.2/js-language-23.1.2.jar:/home/pavel/.cache/coursier/v1/https/repo1.maven.org/maven2/org/graalvm/regex/regex/23.1.2/regex-23.1.2.jar:/home/pavel/.cache/coursier/v1/https/repo1.maven.org/maven2/org/graalvm/shadowed/icu4j/23.1.2/icu4j-23.1.2.jar:/home/pavel/.cache/coursier/v1/https/repo1.maven.org/maven2/org/graalvm/python/python-language/23.1.2/python-language-23.1.2.jar:/home/pavel/.cache/coursier/v1/https/repo1.maven.org/maven2/org/graalvm/python/python-resources/23.1.2/python-resources-23.1.2.jar:/home/pavel/.cache/coursier/v1/https/repo1.maven.org/maven2/org/bouncycastle/bcutil-jdk18on/1.76/bcutil-jdk18on-1.76.jar:/home/pavel/.cache/coursier/v1/https/repo1.maven.org/maven2/org/bouncycastle/bcpkix-jdk18on/1.76/bcpkix-jdk18on-1.76.jar:/home/pavel/.cache/coursier/v1/https/repo1.maven.org/maven2/org/bouncycastle/bcprov-jdk18on/1.76/bcprov-jdk18on-1.76.jar:/home/pavel/.cache/coursier/v1/https/repo1.maven.org/maven2/org/graalvm/llvm/llvm-api/23.1.2/llvm-api-23.1.2.jar:/home/pavel/.cache/coursier/v1/https/repo1.maven.org/maven2/org/graalvm/truffle/truffle-nfi/23.1.2/truffle-nfi-23.1.2.jar:/home/pavel/.cache/coursier/v1/https/repo1.maven.org/maven2/org/graalvm/truffle/truffle-nfi-libffi/23.1.2/truffle-nfi-libffi-23.1.2.jar:/home/pavel/.cache/coursier/v1/https/repo1.maven.org/maven2/org/graalvm/tools/profiler-tool/23.1.2/profiler-tool-23.1.2.jar:/home/pavel/.cache/coursier/v1/https/repo1.maven.org/maven2/org/graalvm/shadowed/json/23.1.2/json-23.1.2.jar:/home/pavel/.cache/coursier/v1/https/repo1.maven.org/maven2/org/tukaani/xz/1.9/xz-1.9.jar:/home/pavel/.cache/coursier/v1/https/repo1.maven.org/maven2/org/slf4j/slf4j-api/2.0.9/slf4j-api-2.0.9.jar:/home/pavel/.cache/coursier/v1/https/repo1.maven.org/maven2/org/slf4j/slf4j-nop/2.0.9/slf4j-nop-2.0.9.jar:/home/pavel/dev/enso/runtime.jar --add-modules=org.enso.runtime --add-exports=org.slf4j.nop/org.slf4j.nop=org.slf4j
[info] # Blackhole mode: compiler (auto-detected, use -Djmh.blackhole.autoDetect=false to disable)
[info] # Warmup: <none>
[info] # Measurement: 1 iterations, 1 s each
[info] # Timeout: 10 min per iteration
[info] # Threads: 1 thread, will synchronize iterations
[info] # Benchmark mode: Average time, time/op
[info] # Benchmark: org.enso.interpreter.bench.benchmarks.semantic.ArrayProxyBenchmarks.sumOverComputingProxy
[info] # Run progress: 0.00% complete, ETA 00:00:01
[info] # Fork: N/A, test runs in the host VM
[info] # *** WARNING: Non-forked runs may silently omit JVM options, mess up profilers, disable compiler hints, etc. ***
[info] # *** WARNING: Use non-forked runs only for debugging purposes, not for actual performance runs. ***
[error] SLF4J: Attempting to load provider "org.slf4j.nop.NOPServiceProvider" specified via "slf4j.provider" system property
[info] Iteration 1: <failure>
[info] java.lang.AssertionError: My error
[info] at org.enso.interpreter.bench.benchmarks.semantic.ArrayProxyBenchmarks.sumOverComputingProxy(ArrayProxyBenchmarks.java:99)
[info] at org.enso.interpreter.bench.benchmarks.semantic.jmh_generated.ArrayProxyBenchmarks_sumOverComputingProxy_jmhTest.sumOverComputingProxy_avgt_jmhStub(ArrayProxyBenchmarks_sumOverComputingProxy_jmhTest.java:232)
[info] at org.enso.interpreter.bench.benchmarks.semantic.jmh_generated.ArrayProxyBenchmarks_sumOverComputingProxy_jmhTest.sumOverComputingProxy_AverageTime(ArrayProxyBenchmarks_sumOverComputingProxy_jmhTest.java:173)
[info] at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
[info] at java.base/java.lang.reflect.Method.invoke(Method.java:580)
[info] at org.openjdk.jmh.runner.BenchmarkHandler$BenchmarkTask.call(BenchmarkHandler.java:475)
[info] at org.openjdk.jmh.runner.BenchmarkHandler$BenchmarkTask.call(BenchmarkHandler.java:458)
[info] at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
[info] at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572)
[info] at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
[info] at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
[info] at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
[error] Benchmark run failed: Benchmark caught the exception
[info] at java.base/java.lang.Thread.run(Thread.java:1583)
[error] org.openjdk.jmh.runner.RunnerException: Benchmark caught the exception
[error] at org.openjdk.jmh.runner.Runner.runBenchmarks(Runner.java:575)
[error] at org.openjdk.jmh.runner.Runner.internalRun(Runner.java:310)
[error] at org.openjdk.jmh.runner.Runner.run(Runner.java:209)
[error] at org.enso.interpreter.bench.BenchmarksRunner.runCompileOnly(BenchmarksRunner.java:93)
[error] at org.enso.interpreter.bench.BenchmarksRunner.run(BenchmarksRunner.java:36)
[error] at org.enso.interpreter.bench.benchmarks.RuntimeBenchmarksRunner.main(RuntimeBenchmarksRunner.java:8)
[error] Caused by: org.openjdk.jmh.runner.BenchmarkException: Benchmark error during the run
[error] at org.openjdk.jmh.runner.BenchmarkHandler.runIteration(BenchmarkHandler.java:424)
[error] at org.openjdk.jmh.runner.BaseRunner.runBenchmark(BaseRunner.java:281)
[error] at org.openjdk.jmh.runner.BaseRunner.runBenchmark(BaseRunner.java:233)
[error] at org.openjdk.jmh.runner.BaseRunner.doSingle(BaseRunner.java:138)
[error] at org.openjdk.jmh.runner.BaseRunner.runBenchmarksEmbedded(BaseRunner.java:110)
[error] at org.openjdk.jmh.runner.Runner.runBenchmarks(Runner.java:555)
[error] ... 5 more
[error] Suppressed: java.lang.AssertionError: My error
[error] at org.enso.interpreter.bench.benchmarks.semantic.ArrayProxyBenchmarks.sumOverComputingProxy(ArrayProxyBenchmarks.java:99)
[error] at org.enso.interpreter.bench.benchmarks.semantic.jmh_generated.ArrayProxyBenchmarks_sumOverComputingProxy_jmhTest.sumOverComputingProxy_avgt_jmhStub(ArrayProxyBenchmarks_sumOverComputingProxy_jmhTest.java:232)
[error] at org.enso.interpreter.bench.benchmarks.semantic.jmh_generated.ArrayProxyBenchmarks_sumOverComputingProxy_jmhTest.sumOverComputingProxy_AverageTime(ArrayProxyBenchmarks_sumOverComputingProxy_jmhTest.java:173)
[error] at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
[error] at java.base/java.lang.reflect.Method.invoke(Method.java:580)
[error] at org.openjdk.jmh.runner.BenchmarkHandler$BenchmarkTask.call(BenchmarkHandler.java:475)
[error] at org.openjdk.jmh.runner.BenchmarkHandler$BenchmarkTask.call(BenchmarkHandler.java:458)
[error] at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
[error] at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572)
[error] at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
[error] at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
[error] at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
[error] at java.base/java.lang.Thread.run(Thread.java:1583)
[error] Nonzero exit code returned from runner: 1
[error] (Compile / run) Nonzero exit code returned from runner: 1
[error] Total time: 5 s, completed Mar 13, 2024, 12:49:59 PM
```
This commit is contained in:
Pavel Marek
GitHub
parent
6017e6802f
commit
c4029eed4f
@ -1850,6 +1850,7 @@ lazy val `runtime-benchmarks` =
|
||||
"-Xlint:unchecked"
|
||||
),
|
||||
Compile / compile := (Compile / compile)
|
||||
.dependsOn(`runtime-fat-jar` / assembly)
|
||||
.dependsOn(Def.task { (Compile / sourceManaged).value.mkdirs })
|
||||
.value,
|
||||
parallelExecution := false,
|
||||
|
||||
@ -453,7 +453,7 @@ impl RunContext {
|
||||
// TODO [mwu] It should be possible to run them through context config option.
|
||||
if self.config.build_benchmarks {
|
||||
tasks.extend([
|
||||
"runtime/Benchmark/compile",
|
||||
"runtime-benchmarks/compile",
|
||||
"language-server/Benchmark/compile",
|
||||
"searcher/Benchmark/compile",
|
||||
"std-benchmarks/Benchmark/compile",
|
||||
|
||||
@ -8,6 +8,7 @@ import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.logging.Level;
|
||||
import org.enso.interpreter.runtime.EnsoContext;
|
||||
import org.enso.polyglot.LanguageInfo;
|
||||
@ -21,6 +22,9 @@ public class Utils {
|
||||
public static Context.Builder createDefaultContextBuilder() {
|
||||
return Context.newBuilder()
|
||||
.allowExperimentalOptions(true)
|
||||
.option(
|
||||
RuntimeOptions.LANGUAGE_HOME_OVERRIDE,
|
||||
Paths.get("../../distribution/component").toFile().getAbsolutePath())
|
||||
.option(RuntimeOptions.LOG_LEVEL, Level.WARNING.getName())
|
||||
.option(RuntimeOptions.DISABLE_IR_CACHES, "true")
|
||||
.option(RuntimeOptions.STRICT_ERRORS, "true")
|
||||
|
||||
@ -3,7 +3,6 @@ package org.enso.compiler.benchmarks.module;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
@ -93,9 +92,6 @@ from Standard.Visualization import all
|
||||
this.out = new ByteArrayOutputStream();
|
||||
this.context =
|
||||
Utils.createDefaultContextBuilder()
|
||||
.option(
|
||||
RuntimeOptions.LANGUAGE_HOME_OVERRIDE,
|
||||
Paths.get("../../distribution/component").toFile().getAbsolutePath())
|
||||
// Enable IR caches - we don't want to compile the imported modules from the standard
|
||||
// libraries
|
||||
.option(RuntimeOptions.DISABLE_IR_CACHES, "false")
|
||||
|
||||
@ -25,8 +25,8 @@ import org.openjdk.jmh.infra.BenchmarkParams;
|
||||
|
||||
@BenchmarkMode(Mode.AverageTime)
|
||||
@Fork(1)
|
||||
@Warmup(iterations = 5, time = 1)
|
||||
@Measurement(iterations = 3, time = 3)
|
||||
@Warmup(iterations = 5)
|
||||
@Measurement(iterations = 3)
|
||||
@OutputTimeUnit(TimeUnit.MILLISECONDS)
|
||||
@State(Scope.Benchmark)
|
||||
public class IfVsCaseBenchmarks {
|
||||
|
||||
@ -3,13 +3,9 @@ package org.enso.interpreter.bench;
|
||||
import jakarta.xml.bind.JAXBException;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
import org.openjdk.jmh.results.RunResult;
|
||||
import org.openjdk.jmh.runner.BenchmarkList;
|
||||
import org.openjdk.jmh.runner.BenchmarkListEntry;
|
||||
import org.openjdk.jmh.runner.Runner;
|
||||
import org.openjdk.jmh.runner.RunnerException;
|
||||
import org.openjdk.jmh.runner.options.CommandLineOptionException;
|
||||
@ -21,15 +17,6 @@ import org.openjdk.jmh.runner.options.TimeValue;
|
||||
public class BenchmarksRunner {
|
||||
public static final File REPORT_FILE = new File("./bench-report.xml");
|
||||
|
||||
/**
|
||||
* @return A list of qualified names of all benchmarks visible to JMH.
|
||||
*/
|
||||
public List<String> getAvailable() {
|
||||
return BenchmarkList.defaultList().getAll(null, new ArrayList<>()).stream()
|
||||
.map(BenchmarkListEntry::getUsername)
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
public static void run(String[] args) throws RunnerException {
|
||||
CommandLineOptions cmdOpts = null;
|
||||
try {
|
||||
@ -75,35 +62,35 @@ public class BenchmarksRunner {
|
||||
}
|
||||
}
|
||||
|
||||
private static Collection<RunResult> runCompileOnly(List<String> includes)
|
||||
throws RunnerException {
|
||||
System.out.println("Running benchmarks " + includes + " in compileOnly mode");
|
||||
/**
|
||||
* Results from compileOnly mode are not reported. Moreover, if some of the benchmarks in this
|
||||
* mode fails, the whole process immediately fails. This behavior is different to *normal*
|
||||
* benchmarks, where a single failure does not stop the whole process.
|
||||
*/
|
||||
private static void runCompileOnly(List<String> includes) {
|
||||
if (includes.isEmpty()) {
|
||||
System.out.println("Running all benchmarks in compileOnly mode");
|
||||
} else {
|
||||
System.out.println("Running benchmarks " + includes + " in compileOnly mode");
|
||||
}
|
||||
var optsBuilder =
|
||||
new OptionsBuilder()
|
||||
.measurementTime(TimeValue.seconds(1))
|
||||
.measurementIterations(1)
|
||||
.warmupIterations(0)
|
||||
.shouldFailOnError(true)
|
||||
.forks(0);
|
||||
includes.forEach(optsBuilder::include);
|
||||
var opts = optsBuilder.build();
|
||||
var runner = new Runner(opts);
|
||||
return runner.run();
|
||||
}
|
||||
|
||||
public static BenchmarkItem runSingle(String label) throws RunnerException, JAXBException {
|
||||
String includeRegex = "^" + label + "$";
|
||||
if (Boolean.getBoolean("bench.compileOnly")) {
|
||||
var results = runCompileOnly(List.of(includeRegex));
|
||||
var firstResult = results.iterator().next();
|
||||
return reportResult(label, firstResult);
|
||||
} else {
|
||||
var opts =
|
||||
new OptionsBuilder()
|
||||
.jvmArgsAppend("-Xss16M", "-Dpolyglot.engine.MultiTier=false")
|
||||
.include(includeRegex)
|
||||
.build();
|
||||
RunResult benchmarksResult = new Runner(opts).runSingle();
|
||||
return reportResult(label, benchmarksResult);
|
||||
try {
|
||||
runner.run();
|
||||
System.out.println(
|
||||
"benchmarks run successfully in compileOnly mode. Results are not reported.");
|
||||
} catch (RunnerException e) {
|
||||
System.err.println("Benchmark run failed: " + e.getMessage());
|
||||
e.printStackTrace(System.err);
|
||||
System.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user