Detect compilation while benchmarking (#10574)

Enables `engine.TruffleCompilation` in `std-benchmarks`, collects the logs and dumps compilation into to `System.err` when a benchmark is influenced by dynamic compilation.

engine/runtime/src/main/java/org/enso/interpreter/runtime/data/Type.java

@@ -8,7 +8,9 @@ import com.oracle.truffle.api.interop.UnsupportedMessageException;
 import com.oracle.truffle.api.library.CachedLibrary;
 import com.oracle.truffle.api.library.ExportLibrary;
 import com.oracle.truffle.api.library.ExportMessage;
+import com.oracle.truffle.api.nodes.ExplodeLoop;
 import com.oracle.truffle.api.nodes.Node;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
@@ -157,21 +159,54 @@ public final class Type implements EnsoObject {
     return supertype;
   }
 
+  /**
+   * All types this type represents including super types.
+   *
+   * @param ctx contexts to get Any type (common super class) from
+   * @return a compilation constant array with all types this type represents
+   */
   public final Type[] allTypes(EnsoContext ctx) {
-    if (supertype == null) {
-      if (builtin) {
-        return new Type[] {this};
+    var types = new Type[3];
+    var realCount = fillInTypes(this, types, ctx);
+    return Arrays.copyOf(types, realCount);
+  }
+
+  /**
+   * Fills the provided {@code fill} array with all types the {@code self} type can represent. E.g.
+   * including super classes.
+   *
+   * @param self the type to "enroll"
+   * @param fill the array to fill
+   * @param ctx context to obtain Any type from
+   * @return number of types put into the {@code fill} array
+   */
+  @ExplodeLoop
+  private static int fillInTypes(Type self, Type[] fill, EnsoContext ctx) {
+    var at = 0;
+    while (at < fill.length) {
+      fill[at++] = self;
+      if (self.supertype == null) {
+        if (self.builtin) {
+          return at;
+        }
+        fill[at++] = ctx.getBuiltins().any();
+        return at;
       }
-      return new Type[] {this, ctx.getBuiltins().any()};
+      if (self.supertype == ctx.getBuiltins().any()) {
+        fill[at++] = ctx.getBuiltins().any();
+        return at;
+      }
+      if (self == self.supertype) {
+        return at;
+      }
+      self = self.supertype;
     }
-    if (supertype == ctx.getBuiltins().any()) {
-      return new Type[] {this, ctx.getBuiltins().any()};
-    }
-    var superTypes = supertype.allTypes(ctx);
-    var allTypes = new Type[superTypes.length + 1];
-    System.arraycopy(superTypes, 0, allTypes, 1, superTypes.length);
-    allTypes[0] = this;
-    return allTypes;
+    throw CompilerDirectives.shouldNotReachHere(invalidInTypes(self));
+  }
+
+  @CompilerDirectives.TruffleBoundary
+  private static String invalidInTypes(Type self) {
+    return "Cannot compute allTypes for " + self;
   }
 
   public void generateGetters(EnsoLanguage language) {

lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java

@@ -45,6 +45,8 @@ public class BenchProcessor extends AbstractProcessor {
           "import java.util.Objects;",
           "import java.util.concurrent.TimeUnit;",
           "import java.util.logging.Level;",
+          "import java.util.logging.LogRecord;",
+          "import java.util.logging.Handler;",
           "import org.openjdk.jmh.annotations.Benchmark;",
           "import org.openjdk.jmh.annotations.BenchmarkMode;",
           "import org.openjdk.jmh.annotations.Mode;",
@@ -52,10 +54,12 @@ public class BenchProcessor extends AbstractProcessor {
           "import org.openjdk.jmh.annotations.Measurement;",
           "import org.openjdk.jmh.annotations.OutputTimeUnit;",
           "import org.openjdk.jmh.annotations.Setup;",
+          "import org.openjdk.jmh.annotations.TearDown;",
           "import org.openjdk.jmh.annotations.State;",
           "import org.openjdk.jmh.annotations.Scope;",
           "import org.openjdk.jmh.annotations.Warmup;",
           "import org.openjdk.jmh.infra.BenchmarkParams;",
+          "import org.openjdk.jmh.infra.IterationParams;",
           "import org.openjdk.jmh.infra.Blackhole;",
           "import org.graalvm.polyglot.Context;",
           "import org.graalvm.polyglot.Value;",
@@ -210,6 +214,13 @@ public class BenchProcessor extends AbstractProcessor {
     out.println("public class " + className + " {");
 
     // Field definitions
+    out.println("  private int warmupCounter = 0;");
+    out.println("  private int measurementCounter = 0;");
+    out.println("  private boolean compilationMessagesFound;");
+    out.println("  private final StringBuilder compilationLog = new StringBuilder();");
+    out.println(
+        "  private final List<LogRecord> messages = new"
+            + " java.util.concurrent.CopyOnWriteArrayList<>();");
     out.println("  private Value groupInputArg;");
     for (var specJavaName : specJavaNames) {
       out.println("  private Value benchFunc_" + specJavaName + ";");
@@ -237,7 +248,7 @@ public class BenchProcessor extends AbstractProcessor {
     out.println("      .allowExperimentalOptions(true)");
     out.println("      .allowIO(IOAccess.ALL)");
     out.println("      .allowAllAccess(true)");
-    out.println("      .option(RuntimeOptions.LOG_LEVEL, Level.WARNING.getName())");
+    out.println("      .option(RuntimeOptions.LOG_LEVEL, Level.FINE.getName())");
     out.println("      .logHandler(System.err)");
     out.println("      .option(");
     out.println("        RuntimeOptions.LANGUAGE_HOME_OVERRIDE,");
@@ -247,6 +258,20 @@ public class BenchProcessor extends AbstractProcessor {
     out.println("        RuntimeOptions.PROJECT_ROOT,");
     out.println("        projectRootDir.getAbsolutePath()");
     out.println("      )");
+    out.println(
+        """
+                      .option("engine.TraceCompilation", "true")
+                      .logHandler(new java.util.logging.Handler() {
+                         @Override
+                         public void publish(LogRecord lr) {
+                           if ("engine".equals(lr.getLoggerName())) {
+                             messages.add(lr);
+                           }
+                         }
+                         @Override public void flush() {}
+                         @Override public void close() {}
+                      })
+                """);
     out.println("      .build();");
     out.println("    ");
     out.println("    Value bindings = ctx.getBindings(LanguageInfo.ID);");
@@ -278,6 +303,55 @@ public class BenchProcessor extends AbstractProcessor {
     out.println("  } "); // end of setup method
     out.println("  ");
 
+    out.println(
+        """
+                  @Setup(org.openjdk.jmh.annotations.Level.Iteration)
+                  public void clearCompilationMessages(IterationParams it) {
+                    var round = round(it);
+                    if (!messages.isEmpty()) {
+                      compilationLog.append("Before " + it.getType() + "#" + round + ". ");
+                      compilationLog.append("Cleaning " + messages.size() + " compilation messages\\n");
+                      messages.clear();
+                    }
+                  }
+
+                  private int round(IterationParams it) {
+                    return switch (it.getType()) {
+                      case WARMUP -> ++warmupCounter;
+                      case MEASUREMENT -> ++measurementCounter;
+                    };
+                  }
+
+                  private void dumpMessages() {
+                    for (var lr : messages) {
+                      compilationLog.append(lr.getMessage() + "\\n");
+                      compilationMessagesFound = true;
+                    }
+                  }
+
+                  @TearDown(org.openjdk.jmh.annotations.Level.Iteration)
+                  public void dumpCompilationMessages(IterationParams it) {
+                    switch (it.getType()) {
+                      case MEASUREMENT -> {
+                        compilationLog.append("After " + it.getType() + "#" + measurementCounter + ". ");
+                        if (!messages.isEmpty()) {
+                          compilationLog.append("Dumping " + messages.size() + " compilation messages:\\n");
+                          dumpMessages();
+                        } else {
+                          compilationLog.append("No compilation messages.\\n");
+                        }
+                      }
+                    }
+                  }
+
+                  @TearDown
+                  public void checkNoTruffleCompilation(BenchmarkParams params) {
+                    if (compilationMessagesFound) {
+                      System.err.println(compilationLog.toString());
+                    }
+                  }
+
+                """);
     // Benchmark methods
     for (var specJavaName : specJavaNames) {
       out.println();

std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/LibBenchRunner.java

@@ -5,14 +5,12 @@ import org.enso.interpreter.bench.BenchmarksRunner;
 import org.openjdk.jmh.runner.RunnerException;
 
 /**
- * The only purpose for this class is to enable the {@link org.enso.benchmarks.processor.BenchProcessor}
- * to generate JMH sources for the benchmarks in {@code test/Benchmarks} project.
- * For more information see {@code docs/infrastructure/benchmarks.md#Standard-library-benchmarks}.
+ * The only purpose for this class is to enable the {@link
+ * org.enso.benchmarks.processor.BenchProcessor} to generate JMH sources for the benchmarks in
+ * {@code test/Benchmarks} project. For more information see {@code
+ * docs/infrastructure/benchmarks.md#Standard-library-benchmarks}.
  */
-@GenerateBenchSources(
-    projectRootPath = "test/Benchmarks",
-    moduleName = "local.Benchmarks.Main"
-)
+@GenerateBenchSources(projectRootPath = "test/Benchmarks", moduleName = "local.Benchmarks.Main")
 public class LibBenchRunner {
 
   public static void main(String[] args) throws RunnerException {

test/Benchmarks/src/Vector/Operations.enso

@@ -9,7 +9,7 @@ import project.Vector.Utils
 polyglot java import java.util.Random as Java_Random
 
 
-options = Bench.options . set_warmup (Bench.phase_conf 2 5) . set_measure (Bench.phase_conf 1 5)
+options = Bench.options . set_warmup (Bench.phase_conf 5 5) . set_measure (Bench.phase_conf 1 5)
 
 
 collect_benches = Bench.build builder->