Optional Espresso support with ENSO_JAVA=espresso env variable

This commit is contained in:
Jaroslav Tulach 2023-09-19 15:10:12 +02:00 committed by GitHub
parent b0c1f3b00e
commit 6cbd111bad
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 256 additions and 72 deletions

View File

@ -5,5 +5,8 @@
"vue.complete.casing.tags": "pascal",
"auto-snippets.snippets": [
{ "language": "vue", "snippet": "Vue single-file component" }
]
],
"files.watcherExclude": {
"**/target": true
}
}

View File

@ -925,6 +925,7 @@
- [Warning.get_all returns only unique warnings][6372]
- [Reimplement `enso_project` as a proper builtin][6352]
- [Limit number of reported warnings per value][6577]
- [Experimental support for Espresso Java interpreter][6966]
- [Suggestions are updated only when the type of the expression changes][6755]
- [Add project creation time to project metadata][6780]
- [Upgrade GraalVM to 22.3.1 JDK17][6750]
@ -1066,6 +1067,7 @@
[6372]: https://github.com/enso-org/enso/pull/6372
[6352]: https://github.com/enso-org/enso/pull/6352
[6577]: https://github.com/enso-org/enso/pull/6577
[6966]: https://github.com/enso-org/enso/pull/6966
[6750]: https://github.com/enso-org/enso/pull/6750
[6755]: https://github.com/enso-org/enso/pull/6755
[6780]: https://github.com/enso-org/enso/pull/6780

View File

@ -1699,11 +1699,25 @@ lazy val `engine-runner` = project
"-H:IncludeResources=.*Main.enso$",
"--macro:truffle",
"--language:js",
// "-g",
// "-g",
// "-H:+DashboardAll",
// "-H:DashboardDump=runner.bgv"
"-Dnic=nic"
),
) ++ (if (
org.graalvm.polyglot.Engine
.create()
.getLanguages()
.containsKey("java")
) {
Seq(
"-Dorg.graalvm.launcher.home=" + System.getProperty(
"java.home"
),
"--language:java"
)
} else {
Seq()
}),
mainClass = Option("org.enso.runner.Main"),
cp = Option("runtime.jar"),
initializeAtRuntime = Seq(
@ -1711,6 +1725,7 @@ lazy val `engine-runner` = project
"io.methvin.watchservice.jna.CarbonAPI",
"org.enso.syntax2.Parser",
"zio.internal.ZScheduler$$anon$4",
"org.enso.runner.Main$",
"sun.awt",
"sun.java2d",
"sun.font",

View File

@ -201,20 +201,71 @@ safely.
### Engine runner Configuration
The Native Image generation for the Engine Runner is currently in a preview
state. Limitations are currently mostly due to
[Java interop](https://www.pivotaltracker.com/story/show/183260380) and loading
of stdlib components. To generate the Native Image for runner simply execute
state. To generate the Native Image for runner simply execute
```
```bash
sbt> engine-runner/buildNativeImage
```
and execute the binary on a sample factorial test program
```
```bash
> runner --run engine/runner-native/src/test/resources/Factorial.enso 6
```
The task that generates the Native Image, along with all the necessary
configuration, reside in a separate project due to a bug in the currently used
GraalVM version.
GraalVM version. As September 2023 it can execute all Enso code, but cannot
invoke `IO.println` or other library functions that require
[polyglot java import](../../docs/polyglot/java.md), but read on...
### Engine with Espresso
Since [PR-6966](https://github.com/enso-org/enso/pull/6966) there is an
experimental support for including
[Espresso Java interpreter](https://www.graalvm.org/jdk17/reference-manual/java-on-truffle/)
to allow use of some library functions (like `IO.println`) in the _Native Image_
built runner.
The support can be enabled by setting environment variable `ENSO_JAVA=espresso`
and making sure Espresso is installed in GraalVM executing the Enso engine -
e.g. by running `graalvm/bin/gu install espresso`. Then execute:
```bash
$ cat >hello.enso
import Standard.Base.IO
main = IO.println <| "Hello World!"
$ ENSO_JAVA=espresso ./enso-x.y.z-dev/bin/enso --run hello.enso
```
Unless you see a warning containing _"No language for id java found."_ your code
has just successfully been executed by
[Espresso](https://www.graalvm.org/jdk17/reference-manual/java-on-truffle/)! To
debug just add `JAVA_OPTS` environment variable set to your IDE favorite value:
```bash
$ JAVA_OPTS=-agentlib:jdwp=transport=dt_socket,address=5005 ENSO_JAVA=espresso enso --run hello.enso
```
Espresso support works also with
[native image support](#engine-runner-configuration). Just make sure Espresso is
installed in your GraalVM (via `gu install espresso`) and then rebuild the
`runner` executable:
```bash
enso$ rm runner
enso$ sbt --java-home /graalvm
sbt> engine-runner/buildNativeImage
```
as suggested in the [native image support](#engine-runner-configuration). The
build script detects presence of Espresso and automatically adds
`--language:java` when creating the image. Then you can use
```bash
$ ENSO_JAVA=espresso ./runner --run hello.enso
```
to execute native image `runner` build of Enso together with Espresso.

View File

@ -50,6 +50,7 @@ import org.enso.logger.akka.AkkaConverter
import org.enso.polyglot.{HostAccessFactory, RuntimeOptions, RuntimeServerInfo}
import org.enso.searcher.sql.{SqlDatabase, SqlSuggestionsRepo}
import org.enso.text.{ContentBasedVersioning, Sha3_224VersionCalculator}
import org.graalvm.polyglot.Engine
import org.graalvm.polyglot.Context
import org.graalvm.polyglot.io.MessageEndpoint
import org.slf4j.event.Level
@ -286,7 +287,7 @@ class MainModule(serverConfig: LanguageServerConfig, logLevel: Level) {
val stdInSink = new ObservableOutputStream
val stdIn = new ObservablePipedInputStream(stdInSink)
val context = Context
val builder = Context
.newBuilder()
.allowAllAccess(true)
.allowHostAccess(new HostAccessFactory().allWithTypeMapping())
@ -319,7 +320,22 @@ class MainModule(serverConfig: LanguageServerConfig, logLevel: Level) {
connection
} else null
})
.build()
if (
Engine
.newBuilder()
.allowExperimentalOptions(true)
.build
.getLanguages()
.containsKey("java")
) {
builder
.option("java.ExposeNativeJavaVM", "true")
.option("java.Polyglot", "true")
.option("java.UseBindingsLoader", "true")
.allowCreateThread(true)
}
val context = builder.build()
log.trace("Created Runtime context [{}].", context)
system.eventStream.setLogLevel(AkkaConverter.toAkka(logLevel))

View File

@ -7,6 +7,7 @@ import org.enso.polyglot.debugger.{
DebuggerSessionManagerEndpoint
}
import org.enso.polyglot.{HostAccessFactory, PolyglotContext, RuntimeOptions}
import org.graalvm.polyglot.Engine
import org.graalvm.polyglot.Context
import org.slf4j.event.Level
@ -51,12 +52,22 @@ class ContextFactory {
executionEnvironment.foreach { name =>
options.put("enso.ExecutionEnvironment", name)
}
var javaHome = System.getenv("JAVA_HOME");
if (javaHome == null) {
javaHome = System.getProperty("java.home");
}
if (javaHome == null) {
throw new IllegalStateException("Specify JAVA_HOME environment property");
}
val logLevelName = Converter.toJavaLevel(logLevel).getName
val builder = Context
.newBuilder()
.allowExperimentalOptions(true)
.allowAllAccess(true)
.allowHostAccess(new HostAccessFactory().allWithTypeMapping())
.allowHostAccess(
new HostAccessFactory()
.allWithTypeMapping()
)
.option(RuntimeOptions.PROJECT_ROOT, projectRoot)
.option(RuntimeOptions.STRICT_ERRORS, strictErrors.toString)
.option(RuntimeOptions.WAIT_FOR_PENDING_SERIALIZATION_JOBS, "true")
@ -95,10 +106,25 @@ class ContextFactory {
"bin"
),
"graalpy"
);
)
if (graalpy.exists()) {
builder.option("python.Executable", graalpy.getAbsolutePath());
}
if (
Engine
.newBuilder()
.allowExperimentalOptions(true)
.build()
.getLanguages()
.containsKey("java")
) {
builder
.option("java.ExposeNativeJavaVM", "true")
.option("java.Polyglot", "true")
.option("java.UseBindingsLoader", "true")
.option("java.JavaHome", javaHome)
.allowCreateThread(true)
}
new PolyglotContext(builder.build)
}
}

View File

@ -79,10 +79,19 @@ public abstract class ExpressionNode extends BaseNode implements InstrumentableN
*/
@Override
public SourceSection getSourceSection() {
var bounds = getSourceSectionBounds();
return bounds == null ? null : EnsoRootNode.findSourceSection(getRootNode(), bounds[0], bounds[1]);
}
public int[] getSourceSectionBounds() {
if (this instanceof ExpressionNodeWrapper wrapper) {
return wrapper.getDelegateNode().getSourceSection();
return wrapper.getDelegateNode().getSourceSectionBounds();
} else {
return EnsoRootNode.findSourceSection(getRootNode(), sourceStartIndex, sourceLength);
if (sourceStartIndex == EnsoRootNode.NO_SOURCE && sourceLength == EnsoRootNode.NO_SOURCE) {
return null;
} else {
return new int[] { sourceStartIndex, sourceLength };
}
}
}

View File

@ -3,7 +3,6 @@ package org.enso.interpreter.node.callable.function;
import com.oracle.truffle.api.frame.VirtualFrame;
import com.oracle.truffle.api.instrumentation.StandardTags;
import com.oracle.truffle.api.instrumentation.Tag;
import com.oracle.truffle.api.source.SourceSection;
import org.enso.interpreter.node.ClosureRootNode;
import org.enso.interpreter.node.ExpressionNode;
import org.enso.interpreter.runtime.tag.AvoidIdInstrumentationTag;
@ -28,13 +27,13 @@ final class StatementNode extends ExpressionNode {
}
@Override
public SourceSection getSourceSection() {
return node.getSourceSection();
public int[] getSourceSectionBounds() {
return node.getSourceSectionBounds();
}
@Override
public boolean isInstrumentable() {
return getSourceSection() != null && node.isInstrumentable();
return getSourceSectionBounds() != null && node.isInstrumentable();
}
@Override

View File

@ -1,5 +1,6 @@
package org.enso.interpreter.node.callable.resolver;
import com.oracle.truffle.api.CompilerDirectives;
import com.oracle.truffle.api.dsl.Cached;
import com.oracle.truffle.api.dsl.Cached.Shared;
import com.oracle.truffle.api.dsl.GenerateUncached;
@ -213,25 +214,25 @@ public abstract class HostMethodCallNode extends Node {
Object[] args,
@Shared("interop") @CachedLibrary(limit = "LIB_LIMIT") InteropLibrary members,
@Shared("hostValueToEnsoNode") @Cached HostValueToEnsoNode hostValueToEnsoNode) {
var ctx = EnsoContext.get(this);
try {
return hostValueToEnsoNode.execute(members.invokeMember(self, symbol, args));
} catch (UnsupportedMessageException | UnknownIdentifierException e) {
throw new IllegalStateException(
"Impossible to reach here. The member is checked to be invocable.");
CompilerDirectives.transferToInterpreter();
var err = ctx.getBuiltins().error().makeNotInvokable(self);
throw new PanicException(err, e, this);
} catch (ArityException e) {
throw new PanicException(
EnsoContext.get(this)
.getBuiltins()
var err =
ctx.getBuiltins()
.error()
.makeArityError(e.getExpectedMinArity(), e.getExpectedMaxArity(), e.getActualArity()),
this);
.makeArityError(e.getExpectedMinArity(), e.getExpectedMaxArity(), e.getActualArity());
throw new PanicException(err, this);
} catch (UnsupportedTypeException e) {
throw new PanicException(
EnsoContext.get(this)
.getBuiltins()
var err =
ctx.getBuiltins()
.error()
.makeUnsupportedArgumentsError(e.getSuppliedValues(), e.getMessage()),
this);
.makeUnsupportedArgumentsError(e.getSuppliedValues(), e.getMessage());
throw new PanicException(err, this);
}
}
@ -268,8 +269,10 @@ public abstract class HostMethodCallNode extends Node {
try {
return hostValueToEnsoNode.execute(instances.instantiate(self, args));
} catch (UnsupportedMessageException e) {
throw new IllegalStateException(
"Impossible to reach here. The member is checked to be instantiable.");
CompilerDirectives.transferToInterpreter();
var ctx = EnsoContext.get(this);
var err = ctx.getBuiltins().error().makeNotInvokable(self);
throw new PanicException(err, e, this);
} catch (ArityException e) {
throw new PanicException(
EnsoContext.get(this)

View File

@ -25,10 +25,9 @@ public abstract class AddToClassPathNode extends Node {
@CompilerDirectives.TruffleBoundary
@Specialization
Object doExecute(Object path, @Cached ExpectStringNode expectStringNode) {
EnsoContext context = EnsoContext.get(this);
context
.getEnvironment()
.addToHostClassPath(context.getTruffleFile(new File(expectStringNode.execute(path))));
return context.getBuiltins().nothing();
var ctx = EnsoContext.get(this);
var file = ctx.getTruffleFile(new File(expectStringNode.execute(path)));
ctx.addToClassPath(file);
return ctx.getBuiltins().nothing();
}
}

View File

@ -1,5 +1,6 @@
package org.enso.interpreter.node.expression.builtin.interop.java;
import com.oracle.truffle.api.CompilerDirectives;
import com.oracle.truffle.api.dsl.Cached;
import com.oracle.truffle.api.dsl.Specialization;
import com.oracle.truffle.api.nodes.Node;
@ -18,6 +19,7 @@ public abstract class LookupClassNode extends Node {
}
@Specialization
@CompilerDirectives.TruffleBoundary
Object doExecute(Object name, @Cached("build()") ExpectStringNode expectStringNode) {
return EnsoContext.get(this).lookupJavaClass(expectStringNode.execute(name));
}

View File

@ -3,7 +3,6 @@ package org.enso.interpreter.node.expression.literal;
import com.oracle.truffle.api.frame.VirtualFrame;
import com.oracle.truffle.api.nodes.Node;
import com.oracle.truffle.api.nodes.NodeInfo;
import com.oracle.truffle.api.source.SourceSection;
import java.util.function.Predicate;
import org.enso.compiler.core.IR;
import org.enso.compiler.core.ir.Expression;
@ -51,8 +50,8 @@ final class PatchableLiteralNode extends ExpressionNode implements Patchable, Pr
}
@Override
public SourceSection getSourceSection() {
return node.getSourceSection();
public int[] getSourceSectionBounds() {
return node.getSourceSectionBounds();
}
@Override

View File

@ -13,6 +13,7 @@ import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.atomic.AtomicLong;
import java.util.logging.Level;
import org.enso.compiler.Compiler;
import org.enso.compiler.PackageRepository;
@ -48,11 +49,13 @@ import com.oracle.truffle.api.TruffleFile;
import com.oracle.truffle.api.TruffleLanguage;
import com.oracle.truffle.api.TruffleLanguage.Env;
import com.oracle.truffle.api.TruffleLogger;
import com.oracle.truffle.api.interop.InteropException;
import com.oracle.truffle.api.interop.InteropLibrary;
import com.oracle.truffle.api.interop.UnknownIdentifierException;
import com.oracle.truffle.api.interop.UnsupportedMessageException;
import com.oracle.truffle.api.nodes.Node;
import com.oracle.truffle.api.object.Shape;
import com.oracle.truffle.api.source.Source;
import scala.jdk.javaapi.OptionConverters;
@ -60,7 +63,7 @@ import scala.jdk.javaapi.OptionConverters;
* The language context is the internal state of the language that is associated with each thread in
* a running Enso program.
*/
public class EnsoContext {
public final class EnsoContext {
private static final TruffleLanguage.ContextReference<EnsoContext> REFERENCE =
TruffleLanguage.ContextReference.create(EnsoLanguage.class);
@ -381,6 +384,27 @@ public class EnsoContext {
.findFirst();
}
/**
* Modifies the classpath to use to lookup {@code polyglot java} imports.
* @param file the file to register
*/
@TruffleBoundary
public void addToClassPath(TruffleFile file) {
if (findGuestJava() == null) {
environment.addToHostClassPath(file);
} else {
try {
var path = new File(file.toUri()).getAbsoluteFile();
if (!path.exists()) {
throw new IllegalStateException("File not found " + path);
}
InteropLibrary.getUncached().invokeMember(findGuestJava(), "addPath", path.getPath());
} catch (InteropException ex) {
throw new IllegalStateException(ex);
}
}
}
/**
* Tries to lookup a Java class (host symbol in Truffle terminology) by its fully qualified name.
* This method also tries to lookup inner classes. More specifically, if the provided name
@ -399,18 +423,63 @@ public class EnsoContext {
List<String> nestedClassPart =
i < items.size() - 1 ? items.subList(i + 1, items.size()) : List.of();
try {
Object hostSymbol = environment.lookupHostSymbol(pkgName + "." + curClassName);
var hostSymbol = lookupHostSymbol(pkgName, curClassName);
if (nestedClassPart.isEmpty()) {
return hostSymbol;
} else {
return getNestedClass(hostSymbol, nestedClassPart);
var fullInnerClassName = curClassName + "$" + String.join("$", nestedClassPart);
return lookupHostSymbol(pkgName, fullInnerClassName);
}
} catch (RuntimeException ignored) {
} catch (RuntimeException | InteropException ex) {
logger.log(Level.WARNING, null, ex);
}
}
return null;
}
private Object lookupHostSymbol(String pkgName, String curClassName)
throws UnknownIdentifierException, UnsupportedMessageException {
if (findGuestJava() == null) {
return environment.lookupHostSymbol(pkgName + "." + curClassName);
} else {
return InteropLibrary.getUncached().readMember(findGuestJava(), pkgName + "." + curClassName);
}
}
private Object guestJava = this;
@TruffleBoundary
private Object findGuestJava() throws IllegalStateException {
if (guestJava != this) {
return guestJava;
}
guestJava = null;
var envJava = System.getenv("ENSO_JAVA");
if (envJava == null) {
return guestJava;
}
if ("espresso".equals(envJava)) {
var src = Source.newBuilder("java", "<Bindings>", "getbindings.java").build();
try {
guestJava = environment.parsePublic(src).call();
logger.log(Level.SEVERE, "Using experimental Espresso support!");
} catch (Exception ex) {
if (ex.getMessage().contains("No language for id java found.")) {
logger.log(Level.SEVERE, "Environment variable ENSO_JAVA=" + envJava + ", but " + ex.getMessage());
logger.log(Level.SEVERE, "Use " + System.getProperty("java.home") + "/bin/gu install espresso");
logger.log(Level.SEVERE, "Continuing in regular Java mode");
} else {
var ise = new IllegalStateException(ex.getMessage());
ise.setStackTrace(ex.getStackTrace());
throw ise;
}
}
} else {
throw new IllegalStateException("Specify ENSO_JAVA=espresso to use Espresso. Was: " + envJava);
}
return guestJava;
}
/**
* Finds the package the provided module belongs to.
*
@ -605,30 +674,6 @@ public class EnsoContext {
return notificationHandler;
}
private Object getNestedClass(Object hostClass, List<String> nestedClassName) {
Object nestedClass = hostClass;
var interop = InteropLibrary.getUncached();
for (String name : nestedClassName) {
if (interop.isMemberReadable(nestedClass, name)) {
Object member;
try {
member = interop.readMember(nestedClass, name);
} catch (UnsupportedMessageException | UnknownIdentifierException e) {
throw new IllegalStateException(e);
}
assert member != null;
if (interop.isMetaObject(member)) {
nestedClass = member;
} else {
return null;
}
} else {
return null;
}
}
return nestedClass;
}
private <T> T getOption(OptionKey<T> key) {
var options = getEnvironment().getOptions();
var safely = false;

View File

@ -6,6 +6,7 @@ import com.oracle.truffle.api.frame.MaterializedFrame;
import com.oracle.truffle.api.interop.InteropLibrary;
import com.oracle.truffle.api.interop.UnknownIdentifierException;
import com.oracle.truffle.api.interop.UnsupportedMessageException;
import com.oracle.truffle.api.library.CachedLibrary;
import com.oracle.truffle.api.library.ExportLibrary;
import com.oracle.truffle.api.library.ExportMessage;
import com.oracle.truffle.api.source.SourceSection;
@ -77,11 +78,13 @@ public class DebugLocalScope implements EnsoObject {
&& this.bindingsByLevelsIdx < this.bindingsByLevels.size());
}
@TruffleBoundary
public static DebugLocalScope createFromFrame(EnsoRootNode rootNode, MaterializedFrame frame) {
return new DebugLocalScope(
rootNode, frame, gatherBindingsByLevels(rootNode.getLocalScope().flattenBindings()), 0);
}
@TruffleBoundary
private static DebugLocalScope createParent(DebugLocalScope childScope) {
return new DebugLocalScope(
childScope.rootNode,
@ -137,6 +140,7 @@ public class DebugLocalScope implements EnsoObject {
/** Returns the members from the current local scope and all the parent scopes. */
@ExportMessage
@TruffleBoundary
ScopeMembers getMembers(boolean includeInternal) {
List<String> members = new ArrayList<>();
bindingsByLevels.stream().skip(bindingsByLevelsIdx).forEach(members::addAll);
@ -144,6 +148,7 @@ public class DebugLocalScope implements EnsoObject {
}
@ExportMessage
@TruffleBoundary
boolean isMemberModifiable(String memberName) {
return allBindings.containsKey(memberName);
}
@ -170,6 +175,7 @@ public class DebugLocalScope implements EnsoObject {
}
@ExportMessage
@TruffleBoundary
boolean isMemberReadable(String memberName) {
// When a value in a frame is null, it means that the corresponding
// AssignmentNode was not run yet, and the slot kind of the
@ -179,13 +185,15 @@ public class DebugLocalScope implements EnsoObject {
}
@ExportMessage
Object readMember(String member) {
@TruffleBoundary
Object readMember(String member, @CachedLibrary("this") InteropLibrary interop) {
FramePointer framePtr = allBindings.get(member);
var value = getValue(frame, framePtr);
return value != null ? value : DataflowError.UNINITIALIZED;
}
@ExportMessage
@TruffleBoundary
void writeMember(String member, Object value) throws UnknownIdentifierException {
if (!allBindings.containsKey(member)) {
throw UnknownIdentifierException.create(member);
@ -225,6 +233,7 @@ public class DebugLocalScope implements EnsoObject {
}
@ExportMessage
@TruffleBoundary
SourceSection getSourceLocation() {
return rootNode.getSourceSection();
}
@ -236,6 +245,7 @@ public class DebugLocalScope implements EnsoObject {
}
@Override
@TruffleBoundary
public String toString() {
return String.format(
"DebugLocalScope{rootNode = '%s', bindingsByLevels = %s, idx = %d}",

View File

@ -181,7 +181,7 @@ private class DefaultPackageRepository(
isLibrary: Boolean
): Unit = {
val extensions = pkg.listPolyglotExtensions("java")
extensions.foreach(context.getEnvironment.addToHostClassPath)
extensions.foreach(context.addToClassPath)
val (regularModules, syntheticModulesMetadata) = pkg
.listSources()

View File

@ -101,6 +101,12 @@ object NativeImage {
"because Native Image component was not found."
)
}
if (additionalOptions.contains("--language:java")) {
log.warn(
s"Building ${artifactName} image with experimental Espresso support!"
)
}
val debugParameters =
if (includeDebugInfo) Seq("-H:GenerateDebugInfo=1") else Seq()

View File

@ -6,7 +6,6 @@ import Standard.Test.Extensions
polyglot java import java.time.ZoneId
polyglot java import java.time.ZoneOffset
polyglot java import java.time.ZoneRegion
spec =
Test.group "Zone" <|