Proper classpath of engine sources in Enso4Igv plugin (#3810)

This PR modifies `sbt` to record options sent to [frgaal](http://frgaal.org) compiler. Then it reads these options (especially exact classpath used during compilation) from IGV or NetBeans.

# Important Notes
![Open project in IGV](https://user-images.githubusercontent.com/26887752/201684275-b3ee7a37-7b55-4290-b426-75df0280ba32.png)

.gitignore

@@ -66,6 +66,8 @@ package-lock.json
 .projections.json
 .nvmrc
 *.iml
+.enso-sources*
+.metals
 
 ############################
 ## Rendered Documentation ##

CHANGELOG.md

@@ -438,6 +438,7 @@
 - [Initialize Builtins at Native Image build time][3821]
 - [Add the `Self` keyword referring to current type][3844]
 - [Split Atom suggestion entry to Type and Constructor][3835]
+- [Connecting IGV 4 Enso with Engine sources][3810]
 - [Support VCS for projects in Language Server][3851]
 
 [3227]: https://github.com/enso-org/enso/pull/3227
@@ -500,6 +501,7 @@
 [3821]: https://github.com/enso-org/enso/pull/3821
 [3844]: https://github.com/enso-org/enso/pull/3844
 [3835]: https://github.com/enso-org/enso/pull/3835
+[3810]: https://github.com/enso-org/enso/pull/3810
 [3851]: https://github.com/enso-org/enso/pull/3851
 
 # Enso 2.0.0-alpha.18 (2021-10-12)

project/FrgaalJavaCompiler.scala

@@ -20,8 +20,11 @@ import xsbti.compile.{IncToolOptions, Output, JavaCompiler => XJavaCompiler}
 import java.io.File
 import java.nio.file.{Path, Paths}
 import scala.sys.process.Process
+import scala.util.Using
+import java.io.FileWriter
 
 object FrgaalJavaCompiler {
+  private val ENSO_SOURCES = ".enso-sources"
 
   val frgaal = "org.frgaal" % "compiler" % "19.0.0" % "provided"
 
@@ -63,6 +66,54 @@ object FrgaalJavaCompiler {
     val sources = sources0 map {
       case x: PathBasedFile => x.toPath.toAbsolutePath.toString
     }
+    val out = output.getSingleOutputAsPath().get()
+    val shared = sources0.fold(out)((a, b) => {
+      var ap = a match {
+        case p: PathBasedFile => p.toPath
+        case p: Path => p
+      }
+      val bp = b match {
+        case p: PathBasedFile => p.toPath
+        case p: Path => p
+      }
+
+      var i = 0
+      while (i < Math.min(ap.getNameCount(), bp.getNameCount()) && ap.getName(i) == bp.getName(i)) {
+        i += 1;
+      }
+
+      while (ap.getNameCount() > i) {
+        ap = ap.getParent()
+      }
+      ap
+    }).asInstanceOf[Path]
+
+    if (shared.toFile().exists()) {
+      val ensoMarker = new File(shared.toFile(), ENSO_SOURCES)
+      val ensoConfig = new File(shared.toFile(), ENSO_SOURCES + "-" + out.getFileName().toString())
+      val ensoProperties = new java.util.Properties()
+
+      def storeArray(name: String, values : Seq[String]) = {
+        values.zipWithIndex.foreach { case (value, idx) => ensoProperties.setProperty(s"$name.$idx", value) }
+      }
+
+      ensoProperties.setProperty("output", out.toString())
+      storeArray("options", options)
+      source.foreach(v => ensoProperties.setProperty("source", v))
+      ensoProperties.setProperty("target", target)
+      javaHome.foreach(v => ensoProperties.setProperty("java.home", v.toString()))
+
+      Using(new FileWriter(ensoConfig)) { w =>
+        ensoProperties.store(w, "# Enso compiler configuration")
+      }
+      Using(new FileWriter(ensoMarker)) { _ =>
+      }
+    } else {
+      throw new IllegalStateException("Cannot write Enso source options to " + shared + " values:\n" +
+        "options: " + options + " sources0: " + sources +" output: " + output
+      )
+    }
+
     val frgaalOptions: Seq[String] = source.map(v => Seq("-source", v)).getOrElse(Seq()) ++ Seq("-target", target)
     val allArguments = outputOption ++ frgaalOptions ++ nonJArgs ++ sources
 

tools/enso4igv/README.md

@@ -53,9 +53,14 @@ to _finish_ the installation.
 
 ## Using the IGV
 
-Get an instance of the Enso runtime engine (see
-[Running Enso](../../docs/CONTRIBUTING.md#running-enso)) and then launch it with
-special `--dump-graphs` option:
+Build an instance of the Enso runtime engine (see
+[Running Enso](../../docs/CONTRIBUTING.md#running-enso)) using:
+
+```bash
+enso$ sbt buildEngineDistribution
+```
+
+and then launch it with special `--dump-graphs` option:
 
 ```bash
 enso$ ./built-distribution/enso-engine-0.0.0-dev-linux-amd64/enso-0.0.0-dev/bin/enso --dump-graphs --run yourprogram.enso
@@ -89,9 +94,18 @@ graal_dumps/2022.06.20.06.18.21.733/TruffleHotSpotCompilation-9935[Primes.next_<
 graal_dumps/2022.06.20.06.18.21.733/TruffleHotSpotCompilation-9935[Primes.next_<split-717d5bdf>].bgv
 ```
 
-Let's launch IGV with Enso integration and let's open graph for one of the
-top-most functions: `TruffleHotSpotCompilation*Primes*next*.bgv`. Choose
-compilation phase _"Before lowering"_:
+Let's launch IGV with Enso integration. Locate the `engine/runtime` directory
+and open it as _"project"_ in IGV:
+
+![Open Project in IGV](https://user-images.githubusercontent.com/26887752/201684275-b3ee7a37-7b55-4290-b426-75df0280ba32.png)
+
+The project directories (not only `runtime`, but also other like
+`runtime-language-epb`, etc.) are recognized only if you have built the Enso
+engine sources with `sbt buildEngineDistribution`.
+
+With such setup let's open graph for one of the top-most functions:
+`TruffleHotSpotCompilation*Primes*next*.bgv`. Choose compilation phase _"Before
+lowering"_:
 
 ![Before Lowering Graph](https://user-images.githubusercontent.com/26887752/174608397-331a4438-1f12-40b0-9fcd-59eda5e53fb6.png)
 
@@ -102,8 +116,13 @@ it got _inlined_(you can use search box in the top-right corner)
 ![Inlining Stacktrace](https://user-images.githubusercontent.com/26887752/174608478-e7002c43-d746-42c0-b61c-92ceb9d9f124.png)
 
 The stack trace shows what methods of the Enso interpreter and Truffle runtime
-are _"inlined on stack"_ when this node is being compiled. This is all regular
-_IGV_ functionality, but now we can switch to _Enso view_:
+are _"inlined on stack"_ when this node is being compiled. However thanks to
+integration with `engine/runtime` sources one can directly jump to the sources
+of the interpreter that represent certain graph nodes:
+
+![Associated Engine Sources](https://user-images.githubusercontent.com/26887752/201688115-4afdb2ac-9a41-4469-8b7b-d7130f74883e.png)
+
+Not only that, but one we can also switch to _Enso view_:
 
 ![Enso Source](https://user-images.githubusercontent.com/26887752/174608595-4ce80b00-949a-4b28-84a7-60d5988bfc70.png)
 

tools/enso4igv/pom.xml

@@ -5,7 +5,7 @@
     <artifactId>enso4igv</artifactId>
     <packaging>nbm</packaging>
     <name>Enso Language Support for NetBeans &amp; Ideal Graph Visualizer</name>
-    <version>1.0-SNAPSHOT</version>
+    <version>1.3-SNAPSHOT</version>
     <build>
         <plugins>
             <plugin>
@@ -30,6 +30,27 @@
                     </archive>
                 </configuration>
             </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>3.8.1</version>
+                <dependencies>
+                    <dependency>
+                        <groupId>org.frgaal</groupId>
+                        <artifactId>compiler-maven-plugin</artifactId>
+                        <version>19.0.0</version>
+                    </dependency>
+                </dependencies>
+                <configuration>
+                    <compilerId>frgaal</compilerId>
+                    <source>19</source>
+                    <target>1.8</target>
+                    <compilerArgs>
+                        <arg>-Xlint:deprecation</arg>
+                        <arg>--enable-preview</arg>
+                    </compilerArgs>
+                </configuration>
+            </plugin>
         </plugins>
     </build>
     <dependencies>
@@ -43,6 +64,11 @@
             <artifactId>org-netbeans-modules-textmate-lexer</artifactId>
             <version>${netbeans.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.netbeans.api</groupId>
+            <artifactId>org-netbeans-modules-projectapi</artifactId>
+            <version>${netbeans.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.netbeans.api</groupId>
             <artifactId>org-netbeans-api-templates</artifactId>
@@ -103,9 +129,40 @@
             <artifactId>org-openide-dialogs</artifactId>
             <version>${netbeans.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.netbeans.api</groupId>
+            <artifactId>org-netbeans-modules-projectuiapi-base</artifactId>
+            <version>RELEASE113</version>
+            <type>jar</type>
+        </dependency>
+        <dependency>
+            <groupId>org.netbeans.api</groupId>
+            <artifactId>org-netbeans-api-java-classpath</artifactId>
+            <version>RELEASE113</version>
+            <type>jar</type>
+        </dependency>
+        <dependency>
+            <groupId>org.netbeans.api</groupId>
+            <artifactId>org-netbeans-modules-java-project</artifactId>
+            <version>RELEASE113</version>
+            <type>jar</type>
+        </dependency>
+        <dependency>
+            <groupId>org.netbeans.api</groupId>
+            <artifactId>org-netbeans-modules-java-platform</artifactId>
+            <version>RELEASE113</version>
+            <type>jar</type>
+        </dependency>
+        <dependency>
+            <groupId>org.netbeans.api</groupId>
+            <artifactId>org-netbeans-api-java</artifactId>
+            <version>RELEASE113</version>
+            <type>jar</type>
+        </dependency>
     </dependencies>
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <netbeans.version>RELEASE113</netbeans.version>
+        <netbeans.compile.on.save>none</netbeans.compile.on.save>
     </properties>
 </project>

tools/enso4igv/src/main/java/org/enso/tools/enso4igv/EnsoSbtClassPathProvider.java

@@ -0,0 +1,185 @@
+package org.enso.tools.enso4igv;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Properties;
+import javax.swing.event.ChangeListener;
+import org.netbeans.api.java.classpath.ClassPath;
+import org.netbeans.api.java.classpath.GlobalPathRegistry;
+import org.netbeans.api.java.platform.JavaPlatform;
+import org.netbeans.spi.java.classpath.ClassPathProvider;
+import org.netbeans.spi.java.classpath.support.ClassPathSupport;
+import org.netbeans.spi.java.project.support.ProjectPlatform;
+import org.netbeans.spi.java.queries.CompilerOptionsQueryImplementation;
+import org.netbeans.spi.java.queries.SourceLevelQueryImplementation2;
+import org.netbeans.spi.project.ui.ProjectOpenedHook;
+import org.openide.filesystems.FileObject;
+import org.openide.filesystems.FileUtil;
+import org.openide.util.Exceptions;
+
+final class EnsoSbtClassPathProvider extends ProjectOpenedHook
+implements ClassPathProvider, SourceLevelQueryImplementation2, CompilerOptionsQueryImplementation {
+    private static final String BOOT = "classpath/boot";
+    private static final String SOURCE = "classpath/source";
+    private static final String COMPILE = "classpath/compile";
+    private final EnsoSbtProject project;
+    private final EnsoSources info;
+
+    EnsoSbtClassPathProvider(EnsoSbtProject prj) {
+        this.project = prj;
+        this.info = computeSbtClassPath(prj);
+    }
+
+    @Override
+    public ClassPath findClassPath(FileObject file, String type) {
+        if (FileUtil.isParentOf(project.getProjectDirectory(), file)) {
+            return switch (type) {
+                case SOURCE -> info.srcCp;
+                case COMPILE -> info.cp;
+                case BOOT -> info.platform.getBootstrapLibraries();
+                default -> null;
+            };
+        } else {
+            return null;
+        }
+    }
+
+    @Override
+    public void projectOpened() {
+        GlobalPathRegistry.getDefault().register(COMPILE, new ClassPath[] { info.cp });
+        GlobalPathRegistry.getDefault().register(SOURCE, new ClassPath[] { info.srcCp });
+    }
+
+    @Override
+    public void projectClosed() {
+        GlobalPathRegistry.getDefault().unregister(COMPILE, new ClassPath[] { info.cp });
+        GlobalPathRegistry.getDefault().unregister(SOURCE, new ClassPath[] { info.srcCp });
+    }
+
+    private static EnsoSources computeSbtClassPath(EnsoSbtProject prj) {
+        var platform = JavaPlatform.getDefault();
+        var roots = new LinkedHashSet<>();
+        var generatedSources = new LinkedHashSet<>();
+        var source = "11";
+        var options = new ArrayList<String>();
+        for (FileObject ch : prj.getProjectDirectory().getChildren()) {
+            if (ch.getNameExt().startsWith(".enso-sources")) {
+                Properties p = new Properties();
+                try (InputStream is = ch.getInputStream()) {
+                    p.load(is);
+                } catch (IOException ex) {
+                    Exceptions.printStackTrace(ex);
+                }
+                if (p.get("java.home") instanceof String javaHome) {
+                    var javaHomeFile = new File(javaHome);
+                    var javaHomeFo = FileUtil.toFileObject(javaHomeFile);
+                    platform = ProjectPlatform.forProject(prj, javaHomeFo, javaHomeFile.getName(), "j2se");
+                }
+
+                for (var i = 0; ; i++) {
+                    final String prop = p.getProperty("options." + i);
+                    if (prop == null) {
+                        break;
+                    }
+                    var next = p.getProperty("options." + (i + 1));
+                    if ("-source".equals(prop) && next != null) {
+                        source = next;
+                        i++;
+                        continue;
+                    }
+                    if ("-classpath".equals(prop) && next != null) {
+                        var paths = next.split(File.pathSeparator);
+                        for (var element : paths) {
+                            File file = new File(element);
+                            FileObject fo = FileUtil.toFileObject(file);
+                            if (fo != null) {
+                                if (fo.isFolder()) {
+                                    roots.add(fo);
+                                } else {
+                                    var jarRoot = FileUtil.getArchiveRoot(fo);
+                                    roots.add(jarRoot);
+                                }
+                            }
+                        }
+                        i++;
+                        continue;
+                    }
+                    if ("-s".equals(prop) && next != null) {
+                        var fo = FileUtil.toFileObject(new File(next));
+                        if (fo != null) {
+                            generatedSources.add(fo);
+                        }
+                    }
+                    options.add(prop);
+                }
+            }
+        }
+
+        var srcRoots = new LinkedHashSet<>();
+        var srcDir = prj.getProjectDirectory().getFileObject("src");
+        if (srcDir != null) {
+            for (var group : srcDir.getChildren()) {
+                if (group.isFolder()) {
+                    for (var ch : group.getChildren()) {
+                        if (ch.isFolder()) {
+                            srcRoots.add(ch);
+                        }
+                    }
+                }
+            }
+        }
+        srcRoots.addAll(generatedSources);
+        var cp = ClassPathSupport.createClassPath(roots.toArray(new FileObject[0]));
+        var srcCp = ClassPathSupport.createClassPath(srcRoots.toArray(new FileObject[0]));
+        return new EnsoSources(cp, srcCp, platform, source, options);
+    }
+
+    @Override
+    public SourceLevelQueryImplementation2.Result getSourceLevel(FileObject fo) {
+        return new SourceLevelQueryImplementation2.Result() {
+            @Override
+            public String getSourceLevel() {
+                return info.source;
+            }
+
+            @Override
+            public void addChangeListener(ChangeListener cl) {
+            }
+
+            @Override
+            public void removeChangeListener(ChangeListener cl) {
+            }
+        };
+    }
+
+    @Override
+    public CompilerOptionsQueryImplementation.Result getOptions(FileObject fo) {
+        return new CompilerOptionsQueryImplementation.Result() {
+            @Override
+            public List<? extends String> getArguments() {
+                return info.options;
+            }
+
+            @Override
+            public void addChangeListener(ChangeListener cl) {
+            }
+
+            @Override
+            public void removeChangeListener(ChangeListener cl) {
+            }
+        };
+    }
+
+
+
+    private record EnsoSources(
+        ClassPath cp, ClassPath srcCp,
+        JavaPlatform platform,
+        String source, List<String> options
+    ) {
+    }
+}

tools/enso4igv/src/main/java/org/enso/tools/enso4igv/EnsoSbtProject.java

@@ -0,0 +1,68 @@
+package org.enso.tools.enso4igv;
+
+import java.io.IOException;
+import org.netbeans.api.project.Project;
+import org.netbeans.api.project.ProjectManager;
+import org.netbeans.spi.project.ProjectFactory;
+import org.netbeans.spi.project.ProjectFactory2;
+import org.netbeans.spi.project.ProjectState;
+import org.openide.filesystems.FileObject;
+import org.openide.util.ImageUtilities;
+import org.openide.util.Lookup;
+import org.openide.util.lookup.Lookups;
+import org.openide.util.lookup.ServiceProvider;
+
+public class EnsoSbtProject implements Project {
+    private final FileObject prj;
+    private final ProjectState ps;
+    private final Lookup lkp;
+
+    private EnsoSbtProject(FileObject fo, ProjectState ps) {
+        this.prj = fo;
+        this.ps = ps;
+        this.lkp = Lookups.fixed(
+            this,
+            new EnsoSbtClassPathProvider(this)
+        );
+    }
+
+    public FileObject getProjectDirectory() {
+        return prj;
+    }
+
+    public Lookup getLookup() {
+        return lkp;
+    }
+
+    @Override
+    public String toString() {
+        return "EnsoSbtProject{prj=" + prj + "}";
+    }
+
+    @ServiceProvider(service = ProjectFactory.class)
+    public static final class Factory implements ProjectFactory2 {
+        public boolean isProject(FileObject fo) {
+            return fo.getFileObject(".enso-sources") != null;
+        }
+
+        public Project loadProject(FileObject fo, ProjectState ps) throws IOException {
+            if (isProject(fo)) {
+                return new EnsoSbtProject(fo, ps);
+            } else {
+                return null;
+            }
+        }
+
+        public void saveProject(Project prjct) throws IOException, ClassCastException {
+        }
+
+        @Override
+        public ProjectManager.Result isProject2(FileObject fo) {
+            if (isProject(fo)) {
+                var img = ImageUtilities.loadImage("org/enso/tools/enso4igv/enso.png");
+                return new ProjectManager.Result(ImageUtilities.image2Icon(img));
+            }
+            return null;
+        }
+    }
+}

tools/enso4igv/src/main/nbm/manifest.mf

@@ -1,4 +1,5 @@
 Manifest-Version: 1.0
 OpenIDE-Module-Layer: org/enso/tools/enso4igv/layer.xml
 OpenIDE-Module-Localizing-Bundle: org/enso/tools/enso4igv/Bundle.properties
-
+Multi-Release: true
+OpenIDE-Module-Recommends: cnb.org.netbeans.modules.java.kit