Updating the code to be more new parser friendly (#3853)

To minimize differences between #3611 and `develop` branch I propose to make following code changes that seem to work fine with the old as well as new parser. In addition to that there are new tests comparing the two parsers. # Important Notes Old parser is still used everywhere except `EnsoCompilerTest` where the test compares `IR`s constructed by both parsers.
2024-12-23 20:16:47 +03:00 · 2022-11-04 19:08:24 +01:00 · 2022-11-04 19:08:24 +01:00 · 1273d8ac91
commit 1273d8ac91
parent b5881efdf0
8 changed files with 192 additions and 92 deletions
--- a/engine/runtime/src/main/java/org/enso/compiler/TreeToIr.java
+++ b/engine/runtime/src/main/java/org/enso/compiler/TreeToIr.java
@ -1,5 +1,6 @@
 package org.enso.compiler;

+import java.util.UUID;
 import org.enso.compiler.core.IR;
 import org.enso.compiler.core.IR$Application$Literal$Sequence;
 import org.enso.compiler.core.IR$Application$Operator$Binary;
@ -166,7 +167,7 @@ final class TreeToIr {
          methodRef,
          args,
          body,
-          getIdentifiedLocation(inputAst, 0, 1),
+          getIdentifiedLocation(inputAst, 0, 1, null),
          meta(), diag()
        );
        yield cons(binding, appendTo);
@ -566,7 +567,7 @@ final class TreeToIr {
              );
            }
            var at = expandToContain(switch (body) {
-              case IR$Expression$Block __ -> getIdentifiedLocation(tree, 0, 1);
+              case IR$Expression$Block __ -> getIdentifiedLocation(tree, 0, 1, null);
              default -> getIdentifiedLocation(tree);
            }, body.location());
            yield new IR$Function$Lambda(args, body, at, true, meta(), diag());
@ -659,10 +660,10 @@ final class TreeToIr {
          expressions.remove(expressions.size()-1);
        }
        var list = CollectionConverters.asScala(expressions.iterator()).toList();
-        var locationWithANewLine = getIdentifiedLocation(body, 0, 1);
+        var locationWithANewLine = getIdentifiedLocation(body, 0, 1, null);
        if (last != null && last.location().isDefined() && last.location().get().end() != locationWithANewLine.get().end()) {
            var patched = new Location(last.location().get().start(), locationWithANewLine.get().end() - 1);
-            var id = new IdentifiedLocation(patched, locationWithANewLine.get().id());
+            var id = new IdentifiedLocation(patched, last.location().get().id());
            last = last.setLocation(Option.apply(id));
        }
        yield new IR$Expression$Block(list, last, locationWithANewLine, false, meta(), diag());
@ -713,7 +714,7 @@ final class TreeToIr {
          yield switch (translateExpression(group.getBody(), false)) {
              case null -> new IR$Error$Syntax(getIdentifiedLocation(group).get(), IR$Error$Syntax$EmptyParentheses$.MODULE$, meta(), diag());
              case IR$Application$Prefix pref -> {
-                  final Option<IdentifiedLocation> groupWithoutParenthesis = getIdentifiedLocation(group, 1, -1);
+                  final Option<IdentifiedLocation> groupWithoutParenthesis = getIdentifiedLocation(group, 1, -1, pref.getExternalId());
                  yield pref.setLocation(groupWithoutParenthesis);
              }
              case IR.Expression in -> in;
@ -780,7 +781,7 @@ final class TreeToIr {
        var methodReference = new IR$CallArgument$Specified(
                Option.empty(),
                methodName,
-                getIdentifiedLocation(sig),
+                methodName.location(),
                meta(), diag()
        );
        var opName = buildName(Option.empty(), sig.getOperator(), true);
@ -1287,17 +1288,20 @@ final class TreeToIr {
  }

  private Option<IdentifiedLocation> getIdentifiedLocation(Tree ast) {
-    return getIdentifiedLocation(ast, 0, 0);
+    var someId = Option.apply(ast.uuid());
+    return getIdentifiedLocation(ast, 0, 0, someId);
  }
-  private Option<IdentifiedLocation> getIdentifiedLocation(Tree ast, int b, int e) {
+  private Option<IdentifiedLocation> getIdentifiedLocation(Tree ast, int b, int e, Option<UUID> someId) {
+    if (someId == null) {
+      someId = Option.apply(ast.uuid());
+    }
    return Option.apply(switch (ast) {
-        case null -> null;
-        default -> {
-            var begin = Math.toIntExact(ast.getStartCode()) + b;
-            var end = Math.toIntExact(ast.getEndCode()) + e;
-            var someId = Option.apply(ast.uuid());
-            yield new IdentifiedLocation(new Location(begin, end), someId);
-        }
+      case null -> null;
+      default -> {
+        var begin = Math.toIntExact(ast.getStartCode()) + b;
+        var end = Math.toIntExact(ast.getEndCode()) + e;
+        yield new IdentifiedLocation(new Location(begin, end), someId);
+      }
    });
  }

--- a/engine/runtime/src/main/scala/org/enso/compiler/Compiler.scala
+++ b/engine/runtime/src/main/scala/org/enso/compiler/Compiler.scala
@ -53,7 +53,8 @@ class Compiler(
  )
  private val serializationManager: SerializationManager =
    new SerializationManager(this)
-  private val logger: TruffleLogger = context.getLogger(getClass)
+  private val logger: TruffleLogger      = context.getLogger(getClass)
+  private val ensoCompiler: EnsoCompiler = new EnsoCompiler();

  /** Run the initialization sequence. */
  def initialize(): Unit = {
@ -437,8 +438,21 @@ class Compiler(
      compilerConfig   = config,
      isGeneratingDocs = isGenDocs
    )
-    val parsedAST = parse(module.getSource)
-    val expr      = generateIR(parsedAST)
+
+    val src = module.getSource
+    def oldParser() = {
+      val tree = parse(src)
+      generateIR(tree)
+    }
+    def newParser() = {
+      System.err.println("Using new parser to process " + src.getURI())
+      val tree = ensoCompiler.parse(src)
+      ensoCompiler.generateIR(tree)
+    }
+    val size = src.getCharacters().length()
+    // change the condition to use old or new parser
+    val expr = if (size >= 0) oldParser() else newParser()
+
    val exprWithModuleExports =
      if (module.isSynthetic)
        expr
--- a/engine/runtime/src/test/java/org/enso/compiler/EnsoCompilerTest.java
+++ b/engine/runtime/src/test/java/org/enso/compiler/EnsoCompilerTest.java
@ -61,6 +61,26 @@ public class EnsoCompilerTest {
    """, true, false, true);
  }

+  @Test
+  public void testLocationsMethodWithComplexBody() throws Exception {
+    parseTest("""
+    foo a b =
+        x : Number
+        x = a + 1
+        y = b - 2
+        x * y
+    """, true, false, true);
+  }
+
+  @Test
+  public void testLocationsBuildFunctionSimple() throws Exception {
+    parseTest("""
+    main =
+        foo a = a + 1
+        foo 42
+    """, true, false, true);
+  }
+
  @Test
  public void testLocationsDeeplyNestedFunctions() throws Exception {
    parseTest("""
@ -350,9 +370,13 @@ public class EnsoCompilerTest {
  @Ignore
  public void testMetadataRaw() throws Exception {
    parseTest("""
-    main = 4
+    main =
+        foo = 42
+
+
    #### METADATA ####
-    [[{"index":{"value":7},"size":{"value":8}},"5bad897e-099b-4b00-9348-64092636746d"]]
+    [[{"index": {"value": 17}, "size": {"value": 2}}, "0270bcdf-26b8-4b99-8745-85b3600c7359"]]
+    []
    """);
  }

--- a/engine/runtime/src/test/java/org/enso/interpreter/test/CodeIdsTestInstrument.java
+++ b/engine/runtime/src/test/java/org/enso/interpreter/test/CodeIdsTestInstrument.java
@ -3,9 +3,10 @@ package org.enso.interpreter.test;
 import com.oracle.truffle.api.frame.VirtualFrame;
 import com.oracle.truffle.api.instrumentation.*;
 import com.oracle.truffle.api.nodes.Node;
+import java.util.LinkedHashMap;
+import java.util.Map;
 import org.enso.interpreter.node.ExpressionNode;
 import org.enso.interpreter.runtime.control.TailCallException;
-import org.enso.interpreter.runtime.tag.IdentifiedTag;

 import java.util.UUID;

@ -37,12 +38,13 @@ public class CodeIdsTestInstrument extends TruffleInstrument {
   * An event listener implementing the behavior of verifying whether the currently executed node is
   * the one expected by the user.
   */
-  public static class IdEventListener implements ExecutionEventListener {
+  public static class IdEventListener implements ExecutionEventNodeFactory {
    private boolean successful = false;
    private final UUID expectedId;
    private final String expectedResult;
+    private final Map<IdEventNode, Object> nodes = new LinkedHashMap<>();

-    public IdEventListener(UUID expectedId, String expectedResult) {
+    IdEventListener(UUID expectedId, String expectedResult) {
      this.expectedId = expectedId;
      this.expectedResult = expectedResult;
    }
@ -65,52 +67,88 @@ public class CodeIdsTestInstrument extends TruffleInstrument {
      return expectedResult;
    }

-    @Override
-    public void onEnter(EventContext context, VirtualFrame frame) {}
-
-    /**
-     * Checks if the node to be executed is the node this listener was created to observe.
-     *
-     * @param context current execution context
-     * @param frame current execution frame
-     * @param result the result of executing the node
-     */
-    @Override
-    public void onReturnValue(EventContext context, VirtualFrame frame, Object result) {
-      if (successful) {
-        return;
-      }
-      Node node = context.getInstrumentedNode();
-      if (!(node instanceof ExpressionNode)) {
-        return;
-      }
-      UUID id = ((ExpressionNode) node).getId();
-      if (!id.equals(expectedId)) {
-        return;
-      }
-      if (expectedResult != null && expectedResult.equals(result.toString())) {
-        successful = true;
+    public String dumpNodes() {
+      var sb = new StringBuilder();
+      for (var n : nodes.entrySet()) {
+        sb.append("\nvalue ").append(n.getValue()).append("  for " ).append(n.getKey().toString());
      }
+      return sb.toString();
    }

-    /**
-     * Checks if the specified was called, if its execution triggered TCO.
-     *
-     * @param context current execution context.
-     * @param frame current execution frame.
-     * @param exception the exception thrown from this node's execution.
-     */
    @Override
-    public void onReturnExceptional(EventContext context, VirtualFrame frame, Throwable exception) {
-      if (!(exception instanceof TailCallException)) {
-        return;
+    public ExecutionEventNode create(EventContext context) {
+      var node = new IdEventNode(context);
+      nodes.put(node, null);
+      return node;
+    }
+
+    private final class IdEventNode extends ExecutionEventNode {
+      private final EventContext context;
+
+      IdEventNode(EventContext context) {
+        this.context = context;
      }
-      if (!(context.getInstrumentedNode() instanceof ExpressionNode)) {
-        return;
+
+      @Override
+      public void onEnter(VirtualFrame frame) {}
+
+      /**
+       * Checks if the node to be executed is the node this listener was created to observe.
+       *
+       * @param context current execution context
+       * @param frame current execution frame
+       * @param result the result of executing the node
+       */
+      @Override
+      public void onReturnValue(VirtualFrame frame, Object result) {
+        if (successful) {
+          return;
+        }
+        Node node = context.getInstrumentedNode();
+        if (!(node instanceof ExpressionNode)) {
+          return;
+        }
+        nodes.put(this, result);
+        UUID id = ((ExpressionNode) node).getId();
+        if (id == null || !id.equals(expectedId)) {
+          return;
+        }
+        if (expectedResult != null && expectedResult.equals(result.toString())) {
+          successful = true;
+        }
      }
-      UUID id = ((ExpressionNode) context.getInstrumentedNode()).getId();
-      if (expectedResult == null) {
-        successful = true;
+
+      /**
+       * Checks if the specified was called, if its execution triggered TCO.
+       *
+       * @param context current execution context.
+       * @param frame current execution frame.
+       * @param exception the exception thrown from this node's execution.
+       */
+      @Override
+      public void onReturnExceptional(VirtualFrame frame, Throwable exception) {
+        if (!(exception instanceof TailCallException)) {
+          return;
+        }
+        if (!(context.getInstrumentedNode() instanceof ExpressionNode)) {
+          return;
+        }
+        UUID id = ((ExpressionNode) context.getInstrumentedNode()).getId();
+        if (expectedResult == null) {
+          successful = true;
+        }
+      }
+
+      @Override
+      public String toString() {
+        var sb = new StringBuilder();
+        sb.append(context.getInstrumentedNode().getClass().getSimpleName());
+        if (context.getInstrumentedNode() instanceof ExpressionNode expr) {
+            sb.append("@").append(expr.getId());
+        }
+        sb.append(" ");
+        sb.append(context.getInstrumentedSourceSection());
+        return sb.toString();
      }
    }
  }
@ -124,10 +162,13 @@ public class CodeIdsTestInstrument extends TruffleInstrument {
   * @return a reference to attached event listener
   */
  public EventBinding<IdEventListener> bindTo(UUID id, String expectedResult) {
-    return env.getInstrumenter()
-        .attachExecutionEventListener(
-            SourceSectionFilter.newBuilder().tagIs(IdentifiedTag.class).build(),
-            new IdEventListener(id, expectedResult));
+    var testSource = SourceFilter.newBuilder().sourceIs((t) -> t.getName().equals("Test")).build();
+    var eventFilter =
+        SourceSectionFilter.newBuilder()
+            .sourceFilter(testSource)
+            .build();
+    var factory = new IdEventListener(id, expectedResult);
+    return env.getInstrumenter().attachExecutionEventFactory(eventFilter, factory);
  }

  /**
--- a/engine/runtime/src/test/java/org/enso/interpreter/test/CodeLocationsTestInstrument.java
+++ b/engine/runtime/src/test/java/org/enso/interpreter/test/CodeLocationsTestInstrument.java
@ -4,6 +4,7 @@ import com.oracle.truffle.api.frame.VirtualFrame;
 import com.oracle.truffle.api.instrumentation.EventBinding;
 import com.oracle.truffle.api.instrumentation.EventContext;
 import com.oracle.truffle.api.instrumentation.ExecutionEventListener;
+import com.oracle.truffle.api.instrumentation.SourceFilter;
 import com.oracle.truffle.api.instrumentation.SourceSectionFilter;
 import com.oracle.truffle.api.instrumentation.TruffleInstrument;
 import com.oracle.truffle.api.nodes.Node;
@ -153,9 +154,13 @@ public class CodeLocationsTestInstrument extends TruffleInstrument {
   */
  public EventBinding<LocationsEventListener> bindTo(
      int sourceStart, int diff, int length, int lengthDiff, Class<?> type) {
+    var testSource = SourceFilter.newBuilder().sourceIs((t) -> t.getName().equals("Test")).build();
    return env.getInstrumenter()
        .attachExecutionEventListener(
-            SourceSectionFilter.newBuilder().indexIn(sourceStart, length).build(),
+            SourceSectionFilter.newBuilder()
+                .sourceFilter(testSource)
+                .indexIn(sourceStart, length)
+                .build(),
            new LocationsEventListener(sourceStart, diff, length, lengthDiff, type));
  }
 }
--- a/engine/runtime/src/test/scala/org/enso/compiler/test/context/SuggestionBuilderTest.scala
+++ b/engine/runtime/src/test/scala/org/enso/compiler/test/context/SuggestionBuilderTest.scala
@ -45,6 +45,9 @@ class SuggestionBuilderTest extends AnyWordSpecLike with Matchers {
    Vector()
  )

+  def endOfLine(line: Int, character: Int): Suggestion.Position =
+    Suggestion.Position(line, character)
+
  "SuggestionBuilder" should {

    "build method without explicit arguments" in {
@ -304,7 +307,7 @@ class SuggestionBuilderTest extends AnyWordSpecLike with Matchers {
                  "x",
                  "Number",
                  Suggestion
-                    .Scope(Suggestion.Position(0, 9), Suggestion.Position(4, 9))
+                    .Scope(Suggestion.Position(0, 9), endOfLine(4, 9))
                ),
                Vector()
              ),
@ -315,7 +318,7 @@ class SuggestionBuilderTest extends AnyWordSpecLike with Matchers {
                  "y",
                  SuggestionBuilder.Any,
                  Suggestion
-                    .Scope(Suggestion.Position(0, 9), Suggestion.Position(4, 9))
+                    .Scope(Suggestion.Position(0, 9), endOfLine(4, 9))
                ),
                Vector()
              )
@ -886,7 +889,8 @@ class SuggestionBuilderTest extends AnyWordSpecLike with Matchers {
      val code =
        """main =
          |    foo a = a + 1
-          |    foo 42""".stripMargin
+          |    foo 42
+          |""".stripMargin
      val module = code.preprocessModule

      build(code, module) shouldEqual Tree.Root(
@ -916,7 +920,7 @@ class SuggestionBuilderTest extends AnyWordSpecLike with Matchers {
                  returnType = SuggestionBuilder.Any,
                  scope = Suggestion.Scope(
                    Suggestion.Position(0, 6),
-                    Suggestion.Position(2, 10)
+                    Suggestion.Position(3, 0)
                  )
                ),
                Vector()
@ -934,7 +938,8 @@ class SuggestionBuilderTest extends AnyWordSpecLike with Matchers {
          |    foo a =
          |        b = a + 1
          |        b
-          |    foo 42""".stripMargin
+          |    foo 42
+          |""".stripMargin
      val module = code.preprocessModule

      build(code, module) shouldEqual Tree.Root(
@ -964,7 +969,7 @@ class SuggestionBuilderTest extends AnyWordSpecLike with Matchers {
                  returnType = SuggestionBuilder.Any,
                  scope = Suggestion.Scope(
                    Suggestion.Position(0, 6),
-                    Suggestion.Position(4, 10)
+                    Suggestion.Position(5, 0)
                  )
                ),
                Vector(
@ -976,7 +981,7 @@ class SuggestionBuilderTest extends AnyWordSpecLike with Matchers {
                      returnType = SuggestionBuilder.Any,
                      scope = Suggestion.Scope(
                        Suggestion.Position(1, 11),
-                        Suggestion.Position(3, 9)
+                        endOfLine(3, 9)
                      )
                    ),
                    Vector()
@ -995,7 +1000,8 @@ class SuggestionBuilderTest extends AnyWordSpecLike with Matchers {
        """main =
          |    foo : Number -> Number
          |    foo a = a + 1
-          |    foo 42""".stripMargin
+          |    foo 42
+          |""".stripMargin
      val module = code.preprocessModule

      build(code, module) shouldEqual Tree.Root(
@ -1024,7 +1030,7 @@ class SuggestionBuilderTest extends AnyWordSpecLike with Matchers {
                  returnType = "Number",
                  scope = Suggestion.Scope(
                    Suggestion.Position(0, 6),
-                    Suggestion.Position(3, 10)
+                    Suggestion.Position(4, 0)
                  )
                ),
                Vector()
@ -1043,7 +1049,8 @@ class SuggestionBuilderTest extends AnyWordSpecLike with Matchers {
          |main =
          |    foo : A -> A
          |    foo a = a + 1
-          |    foo 42""".stripMargin
+          |    foo 42
+          |""".stripMargin
      val module = code.preprocessModule

      build(code, module) shouldEqual Tree.Root(
@ -1091,7 +1098,7 @@ class SuggestionBuilderTest extends AnyWordSpecLike with Matchers {
                  returnType = "Unnamed.Test.A",
                  scope = Suggestion.Scope(
                    Suggestion.Position(2, 6),
-                    Suggestion.Position(5, 10)
+                    Suggestion.Position(6, 0)
                  )
                ),
                Vector()
@ -1107,7 +1114,8 @@ class SuggestionBuilderTest extends AnyWordSpecLike with Matchers {
      val code =
        """main =
          |    foo = 42
-          |    foo""".stripMargin
+          |    foo
+          |""".stripMargin
      val module = code.preprocessModule

      build(code, module) shouldEqual Tree.Root(
@ -1133,7 +1141,7 @@ class SuggestionBuilderTest extends AnyWordSpecLike with Matchers {
                  returnType = SuggestionBuilder.Any,
                  scope = Suggestion.Scope(
                    Suggestion.Position(0, 6),
-                    Suggestion.Position(2, 7)
+                    Suggestion.Position(3, 0)
                  )
                ),
                Vector()
@ -1151,7 +1159,8 @@ class SuggestionBuilderTest extends AnyWordSpecLike with Matchers {
          |    foo =
          |        b = 42
          |        b
-          |    foo""".stripMargin
+          |    foo
+          |""".stripMargin
      val module = code.preprocessModule

      build(code, module) shouldEqual Tree.Root(
@ -1177,7 +1186,7 @@ class SuggestionBuilderTest extends AnyWordSpecLike with Matchers {
                  returnType = SuggestionBuilder.Any,
                  scope = Suggestion.Scope(
                    Suggestion.Position(0, 6),
-                    Suggestion.Position(4, 7)
+                    Suggestion.Position(5, 0)
                  )
                ),
                Vector(
@ -1189,7 +1198,7 @@ class SuggestionBuilderTest extends AnyWordSpecLike with Matchers {
                      returnType = SuggestionBuilder.Any,
                      scope = Suggestion.Scope(
                        Suggestion.Position(1, 9),
-                        Suggestion.Position(3, 9)
+                        endOfLine(3, 9)
                      )
                    ),
                    Vector()
@ -1208,7 +1217,8 @@ class SuggestionBuilderTest extends AnyWordSpecLike with Matchers {
        """main =
          |    foo : Number
          |    foo = 42
-          |    foo""".stripMargin
+          |    foo
+          |""".stripMargin
      val module = code.preprocessModule

      build(code, module) shouldEqual Tree.Root(
@ -1234,7 +1244,7 @@ class SuggestionBuilderTest extends AnyWordSpecLike with Matchers {
                  returnType = "Number",
                  scope = Suggestion.Scope(
                    Suggestion.Position(0, 6),
-                    Suggestion.Position(3, 7)
+                    Suggestion.Position(4, 0)
                  )
                ),
                Vector()
@ -1253,7 +1263,8 @@ class SuggestionBuilderTest extends AnyWordSpecLike with Matchers {
          |main =
          |    foo : A
          |    foo = A
-          |    foo""".stripMargin
+          |    foo
+          |""".stripMargin
      val module = code.preprocessModule

      build(code, module) shouldEqual Tree.Root(
@ -1290,7 +1301,7 @@ class SuggestionBuilderTest extends AnyWordSpecLike with Matchers {
                  returnType = "Unnamed.Test.A",
                  scope = Suggestion.Scope(
                    Suggestion.Position(2, 6),
-                    Suggestion.Position(5, 7)
+                    Suggestion.Position(6, 0)
                  )
                ),
                Vector()
@ -2082,7 +2093,7 @@ class SuggestionBuilderTest extends AnyWordSpecLike with Matchers {
                  returnType = SuggestionBuilder.Any,
                  scope = Suggestion.Scope(
                    Suggestion.Position(0, 6),
-                    Suggestion.Position(2, 28)
+                    endOfLine(2, 28)
                  )
                ),
                Vector()
@ -2131,7 +2142,7 @@ class SuggestionBuilderTest extends AnyWordSpecLike with Matchers {
                  returnType = SuggestionBuilder.Any,
                  scope = Suggestion.Scope(
                    Suggestion.Position(0, 6),
-                    Suggestion.Position(2, 18)
+                    endOfLine(2, 18)
                  )
                ),
                Vector()
--- a/engine/runtime/src/test/scala/org/enso/interpreter/test/InterpreterTest.scala
+++ b/engine/runtime/src/test/scala/org/enso/interpreter/test/InterpreterTest.scala
@ -77,7 +77,8 @@ case class IdsInstrumenter(instrument: CodeIdsTestInstrument) {
      if (!listener.isSuccessful) {
        Assertions.fail(
          s"Node with id ${listener.getId} does not exist or did not return the" +
-          s" correct value (expected ${listener.getExpectedResult}."
+          s" correct value (expected ${listener.getExpectedResult}.\n" +
+          s"${listener.dumpNodes()}"
        )
      }
    }
--- a/engine/runtime/src/test/scala/org/enso/interpreter/test/semantic/ExpressionIdTest.scala
+++ b/engine/runtime/src/test/scala/org/enso/interpreter/test/semantic/ExpressionIdTest.scala
@ -102,7 +102,7 @@ class ExpressionIdTest extends InterpreterTest {
          |    foo x + foo y
          |""".stripMargin
      val meta = new Metadata
-      val id1  = meta.addItem(126, 109)
+      val id1  = meta.addItem(126, 109, "1111")
      val id2  = meta.addItem(172, 7)
      val id3  = meta.addItem(192, 9)
      val id4  = meta.addItem(229, 5)