More robust work with caches (#8393)

engine/runtime-compiler/src/main/scala/org/enso/compiler/Compiler.scala

@@ -18,6 +18,7 @@ import org.enso.compiler.core.ir.{
   Warning,
   Module => IRModule
 }
+import org.enso.compiler.core.ir.MetadataStorage.MetadataPair
 import org.enso.compiler.core.ir.expression.Error
 import org.enso.compiler.core.ir.module.scope.Export
 import org.enso.compiler.core.ir.module.scope.Import
@@ -587,9 +588,17 @@ class Compiler(
         injectSyntheticModuleExports(expr, module.getDirectModulesRefs)
     val discoveredModule =
       recognizeBindings(exprWithModuleExports, moduleContext)
+    if (context.wasLoadedFromCache(module)) {
+      if (module.getBindingsMap() != null) {
+        discoveredModule.passData.update(
+          new MetadataPair(BindingAnalysis, module.getBindingsMap())
+        )
+      }
+    }
     context.updateModule(
       module,
       { u =>
+        u.bindingsMap(null);
         u.ir(discoveredModule)
         u.compilationStage(CompilationStage.AFTER_PARSING)
         u.loadedFromCache(false)

engine/runtime-parser/src/main/java/org/enso/compiler/core/ir/IrPersistance.java

@@ -61,6 +61,10 @@ import scala.collection.immutable.Seq;
 @Persistable(clazz = Name.BuiltinAnnotation.class, id = 783)
 @Persistable(clazz = Type.Error.class, id = 784)
 @Persistable(clazz = Unused.Binding.class, id = 785)
+@Persistable(clazz = Unused.PatternBinding.class, id = 786)
+@Persistable(clazz = Unused.FunctionArgument.class, id = 787)
+@Persistable(clazz = Warning.DuplicatedImport.class, id = 788)
+@Persistable(clazz = Warning.WrongBuiltinMethod.class, id = 789)
 public final class IrPersistance {
   private IrPersistance() {}
 
@@ -389,23 +393,15 @@ public final class IrPersistance {
     }
 
     @Override
-    protected void writeObject(DiagnosticStorage obj, Output out) throws IOException {}
+    protected void writeObject(DiagnosticStorage obj, Output out) throws IOException {
+      out.writeInline(List.class, obj.toList());
+    }
 
     @Override
     @SuppressWarnings("unchecked")
     protected DiagnosticStorage readObject(Input in) throws IOException, ClassNotFoundException {
-      return new DiagnosticStorage(
+      var diags = in.readInline(List.class);
-          (scala.collection.immutable.List) scala.collection.immutable.Nil$.MODULE$);
+      return new DiagnosticStorage(diags);
     }
   }
-
-  private static <T> scala.collection.immutable.List<T> join(
-      T head, scala.collection.immutable.List<T> tail) {
-    return scala.collection.immutable.$colon$colon$.MODULE$.apply(head, tail);
-  }
-
-  @SuppressWarnings("unchecked")
-  private static <E extends Throwable> E raise(Class<E> clazz, Throwable t) throws E {
-    throw (E) t;
-  }
 }

engine/runtime/src/main/java/org/enso/interpreter/runtime/TruffleCompilerContext.java

@@ -276,7 +276,7 @@ final class TruffleCompilerContext implements CompilerContext {
 
   private final class ModuleUpdater implements Updater, AutoCloseable {
     private final Module module;
-    private BindingsMap map;
+    private BindingsMap[] map;
     private org.enso.compiler.core.ir.Module[] ir;
     private CompilationStage stage;
     private Boolean loadedFromCache;
@@ -289,7 +289,7 @@ final class TruffleCompilerContext implements CompilerContext {
 
     @Override
     public void bindingsMap(BindingsMap map) {
-      this.map = map;
+      this.map = new BindingsMap[] { map };
     }
 
     @Override
@@ -320,10 +320,10 @@ final class TruffleCompilerContext implements CompilerContext {
     @Override
     public void close() {
       if (map != null) {
-        if (module.bindings != null) {
+        if (module.bindings != null && map[0] != null) {
           loggerCompiler.log(Level.FINEST, "Reassigining bindings to {0}", module);
         }
-        module.bindings = map;
+        module.bindings = map[0];
       }
       if (ir != null) {
         module.module.unsafeSetIr(ir[0]);

engine/runtime/src/main/scala/org/enso/interpreter/runtime/DefaultPackageRepository.scala

@@ -102,7 +102,7 @@ private class DefaultPackageRepository(
   /** The mapping between the library and its cached bindings, if already laoded. */
   private val loadedLibraryBindings: collection.mutable.Map[
     LibraryName,
-    ImportExportCache.CachedBindings
+    Option[ImportExportCache.CachedBindings]
   ] =
     collection.mutable.LinkedHashMap()
 
@@ -578,15 +578,15 @@ private class DefaultPackageRepository(
     val cache = ensurePackageIsLoaded(libraryName).toOption.flatMap { _ =>
       if (!loadedLibraryBindings.contains(libraryName)) {
         loadedPackages.get(libraryName).flatten.foreach(loadDependencies(_))
-        context
+        val cachedBindingOption = context
           .asInstanceOf[TruffleCompilerContext]
           .getSerializationManager()
           .deserializeLibraryBindings(libraryName)
-          .foreach(cache => loadedLibraryBindings.addOne((libraryName, cache)))
+        loadedLibraryBindings.addOne((libraryName, cachedBindingOption))
       }
       loadedLibraryBindings.get(libraryName)
     }
-    cache.flatMap(_.bindings.findForModule(moduleName))
+    cache.flatMap(_.flatMap(_.bindings.findForModule(moduleName)))
   }
 
   private def loadDependencies(pkg: Package[TruffleFile]): Unit = {

engine/runtime/src/test/java/org/enso/interpreter/caches/ModuleCacheTest.java

@@ -1,22 +1,19 @@
 package org.enso.interpreter.caches;
 
 import org.enso.compiler.CompilerTest;
-import org.enso.compiler.core.IR;
-import org.enso.interpreter.Constants;
 import org.enso.interpreter.runtime.EnsoContext;
-import org.enso.interpreter.runtime.builtin.Builtins;
 import org.enso.interpreter.test.TestBase;
-import org.enso.interpreter.util.ScalaConversions;
 import org.enso.polyglot.CompilationStage;
 import org.enso.polyglot.LanguageInfo;
 import org.enso.polyglot.MethodNames;
 import org.enso.polyglot.RuntimeOptions;
 import org.graalvm.polyglot.Context;
-import org.junit.Test;
+import org.graalvm.polyglot.Source;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 import org.junit.BeforeClass;
+import org.junit.Test;
 
 public class ModuleCacheTest extends TestBase {
   private static Context ctx;
@@ -55,4 +52,30 @@ public class ModuleCacheTest extends TestBase {
     assertNotNull("IR read", cachedIr);
     CompilerTest.assertIR(name, ir, cachedIr.moduleIR());
   }
+
+
+  @Test
+  public void testCompareWithWarning() throws Exception {
+    var ensoCtx = (EnsoContext) ctx.getBindings(LanguageInfo.ID).invokeMember(MethodNames.TopScope.LEAK_CONTEXT).asHostObject();
+    var name = "TestWarning";
+    var code = Source.newBuilder("enso", """
+      empty x = 42
+      """, "TestWarning.enso")
+      .build();
+
+    var v = ctx.eval(code).invokeMember(MethodNames.Module.EVAL_EXPRESSION, "empty").execute(-1);
+    assertEquals(42, v.asInt());
+
+    var option = ensoCtx.findModule(name);
+    assertTrue("Module found", option.isPresent());
+    var module = option.get();
+    var ir = module.getIr().duplicate(true, true, true, true);
+    var cm = new ModuleCache.CachedModule(ir, CompilationStage.AFTER_CODEGEN, module.getSource());
+    byte[] arr = module.getCache().serialize(ensoCtx, cm);
+
+    var meta = new ModuleCache.Metadata("hash", "code", CompilationStage.AFTER_CODEGEN.toString());
+    var cachedIr = module.getCache().deserialize(ensoCtx, arr, meta, null);
+    assertNotNull("IR read", cachedIr);
+    CompilerTest.assertIR(name, ir, cachedIr.moduleIR());
+  }
 }

engine/runtime/src/test/java/org/enso/interpreter/runtime/ModuleTest.java

@@ -1,22 +1,31 @@
 package org.enso.interpreter.runtime;
 
-import static org.junit.Assert.assertTrue;
-
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Paths;
 import java.util.logging.Level;
+
+import org.enso.compiler.data.BindingsMap;
+import org.enso.compiler.data.BindingsMap$ModuleReference$Concrete;
+import org.enso.interpreter.runtime.Module;
+import static org.enso.interpreter.util.ScalaConversions.nil;
 import org.enso.pkg.QualifiedName;
 import org.enso.polyglot.LanguageInfo;
 import org.enso.polyglot.MethodNames;
 import org.enso.polyglot.RuntimeOptions;
 import org.graalvm.polyglot.Context;
 import org.graalvm.polyglot.Engine;
+import org.graalvm.polyglot.Source;
 import org.graalvm.polyglot.io.IOAccess;
 import org.junit.After;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
 import org.junit.Before;
 import org.junit.Test;
 
+import scala.Option;
+
 public class ModuleTest {
 
   private File f;
@@ -60,4 +69,70 @@ public class ModuleTest {
     assertTrue(
         "getPath is non-null", tFile.getPath() != null && module.getPath() == tFile.getPath());
   }
+
+  @Test
+  public void updaterCanNullTheBindings() throws Exception {
+    var name = QualifiedName.simpleName("SimpleExample");
+    var ensoContext =
+        (EnsoContext)
+            ctx.getBindings(LanguageInfo.ID)
+                .invokeMember(MethodNames.TopScope.LEAK_CONTEXT)
+                .asHostObject();
+    var tFile = ensoContext.getTruffleFile(f);
+
+    var code = Source.newBuilder("enso", """
+    main = 42
+    """, name.toString()).build();
+
+    ctx.eval(code);
+    var module = ensoContext.getTopScope().getModule(name.toString()).get().asCompilerModule();
+    ctx.enter();
+    var compilerContext = ensoContext.getCompiler().context();
+
+    assertNull("No bindings map by default", module.getBindingsMap());
+
+    var bindings = new BindingsMap(nil(), new BindingsMap$ModuleReference$Concrete(module));
+    compilerContext.updateModule(module, (u) -> {
+      u.bindingsMap(bindings);
+    });
+    assertEquals("Bindings map has changed", bindings, module.getBindingsMap());
+
+    compilerContext.updateModule(module, (u) -> {
+      u.bindingsMap(null);
+    });
+    assertNull("No bindings map again", module.getBindingsMap());
+  }
+
+  @Test
+  public void updaterCanNullTheIR() throws Exception {
+    var name = QualifiedName.simpleName("AnotherSimpleExample");
+    var ensoContext =
+        (EnsoContext)
+            ctx.getBindings(LanguageInfo.ID)
+                .invokeMember(MethodNames.TopScope.LEAK_CONTEXT)
+                .asHostObject();
+    var tFile = ensoContext.getTruffleFile(f);
+
+    var code = Source.newBuilder("enso", """
+    main = 42
+    """, name.toString()).build();
+
+    ctx.eval(code);
+    var module = ensoContext.getTopScope().getModule(name.toString()).get().asCompilerModule();
+    ctx.enter();
+    var compilerContext = ensoContext.getCompiler().context();
+
+    assertNull("No IR by default", module.getIr());
+
+    var ir = new org.enso.compiler.core.ir.Module(nil(), nil(), nil(), false, Option.empty(), null, null);
+    compilerContext.updateModule(module, (u) -> {
+      u.ir(ir);
+    });
+    assertEquals("IR has changed", ir, module.getIr());
+
+    compilerContext.updateModule(module, (u) -> {
+      u.ir(null);
+    });
+    assertNull("No IR again", module.getIr());
+  }
 }