Don't cancel pending visualization's upserts (#8853)

Uniqueness check of `UpsertVisualizationJob` only involved expressionId. Apparently now GUI sends mutliple visualizations for the same expressions and expects all of them to exist. Since previously we would cancel duplicate jobs, this was problematic.

This change makes sure that uniqueness also takes into account visualization id. Fixed a few logs that were not passing arguments properly.

Closes #8801

# Important Notes
I have not noticed any more problems with loading visualizations so the issue appears to be resolved with this change.
Added a unit test case that would previously fail due to cancellation of a job that upserts visualization.

build.sbt

@@ -1747,9 +1747,10 @@ lazy val runtime = (project in file("engine/runtime"))
       "ENSO_TEST_DISABLE_IR_CACHE" -> "false",
       "ENSO_EDITION_PATH"          -> file("distribution/editions").getCanonicalPath
     ),
-    Test / compile := (Test / compile)
+    Test / compile := {
-      .dependsOn(`runtime-fat-jar` / Compile / compileModuleInfo)
+      (LocalProject("runtime-instrument-common") / Test / compile).value
-      .value
+      (Test / compile).value
+    }
   )
   .settings(
     (Compile / javacOptions) ++= Seq(
@@ -1898,13 +1899,18 @@ lazy val `runtime-instrument-common` =
       Test / fork := true,
       Test / envVars ++= distributionEnvironmentOverrides ++ Map(
         "ENSO_TEST_DISABLE_IR_CACHE" -> "false"
+      ),
+      libraryDependencies ++= Seq(
+        "junit"          % "junit"           % junitVersion     % Test,
+        "com.github.sbt" % "junit-interface" % junitIfVersion   % Test,
+        "org.scalatest" %% "scalatest"       % scalatestVersion % Test
       )
     )
     .dependsOn(`refactoring-utils`)
     .dependsOn(
       LocalProject(
         "runtime"
-      ) % "compile->compile;test->test;runtime->runtime;bench->bench"
+      ) % "compile->compile;runtime->runtime;bench->bench"
     )
 
 lazy val `runtime-instrument-id-execution` =

engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/LibrariesTest.scala

@@ -59,7 +59,7 @@ class LibrariesTest
   )
 
   "LocalLibraryManager" should {
-    "create a library project and include it on the list of local projects" taggedAs Flaky in {
+    "create a library project and include it on the list of local projects" taggedAs SkipOnFailure in {
       val client          = getInitialisedWsClient()
       val testLibraryName = LibraryName("user", "My_Local_Lib")
 

engine/runtime-instrument-common/src/main/scala/org/enso/interpreter/instrument/command/AttachVisualizationCmd.scala

@@ -34,7 +34,7 @@ class AttachVisualizationCmd(
     )
     val maybeFutureExecutable =
       ctx.jobProcessor.run(
-        new UpsertVisualizationJob(
+        upsertVisualization(
           maybeRequestId,
           request.visualizationId,
           request.expressionId,
@@ -48,6 +48,20 @@ class AttachVisualizationCmd(
     }
   }
 
+  def upsertVisualization(
+    maybeRequestId: Option[Api.RequestId],
+    visualizationId: Api.VisualizationId,
+    expressionId: Api.ExpressionId,
+    config: Api.VisualizationConfiguration
+  ): UpsertVisualizationJob = {
+    new UpsertVisualizationJob(
+      maybeRequestId,
+      visualizationId,
+      expressionId,
+      config
+    )
+  }
+
   override def toString: String = {
     "AttachVisualizationCmd(visualizationId: " + request.visualizationId + ",expressionId=" + request.expressionId + ")"
   }

engine/runtime-instrument-common/src/main/scala/org/enso/interpreter/instrument/command/ModifyVisualizationCmd.scala

@@ -33,7 +33,7 @@ class ModifyVisualizationCmd(
     ctx.executionService.getLogger.log(
       Level.FINE,
       "Modify visualization cmd for request id [{}] and visualization id [{}]",
-      Array(maybeRequestId, request.visualizationId)
+      Array[Object](maybeRequestId, request.visualizationId)
     )
     val existingVisualization = ctx.contextManager.getVisualizationById(
       request.visualizationConfig.executionContextId,

engine/runtime-instrument-common/src/main/scala/org/enso/interpreter/instrument/job/DeserializeLibrarySuggestionsJob.scala

@@ -30,7 +30,8 @@ final class DeserializeLibrarySuggestionsJob(
   override def run(implicit ctx: RuntimeContext): Unit = {
     ctx.executionService.getLogger.log(
       Level.FINE,
-      s"Deserializing suggestions for library [$libraryName]."
+      "Deserializing suggestions for library [{}].",
+      libraryName
     )
     val serializationManager = SerializationManager(
       ctx.executionService.getContext.getCompiler.context

engine/runtime-instrument-common/src/main/scala/org/enso/interpreter/instrument/job/EnsureCompiledJob.scala

@@ -429,7 +429,8 @@ final class EnsureCompiledJob(
     if (invalidatedVisualizations.nonEmpty) {
       ctx.executionService.getLogger.log(
         Level.FINEST,
-        s"Invalidated visualizations [${invalidatedVisualizations.map(_.id)}]"
+        "Invalidated visualizations [{}]",
+        invalidatedVisualizations.map(_.id)
       )
     }
 

engine/runtime-instrument-common/src/main/scala/org/enso/interpreter/instrument/job/UpsertVisualizationJob.scala

@@ -53,7 +53,7 @@ class UpsertVisualizationJob(
   override def equalsTo(that: UniqueJob[_]): Boolean =
     that match {
       case that: UpsertVisualizationJob =>
-        this.expressionId == that.expressionId
+        this.expressionId == that.expressionId && this.visualizationId == that.visualizationId
       case _ => false
     }
 
@@ -146,7 +146,7 @@ class UpsertVisualizationJob(
     ctx.executionService.getLogger.log(
       Level.SEVERE,
       "Visualization for expression {0} failed: {1} (evaluation result: {2})",
-      Array(expressionId, message, executionResult)
+      Array[Object](expressionId, message, executionResult)
     )
     ctx.endpoint.sendToClient(
       Api.Response(
@@ -159,6 +159,10 @@ class UpsertVisualizationJob(
     )
   }
 
+  override def toString: String = {
+    s"UpsertVisualizationJob(visualizationId=$visualizationId, expressionId=$expressionId)"
+  }
+
 }
 
 object UpsertVisualizationJob {

engine/runtime-instrument-common/src/test/java/org/enso/interpreter/instrument/MockHandler.java

@@ -1,11 +1,11 @@
 package org.enso.interpreter.instrument;
 
 import org.enso.interpreter.instrument.command.CommandFactory;
-import org.enso.interpreter.instrument.command.MockedCommandFactory$;
+import org.enso.interpreter.instrument.command.MockedCommandFactory;
 
 public class MockHandler extends Handler {
 
-  private CommandFactory _cmdFactory = MockedCommandFactory$.MODULE$;
+  private CommandFactory _cmdFactory = new MockedCommandFactory();
 
   @Override
   public CommandFactory cmdFactory() {

engine/runtime-instrument-common/src/test/java/org/enso/interpreter/instrument/job/JobsTest.java

@@ -0,0 +1,37 @@
+package org.enso.interpreter.instrument.job;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.UUID;
+import org.enso.polyglot.runtime.Runtime$Api$VisualizationConfiguration;
+import org.enso.polyglot.runtime.Runtime$Api$VisualizationExpression$Text;
+import org.junit.Test;
+import scala.Option;
+
+public class JobsTest {
+  @Test
+  public void upsertJobUniqueness() {
+    var config =
+        new Runtime$Api$VisualizationConfiguration(
+            UUID.randomUUID(), new Runtime$Api$VisualizationExpression$Text("foo", "bar"), "test");
+    var expression1 = UUID.randomUUID();
+    var expression2 = UUID.randomUUID();
+    var visualization1 = UUID.randomUUID();
+    var visualization2 = UUID.randomUUID();
+    var visualization3 = UUID.randomUUID();
+
+    var upsert1 = new UpsertVisualizationJob(Option.empty(), visualization1, expression1, config);
+    var upsert2 = new UpsertVisualizationJob(Option.empty(), visualization2, expression1, config);
+    var upsert3 = new UpsertVisualizationJob(Option.empty(), visualization3, expression2, config);
+    var upsert4 = new UpsertVisualizationJob(Option.empty(), visualization1, expression1, config);
+    var upsert5 = new UpsertVisualizationJob(Option.empty(), visualization1, expression2, config);
+
+    assertFalse(upsert1.equalsTo(upsert2));
+    assertFalse(upsert2.equalsTo(upsert3));
+    assertFalse(upsert1.equalsTo(upsert3));
+    assertTrue(upsert1.equalsTo(upsert4));
+    assertFalse(upsert3.equalsTo(upsert4));
+    assertFalse(upsert5.equalsTo(upsert3));
+  }
+}

engine/runtime-instrument-common/src/test/java/org/enso/interpreter/runtime/ModuleTestUtils.java

@@ -0,0 +1,15 @@
+package org.enso.interpreter.runtime;
+
+import org.enso.polyglot.CompilationStage;
+
+public final class ModuleTestUtils {
+  private ModuleTestUtils() {}
+
+  public static void unsafeSetIr(Module m, org.enso.compiler.core.ir.Module ir) {
+    m.unsafeSetIr(ir);
+  }
+
+  public static void unsafeSetCompilationStage(Module m, CompilationStage s) {
+    m.unsafeSetCompilationStage(s);
+  }
+}

engine/runtime-instrument-common/src/test/scala/org/enso/compiler/test/CompilerTestSetup.scala

@@ -0,0 +1,228 @@
+package org.enso.compiler.test
+
+import org.enso.compiler.context.{FreshNameSupply, InlineContext, ModuleContext}
+import org.enso.compiler.core.EnsoParser
+import org.enso.compiler.core.Implicits.AsMetadata
+import org.enso.compiler.core.ir.{Expression, Module}
+import org.enso.compiler.core.ir.MetadataStorage.MetadataPair
+import org.enso.compiler.data.BindingsMap.ModuleReference
+import org.enso.compiler.data.{BindingsMap, CompilerConfig}
+import org.enso.compiler.pass.analyse.BindingAnalysis
+import org.enso.compiler.pass.{PassConfiguration, PassManager}
+import org.enso.interpreter.runtime
+import org.enso.interpreter.runtime.ModuleTestUtils
+import org.enso.compiler.context.LocalScope
+import org.enso.pkg.QualifiedName
+import org.enso.polyglot.CompilationStage
+
+/** A reduced version of [[org.enso.compiler.test.CompilerRunner]] that avoids introducing a cyclic dependency
+  * to `runtime-instrument-common` subject.
+  */
+trait CompilerTestSetup {
+  // === IR Utilities =========================================================
+
+  /** An extension method to allow converting string source code to IR as a
+    * module.
+    *
+    * @param source the source code to convert
+    */
+  implicit private class ToIrModule(source: String) {
+
+    /** Converts program text to a top-level Enso module.
+      *
+      * @return the [[IR]] representing [[source]]
+      */
+    def toIrModule: Module = {
+      val compiler = new EnsoParser()
+      try compiler.compile(source)
+      finally compiler.close()
+    }
+  }
+
+  /** An extension method to allow converting string source code to IR as an
+    * expression.
+    *
+    * @param source the source code to convert
+    */
+  implicit private class ToIrExpression(source: String) {
+
+    /** Converts the program text to an Enso expression.
+      *
+      * @return the [[IR]] representing [[source]], if it is a valid expression
+      */
+    def toIrExpression: Option[Expression] = {
+      val compiler = new EnsoParser()
+      try compiler.generateIRInline(compiler.parse(source))
+      finally compiler.close()
+    }
+  }
+
+  /** Provides an extension method allowing the running of a specified list of
+    * passes on the provided IR.
+    *
+    * @param ir the IR to run the passes on
+    */
+  implicit private class RunPassesOnModule(ir: Module) {
+
+    /** Executes the passes using `passManager` on the input [[ir]].
+      *
+      * @param passManager the pass configuration
+      * @param moduleContext the module context it is executing in
+      * @return the result of executing the passes in `passManager` on [[ir]]
+      */
+    def runPasses(
+      passManager: PassManager,
+      moduleContext: ModuleContext
+    ): Module = {
+      passManager.runPassesOnModule(ir, moduleContext)
+    }
+  }
+
+  /** Provides an extension method allowing the running of a specified list of
+    * passes on the provided IR.
+    *
+    * @param ir the IR to run the passes on
+    */
+  implicit private class RunPassesOnExpression(ir: Expression) {
+
+    /** Executes the passes using `passManager` on the input [[ir]].
+      *
+      * @param passManager the pass configuration
+      * @param inlineContext the inline context it is executing in
+      * @return the result of executing the passes in `passManager` on [[ir]]
+      */
+    def runPasses(
+      passManager: PassManager,
+      inlineContext: InlineContext
+    ): Expression = {
+      passManager.runPassesInline(ir, inlineContext)
+    }
+  }
+
+  /** Adds an extension method to preprocess the source as IR.
+    *
+    * @param source the source code to preprocess
+    */
+  implicit class Preprocess(source: String)(implicit
+    passManager: PassManager
+  ) {
+
+    /** Translates the source code into appropriate IR for testing this pass.
+      *
+      * @return IR appropriate for testing the alias analysis pass as a module
+      */
+    def preprocessModule(implicit moduleContext: ModuleContext): Module = {
+      source.toIrModule.runPasses(passManager, moduleContext)
+    }
+
+    /** Translates the source code into appropriate IR for testing this pass
+      *
+      * @return IR appropriate for testing the alias analysis pass as an
+      *         expression
+      */
+    def preprocessExpression(implicit
+      inlineContext: InlineContext
+    ): Option[Expression] = {
+      source.toIrExpression.map(_.runPasses(passManager, inlineContext))
+    }
+  }
+
+  // === IR Testing Utils =====================================================
+
+  /** Builds a module context with a mocked module for testing purposes.
+    *
+    * @param moduleName the name of the test module.
+    * @param freshNameSupply the fresh name supply to use in tests.
+    * @param passConfiguration any additional pass configuration.
+    * @return an instance of module context.
+    */
+  def buildModuleContext(
+    moduleName: QualifiedName                    = QualifiedName.simpleName("Test_Module"),
+    freshNameSupply: Option[FreshNameSupply]     = None,
+    passConfiguration: Option[PassConfiguration] = None,
+    compilerConfig: CompilerConfig               = defaultConfig,
+    isGeneratingDocs: Boolean                    = false
+  ): ModuleContext = buildModuleContextModule(
+    moduleName,
+    freshNameSupply,
+    passConfiguration,
+    compilerConfig,
+    isGeneratingDocs
+  )._1
+
+  /** Builds a module context with a mocked module for testing purposes.
+    *
+    * @param moduleName the name of the test module.
+    * @param freshNameSupply the fresh name supply to use in tests.
+    * @param passConfiguration any additional pass configuration.
+    * @return an pair of module context and module.
+    */
+  def buildModuleContextModule(
+    moduleName: QualifiedName                    = QualifiedName.simpleName("Test_Module"),
+    freshNameSupply: Option[FreshNameSupply]     = None,
+    passConfiguration: Option[PassConfiguration] = None,
+    compilerConfig: CompilerConfig               = defaultConfig,
+    isGeneratingDocs: Boolean                    = false
+  ): (ModuleContext, runtime.Module) = {
+    val mod = runtime.Module.empty(moduleName, null)
+    val ctx = ModuleContext(
+      module            = mod.asCompilerModule(),
+      freshNameSupply   = freshNameSupply,
+      passConfiguration = passConfiguration,
+      compilerConfig    = compilerConfig,
+      isGeneratingDocs  = isGeneratingDocs
+    )
+    (ctx, mod)
+  }
+
+  /** Builds an inline context with a mocked module for testing purposes.
+    *
+    * @param localScope the local scope for variable resolution.
+    * @param isInTailPosition whether the expression is being evaluated in
+    *                         a tail position.
+    * @param freshNameSupply the fresh name supply to use for name generation.
+    * @param passConfiguration any additional pass configuration.
+    * @return an instance of inline context.
+    */
+  def buildInlineContext(
+    localScope: Option[LocalScope]               = None,
+    isInTailPosition: Option[Boolean]            = None,
+    freshNameSupply: Option[FreshNameSupply]     = None,
+    passConfiguration: Option[PassConfiguration] = None,
+    compilerConfig: CompilerConfig               = defaultConfig
+  ): InlineContext = {
+    val mod =
+      runtime.Module.empty(QualifiedName.simpleName("Test_Module"), null)
+    ModuleTestUtils.unsafeSetIr(
+      mod,
+      Module(List(), List(), List(), false, None)
+        .updateMetadata(
+          new MetadataPair(
+            BindingAnalysis,
+            BindingsMap(
+              List(),
+              ModuleReference.Concrete(mod.asCompilerModule())
+            )
+          )
+        )
+    )
+    ModuleTestUtils.unsafeSetCompilationStage(
+      mod,
+      CompilationStage.AFTER_CODEGEN
+    )
+    val mc = ModuleContext(
+      module         = mod.asCompilerModule(),
+      compilerConfig = compilerConfig
+    )
+    InlineContext(
+      module            = mc,
+      freshNameSupply   = freshNameSupply,
+      passConfiguration = passConfiguration,
+      localScope        = localScope,
+      isInTailPosition  = isInTailPosition,
+      compilerConfig    = compilerConfig
+    )
+  }
+
+  val defaultConfig: CompilerConfig = CompilerConfig()
+}

engine/runtime-instrument-common/src/test/scala/org/enso/compiler/test/context/ChangesetBuilderTest.scala

@@ -13,15 +13,20 @@ import org.enso.compiler.core.ir.expression.Application
 import org.enso.compiler.core.ir.expression.errors
 import org.enso.compiler.core.ir.module.scope.definition
 import org.enso.compiler.pass.PassManager
-import org.enso.compiler.test.CompilerTest
+import org.enso.compiler.test.CompilerTestSetup
 import org.enso.compiler.context.LocalScope
 import org.enso.text.buffer.Rope
 import org.enso.text.editing.JavaEditorAdapter
 import org.enso.text.editing.model.{Position, Range, TextEdit}
+import org.scalatest.matchers.should.Matchers
+import org.scalatest.wordspec.AnyWordSpecLike
 
 import java.util.UUID
 
-class ChangesetBuilderTest extends CompilerTest {
+class ChangesetBuilderTest
+    extends AnyWordSpecLike
+    with Matchers
+    with CompilerTestSetup {
 
   implicit val passManager: PassManager = new Passes(defaultConfig).passManager
 

engine/runtime-instrument-common/src/test/scala/org/enso/interpreter/instrument/command/MockedCommandFactory.scala

@@ -1,16 +1,25 @@
 package org.enso.interpreter.instrument.command
 import org.enso.polyglot.runtime.Runtime.Api
 
-object MockedCommandFactory extends CommandFactory {
+class MockedCommandFactory extends CommandFactory {
 
   private var editRequestCounter         = 0
+  private var attachVisualizationCounter = 0
 
   override def createCommand(request: Api.Request): Command = {
     request.payload match {
       case payload: Api.EditFileNotification =>
-        val cmd = new SlowEditFileCmd(payload, editRequestCounter)
+        val cmd = new SlowEditFileCmd(payload, editRequestCounter % 2 == 0)
         editRequestCounter += 1
         cmd
+      case payload: Api.AttachVisualization =>
+        val cmd = new SlowAttachVisualizationCmd(
+          request.requestId,
+          payload,
+          attachVisualizationCounter % 2 == 0
+        )
+        attachVisualizationCounter += 1
+        cmd
       case _ =>
         super.createCommand(request)
     }

engine/runtime-instrument-common/src/test/scala/org/enso/interpreter/instrument/command/SlowAttachVisualizationCmd.scala

@@ -0,0 +1,35 @@
+package org.enso.interpreter.instrument.command
+
+import org.enso.interpreter.instrument.job.{
+  SlowUpsertVisualizationJob,
+  UpsertVisualizationJob
+}
+import org.enso.polyglot.runtime.Runtime.Api
+import org.slf4j.LoggerFactory
+
+class SlowAttachVisualizationCmd(
+  maybeRequestId: Option[Api.RequestId],
+  request: Api.AttachVisualization,
+  delay: Boolean
+) extends AttachVisualizationCmd(maybeRequestId, request) {
+
+  private val logger =
+    LoggerFactory.getLogger(classOf[SlowAttachVisualizationCmd])
+
+  override def upsertVisualization(
+    maybeRequestId: Option[Api.RequestId],
+    visualizationId: Api.VisualizationId,
+    expressionId: Api.ExpressionId,
+    config: Api.VisualizationConfiguration
+  ): UpsertVisualizationJob = {
+
+    logger.info("Delaying upsert for {}: {}", request.visualizationId, delay)
+    new SlowUpsertVisualizationJob(
+      maybeRequestId,
+      visualizationId,
+      expressionId,
+      config,
+      delay
+    )
+  }
+}

engine/runtime-instrument-common/src/test/scala/org/enso/interpreter/instrument/command/SlowEditFileCmd.scala

@@ -5,7 +5,7 @@ import org.enso.polyglot.runtime.Runtime.Api
 
 import scala.concurrent.ExecutionContext
 
-class SlowEditFileCmd(request: Api.EditFileNotification, counter: Int)
+class SlowEditFileCmd(request: Api.EditFileNotification, delay: Boolean)
     extends EditFileCmd(request) {
 
   override def executeSynchronously(implicit
@@ -13,7 +13,7 @@ class SlowEditFileCmd(request: Api.EditFileNotification, counter: Int)
     ec: ExecutionContext
   ): Unit = {
     if (
-      ctx.executionService.getContext.isRandomDelayedCommandExecution && counter % 2 == 0
+      ctx.executionService.getContext.isRandomDelayedCommandExecution && delay
     ) {
       try {
         Thread.sleep(2000)

engine/runtime-instrument-common/src/test/scala/org/enso/interpreter/instrument/job/SlowUpsertVisualizationJob.scala

@@ -0,0 +1,33 @@
+package org.enso.interpreter.instrument.job
+
+import org.enso.interpreter.instrument.execution.{Executable, RuntimeContext}
+import org.enso.polyglot.runtime.Runtime.Api
+
+import scala.annotation.unused
+
+class SlowUpsertVisualizationJob(
+  @unused requestId: Option[Api.RequestId],
+  visualizationId: Api.VisualizationId,
+  expressionId: Api.ExpressionId,
+  config: Api.VisualizationConfiguration,
+  delay: Boolean
+) extends UpsertVisualizationJob(
+      requestId,
+      visualizationId,
+      expressionId,
+      config
+    ) {
+
+  override val isCancellable: Boolean         = true
+  override val mayInterruptIfRunning: Boolean = true
+
+  override def run(implicit ctx: RuntimeContext): Option[Executable] = {
+    if (
+      ctx.executionService.getContext.isRandomDelayedCommandExecution && delay
+    ) {
+      Thread.sleep(1000)
+    }
+
+    super.run(ctx)
+  }
+}

engine/runtime/src/test/resources/application-test.conf

@@ -1 +1,26 @@
 akka.coordinated-shutdown.run-by-actor-system-terminate = off
+logging-service {
+  logger {
+    akka.actor = info
+    akka.event = error
+    akka.routing = error
+    akka.io = error
+    akka.stream = error
+    slick.jdbc.JdbcBackend.statement = error # log SQL queries on debug level
+    slick."*" = error
+    org.eclipse.jgit = error
+    io.methvin.watcher = error
+  }
+  appenders = [
+    {
+      name = "memory"
+      forward-to = console
+    },
+    {
+      name = "console"
+      pattern = "[%level] [%d{yyyy-MM-ddTHH:mm:ssXXX}] [%logger] %msg%n"
+    }
+  ]
+  default-appender = memory
+  log-level = "error"
+}

engine/runtime/src/test/scala/org/enso/interpreter/test/instrument/RuntimeVisualizationsTest.scala

@@ -8,7 +8,6 @@ import org.enso.polyglot.runtime.Runtime.Api
 import org.enso.text.editing.model
 import org.enso.text.editing.model.TextEdit
 import org.graalvm.polyglot.Context
-import org.scalatest.BeforeAndAfterEach
 import org.scalatest.flatspec.AnyFlatSpec
 import org.scalatest.matchers.should.Matchers
 
@@ -19,16 +18,11 @@ import java.util.UUID
 import java.util.logging.Level
 
 @scala.annotation.nowarn("msg=multiarg infix syntax")
-class RuntimeVisualizationsTest
+class RuntimeVisualizationsTest extends AnyFlatSpec with Matchers {
-    extends AnyFlatSpec
-    with Matchers
-    with BeforeAndAfterEach {
 
   // === Test Utilities =======================================================
 
-  var context: TestContext = _
+  class TestContext(packageName: String, sequentialExecution: Boolean)
-
-  class TestContext(packageName: String)
       extends InstrumentTestContext(packageName) {
 
     val out: ByteArrayOutputStream = new ByteArrayOutputStream()
@@ -39,7 +33,14 @@ class RuntimeVisualizationsTest
         .allowAllAccess(true)
         .option(RuntimeOptions.PROJECT_ROOT, pkg.root.getAbsolutePath)
         .option(RuntimeOptions.LOG_LEVEL, Level.WARNING.getName())
-        .option(RuntimeOptions.INTERPRETER_SEQUENTIAL_COMMAND_EXECUTION, "true")
+        .option(
+          RuntimeOptions.INTERPRETER_SEQUENTIAL_COMMAND_EXECUTION,
+          sequentialExecution.toString
+        )
+        .option(
+          RuntimeOptions.INTERPRETER_RANDOM_DELAYED_COMMAND_EXECUTION,
+          (!sequentialExecution).toString
+        )
         .option(RuntimeOptions.ENABLE_PROJECT_SUGGESTIONS, "false")
         .option(RuntimeOptions.ENABLE_GLOBAL_SUGGESTIONS, "false")
         .option(RuntimeOptions.ENABLE_EXECUTION_TIMER, "false")
@@ -83,7 +84,7 @@ class RuntimeVisualizationsTest
 
     // === The Tests ==========================================================
 
-    object Main {
+    object Main { context =>
 
       val metadata = new Metadata
 
@@ -301,21 +302,24 @@ class RuntimeVisualizationsTest
 
   }
 
-  override protected def beforeEach(): Unit = {
+  def withContext(
-    context = new TestContext("Test")
+    sequentialExecution: Boolean = true
+  )(f: TestContext => Unit): Unit = {
+    val context = new TestContext("Test", sequentialExecution)
+    try {
       context.init()
       val Some(Api.Response(_, Api.InitializedNotification())) = context.receive
-  }
+      f(context)
-
+    } finally {
-  override protected def afterEach(): Unit = {
       if (context != null) {
         context.close()
         context.out.reset()
-      context = null
+      }
     }
   }
 
-  it should "emit visualization update when expression is computed" in {
+  it should "emit visualization update when expression is computed" in withContext() {
+    context =>
       val idMainRes  = context.Main.metadata.addItem(99, 1)
       val contents   = context.Main.code
       val mainFile   = context.writeMain(context.Main.code)
@@ -416,7 +420,10 @@ class RuntimeVisualizationsTest
 
       // recompute
       context.send(
-      Api.Request(requestId, Api.RecomputeContextRequest(contextId, None, None))
+        Api.Request(
+          requestId,
+          Api.RecomputeContextRequest(contextId, None, None)
+        )
       )
 
       val recomputeResponses = context.receiveNIgnoreExpressionUpdates(3)
@@ -441,7 +448,8 @@ class RuntimeVisualizationsTest
       data2.sameElements("50".getBytes) shouldBe true
   }
 
-  it should "emit visualization update when expression is cached" in {
+  it should "emit visualization update when expression is cached" in withContext() {
+    context =>
       val contents   = context.Main.code
       val mainFile   = context.writeMain(context.Main.code)
       val moduleName = "Enso_Test.Test.Main"
@@ -539,7 +547,10 @@ class RuntimeVisualizationsTest
 
       // recompute
       context.send(
-      Api.Request(requestId, Api.RecomputeContextRequest(contextId, None, None))
+        Api.Request(
+          requestId,
+          Api.RecomputeContextRequest(contextId, None, None)
+        )
       )
       context.receiveNIgnoreExpressionUpdates(2) should contain allOf (
         Api.Response(requestId, Api.RecomputeContextResponse(contextId)),
@@ -553,7 +564,9 @@ class RuntimeVisualizationsTest
           Api.RecomputeContextRequest(
             contextId,
             Some(
-            Api.InvalidatedExpressions.Expressions(Vector(context.Main.idMainX))
+              Api.InvalidatedExpressions.Expressions(
+                Vector(context.Main.idMainX)
+              )
             ),
             None
           )
@@ -581,7 +594,8 @@ class RuntimeVisualizationsTest
       data2.sameElements("6".getBytes) shouldBe true
   }
 
-  it should "emit visualization update when expression is modified" in {
+  it should "emit visualization update when expression is modified" in withContext() {
+    context =>
       val contents   = context.Main.code
       val moduleName = "Enso_Test.Test.Main"
       val mainFile   = context.writeMain(contents)
@@ -714,7 +728,8 @@ class RuntimeVisualizationsTest
       data1.sameElements("5".getBytes) shouldBe true
   }
 
-  it should "emit visualization update when transitive expression is modified" in {
+  it should "emit visualization update when transitive expression is modified" in withContext() {
+    context =>
       val contents   = context.Main.code
       val moduleName = "Enso_Test.Test.Main"
       val mainFile   = context.writeMain(contents)
@@ -847,7 +862,8 @@ class RuntimeVisualizationsTest
       data1.sameElements("45".getBytes) shouldBe true
   }
 
-  it should "emit visualization update when frame popped" in {
+  it should "emit visualization update when frame popped" in withContext() {
+    context =>
       val contents   = context.Main.code
       val moduleName = "Enso_Test.Test.Main"
       val mainFile   = context.writeMain(contents)
@@ -1091,7 +1107,7 @@ class RuntimeVisualizationsTest
       new String(data4) shouldEqual "60"
   }
 
-  it should "be able to modify visualizations" in {
+  it should "be able to modify visualizations" in withContext() { context =>
     val contents = context.Main.code
     val mainFile = context.writeMain(contents)
     val visualizationFile =
@@ -1226,7 +1242,193 @@ class RuntimeVisualizationsTest
     dataAfterModification.sameElements("7".getBytes) shouldBe true
   }
 
-  it should "not emit visualization update when visualization is detached" in {
+  it should "be able to modify visualizations for pending visualizations" in withContext(
+    sequentialExecution = false
+  ) { context =>
+    val contents = context.Main.code
+    val mainFile = context.writeMain(contents)
+    val visualizationFile =
+      context.writeInSrcDir("Visualization", context.Visualization.code)
+
+    val contextId        = UUID.randomUUID()
+    val requestId        = UUID.randomUUID()
+    val visualizationId  = UUID.randomUUID()
+    val visualizationId2 = UUID.randomUUID()
+
+    // open files
+    context.send(
+      Api.Request(requestId, Api.OpenFileRequest(mainFile, contents))
+    )
+
+    context.receive shouldEqual Some(
+      Api.Response(Some(requestId), Api.OpenFileResponse)
+    )
+    context.send(
+      Api.Request(
+        requestId,
+        Api.OpenFileRequest(
+          visualizationFile,
+          context.Visualization.code
+        )
+      )
+    )
+    context.receive shouldEqual Some(
+      Api.Response(Some(requestId), Api.OpenFileResponse)
+    )
+
+    // create context
+    context.send(Api.Request(requestId, Api.CreateContextRequest(contextId)))
+    context.receive shouldEqual Some(
+      Api.Response(requestId, Api.CreateContextResponse(contextId))
+    )
+
+    // push main
+    val item1 = Api.StackItem.ExplicitCall(
+      Api.MethodPointer("Enso_Test.Test.Main", "Enso_Test.Test.Main", "main"),
+      None,
+      Vector()
+    )
+    context.send(
+      Api.Request(requestId, Api.PushContextRequest(contextId, item1))
+    )
+    context.receiveNIgnorePendingExpressionUpdates(
+      5
+    ) should contain theSameElementsAs Seq(
+      Api.Response(requestId, Api.PushContextResponse(contextId)),
+      context.Main.Update.mainX(contextId),
+      context.Main.Update.mainY(contextId),
+      context.Main.Update.mainZ(contextId),
+      context.executionComplete(contextId)
+    )
+
+    // attach visualizations
+    context.send(
+      Api.Request(
+        requestId,
+        Api.AttachVisualization(
+          visualizationId,
+          context.Main.idMainX,
+          Api.VisualizationConfiguration(
+            contextId,
+            Api.VisualizationExpression.Text(
+              "Enso_Test.Test.Visualization",
+              "x -> encode x"
+            ),
+            "Enso_Test.Test.Visualization"
+          )
+        )
+      )
+    )
+
+    context.send(
+      Api.Request(
+        Api.EditFileNotification(
+          mainFile,
+          Seq(
+            TextEdit(
+              model.Range(model.Position(4, 8), model.Position(4, 9)),
+              "7"
+            )
+          ),
+          execute = true
+        )
+      )
+    )
+
+    context.send(
+      Api.Request(
+        requestId,
+        Api.AttachVisualization(
+          visualizationId2,
+          context.Main.idMainX,
+          Api.VisualizationConfiguration(
+            contextId,
+            Api.VisualizationExpression.Text(
+              "Enso_Test.Test.Visualization",
+              "x -> encode x"
+            ),
+            "Enso_Test.Test.Visualization"
+          )
+        )
+      )
+    )
+
+    val attachVisualizationResponses = context.receiveN(2)
+
+    attachVisualizationResponses.filter(
+      _.payload.isInstanceOf[Api.VisualizationAttached]
+    ) shouldEqual List(
+      Api.Response(requestId, Api.VisualizationAttached()),
+      Api.Response(requestId, Api.VisualizationAttached())
+    )
+
+    val responses = context.receiveNIgnoreExpressionUpdates(3)
+    val visualizationUpdatesResponses =
+      responses.filter(_.payload.isInstanceOf[Api.VisualizationUpdate])
+    val expectedExpressionId = context.Main.idMainX
+    val visualizationUpdates = visualizationUpdatesResponses.map(
+      _.payload.asInstanceOf[Api.VisualizationUpdate]
+    )
+    val visContexts = visualizationUpdates.map(_.visualizationContext)
+    visContexts should contain allOf (
+      Api.VisualizationContext(
+        `visualizationId`,
+        `contextId`,
+        `expectedExpressionId`
+      ),
+      Api.VisualizationContext(
+        `visualizationId2`,
+        `contextId`,
+        `expectedExpressionId`
+      ),
+    )
+    val data = visualizationUpdates.head.data
+    data.sameElements("6".getBytes) shouldBe true
+
+    // modify visualization
+    context.send(
+      Api.Request(
+        requestId,
+        Api.ModifyVisualization(
+          visualizationId,
+          Api.VisualizationConfiguration(
+            contextId,
+            Api.VisualizationExpression.Text(
+              "Enso_Test.Test.Visualization",
+              "x -> incAndEncode x"
+            ),
+            "Enso_Test.Test.Visualization"
+          )
+        )
+      )
+    )
+    val modifyVisualizationResponses = context.receiveN(5)
+    modifyVisualizationResponses should contain(
+      Api.Response(requestId, Api.VisualizationModified())
+    )
+    val Some((dataAfterModification, foundVisualizatonId)) =
+      modifyVisualizationResponses.collectFirst {
+        case Api.Response(
+              None,
+              Api.VisualizationUpdate(
+                Api.VisualizationContext(
+                  modifiedId,
+                  `contextId`,
+                  `expectedExpressionId`
+                ),
+                data
+              )
+            ) =>
+          (data, modifiedId)
+      }
+
+    List(visualizationId, visualizationId2) should contain(foundVisualizatonId)
+
+    dataAfterModification.sameElements("7".getBytes) shouldBe true
+  }
+
+  it should "not emit visualization update when visualization is detached" in withContext() {
+    context =>
       val contents = context.Main.code
       val mainFile = context.writeMain(contents)
       val visualizationFile =
@@ -1342,7 +1544,10 @@ class RuntimeVisualizationsTest
 
       // recompute
       context.send(
-      Api.Request(requestId, Api.RecomputeContextRequest(contextId, None, None))
+        Api.Request(
+          requestId,
+          Api.RecomputeContextRequest(contextId, None, None)
+        )
       )
       context.receiveNIgnoreExpressionUpdates(
         2
@@ -1358,7 +1563,9 @@ class RuntimeVisualizationsTest
           Api.RecomputeContextRequest(
             contextId,
             Some(
-            Api.InvalidatedExpressions.Expressions(Vector(context.Main.idMainX))
+              Api.InvalidatedExpressions.Expressions(
+                Vector(context.Main.idMainX)
+              )
             ),
             None
           )
@@ -1372,7 +1579,8 @@ class RuntimeVisualizationsTest
       )
   }
 
-  it should "not emit visualization update when expression is not affected by the change" in {
+  it should "not emit visualization update when expression is not affected by the change" in withContext() {
+    context =>
       val contents   = context.Main.code
       val moduleName = "Enso_Test.Test.Main"
       val mainFile   = context.writeMain(contents)
@@ -1491,7 +1699,7 @@ class RuntimeVisualizationsTest
       )
   }
 
-  it should "not reorder visualization commands" in {
+  it should "not reorder visualization commands" in withContext() { context =>
     val contents = context.Main.code
     val mainFile = context.writeMain(contents)
     val visualizationFile =
@@ -1638,7 +1846,8 @@ class RuntimeVisualizationsTest
     dataAfterModification.sameElements("7".getBytes) shouldBe true
   }
 
-  it should "return ModuleNotFound error when attaching visualization" in {
+  it should "return ModuleNotFound error when attaching visualization" in withContext() {
+    context =>
       val idMain     = context.Main.metadata.addItem(99, 1)
       val contents   = context.Main.code
       val mainFile   = context.writeMain(context.Main.code)
@@ -1706,7 +1915,8 @@ class RuntimeVisualizationsTest
       )
   }
 
-  it should "be able to use external libraries if they are needed by the visualization" in {
+  it should "be able to use external libraries if they are needed by the visualization" in withContext() {
+    context =>
       val idMain     = context.Main.metadata.addItem(99, 1)
       val contents   = context.Main.code
       val mainFile   = context.writeMain(context.Main.code)
@@ -1804,7 +2014,8 @@ class RuntimeVisualizationsTest
       loadedLibraries should contain(("Standard", "Visualization"))
   }
 
-  it should "return VisualizationExpressionFailed error when attaching visualization" in {
+  it should "return VisualizationExpressionFailed error when attaching visualization" in withContext() {
+    context =>
       val idMain     = context.Main.metadata.addItem(99, 1)
       val contents   = context.Main.code
       val mainFile   = context.writeMain(context.Main.code)
@@ -1888,7 +2099,8 @@ class RuntimeVisualizationsTest
       )
   }
 
-  it should "return visualization evaluation errors with diagnostic info" in {
+  it should "return visualization evaluation errors with diagnostic info" in withContext() {
+    context =>
       val idMain     = context.Main.metadata.addItem(99, 1)
       val contents   = context.Main.code
       val mainFile   = context.writeMain(context.Main.code)
@@ -1986,7 +2198,8 @@ class RuntimeVisualizationsTest
       )
   }
 
-  it should "return visualization error with a stack trace" in {
+  it should "return visualization error with a stack trace" in withContext() {
+    context =>
       val idMain     = context.Main.metadata.addItem(99, 1)
       val contents   = context.Main.code
       val mainFile   = context.writeMain(context.Main.code)
@@ -2114,7 +2327,8 @@ class RuntimeVisualizationsTest
       )
   }
 
-  it should "run visualization expression catching error" in {
+  it should "run visualization expression catching error" in withContext() {
+    context =>
       val contextId       = UUID.randomUUID()
       val requestId       = UUID.randomUUID()
       val visualizationId = UUID.randomUUID()
@@ -2190,7 +2404,8 @@ class RuntimeVisualizationsTest
           )
         )
       )
-    val attachVisualizationResponses = context.receiveN(4, timeoutSeconds = 60)
+      val attachVisualizationResponses =
+        context.receiveN(4, timeoutSeconds = 60)
       attachVisualizationResponses should contain allOf (
         Api.Response(requestId, Api.VisualizationAttached()),
         context.executionComplete(contextId)
@@ -2212,7 +2427,8 @@ class RuntimeVisualizationsTest
       data.sameElements("42".getBytes) shouldBe true
   }
 
-  it should "run visualization expression propagating panic" in {
+  it should "run visualization expression propagating panic" in withContext() {
+    context =>
       val contextId       = UUID.randomUUID()
       val requestId       = UUID.randomUUID()
       val visualizationId = UUID.randomUUID()
@@ -2320,8 +2536,9 @@ class RuntimeVisualizationsTest
               Api.ExecutionResult.Diagnostic.error(
                 message = "42",
                 file    = Some(mainFile),
-              location =
+                location = Some(
-                Some(model.Range(model.Position(3, 4), model.Position(3, 18))),
+                  model.Range(model.Position(3, 4), model.Position(3, 18))
+                ),
                 expressionId = Some(idMain),
                 stack = Vector(
                   Api.StackTraceElement(
@@ -2341,7 +2558,7 @@ class RuntimeVisualizationsTest
       )
   }
 
-  it should "run visualization error preprocessor" in {
+  it should "run visualization error preprocessor" in withContext() { context =>
     val contextId       = UUID.randomUUID()
     val requestId       = UUID.randomUUID()
     val visualizationId = UUID.randomUUID()
@@ -2451,7 +2668,8 @@ class RuntimeVisualizationsTest
     stringified shouldEqual """{"kind":"Dataflow","message":"The List is empty. (at <enso> Main.main(Enso_Test.Test.Main:6:5-38)"}"""
   }
 
-  it should "run visualization default preprocessor" in {
+  it should "run visualization default preprocessor" in withContext() {
+    context =>
       val contextId       = UUID.randomUUID()
       val requestId       = UUID.randomUUID()
       val visualizationId = UUID.randomUUID()
@@ -2553,7 +2771,8 @@ class RuntimeVisualizationsTest
       stringified shouldEqual "\"Function\""
   }
 
-  it should "attach method pointer visualization without arguments" in {
+  it should "attach method pointer visualization without arguments" in withContext() {
+    context =>
       val idMainRes = context.Main.metadata.addItem(99, 1)
       val contents  = context.Main.code
       val mainFile  = context.writeMain(context.Main.code)
@@ -2657,7 +2876,10 @@ class RuntimeVisualizationsTest
 
       // recompute
       context.send(
-      Api.Request(requestId, Api.RecomputeContextRequest(contextId, None, None))
+        Api.Request(
+          requestId,
+          Api.RecomputeContextRequest(contextId, None, None)
+        )
       )
 
       val recomputeResponses = context.receiveNIgnoreExpressionUpdates(3)
@@ -2682,7 +2904,8 @@ class RuntimeVisualizationsTest
       data2.sameElements("51".getBytes) shouldBe true
   }
 
-  it should "attach method pointer visualization with arguments" in {
+  it should "attach method pointer visualization with arguments" in withContext() {
+    context =>
       val idMainRes  = context.Main.metadata.addItem(99, 1)
       val contents   = context.Main.code
       val mainFile   = context.writeMain(context.Main.code)
@@ -2792,7 +3015,10 @@ class RuntimeVisualizationsTest
 
       // recompute
       context.send(
-      Api.Request(requestId, Api.RecomputeContextRequest(contextId, None, None))
+        Api.Request(
+          requestId,
+          Api.RecomputeContextRequest(contextId, None, None)
+        )
       )
 
       val recomputeResponses = context.receiveNIgnoreExpressionUpdates(3)
@@ -2862,7 +3088,8 @@ class RuntimeVisualizationsTest
       context.consumeOut shouldEqual List("encoding...")
   }
 
-  it should "cache intermediate visualization expressions" in {
+  it should "cache intermediate visualization expressions" in withContext() {
+    context =>
       val idMainRes  = context.Main.metadata.addItem(99, 1)
       val contents   = context.Main.code
       val mainFile   = context.writeMain(context.Main.code)
@@ -2972,7 +3199,10 @@ class RuntimeVisualizationsTest
 
       // recompute
       context.send(
-      Api.Request(requestId, Api.RecomputeContextRequest(contextId, None, None))
+        Api.Request(
+          requestId,
+          Api.RecomputeContextRequest(contextId, None, None)
+        )
       )
 
       val recomputeResponses = context.receiveNIgnoreExpressionUpdates(3)
@@ -3035,7 +3265,8 @@ class RuntimeVisualizationsTest
       context.consumeOut shouldEqual List("encoding...")
   }
 
-  it should "emit visualization update for values annotated with warnings" in {
+  it should "emit visualization update for values annotated with warnings" in withContext() {
+    context =>
       val contextId       = UUID.randomUUID()
       val requestId       = UUID.randomUUID()
       val visualizationId = UUID.randomUUID()
@@ -3134,7 +3365,8 @@ class RuntimeVisualizationsTest
       new String(data, StandardCharsets.UTF_8) shouldEqual "42"
   }
 
-  it should "emit visualization update for values in array annotated with warnings" in {
+  it should "emit visualization update for values in array annotated with warnings" in withContext() {
+    context =>
       val contextId       = UUID.randomUUID()
       val requestId       = UUID.randomUUID()
       val visualizationId = UUID.randomUUID()
@@ -3236,7 +3468,8 @@ class RuntimeVisualizationsTest
       new String(data, StandardCharsets.UTF_8) shouldEqual "[42]"
   }
 
-  it should "emit visualization update for values in atom annotated with warnings" in {
+  it should "emit visualization update for values in atom annotated with warnings" in withContext() {
+    context =>
       val contextId         = UUID.randomUUID()
       val requestId         = UUID.randomUUID()
       val visualizationId   = UUID.randomUUID()
@@ -3313,7 +3546,8 @@ class RuntimeVisualizationsTest
           s"$moduleName.Newtype",
           methodCall = Some(
             Api.MethodCall(
-            Api.MethodPointer(moduleName, s"$moduleName.Newtype", "Mk_Newtype")
+              Api
+                .MethodPointer(moduleName, s"$moduleName.Newtype", "Mk_Newtype")
             )
           ),
           payload = Api.ExpressionUpdate.Payload.Value(
@@ -3366,7 +3600,8 @@ class RuntimeVisualizationsTest
       new String(data, StandardCharsets.UTF_8) shouldEqual "(Mk_Newtype 42)"
   }
 
-  it should "emit visualization update for the target of a method call" in {
+  it should "emit visualization update for the target of a method call" in withContext() {
+    context =>
       val contextId       = UUID.randomUUID()
       val requestId       = UUID.randomUUID()
       val visualizationId = UUID.randomUUID()
@@ -3521,7 +3756,8 @@ class RuntimeVisualizationsTest
       new String(data1, StandardCharsets.UTF_8) shouldEqual "C"
   }
 
-  it should "execute expression in the scope of local expression cached" in {
+  it should "execute expression in the scope of local expression cached" in withContext() {
+    context =>
       val contextId       = UUID.randomUUID()
       val requestId       = UUID.randomUUID()
       val visualizationId = UUID.randomUUID()
@@ -3620,7 +3856,8 @@ class RuntimeVisualizationsTest
       new String(data) shouldEqual "42"
   }
 
-  it should "execute expression in the scope of local expression not cached" in {
+  it should "execute expression in the scope of local expression not cached" in withContext() {
+    context =>
       val contextId       = UUID.randomUUID()
       val requestId       = UUID.randomUUID()
       val visualizationId = UUID.randomUUID()
@@ -3721,7 +3958,8 @@ class RuntimeVisualizationsTest
       new String(data) shouldEqual "42"
   }
 
-  it should "execute expression in the scope of local binding" in {
+  it should "execute expression in the scope of local binding" in withContext() {
+    context =>
       val contextId       = UUID.randomUUID()
       val requestId       = UUID.randomUUID()
       val visualizationId = UUID.randomUUID()
@@ -3825,7 +4063,8 @@ class RuntimeVisualizationsTest
       new String(data) shouldEqual "85"
   }
 
-  it should "execute expression in the scope of main method" in {
+  it should "execute expression in the scope of main method" in withContext() {
+    context =>
       val contextId       = UUID.randomUUID()
       val requestId       = UUID.randomUUID()
       val visualizationId = UUID.randomUUID()
@@ -3926,7 +4165,8 @@ class RuntimeVisualizationsTest
       new String(data) shouldEqual "85"
   }
 
-  it should "execute default visualization preprocessor" in {
+  it should "execute default visualization preprocessor" in withContext() {
+    context =>
       val contextId       = UUID.randomUUID()
       val requestId       = UUID.randomUUID()
       val visualizationId = UUID.randomUUID()
@@ -4015,7 +4255,8 @@ class RuntimeVisualizationsTest
       new String(data) shouldEqual "85"
   }
 
-  it should "execute default visualization preprocessor with a FQN" in {
+  it should "execute default visualization preprocessor with a FQN" in withContext() {
+    context =>
       val contextId       = UUID.randomUUID()
       val requestId       = UUID.randomUUID()
       val visualizationId = UUID.randomUUID()