Cleanup after each test run (#8418)

Reducing leaks when running our test suite.

Potentially fixes #8408.

# Important Notes
Managed to keep thread count and memory in between runs relatively stable.
Initially:
![Screenshot from 2023-11-29 11-06-04](https://github.com/enso-org/enso/assets/292128/af437d8a-9111-4bd6-9033-a59030c7ebed)
Now:
![Screenshot from 2023-11-29 15-57-34](https://github.com/enso-org/enso/assets/292128/3c1f8aef-fe7e-4f5b-a236-12c86ea8b906)

The screenshot illustrates for `RuntimeVisualizationsTest` only. Will need to be applied in other places as well.

Applying the same style to `runtime-with-instruments`.
Before:
![Screenshot from 2023-11-29 16-55-11](https://github.com/enso-org/enso/assets/292128/385e11be-0265-431d-b0d7-c5096df11c50)
After:
![Screenshot from 2023-11-29 16-50-07](https://github.com/enso-org/enso/assets/292128/5e8c28ea-e921-484a-a82c-9f2d3e827e8b)

engine/runtime-instrument-common/src/main/scala/org/enso/interpreter/instrument/execution/CommandExecutionEngine.scala

@@ -103,6 +103,7 @@ class CommandExecutionEngine(interpreterContext: InterpreterContext)
   /** @inheritdoc */
   override def stop(): Unit = {
     jobExecutionEngine.stop()
+    sequentialExecutionService.shutdownNow()
     commandExecutor.shutdownNow()
   }
 

engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/BuiltinTypesTest.scala

@@ -1,6 +1,6 @@
 package org.enso.interpreter.test.instrument
 
-import org.enso.distribution.FileSystem
+import org.apache.commons.io.FileUtils
 import org.enso.distribution.locking.ThreadSafeFileLockManager
 import org.enso.interpreter.runtime.`type`.ConstantsGen
 import org.enso.interpreter.test.Metadata
@@ -16,6 +16,8 @@ import java.io.{ByteArrayOutputStream, File}
 import java.nio.file.{Files, Path, Paths}
 import java.util.UUID
 import java.util.logging.Level
+import scala.concurrent.Await
+import scala.concurrent.duration._
 
 @scala.annotation.nowarn("msg=multiarg infix syntax")
 class BuiltinTypesTest
@@ -30,8 +32,7 @@ class BuiltinTypesTest
   class TestContext(packageName: String) extends InstrumentTestContext {
 
     val tmpDir: Path = Files.createTempDirectory("enso-test-packages")
-    sys.addShutdownHook(FileSystem.removeDirectoryIfExists(tmpDir))
+    val lockManager  = new ThreadSafeFileLockManager(tmpDir.resolve("locks"))
-    val lockManager = new ThreadSafeFileLockManager(tmpDir.resolve("locks"))
     val runtimeServerEmulator =
       new RuntimeServerEmulator(messageQueue, lockManager)
 
@@ -98,8 +99,15 @@ class BuiltinTypesTest
     val Some(Api.Response(_, Api.InitializedNotification())) = context.receive
   }
   override protected def afterEach(): Unit = {
-    context.executionContext.context.close()
+    if (context != null) {
-    context.runtimeServerEmulator.terminate()
+      context.reset()
+      context.executionContext.context.close()
+      Await.ready(context.runtimeServerEmulator.terminate(), 5.seconds)
+      context.lockManager.reset()
+      context.out.reset()
+      FileUtils.deleteQuietly(context.tmpDir.toFile)
+      context = null
+    }
   }
 
   def runCode(contextId: UUID, requestId: UUID, contents: String): Unit = {

engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeAsyncCommandsTest.scala

@@ -1,6 +1,6 @@
 package org.enso.interpreter.test.instrument
 
-import org.enso.distribution.FileSystem
+import org.apache.commons.io.FileUtils
 import org.enso.distribution.locking.ThreadSafeFileLockManager
 import org.enso.interpreter.test.Metadata
 import org.enso.pkg.{Package, PackageManager}
@@ -16,6 +16,8 @@ import java.io.{ByteArrayOutputStream, File}
 import java.nio.file.{Files, Path, Paths}
 import java.util.UUID
 import java.util.logging.Level
+import scala.concurrent.Await
+import scala.concurrent.duration._
 
 @scala.annotation.nowarn("msg=multiarg infix syntax")
 class RuntimeAsyncCommandsTest
@@ -29,8 +31,7 @@ class RuntimeAsyncCommandsTest
 
   class TestContext(packageName: String) extends InstrumentTestContext {
     val tmpDir: Path = Files.createTempDirectory("enso-test-packages")
-    sys.addShutdownHook(FileSystem.removeDirectoryIfExists(tmpDir))
+    val lockManager  = new ThreadSafeFileLockManager(tmpDir.resolve("locks"))
-    val lockManager = new ThreadSafeFileLockManager(tmpDir.resolve("locks"))
     val runtimeServerEmulator =
       new RuntimeServerEmulator(messageQueue, lockManager)
 
@@ -103,8 +104,15 @@ class RuntimeAsyncCommandsTest
     val Some(Api.Response(_, Api.InitializedNotification())) = context.receive
   }
   override protected def afterEach(): Unit = {
-    context.executionContext.context.close()
+    if (context != null) {
-    context.runtimeServerEmulator.terminate()
+      context.reset()
+      context.executionContext.context.close()
+      Await.ready(context.runtimeServerEmulator.terminate(), 5.seconds)
+      context.lockManager.reset()
+      context.out.reset()
+      FileUtils.deleteQuietly(context.tmpDir.toFile)
+      context = null
+    }
   }
 
   it should "interrupt stopped execution context" in {

engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeComponentsTest.scala

@@ -1,6 +1,6 @@
 package org.enso.interpreter.test.instrument
 
-import org.enso.distribution.FileSystem
+import org.apache.commons.io.FileUtils
 import org.enso.distribution.locking.ThreadSafeFileLockManager
 import org.enso.editions.LibraryName
 import org.enso.interpreter.runtime
@@ -28,6 +28,7 @@ import java.nio.file.{Files, Path, Paths}
 import java.util.UUID
 import java.util.concurrent.{LinkedBlockingQueue, TimeUnit}
 import java.util.logging.Level
+import scala.concurrent.Await
 import scala.concurrent.duration._
 
 @scala.annotation.nowarn("msg=multiarg infix syntax")
@@ -74,7 +75,6 @@ class RuntimeComponentsTest
       new LinkedBlockingQueue()
 
     val tmpDir: Path = Files.createTempDirectory("enso-test-packages")
-    sys.addShutdownHook(FileSystem.removeDirectoryIfExists(tmpDir))
     val distributionHome: File =
       Paths.get("../../distribution/component").toFile.getAbsoluteFile
     val editionHome: File =
@@ -224,8 +224,15 @@ class RuntimeComponentsTest
     val Some(Api.Response(_, Api.InitializedNotification())) = context.receive
   }
   override protected def afterEach(): Unit = {
-    context.executionContext.context.close()
+    if (context != null) {
-    context.runtimeServerEmulator.terminate()
+      context.messageQueue.clear()
+      context.executionContext.context.close()
+      Await.ready(context.runtimeServerEmulator.terminate(), 5.seconds)
+      context.lockManager.reset()
+      context.out.reset()
+      FileUtils.deleteQuietly(context.tmpDir.toFile)
+      context = null
+    }
   }
 
   it should "load library extended by the component group" in {

engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeErrorsTest.scala

@@ -1,6 +1,6 @@
 package org.enso.interpreter.test.instrument
 
-import org.enso.distribution.FileSystem
+import org.apache.commons.io.FileUtils
 import org.enso.distribution.locking.ThreadSafeFileLockManager
 import org.enso.interpreter.runtime.`type`.ConstantsGen
 import org.enso.interpreter.test.Metadata
@@ -35,8 +35,7 @@ class RuntimeErrorsTest
   class TestContext(packageName: String) extends InstrumentTestContext {
 
     val tmpDir: Path = Files.createTempDirectory("enso-test-packages")
-    sys.addShutdownHook(FileSystem.removeDirectoryIfExists(tmpDir))
+    val lockManager  = new ThreadSafeFileLockManager(tmpDir.resolve("locks"))
-    val lockManager = new ThreadSafeFileLockManager(tmpDir.resolve("locks"))
     val runtimeServerEmulator =
       new RuntimeServerEmulator(messageQueue, lockManager)
 
@@ -116,8 +115,15 @@ class RuntimeErrorsTest
   }
 
   override protected def afterEach(): Unit = {
-    context.executionContext.context.close()
+    if (context != null) {
-    Await.ready(context.runtimeServerEmulator.terminate(), 5.seconds)
+      context.reset()
+      context.executionContext.context.close()
+      Await.ready(context.runtimeServerEmulator.terminate(), 5.seconds)
+      context.lockManager.reset()
+      context.out.reset()
+      FileUtils.deleteQuietly(context.tmpDir.toFile)
+      context = null
+    }
   }
 
   it should "return panic sentinels in method body" in {

engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeExecutionEnvironmentTest.scala

@@ -1,6 +1,6 @@
 package org.enso.interpreter.test.instrument
 
-import org.enso.distribution.FileSystem
+import org.apache.commons.io.FileUtils
 import org.enso.distribution.locking.ThreadSafeFileLockManager
 import org.enso.interpreter.runtime.EnsoContext
 import org.enso.interpreter.runtime.`type`.ConstantsGen
@@ -19,6 +19,7 @@ import java.io.{ByteArrayOutputStream, File}
 import java.nio.file.{Files, Path, Paths}
 import java.util.UUID
 import java.util.logging.Level
+import scala.concurrent.Await
 import scala.concurrent.duration._
 
 @scala.annotation.nowarn("msg=multiarg infix syntax")
@@ -42,7 +43,6 @@ class RuntimeExecutionEnvironmentTest
   class TestContext(packageName: String) extends InstrumentTestContext {
 
     val tmpDir: Path = Files.createTempDirectory("enso-test-packages")
-    sys.addShutdownHook(FileSystem.removeDirectoryIfExists(tmpDir))
     val distributionHome: File =
       Paths.get("../../distribution/component").toFile.getAbsoluteFile
     val editionHome: File =
@@ -129,8 +129,15 @@ class RuntimeExecutionEnvironmentTest
   }
 
   override protected def afterEach(): Unit = {
-    context.executionContext.context.close()
+    if (context != null) {
-    context.runtimeServerEmulator.terminate()
+      context.reset()
+      context.executionContext.context.close()
+      Await.ready(context.runtimeServerEmulator.terminate(), 5.seconds)
+      context.lockManager.reset()
+      context.out.reset()
+      FileUtils.deleteQuietly(context.tmpDir.toFile)
+      context = null
+    }
   }
 
   it should "panic when output context is not enabled" in {

engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeInstrumentTest.scala

@@ -1,6 +1,6 @@
 package org.enso.interpreter.test.instrument
 
-import org.enso.distribution.FileSystem
+import org.apache.commons.io.FileUtils
 import org.enso.distribution.locking.ThreadSafeFileLockManager
 import org.enso.interpreter.runtime.`type`.{Constants, ConstantsGen}
 import org.enso.interpreter.test.Metadata
@@ -16,6 +16,8 @@ import org.scalatest.matchers.should.Matchers
 import java.io.{ByteArrayOutputStream, File}
 import java.nio.file.{Files, Path, Paths}
 import java.util.UUID
+import scala.concurrent.Await
+import scala.concurrent.duration._
 
 @scala.annotation.nowarn("msg=multiarg infix syntax")
 class RuntimeInstrumentTest
@@ -29,8 +31,7 @@ class RuntimeInstrumentTest
 
   class TestContext(packageName: String) extends InstrumentTestContext {
     val tmpDir: Path = Files.createTempDirectory("enso-test-packages")
-    sys.addShutdownHook(FileSystem.removeDirectoryIfExists(tmpDir))
+    val lockManager  = new ThreadSafeFileLockManager(tmpDir.resolve("locks"))
-    val lockManager = new ThreadSafeFileLockManager(tmpDir.resolve("locks"))
     val runtimeServerEmulator =
       new RuntimeServerEmulator(messageQueue, lockManager)
 
@@ -101,8 +102,15 @@ class RuntimeInstrumentTest
   }
 
   override protected def afterEach(): Unit = {
-    context.executionContext.context.close()
+    if (context != null) {
-    context.runtimeServerEmulator.terminate()
+      context.reset()
+      context.executionContext.context.close()
+      Await.ready(context.runtimeServerEmulator.terminate(), 5.seconds)
+      context.lockManager.reset()
+      context.out.reset()
+      FileUtils.deleteQuietly(context.tmpDir.toFile)
+      context = null
+    }
   }
 
   it should "instrument simple expression" in {

engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeRefactoringTest.scala

@@ -1,7 +1,7 @@
 package org.enso.interpreter.test.instrument
 
+import org.apache.commons.io.FileUtils
 import org.apache.commons.io.output.TeeOutputStream
-import org.enso.distribution.FileSystem
 import org.enso.distribution.locking.ThreadSafeFileLockManager
 import org.enso.interpreter.runtime.`type`.ConstantsGen
 import org.enso.interpreter.test.Metadata
@@ -20,6 +20,8 @@ import java.io.{ByteArrayOutputStream, File}
 import java.nio.file.{Files, Path, Paths}
 import java.util.UUID
 import java.util.logging.Level
+import scala.concurrent.Await
+import scala.concurrent.duration._
 
 @scala.annotation.nowarn("msg=multiarg infix syntax")
 class RuntimeRefactoringTest
@@ -34,8 +36,7 @@ class RuntimeRefactoringTest
   class TestContext(packageName: String) extends InstrumentTestContext {
 
     val tmpDir: Path = Files.createTempDirectory("enso-test-packages")
-    sys.addShutdownHook(FileSystem.removeDirectoryIfExists(tmpDir))
+    val lockManager  = new ThreadSafeFileLockManager(tmpDir.resolve("locks"))
-    val lockManager = new ThreadSafeFileLockManager(tmpDir.resolve("locks"))
     val runtimeServerEmulator =
       new RuntimeServerEmulator(messageQueue, lockManager)
 
@@ -101,8 +102,15 @@ class RuntimeRefactoringTest
   }
 
   override protected def afterEach(): Unit = {
-    context.executionContext.context.close()
+    if (context != null) {
-    context.runtimeServerEmulator.terminate()
+      context.reset()
+      context.executionContext.context.close()
+      Await.ready(context.runtimeServerEmulator.terminate(), 5.seconds)
+      context.lockManager.reset()
+      context.out.reset()
+      FileUtils.deleteQuietly(context.tmpDir.toFile)
+      context = null
+    }
   }
 
   "RuntimeServer" should "rename operator in main body" in {

engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeServerTest.scala

@@ -1,6 +1,6 @@
 package org.enso.interpreter.test.instrument
 
-import org.enso.distribution.FileSystem
+import org.apache.commons.io.FileUtils
 import org.enso.distribution.locking.ThreadSafeFileLockManager
 import org.enso.interpreter.runtime.`type`.{Constants, ConstantsGen, Types}
 import org.enso.interpreter.runtime.EnsoContext
@@ -21,6 +21,9 @@ import java.nio.file.{Files, Path, Paths}
 import java.util.UUID
 import org.apache.commons.io.output.TeeOutputStream
 
+import scala.concurrent.Await
+import scala.concurrent.duration._
+
 @scala.annotation.nowarn("msg=multiarg infix syntax")
 class RuntimeServerTest
     extends AnyFlatSpec
@@ -34,8 +37,7 @@ class RuntimeServerTest
   class TestContext(packageName: String) extends InstrumentTestContext {
 
     val tmpDir: Path = Files.createTempDirectory("enso-test-packages")
-    sys.addShutdownHook(FileSystem.removeDirectoryIfExists(tmpDir))
+    val lockManager  = new ThreadSafeFileLockManager(tmpDir.resolve("locks"))
-    val lockManager = new ThreadSafeFileLockManager(tmpDir.resolve("locks"))
     val runtimeServerEmulator =
       new RuntimeServerEmulator(messageQueue, lockManager)
 
@@ -368,8 +370,15 @@ class RuntimeServerTest
   }
 
   override protected def afterEach(): Unit = {
-    context.executionContext.context.close()
+    if (context != null) {
-    context.runtimeServerEmulator.terminate()
+      context.reset()
+      context.executionContext.context.close()
+      Await.ready(context.runtimeServerEmulator.terminate(), 5.seconds)
+      context.lockManager.reset()
+      context.out.reset()
+      FileUtils.deleteQuietly(context.tmpDir.toFile)
+      context = null
+    }
   }
 
   "RuntimeServer" should "push and pop functions on the stack" in {

engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeStdlibTest.scala

@@ -1,6 +1,6 @@
 package org.enso.interpreter.test.instrument
 
-import org.enso.distribution.FileSystem
+import org.apache.commons.io.FileUtils
 import org.enso.distribution.locking.ThreadSafeFileLockManager
 import org.enso.interpreter.test.Metadata
 import org.enso.pkg.{Package, PackageManager, QualifiedName}
@@ -19,6 +19,7 @@ import java.util.UUID
 import java.util.concurrent.{LinkedBlockingQueue, TimeUnit}
 import java.util.logging.Level
 import scala.collection.mutable
+import scala.concurrent.Await
 import scala.concurrent.duration._
 
 @scala.annotation.nowarn("msg=multiarg infix syntax")
@@ -45,7 +46,6 @@ class RuntimeStdlibTest
       new LinkedBlockingQueue()
 
     val tmpDir: Path = Files.createTempDirectory("enso-test-packages")
-    sys.addShutdownHook(FileSystem.removeDirectoryIfExists(tmpDir))
     val distributionHome: File =
       Paths.get("../../distribution/component").toFile.getAbsoluteFile
     val editionHome: File =
@@ -187,8 +187,15 @@ class RuntimeStdlibTest
   }
 
   override protected def afterEach(): Unit = {
-    context.executionContext.context.close()
+    if (context != null) {
-    context.runtimeServerEmulator.terminate()
+      context.messageQueue.clear()
+      context.executionContext.context.close()
+      Await.ready(context.runtimeServerEmulator.terminate(), 5.seconds)
+      context.lockManager.reset()
+      context.out.reset()
+      FileUtils.deleteQuietly(context.tmpDir.toFile)
+      context = null
+    }
   }
 
   it should "import Base modules" in {

engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeSuggestionUpdatesTest.scala

@@ -1,6 +1,6 @@
 package org.enso.interpreter.test.instrument
 
-import org.enso.distribution.FileSystem
+import org.apache.commons.io.FileUtils
 import org.enso.distribution.locking.ThreadSafeFileLockManager
 import org.enso.interpreter.runtime.`type`.ConstantsGen
 import org.enso.pkg.{Package, PackageManager}
@@ -18,6 +18,8 @@ import java.io.{ByteArrayOutputStream, File}
 import java.nio.file.{Files, Path, Paths}
 import java.util.UUID
 import java.util.logging.Level
+import scala.concurrent.Await
+import scala.concurrent.duration._
 
 @scala.annotation.nowarn("msg=multiarg infix syntax")
 class RuntimeSuggestionUpdatesTest
@@ -30,8 +32,7 @@ class RuntimeSuggestionUpdatesTest
   class TestContext(packageName: String) extends InstrumentTestContext {
 
     val tmpDir: Path = Files.createTempDirectory("enso-test-packages")
-    sys.addShutdownHook(FileSystem.removeDirectoryIfExists(tmpDir))
+    val lockManager  = new ThreadSafeFileLockManager(tmpDir.resolve("locks"))
-    val lockManager = new ThreadSafeFileLockManager(tmpDir.resolve("locks"))
     val runtimeServerEmulator =
       new RuntimeServerEmulator(messageQueue, lockManager)
 
@@ -104,8 +105,15 @@ class RuntimeSuggestionUpdatesTest
   }
 
   override protected def afterEach(): Unit = {
-    context.executionContext.context.close()
+    if (context != null) {
-    context.runtimeServerEmulator.terminate()
+      context.reset()
+      context.executionContext.context.close()
+      Await.ready(context.runtimeServerEmulator.terminate(), 5.seconds)
+      context.lockManager.reset()
+      context.out.reset()
+      FileUtils.deleteQuietly(context.tmpDir.toFile)
+      context = null
+    }
   }
 
   it should "send suggestion updates after file modification" in {

engine/runtime-with-polyglot/src/test/scala/org/enso/interpreter/test/instrument/RuntimeVisualizationsTest.scala

@@ -32,8 +32,7 @@ class RuntimeVisualizationsTest
 
   class TestContext(packageName: String) extends InstrumentTestContext {
     val tmpDir: Path = Files.createTempDirectory("enso-test-packages")
-    sys.addShutdownHook(FileUtils.deleteQuietly(tmpDir.toFile))
+    val lockManager  = new ThreadSafeFileLockManager(tmpDir.resolve("locks"))
-    val lockManager = new ThreadSafeFileLockManager(tmpDir.resolve("locks"))
     val runtimeServerEmulator =
       new RuntimeServerEmulator(messageQueue, lockManager)
 
@@ -316,6 +315,18 @@ class RuntimeVisualizationsTest
     val Some(Api.Response(_, Api.InitializedNotification())) = context.receive
   }
 
+  override protected def afterEach(): Unit = {
+    if (context != null) {
+      context.reset()
+      context.executionContext.context.close()
+      context.runtimeServerEmulator.terminate()
+      context.lockManager.reset()
+      context.out.reset()
+      FileUtils.deleteQuietly(context.tmpDir.toFile)
+      context = null
+    }
+  }
+
   it should "emit visualization update when expression is computed" in {
     val idMainRes  = context.Main.metadata.addItem(99, 1)
     val contents   = context.Main.code

engine/runtime/src/test/scala/org/enso/interpreter/test/instrument/InstrumentTestContext.scala

@@ -106,6 +106,10 @@ class InstrumentTestContext {
         true
     }
 
+  def reset(): Unit = {
+    messageQueue.clear()
+  }
+
 }
 
 object InstrumentTestContext {

lib/scala/distribution-manager/src/main/scala/org/enso/distribution/locking/ThreadSafeFileLockManager.scala

@@ -325,4 +325,6 @@ class ThreadSafeFileLockManager(locksRoot: Path) extends ThreadSafeLockManager {
       case LockType.Shared    => lock.tryAcquireReader()
     }
   }
+
+  override def reset(): Unit = localLocks.clear()
 }

lib/scala/distribution-manager/src/main/scala/org/enso/distribution/locking/ThreadSafeLockManager.scala

@@ -1,4 +1,6 @@
 package org.enso.distribution.locking
 
 /** A [[LockManager]] which guarantees to be thread-safe. */
-trait ThreadSafeLockManager extends LockManager
+trait ThreadSafeLockManager extends LockManager {
+  def reset(): Unit
+}

lib/scala/runtime-version-manager-test/src/main/scala/org/enso/runtimeversionmanager/test/TestLocalLockManager.scala

@@ -58,4 +58,6 @@ class TestLocalLockManager extends ThreadSafeLockManager {
       case LockType.Shared    => rwLock.readLock()
     }
   }
+
+  override def reset(): Unit = {}
 }