Add RetrySpec to tests (#828)

Co-authored-by: Ara Adkins <iamrecursion@users.noreply.github.com>

engine/language-server/src/test/scala/org/enso/languageserver/filemanager/WatcherAdapterSpec.scala

@@ -4,7 +4,7 @@ import java.nio.file.{Files, Path, Paths}
 import java.util.concurrent.{Executors, LinkedBlockingQueue, Semaphore}
 
 import org.apache.commons.io.FileUtils
-import org.enso.jsonrpc.test.FlakySpec
+import org.enso.jsonrpc.test.RetrySpec
 import org.enso.languageserver.effect.Effects
 import org.scalatest.flatspec.AnyFlatSpec
 import org.scalatest.matchers.should.Matchers
@@ -12,21 +12,26 @@ import org.scalatest.matchers.should.Matchers
 import scala.concurrent.duration._
 import scala.util.Try
 
-class WatcherAdapterSpec extends AnyFlatSpec with Matchers with Effects with FlakySpec {
+class WatcherAdapterSpec
+    extends AnyFlatSpec
+    with Matchers
+    with Effects
+    with RetrySpec {
 
   import WatcherAdapter._
 
   final val Timeout: FiniteDuration = 5.seconds
 
-  it should "get create events" in withWatcher { (path, events) =>
+  it should "get create events" taggedAs Retry() in withWatcher {
+    (path, events) =>
       val fileA = Paths.get(path.toString, "a.txt")
-
       Files.createFile(fileA)
       val event = events.poll(Timeout.length, Timeout.unit)
       event shouldBe WatcherAdapter.WatcherEvent(fileA, EventTypeCreate)
   }
 
-  it should "get delete events" in withWatcher { (path, events) =>
+  it should "get delete events" taggedAs Retry() in withWatcher {
+    (path, events) =>
       val fileA = Paths.get(path.toString, "a.txt")
 
       Files.createFile(fileA)
@@ -38,7 +43,8 @@ class WatcherAdapterSpec extends AnyFlatSpec with Matchers with Effects with Fla
       event2 shouldBe WatcherEvent(fileA, EventTypeDelete)
   }
 
-  it should "get modify events" taggedAs (Flaky) in withWatcher { (path, events) =>
+  it should "get modify events" taggedAs Retry() in withWatcher {
+    (path, events) =>
       val fileA = Paths.get(path.toString, "a.txt")
 
       Files.createFile(fileA)
@@ -50,7 +56,8 @@ class WatcherAdapterSpec extends AnyFlatSpec with Matchers with Effects with Fla
       event2 shouldBe WatcherEvent(fileA, EventTypeModify)
   }
 
-  it should "get events from subdirectories" in withWatcher { (path, events) =>
+  it should "get events from subdirectories" taggedAs Retry() in withWatcher {
+    (path, events) =>
       val subdir = Paths.get(path.toString, "subdir")
       val fileA  = Paths.get(path.toString, "subdir", "a.txt")
 
@@ -70,14 +77,12 @@ class WatcherAdapterSpec extends AnyFlatSpec with Matchers with Effects with Fla
     val executor = Executors.newSingleThreadExecutor()
     val tmp      = Files.createTempDirectory(null).toRealPath()
     val queue    = new LinkedBlockingQueue[WatcherEvent]()
-    val watcher  = WatcherAdapter.build(tmp, queue.put(_), println(_))
+    val watcher  = WatcherAdapter.build(tmp, queue.put, println(_))
 
-    executor.submit(new Runnable {
-      def run(): Unit = {
+    executor.submit { () =>
       lock.release()
-        watcher.start().unsafeRunSync(): Unit
+      watcher.start().unsafeRunSync()
     }
-    })
 
     try {
       lock.tryAcquire(Timeout.length, Timeout.unit)
@@ -86,7 +91,7 @@ class WatcherAdapterSpec extends AnyFlatSpec with Matchers with Effects with Fla
       watcher.stop().unsafeRunSync()
       executor.shutdown()
       Try(executor.awaitTermination(Timeout.length, Timeout.unit))
-      Try(FileUtils.deleteDirectory(tmp.toFile)): Unit
+      Try(FileUtils.deleteDirectory(tmp.toFile))
     }
   }
 }

lib/json-rpc-server-test/src/main/scala/org/enso/jsonrpc/test/FlakySpec.scala

@@ -10,7 +10,7 @@ import org.scalatest._
 trait FlakySpec extends TestSuite {
 
   /** Tags test as _flaky_. */
-  object Flaky extends Tag("flaky")
+  object Flaky extends Tag("org.enso.test.flaky")
 
   override def withFixture(test: NoArgTest): Outcome =
     super.withFixture(test) match {

lib/json-rpc-server-test/src/main/scala/org/enso/jsonrpc/test/RetrySpec.scala

@@ -0,0 +1,54 @@
+package org.enso.jsonrpc.test
+
+import org.scalatest._
+
+/** Trait is used to retry the marked test in case of failure. */
+trait RetrySpec extends TestSuite {
+
+  /** Tag the test to be retried a specified number of times until success.
+    *
+    * @param times the number of attempted retries
+    */
+  case class Retry(times: Int) extends Tag(Retry.tagName(times)) {
+    assert(times > 0, "number of retries should be a positive number")
+  }
+
+  case object Retry {
+
+    /** Retry the test a single time. */
+    def apply(): Retry =
+      new Retry(1)
+
+    val Name      = "org.enso.test.retry"
+    val Separator = "-"
+
+    /** Create the tag name. */
+    def tagName(n: Int): String =
+      s"$Name$Separator$n"
+
+    /** Parse the number of retries from the tag name. */
+    def parseRetries(tag: String): Int =
+      tag.drop(Name.length + Separator.length).toInt
+  }
+
+  override def withFixture(test: NoArgTest): Outcome = {
+    @scala.annotation.tailrec
+    def go(n: Int, outcomes: List[Outcome]): Outcome =
+      if (n > 0) {
+        val result = super.withFixture(test)
+        result match {
+          case Failed(_) | Canceled(_) =>
+            go(n - 1, result :: outcomes)
+          case outcome => outcome
+        }
+      } else outcomes.head
+
+    test.tags.find(_.contains(Retry.Name)) match {
+      case Some(tag) =>
+        go(Retry.parseRetries(tag) + 1, Nil)
+      case None =>
+        super.withFixture(test)
+    }
+  }
+
+}