Delay LS shutdown when last client disconnects (#7254)

* Delay LS shutdown when last client disconnects

Rather than closing Language Server immediately, we delay the shutdown
until some timeout hits. This gives a chance for new clients to connect
without paying the price of the initialization again.

More importantly, during hibernation/restart, the connection between
client (IDE) and LS is severed so it appears as if client disconnect. In
fact a few moments later IDE would attempt to re-establish the
connection on the same port. Without this change, LS shutsdown and
further attempts to connect on that particular port will fail.

There are still problems on the IDE-side after waking up from
hibernation but it is not related to Language Server.

* Introduce a separate timeout for delayed shutdown

Can't/shouldn't use the same timeout value as for shutdown timeout for
delaying shutdowns initiated by lack of clients.

* Add test demonstrating the new functionality
This commit is contained in:
Hubert Plociniczak 2023-07-18 08:37:11 +02:00 committed by GitHub
parent 6a6d7dbff3
commit 9252eb5f3a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 144 additions and 9 deletions

View File

@ -65,6 +65,7 @@ project-manager {
request-timeout = 10 seconds
boot-timeout = 40 seconds
shutdown-timeout = 20 seconds
delayed-shutdown-timeout = 8 seconds
socket-close-timeout = 15 seconds
retries = 5
}

View File

@ -65,6 +65,7 @@ object configuration {
* @param requestTimeout a timeout for JSON RPC request timeout
* @param bootTimeout a timeout for booting process
* @param shutdownTimeout a timeout for shutdown request
* @param delayedShutdownTimeout a timeout when shutdown, caused by lack of clients, can be cancelled
* @param socketCloseTimeout a timeout for closing the socket
* @param retries a number of retries attempted when timeout is reached
*/
@ -73,6 +74,7 @@ object configuration {
requestTimeout: FiniteDuration,
bootTimeout: FiniteDuration,
shutdownTimeout: FiniteDuration,
delayedShutdownTimeout: FiniteDuration,
socketCloseTimeout: FiniteDuration,
retries: Int
)

View File

@ -160,10 +160,12 @@ class LanguageServerController(
private def supervising(
connectionInfo: LanguageServerConnectionInfo,
serverProcessManager: ActorRef,
clients: Set[UUID] = Set.empty
clients: Set[UUID] = Set.empty,
scheduledShutdown: Option[Cancellable] = None
): Receive =
LoggingReceive.withLabel("supervising") {
case StartServer(clientId, _, requestedEngineVersion, _, _) =>
scheduledShutdown.foreach(_.cancel())
if (requestedEngineVersion != engineVersion) {
sender() ! ServerBootFailed(
new IllegalStateException(
@ -184,11 +186,13 @@ class LanguageServerController(
supervising(
connectionInfo,
serverProcessManager,
clients + clientId
clients + clientId,
None
)
)
}
case Terminated(_) =>
scheduledShutdown.foreach(_.cancel())
logger.debug("Bootloader for {} terminated.", project)
case StopServer(clientId, _) =>
@ -197,10 +201,15 @@ class LanguageServerController(
serverProcessManager,
clients,
clientId,
Some(sender())
Some(sender()),
scheduledShutdown
)
case ScheduledShutdown(requester) =>
shutDownServer(requester)
case ShutDownServer =>
scheduledShutdown.foreach(_.cancel())
shutDownServer(None)
case ClientDisconnected(clientId) =>
@ -209,10 +218,12 @@ class LanguageServerController(
serverProcessManager,
clients,
clientId,
None
None,
scheduledShutdown
)
case RenameProject(_, namespace, oldName, newName) =>
scheduledShutdown.foreach(_.cancel())
val socket = Socket(connectionInfo.interface, connectionInfo.rpcPort)
context.actorOf(
ProjectRenameAction
@ -230,6 +241,7 @@ class LanguageServerController(
)
case ServerDied =>
scheduledShutdown.foreach(_.cancel())
logger.error("Language server died [{}].", connectionInfo)
context.stop(self)
@ -240,15 +252,40 @@ class LanguageServerController(
serverProcessManager: ActorRef,
clients: Set[UUID],
clientId: UUID,
maybeRequester: Option[ActorRef]
maybeRequester: Option[ActorRef],
shutdownTimeout: Option[Cancellable]
): Unit = {
val updatedClients = clients - clientId
if (updatedClients.isEmpty) {
shutDownServer(maybeRequester)
logger.debug("Delaying shutdown for project {}.", project.id)
val scheduledShutdown =
shutdownTimeout.orElse(
Some(
context.system.scheduler
.scheduleOnce(
timeoutConfig.delayedShutdownTimeout,
self,
ScheduledShutdown(maybeRequester)
)
)
)
context.become(
supervising(
connectionInfo,
serverProcessManager,
Set.empty,
scheduledShutdown
)
)
} else {
sender() ! CannotDisconnectOtherClients
context.become(
supervising(connectionInfo, serverProcessManager, updatedClients)
supervising(
connectionInfo,
serverProcessManager,
updatedClients,
shutdownTimeout
)
)
}
}
@ -306,7 +343,7 @@ class LanguageServerController(
}
private def waitingForChildren(): Receive =
LoggingReceive.withLabel("waitingForChldren") { case Terminated(_) =>
LoggingReceive.withLabel("waitingForChildren") { case Terminated(_) =>
if (context.children.isEmpty) {
context.stop(self)
}
@ -389,4 +426,7 @@ object LanguageServerController {
case class Retry(message: Any)
/** Signals that a (cancellable) shutdown has been initiated */
case class ScheduledShutdown(requester: Option[ActorRef])
}

View File

@ -53,6 +53,7 @@ project-manager {
request-timeout = 10 seconds
boot-timeout = 30 seconds
shutdown-timeout = 20 seconds
delayed-shutdown-timeout = 5 seconds
socket-close-timeout = 10 seconds
}

View File

@ -5,13 +5,13 @@ import io.circe.literal._
import nl.gn0s1s.bump.SemVer
import org.apache.commons.io.FileUtils
import org.enso.editions.SemVerJson._
import org.enso.projectmanager.boot.configuration.TimeoutConfig
import org.enso.projectmanager.{BaseServerSpec, ProjectManagementOps}
import org.enso.testkit.FlakySpec
import java.io.File
import java.nio.file.{Files, Paths}
import java.util.UUID
import scala.concurrent.duration._
import scala.io.Source
@ -29,6 +29,10 @@ class ProjectManagementApiSpec
override val deleteProjectsRootAfterEachTest = false
override lazy val timeoutConfig: TimeoutConfig = {
config.timeout.copy(delayedShutdownTimeout = 1.nanosecond)
}
"project/create" must {
"check if project name is not empty" taggedAs Flaky in {

View File

@ -0,0 +1,87 @@
package org.enso.projectmanager.protocol
import akka.actor.ActorRef
import nl.gn0s1s.bump.SemVer
import org.enso.jsonrpc.ClientControllerFactory
import org.enso.projectmanager.boot.configuration.TimeoutConfig
import org.enso.projectmanager.event.ClientEvent.ClientDisconnected
import zio.{ZAny, ZIO}
import java.util.UUID
import org.enso.projectmanager.{BaseServerSpec, ProjectManagementOps}
import org.enso.testkit.FlakySpec
import scala.concurrent.duration._
class ProjectShutdownSpec
extends BaseServerSpec
with FlakySpec
with ProjectManagementOps {
override def beforeEach(): Unit = {
super.beforeEach()
gen.reset()
}
override val engineToInstall = Some(SemVer(0, 0, 1))
override val deleteProjectsRootAfterEachTest = false
var delayedShutdown = 3.seconds
var clientUUID: UUID = null
override def clientControllerFactory: ClientControllerFactory = {
new ManagerClientControllerFactory[ZIO[ZAny, +*, +*]](
system = system,
projectService = projectService,
globalConfigService = globalConfigService,
runtimeVersionManagementService = runtimeVersionManagementService,
loggingServiceDescriptor = loggingService,
timeoutConfig = timeoutConfig
) {
override def createClientController(clientId: UUID): ActorRef = {
clientUUID = clientId
super.createClientController(clientId)
}
}
}
override lazy val timeoutConfig: TimeoutConfig = {
config.timeout.copy(delayedShutdownTimeout = delayedShutdown)
}
"ensure language server does not shutdown immediately after last client disconnects" in {
val client1 = new WsTestClient(address)
val projectId = createProject("Foo")(client1)
val socket1 = openProject(projectId)(client1)
system.eventStream.publish(
ClientDisconnected(clientUUID)
)
val client2 = new WsTestClient(address)
val socket2 = openProject(projectId)(client2)
socket2 shouldBe socket1
closeProject(projectId)(client2)
deleteProject(projectId)(client2)
}
"ensure language server does eventually shutdown after last client disconnects" in {
val client1 = new WsTestClient(address)
val projectId = createProject("Foo")(client1)
val socket1 = openProject(projectId)(client1)
system.eventStream.publish(
ClientDisconnected(clientUUID)
)
Thread.sleep(
(timeoutConfig.delayedShutdownTimeout + timeoutConfig.shutdownTimeout + 1.second).toMillis
)
val client2 = new WsTestClient(address)
val socket2 = openProject(projectId)(client2)
socket2 shouldNot be(socket1)
closeProject(projectId)(client2)
deleteProject(projectId)(client2)
}
}