Project manager (#11)

build.sbt

@@ -20,7 +20,8 @@ lazy val enso = (project in file("."))
   .aggregate(
     syntax,
     pkg,
-    interpreter
+    interpreter,
+    projectManager
   )
 
 // Sub-Projects
@@ -90,3 +91,26 @@ lazy val interpreter = (project in file("interpreter"))
     bench := (test in Benchmark).value,
     parallelExecution in Benchmark := false
   )
+
+val akkaActor  = "com.typesafe.akka" %% "akka-actor"           % "2.5.23"
+val akkaStream = "com.typesafe.akka" %% "akka-stream"          % "2.5.23"
+val akkaHttp   = "com.typesafe.akka" %% "akka-http"            % "10.1.8"
+val akkaSpray  = "com.typesafe.akka" %% "akka-http-spray-json" % "10.1.8"
+val akkaTyped  = "com.typesafe.akka" %% "akka-actor-typed"     % "2.5.23"
+
+val akka = Seq(akkaActor, akkaStream, akkaHttp, akkaSpray, akkaTyped)
+
+val circe = Seq("circe-core", "circe-generic", "circe-yaml").map(
+  "io.circe" %% _ % "0.10.0"
+)
+
+lazy val projectManager = (project in file("project-manager"))
+  .settings(
+    (Compile / mainClass) := Some("org.enso.projectmanager.Server")
+  )
+  .settings(
+    libraryDependencies ++= akka,
+    libraryDependencies ++= circe,
+    libraryDependencies += "io.spray" %% "spray-json" % "1.3.5"
+  )
+  .dependsOn(pkg)

project-manager/src/main/resources/application.conf

@@ -0,0 +1,19 @@
+project-manager {
+  server {
+    host = "0.0.0.0"
+    port = 30535
+    timeout = 10 seconds
+  }
+
+  storage {
+    projects-root = ${user.home}/enso
+    temporary-projects-path = ${project-manager.storage.projects-root}/tmp
+    local-projects-path = ${project-manager.storage.projects-root}/projects
+    tutorials-path = ${project-manager.storage.projects-root}/tutorials
+    tutorials-cache-path = ${project-manager.storage.projects-root}/.tutorials-cache
+  }
+
+  tutorials {
+    github-organisation = "luna-packages"
+  }
+}

project-manager/src/main/scala/org/enso/projectmanager/RouteHelper.scala

@@ -0,0 +1,33 @@
+package org.enso.projectmanager
+
+import java.util.UUID
+
+import akka.http.scaladsl.model.Uri
+import akka.http.scaladsl.model.Uri.Path
+import akka.http.scaladsl.server.PathMatcher0
+import akka.http.scaladsl.server.PathMatcher1
+import akka.http.scaladsl.server.PathMatchers.JavaUUID
+import org.enso.projectmanager.model.ProjectId
+
+class RouteHelper {
+
+  val tutorials: String = "tutorials"
+  val projects: String  = "projects"
+  val thumb: String     = "thumb"
+
+  val tutorialsPath: Path                = Path / tutorials
+  val tutorialsPathMatcher: PathMatcher0 = tutorials
+
+  val projectsPath: Path                = Path / projects
+  val projectsPathMatcher: PathMatcher0 = projects
+
+  def projectPath(id: ProjectId): Path = projectsPath / id.toString
+
+  val projectPathMatcher: PathMatcher1[ProjectId] =
+    (projectsPathMatcher / JavaUUID).map(ProjectId)
+
+  def thumbPath(id: ProjectId): Path = projectPath(id) / thumb
+  val thumbPathMatcher: PathMatcher1[ProjectId] = projectPathMatcher / thumb
+
+  def uriFor(base: Uri, path: Path): Uri = base.withPath(path)
+}

project-manager/src/main/scala/org/enso/projectmanager/Server.scala

@@ -0,0 +1,189 @@
+package org.enso.projectmanager
+
+import java.io.File
+import java.util.UUID
+import java.util.concurrent.TimeUnit
+
+import akka.actor.ActorSystem
+import akka.actor.Scheduler
+import com.typesafe.config.ConfigFactory
+import akka.actor.typed.ActorRef
+import akka.actor.typed.scaladsl.adapter._
+import akka.actor.typed.scaladsl.AskPattern._
+import akka.http.scaladsl.Http
+import akka.http.scaladsl.model.HttpResponse
+import akka.http.scaladsl.model.StatusCodes
+import akka.http.scaladsl.model.Uri
+import akka.http.scaladsl.server.Directives
+import akka.http.scaladsl.server.Route
+import akka.stream.ActorMaterializer
+import akka.util.Timeout
+import org.enso.projectmanager.api.ProjectFactory
+import org.enso.projectmanager.api.ProjectJsonSupport
+import org.enso.projectmanager.model.Project
+import org.enso.projectmanager.model.ProjectId
+import org.enso.projectmanager.services.CreateTemporary
+import org.enso.projectmanager.services.CreateTemporaryResponse
+import org.enso.projectmanager.services.GetProjectById
+import org.enso.projectmanager.services.GetProjectResponse
+import org.enso.projectmanager.services.ListProjectsRequest
+import org.enso.projectmanager.services.ListProjectsResponse
+import org.enso.projectmanager.services.ListTutorialsRequest
+import org.enso.projectmanager.services.ProjectsCommand
+import org.enso.projectmanager.services.ProjectsService
+import org.enso.projectmanager.services.StorageManager
+import org.enso.projectmanager.services.TutorialsDownloader
+
+import scala.concurrent.ExecutionContext
+import scala.concurrent.Future
+import scala.concurrent.duration._
+import scala.util.Failure
+import scala.util.Success
+
+case class Server(
+  host: String,
+  port: Int,
+  repository: ActorRef[ProjectsCommand],
+  routeHelper: RouteHelper,
+  apiFactory: ProjectFactory
+)(implicit val system: ActorSystem,
+  implicit val executor: ExecutionContext,
+  implicit val materializer: ActorMaterializer,
+  implicit val askTimeout: Timeout)
+    extends Directives
+    with ProjectJsonSupport {
+
+  implicit val scheduler: Scheduler = system.scheduler
+
+  def projectDoesNotExistResponse(id: ProjectId): HttpResponse =
+    HttpResponse(StatusCodes.NotFound, entity = s"Project $id does not exist")
+
+  def thumbDoesNotExistResponse: HttpResponse =
+    HttpResponse(StatusCodes.NotFound, entity = "Thumbnail does not exist")
+
+  def withSuccess[T](
+    fut: Future[T],
+    errorResponse: HttpResponse = HttpResponse(StatusCodes.InternalServerError)
+  )(successHandler: T => Route
+  ): Route = {
+    onComplete(fut) {
+      case Success(r) => successHandler(r)
+      case Failure(_) => complete(errorResponse)
+    }
+  }
+
+  def withProject(id: ProjectId)(route: Project => Route): Route = {
+    val projectFuture =
+      repository
+        .ask(
+          (ref: ActorRef[GetProjectResponse]) => GetProjectById(id, ref)
+        )
+        .map(_.project)
+    withSuccess(projectFuture) {
+      case Some(project) => route(project)
+      case None          => complete(projectDoesNotExistResponse(id))
+    }
+  }
+
+  def listProjectsWith(
+    reqBuilder: ActorRef[ListProjectsResponse] => ProjectsCommand
+  )(baseUri: Uri
+  ): Route = {
+    val projectsFuture = repository.ask(reqBuilder)
+    withSuccess(projectsFuture) { projectsResponse =>
+      val response = projectsResponse.projects.toSeq.map {
+        case (id, project) => apiFactory.fromModel(id, project, baseUri)
+      }
+      complete(response)
+    }
+  }
+
+  def createProject(baseUri: Uri): Route = {
+    val projectFuture = repository.ask(
+      (ref: ActorRef[CreateTemporaryResponse]) =>
+        CreateTemporary("NewProject", ref)
+    )
+    withSuccess(projectFuture) { response =>
+      complete(apiFactory.fromModel(response.id, response.project, baseUri))
+    }
+  }
+
+  def getThumb(projectId: ProjectId): Route = {
+    withProject(projectId) { project =>
+      if (project.pkg.hasThumb) getFromFile(project.pkg.thumbFile)
+      else complete(thumbDoesNotExistResponse)
+    }
+  }
+
+  val route: Route = ignoreTrailingSlash {
+    path(routeHelper.projectsPathMatcher)(
+      (get & extractUri)(listProjectsWith(ListProjectsRequest)) ~
+      (post & extractUri)(createProject)
+    ) ~
+    (get & path(routeHelper.tutorialsPathMatcher) & extractUri)(
+      listProjectsWith(ListTutorialsRequest)
+    ) ~
+    (get & path(routeHelper.thumbPathMatcher))(getThumb)
+  }
+
+  def serve: Future[Http.ServerBinding] = {
+    Http().bindAndHandle(route, host, port)
+  }
+}
+
+object Server {
+
+  def main(args: Array[String]) {
+
+    val config        = ConfigFactory.load.getConfig("project-manager")
+    val serverConfig  = config.getConfig("server")
+    val storageConfig = config.getConfig("storage")
+
+    val host = serverConfig.getString("host")
+    val port = serverConfig.getInt("port")
+
+    val timeout =
+      FiniteDuration(
+        serverConfig.getDuration("timeout").toNanos,
+        TimeUnit.NANOSECONDS
+      )
+
+    implicit val system: ActorSystem             = ActorSystem("project-manager")
+    implicit val executor: ExecutionContext      = system.dispatcher
+    implicit val materializer: ActorMaterializer = ActorMaterializer()
+    implicit val askTimeout: Timeout             = new Timeout(timeout)
+
+    val localProjectsPath =
+      new File(storageConfig.getString("local-projects-path"))
+    val tmpProjectsPath = new File(
+      storageConfig.getString("temporary-projects-path")
+    )
+    val tutorialsPath =
+      new File(storageConfig.getString("tutorials-path"))
+    val tutorialsCachePath =
+      new File(storageConfig.getString("tutorials-cache-path"))
+
+    val tutorialsDownloader =
+      TutorialsDownloader(
+        tutorialsPath,
+        tutorialsCachePath,
+        config.getString("tutorials.github-organisation")
+      )
+    val storageManager = StorageManager(
+      localProjectsPath,
+      tmpProjectsPath,
+      tutorialsPath
+    )
+
+    val repoActor = system.spawn(
+      ProjectsService.behavior(storageManager, tutorialsDownloader),
+      "projects-repository"
+    )
+
+    val routeHelper = new RouteHelper
+    val apiFactory  = ProjectFactory(routeHelper)
+
+    val server = Server(host, port, repoActor, routeHelper, apiFactory)
+    server.serve
+  }
+}

project-manager/src/main/scala/org/enso/projectmanager/api/Project.scala

@@ -0,0 +1,42 @@
+package org.enso.projectmanager.api
+
+import java.util.UUID
+
+import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport
+import akka.http.scaladsl.model.Uri
+import org.enso.projectmanager.RouteHelper
+import org.enso.projectmanager.model
+import org.enso.projectmanager.model.ProjectId
+import spray.json.DefaultJsonProtocol
+
+case class Project(
+  id: String,
+  name: String,
+  path: String,
+  thumb: Option[String],
+  persisted: Boolean)
+
+case class ProjectFactory(routeHelper: RouteHelper) {
+
+  def fromModel(
+    id: ProjectId,
+    project: model.Project,
+    baseUri: Uri
+  ): Project = {
+    val thumbUri =
+      if (project.hasThumb)
+        Some(routeHelper.uriFor(baseUri, routeHelper.thumbPath(id)))
+      else None
+    Project(
+      id.toString,
+      project.pkg.name,
+      project.pkg.root.getAbsolutePath,
+      thumbUri.map(_.toString),
+      project.isPersistent
+    )
+  }
+}
+
+trait ProjectJsonSupport extends SprayJsonSupport with DefaultJsonProtocol {
+  implicit val projectFormat = jsonFormat5(Project.apply)
+}

project-manager/src/main/scala/org/enso/projectmanager/model/Project.scala

@@ -0,0 +1,51 @@
+package org.enso.projectmanager.model
+
+import java.util.UUID
+
+import org.enso.pkg.Package
+
+import scala.collection.immutable.HashMap
+
+sealed trait ProjectType {
+  def isPersistent: Boolean
+}
+case object Local extends ProjectType {
+  override def isPersistent: Boolean = true
+}
+case object Tutorial extends ProjectType {
+  override def isPersistent: Boolean = false
+}
+case object Temporary extends ProjectType {
+  override def isPersistent: Boolean = false
+}
+
+case class ProjectId(uid: UUID) {
+  override def toString: String = uid.toString
+}
+
+case class Project(kind: ProjectType, pkg: Package) {
+  def isPersistent: Boolean = kind.isPersistent
+  def hasThumb:     Boolean = pkg.hasThumb
+}
+
+case class ProjectsRepository(projects: HashMap[ProjectId, Project]) {
+
+  def getById(id: ProjectId): Option[Project] = {
+    projects.get(id)
+  }
+
+  def insert(project: Project): (ProjectId, ProjectsRepository) = {
+    val id      = ProjectsRepository.generateId
+    val newRepo = copy(projects = projects + (id -> project))
+    (id, newRepo)
+  }
+}
+
+case object ProjectsRepository {
+
+  def apply(projects: Seq[Project]): ProjectsRepository = {
+    ProjectsRepository(HashMap(projects.map(generateId -> _): _*))
+  }
+
+  def generateId: ProjectId = ProjectId(UUID.randomUUID)
+}

project-manager/src/main/scala/org/enso/projectmanager/services/ProjectsService.scala

@@ -0,0 +1,78 @@
+package org.enso.projectmanager.services
+
+import java.util.UUID
+
+import akka.actor.typed.scaladsl.Behaviors
+import akka.actor.typed.scaladsl.StashBuffer
+import akka.actor.typed.ActorRef
+import akka.actor.typed.Behavior
+import org.enso.projectmanager.model.Project
+import org.enso.projectmanager.model.ProjectId
+import org.enso.projectmanager.model.ProjectsRepository
+
+import scala.collection.immutable.HashMap
+
+sealed trait ProjectsServiceCommand
+sealed trait ProjectsCommand extends ProjectsServiceCommand
+sealed trait ControlCommand  extends ProjectsServiceCommand
+
+case class ListTutorialsRequest(replyTo: ActorRef[ListProjectsResponse])
+    extends ProjectsCommand
+case class ListProjectsRequest(replyTo: ActorRef[ListProjectsResponse])
+    extends ProjectsCommand
+case class ListProjectsResponse(projects: HashMap[ProjectId, Project])
+
+case class GetProjectById(id: ProjectId, replyTo: ActorRef[GetProjectResponse])
+    extends ProjectsCommand
+case class GetProjectResponse(project: Option[Project])
+
+case class CreateTemporary(
+  name: String,
+  replyTo: ActorRef[CreateTemporaryResponse])
+    extends ProjectsCommand
+case class CreateTemporaryResponse(id: ProjectId, project: Project)
+
+case object TutorialsReady extends ProjectsServiceCommand
+
+object ProjectsService {
+
+  def behavior(
+    storageManager: StorageManager,
+    tutorialsDownloader: TutorialsDownloader
+  ): Behavior[ProjectsServiceCommand] = Behaviors.setup { context =>
+    val buffer = StashBuffer[ProjectsServiceCommand](capacity = 100)
+
+    def handle(
+      localRepo: ProjectsRepository,
+      tutorialsRepo: Option[ProjectsRepository]
+    ): Behavior[ProjectsServiceCommand] = Behaviors.receiveMessage {
+      case ListProjectsRequest(replyTo) =>
+        replyTo ! ListProjectsResponse(localRepo.projects)
+        Behaviors.same
+      case msg: ListTutorialsRequest =>
+        tutorialsRepo match {
+          case Some(repo) => msg.replyTo ! ListProjectsResponse(repo.projects)
+          case None       => buffer.stash(msg)
+        }
+        Behaviors.same
+      case GetProjectById(id, replyTo) =>
+        val project =
+          localRepo.getById(id).orElse(tutorialsRepo.flatMap(_.getById(id)))
+        replyTo ! GetProjectResponse(project)
+        Behaviors.same
+      case TutorialsReady =>
+        val newTutorialsRepo = storageManager.readTutorials
+        buffer.unstashAll(context, handle(localRepo, Some(newTutorialsRepo)))
+      case msg: CreateTemporary =>
+        val project =
+          storageManager.createTemporary(msg.name)
+        val (projectId, newProjectsRepo) = localRepo.insert(project)
+        msg.replyTo ! CreateTemporaryResponse(projectId, project)
+        handle(newProjectsRepo, tutorialsRepo)
+    }
+
+    context.pipeToSelf(tutorialsDownloader.run())(_ => TutorialsReady)
+
+    handle(storageManager.readLocalProjects, None)
+  }
+}

project-manager/src/main/scala/org/enso/projectmanager/services/StorageManager.scala

@@ -0,0 +1,70 @@
+package org.enso.projectmanager.services
+
+import java.io.File
+import java.util.UUID
+
+import org.enso.pkg.Package
+import org.enso.projectmanager.model.Local
+import org.enso.projectmanager.model.Project
+import org.enso.projectmanager.model.ProjectType
+import org.enso.projectmanager.model.ProjectsRepository
+import org.enso.projectmanager.model.Temporary
+import org.enso.projectmanager.model.Tutorial
+
+import scala.collection.immutable.HashMap
+
+case class StorageManager(
+  localProjectsPath: File,
+  tmpProjectsPath: File,
+  tutorialsPath: File) {
+
+  localProjectsPath.mkdirs()
+  tmpProjectsPath.mkdirs()
+  tutorialsPath.mkdirs()
+
+  def moveToLocal(project: Project, newName: Option[String]): Project = {
+    val pkg     = project.pkg
+    val renamed = newName.map(pkg.rename).getOrElse(pkg)
+    val root    = createRootForName(localProjectsPath, renamed.name)
+    val moved   = renamed.move(root)
+    Project(Local, moved)
+  }
+
+  def createRootForName(
+    rootDir: File,
+    name: String,
+    idx: Option[Int] = None
+  ): File = {
+    val idxSuffix = idx.map(idx => s".$idx").getOrElse("")
+    val nameToTry = name + idxSuffix
+    val rootToTry = new File(rootDir, nameToTry)
+
+    if (rootToTry.mkdirs()) rootToTry
+    else {
+      val nextIdx = idx.map(_ + 1).getOrElse(0)
+      createRootForName(rootDir, name, Some(nextIdx))
+    }
+
+  }
+
+  def readLocalProjects: ProjectsRepository =
+    listProjectsInDirectory(Local, localProjectsPath)
+
+  def readTutorials: ProjectsRepository =
+    listProjectsInDirectory(Tutorial, tutorialsPath)
+
+  def listProjectsInDirectory(
+    kind: ProjectType,
+    dir: File
+  ): ProjectsRepository = {
+    val candidates = dir.listFiles(_.isDirectory).toList
+    val projects   = candidates.map(Package.getOrCreate).map(Project(kind, _))
+    ProjectsRepository(projects)
+  }
+
+  def createTemporary(name: String): Project = {
+    val root = createRootForName(tmpProjectsPath, name)
+    val pkg  = Package.create(root, name)
+    Project(Temporary, pkg)
+  }
+}

project-manager/src/main/scala/org/enso/projectmanager/services/TutorialsDownloader.scala

@@ -0,0 +1,143 @@
+package org.enso.projectmanager.services
+
+import java.io.File
+import java.io.FileOutputStream
+import java.time.Instant
+import java.util.zip.ZipFile
+
+import akka.actor.ActorSystem
+import akka.event.LogSource
+import akka.event.Logging
+import akka.http.scaladsl.Http
+import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport
+import akka.http.scaladsl.model.HttpRequest
+import akka.http.scaladsl.model.HttpResponse
+import akka.http.scaladsl.model.StatusCodes
+import akka.http.scaladsl.model.headers.Location
+import akka.http.scaladsl.unmarshalling.Unmarshal
+import akka.stream.ActorMaterializer
+import akka.stream.scaladsl.FileIO
+import org.apache.commons.io.IOUtils
+import spray.json.DefaultJsonProtocol
+import spray.json.JsonParser
+
+import scala.collection.JavaConverters._
+import scala.concurrent.ExecutionContext
+import scala.concurrent.Future
+import scala.util.Try
+
+case class GithubTutorial(name: String, lastPushString: String) {
+  val lastPush: Instant = Instant.parse(lastPushString)
+}
+
+trait GithubJsonProtocol extends SprayJsonSupport with DefaultJsonProtocol {
+  implicit val tutorialFormat =
+    jsonFormat(GithubTutorial, "name", "pushed_at")
+}
+
+case class HttpHelper(
+  implicit val executor: ExecutionContext,
+  implicit val system: ActorSystem,
+  implicit val materializer: ActorMaterializer) {
+
+  def performWithRedirects(request: HttpRequest): Future[HttpResponse] =
+    Http().singleRequest(request).flatMap(followRedirects)
+
+  def followRedirects(response: HttpResponse): Future[HttpResponse] =
+    response.status match {
+      case StatusCodes.Found =>
+        response.entity.discardBytes()
+        val redirectUri = response.header[Location].get.uri
+        val newRequest  = HttpRequest(uri = redirectUri)
+        performWithRedirects(newRequest)
+      case _ => Future(response)
+    }
+}
+
+case class TutorialsDownloader(
+  tutorialsDir: File,
+  cacheDir: File,
+  packagesGithubOrganisation: String
+)(implicit val system: ActorSystem,
+  implicit val executor: ExecutionContext,
+  implicit val materializer: ActorMaterializer)
+    extends GithubJsonProtocol {
+
+  val packagesGithubUrl =
+    s"https://api.github.com/orgs/$packagesGithubOrganisation"
+  val packagesGithubReposUrl = s"$packagesGithubUrl/repos"
+
+  implicit val logSource: LogSource[TutorialsDownloader] = _ =>
+    "TutorialsDownloader"
+  val logging = Logging(system, this)
+
+  tutorialsDir.mkdirs()
+  cacheDir.mkdirs()
+
+  def downloadUrlFor(tutorial: GithubTutorial): String =
+    s"https://github.com/$packagesGithubOrganisation/${tutorial.name}/archive/master.zip"
+
+  def zipFileFor(tutorial: GithubTutorial): File =
+    new File(cacheDir, s"${tutorial.name}.zip")
+
+  def getAvailable: Future[List[GithubTutorial]] =
+    Http()
+      .singleRequest(HttpRequest(uri = packagesGithubReposUrl))
+      .flatMap(resp => Unmarshal(resp.entity).to[String])
+      .map(JsonParser(_).convertTo[List[GithubTutorial]])
+
+  def downloadIfNeeded(tutorial: GithubTutorial): Future[File] =
+    if (needsUpdate(tutorial)) {
+      logging.info(s"Downloading zip archive for ${tutorial.name}")
+      downloadZip(tutorial)
+    }
+    else {
+      logging.info(s"Reusing cached zip archive for ${tutorial.name}")
+      Future { zipFileFor(tutorial) }
+    }
+
+  def needsUpdate(tutorial: GithubTutorial): Boolean =
+    zipFileFor(tutorial).lastModified < tutorial.lastPush.toEpochMilli
+
+  def downloadZip(tutorial: GithubTutorial): Future[File] = {
+    val downloadUrl = downloadUrlFor(tutorial)
+    val request     = HttpRequest(uri = downloadUrl)
+    val response    = HttpHelper().performWithRedirects(request)
+    val target      = zipFileFor(tutorial)
+    response
+      .flatMap(_.entity.dataBytes.runWith(FileIO.toPath(target.toPath)))
+      .map(_ => target)
+  }
+
+  def unzip(source: File): Unit = {
+    logging.debug(s"Unzipping $source\n")
+    val zipFileTry = Try(new ZipFile(source))
+    val result = zipFileTry.map { zipFile =>
+      zipFile.entries.asScala.foreach { entry =>
+        logging.debug(s"Extracting ${entry.getName}\n")
+        val target = new File(tutorialsDir, entry.getName)
+        if (entry.isDirectory) target.mkdirs()
+        else {
+          target.getParentFile.mkdirs()
+          val in  = zipFile.getInputStream(entry)
+          val out = new FileOutputStream(target)
+          IOUtils.copy(in, out)
+          out.close()
+        }
+      }
+    }
+    zipFileTry.map(_.close())
+    result.get
+  }
+
+  def run(): Future[Unit] = {
+    val tutorialsFuture = getAvailable
+    val zipsFuture = tutorialsFuture.flatMap { tutorials =>
+      Future.sequence(
+        tutorials.map(downloadIfNeeded)
+      )
+    }
+    zipsFuture.map(_.foreach(unzip))
+  }
+
+}