From c2a60eb186d8e72109ec2ed2db0f6034c593542d Mon Sep 17 00:00:00 2001 From: Marcin Kostrzewa Date: Wed, 10 Jul 2019 14:13:45 +0200 Subject: [PATCH] Project manager (#11) --- build.sbt | 26 ++- .../src/main/resources/application.conf | 19 ++ .../org/enso/projectmanager/RouteHelper.scala | 33 +++ .../org/enso/projectmanager/Server.scala | 189 ++++++++++++++++++ .../org/enso/projectmanager/api/Project.scala | 42 ++++ .../enso/projectmanager/model/Project.scala | 51 +++++ .../services/ProjectsService.scala | 78 ++++++++ .../services/StorageManager.scala | 70 +++++++ .../services/TutorialsDownloader.scala | 143 +++++++++++++ 9 files changed, 650 insertions(+), 1 deletion(-) create mode 100644 project-manager/src/main/resources/application.conf create mode 100644 project-manager/src/main/scala/org/enso/projectmanager/RouteHelper.scala create mode 100644 project-manager/src/main/scala/org/enso/projectmanager/Server.scala create mode 100644 project-manager/src/main/scala/org/enso/projectmanager/api/Project.scala create mode 100644 project-manager/src/main/scala/org/enso/projectmanager/model/Project.scala create mode 100644 project-manager/src/main/scala/org/enso/projectmanager/services/ProjectsService.scala create mode 100644 project-manager/src/main/scala/org/enso/projectmanager/services/StorageManager.scala create mode 100644 project-manager/src/main/scala/org/enso/projectmanager/services/TutorialsDownloader.scala diff --git a/build.sbt b/build.sbt index a8ee16987f8..439aa4390ae 100644 --- a/build.sbt +++ b/build.sbt @@ -20,7 +20,8 @@ lazy val enso = (project in file(".")) .aggregate( syntax, pkg, - interpreter + interpreter, + projectManager ) // Sub-Projects @@ -90,3 +91,26 @@ lazy val interpreter = (project in file("interpreter")) bench := (test in Benchmark).value, parallelExecution in Benchmark := false ) + +val akkaActor = "com.typesafe.akka" %% "akka-actor" % "2.5.23" +val akkaStream = "com.typesafe.akka" %% "akka-stream" % "2.5.23" +val akkaHttp = "com.typesafe.akka" %% "akka-http" % "10.1.8" +val akkaSpray = "com.typesafe.akka" %% "akka-http-spray-json" % "10.1.8" +val akkaTyped = "com.typesafe.akka" %% "akka-actor-typed" % "2.5.23" + +val akka = Seq(akkaActor, akkaStream, akkaHttp, akkaSpray, akkaTyped) + +val circe = Seq("circe-core", "circe-generic", "circe-yaml").map( + "io.circe" %% _ % "0.10.0" +) + +lazy val projectManager = (project in file("project-manager")) + .settings( + (Compile / mainClass) := Some("org.enso.projectmanager.Server") + ) + .settings( + libraryDependencies ++= akka, + libraryDependencies ++= circe, + libraryDependencies += "io.spray" %% "spray-json" % "1.3.5" + ) + .dependsOn(pkg) diff --git a/project-manager/src/main/resources/application.conf b/project-manager/src/main/resources/application.conf new file mode 100644 index 00000000000..3cde653792c --- /dev/null +++ b/project-manager/src/main/resources/application.conf @@ -0,0 +1,19 @@ +project-manager { + server { + host = "0.0.0.0" + port = 30535 + timeout = 10 seconds + } + + storage { + projects-root = ${user.home}/enso + temporary-projects-path = ${project-manager.storage.projects-root}/tmp + local-projects-path = ${project-manager.storage.projects-root}/projects + tutorials-path = ${project-manager.storage.projects-root}/tutorials + tutorials-cache-path = ${project-manager.storage.projects-root}/.tutorials-cache + } + + tutorials { + github-organisation = "luna-packages" + } +} \ No newline at end of file diff --git a/project-manager/src/main/scala/org/enso/projectmanager/RouteHelper.scala b/project-manager/src/main/scala/org/enso/projectmanager/RouteHelper.scala new file mode 100644 index 00000000000..b6c416f82fb --- /dev/null +++ b/project-manager/src/main/scala/org/enso/projectmanager/RouteHelper.scala @@ -0,0 +1,33 @@ +package org.enso.projectmanager + +import java.util.UUID + +import akka.http.scaladsl.model.Uri +import akka.http.scaladsl.model.Uri.Path +import akka.http.scaladsl.server.PathMatcher0 +import akka.http.scaladsl.server.PathMatcher1 +import akka.http.scaladsl.server.PathMatchers.JavaUUID +import org.enso.projectmanager.model.ProjectId + +class RouteHelper { + + val tutorials: String = "tutorials" + val projects: String = "projects" + val thumb: String = "thumb" + + val tutorialsPath: Path = Path / tutorials + val tutorialsPathMatcher: PathMatcher0 = tutorials + + val projectsPath: Path = Path / projects + val projectsPathMatcher: PathMatcher0 = projects + + def projectPath(id: ProjectId): Path = projectsPath / id.toString + + val projectPathMatcher: PathMatcher1[ProjectId] = + (projectsPathMatcher / JavaUUID).map(ProjectId) + + def thumbPath(id: ProjectId): Path = projectPath(id) / thumb + val thumbPathMatcher: PathMatcher1[ProjectId] = projectPathMatcher / thumb + + def uriFor(base: Uri, path: Path): Uri = base.withPath(path) +} diff --git a/project-manager/src/main/scala/org/enso/projectmanager/Server.scala b/project-manager/src/main/scala/org/enso/projectmanager/Server.scala new file mode 100644 index 00000000000..67532eadf51 --- /dev/null +++ b/project-manager/src/main/scala/org/enso/projectmanager/Server.scala @@ -0,0 +1,189 @@ +package org.enso.projectmanager + +import java.io.File +import java.util.UUID +import java.util.concurrent.TimeUnit + +import akka.actor.ActorSystem +import akka.actor.Scheduler +import com.typesafe.config.ConfigFactory +import akka.actor.typed.ActorRef +import akka.actor.typed.scaladsl.adapter._ +import akka.actor.typed.scaladsl.AskPattern._ +import akka.http.scaladsl.Http +import akka.http.scaladsl.model.HttpResponse +import akka.http.scaladsl.model.StatusCodes +import akka.http.scaladsl.model.Uri +import akka.http.scaladsl.server.Directives +import akka.http.scaladsl.server.Route +import akka.stream.ActorMaterializer +import akka.util.Timeout +import org.enso.projectmanager.api.ProjectFactory +import org.enso.projectmanager.api.ProjectJsonSupport +import org.enso.projectmanager.model.Project +import org.enso.projectmanager.model.ProjectId +import org.enso.projectmanager.services.CreateTemporary +import org.enso.projectmanager.services.CreateTemporaryResponse +import org.enso.projectmanager.services.GetProjectById +import org.enso.projectmanager.services.GetProjectResponse +import org.enso.projectmanager.services.ListProjectsRequest +import org.enso.projectmanager.services.ListProjectsResponse +import org.enso.projectmanager.services.ListTutorialsRequest +import org.enso.projectmanager.services.ProjectsCommand +import org.enso.projectmanager.services.ProjectsService +import org.enso.projectmanager.services.StorageManager +import org.enso.projectmanager.services.TutorialsDownloader + +import scala.concurrent.ExecutionContext +import scala.concurrent.Future +import scala.concurrent.duration._ +import scala.util.Failure +import scala.util.Success + +case class Server( + host: String, + port: Int, + repository: ActorRef[ProjectsCommand], + routeHelper: RouteHelper, + apiFactory: ProjectFactory +)(implicit val system: ActorSystem, + implicit val executor: ExecutionContext, + implicit val materializer: ActorMaterializer, + implicit val askTimeout: Timeout) + extends Directives + with ProjectJsonSupport { + + implicit val scheduler: Scheduler = system.scheduler + + def projectDoesNotExistResponse(id: ProjectId): HttpResponse = + HttpResponse(StatusCodes.NotFound, entity = s"Project $id does not exist") + + def thumbDoesNotExistResponse: HttpResponse = + HttpResponse(StatusCodes.NotFound, entity = "Thumbnail does not exist") + + def withSuccess[T]( + fut: Future[T], + errorResponse: HttpResponse = HttpResponse(StatusCodes.InternalServerError) + )(successHandler: T => Route + ): Route = { + onComplete(fut) { + case Success(r) => successHandler(r) + case Failure(_) => complete(errorResponse) + } + } + + def withProject(id: ProjectId)(route: Project => Route): Route = { + val projectFuture = + repository + .ask( + (ref: ActorRef[GetProjectResponse]) => GetProjectById(id, ref) + ) + .map(_.project) + withSuccess(projectFuture) { + case Some(project) => route(project) + case None => complete(projectDoesNotExistResponse(id)) + } + } + + def listProjectsWith( + reqBuilder: ActorRef[ListProjectsResponse] => ProjectsCommand + )(baseUri: Uri + ): Route = { + val projectsFuture = repository.ask(reqBuilder) + withSuccess(projectsFuture) { projectsResponse => + val response = projectsResponse.projects.toSeq.map { + case (id, project) => apiFactory.fromModel(id, project, baseUri) + } + complete(response) + } + } + + def createProject(baseUri: Uri): Route = { + val projectFuture = repository.ask( + (ref: ActorRef[CreateTemporaryResponse]) => + CreateTemporary("NewProject", ref) + ) + withSuccess(projectFuture) { response => + complete(apiFactory.fromModel(response.id, response.project, baseUri)) + } + } + + def getThumb(projectId: ProjectId): Route = { + withProject(projectId) { project => + if (project.pkg.hasThumb) getFromFile(project.pkg.thumbFile) + else complete(thumbDoesNotExistResponse) + } + } + + val route: Route = ignoreTrailingSlash { + path(routeHelper.projectsPathMatcher)( + (get & extractUri)(listProjectsWith(ListProjectsRequest)) ~ + (post & extractUri)(createProject) + ) ~ + (get & path(routeHelper.tutorialsPathMatcher) & extractUri)( + listProjectsWith(ListTutorialsRequest) + ) ~ + (get & path(routeHelper.thumbPathMatcher))(getThumb) + } + + def serve: Future[Http.ServerBinding] = { + Http().bindAndHandle(route, host, port) + } +} + +object Server { + + def main(args: Array[String]) { + + val config = ConfigFactory.load.getConfig("project-manager") + val serverConfig = config.getConfig("server") + val storageConfig = config.getConfig("storage") + + val host = serverConfig.getString("host") + val port = serverConfig.getInt("port") + + val timeout = + FiniteDuration( + serverConfig.getDuration("timeout").toNanos, + TimeUnit.NANOSECONDS + ) + + implicit val system: ActorSystem = ActorSystem("project-manager") + implicit val executor: ExecutionContext = system.dispatcher + implicit val materializer: ActorMaterializer = ActorMaterializer() + implicit val askTimeout: Timeout = new Timeout(timeout) + + val localProjectsPath = + new File(storageConfig.getString("local-projects-path")) + val tmpProjectsPath = new File( + storageConfig.getString("temporary-projects-path") + ) + val tutorialsPath = + new File(storageConfig.getString("tutorials-path")) + val tutorialsCachePath = + new File(storageConfig.getString("tutorials-cache-path")) + + val tutorialsDownloader = + TutorialsDownloader( + tutorialsPath, + tutorialsCachePath, + config.getString("tutorials.github-organisation") + ) + val storageManager = StorageManager( + localProjectsPath, + tmpProjectsPath, + tutorialsPath + ) + + val repoActor = system.spawn( + ProjectsService.behavior(storageManager, tutorialsDownloader), + "projects-repository" + ) + + val routeHelper = new RouteHelper + val apiFactory = ProjectFactory(routeHelper) + + val server = Server(host, port, repoActor, routeHelper, apiFactory) + server.serve + } +} diff --git a/project-manager/src/main/scala/org/enso/projectmanager/api/Project.scala b/project-manager/src/main/scala/org/enso/projectmanager/api/Project.scala new file mode 100644 index 00000000000..ef265a305cf --- /dev/null +++ b/project-manager/src/main/scala/org/enso/projectmanager/api/Project.scala @@ -0,0 +1,42 @@ +package org.enso.projectmanager.api + +import java.util.UUID + +import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport +import akka.http.scaladsl.model.Uri +import org.enso.projectmanager.RouteHelper +import org.enso.projectmanager.model +import org.enso.projectmanager.model.ProjectId +import spray.json.DefaultJsonProtocol + +case class Project( + id: String, + name: String, + path: String, + thumb: Option[String], + persisted: Boolean) + +case class ProjectFactory(routeHelper: RouteHelper) { + + def fromModel( + id: ProjectId, + project: model.Project, + baseUri: Uri + ): Project = { + val thumbUri = + if (project.hasThumb) + Some(routeHelper.uriFor(baseUri, routeHelper.thumbPath(id))) + else None + Project( + id.toString, + project.pkg.name, + project.pkg.root.getAbsolutePath, + thumbUri.map(_.toString), + project.isPersistent + ) + } +} + +trait ProjectJsonSupport extends SprayJsonSupport with DefaultJsonProtocol { + implicit val projectFormat = jsonFormat5(Project.apply) +} diff --git a/project-manager/src/main/scala/org/enso/projectmanager/model/Project.scala b/project-manager/src/main/scala/org/enso/projectmanager/model/Project.scala new file mode 100644 index 00000000000..70779aee678 --- /dev/null +++ b/project-manager/src/main/scala/org/enso/projectmanager/model/Project.scala @@ -0,0 +1,51 @@ +package org.enso.projectmanager.model + +import java.util.UUID + +import org.enso.pkg.Package + +import scala.collection.immutable.HashMap + +sealed trait ProjectType { + def isPersistent: Boolean +} +case object Local extends ProjectType { + override def isPersistent: Boolean = true +} +case object Tutorial extends ProjectType { + override def isPersistent: Boolean = false +} +case object Temporary extends ProjectType { + override def isPersistent: Boolean = false +} + +case class ProjectId(uid: UUID) { + override def toString: String = uid.toString +} + +case class Project(kind: ProjectType, pkg: Package) { + def isPersistent: Boolean = kind.isPersistent + def hasThumb: Boolean = pkg.hasThumb +} + +case class ProjectsRepository(projects: HashMap[ProjectId, Project]) { + + def getById(id: ProjectId): Option[Project] = { + projects.get(id) + } + + def insert(project: Project): (ProjectId, ProjectsRepository) = { + val id = ProjectsRepository.generateId + val newRepo = copy(projects = projects + (id -> project)) + (id, newRepo) + } +} + +case object ProjectsRepository { + + def apply(projects: Seq[Project]): ProjectsRepository = { + ProjectsRepository(HashMap(projects.map(generateId -> _): _*)) + } + + def generateId: ProjectId = ProjectId(UUID.randomUUID) +} diff --git a/project-manager/src/main/scala/org/enso/projectmanager/services/ProjectsService.scala b/project-manager/src/main/scala/org/enso/projectmanager/services/ProjectsService.scala new file mode 100644 index 00000000000..85a80f56302 --- /dev/null +++ b/project-manager/src/main/scala/org/enso/projectmanager/services/ProjectsService.scala @@ -0,0 +1,78 @@ +package org.enso.projectmanager.services + +import java.util.UUID + +import akka.actor.typed.scaladsl.Behaviors +import akka.actor.typed.scaladsl.StashBuffer +import akka.actor.typed.ActorRef +import akka.actor.typed.Behavior +import org.enso.projectmanager.model.Project +import org.enso.projectmanager.model.ProjectId +import org.enso.projectmanager.model.ProjectsRepository + +import scala.collection.immutable.HashMap + +sealed trait ProjectsServiceCommand +sealed trait ProjectsCommand extends ProjectsServiceCommand +sealed trait ControlCommand extends ProjectsServiceCommand + +case class ListTutorialsRequest(replyTo: ActorRef[ListProjectsResponse]) + extends ProjectsCommand +case class ListProjectsRequest(replyTo: ActorRef[ListProjectsResponse]) + extends ProjectsCommand +case class ListProjectsResponse(projects: HashMap[ProjectId, Project]) + +case class GetProjectById(id: ProjectId, replyTo: ActorRef[GetProjectResponse]) + extends ProjectsCommand +case class GetProjectResponse(project: Option[Project]) + +case class CreateTemporary( + name: String, + replyTo: ActorRef[CreateTemporaryResponse]) + extends ProjectsCommand +case class CreateTemporaryResponse(id: ProjectId, project: Project) + +case object TutorialsReady extends ProjectsServiceCommand + +object ProjectsService { + + def behavior( + storageManager: StorageManager, + tutorialsDownloader: TutorialsDownloader + ): Behavior[ProjectsServiceCommand] = Behaviors.setup { context => + val buffer = StashBuffer[ProjectsServiceCommand](capacity = 100) + + def handle( + localRepo: ProjectsRepository, + tutorialsRepo: Option[ProjectsRepository] + ): Behavior[ProjectsServiceCommand] = Behaviors.receiveMessage { + case ListProjectsRequest(replyTo) => + replyTo ! ListProjectsResponse(localRepo.projects) + Behaviors.same + case msg: ListTutorialsRequest => + tutorialsRepo match { + case Some(repo) => msg.replyTo ! ListProjectsResponse(repo.projects) + case None => buffer.stash(msg) + } + Behaviors.same + case GetProjectById(id, replyTo) => + val project = + localRepo.getById(id).orElse(tutorialsRepo.flatMap(_.getById(id))) + replyTo ! GetProjectResponse(project) + Behaviors.same + case TutorialsReady => + val newTutorialsRepo = storageManager.readTutorials + buffer.unstashAll(context, handle(localRepo, Some(newTutorialsRepo))) + case msg: CreateTemporary => + val project = + storageManager.createTemporary(msg.name) + val (projectId, newProjectsRepo) = localRepo.insert(project) + msg.replyTo ! CreateTemporaryResponse(projectId, project) + handle(newProjectsRepo, tutorialsRepo) + } + + context.pipeToSelf(tutorialsDownloader.run())(_ => TutorialsReady) + + handle(storageManager.readLocalProjects, None) + } +} diff --git a/project-manager/src/main/scala/org/enso/projectmanager/services/StorageManager.scala b/project-manager/src/main/scala/org/enso/projectmanager/services/StorageManager.scala new file mode 100644 index 00000000000..77e6e58bb85 --- /dev/null +++ b/project-manager/src/main/scala/org/enso/projectmanager/services/StorageManager.scala @@ -0,0 +1,70 @@ +package org.enso.projectmanager.services + +import java.io.File +import java.util.UUID + +import org.enso.pkg.Package +import org.enso.projectmanager.model.Local +import org.enso.projectmanager.model.Project +import org.enso.projectmanager.model.ProjectType +import org.enso.projectmanager.model.ProjectsRepository +import org.enso.projectmanager.model.Temporary +import org.enso.projectmanager.model.Tutorial + +import scala.collection.immutable.HashMap + +case class StorageManager( + localProjectsPath: File, + tmpProjectsPath: File, + tutorialsPath: File) { + + localProjectsPath.mkdirs() + tmpProjectsPath.mkdirs() + tutorialsPath.mkdirs() + + def moveToLocal(project: Project, newName: Option[String]): Project = { + val pkg = project.pkg + val renamed = newName.map(pkg.rename).getOrElse(pkg) + val root = createRootForName(localProjectsPath, renamed.name) + val moved = renamed.move(root) + Project(Local, moved) + } + + def createRootForName( + rootDir: File, + name: String, + idx: Option[Int] = None + ): File = { + val idxSuffix = idx.map(idx => s".$idx").getOrElse("") + val nameToTry = name + idxSuffix + val rootToTry = new File(rootDir, nameToTry) + + if (rootToTry.mkdirs()) rootToTry + else { + val nextIdx = idx.map(_ + 1).getOrElse(0) + createRootForName(rootDir, name, Some(nextIdx)) + } + + } + + def readLocalProjects: ProjectsRepository = + listProjectsInDirectory(Local, localProjectsPath) + + def readTutorials: ProjectsRepository = + listProjectsInDirectory(Tutorial, tutorialsPath) + + def listProjectsInDirectory( + kind: ProjectType, + dir: File + ): ProjectsRepository = { + val candidates = dir.listFiles(_.isDirectory).toList + val projects = candidates.map(Package.getOrCreate).map(Project(kind, _)) + ProjectsRepository(projects) + } + + def createTemporary(name: String): Project = { + val root = createRootForName(tmpProjectsPath, name) + val pkg = Package.create(root, name) + Project(Temporary, pkg) + } +} diff --git a/project-manager/src/main/scala/org/enso/projectmanager/services/TutorialsDownloader.scala b/project-manager/src/main/scala/org/enso/projectmanager/services/TutorialsDownloader.scala new file mode 100644 index 00000000000..cefa08283e2 --- /dev/null +++ b/project-manager/src/main/scala/org/enso/projectmanager/services/TutorialsDownloader.scala @@ -0,0 +1,143 @@ +package org.enso.projectmanager.services + +import java.io.File +import java.io.FileOutputStream +import java.time.Instant +import java.util.zip.ZipFile + +import akka.actor.ActorSystem +import akka.event.LogSource +import akka.event.Logging +import akka.http.scaladsl.Http +import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport +import akka.http.scaladsl.model.HttpRequest +import akka.http.scaladsl.model.HttpResponse +import akka.http.scaladsl.model.StatusCodes +import akka.http.scaladsl.model.headers.Location +import akka.http.scaladsl.unmarshalling.Unmarshal +import akka.stream.ActorMaterializer +import akka.stream.scaladsl.FileIO +import org.apache.commons.io.IOUtils +import spray.json.DefaultJsonProtocol +import spray.json.JsonParser + +import scala.collection.JavaConverters._ +import scala.concurrent.ExecutionContext +import scala.concurrent.Future +import scala.util.Try + +case class GithubTutorial(name: String, lastPushString: String) { + val lastPush: Instant = Instant.parse(lastPushString) +} + +trait GithubJsonProtocol extends SprayJsonSupport with DefaultJsonProtocol { + implicit val tutorialFormat = + jsonFormat(GithubTutorial, "name", "pushed_at") +} + +case class HttpHelper( + implicit val executor: ExecutionContext, + implicit val system: ActorSystem, + implicit val materializer: ActorMaterializer) { + + def performWithRedirects(request: HttpRequest): Future[HttpResponse] = + Http().singleRequest(request).flatMap(followRedirects) + + def followRedirects(response: HttpResponse): Future[HttpResponse] = + response.status match { + case StatusCodes.Found => + response.entity.discardBytes() + val redirectUri = response.header[Location].get.uri + val newRequest = HttpRequest(uri = redirectUri) + performWithRedirects(newRequest) + case _ => Future(response) + } +} + +case class TutorialsDownloader( + tutorialsDir: File, + cacheDir: File, + packagesGithubOrganisation: String +)(implicit val system: ActorSystem, + implicit val executor: ExecutionContext, + implicit val materializer: ActorMaterializer) + extends GithubJsonProtocol { + + val packagesGithubUrl = + s"https://api.github.com/orgs/$packagesGithubOrganisation" + val packagesGithubReposUrl = s"$packagesGithubUrl/repos" + + implicit val logSource: LogSource[TutorialsDownloader] = _ => + "TutorialsDownloader" + val logging = Logging(system, this) + + tutorialsDir.mkdirs() + cacheDir.mkdirs() + + def downloadUrlFor(tutorial: GithubTutorial): String = + s"https://github.com/$packagesGithubOrganisation/${tutorial.name}/archive/master.zip" + + def zipFileFor(tutorial: GithubTutorial): File = + new File(cacheDir, s"${tutorial.name}.zip") + + def getAvailable: Future[List[GithubTutorial]] = + Http() + .singleRequest(HttpRequest(uri = packagesGithubReposUrl)) + .flatMap(resp => Unmarshal(resp.entity).to[String]) + .map(JsonParser(_).convertTo[List[GithubTutorial]]) + + def downloadIfNeeded(tutorial: GithubTutorial): Future[File] = + if (needsUpdate(tutorial)) { + logging.info(s"Downloading zip archive for ${tutorial.name}") + downloadZip(tutorial) + } + else { + logging.info(s"Reusing cached zip archive for ${tutorial.name}") + Future { zipFileFor(tutorial) } + } + + def needsUpdate(tutorial: GithubTutorial): Boolean = + zipFileFor(tutorial).lastModified < tutorial.lastPush.toEpochMilli + + def downloadZip(tutorial: GithubTutorial): Future[File] = { + val downloadUrl = downloadUrlFor(tutorial) + val request = HttpRequest(uri = downloadUrl) + val response = HttpHelper().performWithRedirects(request) + val target = zipFileFor(tutorial) + response + .flatMap(_.entity.dataBytes.runWith(FileIO.toPath(target.toPath))) + .map(_ => target) + } + + def unzip(source: File): Unit = { + logging.debug(s"Unzipping $source\n") + val zipFileTry = Try(new ZipFile(source)) + val result = zipFileTry.map { zipFile => + zipFile.entries.asScala.foreach { entry => + logging.debug(s"Extracting ${entry.getName}\n") + val target = new File(tutorialsDir, entry.getName) + if (entry.isDirectory) target.mkdirs() + else { + target.getParentFile.mkdirs() + val in = zipFile.getInputStream(entry) + val out = new FileOutputStream(target) + IOUtils.copy(in, out) + out.close() + } + } + } + zipFileTry.map(_.close()) + result.get + } + + def run(): Future[Unit] = { + val tutorialsFuture = getAvailable + val zipsFuture = tutorialsFuture.flatMap { tutorials => + Future.sequence( + tutorials.map(downloadIfNeeded) + ) + } + zipsFuture.map(_.foreach(unzip)) + } + +}