Update Simple Library Server (#1952)

RELEASES.md

@@ -7,6 +7,13 @@
 - Added the ability to specify cell ranges for reading XLS and XSLX spreadsheets
   ([#1954](https://github.com/enso-org/enso/pull/1954)).
 
+## Tooling
+
+- Updated the Simple Library Server to make it more robust; updated the edition
+  configuration with a proper URL to the Enso Library Repository, making it
+  possible for new libraries to be downloaded from it
+  ([#1952](https://github.com/enso-org/enso/pull/1952)).
+
 # Enso 0.2.24 (2021-08-13)
 
 ## Interpreter/Runtime

docs/libraries/sharing.md

@@ -75,4 +75,6 @@ Then you can use the Enso CLI to upload the project:
 enso publish-library --upload-url <URL> <path to project root>
 ```
 
-See `enso publish-library --help` for more information.
+The `--upload-url` is optional, if not provided, the library will be uploaded to
+the main Enso library repository. See `enso publish-library --help` for more
+information.

engine/launcher/src/main/scala/org/enso/launcher/Constants.scala

@@ -11,4 +11,7 @@ object Constants {
     */
   val uploadIntroducedVersion: SemVer =
     SemVer(0, 2, 17, Some("SNAPSHOT"))
+
+  /** The upload URL associated with the main Enso library repository. */
+  val defaultUploadUrl = "https://publish.libraries.release.enso.org/"
 }

engine/launcher/src/main/scala/org/enso/launcher/Launcher.scala

@@ -368,12 +368,7 @@ case class Launcher(cliOptions: GlobalCLIOptions) {
     val settings = runner
       .uploadLibrary(
         path,
-        uploadUrl.getOrElse {
+        uploadUrl.getOrElse(Constants.defaultUploadUrl),
-          throw new IllegalArgumentException(
-            "The default repository is currently not defined. " +
-            "You need to explicitly specify the `--upload-url`."
-          )
-        },
         authToken.orElse(LauncherEnvironment.getEnvVar("ENSO_AUTH_TOKEN")),
         cliOptions.hideProgress,
         logLevel,

engine/runner/src/main/scala/org/enso/runner/Main.scala

@@ -7,15 +7,16 @@ import org.apache.commons.cli.{Option => CliOption, _}
 import org.enso.editions.DefaultEdition
 import org.enso.languageserver.boot
 import org.enso.languageserver.boot.LanguageServerConfig
+import org.enso.libraryupload.LibraryUploader.UploadFailedError
 import org.enso.loggingservice.LogLevel
 import org.enso.pkg.{Contact, PackageManager, Template}
 import org.enso.polyglot.{LanguageInfo, Module, PolyglotContext}
 import org.enso.version.VersionDescription
 import org.graalvm.polyglot.PolyglotException
+
 import java.io.File
 import java.nio.file.Path
 import java.util.UUID
-
 import scala.Console.err
 import scala.jdk.CollectionConverters._
 import scala.util.Try
@@ -700,13 +701,20 @@ object Main {
             exitFail()
           }
 
-      ProjectUploader.uploadProject(
+      try {
-        projectRoot  = projectRoot,
+        ProjectUploader.uploadProject(
-        uploadUrl    = line.getOptionValue(UPLOAD_OPTION),
+          projectRoot  = projectRoot,
-        authToken    = Option(line.getOptionValue(AUTH_TOKEN)),
+          uploadUrl    = line.getOptionValue(UPLOAD_OPTION),
-        showProgress = !line.hasOption(HIDE_PROGRESS)
+          authToken    = Option(line.getOptionValue(AUTH_TOKEN)),
-      )
+          showProgress = !line.hasOption(HIDE_PROGRESS)
-      exitSuccess()
+        )
+        exitSuccess()
+      } catch {
+        case UploadFailedError(_) =>
+          // We catch this error to avoid printing an unnecessary stack trace.
+          // The error itself is already logged.
+          exitFail()
+      }
     }
 
     if (line.hasOption(RUN_OPTION)) {

lib/scala/cli/src/main/scala/org/enso/cli/task/MappedTask.scala

@@ -7,17 +7,30 @@ import scala.util.Try
   * Used internally by [[TaskProgress.flatMap]].
   */
 private class MappedTask[A, B](source: TaskProgress[A], f: A => Try[B])
-    extends TaskProgress[B] {
+    extends TaskProgress[B] { self =>
-  override def addProgressListener(
-    listener: ProgressListener[B]
-  ): Unit =
-    source.addProgressListener(new ProgressListener[A] {
-      override def progressUpdate(done: Long, total: Option[Long]): Unit =
-        listener.progressUpdate(done, total)
 
-      override def done(result: Try[A]): Unit =
+  var listeners: List[ProgressListener[B]] = Nil
-        listener.done(result.flatMap(f))
+  var savedResult: Option[Try[B]]          = None
-    })
+
+  source.addProgressListener(new ProgressListener[A] {
+    override def progressUpdate(done: Long, total: Option[Long]): Unit =
+      listeners.foreach(_.progressUpdate(done, total))
+
+    override def done(result: Try[A]): Unit = self.synchronized {
+      val mapped = result.flatMap(f)
+      savedResult = Some(mapped)
+      listeners.foreach(_.done(mapped))
+    }
+  })
+
+  override def addProgressListener(listener: ProgressListener[B]): Unit =
+    self.synchronized {
+      listeners ::= listener
+      savedResult match {
+        case Some(saved) => listener.done(saved)
+        case None        =>
+      }
+    }
 
   override def unit: ProgressUnit = source.unit
 }

lib/scala/cli/src/test/scala/org/enso/cli/task/MappedTaskSpec.scala

@@ -0,0 +1,40 @@
+package org.enso.cli.task
+
+import org.scalatest.matchers.should.Matchers
+import org.scalatest.wordspec.AnyWordSpec
+
+import scala.util.{Success, Try}
+
+class MappedTaskSpec extends AnyWordSpec with Matchers {
+  "TaskProgress.map" should {
+    "run only once even with multiple listeners" in {
+      var runs  = 0
+      val task1 = new TaskProgressImplementation[String]()
+      val task2 = task1.map { str =>
+        runs += 1
+        str + "bar"
+      }
+
+      val emptyListener = new ProgressListener[String] {
+        override def progressUpdate(done: Long, total: Option[Long]): Unit = ()
+        override def done(result: Try[String]): Unit                       = ()
+      }
+      task2.addProgressListener(emptyListener)
+      task2.addProgressListener(emptyListener)
+
+      task1.setComplete(Success("foo"))
+
+      task2.addProgressListener(emptyListener)
+      var answer: Option[Try[String]] = None
+      task2.addProgressListener(new ProgressListener[String] {
+        override def progressUpdate(done: Long, total: Option[Long]): Unit = ()
+        override def done(result: Try[String]): Unit = {
+          answer = Some(result)
+        }
+      })
+
+      answer shouldEqual Some(Success("foobar"))
+      runs shouldEqual 1
+    }
+  }
+}

lib/scala/library-manager/src/main/scala/org/enso/libraryupload/LibraryUploader.scala

@@ -92,6 +92,10 @@ object LibraryUploader {
     }
   }
 
+  /** Indicates that the library upload has failed. */
+  case class UploadFailedError(message: String)
+      extends RuntimeException(message)
+
   /** Creates an URL for the upload, including information identifying the
     * library version.
     */
@@ -222,11 +226,7 @@ object LibraryUploader {
         val errorMessage =
           s"Upload failed: $message (Status code: ${response.statusCode})."
         logger.error(errorMessage)
-        Failure(
+        Failure(UploadFailedError(errorMessage))
-          new RuntimeException(
-            errorMessage
-          )
-        )
       }
     }
   }

project/Editions.scala

@@ -26,6 +26,9 @@ object Editions {
     */
   val contribLibraries: Seq[ContribLibrary] = Seq()
 
+  /** The URL to the main library repository. */
+  val mainLibraryRepositoryUrl = "https://libraries.release.enso.org/libraries"
+
   private val editionsRoot = file("distribution") / "editions"
   private val extension    = ".yaml"
 
@@ -68,7 +71,7 @@ object Editions {
         s"""engine-version: $ensoVersion
            |repositories:
            |  - name: main
-           |    url: n/a # Library repository is still a work in progress.
+           |    url: $mainLibraryRepositoryUrl
            |libraries:
            |${librariesConfigs.mkString("\n")}
            |""".stripMargin

tools/simple-library-server/main.js

@@ -1,5 +1,6 @@
 #!/usr/bin/env node
 const express = require("express");
+const crypto = require("crypto");
 const path = require("path");
 const os = require("os");
 const fs = require("fs");
@@ -63,13 +64,34 @@ if (argv.upload == "disabled") {
     console.log("WARNING: Uploads are enabled without any authentication.");
   }
 }
+
+app.get("/health", function (req, res) {
+  res.status(200).send("OK");
+});
+
 app.use(express.static(argv.root));
 
+let port = argv.port;
+if (process.env.PORT) {
+  port = process.env.PORT;
+  console.log(
+    `Overriding the port to ${port} set by the PORT environment variable.`
+  );
+}
 console.log(
-  `Serving the repository located under ${argv.root} on port ${argv.port}.`
+  `Serving the repository located under ${argv.root} on port ${port}.`
 );
 
-app.listen(argv.port);
+const server = app.listen(port);
+
+function handleShutdown() {
+  console.log("Received a signal - shutting down.");
+  server.close(() => {
+    console.log("Server terminated.");
+  });
+}
+process.on("SIGTERM", handleShutdown);
+process.on("SIGINT", handleShutdown);
 
 /// Specifies if a particular file can be compressed in transfer, if supported.
 function shouldCompress(req, res) {
@@ -121,7 +143,21 @@ async function handleUpload(req, res) {
     }
   }
 
-  const libraryPath = path.join(libraryRoot, namespace, name, version);
+  const libraryBasePath = path.join(libraryRoot, namespace, name);
+  const libraryPath = path.join(libraryBasePath, version);
+
+  /** Finds a name for a temporary directory to move the files to,
+      so that the upload can then be committed atomically by renaming
+      a single directory. */
+  function findRandomTemporaryDirectory() {
+    const randomName = crypto.randomBytes(32).toString("hex");
+    const temporaryPath = path.join(libraryBasePath, randomName);
+    if (fs.existsSync(temporaryPath)) {
+      return findRandomTemporaryDirectory();
+    }
+
+    return temporaryPath;
+  }
 
   if (fs.existsSync(libraryPath)) {
     return fail(
@@ -132,11 +168,14 @@ async function handleUpload(req, res) {
     );
   }
 
-  await fsPromises.mkdir(libraryPath, { recursive: true });
+  const temporaryPath = findRandomTemporaryDirectory();
+  await fsPromises.mkdir(libraryBasePath, { recursive: true });
+  await fsPromises.mkdir(temporaryPath, { recursive: true });
 
   console.log(`Uploading library [${namespace}.${name}:${version}].`);
   try {
-    await putFiles(libraryPath, req.files);
+    await putFiles(temporaryPath, req.files);
+    await fsPromises.rename(temporaryPath, libraryPath);
   } catch (error) {
     console.log(`Upload failed: [${error}].`);
     console.error(error.stack);
@@ -154,7 +193,7 @@ function isVersionValid(version) {
 
 /// Checks if the namespace/username is valid.
 function isNamespaceValid(namespace) {
-  return /^[a-z][a-z0-9]*$/.test(namespace) && namespace.length >= 3;
+  return /^[A-Za-z][a-z0-9]*$/.test(namespace) && namespace.length >= 3;
 }
 
 /** Checks if the library name is valid.
@@ -194,6 +233,7 @@ async function putFiles(directory, files) {
     const file = files[i];
     const filename = file.originalname;
     const destination = path.join(directory, filename);
-    await fsPromises.rename(file.path, destination);
+    await fsPromises.copyFile(file.path, destination);
+    await fsPromises.unlink(file.path);
   }
 }

tools/simple-library-server/package.json

@@ -19,5 +19,8 @@
     "multer": "^1.4.2",
     "semver": "^7.3.5",
     "yargs": "^17.0.1"
+  },
+  "engines": {
+    "node": ">=14.17.2"
   }
 }