mirror of
https://github.com/enso-org/enso.git
synced 2024-12-27 22:18:43 +03:00
Revert "Library Publishing MVP (#1898)"
This reverts commit 1bd4e5824e.
This commit is contained in:
Ara Adkins
parent
1bd4e5824e
commit
a7478bc573
2
.github/workflows/scala.yml
vendored
2
.github/workflows/scala.yml
vendored
@ -16,7 +16,7 @@ env:
|
||||
# Please ensure that this is in sync with rustVersion in build.sbt
|
||||
rustToolchain: nightly-2021-05-12
|
||||
# Some moderately recent version of Node.JS is needed to run the library download tests.
|
||||
nodeVersion: 14.17.2
|
||||
nodeVersion: 12.18.4
|
||||
|
||||
jobs:
|
||||
test_and_publish:
|
||||
|
||||
@ -12,10 +12,6 @@
|
||||
- Implemented a basic library downloader
|
||||
([#1885](https://github.com/enso-org/enso/pull/1885)), allowing the
|
||||
downloading of missing libraries.
|
||||
- Implemented a basic library uploader
|
||||
([#1898](https://github.com/enso-org/enso/pull/1898)). It implements the
|
||||
`library/publish` endpoint of the Language Server and adds a `publish-library`
|
||||
subcommand to the Launcher.
|
||||
|
||||
## Libraries
|
||||
|
||||
|
||||
48
build.sbt
48
build.sbt
@ -251,7 +251,6 @@ lazy val enso = (project in file("."))
|
||||
`distribution-manager`,
|
||||
`edition-updater`,
|
||||
`library-manager`,
|
||||
`library-manager-test`,
|
||||
syntax.jvm,
|
||||
testkit
|
||||
)
|
||||
@ -1024,7 +1023,6 @@ lazy val `language-server` = (project in file("engine/language-server"))
|
||||
.dependsOn(`version-output`)
|
||||
.dependsOn(pkg)
|
||||
.dependsOn(testkit % Test)
|
||||
.dependsOn(`library-manager-test` % Test)
|
||||
.dependsOn(`runtime-version-manager-test` % Test)
|
||||
|
||||
lazy val ast = (project in file("lib/scala/ast"))
|
||||
@ -1180,6 +1178,22 @@ lazy val runtime = (project in file("engine/runtime"))
|
||||
case _ => MergeStrategy.first
|
||||
}
|
||||
)
|
||||
.settings(
|
||||
(Compile / compile) := (Compile / compile)
|
||||
.dependsOn(
|
||||
Def.task {
|
||||
Editions.writeEditionConfig(
|
||||
ensoVersion = ensoVersion,
|
||||
editionName = currentEdition,
|
||||
libraryVersion =
|
||||
"0.1.0", // TODO [RW] Once we start releasing the standard libraries, this will be synced with engine version.
|
||||
log = streams.value.log
|
||||
)
|
||||
}
|
||||
)
|
||||
.value,
|
||||
cleanFiles += baseDirectory.value / ".." / ".." / "distribution" / "editions"
|
||||
)
|
||||
.dependsOn(pkg)
|
||||
.dependsOn(`interpreter-dsl`)
|
||||
.dependsOn(syntax.jvm)
|
||||
@ -1256,7 +1270,6 @@ lazy val `engine-runner` = project
|
||||
)
|
||||
.dependsOn(`version-output`)
|
||||
.dependsOn(pkg)
|
||||
.dependsOn(cli)
|
||||
.dependsOn(`library-manager`)
|
||||
.dependsOn(`language-server`)
|
||||
.dependsOn(`polyglot-api`)
|
||||
@ -1346,22 +1359,6 @@ lazy val editions = project
|
||||
"org.scalatest" %% "scalatest" % scalatestVersion % Test
|
||||
)
|
||||
)
|
||||
.settings(
|
||||
(Compile / compile) := (Compile / compile)
|
||||
.dependsOn(
|
||||
Def.task {
|
||||
Editions.writeEditionConfig(
|
||||
ensoVersion = ensoVersion,
|
||||
editionName = currentEdition,
|
||||
libraryVersion =
|
||||
"0.1.0", // TODO [RW] Once we start releasing the standard libraries, this will be synced with engine version.
|
||||
log = streams.value.log
|
||||
)
|
||||
}
|
||||
)
|
||||
.value,
|
||||
cleanFiles += baseDirectory.value / ".." / ".." / "distribution" / "editions"
|
||||
)
|
||||
.dependsOn(testkit % Test)
|
||||
|
||||
lazy val downloader = (project in file("lib/scala/downloader"))
|
||||
@ -1409,19 +1406,6 @@ lazy val `library-manager` = project
|
||||
.dependsOn(testkit % Test)
|
||||
.dependsOn(`logging-service` % Test)
|
||||
|
||||
lazy val `library-manager-test` = project
|
||||
.in(file("lib/scala/library-manager-test"))
|
||||
.configs(Test)
|
||||
.settings(
|
||||
libraryDependencies ++= Seq(
|
||||
"com.typesafe.scala-logging" %% "scala-logging" % scalaLoggingVersion,
|
||||
"org.scalatest" %% "scalatest" % scalatestVersion % Test
|
||||
)
|
||||
)
|
||||
.dependsOn(`library-manager`)
|
||||
.dependsOn(testkit)
|
||||
.dependsOn(`logging-service`)
|
||||
|
||||
lazy val `runtime-version-manager` = project
|
||||
.in(file("lib/scala/runtime-version-manager"))
|
||||
.configs(Test)
|
||||
|
||||
@ -200,7 +200,6 @@ transport formats, please look [here](./protocol-architecture).
|
||||
- [`LibraryDownloadError`](#librarydownloaderror)
|
||||
- [`LocalLibraryNotFound`](#locallibrarynotfound)
|
||||
- [`LibraryNotResolved`](#librarynotresolved)
|
||||
- [`InvalidLibraryName`](#invalidlibraryname)
|
||||
|
||||
<!-- /MarkdownTOC -->
|
||||
|
||||
@ -4329,8 +4328,6 @@ null;
|
||||
|
||||
#### Errors
|
||||
|
||||
- [`InvalidLibraryName`](#invalidlibraryname) to signal that the selected
|
||||
library name is not valid.
|
||||
- [`LibraryAlreadyExists`](#libraryalreadyexists) to signal that a library with
|
||||
the given namespace and name already exists.
|
||||
- [`FileSystemError`](#filesystemerror) to signal a generic, unrecoverable
|
||||
@ -4409,9 +4406,6 @@ versions. This is a temporary solution and in the longer-term it should be
|
||||
replaced with separate settings allowing to arbitrarily modify the library
|
||||
version from the IDE.
|
||||
|
||||
The `uploadUrl` is the URL of the library repository that accepts library
|
||||
uploads.
|
||||
|
||||
The metadata for publishing the library can be set with
|
||||
[`library/setMetadata`](#librarysetmetadata). If it was not set, the publish
|
||||
operation will still proceed, but that metadata will be missing.
|
||||
@ -4423,7 +4417,6 @@ operation will still proceed, but that metadata will be missing.
|
||||
namespace: String;
|
||||
name: String;
|
||||
authToken: String;
|
||||
uploadUrl: String;
|
||||
|
||||
bumpVersionAfterPublish?: Boolean;
|
||||
}
|
||||
@ -4437,8 +4430,6 @@ null;
|
||||
|
||||
#### Errors
|
||||
|
||||
- [`LocalLibraryNotFound`](#locallibrarynotfound) to signal that a local library
|
||||
with the given name does not exist on the local libraries path.
|
||||
- [`LibraryPublishError`](#librarypublisherror) to signal that the server did
|
||||
not accept to publish the library (for example because a library with the same
|
||||
version already exists).
|
||||
@ -5003,21 +4994,3 @@ there either.
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### `InvalidLibraryName`
|
||||
|
||||
Signals that the chosen library name is invalid.
|
||||
|
||||
It contains a suggestion of a similar name that is valid.
|
||||
|
||||
For example for `FooBar` it will suggest `Foo_Bar`.
|
||||
|
||||
```typescript
|
||||
"error" : {
|
||||
"code" : 8009,
|
||||
"message" : "[<name>] is not a valid name: <reason>.",
|
||||
"payload" : {
|
||||
"suggestedName" : "<fixed-name>"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
@ -234,31 +234,6 @@ them), it will result in the following merged directory structure:
|
||||
└── LICENSE.md
|
||||
```
|
||||
|
||||
### Publishing
|
||||
|
||||
To be able to publish libraries to a repository, the repository must provide an
|
||||
upload endpoint which satisfies the following requirements.
|
||||
|
||||
The endpoint should get the library name and version from the query parameters:
|
||||
`namespace`, `name` and `version`.
|
||||
|
||||
It should check any authentication data attached to the query and verify that
|
||||
the user has sufficient privileges to upload the library for that `namespace`.
|
||||
|
||||
Currently, we use a static check which checks an `Auth-Token` header for a
|
||||
pre-determined secret key, but any other authentication schemes can be used, as
|
||||
long as they are supported by the GUI or CLI.
|
||||
|
||||
Then, the server must check if a library with the given name and version
|
||||
combination already exists. If the library already exists, the request should be
|
||||
rejected with `409 Conflict` status code indicating that a conflicting library
|
||||
is already in the repository.
|
||||
|
||||
If the request goes through, the server should create a directory for the
|
||||
library and put any files attached to the request there. Each request should
|
||||
always contain `package.yaml` and `manifest.yaml` files attached and at least
|
||||
one sub-archive, usually called `main.tgz`.
|
||||
|
||||
## Editions Repository
|
||||
|
||||
The Editions repository has a very simple structure.
|
||||
|
||||
@ -62,17 +62,5 @@ import <namespace>.<Project_Name>
|
||||
|
||||
## Publishing
|
||||
|
||||
To publish a library, first you must obtain the upload URL of the repository, if
|
||||
you are hosting the repository locally it will be `http://localhost:8080/upload`
|
||||
(or possibly with a different port if that was overridden).
|
||||
|
||||
If the repository requires authentication, it is best to set it up by setting
|
||||
the `ENSO_AUTH_TOKEN` environment variable to the value of your secret token.
|
||||
|
||||
Then you can use the Enso CLI to upload the project:
|
||||
|
||||
```bash
|
||||
enso publish-library --upload-url <URL> <path to project root>
|
||||
```
|
||||
|
||||
See `enso publish-library --help` for more information.
|
||||
> Soon it will be possible to share the libraries through the Marketplace, but
|
||||
> it is still a work in progress.
|
||||
|
||||
@ -164,7 +164,6 @@ object LibraryApi {
|
||||
namespace: String,
|
||||
name: String,
|
||||
authToken: String,
|
||||
uploadUrl: String,
|
||||
bumpVersionAfterPublish: Option[Boolean]
|
||||
)
|
||||
|
||||
@ -233,16 +232,4 @@ object LibraryApi {
|
||||
} """
|
||||
)
|
||||
}
|
||||
|
||||
case class InvalidLibraryName(
|
||||
originalName: String,
|
||||
suggestedName: String,
|
||||
reason: String
|
||||
) extends Error(8009, s"[$originalName] is not a valid name: $reason.") {
|
||||
override def payload: Option[Json] = Some(
|
||||
json""" {
|
||||
"suggestedName" : $suggestedName
|
||||
} """
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@ -5,12 +5,8 @@ import com.typesafe.scalalogging.LazyLogging
|
||||
import org.enso.distribution.{DistributionManager, FileSystem}
|
||||
import org.enso.editions.{Editions, LibraryName}
|
||||
import org.enso.languageserver.libraries.LocalLibraryManagerProtocol._
|
||||
import org.enso.librarymanager.local.{
|
||||
DefaultLocalLibraryProvider,
|
||||
LocalLibraryProvider
|
||||
}
|
||||
import org.enso.librarymanager.local.LocalLibraryProvider
|
||||
import org.enso.pkg.PackageManager
|
||||
import org.enso.pkg.validation.NameValidation
|
||||
|
||||
import java.io.File
|
||||
import java.nio.file.Files
|
||||
@ -38,18 +34,9 @@ class LocalLibraryManager(
|
||||
sender() ! listLocalLibraries()
|
||||
case Create(libraryName, authors, maintainers, license) =>
|
||||
sender() ! createLibrary(libraryName, authors, maintainers, license)
|
||||
case FindLibrary(libraryName) =>
|
||||
sender() ! findLibrary(libraryName)
|
||||
}
|
||||
}
|
||||
|
||||
/** Checks if the library name is a valid Enso module name. */
|
||||
private def validateLibraryName(libraryName: LibraryName): Unit = {
|
||||
// TODO [RW] more specific exceptions
|
||||
NameValidation.validateName(libraryName.name) match {
|
||||
case Left(error) =>
|
||||
throw new RuntimeException(s"Library name is not valid: [$error].")
|
||||
case Right(_) =>
|
||||
case Publish(_, _, _) =>
|
||||
logger.error("Publishing libraries is currently not implemented.")
|
||||
sender() ! Failure(new NotImplementedError())
|
||||
}
|
||||
}
|
||||
|
||||
@ -67,8 +54,6 @@ class LocalLibraryManager(
|
||||
// TODO [RW] modify protocol to be able to create Contact instances
|
||||
val _ = (authors, maintainers)
|
||||
|
||||
validateLibraryName(libraryName)
|
||||
|
||||
// TODO [RW] make the exceptions more relevant
|
||||
val possibleRoots = LazyList
|
||||
.from(distributionManager.paths.localLibrariesSearchPaths)
|
||||
@ -119,17 +104,6 @@ class LocalLibraryManager(
|
||||
} yield LibraryName(namespace, name)
|
||||
}
|
||||
|
||||
/** Finds the path on the filesystem to a local library. */
|
||||
private def findLibrary(
|
||||
libraryName: LibraryName
|
||||
): Try[FindLibraryResponse] = Try {
|
||||
val localLibraryProvider = new DefaultLocalLibraryProvider(
|
||||
distributionManager.paths.localLibrariesSearchPaths.toList
|
||||
)
|
||||
val pathOpt = localLibraryProvider.findLibrary(libraryName)
|
||||
FindLibraryResponse(pathOpt)
|
||||
}
|
||||
|
||||
/** Finds the edition associated with the current project, if specified in its
|
||||
* config.
|
||||
*/
|
||||
|
||||
@ -2,8 +2,6 @@ package org.enso.languageserver.libraries
|
||||
|
||||
import org.enso.editions.LibraryName
|
||||
|
||||
import java.nio.file.Path
|
||||
|
||||
object LocalLibraryManagerProtocol {
|
||||
|
||||
/** A top class representing any request to the [[LocalLibraryManager]]. */
|
||||
@ -39,9 +37,10 @@ object LocalLibraryManagerProtocol {
|
||||
license: String
|
||||
) extends Request
|
||||
|
||||
/** A request to find the path to a local library. */
|
||||
case class FindLibrary(libraryName: LibraryName) extends Request
|
||||
|
||||
/** A response to [[FindLibrary]]. */
|
||||
case class FindLibraryResponse(libraryRoot: Option[Path])
|
||||
/** A request to publish a library. */
|
||||
case class Publish(
|
||||
libraryName: LibraryName,
|
||||
authToken: String,
|
||||
bumpVersionAfterPublish: Boolean
|
||||
) extends Request
|
||||
}
|
||||
|
||||
@ -1,161 +1,32 @@
|
||||
package org.enso.languageserver.libraries.handler
|
||||
|
||||
import akka.actor.{Actor, ActorRef, Cancellable, Props}
|
||||
import akka.actor.{Actor, Props}
|
||||
import com.typesafe.scalalogging.LazyLogging
|
||||
import org.enso.cli.task.notifications.ActorProgressNotificationForwarder
|
||||
import org.enso.editions.LibraryName
|
||||
import org.enso.jsonrpc.{Id, Request, ResponseError, ResponseResult, Unused}
|
||||
import org.enso.jsonrpc.{Request, ResponseError}
|
||||
import org.enso.languageserver.filemanager.FileManagerApi.FileSystemError
|
||||
import org.enso.languageserver.libraries.LibraryApi._
|
||||
import org.enso.languageserver.libraries.LocalLibraryManagerProtocol.{
|
||||
FindLibrary,
|
||||
FindLibraryResponse
|
||||
}
|
||||
import org.enso.languageserver.requesthandler.RequestTimeout
|
||||
import org.enso.languageserver.util.UnhandledLogging
|
||||
import org.enso.libraryupload.{auth, LibraryUploader}
|
||||
|
||||
import scala.concurrent.duration.FiniteDuration
|
||||
import scala.util.{Failure, Success}
|
||||
|
||||
/** A request handler for the `library/publish` endpoint.
|
||||
*
|
||||
* @param timeout request timeout
|
||||
* @param localLibraryManager a reference to the LocalLibraryManager
|
||||
* It is currently a stub implementation which will be refined later on.
|
||||
*/
|
||||
class LibraryPublishHandler(
|
||||
timeout: FiniteDuration,
|
||||
localLibraryManager: ActorRef
|
||||
) extends Actor
|
||||
class LibraryPublishHandler
|
||||
extends Actor
|
||||
with LazyLogging
|
||||
with UnhandledLogging {
|
||||
override def receive: Receive = requestStage
|
||||
|
||||
import context.dispatcher
|
||||
|
||||
private def requestStage: Receive = {
|
||||
case Request(
|
||||
LibraryPublish,
|
||||
id,
|
||||
LibraryPublish.Params(
|
||||
namespace,
|
||||
name,
|
||||
authToken,
|
||||
uploadUrl,
|
||||
bumpVersionAfterPublish
|
||||
)
|
||||
) =>
|
||||
val shouldBump = bumpVersionAfterPublish.getOrElse(false)
|
||||
val replyTo = sender()
|
||||
val token = auth.SimpleHeaderToken(authToken)
|
||||
val libraryName = LibraryName(namespace, name)
|
||||
localLibraryManager ! FindLibrary(libraryName)
|
||||
|
||||
val timeoutCancellable =
|
||||
context.system.scheduler.scheduleOnce(timeout, self, RequestTimeout)
|
||||
context.become(
|
||||
waitForLibraryResolutionStage(
|
||||
replyTo,
|
||||
libraryName,
|
||||
id,
|
||||
uploadUrl,
|
||||
token,
|
||||
timeoutCancellable,
|
||||
shouldBump
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
private def stop(timeoutCancellable: Cancellable): Unit = {
|
||||
timeoutCancellable.cancel()
|
||||
context.stop(self)
|
||||
}
|
||||
|
||||
/** Waits for the response of LocalLibraryManager and continues the publishing
|
||||
* process.
|
||||
*/
|
||||
private def waitForLibraryResolutionStage(
|
||||
replyTo: ActorRef,
|
||||
libraryName: LibraryName,
|
||||
id: Id,
|
||||
uploadUrl: String,
|
||||
token: auth.Token,
|
||||
timeoutCancellable: Cancellable,
|
||||
shouldBumpAfterPublishing: Boolean
|
||||
): Receive = {
|
||||
case RequestTimeout =>
|
||||
replyTo ! RequestTimeout
|
||||
context.stop(self)
|
||||
|
||||
case Success(FindLibraryResponse(Some(libraryRoot))) =>
|
||||
val progressReporter =
|
||||
ActorProgressNotificationForwarder.translateAndForward(
|
||||
LibraryPublish.name,
|
||||
replyTo
|
||||
)
|
||||
|
||||
val result = LibraryUploader.uploadLibrary(
|
||||
libraryRoot,
|
||||
uploadUrl,
|
||||
token,
|
||||
progressReporter
|
||||
)
|
||||
|
||||
result match {
|
||||
case Failure(exception) =>
|
||||
replyTo ! ResponseError(
|
||||
Some(id),
|
||||
FileSystemError(s"Upload failed: $exception")
|
||||
)
|
||||
case Success(_) =>
|
||||
if (shouldBumpAfterPublishing) {
|
||||
logger.warn(
|
||||
"`bumpVersionAfterPublish` was set to true, but this feature " +
|
||||
"is not currently implemented. Ignoring."
|
||||
)
|
||||
}
|
||||
replyTo ! ResponseResult(LibraryPublish, id, Unused)
|
||||
}
|
||||
|
||||
stop(timeoutCancellable)
|
||||
|
||||
case Success(FindLibraryResponse(None)) =>
|
||||
replyTo ! ResponseError(
|
||||
override def receive: Receive = {
|
||||
case Request(LibraryPublish, id, _: LibraryPublish.Params) =>
|
||||
// TODO [RW] actual implementation
|
||||
sender() ! ResponseError(
|
||||
Some(id),
|
||||
FileSystemError(
|
||||
s"The library [$libraryName] was not found in local libraries " +
|
||||
s"search paths."
|
||||
)
|
||||
FileSystemError("Feature not implemented")
|
||||
)
|
||||
|
||||
stop(timeoutCancellable)
|
||||
|
||||
case Failure(exception) =>
|
||||
replyTo ! ResponseError(
|
||||
Some(id),
|
||||
FileSystemError(
|
||||
s"Failed to find the requested local library: $exception"
|
||||
)
|
||||
)
|
||||
|
||||
stop(timeoutCancellable)
|
||||
}
|
||||
}
|
||||
|
||||
object LibraryPublishHandler {
|
||||
|
||||
/** Creates a configuration object to create [[LibraryPublishHandler]].
|
||||
*
|
||||
* @param timeout request timeout
|
||||
* @param localLibraryManager a reference to the LocalLibraryManager
|
||||
*/
|
||||
def props(
|
||||
timeout: FiniteDuration,
|
||||
localLibraryManager: ActorRef
|
||||
): Props = Props(
|
||||
new LibraryPublishHandler(
|
||||
timeout,
|
||||
localLibraryManager
|
||||
)
|
||||
)
|
||||
/** Creates a configuration object to create [[LibraryPublishHandler]]. */
|
||||
def props(): Props = Props(new LibraryPublishHandler)
|
||||
}
|
||||
|
||||
@ -505,8 +505,7 @@ class JsonConnectionController(
|
||||
.props(requestTimeout, localLibraryManager),
|
||||
LibraryGetMetadata -> LibraryGetMetadataHandler.props(),
|
||||
LibraryPreinstall -> LibraryPreinstallHandler.props(),
|
||||
LibraryPublish -> LibraryPublishHandler
|
||||
.props(requestTimeout, localLibraryManager),
|
||||
LibraryPublish -> LibraryPublishHandler.props(),
|
||||
LibrarySetMetadata -> LibrarySetMetadataHandler.props()
|
||||
)
|
||||
}
|
||||
|
||||
@ -4,9 +4,6 @@ import io.circe.literal._
|
||||
import io.circe.{Json, JsonObject}
|
||||
import org.enso.languageserver.libraries.LibraryEntry
|
||||
import org.enso.languageserver.libraries.LibraryEntry.PublishedLibraryVersion
|
||||
import org.enso.librarymanager.published.repository.EmptyRepository
|
||||
|
||||
import java.nio.file.Files
|
||||
|
||||
class LibrariesTest extends BaseServerTest {
|
||||
"LocalLibraryManager" should {
|
||||
@ -32,8 +29,8 @@ class LibrariesTest extends BaseServerTest {
|
||||
"method": "library/create",
|
||||
"id": 1,
|
||||
"params": {
|
||||
"namespace": "user",
|
||||
"name": "My_Local_Lib",
|
||||
"namespace": "User",
|
||||
"name": "MyLocalLib",
|
||||
"authors": [],
|
||||
"maintainers": [],
|
||||
"license": ""
|
||||
@ -59,8 +56,8 @@ class LibrariesTest extends BaseServerTest {
|
||||
"result": {
|
||||
"localLibraries": [
|
||||
{
|
||||
"namespace": "user",
|
||||
"name": "My_Local_Lib",
|
||||
"namespace": "User",
|
||||
"name": "MyLocalLib",
|
||||
"version": {
|
||||
"type": "LocalLibraryVersion"
|
||||
}
|
||||
@ -75,83 +72,6 @@ class LibrariesTest extends BaseServerTest {
|
||||
"existed" ignore {
|
||||
// TODO [RW] error handling (#1877)
|
||||
}
|
||||
|
||||
"validate the library name" ignore {
|
||||
// TODO [RW] error handling (#1877)
|
||||
}
|
||||
|
||||
def port: Int = 47308
|
||||
|
||||
"create and publish a library" in {
|
||||
val client = getInitialisedWsClient()
|
||||
client.send(json"""
|
||||
{ "jsonrpc": "2.0",
|
||||
"method": "library/create",
|
||||
"id": 0,
|
||||
"params": {
|
||||
"namespace": "user",
|
||||
"name": "Publishable_Lib",
|
||||
"authors": [],
|
||||
"maintainers": [],
|
||||
"license": ""
|
||||
}
|
||||
}
|
||||
""")
|
||||
client.expectJson(json"""
|
||||
{ "jsonrpc": "2.0",
|
||||
"id": 0,
|
||||
"result": null
|
||||
}
|
||||
""")
|
||||
|
||||
val repoRoot = getTestDirectory.resolve("libraries_repo_root")
|
||||
val server = EmptyRepository.startServer(port, repoRoot, uploads = true)
|
||||
try {
|
||||
val uploadUrl = s"http://localhost:$port/upload"
|
||||
client.send(json"""
|
||||
{ "jsonrpc": "2.0",
|
||||
"method": "library/publish",
|
||||
"id": 1,
|
||||
"params": {
|
||||
"namespace": "user",
|
||||
"name": "Publishable_Lib",
|
||||
"authToken": "SOME TOKEN",
|
||||
"uploadUrl": $uploadUrl,
|
||||
"bumpVersionAfterPublish": null
|
||||
}
|
||||
}
|
||||
""")
|
||||
|
||||
var found = false
|
||||
while (!found) {
|
||||
val rawResponse = client.expectSomeJson()
|
||||
val response = rawResponse.asObject.value
|
||||
val idMatches =
|
||||
response("id").flatMap(_.asNumber).flatMap(_.toInt).contains(1)
|
||||
if (idMatches) {
|
||||
rawResponse shouldEqual json"""
|
||||
{ "jsonrpc": "2.0",
|
||||
"id": 1,
|
||||
"result": null
|
||||
}
|
||||
"""
|
||||
|
||||
found = true
|
||||
}
|
||||
}
|
||||
|
||||
val libraryRoot = repoRoot
|
||||
.resolve("libraries")
|
||||
.resolve("user")
|
||||
.resolve("Publishable_Lib")
|
||||
.resolve("0.0.1")
|
||||
val mainPackage = libraryRoot.resolve("main.tgz")
|
||||
assert(Files.exists(mainPackage))
|
||||
} finally {
|
||||
server.kill(killDescendants = true)
|
||||
server.join(waitForDescendants = true)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
"mocked library/preinstall" should {
|
||||
|
||||
@ -1,14 +0,0 @@
|
||||
package org.enso.launcher
|
||||
|
||||
import nl.gn0s1s.bump.SemVer
|
||||
|
||||
object Constants {
|
||||
|
||||
/** The engine version in which the uploads command has been introduced.
|
||||
*
|
||||
* It is used to check by the launcher if the engine can handle this command
|
||||
* and provide better error messages if it cannot.
|
||||
*/
|
||||
val uploadIntroducedVersion: SemVer =
|
||||
SemVer(0, 2, 17, Some("SNAPSHOT"))
|
||||
}
|
||||
@ -341,52 +341,6 @@ case class Launcher(cliOptions: GlobalCLIOptions) {
|
||||
0
|
||||
}
|
||||
|
||||
/** Uploads a library to a repository.
|
||||
*
|
||||
* @param path path to the library, if not specified, the current working
|
||||
* directory and its ancestors are searched for an Enso project
|
||||
* to upload
|
||||
* @param uploadUrl a URL of an upload endpoint of a repository; if not
|
||||
* specified, falls back to the default Enso repository
|
||||
* @param authToken a token to use for authentication
|
||||
* @param logLevel log level for the language server
|
||||
* @param useSystemJVM if set, forces to use the default configured JVM,
|
||||
* instead of the JVM associated with the engine version
|
||||
* @param jvmOpts additional options to pass to the launched JVM
|
||||
* @param additionalArguments additional arguments to pass to the runner
|
||||
* @return exit code of the launched program
|
||||
*/
|
||||
def uploadLibrary(
|
||||
path: Option[Path],
|
||||
uploadUrl: Option[String],
|
||||
authToken: Option[String],
|
||||
logLevel: LogLevel,
|
||||
useSystemJVM: Boolean,
|
||||
jvmOpts: Seq[(String, String)],
|
||||
additionalArguments: Seq[String]
|
||||
): Int = {
|
||||
val settings = runner
|
||||
.uploadLibrary(
|
||||
path,
|
||||
uploadUrl.getOrElse {
|
||||
throw new IllegalArgumentException(
|
||||
"The default repository is currently not defined. " +
|
||||
"You need to explicitly specify the `--upload-url`."
|
||||
)
|
||||
},
|
||||
authToken.orElse(LauncherEnvironment.getEnvVar("ENSO_AUTH_TOKEN")),
|
||||
cliOptions.hideProgress,
|
||||
logLevel,
|
||||
cliOptions.internalOptions.logMasking,
|
||||
additionalArguments
|
||||
)
|
||||
.get
|
||||
|
||||
runner.withCommand(settings, JVMSettings(useSystemJVM, jvmOpts)) {
|
||||
command => command.run().get
|
||||
}
|
||||
}
|
||||
|
||||
/** Prints the value of `key` from the global configuration.
|
||||
*
|
||||
* If the `key` is not set in the config, sets exit code to 1 and prints a
|
||||
|
||||
@ -7,14 +7,14 @@ import org.enso.launcher.InfoLogger
|
||||
/** A [[ProgressReporter]] that displays a progress bar in the console or waits
|
||||
* for the task silently, depending on CLI options.
|
||||
*/
|
||||
class CLIProgressReporter(hideProgress: Boolean) extends ProgressReporter {
|
||||
class CLIProgressReporter(cliOptions: GlobalCLIOptions)
|
||||
extends ProgressReporter {
|
||||
|
||||
/** @inheritdoc */
|
||||
override def trackProgress(message: String, task: TaskProgress[_]): Unit = {
|
||||
InfoLogger.info(message)
|
||||
if (!hideProgress) {
|
||||
ProgressBar.waitWithProgress(task)
|
||||
}
|
||||
if (cliOptions.hideProgress) ()
|
||||
else ProgressBar.waitWithProgress(task)
|
||||
}
|
||||
}
|
||||
|
||||
@ -22,5 +22,5 @@ object CLIProgressReporter {
|
||||
|
||||
/** A helper method to create [[CLIProgressReporter]] instances. */
|
||||
def apply(globalCLIOptions: GlobalCLIOptions): CLIProgressReporter =
|
||||
new CLIProgressReporter(globalCLIOptions.hideProgress)
|
||||
new CLIProgressReporter(globalCLIOptions)
|
||||
}
|
||||
|
||||
@ -18,7 +18,7 @@ import org.enso.runtimeversionmanager.components.{
|
||||
class CLIRuntimeVersionManagementUserInterface(
|
||||
cliOptions: GlobalCLIOptions,
|
||||
alwaysInstallMissing: Boolean
|
||||
) extends CLIProgressReporter(hideProgress = cliOptions.hideProgress)
|
||||
) extends CLIProgressReporter(cliOptions)
|
||||
with RuntimeVersionManagementUserInterface {
|
||||
|
||||
private val logger = Logger[CLIRuntimeVersionManagementUserInterface]
|
||||
|
||||
@ -312,64 +312,6 @@ object LauncherApplication {
|
||||
}
|
||||
}
|
||||
|
||||
private def uploadLibraryCommand: Command[Config => Int] =
|
||||
Command(
|
||||
"publish-library",
|
||||
"Publish an Enso library to a repository. " +
|
||||
"If `auto-confirm` is set, this will install missing engines or " +
|
||||
"runtimes without asking."
|
||||
) {
|
||||
val pathOpt = Opts.optionalArgument[Path](
|
||||
"PATH",
|
||||
"If PATH is provided, the project at this path or the closest one of " +
|
||||
"its ancestors that contains an Enso project, is uploaded. If not, " +
|
||||
"the project to upload is searched for in the current directory and " +
|
||||
"its ancestors."
|
||||
)
|
||||
val uploadUrlOpt = Opts.optionalParameter[String](
|
||||
"upload-url",
|
||||
"URL",
|
||||
"Upload URL of the repository to upload the library to.",
|
||||
showInUsage = true
|
||||
)
|
||||
val authTokenOpt = Opts.optionalParameter[String](
|
||||
"auth-token",
|
||||
"TOKEN",
|
||||
"An optional token to add to request headers for use in " +
|
||||
"authorization. If this parameter is not set, the ENSO_AUTH_TOKEN " +
|
||||
"environment variable is checked."
|
||||
)
|
||||
val additionalArgs = Opts.additionalArguments()
|
||||
(
|
||||
pathOpt,
|
||||
uploadUrlOpt,
|
||||
authTokenOpt,
|
||||
engineLogLevel,
|
||||
systemJVMOverride,
|
||||
jvmOpts,
|
||||
additionalArgs
|
||||
) mapN {
|
||||
(
|
||||
path,
|
||||
uploadUrl,
|
||||
authToken,
|
||||
engineLogLevel,
|
||||
systemJVMOverride,
|
||||
jvmOpts,
|
||||
additionalArgs
|
||||
) => (config: Config) =>
|
||||
Launcher(config).uploadLibrary(
|
||||
path = path,
|
||||
uploadUrl = uploadUrl,
|
||||
authToken = authToken,
|
||||
logLevel = engineLogLevel,
|
||||
useSystemJVM = systemJVMOverride,
|
||||
jvmOpts = jvmOpts,
|
||||
additionalArguments = additionalArgs
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private def installEngineCommand: Command[Config => Int] =
|
||||
Command(
|
||||
"engine",
|
||||
@ -529,7 +471,7 @@ object LauncherApplication {
|
||||
|
||||
private def topLevelOpts: Opts[() => TopLevelBehavior[Config]] = {
|
||||
val version =
|
||||
Opts.flag("version", 'V', "Display version.", showInUsage = false)
|
||||
Opts.flag("version", 'V', "Display version.", showInUsage = true)
|
||||
val json = Opts.flag(
|
||||
GlobalCLIOptions.USE_JSON,
|
||||
"Use JSON instead of plain text for version output.",
|
||||
@ -662,7 +604,6 @@ object LauncherApplication {
|
||||
replCommand,
|
||||
runCommand,
|
||||
languageServerCommand,
|
||||
uploadLibraryCommand,
|
||||
defaultCommand,
|
||||
installCommand,
|
||||
uninstallCommand,
|
||||
|
||||
@ -3,9 +3,7 @@ package org.enso.launcher.components
|
||||
import akka.http.scaladsl.model.Uri
|
||||
import nl.gn0s1s.bump.SemVer
|
||||
import org.enso.distribution.{DistributionManager, EditionManager, Environment}
|
||||
import org.enso.launcher.Constants
|
||||
import org.enso.launcher.project.ProjectManager
|
||||
import org.enso.logger.masking.MaskedPath
|
||||
import org.enso.loggingservice.LogLevel
|
||||
import org.enso.runtimeversionmanager.components.RuntimeVersionManager
|
||||
import org.enso.runtimeversionmanager.config.GlobalConfigurationManager
|
||||
@ -193,51 +191,4 @@ class LauncherRunner(
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/** Creates [[RunSettings]] for uploading a library.
|
||||
*
|
||||
* See [[org.enso.launcher.Launcher.uploadLibrary]] for more details.
|
||||
*/
|
||||
def uploadLibrary(
|
||||
path: Option[Path],
|
||||
uploadUrl: String,
|
||||
token: Option[String],
|
||||
hideProgress: Boolean,
|
||||
logLevel: LogLevel,
|
||||
logMasking: Boolean,
|
||||
additionalArguments: Seq[String]
|
||||
): Try[RunSettings] =
|
||||
Try {
|
||||
val actualPath = path.getOrElse(currentWorkingDirectory)
|
||||
val project = projectManager.findProject(actualPath).get.getOrElse {
|
||||
throw RunnerError(
|
||||
s"Could not find a project at " +
|
||||
s"${MaskedPath(actualPath).applyMasking()} or any of its parent " +
|
||||
s"directories."
|
||||
)
|
||||
}
|
||||
|
||||
val version = resolveVersion(None, Some(project))
|
||||
if (version < Constants.uploadIntroducedVersion) {
|
||||
throw RunnerError(
|
||||
s"Library Upload feature is not available in Enso $version. " +
|
||||
s"Please upgrade your project to a newer version."
|
||||
)
|
||||
}
|
||||
|
||||
val tokenOpts = token.map(Seq("--auth-token", _)).toSeq.flatten
|
||||
val hideProgressOpts =
|
||||
if (hideProgress) Seq("--hide-progress") else Seq.empty
|
||||
|
||||
val arguments =
|
||||
Seq("--upload", uploadUrl) ++
|
||||
Seq("--in-project", project.path.toAbsolutePath.normalize.toString) ++
|
||||
tokenOpts ++ hideProgressOpts
|
||||
RunSettings(
|
||||
version,
|
||||
arguments ++ setLogLevelArgs(logLevel, logMasking)
|
||||
++ additionalArguments,
|
||||
connectLoggerIfAvailable = true
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,27 +0,0 @@
|
||||
package org.enso.launcher.components
|
||||
|
||||
import nl.gn0s1s.bump.SemVer
|
||||
import org.enso.launcher.Constants
|
||||
import org.scalatest.matchers.should.Matchers
|
||||
import org.scalatest.wordspec.AnyWordSpec
|
||||
|
||||
class UploadVersionCheck extends AnyWordSpec with Matchers {
|
||||
"Constants.uploadIntroducedVersion" should {
|
||||
"correctly compare with nearby versions" in {
|
||||
assert(
|
||||
SemVer("0.2.17-SNAPSHOT.2021-07-23").get >=
|
||||
Constants.uploadIntroducedVersion
|
||||
)
|
||||
|
||||
assert(
|
||||
SemVer("0.2.17").get >
|
||||
Constants.uploadIntroducedVersion
|
||||
)
|
||||
|
||||
assert(
|
||||
SemVer("0.2.16").get <
|
||||
Constants.uploadIntroducedVersion
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -14,7 +14,6 @@ import org.enso.version.VersionDescription
|
||||
import org.graalvm.polyglot.PolyglotException
|
||||
|
||||
import java.io.File
|
||||
import java.nio.file.Path
|
||||
import java.util.UUID
|
||||
import scala.Console.err
|
||||
import scala.jdk.CollectionConverters._
|
||||
@ -44,9 +43,6 @@ object Main {
|
||||
private val LOG_LEVEL = "log-level"
|
||||
private val LOGGER_CONNECT = "logger-connect"
|
||||
private val NO_LOG_MASKING = "no-log-masking"
|
||||
private val UPLOAD_OPTION = "upload"
|
||||
private val HIDE_PROGRESS = "hide-progress"
|
||||
private val AUTH_TOKEN = "auth-token"
|
||||
|
||||
private lazy val logger = Logger[Main.type]
|
||||
|
||||
@ -197,27 +193,6 @@ object Main {
|
||||
"variable."
|
||||
)
|
||||
.build()
|
||||
val uploadOption = CliOption.builder
|
||||
.hasArg(true)
|
||||
.numberOfArgs(1)
|
||||
.argName("url")
|
||||
.longOpt(UPLOAD_OPTION)
|
||||
.desc(
|
||||
"Uploads the library to a repository. " +
|
||||
"The url defines the repository to upload to."
|
||||
)
|
||||
.build()
|
||||
val hideProgressOption = CliOption.builder
|
||||
.longOpt(HIDE_PROGRESS)
|
||||
.desc("If specified, progress bars will not be displayed.")
|
||||
.build()
|
||||
val authTokenOption = CliOption.builder
|
||||
.hasArg(true)
|
||||
.numberOfArgs(1)
|
||||
.argName("token")
|
||||
.longOpt(AUTH_TOKEN)
|
||||
.desc("Authentication token for the upload.")
|
||||
.build()
|
||||
|
||||
val options = new Options
|
||||
options
|
||||
@ -242,9 +217,6 @@ object Main {
|
||||
.addOption(logLevelOption)
|
||||
.addOption(loggerConnectOption)
|
||||
.addOption(noLogMaskingOption)
|
||||
.addOption(uploadOption)
|
||||
.addOption(hideProgressOption)
|
||||
.addOption(authTokenOption)
|
||||
|
||||
options
|
||||
}
|
||||
@ -664,27 +636,6 @@ object Main {
|
||||
)
|
||||
}
|
||||
|
||||
if (line.hasOption(UPLOAD_OPTION)) {
|
||||
val projectRoot =
|
||||
Option(line.getOptionValue(IN_PROJECT_OPTION))
|
||||
.map(Path.of(_))
|
||||
.getOrElse {
|
||||
logger.error(
|
||||
s"When uploading, the $IN_PROJECT_OPTION is mandatory " +
|
||||
s"to specify which project to upload."
|
||||
)
|
||||
exitFail()
|
||||
}
|
||||
|
||||
ProjectUploader.uploadProject(
|
||||
projectRoot = projectRoot,
|
||||
uploadUrl = line.getOptionValue(UPLOAD_OPTION),
|
||||
authToken = Option(line.getOptionValue(AUTH_TOKEN)),
|
||||
showProgress = !line.hasOption(HIDE_PROGRESS)
|
||||
)
|
||||
exitSuccess()
|
||||
}
|
||||
|
||||
if (line.hasOption(RUN_OPTION)) {
|
||||
run(
|
||||
line.getOptionValue(RUN_OPTION),
|
||||
|
||||
@ -1,56 +0,0 @@
|
||||
package org.enso.runner
|
||||
|
||||
import com.typesafe.scalalogging.Logger
|
||||
import org.enso.cli.ProgressBar
|
||||
import org.enso.cli.task.{ProgressReporter, TaskProgress}
|
||||
import org.enso.libraryupload.{auth, LibraryUploader}
|
||||
|
||||
import java.nio.file.Path
|
||||
|
||||
/** Gathers helper functions for uploading a library project. */
|
||||
object ProjectUploader {
|
||||
|
||||
private lazy val logger = Logger[ProjectUploader.type]
|
||||
|
||||
/** Uploads a project to a library repository.
|
||||
*
|
||||
* @param projectRoot path to the root of the project
|
||||
* @param uploadUrl URL of upload endpoint of the repository to upload to
|
||||
* @param authToken an optional token used for authentication in the
|
||||
* repository
|
||||
* @param showProgress specifies if CLI progress bars should be displayed
|
||||
* showing progress of compression and upload
|
||||
*/
|
||||
def uploadProject(
|
||||
projectRoot: Path,
|
||||
uploadUrl: String,
|
||||
authToken: Option[String],
|
||||
showProgress: Boolean
|
||||
): Unit = {
|
||||
import scala.concurrent.ExecutionContext.Implicits.global
|
||||
val progressReporter = new ProgressReporter {
|
||||
override def trackProgress(
|
||||
message: String,
|
||||
task: TaskProgress[_]
|
||||
): Unit = {
|
||||
logger.info(message)
|
||||
if (showProgress) {
|
||||
ProgressBar.waitWithProgress(task)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
val token = authToken match {
|
||||
case Some(value) => auth.SimpleHeaderToken(value)
|
||||
case None => auth.NoAuthorization
|
||||
}
|
||||
LibraryUploader
|
||||
.uploadLibrary(
|
||||
projectRoot,
|
||||
uploadUrl,
|
||||
token,
|
||||
progressReporter
|
||||
)
|
||||
.get
|
||||
}
|
||||
}
|
||||
@ -51,7 +51,6 @@ object NotificationHandler {
|
||||
/** @inheritdoc */
|
||||
override def trackProgress(message: String, task: TaskProgress[_]): Unit = {
|
||||
logger.info(message)
|
||||
// TODO [RW] check the hideProgress flag provided by the launcher
|
||||
if (System.console() != null) {
|
||||
ProgressBar.waitWithProgress(task)
|
||||
}
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
package org.enso.cli.task
|
||||
|
||||
import java.util.concurrent.LinkedTransferQueue
|
||||
import scala.concurrent.{ExecutionContext, Future}
|
||||
|
||||
import scala.util.{Failure, Try}
|
||||
|
||||
/** Represents a long-running background task.
|
||||
@ -98,21 +98,6 @@ object TaskProgress {
|
||||
}
|
||||
}
|
||||
|
||||
/** Creates a [[TaskProgress]] from a [[Future]]. */
|
||||
def fromFuture[A](
|
||||
future: Future[A]
|
||||
)(implicit ec: ExecutionContext): TaskProgress[A] = {
|
||||
new TaskProgress[A] {
|
||||
override def addProgressListener(
|
||||
listener: ProgressListener[A]
|
||||
): Unit = {
|
||||
future.onComplete { result =>
|
||||
listener.done(result)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Blocks and waits for the task to complete.
|
||||
*/
|
||||
def waitForTask[A](task: TaskProgress[A]): Try[A] = {
|
||||
|
||||
@ -1,126 +0,0 @@
|
||||
package org.enso.downloader.archive
|
||||
|
||||
import org.apache.commons.compress.archivers.tar.{
|
||||
TarArchiveEntry,
|
||||
TarArchiveOutputStream
|
||||
}
|
||||
import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream
|
||||
import org.enso.cli.task.{TaskProgress, TaskProgressImplementation}
|
||||
|
||||
import java.io.{BufferedOutputStream, FileOutputStream}
|
||||
import java.nio.charset.StandardCharsets
|
||||
import java.nio.file.{Files, Path}
|
||||
import scala.util.{Try, Using}
|
||||
|
||||
/** A helper class for writing TAR archives compressed with gzip. */
|
||||
class TarGzWriter private (archive: TarArchiveOutputStream) {
|
||||
|
||||
/** Adds a text file to the archive.
|
||||
*
|
||||
* @param relativePath path of the file in the archive
|
||||
* @param content the text content to put in the file
|
||||
*/
|
||||
def writeTextFile(relativePath: String, content: String): Unit = {
|
||||
val bytes = content.getBytes(StandardCharsets.UTF_8)
|
||||
val entry = new TarArchiveEntry(relativePath)
|
||||
entry.setSize(bytes.size.toLong)
|
||||
archive.putArchiveEntry(entry)
|
||||
archive.write(bytes)
|
||||
archive.closeArchiveEntry()
|
||||
}
|
||||
|
||||
/** Adds a file from the filesystem to the archive.
|
||||
*
|
||||
* @param relativePath path of the file in the archive
|
||||
* @param filePath a path to a file on the filesystem that will be read and
|
||||
* put into the archive
|
||||
* @return returns the number of bytes that were transferrred from the input
|
||||
* file
|
||||
*/
|
||||
def writeFile(relativePath: String, filePath: Path): Long = {
|
||||
val entry = new TarArchiveEntry(filePath.toFile, relativePath)
|
||||
archive.putArchiveEntry(entry)
|
||||
val bytesTransferred = Files.copy(filePath, archive)
|
||||
archive.closeArchiveEntry()
|
||||
bytesTransferred
|
||||
}
|
||||
}
|
||||
|
||||
object TarGzWriter {
|
||||
|
||||
/** Creates a .tar.gz archive at the specified destination.
|
||||
*
|
||||
* It calls the `actions` callback with a [[TarGzWriter]] instance which can
|
||||
* be used to add files to the archive. The [[TarGzWriter]] instance is only
|
||||
* valid during the call of this callback, it should not be leaked anywhere
|
||||
* else as it will then be invalid.
|
||||
*/
|
||||
def createArchive(
|
||||
destination: Path
|
||||
)(actions: TarGzWriter => Unit): Try[Unit] =
|
||||
Using(new FileOutputStream(destination.toFile)) { outputStream =>
|
||||
Using(new BufferedOutputStream(outputStream)) { bufferedStream =>
|
||||
Using(new GzipCompressorOutputStream(bufferedStream)) { gzipStream =>
|
||||
Using(new TarArchiveOutputStream(gzipStream)) { archive =>
|
||||
val writer = new TarGzWriter(archive)
|
||||
actions(writer)
|
||||
}.get
|
||||
}.get
|
||||
}.get
|
||||
}
|
||||
|
||||
/** Creates a .tar.gz archive from a list of files.
|
||||
*
|
||||
* @param archiveDestination path specifying where to put the archive
|
||||
* @param files list of paths to files that should be compressed; these
|
||||
* should be regular files, the behaviour is undefined if one of
|
||||
* these paths is a directory
|
||||
* @param basePath the base path to compute the relative paths of the
|
||||
* compressed files; all `files` should be inside of the
|
||||
* directory denoted by `basePath`
|
||||
*/
|
||||
def compress(
|
||||
archiveDestination: Path,
|
||||
files: Seq[Path],
|
||||
basePath: Path
|
||||
): TaskProgress[Unit] = {
|
||||
val normalizedBase = basePath.toAbsolutePath.normalize
|
||||
def relativePath(file: Path): String = {
|
||||
val normalized = file.toAbsolutePath.normalize
|
||||
if (!normalized.startsWith(normalizedBase)) {
|
||||
throw new IllegalArgumentException(
|
||||
"TarGzWriter precondition failure: " +
|
||||
"Files should all be inside of the provided basePath."
|
||||
)
|
||||
}
|
||||
normalizedBase.relativize(normalized).toString
|
||||
}
|
||||
|
||||
val taskProgress = new TaskProgressImplementation[Unit]()
|
||||
|
||||
def runCompresion(): Unit = {
|
||||
val sumSize = files.map(Files.size).sum
|
||||
|
||||
val result = TarGzWriter.createArchive(archiveDestination) { writer =>
|
||||
var totalBytesWritten: Long = 0
|
||||
def update(): Unit =
|
||||
taskProgress.reportProgress(totalBytesWritten, Some(sumSize))
|
||||
update()
|
||||
for (file <- files) {
|
||||
// TODO [RW] Ideally we could report progress for each chunk, offering
|
||||
// more granular feedback for big data files.
|
||||
val bytesWritten = writer.writeFile(relativePath(file), file)
|
||||
totalBytesWritten += bytesWritten
|
||||
update()
|
||||
}
|
||||
}
|
||||
|
||||
taskProgress.setComplete(result)
|
||||
}
|
||||
|
||||
val thread = new Thread(() => runCompresion(), "Writing-Archive")
|
||||
thread.start()
|
||||
|
||||
taskProgress
|
||||
}
|
||||
}
|
||||
@ -157,7 +157,6 @@ object HTTPDownload {
|
||||
"akka.library-extensions",
|
||||
ConfigValueFactory.fromAnyRef(Seq.empty.asJava)
|
||||
)
|
||||
.withValue("akka.daemonic", ConfigValueFactory.fromAnyRef("on"))
|
||||
.withValue("akka.loggers", ConfigValueFactory.fromAnyRef(loggers))
|
||||
.withValue(
|
||||
"akka.logging-filter",
|
||||
|
||||
@ -11,16 +11,13 @@ import org.enso.downloader.http
|
||||
*/
|
||||
case class HTTPRequestBuilder private (
|
||||
uri: Uri,
|
||||
headers: Vector[(String, String)],
|
||||
httpEntity: RequestEntity
|
||||
headers: Vector[(String, String)]
|
||||
) {
|
||||
|
||||
/** Builds a GET request with the specified settings. */
|
||||
/** Builds a GET request with the specified settings.
|
||||
*/
|
||||
def GET: HTTPRequest = build(HttpMethods.GET)
|
||||
|
||||
/** Builds a POST request with the specified settings. */
|
||||
def POST: HTTPRequest = build(HttpMethods.POST)
|
||||
|
||||
/** Adds an additional header that will be included in the request.
|
||||
*
|
||||
* @param name name of the header
|
||||
@ -29,14 +26,6 @@ case class HTTPRequestBuilder private (
|
||||
def addHeader(name: String, value: String): HTTPRequestBuilder =
|
||||
copy(headers = headers.appended((name, value)))
|
||||
|
||||
/** Sets the [[RequestEntity]] for the request.
|
||||
*
|
||||
* It can be used for example to specify form data to send for a POST
|
||||
* request.
|
||||
*/
|
||||
def setEntity(entity: RequestEntity): HTTPRequestBuilder =
|
||||
copy(httpEntity = entity)
|
||||
|
||||
private def build(
|
||||
method: HttpMethod
|
||||
): HTTPRequest = {
|
||||
@ -52,12 +41,7 @@ case class HTTPRequestBuilder private (
|
||||
}
|
||||
}
|
||||
http.HTTPRequest(
|
||||
HttpRequest(
|
||||
method = method,
|
||||
uri = uri,
|
||||
headers = httpHeaders,
|
||||
entity = httpEntity
|
||||
)
|
||||
HttpRequest(method = method, uri = uri, headers = httpHeaders)
|
||||
)
|
||||
}
|
||||
}
|
||||
@ -67,7 +51,7 @@ object HTTPRequestBuilder {
|
||||
/** Creates a request builder that will send the request for the given URI.
|
||||
*/
|
||||
def fromURI(uri: Uri): HTTPRequestBuilder =
|
||||
new HTTPRequestBuilder(uri, Vector.empty, HttpEntity.Empty)
|
||||
new HTTPRequestBuilder(uri, Vector.empty)
|
||||
|
||||
/** Tries to parse the URI provided as a [[String]] and returns a request
|
||||
* builder that will send the request to the given `uri`.
|
||||
|
||||
@ -1,11 +1,6 @@
|
||||
package org.enso.yaml
|
||||
|
||||
import io.circe.yaml.Printer
|
||||
import io.circe.{yaml, Decoder, Encoder}
|
||||
|
||||
import java.io.FileReader
|
||||
import java.nio.file.Path
|
||||
import scala.util.{Try, Using}
|
||||
import io.circe.{yaml, Decoder}
|
||||
|
||||
/** A helper for parsing YAML configs. */
|
||||
object YamlHelper {
|
||||
@ -19,17 +14,4 @@ object YamlHelper {
|
||||
.flatMap(_.as[R])
|
||||
.left
|
||||
.map(ParseError(_))
|
||||
|
||||
/** Tries to load and parse a YAML file at the provided path. */
|
||||
def load[R](path: Path)(implicit decoder: Decoder[R]): Try[R] =
|
||||
Using(new FileReader(path.toFile)) { reader =>
|
||||
yaml.parser
|
||||
.parse(reader)
|
||||
.flatMap(_.as[R])
|
||||
.toTry
|
||||
}.flatten
|
||||
|
||||
/** Saves a YAML representation of an object into a string. */
|
||||
def toYaml[A](obj: A)(implicit encoder: Encoder[A]): String =
|
||||
Printer.spaces2.copy(preserveOrder = true).pretty(encoder(obj))
|
||||
}
|
||||
|
||||
@ -1,49 +0,0 @@
|
||||
package org.enso.librarymanager.published.repository
|
||||
|
||||
import org.enso.cli.task.{ProgressReporter, TaskProgress}
|
||||
import org.enso.distribution.TemporaryDirectoryManager
|
||||
import org.enso.distribution.locking.{
|
||||
LockUserInterface,
|
||||
Resource,
|
||||
ResourceManager,
|
||||
ThreadSafeFileLockManager
|
||||
}
|
||||
import org.enso.librarymanager.published.cache.DownloadingLibraryCache
|
||||
import org.enso.testkit.HasTestDirectory
|
||||
|
||||
trait DownloaderTest { self: HasTestDirectory =>
|
||||
def withDownloader[R](action: DownloadingLibraryCache => R): R = {
|
||||
val lockManager =
|
||||
new ThreadSafeFileLockManager(getTestDirectory.resolve("locks"))
|
||||
val resourceManager = new ResourceManager(lockManager)
|
||||
try {
|
||||
val cache = new DownloadingLibraryCache(
|
||||
cacheRoot = getTestDirectory.resolve("cache"),
|
||||
temporaryDirectoryManager = new TemporaryDirectoryManager(
|
||||
getTestDirectory.resolve("tmp"),
|
||||
resourceManager
|
||||
),
|
||||
resourceManager = resourceManager,
|
||||
lockUserInterface = new LockUserInterface {
|
||||
override def startWaitingForResource(resource: Resource): Unit =
|
||||
println(s"Waiting for ${resource.name}")
|
||||
|
||||
override def finishWaitingForResource(resource: Resource): Unit =
|
||||
println(s"${resource.name} is ready")
|
||||
},
|
||||
progressReporter = new ProgressReporter {
|
||||
override def trackProgress(
|
||||
message: String,
|
||||
task: TaskProgress[_]
|
||||
): Unit = {}
|
||||
}
|
||||
)
|
||||
|
||||
action(cache)
|
||||
} finally {
|
||||
resourceManager.releaseMainLock()
|
||||
resourceManager.unlockTemporaryDirectory()
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,5 +0,0 @@
|
||||
package org.enso.librarymanager.published.repository
|
||||
|
||||
object EmptyRepository extends DummyRepository {
|
||||
override def libraries: Seq[EmptyRepository.DummyLibrary] = Seq.empty
|
||||
}
|
||||
@ -1,92 +0,0 @@
|
||||
package org.enso.libraryupload
|
||||
|
||||
import nl.gn0s1s.bump.SemVer
|
||||
import org.enso.cli.task.{ProgressReporter, TaskProgress}
|
||||
import org.enso.editions.{Editions, LibraryName}
|
||||
import org.enso.librarymanager.published.repository.{
|
||||
DownloaderTest,
|
||||
EmptyRepository
|
||||
}
|
||||
import org.enso.libraryupload.auth.SimpleHeaderToken
|
||||
import org.enso.pkg.PackageManager
|
||||
import org.enso.testkit.WithTemporaryDirectory
|
||||
import org.scalatest.matchers.should.Matchers
|
||||
import org.scalatest.wordspec.AnyWordSpec
|
||||
|
||||
import java.nio.file.Files
|
||||
|
||||
class LibraryUploadTest
|
||||
extends AnyWordSpec
|
||||
with Matchers
|
||||
with WithTemporaryDirectory
|
||||
with DownloaderTest {
|
||||
|
||||
def port: Int = 47305
|
||||
|
||||
"LibraryUploader" should {
|
||||
"upload the files to the server" in {
|
||||
val projectRoot = getTestDirectory.resolve("lib_root")
|
||||
val repoRoot = getTestDirectory.resolve("repo")
|
||||
|
||||
val libraryName = LibraryName("tester", "Upload_Test")
|
||||
val libraryVersion = SemVer(1, 2, 3)
|
||||
PackageManager.Default.create(
|
||||
projectRoot.toFile,
|
||||
name = libraryName.name,
|
||||
namespace = libraryName.namespace,
|
||||
version = libraryVersion.toString
|
||||
)
|
||||
|
||||
val server = EmptyRepository.startServer(port, repoRoot, uploads = true)
|
||||
try {
|
||||
val uploadUrl = s"http://localhost:$port/upload"
|
||||
val token = SimpleHeaderToken("TODO")
|
||||
import scala.concurrent.ExecutionContext.Implicits.global
|
||||
LibraryUploader
|
||||
.uploadLibrary(
|
||||
projectRoot,
|
||||
uploadUrl,
|
||||
token,
|
||||
new ProgressReporter {
|
||||
override def trackProgress(message: String, task: TaskProgress[_])
|
||||
: Unit = ()
|
||||
}
|
||||
)
|
||||
.get
|
||||
|
||||
val libRoot = repoRoot
|
||||
.resolve("libraries")
|
||||
.resolve("tester")
|
||||
.resolve("Upload_Test")
|
||||
.resolve("1.2.3")
|
||||
|
||||
PackageManager.Default
|
||||
.loadPackage(libRoot.toFile)
|
||||
.get
|
||||
.name shouldEqual libraryName.name
|
||||
assert(Files.exists(libRoot.resolve("manifest.yaml")))
|
||||
assert(Files.exists(libRoot.resolve("main.tgz")))
|
||||
|
||||
withDownloader { cache =>
|
||||
cache.findCachedLibrary(libraryName, libraryVersion) shouldBe empty
|
||||
|
||||
val repo = Editions.Repository(
|
||||
"test_repo",
|
||||
s"http://localhost:$port/libraries"
|
||||
)
|
||||
val installedRoot =
|
||||
cache.findOrInstallLibrary(libraryName, libraryVersion, repo).get
|
||||
val pkg = PackageManager.Default.loadPackage(installedRoot.toFile).get
|
||||
pkg.name shouldEqual libraryName.name
|
||||
val sources = pkg.listSources
|
||||
sources should have size 1
|
||||
sources.head.file.getName shouldEqual "Main.enso"
|
||||
}
|
||||
|
||||
} finally {
|
||||
server.kill(killDescendants = true)
|
||||
server.join(waitForDescendants = true)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,7 +1,6 @@
|
||||
package org.enso.librarymanager.published.repository
|
||||
|
||||
import io.circe.syntax.EncoderOps
|
||||
import io.circe.{Decoder, Encoder, Json}
|
||||
import io.circe.Decoder
|
||||
import org.enso.editions.LibraryName
|
||||
|
||||
/** The manifest file containing metadata related to a published library.
|
||||
@ -20,17 +19,6 @@ case class LibraryManifest(
|
||||
)
|
||||
|
||||
object LibraryManifest {
|
||||
|
||||
/** Creates an empty manifest.
|
||||
*
|
||||
* Such a manifest is invalid as at least one archive should be specified in
|
||||
* a valid manifest.
|
||||
*
|
||||
* It can however be useful as a temporary value for logic that updates or
|
||||
* creates a new manifest.
|
||||
*/
|
||||
def empty: LibraryManifest = LibraryManifest(Seq.empty, Seq.empty, None, None)
|
||||
|
||||
object Fields {
|
||||
val archives = "archives"
|
||||
val dependencies = "dependencies"
|
||||
@ -55,20 +43,6 @@ object LibraryManifest {
|
||||
)
|
||||
}
|
||||
|
||||
/** An [[Encoder]] instance for parsing [[LibraryManifest]]. */
|
||||
implicit val encoder: Encoder[LibraryManifest] = { manifest =>
|
||||
val baseFields = Seq(
|
||||
Fields.archives -> manifest.archives.asJson,
|
||||
Fields.dependencies -> manifest.dependencies.asJson
|
||||
)
|
||||
|
||||
val allFields = baseFields ++
|
||||
manifest.tagLine.map(Fields.tagLine -> _.asJson).toSeq ++
|
||||
manifest.description.map(Fields.description -> _.asJson).toSeq
|
||||
|
||||
Json.obj(allFields: _*)
|
||||
}
|
||||
|
||||
/** The name of the manifest file as included in the directory associated with
|
||||
* a given library in the library repository.
|
||||
*/
|
||||
|
||||
@ -1,233 +0,0 @@
|
||||
package org.enso.libraryupload
|
||||
|
||||
import akka.http.scaladsl.marshalling.Marshal
|
||||
import akka.http.scaladsl.model._
|
||||
import akka.stream.scaladsl.Source
|
||||
import com.typesafe.scalalogging.Logger
|
||||
import nl.gn0s1s.bump.SemVer
|
||||
import org.enso.cli.task.{ProgressReporter, TaskProgress}
|
||||
import org.enso.distribution.FileSystem
|
||||
import org.enso.distribution.FileSystem.PathSyntax
|
||||
import org.enso.downloader.archive.TarGzWriter
|
||||
import org.enso.downloader.http.{HTTPDownload, HTTPRequestBuilder, URIBuilder}
|
||||
import org.enso.editions.LibraryName
|
||||
import org.enso.librarymanager.published.repository.LibraryManifest
|
||||
import org.enso.pkg.{Package, PackageManager}
|
||||
import org.enso.yaml.YamlHelper
|
||||
|
||||
import java.nio.file.{Files, Path}
|
||||
import scala.concurrent.{ExecutionContext, Future}
|
||||
import scala.jdk.CollectionConverters._
|
||||
import scala.util.{Failure, Success, Try, Using}
|
||||
|
||||
/** Gathers functions used for uploading libraries. */
|
||||
object LibraryUploader {
|
||||
private lazy val logger = Logger[LibraryUploader.type]
|
||||
|
||||
/** Uploads a library to a repository.
|
||||
*
|
||||
* @param projectRoot path to the library project root
|
||||
* @param uploadUrl an URL to the upload endpoint of a library repository
|
||||
* @param authToken a token describing the authentication method to use with
|
||||
* the repository
|
||||
* @param progressReporter a [[ProgressReporter]] to track long running tasks
|
||||
* like compression and upload
|
||||
* @param ec an execution context used for handling Futures
|
||||
*/
|
||||
def uploadLibrary(
|
||||
projectRoot: Path,
|
||||
uploadUrl: String,
|
||||
authToken: auth.Token,
|
||||
progressReporter: ProgressReporter
|
||||
)(implicit ec: ExecutionContext): Try[Unit] = Try {
|
||||
FileSystem.withTemporaryDirectory("enso-upload") { tmpDir =>
|
||||
val pkg = PackageManager.Default.loadPackage(projectRoot.toFile).get
|
||||
val version = SemVer(pkg.config.version).getOrElse {
|
||||
throw new IllegalStateException(
|
||||
s"Project version [${pkg.config.version}] is not a valid semver " +
|
||||
s"string."
|
||||
)
|
||||
}
|
||||
val uri = buildUploadUri(uploadUrl, pkg.libraryName, version)
|
||||
|
||||
val mainArchiveName = "main.tgz"
|
||||
val filesToIgnoreInArchive = Seq(
|
||||
Package.configFileName,
|
||||
LibraryManifest.filename
|
||||
)
|
||||
val archivePath = tmpDir / mainArchiveName
|
||||
val compressing =
|
||||
createMainArchive(projectRoot, filesToIgnoreInArchive, archivePath)
|
||||
progressReporter.trackProgress(
|
||||
s"Creating the [$mainArchiveName] archive.",
|
||||
compressing
|
||||
)
|
||||
compressing.force()
|
||||
|
||||
val manifestPath = projectRoot / LibraryManifest.filename
|
||||
val loadedManifest =
|
||||
loadSavedManifest(manifestPath).getOrElse(LibraryManifest.empty)
|
||||
val updatedManifest =
|
||||
// TODO [RW] update dependencies in the manifest (#1773)
|
||||
loadedManifest.copy(archives = Seq(mainArchiveName))
|
||||
FileSystem.writeTextFile(manifestPath, YamlHelper.toYaml(updatedManifest))
|
||||
|
||||
logger.info(s"Uploading library package to the server at [$uploadUrl].")
|
||||
val upload = uploadFiles(
|
||||
uri,
|
||||
authToken,
|
||||
files = Seq(
|
||||
projectRoot / Package.configFileName,
|
||||
projectRoot / LibraryManifest.filename,
|
||||
archivePath
|
||||
)
|
||||
)
|
||||
progressReporter.trackProgress(
|
||||
s"Uploading packages to [$uploadUrl].",
|
||||
upload
|
||||
)
|
||||
upload.force()
|
||||
|
||||
logger.info(s"Upload complete.")
|
||||
}
|
||||
}
|
||||
|
||||
/** Creates an URL for the upload, including information identifying the
|
||||
* library version.
|
||||
*/
|
||||
private def buildUploadUri(
|
||||
baseUploadUrl: String,
|
||||
libraryName: LibraryName,
|
||||
version: SemVer
|
||||
): Uri = {
|
||||
URIBuilder
|
||||
.fromUri(baseUploadUrl)
|
||||
.addQuery("namespace", libraryName.namespace)
|
||||
.addQuery("name", libraryName.name)
|
||||
.addQuery("version", version.toString)
|
||||
.build()
|
||||
}
|
||||
|
||||
/** Gathers project files to create the main archive.
|
||||
*
|
||||
* For now it just filters out the files like manifest which are uploaded
|
||||
* separately. In the future this may be extended to create separate
|
||||
* sub-archives for platform specific binaries or tests.
|
||||
*
|
||||
* @param projectRoot path to the project root
|
||||
* @param rootFilesToIgnore names of files at the root that should *not* be
|
||||
* included in the archive
|
||||
* @param destination path at which the archive is created
|
||||
* @return
|
||||
*/
|
||||
private def createMainArchive(
|
||||
projectRoot: Path,
|
||||
rootFilesToIgnore: Seq[String],
|
||||
destination: Path
|
||||
): TaskProgress[Unit] = {
|
||||
def relativePath(file: Path): String = projectRoot.relativize(file).toString
|
||||
def shouldBeUploaded(file: Path): Boolean = {
|
||||
def isIgnored = rootFilesToIgnore.contains(relativePath(file))
|
||||
Files.isRegularFile(file) && !isIgnored
|
||||
}
|
||||
|
||||
logger.trace("Gathering files to compress.")
|
||||
val filesToCompress = Using(Files.walk(projectRoot)) { filesStream =>
|
||||
filesStream.iterator().asScala.filter(shouldBeUploaded).toSeq
|
||||
}.get
|
||||
|
||||
logger.info(
|
||||
s"Compressing ${filesToCompress.size} project files " +
|
||||
s"into [${destination.getFileName}]."
|
||||
)
|
||||
|
||||
val compression = TarGzWriter.compress(
|
||||
archiveDestination = destination,
|
||||
files = filesToCompress,
|
||||
basePath = projectRoot
|
||||
)
|
||||
|
||||
compression.map { _ =>
|
||||
logger.info(s"Archive [${destination.getFileName}] created.")
|
||||
}
|
||||
}
|
||||
|
||||
/** Creates a [[RequestEntity]] that will upload the provided files. */
|
||||
private def createRequestEntity(
|
||||
files: Seq[Path]
|
||||
)(implicit ec: ExecutionContext): Future[RequestEntity] = {
|
||||
|
||||
val fileBodies = files.map { path =>
|
||||
val filename = path.getFileName.toString
|
||||
Multipart.FormData.BodyPart(
|
||||
filename,
|
||||
HttpEntity.fromPath(detectContentType(path), path),
|
||||
Map("filename" -> filename)
|
||||
)
|
||||
}
|
||||
|
||||
val formData = Multipart.FormData(Source(fileBodies))
|
||||
Marshal(formData).to[RequestEntity]
|
||||
}
|
||||
|
||||
/** Loads a manifest, if it exists. */
|
||||
private def loadSavedManifest(manifestPath: Path): Option[LibraryManifest] = {
|
||||
if (Files.exists(manifestPath)) {
|
||||
val loaded = YamlHelper.load[LibraryManifest](manifestPath).get
|
||||
Some(loaded)
|
||||
} else None
|
||||
}
|
||||
|
||||
/** Tries to detect the content type of the file to upload.
|
||||
*
|
||||
* If it is not a known type, it falls back to `application/octet-stream`.
|
||||
*/
|
||||
private def detectContentType(path: Path): ContentType = {
|
||||
val filename = path.getFileName.toString
|
||||
if (filename.endsWith(".tgz") || filename.endsWith(".tar.gz"))
|
||||
ContentType(MediaTypes.`application/x-gtar`)
|
||||
else if (filename.endsWith(".yaml") || filename.endsWith(".enso"))
|
||||
ContentTypes.`text/plain(UTF-8)`
|
||||
else ContentTypes.`application/octet-stream`
|
||||
}
|
||||
|
||||
/** Uploads the provided files to the provided url, using the provided token
|
||||
* for authentication.
|
||||
*/
|
||||
private def uploadFiles(
|
||||
uri: Uri,
|
||||
authToken: auth.Token,
|
||||
files: Seq[Path]
|
||||
)(implicit ec: ExecutionContext): TaskProgress[Unit] = {
|
||||
val future = createRequestEntity(files).map { entity =>
|
||||
val request = authToken
|
||||
.alterRequest(HTTPRequestBuilder.fromURI(uri))
|
||||
.setEntity(entity)
|
||||
.POST
|
||||
// TODO [RW] upload progress
|
||||
HTTPDownload.fetchString(request).force()
|
||||
}
|
||||
TaskProgress.fromFuture(future).flatMap { response =>
|
||||
if (response.statusCode == 200) {
|
||||
logger.debug("Server responded with 200 OK.")
|
||||
Success(())
|
||||
} else {
|
||||
// TODO [RW] we may want to have more precise error messages to handle auth errors etc. (#1773)
|
||||
val includedMessage = for {
|
||||
json <- io.circe.parser.parse(response.content).toOption
|
||||
obj <- json.asObject
|
||||
message <- obj("error").flatMap(_.asString)
|
||||
} yield message
|
||||
val message = includedMessage.getOrElse("Unknown error")
|
||||
val errorMessage =
|
||||
s"Upload failed: $message (Status code: ${response.statusCode})."
|
||||
logger.error(errorMessage)
|
||||
Failure(
|
||||
new RuntimeException(
|
||||
errorMessage
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,35 +0,0 @@
|
||||
package org.enso.libraryupload.auth
|
||||
|
||||
import org.enso.downloader.http.HTTPRequestBuilder
|
||||
|
||||
/** Represents an authentication method that can be used to authenticate
|
||||
* requests to the library repository.
|
||||
*/
|
||||
trait Token {
|
||||
|
||||
/** Alters the request adding any properties (like headers) necessary to
|
||||
* successfully authenticate.
|
||||
*/
|
||||
def alterRequest(request: HTTPRequestBuilder): HTTPRequestBuilder
|
||||
}
|
||||
|
||||
/** A simple authentication method that adds an `Auth-Token` header to the
|
||||
* request.
|
||||
*/
|
||||
case class SimpleHeaderToken(value: String) extends Token {
|
||||
|
||||
/** @inheritdoc */
|
||||
override def alterRequest(request: HTTPRequestBuilder): HTTPRequestBuilder =
|
||||
request.addHeader("Auth-Token", value)
|
||||
}
|
||||
|
||||
/** A dummy authentication method that does not do anything.
|
||||
*
|
||||
* It can be used for servers that do not require any authentication.
|
||||
*/
|
||||
case object NoAuthorization extends Token {
|
||||
|
||||
/** @inheritdoc */
|
||||
override def alterRequest(request: HTTPRequestBuilder): HTTPRequestBuilder =
|
||||
request
|
||||
}
|
||||
@ -0,0 +1,50 @@
|
||||
package org.enso.librarymanager.published.repository
|
||||
|
||||
import org.apache.commons.compress.archivers.tar.{
|
||||
TarArchiveEntry,
|
||||
TarArchiveOutputStream
|
||||
}
|
||||
import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream
|
||||
|
||||
import java.io.{BufferedOutputStream, FileOutputStream}
|
||||
import java.nio.file.Path
|
||||
import scala.util.Using
|
||||
|
||||
/** A helper class used for creating TAR-GZ archives in tests. */
|
||||
object ArchiveWriter {
|
||||
|
||||
/** A file to add to the archive. */
|
||||
sealed trait FileToWrite {
|
||||
|
||||
/** The path that this file should have within the archive. */
|
||||
def relativePath: String
|
||||
}
|
||||
|
||||
/** Represents a text file to be added to a test archive.
|
||||
*
|
||||
* @param relativePath the path in the archive
|
||||
* @param content the text contents for the file
|
||||
*/
|
||||
case class TextFile(relativePath: String, content: String) extends FileToWrite
|
||||
|
||||
/** Creates a tar archive at the given path, containing the provided files. */
|
||||
def writeTarArchive(path: Path, files: Seq[FileToWrite]): Unit = {
|
||||
Using(new FileOutputStream(path.toFile)) { outputStream =>
|
||||
Using(new BufferedOutputStream(outputStream)) { bufferedStream =>
|
||||
Using(new GzipCompressorOutputStream(bufferedStream)) { gzipStream =>
|
||||
Using(new TarArchiveOutputStream(gzipStream)) { archive =>
|
||||
for (file <- files) {
|
||||
file match {
|
||||
case TextFile(relativePath, content) =>
|
||||
val entry = new TarArchiveEntry(relativePath)
|
||||
archive.putArchiveEntry(entry)
|
||||
archive.write(content.getBytes)
|
||||
archive.closeArchiveEntry()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -3,7 +3,6 @@ package org.enso.librarymanager.published.repository
|
||||
import nl.gn0s1s.bump.SemVer
|
||||
import org.enso.cli.OS
|
||||
import org.enso.distribution.FileSystem
|
||||
import org.enso.downloader.archive.TarGzWriter
|
||||
import org.enso.editions.Editions.RawEdition
|
||||
import org.enso.editions.{Editions, LibraryName}
|
||||
import org.enso.pkg.{Package, PackageManager}
|
||||
@ -49,13 +48,10 @@ abstract class DummyRepository {
|
||||
.resolve(lib.version.toString)
|
||||
Files.createDirectories(libraryRoot)
|
||||
createLibraryProject(libraryRoot, lib)
|
||||
|
||||
TarGzWriter
|
||||
.createArchive(libraryRoot.resolve("main.tgz")) { writer =>
|
||||
writer.writeTextFile("src/Main.enso", lib.mainContent)
|
||||
}
|
||||
.get
|
||||
|
||||
val files = Seq(
|
||||
ArchiveWriter.TextFile("src/Main.enso", lib.mainContent)
|
||||
)
|
||||
ArchiveWriter.writeTarArchive(libraryRoot.resolve("main.tgz"), files)
|
||||
createManifest(libraryRoot)
|
||||
}
|
||||
}
|
||||
@ -110,18 +106,12 @@ abstract class DummyRepository {
|
||||
* @param port port to listen on
|
||||
* @param root root of the library repository, the same as the argument to
|
||||
* [[createRepository]]
|
||||
* @param uploads specifies whether to enable uploads in the server
|
||||
*/
|
||||
def startServer(
|
||||
port: Int,
|
||||
root: Path,
|
||||
uploads: Boolean = false
|
||||
): WrappedProcess = {
|
||||
def startServer(port: Int, root: Path): WrappedProcess = {
|
||||
val serverDirectory =
|
||||
Path.of("tools/simple-library-server").toAbsolutePath.normalize
|
||||
|
||||
val preinstallCommand =
|
||||
commandPrefix ++ Seq(npmCommand, "install")
|
||||
val preinstallCommand = commandPrefix ++ Seq(npmCommand, "install")
|
||||
val preinstallExitCode = new ProcessBuilder()
|
||||
.command(preinstallCommand: _*)
|
||||
.directory(serverDirectory.toFile)
|
||||
@ -135,7 +125,6 @@ abstract class DummyRepository {
|
||||
s"npm exited with code $preinstallCommand."
|
||||
)
|
||||
|
||||
val uploadsArgs = if (uploads) Seq("--upload", "no-auth") else Seq()
|
||||
val command = commandPrefix ++ Seq(
|
||||
nodeCommand,
|
||||
"main.js",
|
||||
@ -143,7 +132,7 @@ abstract class DummyRepository {
|
||||
port.toString,
|
||||
"--root",
|
||||
root.toAbsolutePath.normalize.toString
|
||||
) ++ uploadsArgs
|
||||
)
|
||||
val rawProcess = (new ProcessBuilder)
|
||||
.command(command: _*)
|
||||
.directory(serverDirectory.toFile)
|
||||
@ -1,8 +1,17 @@
|
||||
package org.enso.librarymanager.published.repository
|
||||
|
||||
import org.enso.cli.task.{ProgressReporter, TaskProgress}
|
||||
import org.enso.distribution.TemporaryDirectoryManager
|
||||
import org.enso.distribution.locking.{
|
||||
LockUserInterface,
|
||||
Resource,
|
||||
ResourceManager,
|
||||
ThreadSafeFileLockManager
|
||||
}
|
||||
import org.enso.editions.Editions
|
||||
import org.enso.loggingservice.{LogLevel, TestLogger}
|
||||
import org.enso.librarymanager.published.cache.DownloadingLibraryCache
|
||||
import org.enso.loggingservice.TestLogger.TestLogMessage
|
||||
import org.enso.loggingservice.{LogLevel, TestLogger}
|
||||
import org.enso.pkg.PackageManager
|
||||
import org.enso.testkit.WithTemporaryDirectory
|
||||
import org.scalatest.matchers.should.Matchers
|
||||
@ -13,8 +22,7 @@ import java.nio.file.Files
|
||||
class LibraryDownloadTest
|
||||
extends AnyWordSpec
|
||||
with Matchers
|
||||
with WithTemporaryDirectory
|
||||
with DownloaderTest {
|
||||
with WithTemporaryDirectory {
|
||||
|
||||
val port: Int = 47306
|
||||
|
||||
@ -24,7 +32,32 @@ class LibraryDownloadTest
|
||||
|
||||
val repoRoot = getTestDirectory.resolve("repo")
|
||||
repo.createRepository(repoRoot)
|
||||
withDownloader { cache =>
|
||||
val lockManager =
|
||||
new ThreadSafeFileLockManager(getTestDirectory.resolve("locks"))
|
||||
val resourceManager = new ResourceManager(lockManager)
|
||||
try {
|
||||
val cache = new DownloadingLibraryCache(
|
||||
cacheRoot = getTestDirectory.resolve("cache"),
|
||||
temporaryDirectoryManager = new TemporaryDirectoryManager(
|
||||
getTestDirectory.resolve("tmp"),
|
||||
resourceManager
|
||||
),
|
||||
resourceManager = resourceManager,
|
||||
lockUserInterface = new LockUserInterface {
|
||||
override def startWaitingForResource(resource: Resource): Unit =
|
||||
println(s"Waiting for ${resource.name}")
|
||||
|
||||
override def finishWaitingForResource(resource: Resource): Unit =
|
||||
println(s"${resource.name} is ready")
|
||||
},
|
||||
progressReporter = new ProgressReporter {
|
||||
override def trackProgress(
|
||||
message: String,
|
||||
task: TaskProgress[_]
|
||||
): Unit = {}
|
||||
}
|
||||
)
|
||||
|
||||
val server = repo.startServer(port, repoRoot)
|
||||
try {
|
||||
cache.findCachedLibrary(
|
||||
@ -62,6 +95,9 @@ class LibraryDownloadTest
|
||||
server.kill(killDescendants = true)
|
||||
server.join(waitForDescendants = true)
|
||||
}
|
||||
} finally {
|
||||
resourceManager.releaseMainLock()
|
||||
resourceManager.unlockTemporaryDirectory()
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,13 +1,7 @@
|
||||
#!/usr/bin/env node
|
||||
const express = require("express");
|
||||
const path = require("path");
|
||||
const os = require("os");
|
||||
const fs = require("fs");
|
||||
const fsPromises = require("fs/promises");
|
||||
const multer = require("multer");
|
||||
const compression = require("compression");
|
||||
const yargs = require("yargs");
|
||||
const semverValid = require("semver/functions/valid");
|
||||
|
||||
const argv = yargs
|
||||
.usage(
|
||||
@ -25,44 +19,11 @@ const argv = yargs
|
||||
type: "string",
|
||||
default: ".",
|
||||
})
|
||||
.option("upload", {
|
||||
description:
|
||||
"Specifies whether to allow uploading libraries and which authentication model to choose.",
|
||||
choices: ["disabled", "no-auth", "constant-token"],
|
||||
default: "disabled",
|
||||
})
|
||||
.help()
|
||||
.alias("help", "h").argv;
|
||||
|
||||
const libraryRoot = path.join(argv.root, "libraries");
|
||||
|
||||
const app = express();
|
||||
const tmpDir = path.join(os.tmpdir(), "enso-library-repo-uploads");
|
||||
const upload = multer({ dest: tmpDir });
|
||||
app.use(compression({ filter: shouldCompress }));
|
||||
|
||||
/** The token to compare against for simple authentication.
|
||||
*
|
||||
* If it is not set, no authentication checks are made.
|
||||
*/
|
||||
let token = null;
|
||||
if (argv.upload == "disabled") {
|
||||
console.log("Uploads are disabled.");
|
||||
} else {
|
||||
app.post("/upload", upload.any(), handleUpload);
|
||||
|
||||
if (argv.upload == "constant-token") {
|
||||
const envVar = "ENSO_AUTH_TOKEN";
|
||||
token = process.env[envVar];
|
||||
if (!token) {
|
||||
throw `${envVar} is not defined.`;
|
||||
} else {
|
||||
console.log(`Checking the ${envVar} to authorize requests.`);
|
||||
}
|
||||
} else {
|
||||
console.log("WARNING: Uploads are enabled without any authentication.");
|
||||
}
|
||||
}
|
||||
app.use(express.static(argv.root));
|
||||
|
||||
console.log(
|
||||
@ -71,7 +32,6 @@ console.log(
|
||||
|
||||
app.listen(argv.port);
|
||||
|
||||
/// Specifies if a particular file can be compressed in transfer, if supported.
|
||||
function shouldCompress(req, res) {
|
||||
if (req.path.endsWith(".yaml")) {
|
||||
return true;
|
||||
@ -79,121 +39,3 @@ function shouldCompress(req, res) {
|
||||
|
||||
return compression.filter(req, res);
|
||||
}
|
||||
|
||||
/** Handles upload of a library. */
|
||||
async function handleUpload(req, res) {
|
||||
function fail(code, message) {
|
||||
res.status(code).json({ error: message });
|
||||
cleanFiles(req.files);
|
||||
}
|
||||
|
||||
if (token !== null) {
|
||||
const userToken = req.get("Auth-Token");
|
||||
if (userToken != token) {
|
||||
return fail(403, "Authorization failed.");
|
||||
}
|
||||
}
|
||||
|
||||
const version = req.query.version;
|
||||
const namespace = req.query.namespace;
|
||||
const name = req.query.name;
|
||||
|
||||
if (version === undefined || namespace == undefined || name === undefined) {
|
||||
return fail(400, "One or more required fields were missing.");
|
||||
}
|
||||
|
||||
if (!isVersionValid(version)) {
|
||||
return fail(400, `Invalid semver version string [${version}].`);
|
||||
}
|
||||
|
||||
if (!isNamespaceValid(namespace)) {
|
||||
return fail(400, `Invalid username [${namespace}].`);
|
||||
}
|
||||
|
||||
if (!isNameValid(name)) {
|
||||
return fail(400, `Invalid library name [${name}].`);
|
||||
}
|
||||
|
||||
for (var i = 0; i < req.files.length; ++i) {
|
||||
const filename = req.files[i].originalname;
|
||||
if (!isFilenameValid(filename)) {
|
||||
return fail(400, `Invalid filename: ${filename}.`);
|
||||
}
|
||||
}
|
||||
|
||||
const libraryPath = path.join(libraryRoot, namespace, name, version);
|
||||
|
||||
if (fs.existsSync(libraryPath)) {
|
||||
return fail(
|
||||
409,
|
||||
"A library with the given name and version " +
|
||||
"combination already exists. Versions are immutable, so you must " +
|
||||
"bump the library version when uploading a newer version."
|
||||
);
|
||||
}
|
||||
|
||||
await fsPromises.mkdir(libraryPath, { recursive: true });
|
||||
|
||||
console.log(`Uploading library [${namespace}.${name}:${version}].`);
|
||||
try {
|
||||
await putFiles(libraryPath, req.files);
|
||||
} catch (error) {
|
||||
console.log(`Upload failed: [${error}].`);
|
||||
console.error(error.stack);
|
||||
return fail(500, "Upload failed due to an internal error.");
|
||||
}
|
||||
|
||||
console.log("Upload complete.");
|
||||
res.status(200).json({ message: "Successfully uploaded the library." });
|
||||
}
|
||||
|
||||
/// Checks if a version complies with the semver specification.
|
||||
function isVersionValid(version) {
|
||||
return semverValid(version) !== null;
|
||||
}
|
||||
|
||||
/// Checks if the namespace/username is valid.
|
||||
function isNamespaceValid(namespace) {
|
||||
return /^[a-z][a-z0-9]*$/.test(namespace) && namespace.length >= 3;
|
||||
}
|
||||
|
||||
/** Checks if the library name is valid.
|
||||
*
|
||||
* It may actually accept more identifiers as valid than Enso would, the actual
|
||||
* check should be done when creating the library. This is just a sanity check
|
||||
* for safety.
|
||||
*/
|
||||
function isNameValid(name) {
|
||||
return /^[A-Za-z0-9_]+$/.test(name);
|
||||
}
|
||||
|
||||
// TODO [RW] for now slashes are not permitted to avoid attacks; later on at least the `meta` directory should be allowed, but not much besides that
|
||||
/// Checks if the uploaded filename is valid.
|
||||
function isFilenameValid(name) {
|
||||
return /^[A-Za-z0-9][A-Za-z0-9\._\-]*$/.test(name);
|
||||
}
|
||||
|
||||
/// Schedules to remove the files, if they still exist.
|
||||
function cleanFiles(files) {
|
||||
files.forEach((file) => {
|
||||
if (fs.existsSync(file.path)) {
|
||||
fs.unlink(file.path, (err) => {
|
||||
if (err) {
|
||||
console.error(
|
||||
`Failed to remove ${file.path} ($file.originalname) from a failed upload: ${err}.`
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Moves the files to the provided destination directory.
|
||||
async function putFiles(directory, files) {
|
||||
for (var i = 0; i < files.length; ++i) {
|
||||
const file = files[i];
|
||||
const filename = file.originalname;
|
||||
const destination = path.join(directory, filename);
|
||||
await fsPromises.rename(file.path, destination);
|
||||
}
|
||||
}
|
||||
|
||||
@ -16,8 +16,6 @@
|
||||
"dependencies": {
|
||||
"compression": "^1.7.4",
|
||||
"express": "^4.17.1",
|
||||
"multer": "^1.4.2",
|
||||
"semver": "^7.3.5",
|
||||
"yargs": "^17.0.1"
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user