enso-org/enso
1
1
Fork 0
mirror of https://github.com/enso-org/enso.git synced 2024-12-31 18:11:33 +03:00
Code Issues Projects Releases Wiki Activity

Preinstalling With Dependencies (#1981)

Browse Source
This commit is contained in:
Radosław Waśko 2021-11-23 09:51:17 +01:00 committed by GitHub
parent d61743d43c
commit 46c31bb9a5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
41 changed files with 1162 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/scala.yml vendored
View File

@ -151,6 +151,8 @@ jobs:
sbt --no-colors "project-manager/assembly"
sbt --no-colors --mem 1536 "project-manager/buildNativeImage"
# The runtime/clean is needed to avoid issues with Truffle Instrumentation.
# It should be removed once #1992 is fixed.
- name: Build the Runner & Runtime Uberjars
run: |
sleep 1
@ -167,7 +169,7 @@ jobs:
- name: Check Language Server Benchmark Compilation
run: |
sleep 1
sbt --no-colors language-server/Benchmark/compile
sbt --no-colors "runtime/clean; language-server/Benchmark/compile"
- name: Check Searcher Benchmark Compilation
run: |
sleep 1

1
.gitignore vendored
View File

@ -127,6 +127,7 @@ distribution/lib/Standard/Table/*/polyglot/
distribution/lib/Standard/Database/*/polyglot/
distribution/lib/Standard/Examples/*/data/spreadsheet.xls
distribution/lib/Standard/Examples/*/data/spreadsheet.xlsx
distribution/lib/*/*/*/manifest.yaml
test/Google_Api_Test/data/secret.json
test/Database_Tests/data/redshift_credentials.json

8
RELEASES.md
View File

@ -1,5 +1,13 @@
# Enso Next
## Tooling
- Added the `enso install dependencies` command to the launcher which installs
any project dependencies, ensuring that `enso run` will not need to download
any libraries ([#1981](https://github.com/enso-org/enso/pull/1981)).
Additionally, made the `library/preinstall` endpoint able to install any
transitive dependencies of the library.
## Enso 0.2.31 (2021-10-01)
## Interpreter/Runtime

47
build.sbt
View File

@ -1,3 +1,4 @@
import LibraryManifestGenerator.BundledLibrary
import org.enso.build.BenchTasks._
import org.enso.build.WithDebugCommand
import sbt.Keys.{libraryDependencies, scalacOptions}
@ -1013,6 +1014,27 @@ lazy val `language-server` = (project in file("engine/language-server"))
new TestFramework("org.scalameter.ScalaMeterFramework")
)
)
.settings(
// These settings are needed by language-server tests that create a runtime context.
Test / fork := true,
Test / javaOptions ++= {
// Note [Classpath Separation]
val runtimeClasspath =
(LocalProject("runtime") / Compile / fullClasspath).value
.map(_.data)
.mkString(File.pathSeparator)
Seq(
s"-Dtruffle.class.path.append=$runtimeClasspath",
s"-Duser.dir=${file(".").getCanonicalPath}"
)
},
Test / compile := (Test / compile)
.dependsOn(LocalProject("enso") / updateLibraryManifests)
.value,
Test / envVars ++= Map(
"ENSO_EDITION_PATH" -> file("distribution/editions").getCanonicalPath
)
)
.dependsOn(`json-rpc-server-test` % Test)
.dependsOn(`json-rpc-server`)
.dependsOn(`task-progress-notifications`)
@ -1632,7 +1654,7 @@ lazy val `std-database` = project
`database-polyglot-root`,
Some("std-database.jar"),
ignoreScalaLibrary = true,
unpackedDeps = Set("aws-java-sdk-core", "httpclient")
unpackedDeps = Set("aws-java-sdk-core", "httpclient")
)
.value
result
@ -1684,7 +1706,8 @@ projectManagerDistributionRoot :=
lazy val buildEngineDistribution =
taskKey[Unit]("Builds the engine distribution")
buildEngineDistribution := {
val _ = (`engine-runner` / assembly).value
val _ = (`engine-runner` / assembly).value
updateLibraryManifests.value
val root = engineDistributionRoot.value
val log = streams.value.log
val cacheFactory = streams.value.cacheStoreFactory
@ -1744,3 +1767,23 @@ buildGraalDistribution := {
DistributionPackage.Architecture.X64
)
}
lazy val updateLibraryManifests =
taskKey[Unit](
"Recomputes dependencies to update manifests bundled with libraries."
)
updateLibraryManifests := {
val _ = (`engine-runner` / assembly).value
val log = streams.value.log
val cacheFactory = streams.value.cacheStoreFactory
val libraries = Editions.standardLibraries.map(libName =>
BundledLibrary(libName, stdLibVersion)
)
LibraryManifestGenerator.generateManifests(
libraries,
file("distribution"),
log,
cacheFactory
)
}

15
docs/language-server/protocol-language-server.md
View File

@ -204,6 +204,7 @@ transport formats, please look [here](./protocol-architecture).
- [`LocalLibraryNotFound`](#locallibrarynotfound)
- [`LibraryNotResolved`](#librarynotresolved)
- [`InvalidLibraryName`](#invalidlibraryname)
- [`DependencyDiscoveryError`](#dependencydiscoveryerror)
<!-- /MarkdownTOC -->
@ -4550,6 +4551,8 @@ null;
- [`LibraryNotResolved`](#librarynotresolved) to signal that the requested
library or one of its dependencies could not be resolved.
- [`DependencyDiscoveryError`](#dependencydiscoveryerror) to signal that
dependencies of the library could not be established.
- [`LibraryDownloadError`](#librarydownloaderror) to signal that the download
operation has failed, for network-related reasons, or because the library was
missing in the repository. The error includes the name and version of the
@ -5085,3 +5088,15 @@ For example for `FooBar` it will suggest `Foo_Bar`.
}
}
```
### `DependencyDiscoveryError`
Signals that the library preinstall endpoint could not properly find
dependencies of the requested library.
```typescript
"error" : {
"code" : 8010,
"message" : "Error occurred while discovering dependencies: <reason>."
}
```

11
engine/language-server/src/main/scala/org/enso/languageserver/boot/MainModule.scala
View File

@ -16,13 +16,7 @@ import org.enso.languageserver.effect.ZioExec
import org.enso.languageserver.filemanager._
import org.enso.languageserver.http.server.BinaryWebSocketServer
import org.enso.languageserver.io._
import org.enso.languageserver.libraries.{
EditionReferenceResolver,
LibraryConfig,
LibraryInstallerConfig,
LocalLibraryManager,
ProjectSettingsManager
}
import org.enso.languageserver.libraries._
import org.enso.languageserver.monitoring.{
HealthCheckEndpoint,
IdlenessEndpoint,
@ -330,7 +324,8 @@ class MainModule(serverConfig: LanguageServerConfig, logLevel: LogLevel) {
installerConfig = LibraryInstallerConfig(
distributionManager,
resourceManager,
Some(languageHome)
Some(languageHome),
new CompilerBasedDependencyExtractor(logLevel)
)
)

70
engine/language-server/src/main/scala/org/enso/languageserver/libraries/CompilerBasedDependencyExtractor.scala Normal file
View File

@ -0,0 +1,70 @@
package org.enso.languageserver.libraries
import org.enso.editions.LibraryName
import org.enso.libraryupload.DependencyExtractor
import org.enso.loggingservice.{JavaLoggingLogHandler, LogLevel}
import org.enso.pkg.Package
import org.enso.pkg.SourceFile
import org.enso.polyglot.{PolyglotContext, RuntimeOptions}
import org.graalvm.polyglot.Context
import java.io.File
/** A dependency extractor that runs the compiler in a mode that only parses the
* source code and runs just the basic preprocessing phases to find out what
* libraries are imported by the project.
*
* @param logLevel the log level to use for the runtime context that will do
* the parsing
*/
class CompilerBasedDependencyExtractor(logLevel: LogLevel)
extends DependencyExtractor[File] {
/** @inheritdoc */
override def findDependencies(pkg: Package[File]): Set[LibraryName] = {
val context = createContextWithProject(pkg)
def findImportedLibraries(file: SourceFile[File]): Set[LibraryName] = {
val module = context.getTopScope.getModule(file.qualifiedName.toString)
val imports = module.gatherImportStatements()
val importedLibraries = imports.map { rawName =>
LibraryName.fromString(rawName) match {
case Left(error) =>
throw new IllegalStateException(error)
case Right(value) => value
}
}
importedLibraries.toSet
}
val sourcesImports = pkg.listSources.toSet.flatMap(findImportedLibraries)
val itself = pkg.libraryName
// Builtins need to be removed from the set of the dependencies, because
// even if they are imported, they are not a typical library.
val builtins = LibraryName("Standard", "Builtins")
sourcesImports - itself - builtins
}
/** Creates a simple runtime context with the given package loaded as its
* project root.
*/
private def createContextWithProject(pkg: Package[File]): PolyglotContext = {
val context = Context
.newBuilder()
.allowExperimentalOptions(true)
.allowAllAccess(true)
.option(RuntimeOptions.PROJECT_ROOT, pkg.root.getCanonicalPath)
.option("js.foreign-object-prototype", "true")
.option(
RuntimeOptions.LOG_LEVEL,
JavaLoggingLogHandler.getJavaLogLevelFor(logLevel).getName
)
.logHandler(
JavaLoggingLogHandler.create(JavaLoggingLogHandler.defaultLevelMapping)
)
.build
new PolyglotContext(context)
}
}

6
engine/language-server/src/main/scala/org/enso/languageserver/libraries/LibraryApi.scala
View File

@ -250,4 +250,10 @@ object LibraryApi {
} """
)
}
case class DependencyDiscoveryError(reason: String)
extends Error(
8010,
s"Error occurred while discovering dependencies: $reason."
)
}

9
engine/language-server/src/main/scala/org/enso/languageserver/libraries/LibraryInstallerConfig.scala
View File

@ -1,7 +1,10 @@
package org.enso.languageserver.libraries
import org.enso.distribution.{DistributionManager, LanguageHome}
import org.enso.distribution.locking.ResourceManager
import org.enso.distribution.{DistributionManager, LanguageHome}
import org.enso.libraryupload.DependencyExtractor
import java.io.File
/** Gathers configuration needed by the library installer used in the
* `library/preinstall` endpoint.
@ -9,9 +12,11 @@ import org.enso.distribution.locking.ResourceManager
* @param distributionManager the distribution manager
* @param resourceManager a resource manager instance
* @param languageHome language home, if detected / applicable
* @param dependencyExtractor a dependency extractor
*/
case class LibraryInstallerConfig(
distributionManager: DistributionManager,
resourceManager: ResourceManager,
languageHome: Option[LanguageHome]
languageHome: Option[LanguageHome],
dependencyExtractor: DependencyExtractor[File]
)

160
engine/language-server/src/main/scala/org/enso/languageserver/libraries/handler/LibraryPreinstallHandler.scala
View File

@ -2,9 +2,14 @@ package org.enso.languageserver.libraries.handler
import akka.actor.{Actor, ActorRef, Props, Status}
import akka.pattern.pipe
import cats.implicits.toTraverseOps
import com.typesafe.scalalogging.LazyLogging
import org.enso.cli.task.notifications.ActorProgressNotificationForwarder
import org.enso.cli.task.{ProgressNotification, ProgressReporter}
import org.enso.cli.task.{
ProgressNotification,
ProgressReporter,
TaskProgressImplementation
}
import org.enso.distribution.ProgressAndLockNotificationForwarder
import org.enso.distribution.locking.LockUserInterface
import org.enso.editions.LibraryName
@ -12,6 +17,8 @@ import org.enso.jsonrpc.{Id, Request, ResponseError, ResponseResult, Unused}
import org.enso.languageserver.filemanager.FileManagerApi.FileSystemError
import org.enso.languageserver.libraries.LibraryApi._
import org.enso.languageserver.libraries.handler.LibraryPreinstallHandler.{
DependencyGatheringError,
InstallationError,
InstallationResult,
InstallerError,
InternalError
@ -19,19 +26,21 @@ import org.enso.languageserver.libraries.handler.LibraryPreinstallHandler.{
import org.enso.languageserver.libraries.{
EditionReference,
EditionReferenceResolver,
LibraryInstallerConfig
LibraryConfig
}
import org.enso.languageserver.util.UnhandledLogging
import org.enso.librarymanager.ResolvingLibraryProvider.Error
import org.enso.librarymanager.dependencies.{Dependency, DependencyResolver}
import org.enso.librarymanager.{
DefaultLibraryProvider,
LibraryResolver,
ResolvedLibrary,
ResolvingLibraryProvider
}
import java.util.concurrent.Executors
import scala.concurrent.{ExecutionContext, Future}
import scala.util.Try
import scala.util.{Success, Try}
/** A request handler for the `library/preinstall` endpoint.
*
@ -41,11 +50,11 @@ import scala.util.Try
* to select a reasonable timeout.
*
* @param editionReferenceResolver an [[EditionReferenceResolver]] instance
* @param installerConfig configuration for the library installer
* @param config configuration for the library subsystem
*/
class LibraryPreinstallHandler(
editionReferenceResolver: EditionReferenceResolver,
installerConfig: LibraryInstallerConfig
config: LibraryConfig
) extends Actor
with LazyLogging
with UnhandledLogging {
@ -71,23 +80,84 @@ class LibraryPreinstallHandler(
.translateProgressNotification(LibraryPreinstall.name, notification)
}
val installation: Future[InstallationResult] = Future {
val result = for {
libraryInstaller <- getLibraryProvider(
notificationForwarder
).toEither.left.map(InternalError)
library <- libraryInstaller
.findLibrary(libraryName)
.left
.map(InstallerError)
} yield library
InstallationResult(result)
}
val installation: Future[InstallationResult] =
installLibraryWithDependencies(libraryName, notificationForwarder)
installation pipeTo self
context.become(responseStage(id, replyTo, libraryName))
}
/** Returns a future that will be completed once all dependencies of the
* library have been installed.
*
* @param libraryName name of the library to install
* @param notificationForwarder a notification handler for reporting progress
*/
private def installLibraryWithDependencies(
libraryName: LibraryName,
notificationForwarder: ProgressAndLockNotificationForwarder
): Future[InstallationResult] = Future {
val result = for {
tools <- instantiateTools(notificationForwarder).toEither.left
.map(InternalError)
dependencies <- tools.dependencyResolver
.findDependencies(libraryName)
.toEither
.left
.map(DependencyGatheringError)
dependenciesToInstall = dependencies.filter(!_.isCached)
_ <- installDependencies(
dependenciesToInstall,
notificationForwarder,
tools.libraryInstaller
)
library <- tools.libraryInstaller
.findLibrary(libraryName)
.left
.map(InstallerError)
} yield library
InstallationResult(result)
}
/** Installs the provided dependencies and reports the overall progress. */
private def installDependencies(
dependencies: Set[Dependency],
notificationForwarder: ProgressAndLockNotificationForwarder,
libraryInstaller: ResolvingLibraryProvider
): Either[InstallationError, Unit] = {
logger.trace(s"Dependencies to install: $dependencies.")
val taskProgress = new TaskProgressImplementation[Unit]()
val message =
if (dependencies.size == 1) s"Installing 1 library."
else s"Installing ${dependencies.size} libraries."
notificationForwarder.trackProgress(
message,
taskProgress
)
val total = Some(dependencies.size.toLong)
taskProgress.reportProgress(0, total)
val results =
dependencies.toList.zipWithIndex.traverse { case (dependency, ix) =>
val result = libraryInstaller.findSpecificLibraryVersion(
dependency.libraryName,
dependency.version
)
taskProgress.reportProgress(ix.toLong + 1, total)
result
}
taskProgress.setComplete(Success(()))
results.map { _ => () }.left.map(InstallerError)
}
private def responseStage(
requestId: Id,
replyTo: ActorRef,
@ -99,6 +169,8 @@ class LibraryPreinstallHandler(
val errorMessage = error match {
case InternalError(throwable) =>
FileSystemError(s"Internal error: ${throwable.getMessage}")
case DependencyGatheringError(throwable) =>
DependencyDiscoveryError(throwable.getMessage)
case InstallerError(Error.NotResolved(_)) =>
LibraryNotResolved(libraryName)
case InstallerError(Error.RequestedLocalLibraryDoesNotExist) =>
@ -120,23 +192,41 @@ class LibraryPreinstallHandler(
self ! Left(InternalError(throwable))
}
private def getLibraryProvider(
case class Tools(
libraryInstaller: ResolvingLibraryProvider,
dependencyResolver: DependencyResolver
)
/** A helper function that creates instances if the library installer and
* dependency resolver that report to the provided notification forwarder.
*/
private def instantiateTools(
notificationReporter: ProgressReporter with LockUserInterface
): Try[ResolvingLibraryProvider] =
): Try[Tools] =
for {
config <- editionReferenceResolver.getCurrentProjectConfig
projectConfig <- editionReferenceResolver.getCurrentProjectConfig
edition <- editionReferenceResolver.resolveEdition(
EditionReference.CurrentProjectEdition
)
} yield DefaultLibraryProvider.make(
distributionManager = installerConfig.distributionManager,
resourceManager = installerConfig.resourceManager,
lockUserInterface = notificationReporter,
progressReporter = notificationReporter,
languageHome = installerConfig.languageHome,
edition = edition,
preferLocalLibraries = config.preferLocalLibraries
)
preferLocalLibraries = projectConfig.preferLocalLibraries
installer = DefaultLibraryProvider.make(
distributionManager = config.installerConfig.distributionManager,
resourceManager = config.installerConfig.resourceManager,
lockUserInterface = notificationReporter,
progressReporter = notificationReporter,
languageHome = config.installerConfig.languageHome,
edition = edition,
preferLocalLibraries = preferLocalLibraries
)
dependencyResolver = new DependencyResolver(
localLibraryProvider = config.localLibraryProvider,
publishedLibraryProvider = config.publishedLibraryCache,
edition = edition,
preferLocalLibraries = preferLocalLibraries,
versionResolver = LibraryResolver(config.localLibraryProvider),
dependencyExtractor = config.installerConfig.dependencyExtractor
)
} yield Tools(installer, dependencyResolver)
}
object LibraryPreinstallHandler {
@ -144,13 +234,13 @@ object LibraryPreinstallHandler {
/** Creates a configuration object to create [[LibraryPreinstallHandler]].
*
* @param editionReferenceResolver an [[EditionReferenceResolver]] instance
* @param installerConfig configuration for the library installer
* @param config configuration for the library subsystem
*/
def props(
editionReferenceResolver: EditionReferenceResolver,
installerConfig: LibraryInstallerConfig
config: LibraryConfig
): Props = Props(
new LibraryPreinstallHandler(editionReferenceResolver, installerConfig)
new LibraryPreinstallHandler(editionReferenceResolver, config)
)
/** An internal message used to pass the installation result from the Future
@ -180,4 +270,10 @@ object LibraryPreinstallHandler {
* could not be established.
*/
case class InstallerError(error: Error) extends InstallationError
/** Indicates an error that occurred when looking for all of the transitive
* dependencies of the library.
*/
case class DependencyGatheringError(throwable: Throwable)
extends InstallationError
}

11
engine/language-server/src/main/scala/org/enso/languageserver/libraries/handler/LibraryPublishHandler.scala
View File

@ -7,15 +7,19 @@ import org.enso.cli.task.notifications.ActorProgressNotificationForwarder
import org.enso.editions.LibraryName
import org.enso.jsonrpc._
import org.enso.languageserver.filemanager.FileManagerApi.FileSystemError
import org.enso.languageserver.libraries.BlockingOperation
import org.enso.languageserver.libraries.LibraryApi._
import org.enso.languageserver.libraries.LocalLibraryManagerProtocol.{
FindLibrary,
FindLibraryResponse
}
import org.enso.languageserver.libraries.{
BlockingOperation,
CompilerBasedDependencyExtractor
}
import org.enso.languageserver.requesthandler.RequestTimeout
import org.enso.languageserver.util.UnhandledLogging
import org.enso.libraryupload.{auth, LibraryUploader}
import org.enso.loggingservice.LoggingServiceManager
import scala.concurrent.Future
import scala.concurrent.duration.FiniteDuration
@ -98,7 +102,10 @@ class LibraryPublishHandler(
)
val future: Future[UploadSucceeded] = BlockingOperation.run {
LibraryUploader
val logLevel = LoggingServiceManager.currentLogLevelForThisApplication()
val dependencyExtractor =
new CompilerBasedDependencyExtractor(logLevel)
LibraryUploader(dependencyExtractor)
.uploadLibrary(
libraryRoot,
uploadUrl,

6
engine/language-server/src/main/scala/org/enso/languageserver/protocol/json/JsonConnectionController.scala
View File

@ -513,10 +513,8 @@ class JsonConnectionController(
.props(requestTimeout, libraryConfig.localLibraryManager),
LibraryGetMetadata -> LibraryGetMetadataHandler
.props(requestTimeout, libraryConfig.localLibraryManager),
LibraryPreinstall -> LibraryPreinstallHandler.props(
libraryConfig.editionReferenceResolver,
libraryConfig.installerConfig
),
LibraryPreinstall -> LibraryPreinstallHandler
.props(libraryConfig.editionReferenceResolver, libraryConfig),
LibraryPublish -> LibraryPublishHandler
.props(requestTimeout, libraryConfig.localLibraryManager),
LibrarySetMetadata -> LibrarySetMetadataHandler

4
engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/BaseServerTest.scala
View File

@ -37,6 +37,7 @@ import org.enso.languageserver.text.BufferRegistry
import org.enso.librarymanager.LibraryLocations
import org.enso.librarymanager.local.DefaultLocalLibraryProvider
import org.enso.librarymanager.published.PublishedLibraryCache
import org.enso.loggingservice.LogLevel
import org.enso.pkg.PackageManager
import org.enso.polyglot.data.TypeGraph
import org.enso.polyglot.runtime.Runtime.Api
@ -277,7 +278,8 @@ class BaseServerTest
installerConfig = LibraryInstallerConfig(
distributionManager,
resourceManager,
Some(languageHome)
Some(languageHome),
new CompilerBasedDependencyExtractor(logLevel = LogLevel.Warning)
)
)

60
engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/LibrariesTest.scala
View File

@ -3,22 +3,39 @@ package org.enso.languageserver.websocket.json
import io.circe.literal._
import io.circe.{Json, JsonObject}
import nl.gn0s1s.bump.SemVer
import org.enso.distribution.FileSystem
import org.enso.editions.{Editions, LibraryName}
import org.enso.languageserver.libraries.LibraryEntry
import org.enso.languageserver.libraries.LibraryEntry.PublishedLibraryVersion
import org.enso.librarymanager.published.bundles.LocalReadOnlyRepository
import org.enso.librarymanager.published.repository.{
EmptyRepository,
ExampleRepository
ExampleRepository,
LibraryManifest
}
import org.enso.pkg.{Contact, PackageManager}
import org.enso.yaml.YamlHelper
import java.nio.file.Files
class LibrariesTest extends BaseServerTest {
private val libraryRepositoryPort: Int = 47308
private val exampleRepo = new ExampleRepository
private val exampleRepo = new ExampleRepository {
override def libraries: Seq[DummyLibrary] = Seq(
DummyLibrary(
LibraryName("Foo", "Bar"),
SemVer(1, 0, 0),
"""import Standard.Base
|
|baz = 42
|
|quux = "foobar"
|""".stripMargin,
dependencies = Seq(LibraryName("Standard", "Base"))
)
)
}
private val baseUrl = s"http://localhost:$libraryRepositoryPort/"
private val repositoryUrl = baseUrl + "libraries"
@ -232,6 +249,21 @@ class LibrariesTest extends BaseServerTest {
}
""")
// Update Main.enso
val libraryRoot = getTestDirectory
.resolve("test_home")
.resolve("libraries")
.resolve("user")
.resolve("Publishable_Lib")
val mainSource = libraryRoot.resolve("src").resolve("Main.enso")
FileSystem.writeTextFile(
mainSource,
"""import Some.Other_Library
|
|main = 42
|""".stripMargin
)
client.send(json"""
{ "jsonrpc": "2.0",
"method": "library/setMetadata",
@ -308,6 +340,18 @@ class LibrariesTest extends BaseServerTest {
Contact(name = Some("only-name"), email = None),
Contact(name = None, email = Some("foo@example.com"))
)
val manifest = YamlHelper
.load[LibraryManifest](
libraryRoot.resolve(LibraryManifest.filename)
)
.get
manifest.archives shouldEqual Seq("main.tgz")
manifest.dependencies shouldEqual Seq(
LibraryName("Some", "Other_Library")
)
manifest.description shouldEqual Some("Description for publication.")
manifest.tagLine shouldEqual Some("published-lib")
client.send(json"""
{ "jsonrpc": "2.0",
@ -387,6 +431,9 @@ class LibrariesTest extends BaseServerTest {
msg("id") match {
case Some(json) =>
json.asNumber.value.toInt.value shouldEqual requestId
msg("error").foreach(err =>
println("Request ended with error: " + err)
)
msg("result").value.asNull.value
waitingForResult = false
case None =>
@ -408,8 +455,6 @@ class LibrariesTest extends BaseServerTest {
.asString
.value shouldEqual "library/preinstall"
taskStart._2("unit").value.asString.value shouldEqual "Bytes"
val updates = messages.filter { case (method, params) =>
method == "task/progress-update" &&
params("taskId").value.asString.value == taskId
@ -417,7 +462,7 @@ class LibrariesTest extends BaseServerTest {
updates should not be empty
updates.head._2("message").value.asString.value should include(
"Downloading"
"Installing"
)
val cachePath = getTestDirectory.resolve("test_data").resolve("lib")
@ -435,6 +480,11 @@ class LibrariesTest extends BaseServerTest {
pkg.listSources.map(
_.file.getName
) should contain theSameElementsAs Seq("Main.enso")
assert(
Files.exists(cachedLibraryRoot.resolve(LibraryManifest.filename)),
"The manifest file of a downloaded library should be saved in the cache too."
)
}
}
}

9
engine/launcher/src/main/scala/org/enso/launcher/Constants.scala
View File

@ -12,6 +12,15 @@ object Constants {
val uploadIntroducedVersion: SemVer =
SemVer(0, 2, 17, Some("SNAPSHOT"))
/** The engine version in which the dependency preinstall command has been
* introduced.
*
* It is used to check by the launcher if the engine can handle this command
* and provide better error messages if it cannot.
*/
val preinstallDependenciesIntroducedVersion: SemVer =
SemVer(0, 2, 28, Some("SNAPSHOT"))
/** The upload URL associated with the main Enso library repository. */
val defaultUploadUrl = "https://publish.libraries.release.enso.org/"
}

39
engine/launcher/src/main/scala/org/enso/launcher/Launcher.scala
View File

@ -293,7 +293,6 @@ case class Launcher(cliOptions: GlobalCLIOptions) {
contentRoot: Path,
versionOverride: Option[SemVer],
logLevel: LogLevel,
logMasking: Boolean,
useSystemJVM: Boolean,
jvmOpts: Seq[(String, String)],
additionalArguments: Seq[String]
@ -306,7 +305,7 @@ case class Launcher(cliOptions: GlobalCLIOptions) {
contentRoot,
versionOverride,
logLevel,
logMasking,
logMasking = cliOptions.internalOptions.logMasking,
additionalArguments
)
.get,
@ -317,6 +316,42 @@ case class Launcher(cliOptions: GlobalCLIOptions) {
exitCode
}
/** Runs the engine associated with the project in dependency installation
* mode.
*
* @param versionOverride if provided, overrides the default engine version
* that would have been used
* @param logLevel log level for the language server
* @param useSystemJVM if set, forces to use the default configured JVM,
* instead of the JVM associated with the engine version
* @param jvmOpts additional options to pass to the launched JVM
* @param additionalArguments additional arguments to pass to the runner
* @return exit code of the launched program
*/
def runInstallDependencies(
versionOverride: Option[SemVer],
logLevel: LogLevel,
useSystemJVM: Boolean,
jvmOpts: Seq[(String, String)],
additionalArguments: Seq[String]
): Int = {
val exitCode = runner.withCommand(
runner
.installDependencies(
versionOverride,
hideProgress = cliOptions.hideProgress,
logLevel,
logMasking = cliOptions.internalOptions.logMasking,
additionalArguments = additionalArguments
)
.get,
JVMSettings(useSystemJVM, jvmOpts)
) { command =>
command.run().get
}
exitCode
}
/** Updates the global configuration.
*
* If `value` is an empty string, the `key` is removed from the configuration

44
engine/launcher/src/main/scala/org/enso/launcher/cli/LauncherApplication.scala
View File

@ -124,7 +124,7 @@ object LauncherApplication {
"(error | warning | info | debug | trace)",
"Sets logging verbosity for the engine. Defaults to info."
)
.withDefault(LogLevel.Warning)
.withDefault(LogLevel.Info)
}
private def runCommand: Command[Config => Int] =
@ -244,8 +244,7 @@ object LauncherApplication {
useSystemJVM = systemJVMOverride,
jvmOpts = jvmOpts,
additionalArguments = additionalArgs,
logLevel = engineLogLevel,
logMasking = config.internalOptions.logMasking
logLevel = engineLogLevel
)
}
}
@ -441,12 +440,47 @@ object LauncherApplication {
}
}
private def installDependenciesCommand: Command[Config => Int] =
Command(
"dependencies",
"Install dependencies of the current project."
) {
val additionalArgs = Opts.additionalArguments()
(
versionOverride,
engineLogLevel,
systemJVMOverride,
jvmOpts,
additionalArgs
) mapN {
(
versionOverride,
engineLogLevel,
systemJVMOverride,
jvmOpts,
additionalArgs
) => (config: Config) =>
Launcher(config).runInstallDependencies(
versionOverride,
engineLogLevel,
useSystemJVM = systemJVMOverride,
jvmOpts,
additionalArgs
)
}
}
private def installCommand: Command[Config => Int] =
Command(
"install",
"Install a new version of engine or install the distribution locally."
"Install a new version of engine, install the distribution locally or " +
"install project dependencies."
) {
Opts.subcommands(installEngineCommand, installDistributionCommand)
Opts.subcommands(
installEngineCommand,
installDistributionCommand,
installDependenciesCommand
)
}
private def uninstallEngineCommand: Command[Config => Int] =

45
engine/launcher/src/main/scala/org/enso/launcher/components/LauncherRunner.scala
View File

@ -241,4 +241,49 @@ class LauncherRunner(
connectLoggerIfAvailable = true
)
}
/** Creates [[RunSettings]] for installing project dependencies.
*
* See [[org.enso.launcher.Launcher.runInstallDependencies]] for more
* details.
*/
def installDependencies(
versionOverride: Option[SemVer],
hideProgress: Boolean,
logLevel: LogLevel,
logMasking: Boolean,
additionalArguments: Seq[String]
): Try[RunSettings] =
Try {
val actualPath = currentWorkingDirectory
val project = projectManager.findProject(actualPath).get.getOrElse {
throw RunnerError(
s"Could not find a project at " +
s"${MaskedPath(actualPath).applyMasking()} or any of its parent " +
s"directories."
)
}
val version = resolveVersion(versionOverride, Some(project))
if (version < Constants.preinstallDependenciesIntroducedVersion) {
throw RunnerError(
s"Project dependency installation feature is not available in Enso " +
s"$version. Please upgrade your project to a newer version to use it."
)
}
val hideProgressOpts =
if (hideProgress) Seq("--hide-progress") else Seq.empty
val arguments =
Seq("--preinstall-dependencies") ++
Seq("--in-project", project.path.toAbsolutePath.normalize.toString) ++
hideProgressOpts
RunSettings(
version,
arguments ++ setLogLevelArgs(logLevel, logMasking)
++ additionalArguments,
connectLoggerIfAvailable = true
)
}
}

1
engine/polyglot-api/src/main/java/org/enso/polyglot/MethodNames.java
View File

@ -19,6 +19,7 @@ public class MethodNames {
public static final String GET_NAME = "get_name";
public static final String REPARSE = "reparse";
public static final String GENERATE_DOCS = "generate_docs";
public static final String GATHER_IMPORT_STATEMENTS = "gather_import_statements";
public static final String SET_SOURCE = "set_source";
public static final String SET_SOURCE_FILE = "set_source_file";
}

10
engine/polyglot-api/src/main/scala/org/enso/polyglot/Module.scala
View File

@ -57,6 +57,16 @@ class Module(private val value: Value) {
value.invokeMember(GENERATE_DOCS)
}
/** Triggers gathering of import statements from module sources.
*
* @return value with `GATHER_IMPORT_STATEMENTS` invoked on it.
*/
def gatherImportStatements(): Seq[String] = {
val array = value.invokeMember(GATHER_IMPORT_STATEMENTS)
val size = array.getArraySize
for (i <- 0L until size) yield array.getArrayElement(i).asString()
}
/** Triggers reparsing of module sources. Used to notify the module that
* sources have changed.
*/

14
engine/polyglot-api/src/test/scala/org/enso/polyglot/ModuleManagementTest.scala
View File

@ -188,4 +188,18 @@ class ModuleManagementTest extends AnyFlatSpec with Matchers {
the[PolyglotException] thrownBy mod2.getAssociatedConstructor
exception.getMessage shouldEqual "Compilation aborted due to errors."
}
subject should "allow gathering imported libraries" in {
val ctx = new TestContext("Test")
ctx.writeMain("""
|import Foo.Bar.Baz
|
|main = 42
|""".stripMargin)
val topScope = ctx.executionContext.getTopScope
val mainModule = topScope.getModule("Enso_Test.Test.Main")
val imports = mainModule.gatherImportStatements()
imports shouldEqual Seq("Foo.Bar")
}
}

129
engine/runner/src/main/scala/org/enso/runner/DependencyPreinstaller.scala Normal file
View File

@ -0,0 +1,129 @@
package org.enso.runner
import cats.implicits.toTraverseOps
import com.typesafe.scalalogging.Logger
import org.enso.cli.ProgressBar
import org.enso.cli.task.{ProgressReporter, TaskProgress}
import org.enso.distribution.locking.{
LockUserInterface,
Resource,
ResourceManager,
ThreadSafeFileLockManager
}
import org.enso.distribution.{DistributionManager, Environment, LanguageHome}
import org.enso.editions.updater.EditionManager
import org.enso.editions.{DefaultEdition, EditionResolver}
import org.enso.languageserver.libraries.CompilerBasedDependencyExtractor
import org.enso.librarymanager.dependencies.DependencyResolver
import org.enso.librarymanager.{DefaultLibraryProvider, LibraryResolver}
import org.enso.loggingservice.LogLevel
import org.enso.pkg.PackageManager
import java.io.File
/** A helper to preinstall all dependencies of a project. */
object DependencyPreinstaller {
/** Parses the project to find out its direct dependencies, uses the resolver
* to find all transitive dependencies and ensures that all of them are
* installed.
*/
def preinstallDependencies(projectRoot: File, logLevel: LogLevel): Unit = {
val logger = Logger[DependencyPreinstaller.type]
val pkg = PackageManager.Default.loadPackage(projectRoot).get
val dependencyExtractor = new CompilerBasedDependencyExtractor(logLevel)
val environment = new Environment {}
val languageHome = LanguageHome.detectFromExecutableLocation(environment)
val distributionManager = new DistributionManager(environment)
val lockManager = new ThreadSafeFileLockManager(
distributionManager.paths.locks
)
val resourceManager = new ResourceManager(lockManager)
val editionProvider = EditionManager.makeEditionProvider(
distributionManager,
Some(languageHome)
)
val editionResolver = EditionResolver(editionProvider)
val edition = editionResolver
.resolve(
pkg.config.edition.getOrElse(DefaultEdition.getDefaultEdition)
) match {
case Left(error) =>
throw new RuntimeException(
s"Cannot resolve current project's edition: ${error.getMessage}"
)
case Right(value) => value
}
val preferLocalLibraries = pkg.config.preferLocalLibraries
val (localLibraryProvider, publishedLibraryProvider) =
DefaultLibraryProvider.makeProviders(
distributionManager,
resourceManager,
new LockUserInterface {
override def startWaitingForResource(resource: Resource): Unit =
logger.warn(resource.waitMessage)
override def finishWaitingForResource(resource: Resource): Unit = ()
},
new ProgressReporter {
override def trackProgress(
message: String,
task: TaskProgress[_]
): Unit = {
logger.info(message)
ProgressBar.waitWithProgress(task)
}
},
Some(languageHome)
)
val dependencyResolver = new DependencyResolver(
localLibraryProvider,
publishedLibraryProvider,
edition,
preferLocalLibraries,
LibraryResolver(localLibraryProvider),
dependencyExtractor
)
val installer = new DefaultLibraryProvider(
localLibraryProvider,
publishedLibraryProvider,
edition,
preferLocalLibraries
)
val immediateDependencies = dependencyExtractor.findDependencies(pkg)
logger.trace(
s"The project imports the following libraries: $immediateDependencies."
)
val allDependencies = immediateDependencies.flatMap { name =>
dependencyResolver.findDependencies(name).get
}
logger.trace(s"The project depends on: $allDependencies.")
val dependenciesToInstall = allDependencies.filter(!_.isCached)
if (dependenciesToInstall.isEmpty) {
logger.info(s"All ${allDependencies.size} dependencies are installed.")
} else {
logger.info(s"Will install ${dependenciesToInstall.size} dependencies.")
val result = dependenciesToInstall.toList.traverse { dependency =>
installer.findSpecificLibraryVersion(
dependency.libraryName,
dependency.version
)
}
result match {
case Left(error) =>
throw new RuntimeException(
s"Some dependencies could not be installed: [$error]."
)
case Right(_) =>
}
}
}
}

68
engine/runner/src/main/scala/org/enso/runner/Main.scala
View File

@ -21,6 +21,7 @@ import java.util.UUID
import scala.Console.err
import scala.jdk.CollectionConverters._
import scala.util.Try
import scala.util.control.NonFatal
/** The main CLI entry point class. */
object Main {
@ -34,6 +35,7 @@ object Main {
private val PROJECT_AUTHOR_EMAIL_OPTION = "new-project-author-email"
private val REPL_OPTION = "repl"
private val DOCS_OPTION = "docs"
private val PREINSTALL_OPTION = "preinstall-dependencies"
private val LANGUAGE_SERVER_OPTION = "server"
private val DAEMONIZE_OPTION = "daemon"
private val INTERFACE_OPTION = "interface"
@ -54,6 +56,7 @@ object Main {
private val LOGGER_CONNECT = "logger-connect"
private val NO_LOG_MASKING = "no-log-masking"
private val UPLOAD_OPTION = "upload"
private val UPDATE_MANIFEST_OPTION = "update-manifest"
private val HIDE_PROGRESS = "hide-progress"
private val AUTH_TOKEN = "auth-token"
private val AUTO_PARALLELISM_OPTION = "with-auto-parallelism"
@ -89,6 +92,10 @@ object Main {
.longOpt(DOCS_OPTION)
.desc("Runs the Enso documentation generator.")
.build
val preinstall = CliOption.builder
.longOpt(PREINSTALL_OPTION)
.desc("Installs dependencies of the project.")
.build
val newOpt = CliOption.builder
.hasArg(true)
.numberOfArgs(1)
@ -230,6 +237,13 @@ object Main {
"The url defines the repository to upload to."
)
.build()
val updateManifestOption = CliOption.builder
.longOpt(UPDATE_MANIFEST_OPTION)
.desc(
"Updates the library manifest with the updated list of direct " +
"dependencies."
)
.build()
val hideProgressOption = CliOption.builder
.longOpt(HIDE_PROGRESS)
.desc("If specified, progress bars will not be displayed.")
@ -299,6 +313,7 @@ object Main {
.addOption(repl)
.addOption(run)
.addOption(docs)
.addOption(preinstall)
.addOption(newOpt)
.addOption(newProjectNameOpt)
.addOption(newProjectTemplateOpt)
@ -318,6 +333,7 @@ object Main {
.addOption(loggerConnectOption)
.addOption(noLogMaskingOption)
.addOption(uploadOption)
.addOption(updateManifestOption)
.addOption(hideProgressOption)
.addOption(authTokenOption)
.addOption(noReadIrCachesOption)
@ -577,6 +593,34 @@ object Main {
}
}
/** Handles the `--preinstall-dependencies` CLI option.
*
* Gathers imported dependencies and ensures that all of them are installed.
*
* @param projectPath path of the project
* @param logLevel log level to set for the engine runtime
*/
private def preinstallDependencies(
projectPath: Option[String],
logLevel: LogLevel
): Unit = projectPath match {
case Some(path) =>
try {
DependencyPreinstaller.preinstallDependencies(new File(path), logLevel)
exitSuccess()
} catch {
case NonFatal(error) =>
logger.error(
s"Dependency installation failed: ${error.getMessage}",
error
)
exitFail()
}
case None =>
println("Dependency installation is only available for projects.")
exitFail()
}
private def runPackage(
context: PolyglotContext,
mainModuleName: String,
@ -850,7 +894,8 @@ object Main {
projectRoot = projectRoot,
uploadUrl = line.getOptionValue(UPLOAD_OPTION),
authToken = Option(line.getOptionValue(AUTH_TOKEN)),
showProgress = !line.hasOption(HIDE_PROGRESS)
showProgress = !line.hasOption(HIDE_PROGRESS),
logLevel = logLevel
)
exitSuccess()
} catch {
@ -861,6 +906,21 @@ object Main {
}
}
if (line.hasOption(UPDATE_MANIFEST_OPTION)) {
val projectRoot =
Option(line.getOptionValue(IN_PROJECT_OPTION))
.map(Path.of(_))
.getOrElse {
logger.error(
s"The $IN_PROJECT_OPTION is mandatory."
)
exitFail()
}
ProjectUploader.updateManifest(projectRoot, logLevel)
exitSuccess()
}
if (line.hasOption(COMPILE_OPTION)) {
val packagePaths = line.getOptionValue(COMPILE_OPTION)
val shouldCompileDependencies =
@ -902,6 +962,12 @@ object Main {
shouldEnableIrCaches(line)
)
}
if (line.hasOption(PREINSTALL_OPTION)) {
preinstallDependencies(
Option(line.getOptionValue(IN_PROJECT_OPTION)),
logLevel
)
}
if (line.hasOption(LANGUAGE_SERVER_OPTION)) {
runLanguageServer(line, logLevel)
}

26
engine/runner/src/main/scala/org/enso/runner/ProjectUploader.scala
View File

@ -3,7 +3,10 @@ package org.enso.runner
import com.typesafe.scalalogging.Logger
import org.enso.cli.ProgressBar
import org.enso.cli.task.{ProgressReporter, TaskProgress}
import org.enso.languageserver.libraries.CompilerBasedDependencyExtractor
import org.enso.libraryupload.{auth, LibraryUploader}
import org.enso.loggingservice.LogLevel
import org.enso.pkg.PackageManager
import java.nio.file.Path
@ -20,12 +23,15 @@ object ProjectUploader {
* repository
* @param showProgress specifies if CLI progress bars should be displayed
* showing progress of compression and upload
* @param logLevel the log level to use for the context gathering
* dependencies
*/
def uploadProject(
projectRoot: Path,
uploadUrl: String,
authToken: Option[String],
showProgress: Boolean
showProgress: Boolean,
logLevel: LogLevel
): Unit = {
import scala.concurrent.ExecutionContext.Implicits.global
val progressReporter = new ProgressReporter {
@ -44,7 +50,10 @@ object ProjectUploader {
case Some(value) => auth.SimpleHeaderToken(value)
case None => auth.NoAuthorization
}
LibraryUploader
val dependencyExtractor = new CompilerBasedDependencyExtractor(logLevel)
LibraryUploader(dependencyExtractor)
.uploadLibrary(
projectRoot,
uploadUrl,
@ -53,4 +62,17 @@ object ProjectUploader {
)
.get
}
/** Updates manifest of the project.
*
* @param projectRoot path to the root of the project
* @param logLevel the log level to use for the context gathering
* dependencies
*/
def updateManifest(projectRoot: Path, logLevel: LogLevel): Unit = {
val pkg = PackageManager.Default.loadPackage(projectRoot.toFile).get
val dependencyExtractor = new CompilerBasedDependencyExtractor(logLevel)
LibraryUploader(dependencyExtractor).updateManifest(pkg).get
}
}

13
engine/runtime/src/main/java/org/enso/interpreter/runtime/Module.java
View File

@ -488,6 +488,11 @@ public class Module implements TruffleObject {
return context.getCompiler().generateDocs(module);
}
private static Object gatherImportStatements(Module module, Context context) {
Object[] imports = context.getCompiler().gatherImportStatements(module);
return new Array(imports);
}
@Specialization
static Object doInvoke(
Module module,
@ -496,26 +501,32 @@ public class Module implements TruffleObject {
@CachedContext(Language.class) Context context,
@Cached LoopingCallOptimiserNode callOptimiserNode)
throws UnknownIdentifierException, ArityException, UnsupportedTypeException {
ModuleScope scope = module.compileScope(context);
ModuleScope scope;
switch (member) {
case MethodNames.Module.GET_NAME:
return module.getName().toString();
case MethodNames.Module.GET_METHOD:
scope = module.compileScope(context);
Function result = getMethod(scope, arguments);
return result == null ? context.getBuiltins().nothing().newInstance() : result;
case MethodNames.Module.GET_CONSTRUCTOR:
scope = module.compileScope(context);
return getConstructor(scope, arguments);
case MethodNames.Module.REPARSE:
return reparse(module, arguments, context);
case MethodNames.Module.GENERATE_DOCS:
return generateDocs(module, context);
case MethodNames.Module.GATHER_IMPORT_STATEMENTS:
return gatherImportStatements(module, context);
case MethodNames.Module.SET_SOURCE:
return setSource(module, arguments, context);
case MethodNames.Module.SET_SOURCE_FILE:
return setSourceFile(module, arguments, context);
case MethodNames.Module.GET_ASSOCIATED_CONSTRUCTOR:
scope = module.compileScope(context);
return getAssociatedConstructor(scope, arguments);
case MethodNames.Module.EVAL_EXPRESSION:
scope = module.compileScope(context);
return evalExpression(scope, arguments, context, callOptimiserNode);
default:
throw UnknownIdentifierException.create(member);

40
engine/runtime/src/main/scala/org/enso/compiler/Compiler.scala
View File

@ -15,6 +15,7 @@ import org.enso.compiler.phase.{
ExportsResolution,
ImportResolver
}
import org.enso.editions.LibraryName
import org.enso.interpreter.node.{ExpressionNode => RuntimeExpression}
import org.enso.interpreter.runtime.builtin.Builtins
import org.enso.interpreter.runtime.scope.{LocalScope, ModuleScope}
@ -321,6 +322,32 @@ class Compiler(
requiredModules
}
/** Runs the initial passes of the compiler to gather the import statements,
* used for dependency resolution.
*
* @param module - the scope from which docs are generated.
*/
def gatherImportStatements(module: Module): Array[String] = {
ensureParsed(module)
val importedModules = module.getIr.imports.flatMap {
case imp: IR.Module.Scope.Import.Module =>
imp.name.parts.take(2).map(_.name) match {
case List(namespace, name) => List(LibraryName(namespace, name))
case _ =>
throw new CompilerError(s"Invalid module name: [${imp.name}].")
}
case _: IR.Module.Scope.Import.Polyglot =>
// Note [Polyglot Imports In Dependency Gathering]
Nil
case other =>
throw new CompilerError(
s"Unexpected import type after processing: [$other]."
)
}
importedModules.distinct.map(_.qualifiedName).toArray
}
private def parseModule(
module: Module,
isGenDocs: Boolean = false
@ -364,6 +391,19 @@ class Compiler(
module.setHasCrossModuleLinks(true)
}
/* Note [Polyglot Imports In Dependency Gathering]
* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
* Currently we just ignore polyglot imports when gathering the dependencies -
* we assume that the project itself or one of its dependencies will contain
* in their `polyglot` directory any JARs that need to be included in the
* classpath for this import to be resolved.
*
* In the future we may want to extend the edition system with some settings
* for automatically resolving the Java dependencies using a system based on
* Maven, but currently the libraries just must include their binary
* dependencies.
*/
/** Gets a module definition by name.
*
* @param name the name of module to look up

2
engine/runtime/src/main/scala/org/enso/compiler/PackageRepository.scala
View File

@ -244,7 +244,7 @@ object PackageRepository {
case Left(error) =>
logger.error(s"Resolution failed with [$error].", error)
case Right(resolved) =>
logger.trace(
logger.info(
s"Found library ${resolved.name} @ ${resolved.version} " +
s"at [${MaskedPath(resolved.location).applyMasking()}]."
)

25
lib/scala/library-manager-test/src/main/scala/org/enso/librarymanager/published/repository/DummyRepository.scala
View File

@ -24,11 +24,14 @@ abstract class DummyRepository {
* @param libraryName name of the library
* @param version version of the library
* @param mainContent contents of the `Main.enso` file
* @param dependencies libraries that this library directly depends on, to be
* included in the manifest
*/
case class DummyLibrary(
libraryName: LibraryName,
version: SemVer,
mainContent: String
mainContent: String,
dependencies: Seq[LibraryName] = Seq.empty
)
/** Name of the repository, as it will be indicated in the generated edition.
@ -62,7 +65,7 @@ abstract class DummyRepository {
}
.get
createManifest(libraryRoot)
createManifest(libraryRoot, lib)
}
val editionsRoot = root.resolve("editions")
@ -96,12 +99,22 @@ abstract class DummyRepository {
pkg
}
private def createManifest(path: Path): Unit = {
FileSystem.writeTextFile(
path.resolve("manifest.yaml"),
private def createManifest(path: Path, lib: DummyLibrary): Unit = {
val dependencies =
if (lib.dependencies.isEmpty) ""
else
lib.dependencies
.map(name => s""" - "${name.qualifiedName}"""")
.mkString("dependencies:\n", "\n", "\n")
val content =
s"""archives:
| - main.tgz
|""".stripMargin
|""".stripMargin + dependencies
FileSystem.writeTextFile(
path.resolve("manifest.yaml"),
content
)
}

9
lib/scala/library-manager-test/src/test/scala/org/enso/libraryupload/LibraryUploadTest.scala
View File

@ -8,11 +8,12 @@ import org.enso.librarymanager.published.repository.{
EmptyRepository
}
import org.enso.libraryupload.auth.SimpleHeaderToken
import org.enso.pkg.PackageManager
import org.enso.pkg.{Package, PackageManager}
import org.enso.testkit.WithTemporaryDirectory
import org.scalatest.matchers.should.Matchers
import org.scalatest.wordspec.AnyWordSpec
import java.io.File
import java.nio.file.Files
class LibraryUploadTest
@ -40,8 +41,12 @@ class LibraryUploadTest
EmptyRepository.withServer(port, repoRoot, uploads = true) {
val uploadUrl = s"http://localhost:$port/upload"
val token = SimpleHeaderToken("TODO")
val dependencyExtractor = new DependencyExtractor[File] {
override def findDependencies(pkg: Package[File]): Set[LibraryName] =
Set(LibraryName("Standard", "Base"))
}
import scala.concurrent.ExecutionContext.Implicits.global
LibraryUploader
LibraryUploader(dependencyExtractor)
.uploadLibrary(
projectRoot,
uploadUrl,

90
lib/scala/library-manager/src/main/scala/org/enso/librarymanager/DefaultLibraryProvider.scala
View File

@ -17,6 +17,7 @@ import org.enso.librarymanager.published.bundles.LocalReadOnlyRepository
import org.enso.librarymanager.published.cache.DownloadingLibraryCache
import org.enso.librarymanager.published.{
DefaultPublishedLibraryProvider,
PublishedLibraryCache,
PublishedLibraryProvider
}
@ -28,7 +29,7 @@ import org.enso.librarymanager.published.{
* @param preferLocalLibraries project setting whether to use local
* libraries
*/
class DefaultLibraryProvider private (
class DefaultLibraryProvider(
localLibraryProvider: LocalLibraryProvider,
publishedLibraryProvider: PublishedLibraryProvider,
edition: Editions.ResolvedEdition,
@ -48,33 +49,43 @@ class DefaultLibraryProvider private (
val resolvedVersion = resolver
.resolveLibraryVersion(libraryName, edition, preferLocalLibraries)
logger.trace(s"Resolved $libraryName to [$resolvedVersion].")
resolvedVersion match {
case Left(reason) =>
Left(ResolvingLibraryProvider.Error.NotResolved(reason))
case Right(LibraryVersion.Local) =>
localLibraryProvider
.findLibrary(libraryName)
.map(ResolvedLibrary(libraryName, LibraryVersion.Local, _))
.toRight {
ResolvingLibraryProvider.Error.NotResolved(
LibraryResolutionError(
s"Edition configuration forces to use the local version, but " +
s"the `$libraryName` library is not present among local " +
s"libraries."
)
)
}
case Right(version @ LibraryVersion.Published(semver, repository)) =>
publishedLibraryProvider
.findLibrary(libraryName, semver, repository)
.map(ResolvedLibrary(libraryName, version, _))
.toEither
.left
.map(ResolvingLibraryProvider.Error.DownloadFailed(version, _))
case Right(version) =>
findSpecificLibraryVersion(libraryName, version)
}
}
/** @inheritdoc */
override def findSpecificLibraryVersion(
libraryName: LibraryName,
version: LibraryVersion
): Either[ResolvingLibraryProvider.Error, ResolvedLibrary] = version match {
case LibraryVersion.Local =>
localLibraryProvider
.findLibrary(libraryName)
.map(ResolvedLibrary(libraryName, LibraryVersion.Local, _))
.toRight {
ResolvingLibraryProvider.Error.NotResolved(
LibraryResolutionError(
s"Edition configuration forces to use the local version, but " +
s"the `$libraryName` library is not present among local " +
s"libraries."
)
)
}
case version @ LibraryVersion.Published(semver, repository) =>
publishedLibraryProvider
.findLibrary(libraryName, semver, repository)
.map(ResolvedLibrary(libraryName, version, _))
.toEither
.left
.map(ResolvingLibraryProvider.Error.DownloadFailed(version, _))
}
}
object DefaultLibraryProvider {
@ -100,6 +111,33 @@ object DefaultLibraryProvider {
edition: Editions.ResolvedEdition,
preferLocalLibraries: Boolean
): ResolvingLibraryProvider = {
val (localLibraryProvider, publishedLibraryProvider) = makeProviders(
distributionManager,
resourceManager,
lockUserInterface,
progressReporter,
languageHome
)
new DefaultLibraryProvider(
localLibraryProvider,
publishedLibraryProvider,
edition,
preferLocalLibraries
)
}
/** Creates a pair of local and published library providers. */
def makeProviders(
distributionManager: DistributionManager,
resourceManager: ResourceManager,
lockUserInterface: LockUserInterface,
progressReporter: ProgressReporter,
languageHome: Option[LanguageHome]
): (
LocalLibraryProvider,
PublishedLibraryProvider with PublishedLibraryCache
) = {
val locations = LibraryLocations.resolve(distributionManager, languageHome)
val primaryCache = new DownloadingLibraryCache(
locations.primaryCacheRoot,
@ -115,12 +153,6 @@ object DefaultLibraryProvider {
new DefaultLocalLibraryProvider(locations.localLibrarySearchPaths)
val publishedLibraryProvider =
new DefaultPublishedLibraryProvider(primaryCache, additionalCaches)
new DefaultLibraryProvider(
localLibraryProvider,
publishedLibraryProvider,
edition,
preferLocalLibraries
)
(localLibraryProvider, publishedLibraryProvider)
}
}

13
lib/scala/library-manager/src/main/scala/org/enso/librarymanager/ResolvingLibraryProvider.scala
View File

@ -16,6 +16,19 @@ trait ResolvingLibraryProvider {
def findLibrary(
name: LibraryName
): Either[ResolvingLibraryProvider.Error, ResolvedLibrary]
/** Locates a specific library version in local libraries or the cache.
*
* If it is not available, a download may be attempted.
*
* @param name name of the library
* @param version requested version of the library
* @return the resolved library containing the resulting version and path
*/
def findSpecificLibraryVersion(
name: LibraryName,
version: LibraryVersion
): Either[ResolvingLibraryProvider.Error, ResolvedLibrary]
}
object ResolvingLibraryProvider {

15
lib/scala/library-manager/src/main/scala/org/enso/librarymanager/dependencies/Dependency.scala Normal file
View File

@ -0,0 +1,15 @@
package org.enso.librarymanager.dependencies
import org.enso.editions.{LibraryName, LibraryVersion}
/** Represents a resolved dependency.
*
* @param libraryName name of the library
* @param version version of the library
* @param isCached whether the library is already present in one of the caches
*/
case class Dependency(
libraryName: LibraryName,
version: LibraryVersion,
isCached: Boolean
)

115
lib/scala/library-manager/src/main/scala/org/enso/librarymanager/dependencies/DependencyResolver.scala Normal file
View File

@ -0,0 +1,115 @@
package org.enso.librarymanager.dependencies
import org.enso.editions.{Editions, LibraryName, LibraryVersion}
import org.enso.librarymanager.LibraryResolver
import org.enso.librarymanager.local.LocalLibraryProvider
import org.enso.librarymanager.published.PublishedLibraryCache
import org.enso.librarymanager.published.repository.LibraryManifest
import org.enso.librarymanager.published.repository.RepositoryHelper.RepositoryMethods
import org.enso.libraryupload.DependencyExtractor
import org.enso.pkg.PackageManager
import org.enso.yaml.YamlHelper
import java.io.File
import java.nio.file.Files
import scala.util.Try
/** A helper class that allows to find all transitive dependencies of a specific
* library.
*/
class DependencyResolver(
localLibraryProvider: LocalLibraryProvider,
publishedLibraryProvider: PublishedLibraryCache,
edition: Editions.ResolvedEdition,
preferLocalLibraries: Boolean,
versionResolver: LibraryResolver,
dependencyExtractor: DependencyExtractor[File]
) {
/** Finds all transitive dependencies of the requested library.
*
* The resulting set of dependencies also includes the library itself.
*/
def findDependencies(libraryName: LibraryName): Try[Set[Dependency]] =
Try(findDependencies(libraryName, Set.empty))
/** A helper function to discover all transitive dependencies, avoiding
* looping on cycles.
*
* It keeps track of libraries that already have been 'visited' and if a
* library that was already visited is queried again (which is caused by
* import cycles), it returns an empty set - that is because since this
* library was already visited, it and its dependencies must have already
* been accounted for in one of the parent calls, so we can return this empty
* set at this point, because later on these dependencies will be included.
* If we didn't quit early here, we would get an infinite loop due to the
* dependency cycle.
*/
private def findDependencies(
libraryName: LibraryName,
parents: Set[LibraryName]
): Set[Dependency] = {
if (parents.contains(libraryName)) {
Set.empty
} else {
val version = versionResolver
.resolveLibraryVersion(libraryName, edition, preferLocalLibraries)
.toTry
.get
version match {
case LibraryVersion.Local =>
val libraryPath = localLibraryProvider.findLibrary(libraryName)
val libraryPackage = libraryPath.map(path =>
PackageManager.Default.loadPackage(path.toFile).get
)
val dependencies = libraryPackage match {
case Some(pkg) =>
dependencyExtractor.findDependencies(pkg)
case None =>
Set.empty
}
val itself = Dependency(libraryName, version, libraryPath.isDefined)
dependencies.flatMap(
findDependencies(_, parents + libraryName)
) + itself
case publishedVersion @ LibraryVersion.Published(semver, _) =>
val itself = Dependency(
libraryName,
version,
publishedLibraryProvider.isLibraryCached(libraryName, semver)
)
val manifest = getManifest(libraryName, publishedVersion)
manifest.dependencies.toSet.flatMap { name: LibraryName =>
findDependencies(name, parents + libraryName)
} + itself
}
}
}
private def getManifest(
libraryName: LibraryName,
version: LibraryVersion.Published
): LibraryManifest = {
val cachedManifest = publishedLibraryProvider
.findCachedLibrary(libraryName, version.version)
.flatMap { libraryPath =>
val manifestPath = libraryPath.resolve(LibraryManifest.filename)
if (Files.exists(manifestPath))
YamlHelper.load[LibraryManifest](manifestPath).toOption
else None
}
cachedManifest.getOrElse {
version.repository
.accessLibrary(libraryName, version.version)
.downloadManifest()
.force()
}
}
}

6
lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/CachedLibraryProvider.scala
View File

@ -31,7 +31,7 @@ class CachedLibraryProvider(caches: List[ReadOnlyLibraryCache])
}
/** Looks for the library in the known caches. */
final protected def findCached(
override def findCachedLibrary(
libraryName: LibraryName,
version: SemVer
): Option[Path] = findCachedHelper(libraryName, version, caches)
@ -42,7 +42,7 @@ class CachedLibraryProvider(caches: List[ReadOnlyLibraryCache])
version: SemVer,
recommendedRepository: Editions.Repository
): Try[Path] =
findCached(libraryName, version)
findCachedLibrary(libraryName, version)
.toRight(
LibraryResolutionError(
s"Library [$libraryName:$version] was not found in the cache."
@ -54,5 +54,5 @@ class CachedLibraryProvider(caches: List[ReadOnlyLibraryCache])
override def isLibraryCached(
libraryName: LibraryName,
version: SemVer
): Boolean = findCached(libraryName, version).isDefined
): Boolean = findCachedLibrary(libraryName, version).isDefined
}

2
lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/DefaultPublishedLibraryProvider.scala
View File

@ -27,7 +27,7 @@ class DefaultPublishedLibraryProvider(
version: SemVer,
recommendedRepository: Editions.Repository
): Try[Path] = {
val cached = findCached(libraryName, version)
val cached = findCachedLibrary(libraryName, version)
cached.map(Success(_)).getOrElse {
logger.trace(
s"$libraryName was not found in any caches, it will need to be " +

3
lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/PublishedLibraryCache.scala
View File

@ -16,6 +16,9 @@ trait PublishedLibraryCache {
* caches.
*/
def isLibraryCached(libraryName: LibraryName, version: SemVer): Boolean
/** Tries to locate a cached version of the requested library. */
def findCachedLibrary(libraryName: LibraryName, version: SemVer): Option[Path]
}
object PublishedLibraryCache {

13
lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala vendored
View File

@ -20,6 +20,7 @@ import org.enso.librarymanager.published.repository.RepositoryHelper.{
}
import org.enso.logger.masking.MaskedPath
import org.enso.pkg.PackageManager
import org.enso.yaml.YamlHelper
import java.nio.file.{Files, Path}
import scala.util.control.NonFatal
@ -72,7 +73,6 @@ class DownloadingLibraryCache(
version: SemVer,
recommendedRepository: Editions.Repository
): Try[Path] = {
val _ = progressReporter // TODO
val cached = findCachedLibrary(libraryName, version)
cached match {
case Some(result) => Success(result)
@ -81,6 +81,16 @@ class DownloadingLibraryCache(
}
}
private def saveManifest(
manifest: LibraryManifest,
destinationDirectory: Path
): Unit = {
FileSystem.writeTextFile(
destinationDirectory / LibraryManifest.filename,
YamlHelper.toYaml(manifest)
)
}
private def installLibrary(
libraryName: LibraryName,
version: SemVer,
@ -111,6 +121,7 @@ class DownloadingLibraryCache(
try {
downloadLooseFiles(libraryName, version, access, localTmpDir)
downloadAndExtractArchives(libraryName, access, manifest, localTmpDir)
saveManifest(manifest, localTmpDir)
verifyPackageIntegrity(localTmpDir)
FileSystem.atomicMove(

13
lib/scala/library-manager/src/main/scala/org/enso/libraryupload/DependencyExtractor.scala Normal file
View File

@ -0,0 +1,13 @@
package org.enso.libraryupload
import org.enso.editions.LibraryName
import org.enso.pkg.Package
/** A general interface for a helper that allows to extract dependencies from
* the project.
*/
trait DependencyExtractor[F] {
/** Finds dependencies of a given project package. */
def findDependencies(pkg: Package[F]): Set[LibraryName]
}

46
lib/scala/library-manager/src/main/scala/org/enso/libraryupload/LibraryUploader.scala
View File

@ -12,17 +12,19 @@ import org.enso.downloader.archive.TarGzWriter
import org.enso.downloader.http.{HTTPDownload, HTTPRequestBuilder, URIBuilder}
import org.enso.editions.LibraryName
import org.enso.librarymanager.published.repository.LibraryManifest
import org.enso.libraryupload.LibraryUploader.UploadFailedError
import org.enso.pkg.{Package, PackageManager}
import org.enso.yaml.YamlHelper
import java.io.File
import java.nio.file.{Files, Path}
import scala.concurrent.{ExecutionContext, Future}
import scala.jdk.CollectionConverters._
import scala.util.{Failure, Success, Try, Using}
/** Gathers functions used for uploading libraries. */
object LibraryUploader {
private lazy val logger = Logger[LibraryUploader.type]
class LibraryUploader(dependencyExtractor: DependencyExtractor[File]) {
private lazy val logger = Logger[LibraryUploader]
/** Uploads a library to a repository.
*
@ -50,7 +52,6 @@ object LibraryUploader {
}
val uri = buildUploadUri(uploadUrl, pkg.libraryName, version)
val mainArchiveName = "main.tgz"
val filesToIgnoreInArchive = Seq(
Package.configFileName,
LibraryManifest.filename
@ -64,13 +65,7 @@ object LibraryUploader {
)
compressing.force()
val manifestPath = projectRoot / LibraryManifest.filename
val loadedManifest =
loadSavedManifest(manifestPath).getOrElse(LibraryManifest.empty)
val updatedManifest =
// TODO [RW] update dependencies in the manifest (#1773)
loadedManifest.copy(archives = Seq(mainArchiveName))
FileSystem.writeTextFile(manifestPath, YamlHelper.toYaml(updatedManifest))
updateManifest(pkg).get
logger.info(s"Uploading library package to the server at [$uploadUrl].")
val upload = uploadFiles(
@ -92,9 +87,24 @@ object LibraryUploader {
}
}
/** Indicates that the library upload has failed. */
case class UploadFailedError(message: String)
extends RuntimeException(message)
/** Updates the project's manifest by computing its dependencies.
*
* @param pkg package of the project that is to be updated
*/
def updateManifest(pkg: Package[File]): Try[Unit] = Try {
val directDependencies = dependencyExtractor.findDependencies(pkg)
val manifestPath = pkg.root.toPath / LibraryManifest.filename
val loadedManifest =
loadSavedManifest(manifestPath).getOrElse(LibraryManifest.empty)
val updatedManifest = loadedManifest.copy(
archives = Seq(mainArchiveName),
dependencies = directDependencies.toSeq
)
FileSystem.writeTextFile(manifestPath, YamlHelper.toYaml(updatedManifest))
}
private val mainArchiveName = "main.tgz"
/** Creates an URL for the upload, including information identifying the
* library version.
@ -231,3 +241,13 @@ object LibraryUploader {
}
}
}
object LibraryUploader {
def apply(dependencyExtractor: DependencyExtractor[File]): LibraryUploader =
new LibraryUploader(dependencyExtractor)
/** Indicates that the library upload has failed. */
case class UploadFailedError(message: String)
extends RuntimeException(message)
}

8
lib/scala/logging-service/src/main/scala/org/enso/loggingservice/internal/service/ThreadProcessingService.scala
View File

@ -4,6 +4,7 @@ import org.enso.loggingservice.LogLevel
import org.enso.loggingservice.internal.protocol.WSLogMessage
import org.enso.loggingservice.internal.{
BlockingConsumerMessageQueue,
DefaultLogMessageRenderer,
InternalLogger
}
@ -53,6 +54,10 @@ trait ThreadProcessingService extends Service {
thread.start()
}
private lazy val renderer = new DefaultLogMessageRenderer(
printExceptions = false
)
/** The runner filters out internal messages that have disabled log levels,
* but passes through all external messages (as their log level is set
* independently and can be lower).
@ -68,6 +73,9 @@ trait ThreadProcessingService extends Service {
InternalLogger.error(
s"One of the printers failed to write a message: $e"
)
InternalLogger.error(
s"The dropped message was: ${renderer.render(message)}"
)
}
}
} catch {

76
project/LibraryManifestGenerator.scala Normal file
View File

@ -0,0 +1,76 @@
import sbt._
import sbt.util.CacheStoreFactory
/** A helper for generating manifests for bundled libraries. */
object LibraryManifestGenerator {
/** Represents a library that will be bundled with the engine and needs to
* have its manifest generated.
*/
case class BundledLibrary(name: String, version: String)
/** Generates manifests for the provided libraries.
*
* It assumes that the engine-runner/assembly task is up to date (as it uses
* its artifacts).
*/
def generateManifests(
libraries: Seq[BundledLibrary],
distributionRoot: File,
log: Logger,
cacheStoreFactory: CacheStoreFactory
): Unit =
for (BundledLibrary(qualifiedName, version) <- libraries) {
val (namespace, name) = qualifiedName.split('.') match {
case Array(namespace, name) => (namespace, name)
case _ =>
throw new IllegalArgumentException(
s"Invalid library name [$qualifiedName]."
)
}
val projectPath =
distributionRoot / "lib" / namespace / name / version
val store =
cacheStoreFactory.make(s"library-manifest-$namespace-$name-$version")
val sources = (projectPath / "src" allPaths).get
Tracked.diffInputs(store, FileInfo.hash)(sources.toSet) { diff =>
def manifestExists = (projectPath / "manifest.yaml").exists()
if (diff.modified.nonEmpty || !manifestExists) {
log.info(s"Regenerating manifest for [$projectPath].")
runGenerator(projectPath, log)
} else {
log.debug(s"[$projectPath] manifest is up to date.")
}
}
}
private def runGenerator(projectPath: File, log: Logger): Unit = {
val javaCommand =
ProcessHandle.current().info().command().asScala.getOrElse("java")
val command = Seq(
javaCommand,
s"-Dtruffle.class.path.append=runtime.jar",
"-jar",
"runner.jar",
"--update-manifest",
"--in-project",
projectPath.getCanonicalPath
)
log.debug(s"Running [$command].")
val exitCode = sys.process
.Process(
command,
None,
"ENSO_EDITION_PATH" -> file("distribution/editions").getCanonicalPath
)
.!
if (exitCode != 0) {
val message = s"Command [$command] has failed with code $exitCode."
log.error(message)
throw new RuntimeException(message)
}
}
}