From 46c31bb9a52008a741f8ff9b71d8395374882e67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Tue, 23 Nov 2021 09:51:17 +0100 Subject: [PATCH] Preinstalling With Dependencies (#1981) --- .github/workflows/scala.yml | 4 +- .gitignore | 1 + RELEASES.md | 8 + build.sbt | 47 ++++- .../protocol-language-server.md | 15 ++ .../enso/languageserver/boot/MainModule.scala | 11 +- .../CompilerBasedDependencyExtractor.scala | 70 ++++++++ .../languageserver/libraries/LibraryApi.scala | 6 + .../libraries/LibraryInstallerConfig.scala | 9 +- .../handler/LibraryPreinstallHandler.scala | 160 ++++++++++++++---- .../handler/LibraryPublishHandler.scala | 11 +- .../json/JsonConnectionController.scala | 6 +- .../websocket/json/BaseServerTest.scala | 4 +- .../websocket/json/LibrariesTest.scala | 60 ++++++- .../scala/org/enso/launcher/Constants.scala | 9 + .../scala/org/enso/launcher/Launcher.scala | 39 ++++- .../launcher/cli/LauncherApplication.scala | 44 ++++- .../launcher/components/LauncherRunner.scala | 45 +++++ .../java/org/enso/polyglot/MethodNames.java | 1 + .../main/scala/org/enso/polyglot/Module.scala | 10 ++ .../enso/polyglot/ModuleManagementTest.scala | 14 ++ .../enso/runner/DependencyPreinstaller.scala | 129 ++++++++++++++ .../src/main/scala/org/enso/runner/Main.scala | 68 +++++++- .../org/enso/runner/ProjectUploader.scala | 26 ++- .../org/enso/interpreter/runtime/Module.java | 13 +- .../scala/org/enso/compiler/Compiler.scala | 40 +++++ .../org/enso/compiler/PackageRepository.scala | 2 +- .../repository/DummyRepository.scala | 25 ++- .../libraryupload/LibraryUploadTest.scala | 9 +- .../DefaultLibraryProvider.scala | 90 ++++++---- .../ResolvingLibraryProvider.scala | 13 ++ .../dependencies/Dependency.scala | 15 ++ .../dependencies/DependencyResolver.scala | 115 +++++++++++++ .../published/CachedLibraryProvider.scala | 6 +- .../DefaultPublishedLibraryProvider.scala | 2 +- .../published/PublishedLibraryCache.scala | 3 + .../cache/DownloadingLibraryCache.scala | 13 +- .../libraryupload/DependencyExtractor.scala | 13 ++ .../enso/libraryupload/LibraryUploader.scala | 46 +++-- .../service/ThreadProcessingService.scala | 8 + project/LibraryManifestGenerator.scala | 76 +++++++++ 41 files changed, 1162 insertions(+), 124 deletions(-) create mode 100644 engine/language-server/src/main/scala/org/enso/languageserver/libraries/CompilerBasedDependencyExtractor.scala create mode 100644 engine/runner/src/main/scala/org/enso/runner/DependencyPreinstaller.scala create mode 100644 lib/scala/library-manager/src/main/scala/org/enso/librarymanager/dependencies/Dependency.scala create mode 100644 lib/scala/library-manager/src/main/scala/org/enso/librarymanager/dependencies/DependencyResolver.scala create mode 100644 lib/scala/library-manager/src/main/scala/org/enso/libraryupload/DependencyExtractor.scala create mode 100644 project/LibraryManifestGenerator.scala diff --git a/.github/workflows/scala.yml b/.github/workflows/scala.yml index 1f091999d79..a951c1bb985 100644 --- a/.github/workflows/scala.yml +++ b/.github/workflows/scala.yml @@ -151,6 +151,8 @@ jobs: sbt --no-colors "project-manager/assembly" sbt --no-colors --mem 1536 "project-manager/buildNativeImage" + # The runtime/clean is needed to avoid issues with Truffle Instrumentation. + # It should be removed once #1992 is fixed. - name: Build the Runner & Runtime Uberjars run: | sleep 1 @@ -167,7 +169,7 @@ jobs: - name: Check Language Server Benchmark Compilation run: | sleep 1 - sbt --no-colors language-server/Benchmark/compile + sbt --no-colors "runtime/clean; language-server/Benchmark/compile" - name: Check Searcher Benchmark Compilation run: | sleep 1 diff --git a/.gitignore b/.gitignore index 5dde39fde3c..7c943718664 100644 --- a/.gitignore +++ b/.gitignore @@ -127,6 +127,7 @@ distribution/lib/Standard/Table/*/polyglot/ distribution/lib/Standard/Database/*/polyglot/ distribution/lib/Standard/Examples/*/data/spreadsheet.xls distribution/lib/Standard/Examples/*/data/spreadsheet.xlsx +distribution/lib/*/*/*/manifest.yaml test/Google_Api_Test/data/secret.json test/Database_Tests/data/redshift_credentials.json diff --git a/RELEASES.md b/RELEASES.md index 9ab0d6a41e3..60812ddd5d3 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -1,5 +1,13 @@ # Enso Next +## Tooling + +- Added the `enso install dependencies` command to the launcher which installs + any project dependencies, ensuring that `enso run` will not need to download + any libraries ([#1981](https://github.com/enso-org/enso/pull/1981)). + Additionally, made the `library/preinstall` endpoint able to install any + transitive dependencies of the library. + ## Enso 0.2.31 (2021-10-01) ## Interpreter/Runtime diff --git a/build.sbt b/build.sbt index 0c699c3903f..b242050ea34 100644 --- a/build.sbt +++ b/build.sbt @@ -1,3 +1,4 @@ +import LibraryManifestGenerator.BundledLibrary import org.enso.build.BenchTasks._ import org.enso.build.WithDebugCommand import sbt.Keys.{libraryDependencies, scalacOptions} @@ -1013,6 +1014,27 @@ lazy val `language-server` = (project in file("engine/language-server")) new TestFramework("org.scalameter.ScalaMeterFramework") ) ) + .settings( + // These settings are needed by language-server tests that create a runtime context. + Test / fork := true, + Test / javaOptions ++= { + // Note [Classpath Separation] + val runtimeClasspath = + (LocalProject("runtime") / Compile / fullClasspath).value + .map(_.data) + .mkString(File.pathSeparator) + Seq( + s"-Dtruffle.class.path.append=$runtimeClasspath", + s"-Duser.dir=${file(".").getCanonicalPath}" + ) + }, + Test / compile := (Test / compile) + .dependsOn(LocalProject("enso") / updateLibraryManifests) + .value, + Test / envVars ++= Map( + "ENSO_EDITION_PATH" -> file("distribution/editions").getCanonicalPath + ) + ) .dependsOn(`json-rpc-server-test` % Test) .dependsOn(`json-rpc-server`) .dependsOn(`task-progress-notifications`) @@ -1632,7 +1654,7 @@ lazy val `std-database` = project `database-polyglot-root`, Some("std-database.jar"), ignoreScalaLibrary = true, - unpackedDeps = Set("aws-java-sdk-core", "httpclient") + unpackedDeps = Set("aws-java-sdk-core", "httpclient") ) .value result @@ -1684,7 +1706,8 @@ projectManagerDistributionRoot := lazy val buildEngineDistribution = taskKey[Unit]("Builds the engine distribution") buildEngineDistribution := { - val _ = (`engine-runner` / assembly).value + val _ = (`engine-runner` / assembly).value + updateLibraryManifests.value val root = engineDistributionRoot.value val log = streams.value.log val cacheFactory = streams.value.cacheStoreFactory @@ -1744,3 +1767,23 @@ buildGraalDistribution := { DistributionPackage.Architecture.X64 ) } + +lazy val updateLibraryManifests = + taskKey[Unit]( + "Recomputes dependencies to update manifests bundled with libraries." + ) +updateLibraryManifests := { + val _ = (`engine-runner` / assembly).value + val log = streams.value.log + val cacheFactory = streams.value.cacheStoreFactory + val libraries = Editions.standardLibraries.map(libName => + BundledLibrary(libName, stdLibVersion) + ) + + LibraryManifestGenerator.generateManifests( + libraries, + file("distribution"), + log, + cacheFactory + ) +} diff --git a/docs/language-server/protocol-language-server.md b/docs/language-server/protocol-language-server.md index f1f51945975..7cb6fa99e07 100644 --- a/docs/language-server/protocol-language-server.md +++ b/docs/language-server/protocol-language-server.md @@ -204,6 +204,7 @@ transport formats, please look [here](./protocol-architecture). - [`LocalLibraryNotFound`](#locallibrarynotfound) - [`LibraryNotResolved`](#librarynotresolved) - [`InvalidLibraryName`](#invalidlibraryname) + - [`DependencyDiscoveryError`](#dependencydiscoveryerror) @@ -4550,6 +4551,8 @@ null; - [`LibraryNotResolved`](#librarynotresolved) to signal that the requested library or one of its dependencies could not be resolved. +- [`DependencyDiscoveryError`](#dependencydiscoveryerror) to signal that + dependencies of the library could not be established. - [`LibraryDownloadError`](#librarydownloaderror) to signal that the download operation has failed, for network-related reasons, or because the library was missing in the repository. The error includes the name and version of the @@ -5085,3 +5088,15 @@ For example for `FooBar` it will suggest `Foo_Bar`. } } ``` + +### `DependencyDiscoveryError` + +Signals that the library preinstall endpoint could not properly find +dependencies of the requested library. + +```typescript +"error" : { + "code" : 8010, + "message" : "Error occurred while discovering dependencies: ." +} +``` diff --git a/engine/language-server/src/main/scala/org/enso/languageserver/boot/MainModule.scala b/engine/language-server/src/main/scala/org/enso/languageserver/boot/MainModule.scala index 2531988e620..cb7d1e662cb 100644 --- a/engine/language-server/src/main/scala/org/enso/languageserver/boot/MainModule.scala +++ b/engine/language-server/src/main/scala/org/enso/languageserver/boot/MainModule.scala @@ -16,13 +16,7 @@ import org.enso.languageserver.effect.ZioExec import org.enso.languageserver.filemanager._ import org.enso.languageserver.http.server.BinaryWebSocketServer import org.enso.languageserver.io._ -import org.enso.languageserver.libraries.{ - EditionReferenceResolver, - LibraryConfig, - LibraryInstallerConfig, - LocalLibraryManager, - ProjectSettingsManager -} +import org.enso.languageserver.libraries._ import org.enso.languageserver.monitoring.{ HealthCheckEndpoint, IdlenessEndpoint, @@ -330,7 +324,8 @@ class MainModule(serverConfig: LanguageServerConfig, logLevel: LogLevel) { installerConfig = LibraryInstallerConfig( distributionManager, resourceManager, - Some(languageHome) + Some(languageHome), + new CompilerBasedDependencyExtractor(logLevel) ) ) diff --git a/engine/language-server/src/main/scala/org/enso/languageserver/libraries/CompilerBasedDependencyExtractor.scala b/engine/language-server/src/main/scala/org/enso/languageserver/libraries/CompilerBasedDependencyExtractor.scala new file mode 100644 index 00000000000..d5a58672b60 --- /dev/null +++ b/engine/language-server/src/main/scala/org/enso/languageserver/libraries/CompilerBasedDependencyExtractor.scala @@ -0,0 +1,70 @@ +package org.enso.languageserver.libraries + +import org.enso.editions.LibraryName +import org.enso.libraryupload.DependencyExtractor +import org.enso.loggingservice.{JavaLoggingLogHandler, LogLevel} +import org.enso.pkg.Package +import org.enso.pkg.SourceFile +import org.enso.polyglot.{PolyglotContext, RuntimeOptions} +import org.graalvm.polyglot.Context + +import java.io.File + +/** A dependency extractor that runs the compiler in a mode that only parses the + * source code and runs just the basic preprocessing phases to find out what + * libraries are imported by the project. + * + * @param logLevel the log level to use for the runtime context that will do + * the parsing + */ +class CompilerBasedDependencyExtractor(logLevel: LogLevel) + extends DependencyExtractor[File] { + + /** @inheritdoc */ + override def findDependencies(pkg: Package[File]): Set[LibraryName] = { + val context = createContextWithProject(pkg) + + def findImportedLibraries(file: SourceFile[File]): Set[LibraryName] = { + val module = context.getTopScope.getModule(file.qualifiedName.toString) + val imports = module.gatherImportStatements() + val importedLibraries = imports.map { rawName => + LibraryName.fromString(rawName) match { + case Left(error) => + throw new IllegalStateException(error) + case Right(value) => value + } + } + importedLibraries.toSet + } + + val sourcesImports = pkg.listSources.toSet.flatMap(findImportedLibraries) + val itself = pkg.libraryName + + // Builtins need to be removed from the set of the dependencies, because + // even if they are imported, they are not a typical library. + val builtins = LibraryName("Standard", "Builtins") + + sourcesImports - itself - builtins + } + + /** Creates a simple runtime context with the given package loaded as its + * project root. + */ + private def createContextWithProject(pkg: Package[File]): PolyglotContext = { + val context = Context + .newBuilder() + .allowExperimentalOptions(true) + .allowAllAccess(true) + .option(RuntimeOptions.PROJECT_ROOT, pkg.root.getCanonicalPath) + .option("js.foreign-object-prototype", "true") + .option( + RuntimeOptions.LOG_LEVEL, + JavaLoggingLogHandler.getJavaLogLevelFor(logLevel).getName + ) + .logHandler( + JavaLoggingLogHandler.create(JavaLoggingLogHandler.defaultLevelMapping) + ) + .build + new PolyglotContext(context) + } +} diff --git a/engine/language-server/src/main/scala/org/enso/languageserver/libraries/LibraryApi.scala b/engine/language-server/src/main/scala/org/enso/languageserver/libraries/LibraryApi.scala index eba5707bd58..a8840086a26 100644 --- a/engine/language-server/src/main/scala/org/enso/languageserver/libraries/LibraryApi.scala +++ b/engine/language-server/src/main/scala/org/enso/languageserver/libraries/LibraryApi.scala @@ -250,4 +250,10 @@ object LibraryApi { } """ ) } + + case class DependencyDiscoveryError(reason: String) + extends Error( + 8010, + s"Error occurred while discovering dependencies: $reason." + ) } diff --git a/engine/language-server/src/main/scala/org/enso/languageserver/libraries/LibraryInstallerConfig.scala b/engine/language-server/src/main/scala/org/enso/languageserver/libraries/LibraryInstallerConfig.scala index 2b6f9be1cea..7de702db3d5 100644 --- a/engine/language-server/src/main/scala/org/enso/languageserver/libraries/LibraryInstallerConfig.scala +++ b/engine/language-server/src/main/scala/org/enso/languageserver/libraries/LibraryInstallerConfig.scala @@ -1,7 +1,10 @@ package org.enso.languageserver.libraries -import org.enso.distribution.{DistributionManager, LanguageHome} import org.enso.distribution.locking.ResourceManager +import org.enso.distribution.{DistributionManager, LanguageHome} +import org.enso.libraryupload.DependencyExtractor + +import java.io.File /** Gathers configuration needed by the library installer used in the * `library/preinstall` endpoint. @@ -9,9 +12,11 @@ import org.enso.distribution.locking.ResourceManager * @param distributionManager the distribution manager * @param resourceManager a resource manager instance * @param languageHome language home, if detected / applicable + * @param dependencyExtractor a dependency extractor */ case class LibraryInstallerConfig( distributionManager: DistributionManager, resourceManager: ResourceManager, - languageHome: Option[LanguageHome] + languageHome: Option[LanguageHome], + dependencyExtractor: DependencyExtractor[File] ) diff --git a/engine/language-server/src/main/scala/org/enso/languageserver/libraries/handler/LibraryPreinstallHandler.scala b/engine/language-server/src/main/scala/org/enso/languageserver/libraries/handler/LibraryPreinstallHandler.scala index 35e2901877f..486d87a3c23 100644 --- a/engine/language-server/src/main/scala/org/enso/languageserver/libraries/handler/LibraryPreinstallHandler.scala +++ b/engine/language-server/src/main/scala/org/enso/languageserver/libraries/handler/LibraryPreinstallHandler.scala @@ -2,9 +2,14 @@ package org.enso.languageserver.libraries.handler import akka.actor.{Actor, ActorRef, Props, Status} import akka.pattern.pipe +import cats.implicits.toTraverseOps import com.typesafe.scalalogging.LazyLogging import org.enso.cli.task.notifications.ActorProgressNotificationForwarder -import org.enso.cli.task.{ProgressNotification, ProgressReporter} +import org.enso.cli.task.{ + ProgressNotification, + ProgressReporter, + TaskProgressImplementation +} import org.enso.distribution.ProgressAndLockNotificationForwarder import org.enso.distribution.locking.LockUserInterface import org.enso.editions.LibraryName @@ -12,6 +17,8 @@ import org.enso.jsonrpc.{Id, Request, ResponseError, ResponseResult, Unused} import org.enso.languageserver.filemanager.FileManagerApi.FileSystemError import org.enso.languageserver.libraries.LibraryApi._ import org.enso.languageserver.libraries.handler.LibraryPreinstallHandler.{ + DependencyGatheringError, + InstallationError, InstallationResult, InstallerError, InternalError @@ -19,19 +26,21 @@ import org.enso.languageserver.libraries.handler.LibraryPreinstallHandler.{ import org.enso.languageserver.libraries.{ EditionReference, EditionReferenceResolver, - LibraryInstallerConfig + LibraryConfig } import org.enso.languageserver.util.UnhandledLogging import org.enso.librarymanager.ResolvingLibraryProvider.Error +import org.enso.librarymanager.dependencies.{Dependency, DependencyResolver} import org.enso.librarymanager.{ DefaultLibraryProvider, + LibraryResolver, ResolvedLibrary, ResolvingLibraryProvider } import java.util.concurrent.Executors import scala.concurrent.{ExecutionContext, Future} -import scala.util.Try +import scala.util.{Success, Try} /** A request handler for the `library/preinstall` endpoint. * @@ -41,11 +50,11 @@ import scala.util.Try * to select a reasonable timeout. * * @param editionReferenceResolver an [[EditionReferenceResolver]] instance - * @param installerConfig configuration for the library installer + * @param config configuration for the library subsystem */ class LibraryPreinstallHandler( editionReferenceResolver: EditionReferenceResolver, - installerConfig: LibraryInstallerConfig + config: LibraryConfig ) extends Actor with LazyLogging with UnhandledLogging { @@ -71,23 +80,84 @@ class LibraryPreinstallHandler( .translateProgressNotification(LibraryPreinstall.name, notification) } - val installation: Future[InstallationResult] = Future { - val result = for { - libraryInstaller <- getLibraryProvider( - notificationForwarder - ).toEither.left.map(InternalError) - library <- libraryInstaller - .findLibrary(libraryName) - .left - .map(InstallerError) - } yield library - InstallationResult(result) - } + val installation: Future[InstallationResult] = + installLibraryWithDependencies(libraryName, notificationForwarder) installation pipeTo self context.become(responseStage(id, replyTo, libraryName)) } + /** Returns a future that will be completed once all dependencies of the + * library have been installed. + * + * @param libraryName name of the library to install + * @param notificationForwarder a notification handler for reporting progress + */ + private def installLibraryWithDependencies( + libraryName: LibraryName, + notificationForwarder: ProgressAndLockNotificationForwarder + ): Future[InstallationResult] = Future { + val result = for { + tools <- instantiateTools(notificationForwarder).toEither.left + .map(InternalError) + dependencies <- tools.dependencyResolver + .findDependencies(libraryName) + .toEither + .left + .map(DependencyGatheringError) + dependenciesToInstall = dependencies.filter(!_.isCached) + _ <- installDependencies( + dependenciesToInstall, + notificationForwarder, + tools.libraryInstaller + ) + library <- tools.libraryInstaller + .findLibrary(libraryName) + .left + .map(InstallerError) + } yield library + InstallationResult(result) + } + + /** Installs the provided dependencies and reports the overall progress. */ + private def installDependencies( + dependencies: Set[Dependency], + notificationForwarder: ProgressAndLockNotificationForwarder, + libraryInstaller: ResolvingLibraryProvider + ): Either[InstallationError, Unit] = { + + logger.trace(s"Dependencies to install: $dependencies.") + + val taskProgress = new TaskProgressImplementation[Unit]() + + val message = + if (dependencies.size == 1) s"Installing 1 library." + else s"Installing ${dependencies.size} libraries." + + notificationForwarder.trackProgress( + message, + taskProgress + ) + + val total = Some(dependencies.size.toLong) + taskProgress.reportProgress(0, total) + + val results = + dependencies.toList.zipWithIndex.traverse { case (dependency, ix) => + val result = libraryInstaller.findSpecificLibraryVersion( + dependency.libraryName, + dependency.version + ) + + taskProgress.reportProgress(ix.toLong + 1, total) + result + } + + taskProgress.setComplete(Success(())) + + results.map { _ => () }.left.map(InstallerError) + } + private def responseStage( requestId: Id, replyTo: ActorRef, @@ -99,6 +169,8 @@ class LibraryPreinstallHandler( val errorMessage = error match { case InternalError(throwable) => FileSystemError(s"Internal error: ${throwable.getMessage}") + case DependencyGatheringError(throwable) => + DependencyDiscoveryError(throwable.getMessage) case InstallerError(Error.NotResolved(_)) => LibraryNotResolved(libraryName) case InstallerError(Error.RequestedLocalLibraryDoesNotExist) => @@ -120,23 +192,41 @@ class LibraryPreinstallHandler( self ! Left(InternalError(throwable)) } - private def getLibraryProvider( + case class Tools( + libraryInstaller: ResolvingLibraryProvider, + dependencyResolver: DependencyResolver + ) + + /** A helper function that creates instances if the library installer and + * dependency resolver that report to the provided notification forwarder. + */ + private def instantiateTools( notificationReporter: ProgressReporter with LockUserInterface - ): Try[ResolvingLibraryProvider] = + ): Try[Tools] = for { - config <- editionReferenceResolver.getCurrentProjectConfig + projectConfig <- editionReferenceResolver.getCurrentProjectConfig edition <- editionReferenceResolver.resolveEdition( EditionReference.CurrentProjectEdition ) - } yield DefaultLibraryProvider.make( - distributionManager = installerConfig.distributionManager, - resourceManager = installerConfig.resourceManager, - lockUserInterface = notificationReporter, - progressReporter = notificationReporter, - languageHome = installerConfig.languageHome, - edition = edition, - preferLocalLibraries = config.preferLocalLibraries - ) + preferLocalLibraries = projectConfig.preferLocalLibraries + installer = DefaultLibraryProvider.make( + distributionManager = config.installerConfig.distributionManager, + resourceManager = config.installerConfig.resourceManager, + lockUserInterface = notificationReporter, + progressReporter = notificationReporter, + languageHome = config.installerConfig.languageHome, + edition = edition, + preferLocalLibraries = preferLocalLibraries + ) + dependencyResolver = new DependencyResolver( + localLibraryProvider = config.localLibraryProvider, + publishedLibraryProvider = config.publishedLibraryCache, + edition = edition, + preferLocalLibraries = preferLocalLibraries, + versionResolver = LibraryResolver(config.localLibraryProvider), + dependencyExtractor = config.installerConfig.dependencyExtractor + ) + } yield Tools(installer, dependencyResolver) } object LibraryPreinstallHandler { @@ -144,13 +234,13 @@ object LibraryPreinstallHandler { /** Creates a configuration object to create [[LibraryPreinstallHandler]]. * * @param editionReferenceResolver an [[EditionReferenceResolver]] instance - * @param installerConfig configuration for the library installer + * @param config configuration for the library subsystem */ def props( editionReferenceResolver: EditionReferenceResolver, - installerConfig: LibraryInstallerConfig + config: LibraryConfig ): Props = Props( - new LibraryPreinstallHandler(editionReferenceResolver, installerConfig) + new LibraryPreinstallHandler(editionReferenceResolver, config) ) /** An internal message used to pass the installation result from the Future @@ -180,4 +270,10 @@ object LibraryPreinstallHandler { * could not be established. */ case class InstallerError(error: Error) extends InstallationError + + /** Indicates an error that occurred when looking for all of the transitive + * dependencies of the library. + */ + case class DependencyGatheringError(throwable: Throwable) + extends InstallationError } diff --git a/engine/language-server/src/main/scala/org/enso/languageserver/libraries/handler/LibraryPublishHandler.scala b/engine/language-server/src/main/scala/org/enso/languageserver/libraries/handler/LibraryPublishHandler.scala index 27d30fecf7f..358bf61ee37 100644 --- a/engine/language-server/src/main/scala/org/enso/languageserver/libraries/handler/LibraryPublishHandler.scala +++ b/engine/language-server/src/main/scala/org/enso/languageserver/libraries/handler/LibraryPublishHandler.scala @@ -7,15 +7,19 @@ import org.enso.cli.task.notifications.ActorProgressNotificationForwarder import org.enso.editions.LibraryName import org.enso.jsonrpc._ import org.enso.languageserver.filemanager.FileManagerApi.FileSystemError -import org.enso.languageserver.libraries.BlockingOperation import org.enso.languageserver.libraries.LibraryApi._ import org.enso.languageserver.libraries.LocalLibraryManagerProtocol.{ FindLibrary, FindLibraryResponse } +import org.enso.languageserver.libraries.{ + BlockingOperation, + CompilerBasedDependencyExtractor +} import org.enso.languageserver.requesthandler.RequestTimeout import org.enso.languageserver.util.UnhandledLogging import org.enso.libraryupload.{auth, LibraryUploader} +import org.enso.loggingservice.LoggingServiceManager import scala.concurrent.Future import scala.concurrent.duration.FiniteDuration @@ -98,7 +102,10 @@ class LibraryPublishHandler( ) val future: Future[UploadSucceeded] = BlockingOperation.run { - LibraryUploader + val logLevel = LoggingServiceManager.currentLogLevelForThisApplication() + val dependencyExtractor = + new CompilerBasedDependencyExtractor(logLevel) + LibraryUploader(dependencyExtractor) .uploadLibrary( libraryRoot, uploadUrl, diff --git a/engine/language-server/src/main/scala/org/enso/languageserver/protocol/json/JsonConnectionController.scala b/engine/language-server/src/main/scala/org/enso/languageserver/protocol/json/JsonConnectionController.scala index 4cde27c762f..3ec8dd40dc6 100644 --- a/engine/language-server/src/main/scala/org/enso/languageserver/protocol/json/JsonConnectionController.scala +++ b/engine/language-server/src/main/scala/org/enso/languageserver/protocol/json/JsonConnectionController.scala @@ -513,10 +513,8 @@ class JsonConnectionController( .props(requestTimeout, libraryConfig.localLibraryManager), LibraryGetMetadata -> LibraryGetMetadataHandler .props(requestTimeout, libraryConfig.localLibraryManager), - LibraryPreinstall -> LibraryPreinstallHandler.props( - libraryConfig.editionReferenceResolver, - libraryConfig.installerConfig - ), + LibraryPreinstall -> LibraryPreinstallHandler + .props(libraryConfig.editionReferenceResolver, libraryConfig), LibraryPublish -> LibraryPublishHandler .props(requestTimeout, libraryConfig.localLibraryManager), LibrarySetMetadata -> LibrarySetMetadataHandler diff --git a/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/BaseServerTest.scala b/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/BaseServerTest.scala index 9fc7e23e6f1..c6316964102 100644 --- a/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/BaseServerTest.scala +++ b/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/BaseServerTest.scala @@ -37,6 +37,7 @@ import org.enso.languageserver.text.BufferRegistry import org.enso.librarymanager.LibraryLocations import org.enso.librarymanager.local.DefaultLocalLibraryProvider import org.enso.librarymanager.published.PublishedLibraryCache +import org.enso.loggingservice.LogLevel import org.enso.pkg.PackageManager import org.enso.polyglot.data.TypeGraph import org.enso.polyglot.runtime.Runtime.Api @@ -277,7 +278,8 @@ class BaseServerTest installerConfig = LibraryInstallerConfig( distributionManager, resourceManager, - Some(languageHome) + Some(languageHome), + new CompilerBasedDependencyExtractor(logLevel = LogLevel.Warning) ) ) diff --git a/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/LibrariesTest.scala b/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/LibrariesTest.scala index 1849cc8ef96..e908669030e 100644 --- a/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/LibrariesTest.scala +++ b/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/LibrariesTest.scala @@ -3,22 +3,39 @@ package org.enso.languageserver.websocket.json import io.circe.literal._ import io.circe.{Json, JsonObject} import nl.gn0s1s.bump.SemVer +import org.enso.distribution.FileSystem import org.enso.editions.{Editions, LibraryName} import org.enso.languageserver.libraries.LibraryEntry import org.enso.languageserver.libraries.LibraryEntry.PublishedLibraryVersion import org.enso.librarymanager.published.bundles.LocalReadOnlyRepository import org.enso.librarymanager.published.repository.{ EmptyRepository, - ExampleRepository + ExampleRepository, + LibraryManifest } import org.enso.pkg.{Contact, PackageManager} +import org.enso.yaml.YamlHelper import java.nio.file.Files class LibrariesTest extends BaseServerTest { private val libraryRepositoryPort: Int = 47308 - private val exampleRepo = new ExampleRepository + private val exampleRepo = new ExampleRepository { + override def libraries: Seq[DummyLibrary] = Seq( + DummyLibrary( + LibraryName("Foo", "Bar"), + SemVer(1, 0, 0), + """import Standard.Base + | + |baz = 42 + | + |quux = "foobar" + |""".stripMargin, + dependencies = Seq(LibraryName("Standard", "Base")) + ) + ) + } private val baseUrl = s"http://localhost:$libraryRepositoryPort/" private val repositoryUrl = baseUrl + "libraries" @@ -232,6 +249,21 @@ class LibrariesTest extends BaseServerTest { } """) + // Update Main.enso + val libraryRoot = getTestDirectory + .resolve("test_home") + .resolve("libraries") + .resolve("user") + .resolve("Publishable_Lib") + val mainSource = libraryRoot.resolve("src").resolve("Main.enso") + FileSystem.writeTextFile( + mainSource, + """import Some.Other_Library + | + |main = 42 + |""".stripMargin + ) + client.send(json""" { "jsonrpc": "2.0", "method": "library/setMetadata", @@ -308,6 +340,18 @@ class LibrariesTest extends BaseServerTest { Contact(name = Some("only-name"), email = None), Contact(name = None, email = Some("foo@example.com")) ) + val manifest = YamlHelper + .load[LibraryManifest]( + libraryRoot.resolve(LibraryManifest.filename) + ) + .get + + manifest.archives shouldEqual Seq("main.tgz") + manifest.dependencies shouldEqual Seq( + LibraryName("Some", "Other_Library") + ) + manifest.description shouldEqual Some("Description for publication.") + manifest.tagLine shouldEqual Some("published-lib") client.send(json""" { "jsonrpc": "2.0", @@ -387,6 +431,9 @@ class LibrariesTest extends BaseServerTest { msg("id") match { case Some(json) => json.asNumber.value.toInt.value shouldEqual requestId + msg("error").foreach(err => + println("Request ended with error: " + err) + ) msg("result").value.asNull.value waitingForResult = false case None => @@ -408,8 +455,6 @@ class LibrariesTest extends BaseServerTest { .asString .value shouldEqual "library/preinstall" - taskStart._2("unit").value.asString.value shouldEqual "Bytes" - val updates = messages.filter { case (method, params) => method == "task/progress-update" && params("taskId").value.asString.value == taskId @@ -417,7 +462,7 @@ class LibrariesTest extends BaseServerTest { updates should not be empty updates.head._2("message").value.asString.value should include( - "Downloading" + "Installing" ) val cachePath = getTestDirectory.resolve("test_data").resolve("lib") @@ -435,6 +480,11 @@ class LibrariesTest extends BaseServerTest { pkg.listSources.map( _.file.getName ) should contain theSameElementsAs Seq("Main.enso") + + assert( + Files.exists(cachedLibraryRoot.resolve(LibraryManifest.filename)), + "The manifest file of a downloaded library should be saved in the cache too." + ) } } } diff --git a/engine/launcher/src/main/scala/org/enso/launcher/Constants.scala b/engine/launcher/src/main/scala/org/enso/launcher/Constants.scala index 7f6582f4a70..3c036af13b5 100644 --- a/engine/launcher/src/main/scala/org/enso/launcher/Constants.scala +++ b/engine/launcher/src/main/scala/org/enso/launcher/Constants.scala @@ -12,6 +12,15 @@ object Constants { val uploadIntroducedVersion: SemVer = SemVer(0, 2, 17, Some("SNAPSHOT")) + /** The engine version in which the dependency preinstall command has been + * introduced. + * + * It is used to check by the launcher if the engine can handle this command + * and provide better error messages if it cannot. + */ + val preinstallDependenciesIntroducedVersion: SemVer = + SemVer(0, 2, 28, Some("SNAPSHOT")) + /** The upload URL associated with the main Enso library repository. */ val defaultUploadUrl = "https://publish.libraries.release.enso.org/" } diff --git a/engine/launcher/src/main/scala/org/enso/launcher/Launcher.scala b/engine/launcher/src/main/scala/org/enso/launcher/Launcher.scala index 12b7a744c54..d316cb79e33 100644 --- a/engine/launcher/src/main/scala/org/enso/launcher/Launcher.scala +++ b/engine/launcher/src/main/scala/org/enso/launcher/Launcher.scala @@ -293,7 +293,6 @@ case class Launcher(cliOptions: GlobalCLIOptions) { contentRoot: Path, versionOverride: Option[SemVer], logLevel: LogLevel, - logMasking: Boolean, useSystemJVM: Boolean, jvmOpts: Seq[(String, String)], additionalArguments: Seq[String] @@ -306,7 +305,7 @@ case class Launcher(cliOptions: GlobalCLIOptions) { contentRoot, versionOverride, logLevel, - logMasking, + logMasking = cliOptions.internalOptions.logMasking, additionalArguments ) .get, @@ -317,6 +316,42 @@ case class Launcher(cliOptions: GlobalCLIOptions) { exitCode } + /** Runs the engine associated with the project in dependency installation + * mode. + * + * @param versionOverride if provided, overrides the default engine version + * that would have been used + * @param logLevel log level for the language server + * @param useSystemJVM if set, forces to use the default configured JVM, + * instead of the JVM associated with the engine version + * @param jvmOpts additional options to pass to the launched JVM + * @param additionalArguments additional arguments to pass to the runner + * @return exit code of the launched program + */ + def runInstallDependencies( + versionOverride: Option[SemVer], + logLevel: LogLevel, + useSystemJVM: Boolean, + jvmOpts: Seq[(String, String)], + additionalArguments: Seq[String] + ): Int = { + val exitCode = runner.withCommand( + runner + .installDependencies( + versionOverride, + hideProgress = cliOptions.hideProgress, + logLevel, + logMasking = cliOptions.internalOptions.logMasking, + additionalArguments = additionalArguments + ) + .get, + JVMSettings(useSystemJVM, jvmOpts) + ) { command => + command.run().get + } + exitCode + } + /** Updates the global configuration. * * If `value` is an empty string, the `key` is removed from the configuration diff --git a/engine/launcher/src/main/scala/org/enso/launcher/cli/LauncherApplication.scala b/engine/launcher/src/main/scala/org/enso/launcher/cli/LauncherApplication.scala index 8482ce4990e..8b90dba2195 100644 --- a/engine/launcher/src/main/scala/org/enso/launcher/cli/LauncherApplication.scala +++ b/engine/launcher/src/main/scala/org/enso/launcher/cli/LauncherApplication.scala @@ -124,7 +124,7 @@ object LauncherApplication { "(error | warning | info | debug | trace)", "Sets logging verbosity for the engine. Defaults to info." ) - .withDefault(LogLevel.Warning) + .withDefault(LogLevel.Info) } private def runCommand: Command[Config => Int] = @@ -244,8 +244,7 @@ object LauncherApplication { useSystemJVM = systemJVMOverride, jvmOpts = jvmOpts, additionalArguments = additionalArgs, - logLevel = engineLogLevel, - logMasking = config.internalOptions.logMasking + logLevel = engineLogLevel ) } } @@ -441,12 +440,47 @@ object LauncherApplication { } } + private def installDependenciesCommand: Command[Config => Int] = + Command( + "dependencies", + "Install dependencies of the current project." + ) { + val additionalArgs = Opts.additionalArguments() + ( + versionOverride, + engineLogLevel, + systemJVMOverride, + jvmOpts, + additionalArgs + ) mapN { + ( + versionOverride, + engineLogLevel, + systemJVMOverride, + jvmOpts, + additionalArgs + ) => (config: Config) => + Launcher(config).runInstallDependencies( + versionOverride, + engineLogLevel, + useSystemJVM = systemJVMOverride, + jvmOpts, + additionalArgs + ) + } + } + private def installCommand: Command[Config => Int] = Command( "install", - "Install a new version of engine or install the distribution locally." + "Install a new version of engine, install the distribution locally or " + + "install project dependencies." ) { - Opts.subcommands(installEngineCommand, installDistributionCommand) + Opts.subcommands( + installEngineCommand, + installDistributionCommand, + installDependenciesCommand + ) } private def uninstallEngineCommand: Command[Config => Int] = diff --git a/engine/launcher/src/main/scala/org/enso/launcher/components/LauncherRunner.scala b/engine/launcher/src/main/scala/org/enso/launcher/components/LauncherRunner.scala index 74d17119ecb..9e3884700bf 100644 --- a/engine/launcher/src/main/scala/org/enso/launcher/components/LauncherRunner.scala +++ b/engine/launcher/src/main/scala/org/enso/launcher/components/LauncherRunner.scala @@ -241,4 +241,49 @@ class LauncherRunner( connectLoggerIfAvailable = true ) } + + /** Creates [[RunSettings]] for installing project dependencies. + * + * See [[org.enso.launcher.Launcher.runInstallDependencies]] for more + * details. + */ + def installDependencies( + versionOverride: Option[SemVer], + hideProgress: Boolean, + logLevel: LogLevel, + logMasking: Boolean, + additionalArguments: Seq[String] + ): Try[RunSettings] = + Try { + val actualPath = currentWorkingDirectory + val project = projectManager.findProject(actualPath).get.getOrElse { + throw RunnerError( + s"Could not find a project at " + + s"${MaskedPath(actualPath).applyMasking()} or any of its parent " + + s"directories." + ) + } + + val version = resolveVersion(versionOverride, Some(project)) + if (version < Constants.preinstallDependenciesIntroducedVersion) { + throw RunnerError( + s"Project dependency installation feature is not available in Enso " + + s"$version. Please upgrade your project to a newer version to use it." + ) + } + + val hideProgressOpts = + if (hideProgress) Seq("--hide-progress") else Seq.empty + + val arguments = + Seq("--preinstall-dependencies") ++ + Seq("--in-project", project.path.toAbsolutePath.normalize.toString) ++ + hideProgressOpts + RunSettings( + version, + arguments ++ setLogLevelArgs(logLevel, logMasking) + ++ additionalArguments, + connectLoggerIfAvailable = true + ) + } } diff --git a/engine/polyglot-api/src/main/java/org/enso/polyglot/MethodNames.java b/engine/polyglot-api/src/main/java/org/enso/polyglot/MethodNames.java index 8440e037d29..163951d1af1 100644 --- a/engine/polyglot-api/src/main/java/org/enso/polyglot/MethodNames.java +++ b/engine/polyglot-api/src/main/java/org/enso/polyglot/MethodNames.java @@ -19,6 +19,7 @@ public class MethodNames { public static final String GET_NAME = "get_name"; public static final String REPARSE = "reparse"; public static final String GENERATE_DOCS = "generate_docs"; + public static final String GATHER_IMPORT_STATEMENTS = "gather_import_statements"; public static final String SET_SOURCE = "set_source"; public static final String SET_SOURCE_FILE = "set_source_file"; } diff --git a/engine/polyglot-api/src/main/scala/org/enso/polyglot/Module.scala b/engine/polyglot-api/src/main/scala/org/enso/polyglot/Module.scala index 38badf8be1c..83a172f6ee9 100644 --- a/engine/polyglot-api/src/main/scala/org/enso/polyglot/Module.scala +++ b/engine/polyglot-api/src/main/scala/org/enso/polyglot/Module.scala @@ -57,6 +57,16 @@ class Module(private val value: Value) { value.invokeMember(GENERATE_DOCS) } + /** Triggers gathering of import statements from module sources. + * + * @return value with `GATHER_IMPORT_STATEMENTS` invoked on it. + */ + def gatherImportStatements(): Seq[String] = { + val array = value.invokeMember(GATHER_IMPORT_STATEMENTS) + val size = array.getArraySize + for (i <- 0L until size) yield array.getArrayElement(i).asString() + } + /** Triggers reparsing of module sources. Used to notify the module that * sources have changed. */ diff --git a/engine/polyglot-api/src/test/scala/org/enso/polyglot/ModuleManagementTest.scala b/engine/polyglot-api/src/test/scala/org/enso/polyglot/ModuleManagementTest.scala index 5efbd0fc853..fbdb46470d6 100644 --- a/engine/polyglot-api/src/test/scala/org/enso/polyglot/ModuleManagementTest.scala +++ b/engine/polyglot-api/src/test/scala/org/enso/polyglot/ModuleManagementTest.scala @@ -188,4 +188,18 @@ class ModuleManagementTest extends AnyFlatSpec with Matchers { the[PolyglotException] thrownBy mod2.getAssociatedConstructor exception.getMessage shouldEqual "Compilation aborted due to errors." } + + subject should "allow gathering imported libraries" in { + val ctx = new TestContext("Test") + ctx.writeMain(""" + |import Foo.Bar.Baz + | + |main = 42 + |""".stripMargin) + + val topScope = ctx.executionContext.getTopScope + val mainModule = topScope.getModule("Enso_Test.Test.Main") + val imports = mainModule.gatherImportStatements() + imports shouldEqual Seq("Foo.Bar") + } } diff --git a/engine/runner/src/main/scala/org/enso/runner/DependencyPreinstaller.scala b/engine/runner/src/main/scala/org/enso/runner/DependencyPreinstaller.scala new file mode 100644 index 00000000000..eb1f52d4c58 --- /dev/null +++ b/engine/runner/src/main/scala/org/enso/runner/DependencyPreinstaller.scala @@ -0,0 +1,129 @@ +package org.enso.runner + +import cats.implicits.toTraverseOps +import com.typesafe.scalalogging.Logger +import org.enso.cli.ProgressBar +import org.enso.cli.task.{ProgressReporter, TaskProgress} +import org.enso.distribution.locking.{ + LockUserInterface, + Resource, + ResourceManager, + ThreadSafeFileLockManager +} +import org.enso.distribution.{DistributionManager, Environment, LanguageHome} +import org.enso.editions.updater.EditionManager +import org.enso.editions.{DefaultEdition, EditionResolver} +import org.enso.languageserver.libraries.CompilerBasedDependencyExtractor +import org.enso.librarymanager.dependencies.DependencyResolver +import org.enso.librarymanager.{DefaultLibraryProvider, LibraryResolver} +import org.enso.loggingservice.LogLevel +import org.enso.pkg.PackageManager + +import java.io.File + +/** A helper to preinstall all dependencies of a project. */ +object DependencyPreinstaller { + + /** Parses the project to find out its direct dependencies, uses the resolver + * to find all transitive dependencies and ensures that all of them are + * installed. + */ + def preinstallDependencies(projectRoot: File, logLevel: LogLevel): Unit = { + val logger = Logger[DependencyPreinstaller.type] + val pkg = PackageManager.Default.loadPackage(projectRoot).get + + val dependencyExtractor = new CompilerBasedDependencyExtractor(logLevel) + val environment = new Environment {} + val languageHome = LanguageHome.detectFromExecutableLocation(environment) + + val distributionManager = new DistributionManager(environment) + val lockManager = new ThreadSafeFileLockManager( + distributionManager.paths.locks + ) + val resourceManager = new ResourceManager(lockManager) + + val editionProvider = EditionManager.makeEditionProvider( + distributionManager, + Some(languageHome) + ) + val editionResolver = EditionResolver(editionProvider) + val edition = editionResolver + .resolve( + pkg.config.edition.getOrElse(DefaultEdition.getDefaultEdition) + ) match { + case Left(error) => + throw new RuntimeException( + s"Cannot resolve current project's edition: ${error.getMessage}" + ) + case Right(value) => value + } + + val preferLocalLibraries = pkg.config.preferLocalLibraries + + val (localLibraryProvider, publishedLibraryProvider) = + DefaultLibraryProvider.makeProviders( + distributionManager, + resourceManager, + new LockUserInterface { + override def startWaitingForResource(resource: Resource): Unit = + logger.warn(resource.waitMessage) + + override def finishWaitingForResource(resource: Resource): Unit = () + }, + new ProgressReporter { + override def trackProgress( + message: String, + task: TaskProgress[_] + ): Unit = { + logger.info(message) + ProgressBar.waitWithProgress(task) + } + }, + Some(languageHome) + ) + + val dependencyResolver = new DependencyResolver( + localLibraryProvider, + publishedLibraryProvider, + edition, + preferLocalLibraries, + LibraryResolver(localLibraryProvider), + dependencyExtractor + ) + val installer = new DefaultLibraryProvider( + localLibraryProvider, + publishedLibraryProvider, + edition, + preferLocalLibraries + ) + val immediateDependencies = dependencyExtractor.findDependencies(pkg) + logger.trace( + s"The project imports the following libraries: $immediateDependencies." + ) + val allDependencies = immediateDependencies.flatMap { name => + dependencyResolver.findDependencies(name).get + } + logger.trace(s"The project depends on: $allDependencies.") + + val dependenciesToInstall = allDependencies.filter(!_.isCached) + + if (dependenciesToInstall.isEmpty) { + logger.info(s"All ${allDependencies.size} dependencies are installed.") + } else { + logger.info(s"Will install ${dependenciesToInstall.size} dependencies.") + val result = dependenciesToInstall.toList.traverse { dependency => + installer.findSpecificLibraryVersion( + dependency.libraryName, + dependency.version + ) + } + result match { + case Left(error) => + throw new RuntimeException( + s"Some dependencies could not be installed: [$error]." + ) + case Right(_) => + } + } + } +} diff --git a/engine/runner/src/main/scala/org/enso/runner/Main.scala b/engine/runner/src/main/scala/org/enso/runner/Main.scala index 464e3ccc70e..bd0c0c3bff3 100644 --- a/engine/runner/src/main/scala/org/enso/runner/Main.scala +++ b/engine/runner/src/main/scala/org/enso/runner/Main.scala @@ -21,6 +21,7 @@ import java.util.UUID import scala.Console.err import scala.jdk.CollectionConverters._ import scala.util.Try +import scala.util.control.NonFatal /** The main CLI entry point class. */ object Main { @@ -34,6 +35,7 @@ object Main { private val PROJECT_AUTHOR_EMAIL_OPTION = "new-project-author-email" private val REPL_OPTION = "repl" private val DOCS_OPTION = "docs" + private val PREINSTALL_OPTION = "preinstall-dependencies" private val LANGUAGE_SERVER_OPTION = "server" private val DAEMONIZE_OPTION = "daemon" private val INTERFACE_OPTION = "interface" @@ -54,6 +56,7 @@ object Main { private val LOGGER_CONNECT = "logger-connect" private val NO_LOG_MASKING = "no-log-masking" private val UPLOAD_OPTION = "upload" + private val UPDATE_MANIFEST_OPTION = "update-manifest" private val HIDE_PROGRESS = "hide-progress" private val AUTH_TOKEN = "auth-token" private val AUTO_PARALLELISM_OPTION = "with-auto-parallelism" @@ -89,6 +92,10 @@ object Main { .longOpt(DOCS_OPTION) .desc("Runs the Enso documentation generator.") .build + val preinstall = CliOption.builder + .longOpt(PREINSTALL_OPTION) + .desc("Installs dependencies of the project.") + .build val newOpt = CliOption.builder .hasArg(true) .numberOfArgs(1) @@ -230,6 +237,13 @@ object Main { "The url defines the repository to upload to." ) .build() + val updateManifestOption = CliOption.builder + .longOpt(UPDATE_MANIFEST_OPTION) + .desc( + "Updates the library manifest with the updated list of direct " + + "dependencies." + ) + .build() val hideProgressOption = CliOption.builder .longOpt(HIDE_PROGRESS) .desc("If specified, progress bars will not be displayed.") @@ -299,6 +313,7 @@ object Main { .addOption(repl) .addOption(run) .addOption(docs) + .addOption(preinstall) .addOption(newOpt) .addOption(newProjectNameOpt) .addOption(newProjectTemplateOpt) @@ -318,6 +333,7 @@ object Main { .addOption(loggerConnectOption) .addOption(noLogMaskingOption) .addOption(uploadOption) + .addOption(updateManifestOption) .addOption(hideProgressOption) .addOption(authTokenOption) .addOption(noReadIrCachesOption) @@ -577,6 +593,34 @@ object Main { } } + /** Handles the `--preinstall-dependencies` CLI option. + * + * Gathers imported dependencies and ensures that all of them are installed. + * + * @param projectPath path of the project + * @param logLevel log level to set for the engine runtime + */ + private def preinstallDependencies( + projectPath: Option[String], + logLevel: LogLevel + ): Unit = projectPath match { + case Some(path) => + try { + DependencyPreinstaller.preinstallDependencies(new File(path), logLevel) + exitSuccess() + } catch { + case NonFatal(error) => + logger.error( + s"Dependency installation failed: ${error.getMessage}", + error + ) + exitFail() + } + case None => + println("Dependency installation is only available for projects.") + exitFail() + } + private def runPackage( context: PolyglotContext, mainModuleName: String, @@ -850,7 +894,8 @@ object Main { projectRoot = projectRoot, uploadUrl = line.getOptionValue(UPLOAD_OPTION), authToken = Option(line.getOptionValue(AUTH_TOKEN)), - showProgress = !line.hasOption(HIDE_PROGRESS) + showProgress = !line.hasOption(HIDE_PROGRESS), + logLevel = logLevel ) exitSuccess() } catch { @@ -861,6 +906,21 @@ object Main { } } + if (line.hasOption(UPDATE_MANIFEST_OPTION)) { + val projectRoot = + Option(line.getOptionValue(IN_PROJECT_OPTION)) + .map(Path.of(_)) + .getOrElse { + logger.error( + s"The $IN_PROJECT_OPTION is mandatory." + ) + exitFail() + } + + ProjectUploader.updateManifest(projectRoot, logLevel) + exitSuccess() + } + if (line.hasOption(COMPILE_OPTION)) { val packagePaths = line.getOptionValue(COMPILE_OPTION) val shouldCompileDependencies = @@ -902,6 +962,12 @@ object Main { shouldEnableIrCaches(line) ) } + if (line.hasOption(PREINSTALL_OPTION)) { + preinstallDependencies( + Option(line.getOptionValue(IN_PROJECT_OPTION)), + logLevel + ) + } if (line.hasOption(LANGUAGE_SERVER_OPTION)) { runLanguageServer(line, logLevel) } diff --git a/engine/runner/src/main/scala/org/enso/runner/ProjectUploader.scala b/engine/runner/src/main/scala/org/enso/runner/ProjectUploader.scala index 6012df69a91..9b21fae7fe3 100644 --- a/engine/runner/src/main/scala/org/enso/runner/ProjectUploader.scala +++ b/engine/runner/src/main/scala/org/enso/runner/ProjectUploader.scala @@ -3,7 +3,10 @@ package org.enso.runner import com.typesafe.scalalogging.Logger import org.enso.cli.ProgressBar import org.enso.cli.task.{ProgressReporter, TaskProgress} +import org.enso.languageserver.libraries.CompilerBasedDependencyExtractor import org.enso.libraryupload.{auth, LibraryUploader} +import org.enso.loggingservice.LogLevel +import org.enso.pkg.PackageManager import java.nio.file.Path @@ -20,12 +23,15 @@ object ProjectUploader { * repository * @param showProgress specifies if CLI progress bars should be displayed * showing progress of compression and upload + * @param logLevel the log level to use for the context gathering + * dependencies */ def uploadProject( projectRoot: Path, uploadUrl: String, authToken: Option[String], - showProgress: Boolean + showProgress: Boolean, + logLevel: LogLevel ): Unit = { import scala.concurrent.ExecutionContext.Implicits.global val progressReporter = new ProgressReporter { @@ -44,7 +50,10 @@ object ProjectUploader { case Some(value) => auth.SimpleHeaderToken(value) case None => auth.NoAuthorization } - LibraryUploader + + val dependencyExtractor = new CompilerBasedDependencyExtractor(logLevel) + + LibraryUploader(dependencyExtractor) .uploadLibrary( projectRoot, uploadUrl, @@ -53,4 +62,17 @@ object ProjectUploader { ) .get } + + /** Updates manifest of the project. + * + * @param projectRoot path to the root of the project + * @param logLevel the log level to use for the context gathering + * dependencies + */ + def updateManifest(projectRoot: Path, logLevel: LogLevel): Unit = { + val pkg = PackageManager.Default.loadPackage(projectRoot.toFile).get + + val dependencyExtractor = new CompilerBasedDependencyExtractor(logLevel) + LibraryUploader(dependencyExtractor).updateManifest(pkg).get + } } diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/Module.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/Module.java index 9a1df38f5ce..05ef2b291df 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/Module.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/Module.java @@ -488,6 +488,11 @@ public class Module implements TruffleObject { return context.getCompiler().generateDocs(module); } + private static Object gatherImportStatements(Module module, Context context) { + Object[] imports = context.getCompiler().gatherImportStatements(module); + return new Array(imports); + } + @Specialization static Object doInvoke( Module module, @@ -496,26 +501,32 @@ public class Module implements TruffleObject { @CachedContext(Language.class) Context context, @Cached LoopingCallOptimiserNode callOptimiserNode) throws UnknownIdentifierException, ArityException, UnsupportedTypeException { - ModuleScope scope = module.compileScope(context); + ModuleScope scope; switch (member) { case MethodNames.Module.GET_NAME: return module.getName().toString(); case MethodNames.Module.GET_METHOD: + scope = module.compileScope(context); Function result = getMethod(scope, arguments); return result == null ? context.getBuiltins().nothing().newInstance() : result; case MethodNames.Module.GET_CONSTRUCTOR: + scope = module.compileScope(context); return getConstructor(scope, arguments); case MethodNames.Module.REPARSE: return reparse(module, arguments, context); case MethodNames.Module.GENERATE_DOCS: return generateDocs(module, context); + case MethodNames.Module.GATHER_IMPORT_STATEMENTS: + return gatherImportStatements(module, context); case MethodNames.Module.SET_SOURCE: return setSource(module, arguments, context); case MethodNames.Module.SET_SOURCE_FILE: return setSourceFile(module, arguments, context); case MethodNames.Module.GET_ASSOCIATED_CONSTRUCTOR: + scope = module.compileScope(context); return getAssociatedConstructor(scope, arguments); case MethodNames.Module.EVAL_EXPRESSION: + scope = module.compileScope(context); return evalExpression(scope, arguments, context, callOptimiserNode); default: throw UnknownIdentifierException.create(member); diff --git a/engine/runtime/src/main/scala/org/enso/compiler/Compiler.scala b/engine/runtime/src/main/scala/org/enso/compiler/Compiler.scala index 27fe13b0d93..7db6c461fc1 100644 --- a/engine/runtime/src/main/scala/org/enso/compiler/Compiler.scala +++ b/engine/runtime/src/main/scala/org/enso/compiler/Compiler.scala @@ -15,6 +15,7 @@ import org.enso.compiler.phase.{ ExportsResolution, ImportResolver } +import org.enso.editions.LibraryName import org.enso.interpreter.node.{ExpressionNode => RuntimeExpression} import org.enso.interpreter.runtime.builtin.Builtins import org.enso.interpreter.runtime.scope.{LocalScope, ModuleScope} @@ -321,6 +322,32 @@ class Compiler( requiredModules } + /** Runs the initial passes of the compiler to gather the import statements, + * used for dependency resolution. + * + * @param module - the scope from which docs are generated. + */ + def gatherImportStatements(module: Module): Array[String] = { + ensureParsed(module) + val importedModules = module.getIr.imports.flatMap { + case imp: IR.Module.Scope.Import.Module => + imp.name.parts.take(2).map(_.name) match { + case List(namespace, name) => List(LibraryName(namespace, name)) + case _ => + throw new CompilerError(s"Invalid module name: [${imp.name}].") + } + + case _: IR.Module.Scope.Import.Polyglot => + // Note [Polyglot Imports In Dependency Gathering] + Nil + case other => + throw new CompilerError( + s"Unexpected import type after processing: [$other]." + ) + } + importedModules.distinct.map(_.qualifiedName).toArray + } + private def parseModule( module: Module, isGenDocs: Boolean = false @@ -364,6 +391,19 @@ class Compiler( module.setHasCrossModuleLinks(true) } + /* Note [Polyglot Imports In Dependency Gathering] + * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + * Currently we just ignore polyglot imports when gathering the dependencies - + * we assume that the project itself or one of its dependencies will contain + * in their `polyglot` directory any JARs that need to be included in the + * classpath for this import to be resolved. + * + * In the future we may want to extend the edition system with some settings + * for automatically resolving the Java dependencies using a system based on + * Maven, but currently the libraries just must include their binary + * dependencies. + */ + /** Gets a module definition by name. * * @param name the name of module to look up diff --git a/engine/runtime/src/main/scala/org/enso/compiler/PackageRepository.scala b/engine/runtime/src/main/scala/org/enso/compiler/PackageRepository.scala index 1a542ac6de0..280eadf3493 100644 --- a/engine/runtime/src/main/scala/org/enso/compiler/PackageRepository.scala +++ b/engine/runtime/src/main/scala/org/enso/compiler/PackageRepository.scala @@ -244,7 +244,7 @@ object PackageRepository { case Left(error) => logger.error(s"Resolution failed with [$error].", error) case Right(resolved) => - logger.trace( + logger.info( s"Found library ${resolved.name} @ ${resolved.version} " + s"at [${MaskedPath(resolved.location).applyMasking()}]." ) diff --git a/lib/scala/library-manager-test/src/main/scala/org/enso/librarymanager/published/repository/DummyRepository.scala b/lib/scala/library-manager-test/src/main/scala/org/enso/librarymanager/published/repository/DummyRepository.scala index 93c09d41521..fcc7f71670e 100644 --- a/lib/scala/library-manager-test/src/main/scala/org/enso/librarymanager/published/repository/DummyRepository.scala +++ b/lib/scala/library-manager-test/src/main/scala/org/enso/librarymanager/published/repository/DummyRepository.scala @@ -24,11 +24,14 @@ abstract class DummyRepository { * @param libraryName name of the library * @param version version of the library * @param mainContent contents of the `Main.enso` file + * @param dependencies libraries that this library directly depends on, to be + * included in the manifest */ case class DummyLibrary( libraryName: LibraryName, version: SemVer, - mainContent: String + mainContent: String, + dependencies: Seq[LibraryName] = Seq.empty ) /** Name of the repository, as it will be indicated in the generated edition. @@ -62,7 +65,7 @@ abstract class DummyRepository { } .get - createManifest(libraryRoot) + createManifest(libraryRoot, lib) } val editionsRoot = root.resolve("editions") @@ -96,12 +99,22 @@ abstract class DummyRepository { pkg } - private def createManifest(path: Path): Unit = { - FileSystem.writeTextFile( - path.resolve("manifest.yaml"), + private def createManifest(path: Path, lib: DummyLibrary): Unit = { + val dependencies = + if (lib.dependencies.isEmpty) "" + else + lib.dependencies + .map(name => s""" - "${name.qualifiedName}"""") + .mkString("dependencies:\n", "\n", "\n") + + val content = s"""archives: | - main.tgz - |""".stripMargin + |""".stripMargin + dependencies + + FileSystem.writeTextFile( + path.resolve("manifest.yaml"), + content ) } diff --git a/lib/scala/library-manager-test/src/test/scala/org/enso/libraryupload/LibraryUploadTest.scala b/lib/scala/library-manager-test/src/test/scala/org/enso/libraryupload/LibraryUploadTest.scala index 181b779faf7..9a70c1c9786 100644 --- a/lib/scala/library-manager-test/src/test/scala/org/enso/libraryupload/LibraryUploadTest.scala +++ b/lib/scala/library-manager-test/src/test/scala/org/enso/libraryupload/LibraryUploadTest.scala @@ -8,11 +8,12 @@ import org.enso.librarymanager.published.repository.{ EmptyRepository } import org.enso.libraryupload.auth.SimpleHeaderToken -import org.enso.pkg.PackageManager +import org.enso.pkg.{Package, PackageManager} import org.enso.testkit.WithTemporaryDirectory import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec +import java.io.File import java.nio.file.Files class LibraryUploadTest @@ -40,8 +41,12 @@ class LibraryUploadTest EmptyRepository.withServer(port, repoRoot, uploads = true) { val uploadUrl = s"http://localhost:$port/upload" val token = SimpleHeaderToken("TODO") + val dependencyExtractor = new DependencyExtractor[File] { + override def findDependencies(pkg: Package[File]): Set[LibraryName] = + Set(LibraryName("Standard", "Base")) + } import scala.concurrent.ExecutionContext.Implicits.global - LibraryUploader + LibraryUploader(dependencyExtractor) .uploadLibrary( projectRoot, uploadUrl, diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/DefaultLibraryProvider.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/DefaultLibraryProvider.scala index e0343bc2192..d59e6ba482d 100644 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/DefaultLibraryProvider.scala +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/DefaultLibraryProvider.scala @@ -17,6 +17,7 @@ import org.enso.librarymanager.published.bundles.LocalReadOnlyRepository import org.enso.librarymanager.published.cache.DownloadingLibraryCache import org.enso.librarymanager.published.{ DefaultPublishedLibraryProvider, + PublishedLibraryCache, PublishedLibraryProvider } @@ -28,7 +29,7 @@ import org.enso.librarymanager.published.{ * @param preferLocalLibraries project setting whether to use local * libraries */ -class DefaultLibraryProvider private ( +class DefaultLibraryProvider( localLibraryProvider: LocalLibraryProvider, publishedLibraryProvider: PublishedLibraryProvider, edition: Editions.ResolvedEdition, @@ -48,33 +49,43 @@ class DefaultLibraryProvider private ( val resolvedVersion = resolver .resolveLibraryVersion(libraryName, edition, preferLocalLibraries) logger.trace(s"Resolved $libraryName to [$resolvedVersion].") + resolvedVersion match { case Left(reason) => Left(ResolvingLibraryProvider.Error.NotResolved(reason)) - case Right(LibraryVersion.Local) => - localLibraryProvider - .findLibrary(libraryName) - .map(ResolvedLibrary(libraryName, LibraryVersion.Local, _)) - .toRight { - ResolvingLibraryProvider.Error.NotResolved( - LibraryResolutionError( - s"Edition configuration forces to use the local version, but " + - s"the `$libraryName` library is not present among local " + - s"libraries." - ) - ) - } - - case Right(version @ LibraryVersion.Published(semver, repository)) => - publishedLibraryProvider - .findLibrary(libraryName, semver, repository) - .map(ResolvedLibrary(libraryName, version, _)) - .toEither - .left - .map(ResolvingLibraryProvider.Error.DownloadFailed(version, _)) + case Right(version) => + findSpecificLibraryVersion(libraryName, version) } } + + /** @inheritdoc */ + override def findSpecificLibraryVersion( + libraryName: LibraryName, + version: LibraryVersion + ): Either[ResolvingLibraryProvider.Error, ResolvedLibrary] = version match { + case LibraryVersion.Local => + localLibraryProvider + .findLibrary(libraryName) + .map(ResolvedLibrary(libraryName, LibraryVersion.Local, _)) + .toRight { + ResolvingLibraryProvider.Error.NotResolved( + LibraryResolutionError( + s"Edition configuration forces to use the local version, but " + + s"the `$libraryName` library is not present among local " + + s"libraries." + ) + ) + } + + case version @ LibraryVersion.Published(semver, repository) => + publishedLibraryProvider + .findLibrary(libraryName, semver, repository) + .map(ResolvedLibrary(libraryName, version, _)) + .toEither + .left + .map(ResolvingLibraryProvider.Error.DownloadFailed(version, _)) + } } object DefaultLibraryProvider { @@ -100,6 +111,33 @@ object DefaultLibraryProvider { edition: Editions.ResolvedEdition, preferLocalLibraries: Boolean ): ResolvingLibraryProvider = { + val (localLibraryProvider, publishedLibraryProvider) = makeProviders( + distributionManager, + resourceManager, + lockUserInterface, + progressReporter, + languageHome + ) + + new DefaultLibraryProvider( + localLibraryProvider, + publishedLibraryProvider, + edition, + preferLocalLibraries + ) + } + + /** Creates a pair of local and published library providers. */ + def makeProviders( + distributionManager: DistributionManager, + resourceManager: ResourceManager, + lockUserInterface: LockUserInterface, + progressReporter: ProgressReporter, + languageHome: Option[LanguageHome] + ): ( + LocalLibraryProvider, + PublishedLibraryProvider with PublishedLibraryCache + ) = { val locations = LibraryLocations.resolve(distributionManager, languageHome) val primaryCache = new DownloadingLibraryCache( locations.primaryCacheRoot, @@ -115,12 +153,6 @@ object DefaultLibraryProvider { new DefaultLocalLibraryProvider(locations.localLibrarySearchPaths) val publishedLibraryProvider = new DefaultPublishedLibraryProvider(primaryCache, additionalCaches) - - new DefaultLibraryProvider( - localLibraryProvider, - publishedLibraryProvider, - edition, - preferLocalLibraries - ) + (localLibraryProvider, publishedLibraryProvider) } } diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/ResolvingLibraryProvider.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/ResolvingLibraryProvider.scala index 90391bb717f..b596de605b2 100644 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/ResolvingLibraryProvider.scala +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/ResolvingLibraryProvider.scala @@ -16,6 +16,19 @@ trait ResolvingLibraryProvider { def findLibrary( name: LibraryName ): Either[ResolvingLibraryProvider.Error, ResolvedLibrary] + + /** Locates a specific library version in local libraries or the cache. + * + * If it is not available, a download may be attempted. + * + * @param name name of the library + * @param version requested version of the library + * @return the resolved library containing the resulting version and path + */ + def findSpecificLibraryVersion( + name: LibraryName, + version: LibraryVersion + ): Either[ResolvingLibraryProvider.Error, ResolvedLibrary] } object ResolvingLibraryProvider { diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/dependencies/Dependency.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/dependencies/Dependency.scala new file mode 100644 index 00000000000..7bbb200c2de --- /dev/null +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/dependencies/Dependency.scala @@ -0,0 +1,15 @@ +package org.enso.librarymanager.dependencies + +import org.enso.editions.{LibraryName, LibraryVersion} + +/** Represents a resolved dependency. + * + * @param libraryName name of the library + * @param version version of the library + * @param isCached whether the library is already present in one of the caches + */ +case class Dependency( + libraryName: LibraryName, + version: LibraryVersion, + isCached: Boolean +) diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/dependencies/DependencyResolver.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/dependencies/DependencyResolver.scala new file mode 100644 index 00000000000..388fcffae7c --- /dev/null +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/dependencies/DependencyResolver.scala @@ -0,0 +1,115 @@ +package org.enso.librarymanager.dependencies + +import org.enso.editions.{Editions, LibraryName, LibraryVersion} +import org.enso.librarymanager.LibraryResolver +import org.enso.librarymanager.local.LocalLibraryProvider +import org.enso.librarymanager.published.PublishedLibraryCache +import org.enso.librarymanager.published.repository.LibraryManifest +import org.enso.librarymanager.published.repository.RepositoryHelper.RepositoryMethods +import org.enso.libraryupload.DependencyExtractor +import org.enso.pkg.PackageManager +import org.enso.yaml.YamlHelper + +import java.io.File +import java.nio.file.Files +import scala.util.Try + +/** A helper class that allows to find all transitive dependencies of a specific + * library. + */ +class DependencyResolver( + localLibraryProvider: LocalLibraryProvider, + publishedLibraryProvider: PublishedLibraryCache, + edition: Editions.ResolvedEdition, + preferLocalLibraries: Boolean, + versionResolver: LibraryResolver, + dependencyExtractor: DependencyExtractor[File] +) { + + /** Finds all transitive dependencies of the requested library. + * + * The resulting set of dependencies also includes the library itself. + */ + def findDependencies(libraryName: LibraryName): Try[Set[Dependency]] = + Try(findDependencies(libraryName, Set.empty)) + + /** A helper function to discover all transitive dependencies, avoiding + * looping on cycles. + * + * It keeps track of libraries that already have been 'visited' and if a + * library that was already visited is queried again (which is caused by + * import cycles), it returns an empty set - that is because since this + * library was already visited, it and its dependencies must have already + * been accounted for in one of the parent calls, so we can return this empty + * set at this point, because later on these dependencies will be included. + * If we didn't quit early here, we would get an infinite loop due to the + * dependency cycle. + */ + private def findDependencies( + libraryName: LibraryName, + parents: Set[LibraryName] + ): Set[Dependency] = { + if (parents.contains(libraryName)) { + Set.empty + } else { + val version = versionResolver + .resolveLibraryVersion(libraryName, edition, preferLocalLibraries) + .toTry + .get + + version match { + case LibraryVersion.Local => + val libraryPath = localLibraryProvider.findLibrary(libraryName) + val libraryPackage = libraryPath.map(path => + PackageManager.Default.loadPackage(path.toFile).get + ) + + val dependencies = libraryPackage match { + case Some(pkg) => + dependencyExtractor.findDependencies(pkg) + case None => + Set.empty + } + + val itself = Dependency(libraryName, version, libraryPath.isDefined) + + dependencies.flatMap( + findDependencies(_, parents + libraryName) + ) + itself + + case publishedVersion @ LibraryVersion.Published(semver, _) => + val itself = Dependency( + libraryName, + version, + publishedLibraryProvider.isLibraryCached(libraryName, semver) + ) + + val manifest = getManifest(libraryName, publishedVersion) + + manifest.dependencies.toSet.flatMap { name: LibraryName => + findDependencies(name, parents + libraryName) + } + itself + } + } + } + + private def getManifest( + libraryName: LibraryName, + version: LibraryVersion.Published + ): LibraryManifest = { + val cachedManifest = publishedLibraryProvider + .findCachedLibrary(libraryName, version.version) + .flatMap { libraryPath => + val manifestPath = libraryPath.resolve(LibraryManifest.filename) + if (Files.exists(manifestPath)) + YamlHelper.load[LibraryManifest](manifestPath).toOption + else None + } + cachedManifest.getOrElse { + version.repository + .accessLibrary(libraryName, version.version) + .downloadManifest() + .force() + } + } +} diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/CachedLibraryProvider.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/CachedLibraryProvider.scala index 4cb49595565..b73d8d4da56 100644 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/CachedLibraryProvider.scala +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/CachedLibraryProvider.scala @@ -31,7 +31,7 @@ class CachedLibraryProvider(caches: List[ReadOnlyLibraryCache]) } /** Looks for the library in the known caches. */ - final protected def findCached( + override def findCachedLibrary( libraryName: LibraryName, version: SemVer ): Option[Path] = findCachedHelper(libraryName, version, caches) @@ -42,7 +42,7 @@ class CachedLibraryProvider(caches: List[ReadOnlyLibraryCache]) version: SemVer, recommendedRepository: Editions.Repository ): Try[Path] = - findCached(libraryName, version) + findCachedLibrary(libraryName, version) .toRight( LibraryResolutionError( s"Library [$libraryName:$version] was not found in the cache." @@ -54,5 +54,5 @@ class CachedLibraryProvider(caches: List[ReadOnlyLibraryCache]) override def isLibraryCached( libraryName: LibraryName, version: SemVer - ): Boolean = findCached(libraryName, version).isDefined + ): Boolean = findCachedLibrary(libraryName, version).isDefined } diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/DefaultPublishedLibraryProvider.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/DefaultPublishedLibraryProvider.scala index 546b762cbfd..06ddddb7884 100644 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/DefaultPublishedLibraryProvider.scala +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/DefaultPublishedLibraryProvider.scala @@ -27,7 +27,7 @@ class DefaultPublishedLibraryProvider( version: SemVer, recommendedRepository: Editions.Repository ): Try[Path] = { - val cached = findCached(libraryName, version) + val cached = findCachedLibrary(libraryName, version) cached.map(Success(_)).getOrElse { logger.trace( s"$libraryName was not found in any caches, it will need to be " + diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/PublishedLibraryCache.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/PublishedLibraryCache.scala index 13e784c64cb..50d661a447e 100644 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/PublishedLibraryCache.scala +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/PublishedLibraryCache.scala @@ -16,6 +16,9 @@ trait PublishedLibraryCache { * caches. */ def isLibraryCached(libraryName: LibraryName, version: SemVer): Boolean + + /** Tries to locate a cached version of the requested library. */ + def findCachedLibrary(libraryName: LibraryName, version: SemVer): Option[Path] } object PublishedLibraryCache { diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala index 75c04d08883..930b63e8e92 100644 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/published/cache/DownloadingLibraryCache.scala @@ -20,6 +20,7 @@ import org.enso.librarymanager.published.repository.RepositoryHelper.{ } import org.enso.logger.masking.MaskedPath import org.enso.pkg.PackageManager +import org.enso.yaml.YamlHelper import java.nio.file.{Files, Path} import scala.util.control.NonFatal @@ -72,7 +73,6 @@ class DownloadingLibraryCache( version: SemVer, recommendedRepository: Editions.Repository ): Try[Path] = { - val _ = progressReporter // TODO val cached = findCachedLibrary(libraryName, version) cached match { case Some(result) => Success(result) @@ -81,6 +81,16 @@ class DownloadingLibraryCache( } } + private def saveManifest( + manifest: LibraryManifest, + destinationDirectory: Path + ): Unit = { + FileSystem.writeTextFile( + destinationDirectory / LibraryManifest.filename, + YamlHelper.toYaml(manifest) + ) + } + private def installLibrary( libraryName: LibraryName, version: SemVer, @@ -111,6 +121,7 @@ class DownloadingLibraryCache( try { downloadLooseFiles(libraryName, version, access, localTmpDir) downloadAndExtractArchives(libraryName, access, manifest, localTmpDir) + saveManifest(manifest, localTmpDir) verifyPackageIntegrity(localTmpDir) FileSystem.atomicMove( diff --git a/lib/scala/library-manager/src/main/scala/org/enso/libraryupload/DependencyExtractor.scala b/lib/scala/library-manager/src/main/scala/org/enso/libraryupload/DependencyExtractor.scala new file mode 100644 index 00000000000..e4b6494a14c --- /dev/null +++ b/lib/scala/library-manager/src/main/scala/org/enso/libraryupload/DependencyExtractor.scala @@ -0,0 +1,13 @@ +package org.enso.libraryupload + +import org.enso.editions.LibraryName +import org.enso.pkg.Package + +/** A general interface for a helper that allows to extract dependencies from + * the project. + */ +trait DependencyExtractor[F] { + + /** Finds dependencies of a given project package. */ + def findDependencies(pkg: Package[F]): Set[LibraryName] +} diff --git a/lib/scala/library-manager/src/main/scala/org/enso/libraryupload/LibraryUploader.scala b/lib/scala/library-manager/src/main/scala/org/enso/libraryupload/LibraryUploader.scala index be9e98ec056..4e7858fe329 100644 --- a/lib/scala/library-manager/src/main/scala/org/enso/libraryupload/LibraryUploader.scala +++ b/lib/scala/library-manager/src/main/scala/org/enso/libraryupload/LibraryUploader.scala @@ -12,17 +12,19 @@ import org.enso.downloader.archive.TarGzWriter import org.enso.downloader.http.{HTTPDownload, HTTPRequestBuilder, URIBuilder} import org.enso.editions.LibraryName import org.enso.librarymanager.published.repository.LibraryManifest +import org.enso.libraryupload.LibraryUploader.UploadFailedError import org.enso.pkg.{Package, PackageManager} import org.enso.yaml.YamlHelper +import java.io.File import java.nio.file.{Files, Path} import scala.concurrent.{ExecutionContext, Future} import scala.jdk.CollectionConverters._ import scala.util.{Failure, Success, Try, Using} /** Gathers functions used for uploading libraries. */ -object LibraryUploader { - private lazy val logger = Logger[LibraryUploader.type] +class LibraryUploader(dependencyExtractor: DependencyExtractor[File]) { + private lazy val logger = Logger[LibraryUploader] /** Uploads a library to a repository. * @@ -50,7 +52,6 @@ object LibraryUploader { } val uri = buildUploadUri(uploadUrl, pkg.libraryName, version) - val mainArchiveName = "main.tgz" val filesToIgnoreInArchive = Seq( Package.configFileName, LibraryManifest.filename @@ -64,13 +65,7 @@ object LibraryUploader { ) compressing.force() - val manifestPath = projectRoot / LibraryManifest.filename - val loadedManifest = - loadSavedManifest(manifestPath).getOrElse(LibraryManifest.empty) - val updatedManifest = - // TODO [RW] update dependencies in the manifest (#1773) - loadedManifest.copy(archives = Seq(mainArchiveName)) - FileSystem.writeTextFile(manifestPath, YamlHelper.toYaml(updatedManifest)) + updateManifest(pkg).get logger.info(s"Uploading library package to the server at [$uploadUrl].") val upload = uploadFiles( @@ -92,9 +87,24 @@ object LibraryUploader { } } - /** Indicates that the library upload has failed. */ - case class UploadFailedError(message: String) - extends RuntimeException(message) + /** Updates the project's manifest by computing its dependencies. + * + * @param pkg package of the project that is to be updated + */ + def updateManifest(pkg: Package[File]): Try[Unit] = Try { + val directDependencies = dependencyExtractor.findDependencies(pkg) + + val manifestPath = pkg.root.toPath / LibraryManifest.filename + val loadedManifest = + loadSavedManifest(manifestPath).getOrElse(LibraryManifest.empty) + val updatedManifest = loadedManifest.copy( + archives = Seq(mainArchiveName), + dependencies = directDependencies.toSeq + ) + FileSystem.writeTextFile(manifestPath, YamlHelper.toYaml(updatedManifest)) + } + + private val mainArchiveName = "main.tgz" /** Creates an URL for the upload, including information identifying the * library version. @@ -231,3 +241,13 @@ object LibraryUploader { } } } + +object LibraryUploader { + def apply(dependencyExtractor: DependencyExtractor[File]): LibraryUploader = + new LibraryUploader(dependencyExtractor) + + /** Indicates that the library upload has failed. */ + case class UploadFailedError(message: String) + extends RuntimeException(message) + +} diff --git a/lib/scala/logging-service/src/main/scala/org/enso/loggingservice/internal/service/ThreadProcessingService.scala b/lib/scala/logging-service/src/main/scala/org/enso/loggingservice/internal/service/ThreadProcessingService.scala index 8e3a9ca1443..9976878ec19 100644 --- a/lib/scala/logging-service/src/main/scala/org/enso/loggingservice/internal/service/ThreadProcessingService.scala +++ b/lib/scala/logging-service/src/main/scala/org/enso/loggingservice/internal/service/ThreadProcessingService.scala @@ -4,6 +4,7 @@ import org.enso.loggingservice.LogLevel import org.enso.loggingservice.internal.protocol.WSLogMessage import org.enso.loggingservice.internal.{ BlockingConsumerMessageQueue, + DefaultLogMessageRenderer, InternalLogger } @@ -53,6 +54,10 @@ trait ThreadProcessingService extends Service { thread.start() } + private lazy val renderer = new DefaultLogMessageRenderer( + printExceptions = false + ) + /** The runner filters out internal messages that have disabled log levels, * but passes through all external messages (as their log level is set * independently and can be lower). @@ -68,6 +73,9 @@ trait ThreadProcessingService extends Service { InternalLogger.error( s"One of the printers failed to write a message: $e" ) + InternalLogger.error( + s"The dropped message was: ${renderer.render(message)}" + ) } } } catch { diff --git a/project/LibraryManifestGenerator.scala b/project/LibraryManifestGenerator.scala new file mode 100644 index 00000000000..121e75e42f7 --- /dev/null +++ b/project/LibraryManifestGenerator.scala @@ -0,0 +1,76 @@ +import sbt._ +import sbt.util.CacheStoreFactory + +/** A helper for generating manifests for bundled libraries. */ +object LibraryManifestGenerator { + + /** Represents a library that will be bundled with the engine and needs to + * have its manifest generated. + */ + case class BundledLibrary(name: String, version: String) + + /** Generates manifests for the provided libraries. + * + * It assumes that the engine-runner/assembly task is up to date (as it uses + * its artifacts). + */ + def generateManifests( + libraries: Seq[BundledLibrary], + distributionRoot: File, + log: Logger, + cacheStoreFactory: CacheStoreFactory + ): Unit = + for (BundledLibrary(qualifiedName, version) <- libraries) { + val (namespace, name) = qualifiedName.split('.') match { + case Array(namespace, name) => (namespace, name) + case _ => + throw new IllegalArgumentException( + s"Invalid library name [$qualifiedName]." + ) + } + val projectPath = + distributionRoot / "lib" / namespace / name / version + + val store = + cacheStoreFactory.make(s"library-manifest-$namespace-$name-$version") + val sources = (projectPath / "src" allPaths).get + Tracked.diffInputs(store, FileInfo.hash)(sources.toSet) { diff => + def manifestExists = (projectPath / "manifest.yaml").exists() + if (diff.modified.nonEmpty || !manifestExists) { + log.info(s"Regenerating manifest for [$projectPath].") + runGenerator(projectPath, log) + } else { + log.debug(s"[$projectPath] manifest is up to date.") + } + } + } + + private def runGenerator(projectPath: File, log: Logger): Unit = { + val javaCommand = + ProcessHandle.current().info().command().asScala.getOrElse("java") + val command = Seq( + javaCommand, + s"-Dtruffle.class.path.append=runtime.jar", + "-jar", + "runner.jar", + "--update-manifest", + "--in-project", + projectPath.getCanonicalPath + ) + + log.debug(s"Running [$command].") + val exitCode = sys.process + .Process( + command, + None, + "ENSO_EDITION_PATH" -> file("distribution/editions").getCanonicalPath + ) + .! + if (exitCode != 0) { + val message = s"Command [$command] has failed with code $exitCode." + log.error(message) + throw new RuntimeException(message) + } + } + +}