enso/project/DistributionPackage.scala
Hubert Plociniczak 72c4006cb1
Additional sbt commands to build stdlib packages (#3431)
Auxiliary sbt commands for building individual
stdlib packages.
The commands check if the engine distribution was built at least once,
and only copy the necessary package files if necessary.
So far added:
- `buildStdLibBase`
- `buildStdLibDatabase`
- `buildStdLibTable`
- `buildStdLibImage`
- `buildStdLibGoogle_Api`

Related to [#182014385](https://www.pivotaltracker.com/story/show/182014385)
2022-05-11 10:12:18 +00:00

717 lines
21 KiB
Scala

import io.circe.yaml
import io.circe.syntax._
import sbt.internal.util.ManagedLogger
import sbt._
import sbt.io.syntax.fileToRichFile
import sbt.util.{CacheStore, CacheStoreFactory, FileInfo, Tracked}
import scala.sys.process._
object DistributionPackage {
/** File extensions. */
implicit class FileExtensions(file: File) {
/** Get the outermost directory of this file. For absolute paths this
* function always returns root.
*
* == Example ==
* Get top directory of the relative path.
* {{{
* file("foo/bar/baz").getTopDirectory == file("foo")
* }}}
*
* Get top directory of the absolute path.
* {{{
* file(/foo/bar/baz").getTopDirectory == file("/")
* }}}
*
* @return the outermost directory of this file.
*/
def getTopDirectory: File = {
@scala.annotation.tailrec
def go(path: File): File = {
val parent = path.getParentFile
if (parent == null) path else go(parent)
}
go(file)
}
}
/**
* Conditional copying, based on the contents of cache and timestamps of files.
*
* @param source source directory
* @param destination target directory
* @param cache cache used for persisting the cached information
* @return true, if copying was necessary, false if no change was detected between the directories
*/
def copyDirectoryIncremental(
source: File,
destination: File,
cache: CacheStore
): Boolean = {
val allFiles = source.allPaths.get().toSet
Tracked.diffInputs(cache, FileInfo.lastModified)(allFiles) { diff =>
val missing = diff.unmodified.exists { f =>
val relativePath = f.relativeTo(source).get
val destinationFile =
destination.toPath.resolve(relativePath.toPath).toFile
!destinationFile.exists()
}
if (diff.modified.nonEmpty || diff.removed.nonEmpty || missing) {
IO.delete(destination)
IO.copyDirectory(source, destination)
true
} else false
}
}
def copyFilesIncremental(
sources: Seq[File],
destinationDirectory: File,
cache: CacheStore
): Unit = {
val allFiles = sources.toSet
IO.createDirectory(destinationDirectory)
Tracked.diffInputs(cache, FileInfo.lastModified)(allFiles) { diff =>
for (f <- diff.removed) {
IO.delete(destinationDirectory / f.getName)
}
for (f <- diff.modified -- diff.removed) {
IO.copyFile(f, destinationDirectory / f.getName)
}
for (f <- diff.unmodified) {
val destinationFile = destinationDirectory / f.getName
if (!destinationFile.exists()) {
IO.copyFile(f, destinationDirectory / f.getName)
}
}
}
}
def executableName(baseName: String): String =
if (Platform.isWindows) baseName + ".exe" else baseName
def createProjectManagerPackage(
distributionRoot: File,
cacheFactory: CacheStoreFactory
): Unit = {
copyDirectoryIncremental(
file("distribution/project-manager/THIRD-PARTY"),
distributionRoot / "THIRD-PARTY",
cacheFactory.make("project-manager-third-party")
)
copyFilesIncremental(
Seq(file(executableName("project-manager"))),
distributionRoot / "bin",
cacheFactory.make("project-manager-exe")
)
}
def createEnginePackage(
distributionRoot: File,
cacheFactory: CacheStoreFactory,
log: Logger,
graalVersion: String,
javaVersion: String,
ensoVersion: String,
editionName: String,
sourceStdlibVersion: String,
targetStdlibVersion: String
): Unit = {
copyDirectoryIncremental(
file("distribution/engine/THIRD-PARTY"),
distributionRoot / "THIRD-PARTY",
cacheFactory.make("engine-third-party")
)
copyFilesIncremental(
Seq(file("runtime.jar"), file("runner.jar")),
distributionRoot / "component",
cacheFactory.make("engine-jars")
)
(distributionRoot / "editions").mkdirs()
Editions.writeEditionConfig(
editionsRoot = distributionRoot / "editions",
ensoVersion = ensoVersion,
editionName = editionName,
libraryVersion = targetStdlibVersion,
log = log
)
copyLibraryCacheIncremental(
sourceRoot = file("distribution/lib"),
destinationRoot = distributionRoot / "lib",
sourceVersion = sourceStdlibVersion,
targetVersion = targetStdlibVersion,
cacheFactory = cacheFactory.sub("engine-libraries"),
log = log
)
copyDirectoryIncremental(
file("distribution/bin"),
distributionRoot / "bin",
cacheFactory.make("engine-bin")
)
buildEngineManifest(
template = file("distribution/manifest.template.yaml"),
destination = distributionRoot / "manifest.yaml",
graalVersion = graalVersion,
javaVersion = javaVersion
)
}
def fixLibraryManifest(
packageRoot: File,
targetVersion: String,
log: Logger
): Unit = {
val packageConfig = packageRoot / "package.yaml"
val originalContent = IO.read(packageConfig)
yaml.parser.parse(originalContent) match {
case Left(error) =>
log.error(s"Failed to parse $packageConfig: $error")
throw error
case Right(parsed) =>
val obj = parsed.asObject.getOrElse {
throw new IllegalStateException(s"Incorrect format of $packageConfig")
}
val key = "version"
val updated = obj.remove(key).add(key, targetVersion.asJson)
val serialized = yaml.printer.print(updated.asJson)
if (serialized == originalContent) {
log.info(
s"No need to update $packageConfig, already in correct version."
)
} else {
IO.write(packageConfig, serialized)
log.debug(s"Updated $packageConfig to $targetVersion")
}
}
}
def copyLibraryCacheIncremental(
sourceRoot: File,
destinationRoot: File,
sourceVersion: String,
targetVersion: String,
cacheFactory: CacheStoreFactory,
log: Logger
): Unit = {
val existingLibraries =
collection.mutable.ArrayBuffer.empty[(String, String)]
for (prefix <- sourceRoot.list()) {
for (libName <- (sourceRoot / prefix).list()) {
val targetPackageRoot =
destinationRoot / prefix / libName / targetVersion
copyDirectoryIncremental(
source = sourceRoot / prefix / libName / sourceVersion,
destination = targetPackageRoot,
cache = cacheFactory.make(s"$prefix.$libName")
)
fixLibraryManifest(targetPackageRoot, targetVersion, log)
existingLibraries.append((prefix, libName))
}
}
val existingLibrariesSet = existingLibraries.toSet
for (prefix <- destinationRoot.list()) {
for (libName <- (destinationRoot / prefix).list()) {
if (!existingLibrariesSet.contains((prefix, libName))) {
log.info(
s"Removing a library $prefix.$libName from the distribution, " +
s"because it does not exist in the sources anymore."
)
}
}
}
}
private def buildEngineManifest(
template: File,
destination: File,
graalVersion: String,
javaVersion: String
): Unit = {
val base = IO.read(template)
val extensions =
s"""graal-vm-version: $graalVersion
|graal-java-version: $javaVersion
|""".stripMargin
IO.write(destination, base + extensions)
}
def createLauncherPackage(
distributionRoot: File,
cacheFactory: CacheStoreFactory
): Unit = {
copyDirectoryIncremental(
file("distribution/launcher/THIRD-PARTY"),
distributionRoot / "THIRD-PARTY",
cacheFactory.make("launcher-third-party")
)
copyFilesIncremental(
Seq(file(executableName("enso"))),
distributionRoot / "bin",
cacheFactory.make("launcher-exe")
)
IO.createDirectory(distributionRoot / "dist")
IO.createDirectory(distributionRoot / "runtime")
copyFilesIncremental(
Seq(
file("distribution/launcher/.enso.portable"),
file("distribution/launcher/README.md")
),
distributionRoot,
cacheFactory.make("launcher-rootfiles")
)
}
sealed trait OS {
def name: String
def hasSupportForSulong: Boolean
def graalName: String = name
def executableName(base: String): String = base
def archiveExt: String = ".tar.gz"
def isUNIX: Boolean = true
}
object OS {
case object Linux extends OS {
override val name: String = "linux"
override val hasSupportForSulong: Boolean = true
}
case object MacOS extends OS {
override val name: String = "macos"
override val hasSupportForSulong: Boolean = true
override def graalName: String = "darwin"
}
case object Windows extends OS {
override val name: String = "windows"
override val hasSupportForSulong: Boolean = false
override def executableName(base: String): String = base + ".exe"
override def archiveExt: String = ".zip"
override def isUNIX: Boolean = false
}
val platforms = Seq(Linux, MacOS, Windows)
def apply(name: String): Option[OS] =
name.toLowerCase match {
case Linux.`name` => Some(Linux)
case MacOS.`name` => Some(MacOS)
case Windows.`name` => Some(Windows)
case _ => None
}
}
sealed trait Architecture {
def name: String
}
object Architecture {
case object X64 extends Architecture {
override def name: String = "amd64"
}
val archs = Seq(X64)
}
/** A helper class that manages building distribution artifacts. */
class Builder(
ensoVersion: String,
graalVersion: String,
graalJavaVersion: String,
artifactRoot: File
) {
def artifactName(
component: String,
os: OS,
architecture: Architecture
): String =
s"enso-$component-$ensoVersion-${os.name}-${architecture.name}"
def graalInPackageName: String =
s"graalvm-ce-java$graalJavaVersion-$graalVersion"
private def extractZip(archive: File, root: File): Unit = {
IO.createDirectory(root)
val exitCode = Process(
Seq("unzip", "-q", archive.toPath.toAbsolutePath.normalize.toString),
cwd = Some(root)
).!
if (exitCode != 0) {
throw new RuntimeException(s"Cannot extract $archive.")
}
}
private def listZip(archive: File): Seq[File] = {
val suppressStdErr = ProcessLogger(_ => ())
val zipList = Process(
Seq("zip", "-l", archive.toPath.toAbsolutePath.normalize.toString)
)
zipList.lineStream(suppressStdErr).map(file)
}
private def extractTarGz(archive: File, root: File): Unit = {
IO.createDirectory(root)
val exitCode = Process(
Seq(
"tar",
"xf",
archive.toPath.toAbsolutePath.toString
),
cwd = Some(root)
).!
if (exitCode != 0) {
throw new RuntimeException(s"Cannot extract $archive.")
}
}
private def listTarGz(archive: File): Seq[File] = {
val suppressStdErr = ProcessLogger(_ => ())
val tarList =
Process(Seq("tar", "tf", archive.toPath.toAbsolutePath.toString))
tarList.lineStream(suppressStdErr).map(file)
}
private def extract(archive: File, root: File): Unit = {
if (archive.getName.endsWith("zip")) {
extractZip(archive, root)
} else {
extractTarGz(archive, root)
}
}
private def list(archive: File): Seq[File] = {
if (archive.getName.endsWith("zip")) {
listZip(archive)
} else {
listTarGz(archive)
}
}
private def graalArchive(os: OS, architecture: Architecture): File = {
val packageDir =
artifactRoot / s"graalvm-$graalVersion-${os.name}-${architecture.name}"
if (!packageDir.exists()) {
IO.createDirectory(packageDir)
}
val archiveName = s"graalvm-${os.name}-${architecture.name}-" +
s"$graalVersion-$graalJavaVersion"
packageDir / (archiveName + os.archiveExt)
}
private def downloadGraal(
log: ManagedLogger,
os: OS,
architecture: Architecture
): File = {
val archive = graalArchive(os, architecture)
if (!archive.exists()) {
log.info(
s"Downloading GraalVM $graalVersion Java $graalJavaVersion " +
s"for $os $architecture"
)
val graalUrl =
s"https://github.com/graalvm/graalvm-ce-builds/releases/download/" +
s"vm-$graalVersion/" +
s"graalvm-ce-java$graalJavaVersion-${os.graalName}-" +
s"${architecture.name}-$graalVersion${os.archiveExt}"
val exitCode = (url(graalUrl) #> archive).!
if (exitCode != 0) {
throw new RuntimeException(s"Graal download from $graalUrl failed.")
}
}
archive
}
private def copyGraal(
os: OS,
architecture: Architecture,
runtimeDir: File
): Unit = {
val archive = graalArchive(os, architecture)
extract(archive, runtimeDir)
}
/** Prepare the GraalVM package.
*
* @param log the logger
* @param os the system type
* @param architecture the architecture type
* @return the path to the created GraalVM package
*/
def createGraalPackage(
log: ManagedLogger,
os: OS,
architecture: Architecture
): File = {
log.info("Building GraalVM distribution")
val archive = downloadGraal(log, os, architecture)
if (os.hasSupportForSulong) {
val packageDir = archive.getParentFile
val archiveRootDir = list(archive).head.getTopDirectory.getName
val extractedGraalDir = packageDir / archiveRootDir
if (extractedGraalDir.exists()) {
IO.delete(extractedGraalDir)
}
log.info(s"Extracting $archive to $packageDir")
extract(archive, packageDir)
log.info("Installing components")
gu(log, os, extractedGraalDir, "install", "python", "R")
log.info(s"Re-creating $archive")
IO.delete(archive)
makeArchive(packageDir, archiveRootDir, archive)
log.info(s"Cleaning up $extractedGraalDir")
IO.delete(extractedGraalDir)
}
archive
}
/** Run the `gu` executable from the GraalVM distribution.
*
* @param log the logger
* @param os the system type
* @param graalDir the directory with a GraalVM distribution
* @param arguments the command arguments
*/
def gu(
log: ManagedLogger,
os: OS,
graalDir: File,
arguments: String*
): Unit = {
val executableFile = os match {
case OS.Linux =>
graalDir / "bin" / "gu"
case OS.MacOS =>
graalDir / "Contents" / "Home" / "bin" / "gu"
case OS.Windows =>
graalDir / "bin" / "gu.cmd"
}
val javaHomeFile = executableFile.getParentFile.getParentFile
val command =
executableFile.toPath.toAbsolutePath.toString +: arguments
val exitCode = Process(
command,
Some(graalDir),
("JAVA_HOME", javaHomeFile.toPath.toAbsolutePath.toString),
("GRAALVM_HOME", javaHomeFile.toPath.toAbsolutePath.toString)
).!
if (exitCode != 0) {
throw new RuntimeException(
s"Failed to run '${command.mkString(" ")}'"
)
}
}
def copyEngine(os: OS, architecture: Architecture, distDir: File): Unit = {
val engine = builtArtifact("engine", os, architecture)
if (!engine.exists()) {
throw new IllegalStateException(
s"Cannot create bundle for $os / $architecture because corresponding " +
s"engine has not been built."
)
}
IO.copyDirectory(engine / s"enso-$ensoVersion", distDir / ensoVersion)
}
def makeExecutable(file: File): Unit = {
val ownerOnly = false
file.setExecutable(true, ownerOnly)
}
def fixLauncher(root: File, os: OS): Unit = {
makeExecutable(root / "enso" / "bin" / os.executableName("enso"))
IO.createDirectories(
Seq("dist", "config", "runtime").map(root / "enso" / _)
)
}
def makeArchive(root: File, rootDir: String, target: File): Unit = {
val exitCode = if (target.getName.endsWith("zip")) {
Process(
Seq(
"zip",
"-9",
"-q",
"-r",
target.toPath.toAbsolutePath.normalize.toString,
rootDir
),
cwd = Some(root)
).!
} else {
Process(
Seq(
"tar",
"--use-compress-program=gzip -9",
"-cf",
target.toPath.toAbsolutePath.normalize.toString,
rootDir
),
cwd = Some(root)
).!
}
if (exitCode != 0) {
throw new RuntimeException(s"Failed to create archive $target")
}
}
/** Path to an arbitrary built artifact. */
def builtArtifact(
component: String,
os: OS,
architecture: Architecture
): File = artifactRoot / artifactName(component, os, architecture)
/** Path to the artifact that is built on this local machine. */
def localArtifact(component: String): File = {
val architecture = Architecture.X64
val os =
if (Platform.isWindows) OS.Windows
else if (Platform.isLinux) OS.Linux
else if (Platform.isMacOS) OS.MacOS
else throw new IllegalStateException("Unknown OS")
artifactRoot / artifactName(component, os, architecture)
}
/** Path to a built archive.
*
* These archives are built by [[makePackages]] and [[makeBundles]].
*/
def builtArchive(
component: String,
os: OS,
architecture: Architecture
): File =
artifactRoot / (artifactName(
component,
os,
architecture
) + os.archiveExt)
private def cleanDirectory(dir: File): Unit = {
for (f <- IO.listFiles(dir)) {
IO.delete(f)
}
}
/** Creates compressed and ready for release packages for the launcher and
* engine.
*
* A project manager package is not created, as we release only its bundle.
* See [[makeBundles]].
*
* It does not trigger any builds. Instead, it uses available artifacts
* placed in `artifactRoot`. These artifacts may be created using the
* `enso/build*Distribution` tasks or they may come from other workers (as
* is the case in the release CI where the artifacts are downloaded from
* other jobs).
*/
def makePackages = Command.command("makePackages") { state =>
val log = state.log
for {
os <- OS.platforms
arch <- Architecture.archs
} {
val launcher = builtArtifact("launcher", os, arch)
if (launcher.exists()) {
fixLauncher(launcher, os)
val archive = builtArchive("launcher", os, arch)
makeArchive(launcher, "enso", archive)
log.info(s"Created $archive")
}
val engine = builtArtifact("engine", os, arch)
if (engine.exists()) {
if (os.isUNIX) {
makeExecutable(engine / s"enso-$ensoVersion" / "bin" / "enso")
}
val archive = builtArchive("engine", os, arch)
makeArchive(engine, s"enso-$ensoVersion", archive)
log.info(s"Created $archive")
}
}
state
}
/** Creates launcher and project-manager bundles that include the component
* itself, the engine and a Graal runtime.
*
* It will download the GraalVM runtime and cache it in `artifactRoot` so
* further invocations for the same version will not need to download it.
*
* It does not trigger any builds. Instead, it uses available artifacts
* placed in `artifactRoot`. These artifacts may be created using the
* `enso/build*Distribution` tasks or they may come from other workers (as
* is the case in the release CI where the artifacts are downloaded from
* other jobs).
*/
def makeBundles = Command.command("makeBundles") { state =>
val log = state.log
for {
os <- OS.platforms
arch <- Architecture.archs
} {
val launcher = builtArtifact("launcher", os, arch)
if (launcher.exists()) {
fixLauncher(launcher, os)
copyEngine(os, arch, launcher / "enso" / "dist")
copyGraal(os, arch, launcher / "enso" / "runtime")
val archive = builtArchive("bundle", os, arch)
makeArchive(launcher, "enso", archive)
cleanDirectory(launcher / "enso" / "dist")
cleanDirectory(launcher / "enso" / "runtime")
log.info(s"Created $archive")
}
val pm = builtArtifact("project-manager", os, arch)
if (pm.exists()) {
if (os.isUNIX) {
makeExecutable(pm / "enso" / "bin" / "project-manager")
}
copyEngine(os, arch, pm / "enso" / "dist")
copyGraal(os, arch, pm / "enso" / "runtime")
IO.copyFile(
file("distribution/enso.bundle.template"),
pm / "enso" / ".enso.bundle"
)
val archive = builtArchive("project-manager", os, arch)
makeArchive(pm, "enso", archive)
cleanDirectory(pm / "enso" / "dist")
cleanDirectory(pm / "enso" / "runtime")
log.info(s"Created $archive")
}
}
state
}
}
}