Add Python and R to the GraalVM Bundle (#1644)

CI release pipeline is updated to install
python and R GraalVM components

.github/workflows/release.yml

@@ -142,7 +142,8 @@ jobs:
         working-directory: repo
         shell: bash
         run: >
-          DIST_OS=$(echo ${{ runner.os }} | awk '{print tolower($0)}') bash
+          GRAAL_VERSION=$(echo ${{ env.graalVersion }}) DIST_OS=$(echo
+          ${{runner.os }} | awk '{print tolower($0)}') bash
           tools/ci/prepare-distribution-env.sh
 
       - name: Prepare Launcher Distribution
@@ -163,6 +164,12 @@ jobs:
         run: |
           sleep 1
           sbt buildProjectManagerDistribution
+      - name: Prepare GraalVM Distribution
+        working-directory: repo
+        shell: bash
+        run: |
+          sleep 1
+          sbt buildGraalDistribution
 
       # Ensure that the versions encoded in the binary and in the release match
       - name: Check Versions (Unix)
@@ -216,6 +223,11 @@ jobs:
         with:
           name: ${{ env.PROJECTMANAGER_DIST_NAME }}
           path: repo/${{ env.PROJECTMANAGER_DIST_ROOT }}
+      - name: Upload the GraalVM Artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{ env.GRAAL_DIST_NAME }}
+          path: repo/${{ env.GRAAL_DIST_ROOT }}
       - name: Upload the Manifest Artifact
         uses: actions/upload-artifact@v2
         with:

build.sbt

@@ -1362,3 +1362,28 @@ buildProjectManagerDistribution := {
   DistributionPackage.createProjectManagerPackage(root, cacheFactory)
   log.info(s"Project Manager package created at $root")
 }
+
+lazy val buildGraalDistribution =
+  taskKey[Unit]("Builds the GraalVM distribution")
+buildGraalDistribution := {
+  val log    = streams.value.log
+  val distOs = "DIST_OS"
+  val osName = "os.name"
+  val distName = sys.env.get(distOs).getOrElse {
+    val name = sys.props(osName).takeWhile(!_.isWhitespace)
+    if (sys.env.contains("CI")) {
+      log.warn(
+        s"$distOs env var is empty. Fallback to system property $osName=$name."
+      )
+    }
+    name
+  }
+  val os = DistributionPackage.OS(distName).getOrElse {
+    throw new RuntimeException(s"Failed to determine OS: $distName.")
+  }
+  packageBuilder.createGraalPackage(
+    log,
+    os,
+    DistributionPackage.Architecture.X64
+  )
+}

project/DistributionPackage.scala

@@ -6,6 +6,36 @@ import sbt.util.{CacheStore, CacheStoreFactory, FileInfo, Tracked}
 import scala.sys.process._
 
 object DistributionPackage {
+
+  /** File extensions. */
+  implicit class FileExtensions(file: File) {
+
+    /** Get the outermost directory of this file. For absolute paths this
+      * function always returns root.
+      *
+      * == Example ==
+      * Get top directory of the relative path.
+      * {{{
+      *   file("foo/bar/baz").getTopDirectory == file("foo")
+      * }}}
+      *
+      * Get top directory of the absolute path.
+      * {{{
+      *   file(/foo/bar/baz").getTopDirectory == file("/")
+      * }}}
+      *
+      * @return the outermost directory of this file.
+      */
+    def getTopDirectory: File = {
+      @scala.annotation.tailrec
+      def go(path: File): File = {
+        val parent = path.getParentFile
+        if (parent == null) path else go(parent)
+      }
+      go(file)
+    }
+  }
+
   def copyDirectoryIncremental(
     source: File,
     destination: File,
@@ -150,6 +180,7 @@ object DistributionPackage {
 
   sealed trait OS {
     def name:                String
+    def hasSupportForSulong: Boolean
     def graalName: String                    = name
     def executableName(base: String): String = base
     def archiveExt: String                   = ".tar.gz"
@@ -157,20 +188,31 @@ object DistributionPackage {
   }
   object OS {
     case object Linux extends OS {
-      override def name: String = "linux"
+      override val name: String                 = "linux"
+      override val hasSupportForSulong: Boolean = true
     }
     case object MacOS extends OS {
-      override def name: String      = "macos"
+      override val name: String                 = "macos"
+      override val hasSupportForSulong: Boolean = true
       override def graalName: String            = "darwin"
     }
     case object Windows extends OS {
-      override def name: String                         = "windows"
+      override val name: String                         = "windows"
+      override val hasSupportForSulong: Boolean         = false
       override def executableName(base: String): String = base + ".exe"
       override def archiveExt: String                   = ".zip"
       override def isUNIX: Boolean                      = false
     }
 
     val platforms = Seq(Linux, MacOS, Windows)
+
+    def apply(name: String): Option[OS] =
+      name.toLowerCase match {
+        case Linux.`name`   => Some(Linux)
+        case MacOS.`name`   => Some(MacOS)
+        case Windows.`name` => Some(Windows)
+        case _              => None
+      }
   }
 
   sealed trait Architecture {
@@ -213,6 +255,14 @@ object DistributionPackage {
       }
     }
 
+    private def listZip(archive: File): Seq[File] = {
+      val suppressStdErr = ProcessLogger(_ => ())
+      val zipList = Process(
+        Seq("zip", "-l", archive.toPath.toAbsolutePath.normalize.toString)
+      )
+      zipList.lineStream(suppressStdErr).map(file)
+    }
+
     private def extractTarGz(archive: File, root: File): Unit = {
       IO.createDirectory(root)
       val exitCode = Process(
@@ -228,6 +278,13 @@ object DistributionPackage {
       }
     }
 
+    private def listTarGz(archive: File): Seq[File] = {
+      val suppressStdErr = ProcessLogger(_ => ())
+      val tarList =
+        Process(Seq("tar", "tf", archive.toPath.toAbsolutePath.toString))
+      tarList.lineStream(suppressStdErr).map(file)
+    }
+
     private def extract(archive: File, root: File): Unit = {
       if (archive.getName.endsWith("zip")) {
         extractZip(archive, root)
@@ -236,15 +293,31 @@ object DistributionPackage {
       }
     }
 
-    def copyGraal(
+    private def list(archive: File): Seq[File] = {
+      if (archive.getName.endsWith("zip")) {
+        listZip(archive)
+      } else {
+        listTarGz(archive)
+      }
+    }
+
+    private def graalArchive(os: OS, architecture: Architecture): File = {
+      val packageDir =
+        artifactRoot / s"graalvm-$graalVersion-${os.name}-${architecture.name}"
+      if (!packageDir.exists()) {
+        IO.createDirectory(packageDir)
+      }
+      val archiveName = s"graalvm-${os.name}-${architecture.name}-" +
+        s"$graalVersion-$graalJavaVersion"
+      packageDir / (archiveName + os.archiveExt)
+    }
+
+    private def downloadGraal(
       log: ManagedLogger,
       os: OS,
-      architecture: Architecture,
+      architecture: Architecture
-      runtimeDir: File
+    ): File = {
-    ): Unit = {
+      val archive = graalArchive(os, architecture)
-      val packageName = s"graalvm-${os.name}-${architecture.name}-" +
-        s"$graalVersion-$graalJavaVersion"
-      val archive = artifactRoot / (packageName + os.archiveExt)
       if (!archive.exists()) {
         log.info(
           s"Downloading GraalVM $graalVersion Java $graalJavaVersion " +
@@ -261,9 +334,94 @@ object DistributionPackage {
         }
       }
 
+      archive
+    }
+
+    private def copyGraal(
+      os: OS,
+      architecture: Architecture,
+      runtimeDir: File
+    ): Unit = {
+      val archive = graalArchive(os, architecture)
       extract(archive, runtimeDir)
     }
 
+    /** Prepare the GraalVM package.
+      *
+      * @param log the logger
+      * @param os the system type
+      * @param architecture the architecture type
+      * @return the path to the created GraalVM package
+      */
+    def createGraalPackage(
+      log: ManagedLogger,
+      os: OS,
+      architecture: Architecture
+    ): File = {
+      log.info("Building GraalVM distribution")
+      val archive = downloadGraal(log, os, architecture)
+
+      if (os.hasSupportForSulong) {
+        val packageDir        = archive.getParentFile
+        val archiveRootDir    = list(archive).head.getTopDirectory.getName
+        val extractedGraalDir = packageDir / archiveRootDir
+        if (extractedGraalDir.exists()) {
+          IO.delete(extractedGraalDir)
+        }
+
+        log.info(s"Extracting $archive to $packageDir")
+        extract(archive, packageDir)
+
+        log.info("Installing components")
+        gu(log, os, extractedGraalDir, "install", "python", "R")
+
+        log.info(s"Re-creating $archive")
+        IO.delete(archive)
+        makeArchive(packageDir, archiveRootDir, archive)
+
+        log.info(s"Cleaning up $extractedGraalDir")
+        IO.delete(extractedGraalDir)
+      }
+      archive
+    }
+
+    /** Run the `gu` executable from the GraalVM distribution.
+      *
+      * @param log the logger
+      * @param os the system type
+      * @param graalDir the directory with a GraalVM distribution
+      * @param arguments the command arguments
+      */
+    def gu(
+      log: ManagedLogger,
+      os: OS,
+      graalDir: File,
+      arguments: String*
+    ): Unit = {
+      val executableFile = os match {
+        case OS.Linux =>
+          graalDir / "bin" / "gu"
+        case OS.MacOS =>
+          graalDir / "Contents" / "Home" / "bin" / "gu"
+        case OS.Windows =>
+          graalDir / "bin" / "gu.cmd"
+      }
+      val javaHomeFile = executableFile.getParentFile.getParentFile
+      val command =
+        executableFile.toPath.toAbsolutePath.toString +: arguments
+      val exitCode = Process(
+        command,
+        Some(graalDir),
+        ("JAVA_HOME", javaHomeFile.toPath.toAbsolutePath.toString),
+        ("GRAALVM_HOME", javaHomeFile.toPath.toAbsolutePath.toString)
+      ).!
+      if (exitCode != 0) {
+        throw new RuntimeException(
+          s"Failed to run '${command.mkString(" ")}'"
+        )
+      }
+    }
+
     def copyEngine(os: OS, architecture: Architecture, distDir: File): Unit = {
       val engine = builtArtifact("engine", os, architecture)
       if (!engine.exists()) {
@@ -351,6 +509,12 @@ object DistributionPackage {
         architecture
       ) + os.archiveExt)
 
+    private def cleanDirectory(dir: File): Unit = {
+      for (f <- IO.listFiles(dir)) {
+        IO.delete(f)
+      }
+    }
+
     /** Creates compressed and ready for release packages for the launcher and
       * engine.
       *
@@ -390,12 +554,6 @@ object DistributionPackage {
       state
     }
 
-    private def cleanDirectory(dir: File): Unit = {
-      for (f <- IO.listFiles(dir)) {
-        IO.delete(f)
-      }
-    }
-
     /** Creates launcher and project-manager bundles that include the component
       * itself, the engine and a Graal runtime.
       *
@@ -418,7 +576,7 @@ object DistributionPackage {
         if (launcher.exists()) {
           fixLauncher(launcher, os)
           copyEngine(os, arch, launcher / "enso" / "dist")
-          copyGraal(log, os, arch, launcher / "enso" / "runtime")
+          copyGraal(os, arch, launcher / "enso" / "runtime")
 
           val archive = builtArchive("bundle", os, arch)
           makeArchive(launcher, "enso", archive)
@@ -436,7 +594,8 @@ object DistributionPackage {
           }
 
           copyEngine(os, arch, pm / "enso" / "dist")
-          copyGraal(log, os, arch, pm / "enso" / "runtime")
+          copyGraal(os, arch, pm / "enso" / "runtime")
+
           IO.copyFile(
             file("distribution/enso.bundle.template"),
             pm / "enso" / ".enso.bundle"

tools/ci/prepare-distribution-env.sh

@@ -10,6 +10,10 @@ ENGINE_DIST_DIR=$ENGINE_DIST_ROOT/enso-$DIST_VERSION
 PROJECTMANAGER_DIST_NAME=enso-project-manager-$DIST_VERSION-$DIST_OS-$DIST_ARCH
 PROJECTMANAGER_DIST_ROOT=$BUILD_ROOT/$PROJECTMANAGER_DIST_NAME
 PROJECTMANAGER_DIST_DIR=$PROJECTMANAGER_DIST_ROOT/enso
+GRAAL_DIST_NAME=graalvm-$GRAAL_VERSION-$DIST_OS-$DIST_ARCH
+GRAAL_DIST_ROOT=$BUILD_ROOT/$GRAAL_DIST_NAME
+echo "DIST_OS=$DIST_OS" >> $GITHUB_ENV
+echo "DIST_ARCH=$DIST_ARCH" >> $GITHUB_ENV
 echo "LAUNCHER_DIST_NAME=$LAUNCHER_DIST_NAME" >> $GITHUB_ENV
 echo "LAUNCHER_DIST_DIR=$LAUNCHER_DIST_DIR" >> $GITHUB_ENV
 echo "LAUNCHER_DIST_ROOT=$LAUNCHER_DIST_ROOT" >> $GITHUB_ENV
@@ -19,3 +23,5 @@ echo "ENGINE_DIST_ROOT=$ENGINE_DIST_ROOT" >> $GITHUB_ENV
 echo "PROJECTMANAGER_DIST_NAME=$PROJECTMANAGER_DIST_NAME" >> $GITHUB_ENV
 echo "PROJECTMANAGER_DIST_DIR=$PROJECTMANAGER_DIST_DIR" >> $GITHUB_ENV
 echo "PROJECTMANAGER_DIST_ROOT=$PROJECTMANAGER_DIST_ROOT" >> $GITHUB_ENV
+echo "GRAAL_DIST_NAME=$GRAAL_DIST_NAME" >> $GITHUB_ENV
+echo "GRAAL_DIST_ROOT=$GRAAL_DIST_ROOT" >> $GITHUB_ENV