From 125af6b7fef88f5ef79cda2ec217536b0cf36b95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Fri, 28 Aug 2020 13:03:09 +0200 Subject: [PATCH] Bump GraalVM Version to 20.2.0 (#1094) --- .github/ISSUE_TEMPLATE/bug-report.md | 10 +- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/rust.yml | 106 +++++++-------- .github/workflows/scala.yml | 7 +- build.sbt | 33 ++++- docs/CONTRIBUTING.md | 8 +- docs/distribution/packaging.md | 2 +- docs/distribution/release-policy.md | 2 +- docs/getting-enso.md | 6 +- docs/infrastructure/README.md | 3 + docs/infrastructure/native-image.md | 19 ++- docs/infrastructure/rust.md | 4 +- docs/infrastructure/upgrading-graalvm.md | 27 ++++ .../workarounds/ReplacementStatics.java | 24 ++++ .../scala/org/enso/launcher/Launcher.scala | 5 +- .../enso/launcher/workarounds/Unsafe.scala | 22 +++ .../dispatch/LoopingCallOptimiserNode.java | 4 +- .../org/enso/version/VersionDescription.scala | 10 +- project/NativeImage.scala | 125 ++++++++++++++---- 20 files changed, 310 insertions(+), 111 deletions(-) create mode 100644 docs/infrastructure/upgrading-graalvm.md create mode 100644 engine/launcher/src/main/java/org/enso/launcher/workarounds/ReplacementStatics.java create mode 100644 engine/launcher/src/main/scala/org/enso/launcher/workarounds/Unsafe.scala diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md index 84ac0706fa..7f2e9fc048 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.md +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -58,10 +58,10 @@ Please list the reproduction steps for your bug. For example: For example: ``` Enso Compiler and Runtime -Version: 0.0.1 -Built with: scala-2.13.3 for GraalVM 20.1.0 -Built from: main @ 919ffbdfacc44cc35a1b38f1bad5b573acdbe358 -Running on: OpenJDK 64-Bit Server VM, GraalVM Community, JDK 11.0.7+10-jvmci-20.1-b02 - Linux 4.15.0-108-generic (amd64) +Version: 0.1.0 +Built with: scala-2.13.3 for GraalVM 20.2.0 +Built from: main* @ 919ffbdfacc44cc35a1b38f1bad5b573acdbe358 +Running on: OpenJDK 64-Bit Server VM, GraalVM Community, JDK 11.0.8+10-jvmci-20.2-b03 + Linux 4.15.0-112-generic (amd64) ``` --> diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index c3a5a4abbc..62d059bfe8 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -11,7 +11,7 @@ on: env: # Please ensure that this is in sync with graalAPIVersion in build.sbt - graalVersion: 20.1.0 + graalVersion: 20.2.0 javaVersion: java11 # Please ensure that this is in sync with project/build.properties sbtVersion: 1.3.13 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2db00239d7..17e6faee1f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,7 +7,7 @@ on: env: # Please ensure that this is in sync with graalVersion in build.sbt - graalVersion: 20.1.0 + graalVersion: 20.2.0 # Please ensure that this is in sync with javaVersion in build.sbt javaVersion: 11 # Please ensure that this is in sync with project/build.properties diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 832d5ac08b..9f46379e2f 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -29,19 +29,21 @@ jobs: toolchain: ${{ env.rustToolchain }} override: true - # Caches - - name: Cache Cargo Registry - uses: actions/cache@v2 - with: - path: ~/.cargo/registry - key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**Cargo.toml') }} - restore-keys: ${{ runner.os }}-cargo-registry - - name: Cache Cargo Test - uses: actions/cache@v2 - with: - path: ./target/rust - key: ${{ runner.os }}-cargo-build-${{ hashFiles('**Cargo.toml') }} - restore-keys: ${{ runner.os }}-cargo-build + # TODO [AA] at some point when the caching bug is fixed we will want to + # re-enable the caches + # # Caches + # - name: Cache Cargo Registry + # uses: actions/cache@v2 + # with: + # path: ~/.cargo/registry + # key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**Cargo.toml') }} + # restore-keys: ${{ runner.os }}-cargo-registry + # - name: Cache Cargo Test + # uses: actions/cache@v2 + # with: + # path: ./target/rust + # key: ${{ runner.os }}-cargo-build-${{ hashFiles('**Cargo.toml') }} + # restore-keys: ${{ runner.os }}-cargo-build # Lint - name: Check Code @@ -69,19 +71,19 @@ jobs: - name: Install Clippy run: rustup component add clippy - # Caches - - name: Cache Cargo Registry - uses: actions/cache@v2 - with: - path: ~/.cargo/registry - key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**Cargo.toml') }} - restore-keys: ${{ runner.os }}-cargo-registry - - name: Cache Cargo Test - uses: actions/cache@v2 - with: - path: ./target/rust - key: ${{ runner.os }}-cargo-build-${{ hashFiles('**Cargo.toml') }} - restore-keys: ${{ runner.os }}-cargo-build + # # Caches + # - name: Cache Cargo Registry + # uses: actions/cache@v2 + # with: + # path: ~/.cargo/registry + # key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**Cargo.toml') }} + # restore-keys: ${{ runner.os }}-cargo-registry + # - name: Cache Cargo Test + # uses: actions/cache@v2 + # with: + # path: ./target/rust + # key: ${{ runner.os }}-cargo-build-${{ hashFiles('**Cargo.toml') }} + # restore-keys: ${{ runner.os }}-cargo-build # Lint - name: Lint Code @@ -109,19 +111,19 @@ jobs: toolchain: ${{ env.rustToolchain }} override: true - # Caches - - name: Cache Cargo Registry - uses: actions/cache@v2 - with: - path: ~/.cargo/registry - key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**Cargo.toml') }} - restore-keys: ${{ runner.os }}-cargo-registry - - name: Cache Cargo Test - uses: actions/cache@v2 - with: - path: ./target/rust - key: ${{ runner.os }}-cargo-build-${{ hashFiles('**Cargo.toml') }} - restore-keys: ${{ runner.os }}-cargo-build + # # Caches + # - name: Cache Cargo Registry + # uses: actions/cache@v2 + # with: + # path: ~/.cargo/registry + # key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**Cargo.toml') }} + # restore-keys: ${{ runner.os }}-cargo-registry + # - name: Cache Cargo Test + # uses: actions/cache@v2 + # with: + # path: ./target/rust + # key: ${{ runner.os }}-cargo-build-${{ hashFiles('**Cargo.toml') }} + # restore-keys: ${{ runner.os }}-cargo-build # Tests - name: Test Native @@ -165,19 +167,19 @@ jobs: mv $WASMPACKDIR/wasm-pack ~/.cargo/bin rm -r $WASMPACKDIR - # Caches - - name: Cache Cargo Registry - uses: actions/cache@v2 - with: - path: ~/.cargo/registry - key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**Cargo.toml') }} - restore-keys: ${{ runner.os }}-cargo-registry - - name: Cache Cargo Test - uses: actions/cache@v2 - with: - path: ./target/rust - key: ${{ runner.os }}-cargo-build-${{ hashFiles('**Cargo.toml') }} - restore-keys: ${{ runner.os }}-cargo-build + # # Caches + # - name: Cache Cargo Registry + # uses: actions/cache@v2 + # with: + # path: ~/.cargo/registry + # key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**Cargo.toml') }} + # restore-keys: ${{ runner.os }}-cargo-registry + # - name: Cache Cargo Test + # uses: actions/cache@v2 + # with: + # path: ./target/rust + # key: ${{ runner.os }}-cargo-build-${{ hashFiles('**Cargo.toml') }} + # restore-keys: ${{ runner.os }}-cargo-build # Tests - name: Test WASM diff --git a/.github/workflows/scala.yml b/.github/workflows/scala.yml index cd9abcd03b..b6aba3fa5c 100644 --- a/.github/workflows/scala.yml +++ b/.github/workflows/scala.yml @@ -8,7 +8,7 @@ on: env: # Please ensure that this is in sync with graalVersion in build.sbt - graalVersion: 20.1.0 + graalVersion: 20.2.0 # Please ensure that this is in sync with javaVersion in build.sbt javaVersion: 11 # Please ensure that this is in sync with project/build.properties @@ -100,6 +100,11 @@ jobs: run: | echo '::set-env name=CI_TEST_TIMEFACTOR::2' echo '::set-env name=CI_TEST_FLAKY_ENABLE::true' + - name: Build the Launcher + run: | + sleep 1 + sbt --no-colors launcher/buildNativeImage + echo '::set-env name=LAUNCHER_NATIVE_IMAGE_TEST_SKIP_BUILD::true' - name: Test Enso run: | sleep 1 diff --git a/build.sbt b/build.sbt index 3218da89eb..3501f80713 100644 --- a/build.sbt +++ b/build.sbt @@ -18,7 +18,7 @@ import com.typesafe.sbt.license.DepModuleInfo val scalacVersion = "2.13.3" val rustVersion = "1.40.0-nightly (b520af6fd 2019-11-03)" -val graalVersion = "20.1.0" +val graalVersion = "20.2.0" val javaVersion = "11" val ensoVersion = "0.1.0" // Note [Engine And Launcher Version] @@ -1002,7 +1002,11 @@ lazy val runner = project ) .settings( buildNativeImage := NativeImage - .buildNativeImage("enso", staticOnLinux = false) + .buildNativeImage( + "enso", + staticOnLinux = false, + Seq("-H:IncludeResources=.*Main.enso$") + ) .value ) .settings( @@ -1036,10 +1040,13 @@ lazy val launcher = project staticOnLinux = true, Seq( "--enable-all-security-services", // Note [HTTPS in the Launcher] - "-Dorg.apache.commons.logging.Log=org.apache.commons.logging.impl.NoOpLog" + "-Dorg.apache.commons.logging.Log=org.apache.commons.logging.impl.NoOpLog", + "-H:IncludeResources=.*Main.enso$" ) ) .value, + // Note [Native Image Workaround for GraalVM 20.2] + libraryDependencies += "org.graalvm.nativeimage" % "svm" % "20.2.0" % "provided", test in assembly := {}, assemblyOutputPath in assembly := file("launcher.jar") ) @@ -1069,3 +1076,23 @@ lazy val launcher = project * `--enable-all-security-services` flag is used to ensure it is available in * the built executable. */ + +/* Note [Native Image Workaround for GraalVM 20.2] + * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + * In GraalVM 20.2 the Native Image build of even simple Scala programs has + * started to fail on a call to `Statics.releaseFence`. It has been reported as + * a bug in the GraalVM repository: https://github.com/oracle/graal/issues/2770 + * + * A proposed workaround for this bug is to substitute the original function + * with a different implementation that does not use the problematic + * MethodHandle. This is implemented in class + * `org.enso.launcher.workarounds.ReplacementStatics` using + * `org.enso.launcher.workarounds.Unsafe` which gives access to + * `sun.misc.Unsafe` which contains a low-level function corresponding to the + * required "release fence". + * + * To allow for that substitution, the launcher code requires annotations from + * the `svm` module and that is why this additional dependency is needed as long + * as that workaround is in-place. The dependency is marked as "provided" + * because it is included within the native-image build. + */ diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index ac1e8a12dc..545509d4b1 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -149,6 +149,12 @@ In order to build and run Enso you will need the following tools: - [Rustup](https://rustup.rs), the rust toolchain management utility. - On MacOS and Linux, the `tar` command is required for running some tests. It should be installed by default on most distributions. +- If you want to be able to build the Launcher Native Image, you will need a + native C compiler for your platform as described in the + [Native Image Prerequisites](https://www.graalvm.org/reference-manual/native-image/#prerequisites). + On Linux that will be `gcc`, on macOS you may need `xcode` and on Windows you + need to configure the Developer Command Prompt for Microsoft Visual C++ for + the x64 architecture. Managing multiple JVM installations can be a pain, so some of the team use [Jenv](http://www.jenv.be/): A useful tool for managing multiple JVMs. @@ -296,7 +302,7 @@ Native Image component is installed in your GraalVM distribution. To install it, run: ```bash -gu install native-image +/bin/gu install native-image ``` Then, you can build the launcher using: diff --git a/docs/distribution/packaging.md b/docs/distribution/packaging.md index c6587eee56..0c43b823e3 100644 --- a/docs/distribution/packaging.md +++ b/docs/distribution/packaging.md @@ -42,7 +42,7 @@ My_Package │ ├── Helper.enso │ └── Util.enso └── visualization (optional) - └── + └── ... ``` ### The `src` Directory diff --git a/docs/distribution/release-policy.md b/docs/distribution/release-policy.md index efad8d2f5b..e6afd3aaff 100644 --- a/docs/distribution/release-policy.md +++ b/docs/distribution/release-policy.md @@ -120,7 +120,7 @@ For example: ```yaml minimum-launcher-version: 0.0.1 -graal-vm-version: 20.1.0 +graal-vm-version: 20.2.0 graal-java-version: 11 jvm-options: - value: "-Dpolyglot.engine.IterativePartialEscape=true" diff --git a/docs/getting-enso.md b/docs/getting-enso.md index c8724e052f..5f6e62be51 100644 --- a/docs/getting-enso.md +++ b/docs/getting-enso.md @@ -31,7 +31,7 @@ GraalVM. You can get the Community Edition pre-built distributions from It is important to run Enso with exactly the version specified here. Given that Graal is still a relatively young project, even the minor version changes introduce breaking API changes. The current version of GraalVM required for Enso -is `20.1.0`, and it must be the Java 11 build. +is `20.2.0`, and it must be the Java 11 build. Before running the Enso packages, make sure that the `JAVA_HOME` environment variable points to the correct home location of the Graal distribution. @@ -64,8 +64,8 @@ This section lists the most common failures and their probable causes. section. It should be similar to: ``` - Running on: OpenJDK 64-Bit Server VM, GraalVM Community, JDK 11.0.7+10-jvmci-20.1-b02 - Linux 4.15.0-106-generic (amd64) + Running on: OpenJDK 64-Bit Server VM, GraalVM Community, JDK 11.0.8+10-jvmci-20.2-b03 + Linux 4.15.0-112-generic (amd64) ``` It could also be caused by not using the launcher scripts and trying to run diff --git a/docs/infrastructure/README.md b/docs/infrastructure/README.md index bf14a0334f..e90013ad03 100644 --- a/docs/infrastructure/README.md +++ b/docs/infrastructure/README.md @@ -17,3 +17,6 @@ up as follows: migration. - [**Native Image:**](native-image.md) Description of the Native Image build used for building the launcher native binary. +- [**Rust:**](rust.md) Description of integration of the Scala project with the + Rust components. +- [**Upgrading GraalVM:**](upgrading-graalvm.md) diff --git a/docs/infrastructure/native-image.md b/docs/infrastructure/native-image.md index cef8d565fc..5d8a45bef2 100644 --- a/docs/infrastructure/native-image.md +++ b/docs/infrastructure/native-image.md @@ -28,7 +28,8 @@ Native Image is used for building the Launcher. ### Native Image Component The Native Image component has to be installed within the used GraalVM -distribution. It can be installed by running `gu install native-image`. +distribution. It can be installed by running +`/bin/gu install native-image`. ### Additional Linux Dependencies @@ -41,13 +42,19 @@ suggested by Graal as an alternative. The sbt task automatically downloads a bundle containing all requirements for a static build with `musl`. It only requires a `tar` command to be available to extract the bundle. -Currently, to enable compiling with `musl`, `-H:UseMuslC=/path/to/musl/bundle` -option is added to the `native-image` command. In the future, the command may -use a different option for enabling `musl` or even enable it by default, so the -Native Image task may need updating with a newer Graal release. More information -may be found in +Currently, to use `musl`, the `--libc=musl` option has to be added to the build +and `gcc-musl` must be available in the system PATH for the native-image. In the +future it is possible that a different option will be used or that the bundle +will not be required anymore if it became prepackaged. This task may thus need +an update when moving to a newer version of Graal. More information may be found +in [the Native Image documentation](https://github.com/oracle/graal/blob/master/substratevm/STATIC-IMAGES.md). +To make the bundle work correctly with GraalVM 20.2, a shell script called +`gcc-musl` which loads the bundle's configuration is created by the task and the +paths starting with `/build/bundle` in `musl-gcc.specs` are replaced with +absolute paths to the bundle location. + ## Static Builds The task is parametrized with `staticOnLinux` parameter which if set to `true`, diff --git a/docs/infrastructure/rust.md b/docs/infrastructure/rust.md index c8ba34f3a6..9d5a44a4ce 100644 --- a/docs/infrastructure/rust.md +++ b/docs/infrastructure/rust.md @@ -2,8 +2,8 @@ layout: developer-doc title: Rust category: infrastructure -tags: [infrastructure, build] -order: 1 +tags: [infrastructure, build, rust] +order: 4 --- # Rust diff --git a/docs/infrastructure/upgrading-graalvm.md b/docs/infrastructure/upgrading-graalvm.md new file mode 100644 index 0000000000..5e5cff250a --- /dev/null +++ b/docs/infrastructure/upgrading-graalvm.md @@ -0,0 +1,27 @@ +--- +layout: developer-doc +title: Upgrading GraalVM +category: infrastructure +tags: [infrastructure, build, graalvm, graal, jvm] +order: 5 +--- + +# Upgrading GraalVM + +After upgrading the project to a newer version of GraalVM, all developers must +take the following actions to be able to continue development after the upgrade: + +1. Download the new JVM version and set it as the default for the project. If + you use IntelliJ, you will also need to update the JVM used for the project + in the project settings. +2. Re-run `sbt bootstrap` to get the updated Truffle JAR (if there are issues + updating, removing `engine/runtime/build-cache` directory may help). +3. Do a full clean (it may not _always_ be required, but not doing it often + leads to problems so it is much safer to do it) by running `enso/clean`. +4. To be able to build or run tests for the `launcher` project, Native Image for + the new GraalVM version has to be installed, as it is not included by + default. This can be done with + `/bin/gu install native-image`. + - If there are problems building the Native Image, removing + `engine/launcher/build-cache` (which contains the downloaded `musl` + package) may help. diff --git a/engine/launcher/src/main/java/org/enso/launcher/workarounds/ReplacementStatics.java b/engine/launcher/src/main/java/org/enso/launcher/workarounds/ReplacementStatics.java new file mode 100644 index 0000000000..52ed4dd851 --- /dev/null +++ b/engine/launcher/src/main/java/org/enso/launcher/workarounds/ReplacementStatics.java @@ -0,0 +1,24 @@ +package org.enso.launcher.workarounds; + +import com.oracle.svm.core.annotate.Substitute; +import com.oracle.svm.core.annotate.TargetClass; + +/** + * Uses Native Image substitution capability to substitute the + * {@link scala.runtime.Statics#releaseFence()} function which causes problems + * when building the Native Image on GraalVM 20.2.0. + */ +@TargetClass(className = "scala.runtime.Statics") +final class ReplacementStatics { + + /** + * Implements a "release fence" without using an unsupported + * {@link java.lang.invoke.MethodHandle} like the original one. + * + * Instead, uses {@link sun.misc.Unsafe#storeFence()} under the hood. + */ + @Substitute + public static void releaseFence() { + Unsafe.unsafeInstance().storeFence(); + } +} diff --git a/engine/launcher/src/main/scala/org/enso/launcher/Launcher.scala b/engine/launcher/src/main/scala/org/enso/launcher/Launcher.scala index 403ccbf499..38e8e39991 100644 --- a/engine/launcher/src/main/scala/org/enso/launcher/Launcher.scala +++ b/engine/launcher/src/main/scala/org/enso/launcher/Launcher.scala @@ -253,8 +253,9 @@ case class Launcher(cliOptions: GlobalCLIOptions) { val versionDescription = VersionDescription.make( "Enso Launcher", - includeRuntimeJVMInfo = false, - additionalParameters = runtimeVersionParameter.toSeq + includeRuntimeJVMInfo = false, + enableNativeImageOSWorkaround = true, + additionalParameters = runtimeVersionParameter.toSeq ) println(versionDescription.asString(useJSON)) diff --git a/engine/launcher/src/main/scala/org/enso/launcher/workarounds/Unsafe.scala b/engine/launcher/src/main/scala/org/enso/launcher/workarounds/Unsafe.scala new file mode 100644 index 0000000000..ee9b638a8c --- /dev/null +++ b/engine/launcher/src/main/scala/org/enso/launcher/workarounds/Unsafe.scala @@ -0,0 +1,22 @@ +package org.enso.launcher.workarounds + +/** + * Gives access to an instance of [[sun.misc.Unsafe]] which contains low-level + * functions that are used to replace the ones that cause problems in the + * Native Image build. + * + * Workaround based on + * https://github.com/plokhotnyuk/jsoniter-scala/blob/master/jsoniter-scala-examples/src/main/scala-2.13/com/github/plokhotnyuk/jsoniter_scala/examples/UnsafeUtils.java + */ +object Unsafe { + + /** + * Instance of the [[sun.misc.Unsafe]] acquired using reflection that allows + * to run the unsafe functions needed by the workaround. + */ + val unsafeInstance: sun.misc.Unsafe = { + val field = classOf[sun.misc.Unsafe].getDeclaredField("theUnsafe") + field.setAccessible(true) + field.get(null).asInstanceOf[sun.misc.Unsafe] + } +} diff --git a/engine/runtime/src/main/java/org/enso/interpreter/node/callable/dispatch/LoopingCallOptimiserNode.java b/engine/runtime/src/main/java/org/enso/interpreter/node/callable/dispatch/LoopingCallOptimiserNode.java index 0caffc09c5..751614a7c3 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/node/callable/dispatch/LoopingCallOptimiserNode.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/node/callable/dispatch/LoopingCallOptimiserNode.java @@ -54,9 +54,7 @@ public abstract class LoopingCallOptimiserNode extends CallOptimiserNode { * @param loopNode a cached instance of the loop node used by this node * @return the result of executing {@code function} using {@code arguments} */ - // TODO[MK]: Remove the guard when https://github.com/oracle/graal/pull/2567 is released. - // Probably 20.2. - @Specialization(guards = "true") + @Specialization public Stateful dispatch( Object function, CallerInfo callerInfo, diff --git a/lib/scala/version-output/src/main/scala/org/enso/version/VersionDescription.scala b/lib/scala/version-output/src/main/scala/org/enso/version/VersionDescription.scala index e28e6e8ab4..22cd1c8417 100644 --- a/lib/scala/version-output/src/main/scala/org/enso/version/VersionDescription.scala +++ b/lib/scala/version-output/src/main/scala/org/enso/version/VersionDescription.scala @@ -39,6 +39,7 @@ object VersionDescription { def make( header: String, includeRuntimeJVMInfo: Boolean, + enableNativeImageOSWorkaround: Boolean = false, additionalParameters: Seq[VersionDescriptionParameter] = Seq.empty ): VersionDescription = { val version = Info.ensoVersion @@ -63,8 +64,13 @@ object VersionDescription { if (includeRuntimeJVMInfo) s"""Running on: $vmName, $vmVendor, JDK $jreVersion | $osName $osVersion ($osArch)""".stripMargin - else - s"Running on: $osName $osVersion ($osArch)" + else if (enableNativeImageOSWorkaround) { + // TODO [RW] Currently the `os.name` property seems to be set to the + // OS the program has been built on, instead of the OS that is + // currently running. A workaround should be implemented in #1100 + // that will use other means to query the OS name and version. + s"Built on: $osName ($osArch)" + } else s"Running on: $osName $osVersion ($osArch)" val dirtyStr = if (Info.isDirty) "*" else "" val parameters = diff --git a/project/NativeImage.scala b/project/NativeImage.scala index 4f92624242..d5287ca97d 100644 --- a/project/NativeImage.scala +++ b/project/NativeImage.scala @@ -51,18 +51,16 @@ object NativeImage { } val debugParameters = - if (includeDebugInfo) "-H:GenerateDebugInfo=1" else "" + if (includeDebugInfo) Seq("-H:GenerateDebugInfo=1") else Seq() - val staticParameters = + val (staticParameters, pathExts) = if (staticOnLinux && isLinux) { // Note [Static Build On Linux] val buildCache = subProjectRoot / "build-cache" val path = ensureMuslIsInstalled(buildCache, log) - s"--static -H:UseMuslC=$path" - } else "" - - val resourcesGlobOpt = "-H:IncludeResources=.*Main.enso$" + (Seq("--static", "--libc=musl"), Seq(path.toString)) + } else (Seq(), Seq()) val configLocation = subProjectRoot / "native-image-config" @@ -70,24 +68,30 @@ object NativeImage { if (configLocation.exists()) { val path = configLocation.toPath.toAbsolutePath log.debug(s"Picking up Native Image configuration from `$path`.") - s"-H:ConfigurationFileDirectories=$path" + Seq(s"-H:ConfigurationFileDirectories=$path") } else { log.debug( "No Native Image configuration found, proceeding without it." ) - "" + Seq() } val cmd = - s"$nativeImagePath $staticParameters $debugParameters " + - s"$resourcesGlobOpt $configs " + - s"--no-fallback --initialize-at-build-time " + - s"${additionalOptions.mkString(" ")} " + - s"-cp $classPath ${(Compile / mainClass).value.get} enso" + Seq(nativeImagePath) ++ + debugParameters ++ staticParameters ++ configs ++ + Seq("--no-fallback", "--initialize-at-build-time", "--no-server") ++ + additionalOptions ++ + Seq("-cp", classPath, (Compile / mainClass).value.get, artifactName) - log.debug(cmd) + val pathParts = pathExts ++ Option(System.getenv("PATH")).toSeq + val newPath = pathParts.mkString(File.pathSeparator) - if (cmd.! != 0) { + log.debug(s"""PATH="$newPath" ${cmd.mkString(" ")}""") + + val process = + Process(cmd, None, "PATH" -> newPath) + + if (process.! != 0) { log.error("Native Image build failed.") throw new RuntimeException("Native Image build failed") } @@ -127,7 +131,16 @@ object NativeImage { streams.value.cacheStoreFactory.make("incremental_native_image") Tracked.diffInputs(store, FileInfo.hash)(filesSet) { sourcesDiff: ChangeReport[File] => - if (sourcesDiff.modified.nonEmpty) + if (System.getenv("LAUNCHER_NATIVE_IMAGE_TEST_SKIP_BUILD") == "true") + Def.task { + streams.value.log.warn( + "LAUNCHER_NATIVE_IMAGE_TEST_SKIP_BUILD set to true, " + + "Native Image will not be rebuilt. Use with caution as using " + + "this override without building the launcher manually may " + + "lead to launcher tests failure." + ) + } + else if (sourcesDiff.modified.nonEmpty) rebuild("Native Image is not up to date") else if (!artifactFile(artifactName).exists()) rebuild("Native Image does not exist") @@ -157,12 +170,16 @@ object NativeImage { * Ensures that the `musl` bundle is installed. * * Checks for existence of its directory and if it does not exist, downloads - * and extracts the bundle. `musl` is needed for static builds on Linux. + * and extracts the bundle. After extracting it does the required + * initialization (renaming paths to be absolute and creating a shell script + * called `musl-gcc`). + * + * `musl` is needed for static builds on Linux. * * @param buildCache build-cache directory for the current project * @param log a logger instance - * @return path to the `musl` bundle that can be passed to the Native Image - * as a parameter + * @return path to the `musl` bundle binary directory which should be added + * to PATH of the launched native-image */ private def ensureMuslIsInstalled( buildCache: File, @@ -170,7 +187,11 @@ object NativeImage { ): Path = { val muslRoot = buildCache / "musl-1.2.0" val bundleLocation = muslRoot / "bundle" - if (!bundleLocation.exists()) { + val binaryLocation = bundleLocation / "bin" + val gccLocation = binaryLocation / "musl-gcc" + def isMuslInstalled = + gccLocation.exists() && gccLocation.isOwnerExecutable + if (!isMuslInstalled) { log.info( "`musl` is required for a static build, but it is not installed for " + "this subproject." @@ -201,6 +222,12 @@ object NativeImage { throw new RuntimeException(s"Cannot extract $bundle.") } + replacePathsInSpecs( + bundleLocation / "lib" / "musl-gcc.specs", + bundleLocation + ) + createGCCWrapper(bundleLocation) + log.info("Installed `musl`.") } catch { case e: Exception => @@ -213,7 +240,39 @@ object NativeImage { } - bundleLocation.toPath.toAbsolutePath + binaryLocation.toPath.toAbsolutePath.normalize + } + + /** + * Replaces paths in `musl-gcc.specs` with absolute paths to the bundle. + * + * The paths in `musl-gcc.specs` start with `/build/bundle` which is not a + * valid path by default. Instead, these prefixes are replaced with an + * absolute path to the bundle. + * + * @param specs reference to `musl-gcc.specs` file + * @param bundleLocation location of the bundle root + */ + private def replacePathsInSpecs(specs: File, bundleLocation: File): Unit = { + val content = IO.read(specs) + val bundlePath = bundleLocation.toPath.toAbsolutePath.normalize.toString + val replaced = content.replace("/build/bundle", bundlePath) + IO.write(specs, replaced) + } + + /** + * Creates a simple shell script called `musl-gcc` which calls the original + * `gcc` and ensures the bundle's configuration (`musl-gcc.specs`) is loaded. + */ + private def createGCCWrapper(bundleLocation: File): Unit = { + val bundlePath = bundleLocation.toPath.toAbsolutePath.normalize.toString + val content = + s"""#!/bin/sh + |exec "$${REALGCC:-gcc}" "$$@" -specs "$bundlePath/lib/musl-gcc.specs" + |""".stripMargin + val wrapper = bundleLocation / "bin" / "musl-gcc" + IO.write(wrapper, content) + wrapper.setExecutable(true) } } @@ -226,10 +285,22 @@ object NativeImage { * automatically downloads a bundle containing all requirements for a static * build with `musl`. * - * Currently, to use `musl`, the `-H:UseMuslC=/path/to/musl/bundle` option has - * to be added to the build. In the future, a `--libc=musl` option may be - * preferred instead, as described at - * https://github.com/oracle/graal/blob/master/substratevm/STATIC-IMAGES.md - * or even `musl` may be included by default. This task may thus need an update - * when moving to a newer version of Graal. + * The `musl` bundle that we use is not guaranteed to be maintained, so if in + * the future a new version of `musl` comes out and we need to upgrade, we may + * need to create our own infrastructure (a repository with CI jobs for creating + * such bundles). It is especially important to note that the libstdc++ that is + * included in this bundle should also be built using `musl` as otherwise linker + * errors may arise. + * + * Currently, to use `musl`, the `--libc=musl` option has to be added to the + * build and `gcc-musl` must be available in the system PATH for the + * native-image. In the future it is possible that a different option will be + * used or that the bundle will not be required anymore if it became + * prepackaged. This task may thus need an update when moving to a newer version + * of Graal. + * + * Currently to make the bundle work correctly with GraalVM 20.2, a shell script + * called `gcc-musl` which loads the bundle's configuration is created by the + * task and the paths starting with `/build/bundle` in `musl-gcc.specs` are + * replaced with absolute paths to the bundle location. */