From a5a729a7d680cda44905ec6c701b3100e8e1b7f9 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Tue, 12 Mar 2024 20:25:26 +0100 Subject: [PATCH] Add some engine jobs that run with Oracle GraalVM (#9322) Adds `Oracle GraalVM` configuration for some backend jobs. `Oracle GraalVM` jobs run only on Linux so far. The old jobs use `GraalVM CE`. ### Important Notes - The JDK to download and use is deduced from the `JAVA_VENDOR` environment variable. By default, `GraalVM CE` is used. - sbt can be started with both GraalVM CE and Oracle GraalVM without any warnings. - If you try to start sbt with JDK from a different vendor, but with the same Java version, a warning is printed. Current list of jobs in the `Engine CI` workflow (these jobs are visible on this PR, because they are scheduled to run on every PR): - Engine (GraalVM CE) (linux, x86_64) - Engine (GraalVM CE) (macos, x86_64) - Engine (GraalVM CE) (windows, x86_64) - **Engine (Oracle GraalVM) (linux, x86_64)** - Scala Tests (GraalVM CE) (linux, x86_64) - Scala Tests (GraalVM CE) (macos, x86_64) - Scala Tests (GraalVM CE) (windows, x86_64) - **Scala Tests (Oracle GraalVM) (linux, x86_64)** - Standard Library Tests (GraalVM CE) (linux, x86_64) - Standard Library Tests (GraalVM CE) (macos, x86_64) - Standard Library Tests (GraalVM CE) (windows, x86_64) - **Standard Library Tests (Oracle GraalVM) (linux x86_64)** - Verify License Packages (linux, x86_64) Benchmark Engine workflow (not visible on this PR, cannot schedule manually yet): - Benchmark Engine (GraalVM CE) - **Benchmark Engine (Oracle GraalVM)** Benchmark Standard Libraries workflow (not visible on this PR, cannot schedule manually yet): - Benchmark Standard Libraries (GraalVM CE) - **Benchmark Standard Libraries (Oracle GraalVM)** --- .github/workflows/engine-benchmark.yml | 57 ++++- .github/workflows/scala-new.yml | 244 +++++++++++++++++++-- .github/workflows/std-libs-benchmark.yml | 57 ++++- CHANGELOG.md | 2 + Cargo.lock | 1 + build/build/Cargo.toml | 1 + build/build/src/ci_gen.rs | 53 ++++- build/build/src/ci_gen/job.rs | 83 ++++++- build/build/src/ci_gen/step.rs | 9 +- build/build/src/engine.rs | 4 + build/build/src/engine/env.rs | 4 + build/ci_utils/src/cache/goodie/graalvm.rs | 185 ++++++++++++++-- project/GraalVM.scala | 9 +- 13 files changed, 629 insertions(+), 80 deletions(-) diff --git a/.github/workflows/engine-benchmark.yml b/.github/workflows/engine-benchmark.yml index 81547e39e3..01f675ef4e 100644 --- a/.github/workflows/engine-benchmark.yml +++ b/.github/workflows/engine-benchmark.yml @@ -13,8 +13,8 @@ on: type: boolean default: false jobs: - benchmark-engine: - name: Benchmark Engine + benchmark-engine-graal-vm-ce: + name: Benchmark Engine (GraalVM CE) runs-on: - benchmark steps: @@ -61,6 +61,59 @@ jobs: run: ./run git-clean env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + env: + GRAAL_EDITION: GraalVM CE + timeout-minutes: 240 + benchmark-engine-oracle-graal-vm: + name: Benchmark Engine (Oracle GraalVM) + runs-on: + - benchmark + steps: + - if: startsWith(runner.name, 'GitHub Actions') || startsWith(runner.name, 'Hosted Agent') + name: Setup conda (GH runners only) + uses: s-weigand/setup-conda@v1.2.1 + with: + update-conda: false + conda-channels: anaconda, conda-forge + - if: startsWith(runner.name, 'GitHub Actions') || startsWith(runner.name, 'Hosted Agent') + name: Installing wasm-pack + uses: jetli/wasm-pack-action@v0.4.0 + with: + version: v0.10.2 + - name: Expose Artifact API and context information. + uses: actions/github-script@v7 + with: + script: "\n core.exportVariable(\"ACTIONS_RUNTIME_TOKEN\", process.env[\"ACTIONS_RUNTIME_TOKEN\"])\n core.exportVariable(\"ACTIONS_RUNTIME_URL\", process.env[\"ACTIONS_RUNTIME_URL\"])\n core.exportVariable(\"GITHUB_RETENTION_DAYS\", process.env[\"GITHUB_RETENTION_DAYS\"])\n console.log(context)\n " + - name: Checking out the repository + uses: actions/checkout@v4 + with: + clean: false + submodules: recursive + - name: Build Script Setup + run: ./run --help + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - if: (always()) + name: Clean before + run: ./run git-clean + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - run: ./run backend benchmark runtime + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - if: failure() && runner.os == 'Windows' + name: List files if failed (Windows) + run: Get-ChildItem -Force -Recurse + - if: failure() && runner.os != 'Windows' + name: List files if failed (non-Windows) + run: ls -lAR + - if: (always()) + name: Clean after + run: ./run git-clean + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + env: + GRAAL_EDITION: Oracle GraalVM timeout-minutes: 240 env: ENSO_BUILD_MINIMAL_RUN: ${{ true == inputs.just-check }} diff --git a/.github/workflows/scala-new.yml b/.github/workflows/scala-new.yml index ce3cc69591..31727df866 100644 --- a/.github/workflows/scala-new.yml +++ b/.github/workflows/scala-new.yml @@ -27,8 +27,8 @@ jobs: access_token: ${{ github.token }} permissions: actions: write - enso-build-ci-gen-job-ci-check-backend-linux-x86_64: - name: Engine (linux, x86_64) + enso-build-ci-gen-job-ci-check-backend-graal-vm-ce-linux-x86_64: + name: Engine (GraalVM CE) (linux, x86_64) runs-on: - self-hosted - Linux @@ -76,8 +76,10 @@ jobs: run: ./run git-clean env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - enso-build-ci-gen-job-ci-check-backend-macos-x86_64: - name: Engine (macos, x86_64) + env: + GRAAL_EDITION: GraalVM CE + enso-build-ci-gen-job-ci-check-backend-graal-vm-ce-macos-x86_64: + name: Engine (GraalVM CE) (macos, x86_64) runs-on: - macos-latest steps: @@ -124,8 +126,10 @@ jobs: run: ./run git-clean env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - enso-build-ci-gen-job-ci-check-backend-windows-x86_64: - name: Engine (windows, x86_64) + env: + GRAAL_EDITION: GraalVM CE + enso-build-ci-gen-job-ci-check-backend-graal-vm-ce-windows-x86_64: + name: Engine (GraalVM CE) (windows, x86_64) runs-on: - self-hosted - Windows @@ -173,8 +177,61 @@ jobs: run: ./run git-clean env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - enso-build-ci-gen-job-scala-tests-linux-x86_64: - name: Scala Tests (linux, x86_64) + env: + GRAAL_EDITION: GraalVM CE + enso-build-ci-gen-job-ci-check-backend-oracle-graal-vm-linux-x86_64: + name: Engine (Oracle GraalVM) (linux, x86_64) + runs-on: + - self-hosted + - Linux + steps: + - if: startsWith(runner.name, 'GitHub Actions') || startsWith(runner.name, 'Hosted Agent') + name: Setup conda (GH runners only) + uses: s-weigand/setup-conda@v1.2.1 + with: + update-conda: false + conda-channels: anaconda, conda-forge + - if: startsWith(runner.name, 'GitHub Actions') || startsWith(runner.name, 'Hosted Agent') + name: Installing wasm-pack + uses: jetli/wasm-pack-action@v0.4.0 + with: + version: v0.10.2 + - name: Expose Artifact API and context information. + uses: actions/github-script@v7 + with: + script: "\n core.exportVariable(\"ACTIONS_RUNTIME_TOKEN\", process.env[\"ACTIONS_RUNTIME_TOKEN\"])\n core.exportVariable(\"ACTIONS_RUNTIME_URL\", process.env[\"ACTIONS_RUNTIME_URL\"])\n core.exportVariable(\"GITHUB_RETENTION_DAYS\", process.env[\"GITHUB_RETENTION_DAYS\"])\n console.log(context)\n " + - name: Checking out the repository + uses: actions/checkout@v4 + with: + clean: false + submodules: recursive + - name: Build Script Setup + run: ./run --help + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - if: "(contains(github.event.pull_request.labels.*.name, 'CI: Clean build required') || inputs.clean_build_required)" + name: Clean before + run: ./run git-clean + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - run: ./run backend ci-check + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - if: failure() && runner.os == 'Windows' + name: List files if failed (Windows) + run: Get-ChildItem -Force -Recurse + - if: failure() && runner.os != 'Windows' + name: List files if failed (non-Windows) + run: ls -lAR + - if: "(always()) && (contains(github.event.pull_request.labels.*.name, 'CI: Clean build required') || inputs.clean_build_required)" + name: Clean after + run: ./run git-clean + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + env: + GRAAL_EDITION: Oracle GraalVM + enso-build-ci-gen-job-scala-tests-graal-vm-ce-linux-x86_64: + name: Scala Tests (GraalVM CE) (linux, x86_64) runs-on: - self-hosted - Linux @@ -216,7 +273,7 @@ jobs: uses: dorny/test-reporter@v1 with: max-annotations: 50 - name: Engine Tests Report (linux, x86_64) + name: Engine Tests Report (GraalVM CE, linux, x86_64) path: ${{ env.ENSO_TEST_JUNIT_DIR }}/*.xml path-replace-backslashes: true reporter: java-junit @@ -231,10 +288,12 @@ jobs: run: ./run git-clean env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + env: + GRAAL_EDITION: GraalVM CE permissions: checks: write - enso-build-ci-gen-job-scala-tests-macos-x86_64: - name: Scala Tests (macos, x86_64) + enso-build-ci-gen-job-scala-tests-graal-vm-ce-macos-x86_64: + name: Scala Tests (GraalVM CE) (macos, x86_64) runs-on: - macos-latest steps: @@ -275,7 +334,7 @@ jobs: uses: dorny/test-reporter@v1 with: max-annotations: 50 - name: Engine Tests Report (macos, x86_64) + name: Engine Tests Report (GraalVM CE, macos, x86_64) path: ${{ env.ENSO_TEST_JUNIT_DIR }}/*.xml path-replace-backslashes: true reporter: java-junit @@ -290,10 +349,12 @@ jobs: run: ./run git-clean env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + env: + GRAAL_EDITION: GraalVM CE permissions: checks: write - enso-build-ci-gen-job-scala-tests-windows-x86_64: - name: Scala Tests (windows, x86_64) + enso-build-ci-gen-job-scala-tests-graal-vm-ce-windows-x86_64: + name: Scala Tests (GraalVM CE) (windows, x86_64) runs-on: - self-hosted - Windows @@ -335,7 +396,7 @@ jobs: uses: dorny/test-reporter@v1 with: max-annotations: 50 - name: Engine Tests Report (windows, x86_64) + name: Engine Tests Report (GraalVM CE, windows, x86_64) path: ${{ env.ENSO_TEST_JUNIT_DIR }}/*.xml path-replace-backslashes: true reporter: java-junit @@ -350,10 +411,74 @@ jobs: run: ./run git-clean env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + env: + GRAAL_EDITION: GraalVM CE permissions: checks: write - enso-build-ci-gen-job-standard-library-tests-linux-x86_64: - name: Standard Library Tests (linux, x86_64) + enso-build-ci-gen-job-scala-tests-oracle-graal-vm-linux-x86_64: + name: Scala Tests (Oracle GraalVM) (linux, x86_64) + runs-on: + - self-hosted + - Linux + steps: + - if: startsWith(runner.name, 'GitHub Actions') || startsWith(runner.name, 'Hosted Agent') + name: Setup conda (GH runners only) + uses: s-weigand/setup-conda@v1.2.1 + with: + update-conda: false + conda-channels: anaconda, conda-forge + - if: startsWith(runner.name, 'GitHub Actions') || startsWith(runner.name, 'Hosted Agent') + name: Installing wasm-pack + uses: jetli/wasm-pack-action@v0.4.0 + with: + version: v0.10.2 + - name: Expose Artifact API and context information. + uses: actions/github-script@v7 + with: + script: "\n core.exportVariable(\"ACTIONS_RUNTIME_TOKEN\", process.env[\"ACTIONS_RUNTIME_TOKEN\"])\n core.exportVariable(\"ACTIONS_RUNTIME_URL\", process.env[\"ACTIONS_RUNTIME_URL\"])\n core.exportVariable(\"GITHUB_RETENTION_DAYS\", process.env[\"GITHUB_RETENTION_DAYS\"])\n console.log(context)\n " + - name: Checking out the repository + uses: actions/checkout@v4 + with: + clean: false + submodules: recursive + - name: Build Script Setup + run: ./run --help + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - if: "(contains(github.event.pull_request.labels.*.name, 'CI: Clean build required') || inputs.clean_build_required)" + name: Clean before + run: ./run git-clean + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - run: ./run backend test scala + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - if: success() || failure() + name: Engine Test Reporter + uses: dorny/test-reporter@v1 + with: + max-annotations: 50 + name: Engine Tests Report (Oracle GraalVM, linux, x86_64) + path: ${{ env.ENSO_TEST_JUNIT_DIR }}/*.xml + path-replace-backslashes: true + reporter: java-junit + - if: failure() && runner.os == 'Windows' + name: List files if failed (Windows) + run: Get-ChildItem -Force -Recurse + - if: failure() && runner.os != 'Windows' + name: List files if failed (non-Windows) + run: ls -lAR + - if: "(always()) && (contains(github.event.pull_request.labels.*.name, 'CI: Clean build required') || inputs.clean_build_required)" + name: Clean after + run: ./run git-clean + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + env: + GRAAL_EDITION: Oracle GraalVM + permissions: + checks: write + enso-build-ci-gen-job-standard-library-tests-graal-vm-ce-linux-x86_64: + name: Standard Library Tests (GraalVM CE) (linux, x86_64) runs-on: - self-hosted - Linux @@ -398,7 +523,7 @@ jobs: uses: dorny/test-reporter@v1 with: max-annotations: 50 - name: Standard Library Tests Report (linux, x86_64) + name: Standard Library Tests Report (GraalVM CE, linux, x86_64) path: ${{ env.ENSO_TEST_JUNIT_DIR }}/*/*.xml path-replace-backslashes: true reporter: java-junit @@ -413,10 +538,12 @@ jobs: run: ./run git-clean env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + env: + GRAAL_EDITION: GraalVM CE permissions: checks: write - enso-build-ci-gen-job-standard-library-tests-macos-x86_64: - name: Standard Library Tests (macos, x86_64) + enso-build-ci-gen-job-standard-library-tests-graal-vm-ce-macos-x86_64: + name: Standard Library Tests (GraalVM CE) (macos, x86_64) runs-on: - macos-latest steps: @@ -460,7 +587,7 @@ jobs: uses: dorny/test-reporter@v1 with: max-annotations: 50 - name: Standard Library Tests Report (macos, x86_64) + name: Standard Library Tests Report (GraalVM CE, macos, x86_64) path: ${{ env.ENSO_TEST_JUNIT_DIR }}/*/*.xml path-replace-backslashes: true reporter: java-junit @@ -475,10 +602,12 @@ jobs: run: ./run git-clean env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + env: + GRAAL_EDITION: GraalVM CE permissions: checks: write - enso-build-ci-gen-job-standard-library-tests-windows-x86_64: - name: Standard Library Tests (windows, x86_64) + enso-build-ci-gen-job-standard-library-tests-graal-vm-ce-windows-x86_64: + name: Standard Library Tests (GraalVM CE) (windows, x86_64) runs-on: - self-hosted - Windows @@ -523,7 +652,7 @@ jobs: uses: dorny/test-reporter@v1 with: max-annotations: 50 - name: Standard Library Tests Report (windows, x86_64) + name: Standard Library Tests Report (GraalVM CE, windows, x86_64) path: ${{ env.ENSO_TEST_JUNIT_DIR }}/*/*.xml path-replace-backslashes: true reporter: java-junit @@ -538,6 +667,73 @@ jobs: run: ./run git-clean env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + env: + GRAAL_EDITION: GraalVM CE + permissions: + checks: write + enso-build-ci-gen-job-standard-library-tests-oracle-graal-vm-linux-x86_64: + name: Standard Library Tests (Oracle GraalVM) (linux, x86_64) + runs-on: + - self-hosted + - Linux + steps: + - if: startsWith(runner.name, 'GitHub Actions') || startsWith(runner.name, 'Hosted Agent') + name: Setup conda (GH runners only) + uses: s-weigand/setup-conda@v1.2.1 + with: + update-conda: false + conda-channels: anaconda, conda-forge + - if: startsWith(runner.name, 'GitHub Actions') || startsWith(runner.name, 'Hosted Agent') + name: Installing wasm-pack + uses: jetli/wasm-pack-action@v0.4.0 + with: + version: v0.10.2 + - name: Expose Artifact API and context information. + uses: actions/github-script@v7 + with: + script: "\n core.exportVariable(\"ACTIONS_RUNTIME_TOKEN\", process.env[\"ACTIONS_RUNTIME_TOKEN\"])\n core.exportVariable(\"ACTIONS_RUNTIME_URL\", process.env[\"ACTIONS_RUNTIME_URL\"])\n core.exportVariable(\"GITHUB_RETENTION_DAYS\", process.env[\"GITHUB_RETENTION_DAYS\"])\n console.log(context)\n " + - name: Checking out the repository + uses: actions/checkout@v4 + with: + clean: false + submodules: recursive + - name: Build Script Setup + run: ./run --help + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - if: "(contains(github.event.pull_request.labels.*.name, 'CI: Clean build required') || inputs.clean_build_required)" + name: Clean before + run: ./run git-clean + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - run: ./run backend test standard-library + env: + AWS_ACCESS_KEY_ID: ${{ secrets.ENSO_LIB_S3_AWS_ACCESS_KEY_ID }} + AWS_REGION: ${{ secrets.ENSO_LIB_S3_AWS_REGION }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.ENSO_LIB_S3_AWS_SECRET_ACCESS_KEY }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - if: success() || failure() + name: Standard Library Test Reporter + uses: dorny/test-reporter@v1 + with: + max-annotations: 50 + name: Standard Library Tests Report (Oracle GraalVM, linux, x86_64) + path: ${{ env.ENSO_TEST_JUNIT_DIR }}/*/*.xml + path-replace-backslashes: true + reporter: java-junit + - if: failure() && runner.os == 'Windows' + name: List files if failed (Windows) + run: Get-ChildItem -Force -Recurse + - if: failure() && runner.os != 'Windows' + name: List files if failed (non-Windows) + run: ls -lAR + - if: "(always()) && (contains(github.event.pull_request.labels.*.name, 'CI: Clean build required') || inputs.clean_build_required)" + name: Clean after + run: ./run git-clean + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + env: + GRAAL_EDITION: Oracle GraalVM permissions: checks: write enso-build-ci-gen-job-verify-license-packages-linux-x86_64: diff --git a/.github/workflows/std-libs-benchmark.yml b/.github/workflows/std-libs-benchmark.yml index ca34a14db1..75ff746a7c 100644 --- a/.github/workflows/std-libs-benchmark.yml +++ b/.github/workflows/std-libs-benchmark.yml @@ -13,8 +13,8 @@ on: type: boolean default: false jobs: - benchmark-standard-libraries: - name: Benchmark Standard Libraries + benchmark-standard-libraries-graal-vm-ce: + name: Benchmark Standard Libraries (GraalVM CE) runs-on: - benchmark steps: @@ -61,6 +61,59 @@ jobs: run: ./run git-clean env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + env: + GRAAL_EDITION: GraalVM CE + timeout-minutes: 240 + benchmark-standard-libraries-oracle-graal-vm: + name: Benchmark Standard Libraries (Oracle GraalVM) + runs-on: + - benchmark + steps: + - if: startsWith(runner.name, 'GitHub Actions') || startsWith(runner.name, 'Hosted Agent') + name: Setup conda (GH runners only) + uses: s-weigand/setup-conda@v1.2.1 + with: + update-conda: false + conda-channels: anaconda, conda-forge + - if: startsWith(runner.name, 'GitHub Actions') || startsWith(runner.name, 'Hosted Agent') + name: Installing wasm-pack + uses: jetli/wasm-pack-action@v0.4.0 + with: + version: v0.10.2 + - name: Expose Artifact API and context information. + uses: actions/github-script@v7 + with: + script: "\n core.exportVariable(\"ACTIONS_RUNTIME_TOKEN\", process.env[\"ACTIONS_RUNTIME_TOKEN\"])\n core.exportVariable(\"ACTIONS_RUNTIME_URL\", process.env[\"ACTIONS_RUNTIME_URL\"])\n core.exportVariable(\"GITHUB_RETENTION_DAYS\", process.env[\"GITHUB_RETENTION_DAYS\"])\n console.log(context)\n " + - name: Checking out the repository + uses: actions/checkout@v4 + with: + clean: false + submodules: recursive + - name: Build Script Setup + run: ./run --help + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - if: (always()) + name: Clean before + run: ./run git-clean + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - run: ./run backend benchmark enso-jmh + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - if: failure() && runner.os == 'Windows' + name: List files if failed (Windows) + run: Get-ChildItem -Force -Recurse + - if: failure() && runner.os != 'Windows' + name: List files if failed (non-Windows) + run: ls -lAR + - if: (always()) + name: Clean after + run: ./run git-clean + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + env: + GRAAL_EDITION: Oracle GraalVM timeout-minutes: 240 env: ENSO_BUILD_MINIMAL_RUN: ${{ true == inputs.just-check }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 350c4bf307..b9ceed47b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1073,6 +1073,7 @@ - [Execute and debug individual Enso files in VSCode extension][8923] - [Check type of `self` when calling a method using the static syntax][8867] - [Autoscoped constructors][9190] +- [Allow Oracle GraalVM JDK][9322] [3227]: https://github.com/enso-org/enso/pull/3227 [3248]: https://github.com/enso-org/enso/pull/3248 @@ -1232,6 +1233,7 @@ [8923]: https://github.com/enso-org/enso/pull/8923 [8867]: https://github.com/enso-org/enso/pull/8867 [9190]: https://github.com/enso-org/enso/pull/9190 +[9322]: https://github.com/enso-org/enso/pull/9322 # Enso 2.0.0-alpha.18 (2021-10-12) diff --git a/Cargo.lock b/Cargo.lock index 86b0e7fe3c..4c865a9178 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1204,6 +1204,7 @@ dependencies = [ "futures-util", "glob", "handlebars", + "heck", "ide-ci", "mime", "new_mime_guess", diff --git a/build/build/Cargo.toml b/build/build/Cargo.toml index 4a55958c5c..be6599f884 100644 --- a/build/build/Cargo.toml +++ b/build/build/Cargo.toml @@ -21,6 +21,7 @@ futures = { workspace = true } futures-util = "0.3.17" glob = "0.3.0" handlebars = "4.3.5" +heck = "0.4.0" enso-build-base = { path = "../base" } enso-enso-font = { path = "../../lib/rust/enso-font" } enso-font = { path = "../../lib/rust/font" } diff --git a/build/build/src/ci_gen.rs b/build/build/src/ci_gen.rs index 143542a07a..f13fb3730f 100644 --- a/build/build/src/ci_gen.rs +++ b/build/build/src/ci_gen.rs @@ -3,6 +3,7 @@ use crate::prelude::*; use crate::ci_gen::job::plain_job; use crate::ci_gen::job::with_packaging_steps; use crate::ci_gen::job::RunsOn; +use crate::engine::env; use crate::version::promote::Designation; use crate::version::ENSO_EDITION; use crate::version::ENSO_RELEASE_MODE; @@ -37,6 +38,7 @@ use ide_ci::actions::workflow::definition::WorkflowDispatch; use ide_ci::actions::workflow::definition::WorkflowDispatchInput; use ide_ci::actions::workflow::definition::WorkflowDispatchInputType; use ide_ci::actions::workflow::definition::WorkflowToWrite; +use ide_ci::cache::goodie::graalvm; use strum::IntoEnumIterator; @@ -609,22 +611,34 @@ pub fn backend() -> Result { workflow.add(PRIMARY_TARGET, job::CancelWorkflow); workflow.add(PRIMARY_TARGET, job::VerifyLicensePackages); for target in CHECKED_TARGETS { - workflow.add(target, job::CiCheckBackend); - workflow.add(target, job::ScalaTests); - workflow.add(target, job::StandardLibraryTests); + workflow.add(target, job::CiCheckBackend { graal_edition: graalvm::Edition::Community }); + workflow.add(target, job::ScalaTests { graal_edition: graalvm::Edition::Community }); + workflow + .add(target, job::StandardLibraryTests { graal_edition: graalvm::Edition::Community }); } + // Oracle GraalVM jobs run only on Linux + workflow + .add(PRIMARY_TARGET, job::CiCheckBackend { graal_edition: graalvm::Edition::Enterprise }); + workflow.add(PRIMARY_TARGET, job::ScalaTests { graal_edition: graalvm::Edition::Enterprise }); + workflow.add(PRIMARY_TARGET, job::StandardLibraryTests { + graal_edition: graalvm::Edition::Enterprise, + }); Ok(workflow) } pub fn engine_benchmark() -> Result { - benchmark("Benchmark Engine", "backend benchmark runtime", Some(4 * 60)) + benchmark_workflow("Benchmark Engine", "backend benchmark runtime", Some(4 * 60)) } pub fn std_libs_benchmark() -> Result { - benchmark("Benchmark Standard Libraries", "backend benchmark enso-jmh", Some(4 * 60)) + benchmark_workflow("Benchmark Standard Libraries", "backend benchmark enso-jmh", Some(4 * 60)) } -fn benchmark(name: &str, command_line: &str, timeout_minutes: Option) -> Result { +fn benchmark_workflow( + name: &str, + command_line: &str, + timeout_minutes: Option, +) -> Result { let just_check_input_name = "just-check"; let just_check_input = WorkflowDispatchInput { r#type: WorkflowDispatchInputType::Boolean { default: Some(false) }, @@ -645,14 +659,31 @@ fn benchmark(name: &str, command_line: &str, timeout_minutes: Option) -> Re wrap_expression(format!("true == inputs.{just_check_input_name}")), ); - let mut benchmark_job = RunStepsBuilder::new(command_line) - .cleaning(CleaningCondition::Always) - .build_job(name, BenchmarkRunner); - benchmark_job.timeout_minutes = timeout_minutes; - workflow.add_job(benchmark_job); + for graal_edition in [graalvm::Edition::Community, graalvm::Edition::Enterprise] { + let job_name = format!("{name} ({graal_edition})"); + let job = benchmark_job(&job_name, command_line, timeout_minutes, graal_edition); + workflow.add_job(job); + } Ok(workflow) } +fn benchmark_job( + job_name: &str, + command_line: &str, + timeout_minutes: Option, + graal_edition: graalvm::Edition, +) -> Job { + let mut job = RunStepsBuilder::new(command_line) + .cleaning(CleaningCondition::Always) + .build_job(job_name, BenchmarkRunner); + job.timeout_minutes = timeout_minutes; + match graal_edition { + graalvm::Edition::Community => job.env(env::GRAAL_EDITION, graalvm::Edition::Community), + graalvm::Edition::Enterprise => job.env(env::GRAAL_EDITION, graalvm::Edition::Enterprise), + } + job +} + /// Generate workflows for the CI. pub fn generate( diff --git a/build/build/src/ci_gen/job.rs b/build/build/src/ci_gen/job.rs index 046b421934..5b6d013ecf 100644 --- a/build/build/src/ci_gen/job.rs +++ b/build/build/src/ci_gen/job.rs @@ -7,7 +7,9 @@ use crate::ci_gen::step; use crate::ci_gen::RunStepsBuilder; use crate::ci_gen::RunnerType; use crate::ci_gen::RELEASE_CLEANING_POLICY; +use crate::engine::env; +use heck::ToKebabCase; use ide_ci::actions::workflow::definition::cancel_workflow_action; use ide_ci::actions::workflow::definition::Access; use ide_ci::actions::workflow::definition::Job; @@ -17,6 +19,7 @@ use ide_ci::actions::workflow::definition::RunnerLabel; use ide_ci::actions::workflow::definition::Step; use ide_ci::actions::workflow::definition::Strategy; use ide_ci::actions::workflow::definition::Target; +use ide_ci::cache::goodie::graalvm; @@ -172,21 +175,46 @@ impl JobArchetype for VerifyLicensePackages { } #[derive(Clone, Copy, Debug)] -pub struct ScalaTests; +pub struct ScalaTests { + pub graal_edition: graalvm::Edition, +} + + impl JobArchetype for ScalaTests { fn job(&self, target: Target) -> Job { - RunStepsBuilder::new("backend test scala") - .customize(move |step| vec![step, step::engine_test_reporter(target)]) - .build_job("Scala Tests", target) - .with_permission(Permission::Checks, Access::Write) + let graal_edition = self.graal_edition; + let job_name = format!("Scala Tests ({graal_edition})"); + let mut job = RunStepsBuilder::new("backend test scala") + .customize(move |step| vec![step, step::engine_test_reporter(target, graal_edition)]) + .build_job(job_name, target) + .with_permission(Permission::Checks, Access::Write); + match graal_edition { + graalvm::Edition::Community => job.env(env::GRAAL_EDITION, graalvm::Edition::Community), + graalvm::Edition::Enterprise => + job.env(env::GRAAL_EDITION, graalvm::Edition::Enterprise), + } + job + } + + fn key(&self, (os, arch): Target) -> String { + format!( + "{}-{}-{os}-{arch}", + self.id_key_base(), + self.graal_edition.to_string().to_kebab_case() + ) } } #[derive(Clone, Copy, Debug)] -pub struct StandardLibraryTests; +pub struct StandardLibraryTests { + pub graal_edition: graalvm::Edition, +} + impl JobArchetype for StandardLibraryTests { fn job(&self, target: Target) -> Job { - RunStepsBuilder::new("backend test standard-library") + let graal_edition = self.graal_edition; + let job_name = format!("Standard Library Tests ({graal_edition})"); + let mut job = RunStepsBuilder::new("backend test standard-library") .customize(move |step| { let main_step = step .with_secret_exposed_as( @@ -201,10 +229,24 @@ impl JobArchetype for StandardLibraryTests { secret::ENSO_LIB_S3_AWS_SECRET_ACCESS_KEY, crate::aws::env::AWS_SECRET_ACCESS_KEY, ); - vec![main_step, step::stdlib_test_reporter(target)] + vec![main_step, step::stdlib_test_reporter(target, graal_edition)] }) - .build_job("Standard Library Tests", target) - .with_permission(Permission::Checks, Access::Write) + .build_job(job_name, target) + .with_permission(Permission::Checks, Access::Write); + match graal_edition { + graalvm::Edition::Community => job.env(env::GRAAL_EDITION, graalvm::Edition::Community), + graalvm::Edition::Enterprise => + job.env(env::GRAAL_EDITION, graalvm::Edition::Enterprise), + } + job + } + + fn key(&self, (os, arch): Target) -> String { + format!( + "{}-{}-{os}-{arch}", + self.id_key_base(), + self.graal_edition.to_string().to_kebab_case() + ) } } @@ -418,10 +460,27 @@ impl JobArchetype for PackageNewIde { } #[derive(Clone, Copy, Debug)] -pub struct CiCheckBackend; +pub struct CiCheckBackend { + pub graal_edition: graalvm::Edition, +} impl JobArchetype for CiCheckBackend { fn job(&self, target: Target) -> Job { - RunStepsBuilder::new("backend ci-check").build_job("Engine", target) + let job_name = format!("Engine ({})", self.graal_edition); + let mut job = RunStepsBuilder::new("backend ci-check").build_job(job_name, target); + match self.graal_edition { + graalvm::Edition::Community => job.env(env::GRAAL_EDITION, graalvm::Edition::Community), + graalvm::Edition::Enterprise => + job.env(env::GRAAL_EDITION, graalvm::Edition::Enterprise), + } + job + } + + fn key(&self, (os, arch): Target) -> String { + format!( + "{}-{}-{os}-{arch}", + self.id_key_base(), + self.graal_edition.to_string().to_kebab_case() + ) } } diff --git a/build/build/src/ci_gen/step.rs b/build/build/src/ci_gen/step.rs index 978e8b46b7..c81aa60d36 100644 --- a/build/build/src/ci_gen/step.rs +++ b/build/build/src/ci_gen/step.rs @@ -5,6 +5,7 @@ use crate::paths; use ide_ci::actions::workflow::definition::env_expression; use ide_ci::actions::workflow::definition::Step; use ide_ci::actions::workflow::definition::Target; +use ide_ci::cache::goodie::graalvm; @@ -26,16 +27,16 @@ pub fn test_reporter( .with_custom_argument("name", report_name) } -pub fn stdlib_test_reporter((os, arch): Target) -> Step { +pub fn stdlib_test_reporter((os, arch): Target, graal_edition: graalvm::Edition) -> Step { let step_name = "Standard Library Test Reporter"; - let report_name = format!("Standard Library Tests Report ({os}, {arch})"); + let report_name = format!("Standard Library Tests Report ({graal_edition}, {os}, {arch})"); let path = format!("{}/*/*.xml", env_expression(&paths::ENSO_TEST_JUNIT_DIR)); test_reporter(step_name, report_name, path) } -pub fn engine_test_reporter((os, arch): Target) -> Step { +pub fn engine_test_reporter((os, arch): Target, graal_edition: graalvm::Edition) -> Step { let step_name = "Engine Test Reporter"; - let report_name = format!("Engine Tests Report ({os}, {arch})"); + let report_name = format!("Engine Tests Report ({graal_edition}, {os}, {arch})"); let path = format!("{}/*.xml", env_expression(&paths::ENSO_TEST_JUNIT_DIR)); test_reporter(step_name, report_name, path) } diff --git a/build/build/src/engine.rs b/build/build/src/engine.rs index 7467abd866..ad72cf3338 100644 --- a/build/build/src/engine.rs +++ b/build/build/src/engine.rs @@ -11,6 +11,7 @@ use crate::paths::generated; use artifact::IsArtifact; use bundle::IsBundle; +use ide_ci::cache::goodie::graalvm::Edition; use ide_ci::future::AsyncPolicy; use ide_ci::github::Repo; use package::IsPackage; @@ -310,9 +311,12 @@ pub async fn deduce_graal( build_sbt: &generated::RepoRootBuildSbt, ) -> Result { let build_sbt_content = ide_ci::fs::tokio::read_to_string(build_sbt).await?; + let graal_edition = env::GRAAL_EDITION.get().map_or(Edition::default(), |e| e); + Ok(ide_ci::cache::goodie::graalvm::GraalVM { client, graal_version: get_graal_version(&build_sbt_content)?, + edition: graal_edition, os: TARGET_OS, arch: TARGET_ARCH, }) diff --git a/build/build/src/engine/env.rs b/build/build/src/engine/env.rs index 0163a88468..0f42b2916e 100644 --- a/build/build/src/engine/env.rs +++ b/build/build/src/engine/env.rs @@ -2,6 +2,7 @@ //use crate::prelude::*; +use ide_ci::cache::goodie::graalvm; use ide_ci::define_env_var; @@ -12,4 +13,7 @@ define_env_var! { /// Whether flaku tests should be run. CI_TEST_FLAKY_ENABLE, bool; + + /// GraalVM edition. Either Community or Enterprise. + GRAAL_EDITION, graalvm::Edition; } diff --git a/build/ci_utils/src/cache/goodie/graalvm.rs b/build/ci_utils/src/cache/goodie/graalvm.rs index ae82cd6aba..d04e0dc7cf 100644 --- a/build/ci_utils/src/cache/goodie/graalvm.rs +++ b/build/ci_utils/src/cache/goodie/graalvm.rs @@ -22,24 +22,80 @@ crate::define_env_var! { GRAALVM_HOME, PathBuf; } -pub fn graal_version_from_version_string(version_string: &str) -> Result { - let line = version_string.lines().find(|line| line.contains("GraalVM CE")).context( - "There is a Java environment available but it is not recognizable as GraalVM one.", - )?; - Version::find_in_text(line) +const CE_JAVA_VENDOR: &str = "GraalVM CE"; +const EE_JAVA_VENDOR: &str = "Oracle GraalVM"; + +#[derive(Copy, Clone, Debug, PartialEq, Default)] +pub enum Edition { + /// GraalVM CE (Community Edition). + #[default] + Community, + /// Oracle GraalVM (Formerly GraalVM EE, Enterprise Edition) + Enterprise, } -pub async fn find_graal_version() -> Result { +impl std::str::FromStr for Edition { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + match s { + CE_JAVA_VENDOR => Ok(Edition::Community), + EE_JAVA_VENDOR => Ok(Edition::Enterprise), + _ => bail!("Unknown GraalVM edition: {}", s), + } + } +} + +impl From for String { + fn from(edition: Edition) -> String { + match edition { + Edition::Community => CE_JAVA_VENDOR.to_string(), + Edition::Enterprise => EE_JAVA_VENDOR.to_string(), + } + } +} + +impl Display for Edition { + fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { + match *self { + Edition::Community => write!(f, "{CE_JAVA_VENDOR}"), + Edition::Enterprise => write!(f, "{EE_JAVA_VENDOR}"), + } + } +} + + +pub fn graal_version_from_version_string(version_string: &str) -> Result<(Version, Edition)> { + let line = version_string + .lines() + .find(|line| line.contains(CE_JAVA_VENDOR) || line.contains(EE_JAVA_VENDOR)) + .context( + "There is a Java environment available but it is not recognizable as GraalVM one.", + )?; + let edition = if line.contains(CE_JAVA_VENDOR) { + Edition::Community + } else if line.contains(EE_JAVA_VENDOR) { + Edition::Enterprise + } else { + bail!("Unknown GraalVM edition") + }; + let version = Version::find_in_text(line); + version.map(|version| (version, edition)).context("Failed to find GraalVM version.") +} + +pub async fn find_graal_version() -> Result<(Version, Edition)> { let text = Java.version_string().await?; graal_version_from_version_string(&text) } + /// Description necessary to download and install GraalVM. #[derive(Clone, Debug)] pub struct GraalVM { /// Used to query GitHub about releases. pub client: Octocrab, pub graal_version: Version, + pub edition: Edition, pub os: OS, pub arch: Arch, } @@ -52,9 +108,11 @@ impl Goodie for GraalVM { fn is_active(&self) -> BoxFuture<'static, Result> { let expected_graal_version = self.graal_version.clone(); + let expected_graal_edition = self.edition; async move { - let found_version = find_graal_version().await?; + let (found_version, found_edition) = find_graal_version().await?; ensure!(found_version == expected_graal_version, "GraalVM version mismatch. Expected {expected_graal_version}, found {found_version}."); + ensure!(found_edition == expected_graal_edition, "GraalVM edition mismatch. Expected {expected_graal_edition}, found {found_edition}."); Result::Ok(true) } .boxed() @@ -90,32 +148,50 @@ impl Goodie for GraalVM { impl GraalVM { pub fn url(&self) -> BoxFuture<'static, Result> { - let platform_string = self.platform_string(); - let graal_version_tag = self.graal_version.to_string_core(); - let client = self.client.clone(); - async move { - let repo = CE_BUILDS_REPOSITORY.handle(&client); - let release = repo.find_release_by_text(&graal_version_tag).await?; - crate::github::find_asset_url_by_text(&release, &platform_string).cloned() + match self.edition { + Edition::Community => { + let platform_string = self.platform_string(); + let graal_version_tag = self.graal_version.to_string_core(); + let client = self.client.clone(); + async move { + let repo = CE_BUILDS_REPOSITORY.handle(&client); + let release = repo.find_release_by_text(&graal_version_tag).await?; + crate::github::find_asset_url_by_text(&release, &platform_string).cloned() + } + .boxed() + } + Edition::Enterprise => { + let graal_version_tag = self.graal_version.to_string_core(); + let url = format!( + "https://download.oracle.com/graalvm/{}/archive/graalvm-jdk-{}_{}_bin.tar.gz", + self.graal_version.major, + graal_version_tag, + self.os_arch_string(), + ); + Box::pin(ready(Url::parse(&url).context("Failed to parse URL."))) + } } - .boxed() } - pub fn platform_string(&self) -> String { - let Self { graal_version: _graal_version, arch, os, client: _client } = &self; - let os_name = match *os { + fn os_arch_string(&self) -> String { + let os_name = match self.os { OS::Linux => "linux", OS::Windows => "windows", OS::MacOS => "macos", other_os => unimplemented!("System `{}` is not supported!", other_os), }; - let arch_name = match *arch { + let arch_name = match self.arch { Arch::X86_64 => "x64", Arch::AArch64 => "aarch64", other_arch => unimplemented!("Architecture `{}` is not supported!", other_arch), }; - let java_version = format!("jdk-{}", _graal_version.to_string_core()); - format!("{PACKAGE_PREFIX_URL}-{java_version}_{os_name}-{arch_name}") + format!("{os_name}-{arch_name}") + } + + pub fn platform_string(&self) -> String { + let os_arch = self.os_arch_string(); + let java_version = format!("jdk-{}", self.graal_version.to_string_core()); + format!("{PACKAGE_PREFIX_URL}-{java_version}_{os_arch}") } } @@ -147,7 +223,8 @@ mod tests { OpenJDK Runtime Environment GraalVM CE 17.0.7+7.1 (build 17.0.7+7-jvmci-23.0-b12) OpenJDK 64-Bit Server VM GraalVM CE 17.0.7+7.1 (build 17.0.7+7-jvmci-23.0-b12, mixed mode, sharing)"#; - let found_graal = graal_version_from_version_string(version_string).unwrap(); + let (found_graal_version, found_graal_edition) = + graal_version_from_version_string(version_string).unwrap(); let expected_graal_version = Version { major: 17, minor: 0, @@ -155,12 +232,35 @@ OpenJDK 64-Bit Server VM GraalVM CE 17.0.7+7.1 (build 17.0.7+7-jvmci-23.0-b12, m pre: Prerelease::EMPTY, build: BuildMetadata::new("7.1").unwrap(), }; - assert_eq!(found_graal, expected_graal_version); + assert_eq!(found_graal_version, expected_graal_version); + assert_eq!(found_graal_edition, Edition::Community); let found_java = Java.parse_version(version_string).unwrap(); assert_eq!(found_java, Version::new(17, 0, 7)); } + #[test] + fn enterprise_version_recognize() { + let version_string = r#"java version "21.0.2" 2024-01-16 LTS +Java(TM) SE Runtime Environment Oracle GraalVM 21.0.2+13.1 (build 21.0.2+13-LTS-jvmci-23.1-b30) +Java HotSpot(TM) 64-Bit Server VM Oracle GraalVM 21.0.2+13.1 (build 21.0.2+13-LTS-jvmci-23.1-b30, mixed mode, sharing)"#; + + let (found_graal_version, found_graal_edition) = + graal_version_from_version_string(version_string).unwrap(); + let expected_graal_version = Version { + major: 21, + minor: 0, + patch: 2, + pre: Prerelease::EMPTY, + build: BuildMetadata::new("13.1").unwrap(), + }; + assert_eq!(found_graal_version, expected_graal_version); + assert_eq!(found_graal_edition, Edition::Enterprise); + + let found_java = Java.parse_version(version_string).unwrap(); + assert_eq!(found_java, Version::new(21, 0, 2)); + } + #[test] fn recognize_oneline_version() { let version_line = @@ -175,4 +275,43 @@ OpenJDK 64-Bit Server VM GraalVM CE 17.0.7+7.1 (build 17.0.7+7-jvmci-23.0-b12, m }; assert_eq!(graal_version, expected_graal_version); } + + #[tokio::test] + async fn correct_url_for_enterprise_edition_21() { + let graalvm = GraalVM { + client: Octocrab::default(), + graal_version: Version::new(21, 0, 2), + edition: Edition::Enterprise, + os: OS::Linux, + arch: Arch::X86_64, + }; + let url = graalvm.url().await.unwrap(); + assert_eq!(url.to_string(), "https://download.oracle.com/graalvm/21/archive/graalvm-jdk-21.0.2_linux-x64_bin.tar.gz"); + } + + #[tokio::test] + async fn correct_url_for_enterprise_edition_17() { + let graalvm = GraalVM { + client: Octocrab::default(), + graal_version: Version::new(17, 0, 7), + edition: Edition::Enterprise, + os: OS::Linux, + arch: Arch::X86_64, + }; + let url = graalvm.url().await.unwrap(); + assert_eq!(url.to_string(), "https://download.oracle.com/graalvm/17/archive/graalvm-jdk-17.0.7_linux-x64_bin.tar.gz"); + } + + #[tokio::test] + async fn correct_url_for_community_edition() { + let graalvm = GraalVM { + client: Octocrab::default(), + graal_version: Version::new(21, 0, 2), + edition: Edition::Community, + os: OS::Linux, + arch: Arch::X86_64, + }; + let url = graalvm.url().await.unwrap(); + assert_eq!(url.to_string(), "https://github.com/graalvm/graalvm-ce-builds/releases/download/jdk-21.0.2/graalvm-community-jdk-21.0.2_linux-x64_bin.tar.gz"); + } } diff --git a/project/GraalVM.scala b/project/GraalVM.scala index 2aab193505..5c9ff49d16 100644 --- a/project/GraalVM.scala +++ b/project/GraalVM.scala @@ -101,6 +101,11 @@ object GraalVM { val langsPkgs = jsPkgs ++ pythonPkgs ++ espressoPkgs + private val allowedJavaVendors = Seq( + "GraalVM Community", + "Oracle Corporation" + ) + /** Augments a state transition to do GraalVM version check. * * @param graalVersion the GraalVM version that should be used for @@ -125,10 +130,10 @@ object GraalVM { throw new IllegalStateException("GraalVM version check failed") } val javaVendor = System.getProperty("java.vendor") - if (javaVendor != "GraalVM Community") { + if (!allowedJavaVendors.contains(javaVendor)) { log.warn( s"Running on non-GraalVM JVM (The actual java.vendor is $javaVendor). " + - s"Expected GraalVM Community java.vendor." + s"Expected Java vendors: ${allowedJavaVendors.mkString(", ")}." ) }