Self-hosted Engine CI for Windows and Linux (#3182)

.github/workflows/gui.yml

@@ -108,15 +108,6 @@ jobs:
           node-version: 14.15.0
       - name: Install TypeScript
         run: npm install -g ts-node@10.1.0
-      - name: Install Rust
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: nightly-2021-11-29
-          override: true
-      - name: Install Clippy
-        run: rustup component add clippy
-      - name: Install Clippy
-        run: rustup component add rustfmt
       - name: Lint Rust sources
         run: node ./run lint --skip-version-validation
   test:
@@ -138,11 +129,6 @@ jobs:
           node-version: 14.15.0
       - name: Install TypeScript
         run: npm install -g ts-node@10.1.0
-      - name: Install Rust
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: nightly-2021-11-29
-          override: true
       - name: Run tests (no WASM)
         run: node ./run test --no-wasm --skip-version-validation
   wasm-test:
@@ -164,11 +150,6 @@ jobs:
           node-version: 14.15.0
       - name: Install TypeScript
         run: npm install -g ts-node@10.1.0
-      - name: Install Rust
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: nightly-2021-11-29
-          override: true
       - name: Install wasm-pack (macOS)
         env:
           WASMPACKURL: https://github.com/rustwasm/wasm-pack/releases/download/v0.9.1
@@ -223,11 +204,6 @@ jobs:
           node-version: 14.15.0
       - name: Install TypeScript
         run: npm install -g ts-node@10.1.0
-      - name: Install Rust
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: nightly-2021-11-29
-          override: true
       - name: Install wasm-pack (macOS)
         env:
           WASMPACKURL: https://github.com/rustwasm/wasm-pack/releases/download/v0.9.1
@@ -301,11 +277,6 @@ jobs:
           node-version: 14.15.0
       - name: Install TypeScript
         run: npm install -g ts-node@10.1.0
-      - name: Install Rust
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: nightly-2021-11-29
-          override: true
       - name: Install wasm-pack (macOS)
         env:
           WASMPACKURL: https://github.com/rustwasm/wasm-pack/releases/download/v0.9.1

.github/workflows/scala-new.yml

@@ -0,0 +1,66 @@
+name: Engine CI V2
+
+on:
+  push:
+    branches: [develop, "release/*"]
+  pull_request:
+    branches: ["*"]
+  workflow_dispatch:
+
+jobs:
+  test_and_publish:
+    name: Engine
+    runs-on:
+      - ${{ matrix.os }}
+      - self-hosted
+      - engine
+    timeout-minutes: 120
+    strategy:
+      matrix:
+        os: [Windows, Linux]
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v2
+      - run: cargo run --release --bin build-enso -- ${{ github.workspace }}
+        working-directory: C:\ci
+        if: runner.os == 'Windows'
+      - run: cargo run --release --bin build-enso -- ${{ github.workspace }}
+        working-directory: /ci
+        if: runner.os != 'Windows'
+
+      #####################
+      # PUBLISH ARTIFACTS #
+      #####################
+      - name: Publish the Engine Distribution Artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{ env.ENGINE_DIST_NAME }}
+          path: ${{ env.ENGINE_DIST_ROOT }}/${{ env.ENGINE_DIST_NAME }}.zip
+      - name: Publish the Launcher
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{ env.LAUNCHER_DIST_NAME }}
+          path: ${{ env.LAUNCHER_DIST_ROOT }}
+      - name: Publish the Project Manager
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{ env.PROJECTMANAGER_DIST_NAME }}
+          path: ${{ env.PROJECTMANAGER_DIST_ROOT }}
+      - name: Publish the FlatBuffers Schemas
+        if: runner.os == 'Linux'
+        uses: actions/upload-artifact@v2
+        with:
+          name: Engine Protocol FlatBuffers Schemas
+          path: ${{ env.TARGET_DIR }}/fbs-upload/fbs-schema.zip
+      - name: Publish the Parser JS Bundle
+        if: runner.os == 'Linux'
+        uses: actions/upload-artifact@v2
+        with:
+          name: Parser JS Bundle
+          path: ${{ env.TARGET_DIR }}/scala-parser.js
+      - name: Publish the Manifest
+        if: runner.os == 'Linux'
+        uses: actions/upload-artifact@v2
+        with:
+          name: manifest
+          path: ${{ env.ENGINE_DIST_DIR }}/manifest.yaml

app/gui/docs/CONTRIBUTING.md

@@ -56,11 +56,12 @@ setup:
 
   This project uses several features available only in the nightly Rust
   toolchain. Please use the [the Rust toolchain installer](https://rustup.rs) to
-  install it:
+  manage Rust toolchains. It will automatically download the toolchain needed to
+  build the project.
+
+  In addition, some custom CLI programs need to be installed manually:
 
   ```bash
-  rustup toolchain install nightly-2021-05-12     # Install the nightly channel.
-  rustup component add clippy                     # Install the linter.
   rustup toolchain install stable                 # Stable toolchain required for the following tools.
   cargo +stable install wasm-pack --version 0.9.1 # Install the wasm-pack toolkit.
   cargo +stable install cargo-watch               # To enable ./run watch utility

build.sbt

@@ -16,7 +16,6 @@ import java.io.File
 // ============================================================================
 
 val scalacVersion  = "2.13.6"
-val rustVersion    = "1.59.0-nightly"
 val graalVersion   = "21.1.0"
 val javaVersion    = "11"
 val ensoVersion    = "0.2.32-SNAPSHOT"  // Note [Engine And Launcher Version]
@@ -1054,14 +1053,12 @@ lazy val `language-server` = (project in file("engine/language-server"))
 lazy val ast = (project in file("lib/scala/ast"))
   .settings(
     version := ensoVersion,
-    Cargo.rustVersion := rustVersion,
     Compile / sourceGenerators += GenerateAST.task
   )
 
 lazy val parser = (project in file("lib/scala/parser"))
   .settings(
     fork := true,
-    Cargo.rustVersion := rustVersion,
     Compile / compile / compileInputs := (Compile / compile / compileInputs)
       .dependsOn(Cargo("build --project parser"))
       .value,
@@ -1335,7 +1332,7 @@ lazy val launcher = project
   .settings(
     (Test / test) := (Test / test)
       .dependsOn(buildNativeImage)
-      .dependsOn(LauncherShimsForTest.prepare(rustcVersion = rustVersion))
+      .dependsOn(LauncherShimsForTest.prepare())
       .value,
     Test / parallelExecution := false
   )

build/workflow.js

@@ -12,7 +12,6 @@ const yaml = require('js-yaml')
 // =================
 
 const NODE_VERSION = '14.15.0'
-const RUST_VERSION = read_rust_toolchain_version()
 const WASM_PACK_VERSION = '0.9.1'
 const FLAG_NO_CHANGELOG_NEEDED = '[ci no changelog needed]'
 const FLAG_FORCE_CI_BUILD = '[ci build]'
@@ -29,13 +28,6 @@ function cached_linux_runner(cache_label) {
     return ['Linux', cache_label]
 }
 
-function read_rust_toolchain_version() {
-    return fss
-        .readFileSync(paths.root + '/rust-toolchain')
-        .toString()
-        .trim()
-}
-
 function job(runners, name, steps, cfg) {
     if (!cfg) {
         cfg = {}
@@ -101,15 +93,6 @@ dumpGitHubContext = {
 // === Dependencies ===
 // ====================
 
-let installRust = {
-    name: 'Install Rust',
-    uses: 'actions-rs/toolchain@v1',
-    with: {
-        toolchain: RUST_VERSION,
-        override: true,
-    },
-}
-
 let installNode = {
     name: 'Install Node',
     uses: 'actions/setup-node@v1',
@@ -118,16 +101,6 @@ let installNode = {
     },
 }
 
-let installClippy = {
-    name: 'Install Clippy',
-    run: 'rustup component add clippy',
-}
-
-let installFmt = {
-    name: 'Install Clippy',
-    run: 'rustup component add rustfmt',
-}
-
 // Install fixed version to avoid upgrading to a breaking version.
 // Should be removed once this has a better solution as described here:
 // https://github.com/enso-org/ide/issues/1772
@@ -435,31 +408,21 @@ let workflow = {
             getCurrentReleaseChangelogInfo,
             assertions,
         ]),
-        lint: job_on_linux_cached('linter', 'Linter', [
+        lint: job_on_linux_cached('linter', 'Linter', [installNode, installTypeScript, lintRust]),
-            installNode,
-            installTypeScript,
-            installRust,
-            installClippy,
-            installFmt,
-            lintRust,
-        ]),
         test: job_on_linux_cached('test_native', 'Native Tests', [
             installNode,
             installTypeScript,
-            installRust,
             testNoWASM,
         ]),
         'wasm-test': job_on_linux_cached('test_wasm', 'WASM Tests', [
             installNode,
             installTypeScript,
-            installRust,
             installWasmPack,
             testWASM,
         ]),
         build_wasm: job_on_linux_cached('build_wasm', 'Build WASM', [
             installNode,
             installTypeScript,
-            installRust,
             installWasmPack,
             installJava,
             buildWASM,
@@ -476,7 +439,6 @@ let workflow = {
                 getCurrentReleaseChangelogInfo,
                 installNode,
                 installTypeScript,
-                installRust,
                 installWasmPack,
                 installJava,
 

docs/CONTRIBUTING.md

@@ -192,15 +192,10 @@ git clone git@github.com:enso-org/enso.git
 
 ### Getting Set Up (Rust)
 
-The SBT project requires a specific nightly rust toolchain. To get it set up,
+The Rust code in this repository requires a specific nightly rust toolchain, as
-you will need to install [rustup](https://rustup.rs/) and then run the following
+defined by [rust-toolchain](../rust-toolchain.toml) override file. The `rustup`
-commands:
+will automatically download the appropriate compiler version along with the
-
+necessary components.
-```bash
-rustup toolchain install nightly-2021-05-12
-rustup override set nightly-2021-05-12
-rustup component add clippy
-```
 
 You will also need `node` in order to run the `wasm` tests. We only support the
 latest LTS version of [NodeJS](https://nodejs.org/en/download) and NPM. We

project/Cargo.scala

@@ -6,29 +6,23 @@ import scala.sys.process._
 
 /** A wrapper for executing the command `cargo`. */
 object Cargo {
-
-  /** The version of rust that needs to be installed. */
-  val rustVersion = settingKey[String]("rustc version used in the project")
-
   private val cargoCmd = "cargo"
 
-  /** Checks rust version and executes the command `cargo $args`. */
+  /** Executes the command `cargo $args`. */
   def apply(args: String): Def.Initialize[Task[Unit]] =
     Def.task {
-      run(args, rustVersion.value, state.value.log)
+      run(args, state.value.log)
     }
 
-  /** Checks rust version and executes the command `cargo $args`.
+  /** Executes the command `cargo $args`.
     *
     * @param args arguments to pass to cargo
-    * @param rustVersion Rust version that should be used
     * @param log a logger instance for diagnostics
     * @param extraEnv additional environment variables that should be set for
     *                 the cargo process
     */
   def run(
     args: String,
-    rustVersion: String,
     log: ManagedLogger,
     extraEnv: Seq[(String, String)] = Seq()
   ): Unit = {
@@ -37,9 +31,6 @@ object Cargo {
     if (!cargoOk(log))
       throw new RuntimeException("Cargo isn't installed!")
 
-    if (!EnvironmentCheck.rustVersionOk(rustVersion, log))
-      throw new RuntimeException("Rust version mismatch!")
-
     log.info(cmd)
 
     val exitCode =

project/EnvironmentCheck.scala

@@ -64,36 +64,6 @@ object EnvironmentCheck {
     graalOk && javaOk
   }
 
-  /** Runs `rustc --version` to ensure that it is properly installed and
-    * checks if the reported version is consistent with expectations.
-    *
-    * @param expectedVersion rust version that is expected to be installed,
-    *                        should be based on project settings
-    * @return either an error message explaining what is wrong with the rust
-    *         version or Unit meaning it is correct
-    */
-  def rustVersionOk(expectedVersion: String, log: ManagedLogger): Boolean = {
-    val cmd = "rustc --version"
-
-    try {
-      val versionStr = cmd.!!.trim
-
-      val contained = versionStr.contains(expectedVersion)
-
-      if (!contained) {
-        log.error(
-          s"Rust version mismatch. $expectedVersion is expected, " +
-          s"but it seems $versionStr is installed."
-        )
-      }
-      contained
-    } catch {
-      case _ @(_: RuntimeException | _: IOException) =>
-        log.error("Rust version check failed. Make sure rustc is in your PATH.")
-        false
-    }
-  }
-
   /** Augments a state transition to do a Rust and GraalVM version check.
     *
     * @param graalVersion the GraalVM version that should be used for

project/GenerateAST.scala

@@ -13,10 +13,9 @@ object GenerateAST {
 
     Tracked.diffInputs(cache, FileInfo.lastModified)(Set(source)) {
       source: ChangeReport[File] =>
-        val rustVersion = Cargo.rustVersion.value
         if (source.modified.nonEmpty) {
           output.getParentFile.mkdirs
-          generateAST(rustVersion, output, log)
+          generateAST(output, log)
         }
     }
 
@@ -28,12 +27,12 @@ object GenerateAST {
     *
     * @param out the file where the generated AST is going to be placed
     */
-  def generateAST(rustVersion: String, out: File, log: ManagedLogger): Unit = {
+  def generateAST(out: File, log: ManagedLogger): Unit = {
     val args = s"run -p ast -- --generate-scala-ast $out"
 
     log.info(s"Generating Scala AST from Rust definitions.")
 
-    try Cargo.run(args, rustVersion, log)
+    try Cargo.run(args, log)
     catch {
       case ex: RuntimeException =>
         log.error(s"Generation of the Scala AST failed.")

project/LauncherShimsForTest.scala

@@ -5,15 +5,12 @@ object LauncherShimsForTest {
 
   /** Creates a task that compiles the launcher shims which are used for some of
     * the launcher tests.
-    *
-    * @param rustcVersion Rust version that should be used
     */
-  def prepare(rustcVersion: String): Def.Initialize[Task[Unit]] =
+  def prepare(): Def.Initialize[Task[Unit]] =
     Def.task {
       val log = state.value.log
       Cargo.run(
         "build -p launcher-shims",
-        rustVersion = rustcVersion,
         log         = log
       )
     }

rust-toolchain

@@ -1 +0,0 @@
-nightly-2021-11-29

rust-toolchain.toml

@@ -0,0 +1,5 @@
+[toolchain]
+channel = "nightly-2021-11-29"
+components = ["clippy"]
+profile = "default"
+targets = [ "wasm32-unknown-unknown" ]