enso/.github/workflows/scala.yml

name: Engine CI

on:
  push:
    branches: [main, "release/*"]
  pull_request:
    branches: ["*"]

env:
  # Please ensure that this is in sync with graalVersion in build.sbt
  graalVersion: 20.2.0
  # Please ensure that this is in sync with javaVersion in build.sbt
  javaVersion: 11
  # Please ensure that this is in sync with project/build.properties
  sbtVersion: 1.4.1
  # Please ensure that this is in sync with rustVersion in build.sbt
  rustToolchain: nightly-2019-11-04
  excludedPaths: |
    .github/PULL_REQUEST_TEMPLATE.md
    .github/CODEOWNERS
    .github/ISSUE_TEMPLATE/*
    /doc/*
    .gitignore
    .scalafmt.conf
    CODE_OF_CONDUCT.md
    CONTRIBUTING.md
    LICENSE
    README.md    

jobs:
  test_and_publish:
    name: Build and Test
    runs-on: ${{ matrix.os }}
    timeout-minutes: 45
    strategy:
      matrix:
        os: [macOS-latest, ubuntu-latest, windows-latest]
      fail-fast: false
    steps:
      - uses: actions/checkout@v2
      - name: Enable Developer Command Prompt (Windows)
        uses: ilammy/msvc-dev-cmd@v1.4.1
      - name: Setup Go
        uses: actions/setup-go@v2
      - name: Disable TCP/UDP Offloading (macOS)
        if: runner.os == 'macOS'
        shell: bash
        run: |
          sudo sysctl -w net.link.generic.system.hwcksum_tx=0
          sudo sysctl -w net.link.generic.system.hwcksum_rx=0          
      - name: Disable TCP/UDP Offloading (Linux)
        if: runner.os == 'Linux'
        shell: bash
        run: sudo ethtool -K eth0 tx off rx off
      - name: Disable TCP/UDP Offloading (Windows)
        if: runner.os == 'Windows'
        shell: powershell
        run: >
          Disable-NetAdapterChecksumOffload -Name * -TcpIPv4 -UdpIPv4 -TcpIPv6
          -UdpIPv6          
      - name: Install Rust
        uses: actions-rs/toolchain@v1.0.6
        with:
          toolchain: ${{ env.rustToolchain }}
          override: true
      - name: Setup conda
        uses: s-weigand/setup-conda@v1.0.5
        with:
          update-conda: false
          conda-channels: anaconda, conda-forge
      - name: Setup Conda Environment on Windows
        if: runner.os == 'Windows'
        run: |
          conda create --name enso
          conda init powershell          
      - name: Activate Conda Environment on Windows
        if: runner.os == 'Windows'
        run: conda activate enso
      - name: Install FlatBuffers Compiler
        run: conda install --freeze-installed flatbuffers=1.12.0
      - name: Setup GraalVM Environment
        uses: ayltai/setup-graalvm@v1
        with:
          graalvm-version: ${{ env.graalVersion }}
          java-version: ${{ env.javaVersion }}
          native-image: true
      - name: Set Up SBT
        shell: bash
        run: |
          curl --retry 4 --retry-connrefused -fsSL -o sbt.tgz https://github.com/sbt/sbt/releases/download/v${{env.sbtVersion}}/sbt-${{env.sbtVersion}}.tgz
          tar -xzf sbt.tgz
          echo $GITHUB_WORKSPACE/sbt/bin/ >> $GITHUB_PATH          

      # Caches
      - name: Cache SBT
        uses: actions/cache@v2
        with:
          path: |
            ~/.sbt
            ~/.ivy2/cache
            ~/.cache            
          key: ${{ runner.os }}-sbt-${{ hashFiles('**build.sbt') }}
          restore-keys: ${{ runner.os }}-sbt-

      # Compile
      - name: Bootstrap Enso project
        run: |
          sleep 1
          sbt --no-colors bootstrap          
      - name: Build Enso
        run: |
          sleep 1
          sbt --no-colors compile          

      # Tests
      - name: Setup Tests on Windows
        if: runner.os == 'Windows'
        shell: bash
        run: |
          echo "CI_TEST_TIMEFACTOR=2" >> $GITHUB_ENV
          echo "CI_TEST_FLAKY_ENABLE=true" >> $GITHUB_ENV          
      - name: Build Base Java Extensions
        shell: bash
        run: |
          sleep 1
          sbt --no-colors std-bits/package          
      - name: Build the Launcher
        shell: bash
        run: |
          sleep 1
          sbt --no-colors launcher/buildNativeImage
          # Prevents launcher/test from re-building the Native Image during
          # test run. It is built before starting the tests to conserve system
          # memory
          echo "LAUNCHER_NATIVE_IMAGE_TEST_SKIP_BUILD=true" >> $GITHUB_ENV          
      - name: Test Enso
        run: |
          sleep 1
          sbt --no-colors "set Global / parallelExecution := false; test"          
      - name: Check Runtime Benchmark Compilation
        run: |
          sleep 1
          sbt --no-colors runtime/Benchmark/compile          
      - name: Check Language Server Benchmark Compilation
        run: |
          sleep 1
          sbt --no-colors language-server/Benchmark/compile          
      - name: Check Searcher Benchmark Compilation
        run: |
          sleep 1
          sbt --no-colors searcher/Benchmark/compile          

      # Build Distribution
      - name: Build the Runner & Runtime Uberjars
        run: |
          sleep 1
          sbt --no-colors engine-runner/assembly          
      - name: Build the Project Manager Uberjar
        run: |
          sleep 1
          sbt --no-colors project-manager/assembly          
      - name: Build the Manifest
        run: |
          cp distribution/manifest.template.yaml manifest.yaml
          echo "graal-vm-version: $graalVersion" >> manifest.yaml
          echo "graal-java-version: $javaVersion" >> manifest.yaml          
      - name: Build the Parser JS Bundle
        # The builds are run on 3 platforms, but Flatbuffer schemas are platform
        # agnostic, so they just need to be uploaded from one of the runners.
        if: runner.os == 'Linux'
        run: sbt -no-colors syntaxJS/fullOptJS

      # Prepare distributions
      # The version used in filenames is based on the version of the launcher.
      # Currently launcher and engine versions are tied to each other so they
      # can be used interchangeably like this. If in the future the versions
      # become independent, this may require updating to use proper versions
      # for each component.
      - name: Prepare Distribution Version (Unix)
        if: runner.os != 'Windows'
        shell: bash
        run: |
          DIST_VERSION=$(./enso version --json --only-launcher | jq -r '.version')
          echo "DIST_VERSION=$DIST_VERSION" >> $GITHUB_ENV          

      - name: Prepare Distribution Version (Windows)
        if: runner.os == 'Windows'
        shell: bash
        run: |
          DIST_VERSION=$(./enso.exe version --json --only-launcher | jq -r '.version')
          echo "DIST_VERSION=$DIST_VERSION" >> $GITHUB_ENV          

      # Currently the only architecture supported by Github runners is amd64
      - name: Prepare Distribution Environment
        shell: bash
        run: |
          DIST_ARCH=amd64
          DIST_OS=$(echo ${{ runner.os }} | awk '{print tolower($0)}')
          LAUNCHER_DIST_ROOT=enso-launcher-$DIST_VERSION-$DIST_OS-$DIST_ARCH
          LAUNCHER_DIST_DIR=$LAUNCHER_DIST_ROOT/enso
          ENGINE_DIST_ROOT=enso-engine-$DIST_VERSION-$DIST_OS-$DIST_ARCH
          ENGINE_DIST_DIR=$ENGINE_DIST_ROOT/enso-$DIST_VERSION
          echo "LAUNCHER_DIST_DIR=$LAUNCHER_DIST_DIR" >> $GITHUB_ENV
          echo "LAUNCHER_DIST_ROOT=$LAUNCHER_DIST_ROOT" >> $GITHUB_ENV
          echo "ENGINE_DIST_DIR=$ENGINE_DIST_DIR" >> $GITHUB_ENV
          echo "ENGINE_DIST_ROOT=$ENGINE_DIST_ROOT" >> $GITHUB_ENV          

      - name: Prepare Launcher Distribution (Common)
        shell: bash
        run: |
          mkdir -p ${{ env.LAUNCHER_DIST_DIR }}
          mkdir ${{ env.LAUNCHER_DIST_DIR }}/bin
          mkdir ${{ env.LAUNCHER_DIST_DIR }}/dist
          mkdir ${{ env.LAUNCHER_DIST_DIR }}/runtime
          cp distribution/launcher/.enso.portable ${{ env.LAUNCHER_DIST_DIR }}
          cp distribution/launcher/README.md ${{ env.LAUNCHER_DIST_DIR }}
          cp -r distribution/launcher/THIRD-PARTY ${{ env.LAUNCHER_DIST_DIR }}          

      - name: Prepare Launcher Distribution (Unix)
        if: runner.os != 'Windows'
        shell: bash
        run: |
          cp enso ${{ env.LAUNCHER_DIST_DIR }}/bin/          

      - name: Prepare Launcher Distribution (Windows)
        if: runner.os == 'Windows'
        shell: bash
        run: |
          cp enso.exe ${{ env.LAUNCHER_DIST_DIR }}/bin/          

      # The way artifacts are uploaded currently does not preserve the
      # executable bits for Unix. However putting artifacts into a ZIP would
      # create a twice nested ZIP file. For now, users downloading artifacts
      # from the CI builds have to set the bit themselves.
      # So the following two lines have been removed from this step, as they do
      # nothing useful:
      #  chmod +x $ENGINE_DIST_DIR/bin/enso
      #  chmod +x $ENGINE_DIST_DIR/bin/project-manager
      - name: Prepare Engine Distribution
        shell: bash
        run: |
          mkdir -p $ENGINE_DIST_DIR
          mkdir $ENGINE_DIST_DIR/component
          cp runtime.jar $ENGINE_DIST_DIR/component
          mv runner.jar $ENGINE_DIST_DIR/component
          mv project-manager.jar $ENGINE_DIST_DIR/component
          cp -r distribution/std-lib $ENGINE_DIST_DIR/std-lib
          cp -r distribution/bin $ENGINE_DIST_DIR/bin
          cp -r distribution/engine/THIRD-PARTY $ENGINE_DIST_DIR/THIRD-PARTY
          cp manifest.yaml $ENGINE_DIST_DIR          

      # Test Distribution
      - name: Prepare Engine Test Environment
        shell: bash
        run: |
          go get -v github.com/ahmetb/go-httpbin/cmd/httpbin
          $(go env GOPATH)/bin/httpbin -host :8080 &          

      - name: Test Engine Distribution (Unix)
        shell: bash
        if: runner.os != 'Windows'
        run: |
          $ENGINE_DIST_DIR/bin/enso --run test/Test          

      - name: Test Engine Distribution (Windows)
        shell: bash
        if: runner.os == 'Windows'
        run: |
          $ENGINE_DIST_DIR/bin/enso.bat --run test/Test          

      # Publish
      - name: Publish the Engine Distribution Artifact
        uses: actions/upload-artifact@v2
        with:
          name: ${{ env.ENGINE_DIST_ROOT }}
          path: ${{ env.ENGINE_DIST_ROOT }}
      - name: Publish the Launcher Binary
        uses: actions/upload-artifact@v2
        with:
          name: ${{ env.LAUNCHER_DIST_ROOT }}
          path: ${{ env.LAUNCHER_DIST_ROOT }}

      - name: Prepare the FlatBuffers Schemas for Upload
        # The builds are run on 3 platforms, but Flatbuffer schemas are platform
        # agnostic, so they just need to be uploaded from one of the runners.
        if: runner.os == 'Linux'
        run: |
          mkdir fbs-upload
          cp -r "engine/language-server/src/main/schema" fbs-upload/fbs-schema/
          zip -r -m -ll "fbs-upload/fbs-schema.zip" "fbs-upload/fbs-schema/"          
      - name: Publish the FlatBuffers Schemas
        if: runner.os == 'Linux'
        uses: actions/upload-artifact@v2
        with:
          name: Engine Protocol FlatBuffers Schemas
          path: ./fbs-upload/fbs-schema.zip
      - name: Prepare Parser JS Bundle for Upload
        if: runner.os == 'Linux'
        run: |
          mkdir parser-upload
          cp ./target/scala-parser.js parser-upload          
      - name: Publish the Parser JS Bundle
        if: runner.os == 'Linux'
        uses: actions/upload-artifact@v2
        with:
          name: Parser JS Bundle
          path: ./target/scala-parser.js
      - name: Publish the Manifest
        if: runner.os == 'Linux'
        uses: actions/upload-artifact@v2
        with:
          name: manifest
          path: manifest.yaml

      # Publish FlatBuffer Schemas and Parser Bundle to S3
      - name: Prepare AWS Session
        shell: bash
        if: runner.os == 'Linux'
        run: |
          aws configure --profile s3-upload <<-EOF > /dev/null 2>&1
          ${{ secrets.ARTEFACT_S3_ACCESS_KEY_ID }}
          ${{ secrets.ARTEFACT_S3_SECRET_ACCESS_KEY }}
          us-west-2
          text
          EOF          
      - name: Upload Parser JS Bundle to S3
        shell: bash
        if: runner.os == 'Linux'
        run: |
          aws s3 sync ./parser-upload s3://packages-luna/parser-js/nightly/`git rev-parse HEAD` --profile s3-upload --acl public-read --delete          
      - name: Upload FlatBuffers Schemas to S3
        shell: bash
        if: runner.os == 'Linux'
        run: |
          aws s3 sync ./fbs-upload s3://packages-luna/fbs-schema/nightly/`git rev-parse HEAD` --profile s3-upload --acl public-read --delete          
      - name: Teardown AWS Session
        shell: bash
        if: runner.os == 'Linux'
        run: |
          aws configure --profile s3-upload <<-EOF > /dev/null 2>&1
          null
          null
          null
          text
          EOF