GitHub actions (#895)

* Add Github action for benchmarks

* Change action name to benchmark

* Fix - remove empty env section

* Rename step

* Add steps to print and upload results

* Shrink the matrix of versions for benchmarking

* Enable benchmarks

* rename job

* Fix fetch

* bump actions/setup-haskell

* disable windows - bench script requires Cairo

* Delete Azure bench script

* add comment on git fetch call

* clean up cache key

* Update archive step

.azure/linux-bench.yml

@@ -1,51 +0,0 @@
-jobs:
-- job: ghcide_bench_linux
-  timeoutInMinutes: 120
-  pool:
-    vmImage: 'ubuntu-latest'
-  strategy:
-    matrix:
-      stack:
-        STACK_YAML: "stack.yaml"
-  variables:
-      STACK_ROOT: $(Pipeline.Workspace)/.stack
-  steps:
-  - checkout: self
-  - task: Cache@2
-    inputs:
-      key: stack-root-cache | $(Agent.OS) | $(Build.SourcesDirectory)/$(STACK_YAML) | $(Build.SourcesDirectory)/ghcide.cabal
-      path: $(STACK_ROOT)
-      cacheHitVar: STACK_ROOT_CACHE_RESTORED
-    displayName: "Cache stack root"
-  - task: Cache@2
-    inputs:
-      key: stack-work-cache | $(Agent.OS) | $(Build.SourcesDirectory)/$(STACK_YAML) | $(Build.SourcesDirectory)/ghcide.cabal
-      path: .stack-work
-      cacheHitVar: STACK_WORK_CACHE_RESTORED
-    displayName: "Cache stack work"
-  - bash: |
-      sudo add-apt-repository ppa:hvr/ghc
-      sudo apt-get update
-      sudo apt-get install -y g++ gcc libc6-dev libffi-dev libgmp-dev zlib1g-dev
-      if ! which stack >/dev/null 2>&1; then
-         curl -sSL https://get.haskellstack.org/ | sh
-      fi
-      mkdir -p $STACK_ROOT
-    displayName: 'Install Stack'
-  - bash: stack --version
-    displayName: 'stack version'
-  - bash: stack setup --stack-yaml=$STACK_YAML
-    displayName: 'stack setup'
-  - bash: stack build --bench --only-dependencies --stack-yaml=$STACK_YAML
-    displayName: 'stack build --bench --only-dependencies'
-  - bash: |
-      export PATH=/opt/cabal/bin:$PATH
-      # Retry to avoid fpcomplete servers timeouts
-      stack bench --ghc-options=-Werror --stack-yaml=$STACK_YAML || stack bench --ghc-options=-Werror --stack-yaml=$STACK_YAML  || stack bench --ghc-options=-Werror --stack-yaml=$STACK_YAML
-    displayName: 'stack bench --ghc-options=-Werror'
-  - bash: |
-      column -s, -t < bench-results/results.csv
-    displayName: "cat results"
-  - publish: bench-results
-    artifact: benchmarks
-    displayName: "publish"

.github/workflows/bench.yml

@@ -0,0 +1,62 @@
+name: Benchmark
+
+on: [push, pull_request]
+jobs:
+  bench:
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        ghc: ['8.10.2', '8.8.4', '8.6.5']
+        os: [ubuntu-latest, macOS-latest]
+
+    steps:
+    - uses: actions/checkout@v2
+    - run: git fetch origin master # check the master branch for benchmarking
+    - uses: actions/setup-haskell@v1.1.3
+      with:
+        ghc-version: ${{ matrix.ghc }}
+        cabal-version: '3.2'
+        enable-stack: false
+
+    - name: Cache Cabal
+      uses: actions/cache@v2
+      env:
+        cache-name: cache-cabal
+      with:
+        path: ~/.cabal/
+        key: ${{ runner.os }}-${{ matrix.ghc }}-build-${{ env.cache-name }}-${{ hashFiles('ghcide.cabal', 'cabal.project') }}
+        restore-keys: |
+          ${{ runner.os }}-${{ matrix.ghc }}-build-${{ env.cache-name }}-
+          ${{ runner.os }}-${{ matrix.ghc }}-build-
+          ${{ runner.os }}-${{ matrix.ghc }}
+
+    - run: cabal update
+
+    - run: cabal configure --enable-benchmarks
+
+    - name: Build
+      shell: bash
+      # Retry it three times to workaround compiler segfaults in windows
+      run: cabal build || cabal build || cabal build
+
+    - name: Bench
+      shell: bash
+      # run the tests without parallelism, otherwise tasty will attempt to run
+      # all test cases simultaneously which causes way too many hls
+      # instances to be spun up for the poor github actions runner to handle
+      run: cabal bench
+
+    - name: Display results
+      shell: bash
+      run: |
+        column -s, -t < bench-results/results.csv | tee bench-results/results.txt
+
+    - name: Archive benchmarking artifacts
+      uses: actions/upload-artifact@v2
+      with:
+        name: bench-results-${{ runner.os }}-${{ matrix.ghc }}
+        path: |
+           bench-results/results.*
+           bench-results/*.svg

azure-pipelines.yml

@@ -16,6 +16,3 @@ pr:
 jobs:
   - template: ./.azure/linux-stack.yml
   - template: ./.azure/windows-stack.yml
-
-# Disable benchmarks until we can figure out why they get stuck
-  # - template: ./.azure/linux-bench.yml

bench/config.yaml

@@ -2,7 +2,7 @@
 # At least 100 is recommended in order to observe space leaks
 samples: 100
 
-buildTool: stack
+buildTool: cabal
 
 # Path to the ghcide-bench binary to use for experiments
 ghcideBench: ghcide-bench