Enable all working test suites and add linux-cabal job in Azure (#1571)

* Enabled test-suites:
  * Linux, with stack or cabal, all test suites are enabled
  * Macos:
      * ghc-8.6.*: all suites enabled
      * ghc-8.4.*: func-test and dispatcher-test disabled
  * Windows
      * unit-test enabled
             * Except CabalHelper ones for ghc-8.6.4
      * func-test enabled with almost all tests
* cabal job:
   * Aadded one for linux

Make progress towards #1401
Closes #1431

.azure/linux-cabal.bashrc

@@ -0,0 +1 @@
+export PATH=$HOME/.cabal/bin:/opt/cabal/$CABAL_VERSION/bin:/opt/ghc/$GHC_VERSION/bin:$HOME/.local/bin:$PATH

.azure/linux-cabal.yml

@@ -0,0 +1,76 @@
+jobs:
+- job: Linux_Cabal
+  timeoutInMinutes: 0
+  pool:
+    vmImage: ubuntu-16.04
+  strategy:
+    matrix:
+      ghc-8.6.5:
+        GHC_VERSION: "8.6.5"
+      ghc-8.4.4:
+        GHC_VERSION: "8.4.4"
+  variables:
+    CABAL_VERSION: "3.0"
+  steps:
+  - task: Cache@2
+    inputs:
+      key: '"cabal-store" | "$(Agent.OS)" | "$(CABAL_VERSION)" | "$(GHC_VERSION)" | $(Build.SourcesDirectory)/cabal.project | $(Build.SourcesDirectory)/haskell-ide-engine.cabal | $(Build.SourcesDirectory)/hie-plugin-api/hie-plugin-api.cabal'
+      path: .azure-cache
+      cacheHitVar: CACHE_RESTORED
+    displayName: "Download cache"
+  - bash: |
+      mkdir -p $HOME/.cabal
+      tar -vxzf .azure-cache/cabal-root.tar.gz -C /
+      mkdir -p $HOME/.ghc
+      tar -vxzf .azure-cache/ghc-root.tar.gz -C /
+      mkdir -p dist-newtyle
+      tar -vxzf .azure-cache/cabal-dist.tar.gz
+    displayName: "Unpack cache"
+    condition: eq(variables.CACHE_RESTORED, 'true')
+  - bash: |
+      git submodule sync
+      git submodule update --init
+    displayName: Sync submodules
+  - bash: |
+      source .azure/linux-cabal.bashrc
+      cabal v2-update
+      cabal v2-build --only-dependencies
+    displayName: Build dependencies
+  - bash: |
+      source .azure/linux-cabal.bashrc
+      cabal v2-build
+    displayName: Build `hie`
+  - bash: |
+      source .azure/linux-cabal.bashrc
+      cabal v2-install --overwrite-policy=always # `hie` binary required locally for tests
+    displayName: Install `hie`
+  - bash: |
+      source .azure/linux-cabal.bashrc
+      cabal v2-build --enable-tests --enable-benchmarks --only-dependencies
+    displayName: Build Test-dependencies
+  - bash: |
+      sudo apt update
+      sudo apt install z3
+    displayName: "Install Runtime Test-Dependencies: z3"
+  - bash: |
+      source .azure/linux-cabal.bashrc
+      # to not reinstall hie
+      cd $(Agent.TempDirectory)
+      cabal v2-install liquidhaskell-0.8.6.2 -w /opt/ghc/8.6.5/bin/ghc --overwrite-policy=always
+    displayName: "Install Runtime Test-Dependencies: liquidhaskell"
+  - bash: |
+      source .azure/linux-cabal.bashrc
+      cabal v2-build hoogle
+      cabal v2-exec hoogle generate
+    displayName: "Install Runtime Test-Dependencies: hoogle database"
+  - bash: |
+      source .azure/linux-cabal.bashrc
+      # TODO: Investigate why the test suite can't be run in parallel
+      cabal v2-test -j1
+    displayName: Run Test
+  - bash: |
+      mkdir -p .azure-cache
+      tar -vczf .azure-cache/cabal-root.tar.gz $HOME/.cabal
+      tar -vczf .azure-cache/ghc-root.tar.gz $HOME/.ghc
+      tar -vczf .azure-cache/cabal-dist.tar.gz dist-newstyle
+    displayName: "Pack cache"

.azure/linux-stack.yml

@@ -48,19 +48,19 @@ jobs:
         tar vxz --wildcards --strip-components=1 -C ~/.local/bin '*/stack'
     displayName: Install stack
   - bash: |
-      source .azure/linux.bashrc
+      source .azure/linux-stack.bashrc
       stack setup --stack-yaml $(YAML_FILE)
     displayName: Install GHC
   - bash: |
-      source .azure/linux.bashrc
+      source .azure/linux-stack.bashrc
       stack  --stack-yaml $(YAML_FILE) --install-ghc build --only-dependencies
     displayName: Build dependencies
   - bash: |
-      source .azure/linux.bashrc
+      source .azure/linux-stack.bashrc
       stack build --stack-yaml $(YAML_FILE)
     displayName: Build `hie`
   - bash: |
-      source .azure/linux.bashrc
+      source .azure/linux-stack.bashrc
       stack install --stack-yaml $(YAML_FILE) # `hie` binary required locally for tests
       mkdir .azure-deploy
       stack install --stack-yaml $(YAML_FILE) --local-bin-path .azure-deploy 
@@ -80,7 +80,7 @@ jobs:
       tar -vczf $(Build.ArtifactStagingDirectory)/$ARTIFACT_NAME.tar.xz *
     displayName: Install `hie` 
   - bash: |
-      source .azure/linux.bashrc
+      source .azure/linux-stack.bashrc
       stack build --stack-yaml $(YAML_FILE) --test --bench --only-dependencies
     displayName: Build Test-dependencies
   - bash: |
@@ -88,18 +88,20 @@ jobs:
       sudo apt install z3
     displayName: "Install Runtime Test-Dependencies: z3"
   - bash: |
-      source .azure/linux.bashrc
-      stack install --resolver=lts-11.18 liquid-fixpoint-0.7.0.7 dotgen-0.4.2 fgl-visualize-0.1.0.1 located-base-0.1.1.1 liquidhaskell-0.8.2.4
+      source .azure/linux-stack.bashrc
+      stack install --resolver=lts-13.20 liquid-fixpoint-0.8.0.2 liquidhaskell-0.8.6.2
     displayName: "Install Runtime Test-Dependencies: liquidhaskell"
   - bash: |
-      source .azure/linux.bashrc
+      source .azure/linux-stack.bashrc
       stack build hoogle --stack-yaml=$(YAML_FILE)
       stack exec hoogle generate --stack-yaml=$(YAML_FILE)
     displayName: "Install Runtime Test-Dependencies: hoogle database"
-  # - bash: |
-  #     source .azure/linux.bashrc
-  #     stack test --stack-yaml $(YAML_FILE)
-  #   displayName: Run Test
+  - bash: |
+      source .azure/linux-stack.bashrc
+      # cabal is also a test runtime dependency
+      export PATH=/opt/cabal/3.0/bin:$PATH
+      stack test --stack-yaml $(YAML_FILE)
+    displayName: Run Test
   - task: PublishBuildArtifacts@1
     inputs:
       pathtoPublish: '$(Build.ArtifactStagingDirectory)'

.azure/macos-stack.yml

@@ -88,17 +88,27 @@ jobs:
     displayName: "Install Runtime Test-Dependencies: z3"
   - bash: |
       source .azure/macos.bashrc
-      stack install --resolver=lts-11.18 liquid-fixpoint-0.7.0.7 dotgen-0.4.2 fgl-visualize-0.1.0.1 located-base-0.1.1.1 liquidhaskell-0.8.2.4
+      stack install --resolver=lts-13.20 liquid-fixpoint-0.8.0.2 liquidhaskell-0.8.6.2
     displayName: "Install Runtime Test-Dependencies: liquidhaskell"
   - bash: |
       source .azure/macos.bashrc
       stack build hoogle --stack-yaml=$(YAML_FILE)
       stack exec hoogle generate --stack-yaml=$(YAML_FILE)
     displayName: "Install Runtime Test-Dependencies: hoogle database"
-  # - bash: |
-  #     source .azure/macos.bashrc
-  #     stack test --stack-yaml $(YAML_FILE)
-  #   displayName: Run Test
+  - bash: |
+      source .azure/macos.bashrc
+      brew install cabal-install
+    displayName: "Install Runtime Test-Dependencies: cabal"
+  - bash: |
+      source .azure/macos.bashrc
+      GHC_MAJOR_VERSION=${YAML_FILE:6:3}
+      if [ $GHC_MAJOR_VERSION != "8.4" ]; then
+        stack test --stack-yaml $(YAML_FILE)
+      else
+        #TODO Enable dispatcher-test and func-test for ghc-8.4.*
+        stack test --stack-yaml $(YAML_FILE) :unit-test :plugin-dispatcher-test :wrapper-test
+      fi
+    displayName: Run Test
   - task: PublishBuildArtifacts@1
     inputs:
       pathtoPublish: '$(Build.ArtifactStagingDirectory)'

.azure/windows-installhs-stack.yml

@@ -29,5 +29,6 @@ jobs:
     displayName: Run help of `install.hs`
   - bash: |
       source .azure/windows.bashrc
-      stack install.hs latest
+      # Some executions fails with spurious errors installing the exe
+      stack install.hs latest || stack install.hs latest
     displayName: Run latest target of `install.hs`

.azure/windows-stack.yml

@@ -90,18 +90,33 @@ jobs:
     displayName: "Install Runtime Test-Dependencies: z3"
   - bash: |
       source .azure/windows.bashrc
-      stack install --resolver=lts-11.18 liquid-fixpoint-0.7.0.7 dotgen-0.4.2 fgl-visualize-0.1.0.1 located-base-0.1.1.1 liquidhaskell-0.8.2.4
-      liquid -v
+      stack install --resolver=lts-13.20 liquid-fixpoint-0.8.0.2 liquidhaskell-0.8.6.2
     displayName: "Install Runtime Test-Dependencies: liquidhaskell"
   - bash: |
       source .azure/windows.bashrc
       stack build hoogle --stack-yaml=$(YAML_FILE)
       stack exec hoogle generate --stack-yaml=$(YAML_FILE)
     displayName: "Install Runtime Test-Dependencies: hoogle database"
-#  - bash: |
-#      source .azure/windows.bashrc
-#      stack test --stack-yaml $(YAML_FILE) :unit-test
-#    displayName: Run Test
+  - bash: |
+      source .azure/windows.bashrc
+      choco install cabal
+      /C/ProgramData/chocolatey/bin/RefreshEnv.cmd
+    displayName: "Install Runtime Test-Dependencies: cabal"
+  - bash: |
+      source .azure/windows.bashrc
+      # TODO: Enable CabalHelper unit tests, see https://github.com/DanielG/cabal-helper/issues/91
+      if [ $YAML_FILE = "stack-8.6.4.yaml" ]; then
+        TEST_ARGS=--test-arguments="--skip=CabalHelper"
+      fi
+      # TODO: Enable rest of test suites
+      stack test --stack-yaml $(YAML_FILE) :unit-test :plugin-dispatcher-test $TEST_ARGS # :dispatcher-test  :wrapper-test
+      # TODO: Enable failing functional test
+      if [ $YAML_FILE != "stack-8.6.4.yaml" ]; then
+          stack test --stack-yaml $(YAML_FILE) :func-test
+      else
+          stack test --stack-yaml $(YAML_FILE) :func-test --ta="--skip \"/Hover/hover/works\""
+      fi
+    displayName: Run Test
   - task: PublishBuildArtifacts@1
     inputs:
       pathtoPublish: '$(Build.ArtifactStagingDirectory)'

.gitignore

@@ -36,6 +36,7 @@
 /docs/source/__pycache__/
 /docs/source/plugins/
 /elisp/.cask/
+/test/testdata/*.exe
 /test/testdata/FuncTest.refactored.hs
 /test/testdata/HaReAddRmParam.refactored.hs
 /test/testdata/HaReCase.refactored.hs

azure-pipelines.yml

@@ -26,6 +26,7 @@ pr:
 
 jobs:
 - template: ./.azure/linux-stack.yml
+- template: ./.azure/linux-cabal.yml
 - template: ./.azure/windows-stack.yml
 - template: ./.azure/macos-stack.yml
 - template: ./.azure/linux-installhs-stack.yml

test/functional/DiagnosticsSpec.hs

@@ -40,25 +40,18 @@ spec = describe "diagnostics providers" $ do
           reduceDiag ^. LSP.source `shouldBe` Just "hlint"
 
         diags2a <- waitForDiagnostics
-        -- liftIO $ show diags2a `shouldBe` ""
+        
         liftIO $ length diags2a `shouldBe` 2
 
-        -- docItem <- getDocItem file languageId
         sendNotification TextDocumentDidSave (DidSaveTextDocumentParams doc)
-        -- diags2hlint <- waitForDiagnostics
-        -- -- liftIO $ show diags2hlint `shouldBe` ""
-        -- liftIO $ length diags2hlint `shouldBe` 3
-        -- diags2liquid <- waitForDiagnostics
-        -- liftIO $ length diags2liquid `shouldBe` 3
-        -- -- liftIO $ show diags2 `shouldBe` ""
-        diags3@(d:_) <- waitForDiagnostics
-        -- liftIO $ show diags3 `shouldBe` ""
+        
+        diags3@(d:_) <- waitForDiagnosticsSource "eg2"
+        
         liftIO $ do
-          length diags3 `shouldBe` 3
+          length diags3 `shouldBe` 1
           d ^. LSP.range `shouldBe` Range (Position 0 0) (Position 1 0)
           d ^. LSP.severity `shouldBe` Nothing
           d ^. LSP.code `shouldBe` Nothing
-          d ^. LSP.source `shouldBe` Just "eg2"
           d ^. LSP.message `shouldBe` T.pack "Example plugin diagnostic, triggered byDiagnosticOnSave"
 
   describe "typed hole errors" $