From 5a358d689431c17a200b9e6f11980833dcd60217 Mon Sep 17 00:00:00 2001 From: Timothy Clem Date: Thu, 20 Feb 2020 09:58:47 -0800 Subject: [PATCH 01/11] Fully remove reference to submodules --- script/bootstrap | 1 - semantic-go/vendor/tree-sitter-go | 1 - semantic-java/vendor/tree-sitter-java | 1 - semantic-json/vendor/tree-sitter-json | 1 - semantic-python/vendor/tree-sitter-python | 1 - semantic-ruby/vendor/tree-sitter-ruby | 1 - semantic-tsx/vendor/tree-sitter-typescript | 1 - semantic-typescript/vendor/tree-sitter-typescript | 1 - 8 files changed, 8 deletions(-) delete mode 160000 semantic-go/vendor/tree-sitter-go delete mode 160000 semantic-java/vendor/tree-sitter-java delete mode 160000 semantic-json/vendor/tree-sitter-json delete mode 160000 semantic-python/vendor/tree-sitter-python delete mode 160000 semantic-ruby/vendor/tree-sitter-ruby delete mode 160000 semantic-tsx/vendor/tree-sitter-typescript delete mode 160000 semantic-typescript/vendor/tree-sitter-typescript diff --git a/script/bootstrap b/script/bootstrap index 659a0df3c..910c564a3 100755 --- a/script/bootstrap +++ b/script/bootstrap @@ -1,4 +1,3 @@ #!/bin/bash -git submodule sync --recursive && git submodule update --init --recursive --force cabal v2-update diff --git a/semantic-go/vendor/tree-sitter-go b/semantic-go/vendor/tree-sitter-go deleted file mode 160000 index 689cc8fbd..000000000 --- a/semantic-go/vendor/tree-sitter-go +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 689cc8fbdc0613d267434f221af85aff91a31f8c diff --git a/semantic-java/vendor/tree-sitter-java b/semantic-java/vendor/tree-sitter-java deleted file mode 160000 index afc4cec79..000000000 --- a/semantic-java/vendor/tree-sitter-java +++ /dev/null @@ -1 +0,0 @@ -Subproject commit afc4cec799f6594390aeb0ca5e16ec89e73d488e diff --git a/semantic-json/vendor/tree-sitter-json b/semantic-json/vendor/tree-sitter-json deleted file mode 160000 index 7b6a33f30..000000000 --- a/semantic-json/vendor/tree-sitter-json +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 7b6a33f300e3e88c3017e0a9d88c77b50ea6d149 diff --git a/semantic-python/vendor/tree-sitter-python b/semantic-python/vendor/tree-sitter-python deleted file mode 160000 index 899ac8d5d..000000000 --- a/semantic-python/vendor/tree-sitter-python +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 899ac8d5d6c883b2f05362c9953e14e78aac474c diff --git a/semantic-ruby/vendor/tree-sitter-ruby b/semantic-ruby/vendor/tree-sitter-ruby deleted file mode 160000 index eb2b6225b..000000000 --- a/semantic-ruby/vendor/tree-sitter-ruby +++ /dev/null @@ -1 +0,0 @@ -Subproject commit eb2b6225bfb80010f2e4cbd27db8c6f3775230b5 diff --git a/semantic-tsx/vendor/tree-sitter-typescript b/semantic-tsx/vendor/tree-sitter-typescript deleted file mode 160000 index aa950f58e..000000000 --- a/semantic-tsx/vendor/tree-sitter-typescript +++ /dev/null @@ -1 +0,0 @@ -Subproject commit aa950f58ea8aa112bc72f3481b98fc2d3c07b3e0 diff --git a/semantic-typescript/vendor/tree-sitter-typescript b/semantic-typescript/vendor/tree-sitter-typescript deleted file mode 160000 index 40320d8e0..000000000 --- a/semantic-typescript/vendor/tree-sitter-typescript +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 40320d8e0953db5e173e6dfbb596d375a085ca65 From a697d2109c2f8d87cb391491ef6988bd64f0c733 Mon Sep 17 00:00:00 2001 From: Timothy Clem Date: Thu, 20 Feb 2020 10:01:46 -0800 Subject: [PATCH 02/11] Get the tests and benchmarks building again --- semantic-go/semantic-go.cabal | 7 +++---- semantic-go/test/Test.hs | 8 ++++---- semantic-java/semantic-java.cabal | 7 +++---- semantic-java/test/Test.hs | 8 ++++---- semantic-json/semantic-json.cabal | 7 +++---- semantic-json/test/Test.hs | 8 ++++---- semantic-parse/app/Main.hs | 2 +- semantic-python/bench/Bench.hs | 2 +- semantic-python/semantic-python.cabal | 3 +-- semantic-ruby/semantic-ruby.cabal | 13 +++++-------- semantic-ruby/test/Test.hs | 8 ++++---- semantic-tsx/semantic-tsx.cabal | 7 +++---- semantic-tsx/test/Test.hs | 10 +++++----- semantic-typescript/semantic-typescript.cabal | 7 +++---- semantic-typescript/test/Test.hs | 8 ++++---- 15 files changed, 48 insertions(+), 57 deletions(-) diff --git a/semantic-go/semantic-go.cabal b/semantic-go/semantic-go.cabal index 794a73cd7..03baa3119 100644 --- a/semantic-go/semantic-go.cabal +++ b/semantic-go/semantic-go.cabal @@ -63,13 +63,12 @@ test-suite test hs-source-dirs: test main-is: Test.hs build-depends: base - , tree-sitter - , tree-sitter-go - , semantic-ast , bytestring ^>= 0.10.8.2 , hedgehog >= 0.6 && <2 , pathtype ^>= 0.8.1 - , text + , semantic-ast + , semantic-go , tasty , tasty-hedgehog , tasty-hunit + , text diff --git a/semantic-go/test/Test.hs b/semantic-go/test/Test.hs index dc42ef144..2c04d8cc4 100644 --- a/semantic-go/test/Test.hs +++ b/semantic-go/test/Test.hs @@ -1,12 +1,12 @@ {-# LANGUAGE DisambiguateRecordFields, OverloadedStrings, TypeApplications #-} module Main (main) where -import qualified System.Path as Path -import Test.Tasty -import Language.Go.Grammar -import qualified Language.Go.AST as Go import AST.Test import AST.Unmarshal +import qualified Language.Go.AST as Go +import Language.Go.Grammar +import qualified System.Path as Path +import Test.Tasty main :: IO () main diff --git a/semantic-java/semantic-java.cabal b/semantic-java/semantic-java.cabal index 179824e96..0557e17ae 100644 --- a/semantic-java/semantic-java.cabal +++ b/semantic-java/semantic-java.cabal @@ -63,11 +63,10 @@ test-suite test hs-source-dirs: test main-is: Test.hs build-depends: base - , tree-sitter - , tree-sitter-java - , semantic-ast , bytestring ^>= 0.10.8.2 , pathtype ^>= 0.8.1 - , text + , semantic-ast + , semantic-java , tasty , tasty-hunit + , text diff --git a/semantic-java/test/Test.hs b/semantic-java/test/Test.hs index 3470a81c4..0cb815697 100644 --- a/semantic-java/test/Test.hs +++ b/semantic-java/test/Test.hs @@ -1,12 +1,12 @@ {-# LANGUAGE OverloadedStrings, TypeApplications #-} module Main (main) where -import qualified System.Path as Path -import Test.Tasty -import TreeSitter.Java -import qualified TreeSitter.Java.AST as Java import AST.Test import AST.Unmarshal +import qualified Language.Java.AST as Java +import Language.Java.Grammar +import qualified System.Path as Path +import Test.Tasty main :: IO () main diff --git a/semantic-json/semantic-json.cabal b/semantic-json/semantic-json.cabal index b8d663866..4a590382a 100644 --- a/semantic-json/semantic-json.cabal +++ b/semantic-json/semantic-json.cabal @@ -63,13 +63,12 @@ test-suite test hs-source-dirs: test main-is: Test.hs build-depends: base - , tree-sitter - , tree-sitter-json - , semantic-ast , bytestring ^>= 0.10.8.2 , hedgehog >= 0.6 && <2 , pathtype ^>= 0.8.1 - , text + , semantic-ast + , semantic-json , tasty , tasty-hedgehog , tasty-hunit + , text diff --git a/semantic-json/test/Test.hs b/semantic-json/test/Test.hs index b872065ce..592d4ea73 100644 --- a/semantic-json/test/Test.hs +++ b/semantic-json/test/Test.hs @@ -1,12 +1,12 @@ {-# LANGUAGE DisambiguateRecordFields, OverloadedStrings, TypeApplications #-} module Main (main) where -import qualified System.Path as Path -import Test.Tasty -import TreeSitter.JSON -import qualified TreeSitter.JSON.AST as JSON import AST.Test import AST.Unmarshal +import qualified Language.JSON.AST as JSON +import Language.JSON.Grammar +import qualified System.Path as Path +import Test.Tasty main :: IO () main diff --git a/semantic-parse/app/Main.hs b/semantic-parse/app/Main.hs index b94791cf0..475b6ec97 100644 --- a/semantic-parse/app/Main.hs +++ b/semantic-parse/app/Main.hs @@ -14,7 +14,7 @@ import Options.Applicative hiding (style) import Text.Pretty.Simple (pPrint, pPrintNoColor) import Data.Foldable (traverse_) import Control.Monad ((>=>)) -import Marshal.JSON (marshal) +import AST.Marshal.JSON (marshal) import Data.ByteString.Lazy.Char8 (putStrLn) import Data.Aeson.Encode.Pretty (encodePretty) diff --git a/semantic-python/bench/Bench.hs b/semantic-python/bench/Bench.hs index 078e830df..2e8b1d42b 100644 --- a/semantic-python/bench/Bench.hs +++ b/semantic-python/bench/Bench.hs @@ -8,8 +8,8 @@ import qualified Data.ByteString as B import Gauge import System.Exit (die) import System.Environment (getArgs) -import Language.Python.Grammar import qualified Language.Python.AST as Py +import Language.Python.Grammar import AST.Unmarshal main :: IO () diff --git a/semantic-python/semantic-python.cabal b/semantic-python/semantic-python.cabal index f59ca0f57..42fc306b6 100644 --- a/semantic-python/semantic-python.cabal +++ b/semantic-python/semantic-python.cabal @@ -138,5 +138,4 @@ executable benchmark base , gauge ^>= 0.2.5 , bytestring - , tree-sitter - , tree-sitter-python + , semantic-python diff --git a/semantic-ruby/semantic-ruby.cabal b/semantic-ruby/semantic-ruby.cabal index abfb46d3c..293352647 100644 --- a/semantic-ruby/semantic-ruby.cabal +++ b/semantic-ruby/semantic-ruby.cabal @@ -63,16 +63,15 @@ test-suite test hs-source-dirs: test main-is: Test.hs build-depends: base - , tree-sitter - , tree-sitter-ruby - , semantic-ast , bytestring ^>= 0.10.8.2 , hedgehog >= 0.6 && <2 , pathtype ^>= 0.8.1 - , text + , semantic-ast + , semantic-ruby , tasty , tasty-hedgehog , tasty-hunit + , text executable benchmarks import: haskell @@ -87,8 +86,7 @@ executable benchmarks , Glob , lens >= 4.17 && < 4.19 , pathtype ^>= 0.8.1 - , tree-sitter - , tree-sitter-ruby + , semantic-ruby executable tree-sitter-ruby import: haskell @@ -97,5 +95,4 @@ executable tree-sitter-ruby main-is: Main.hs build-depends: base , bytestring - , tree-sitter - , tree-sitter-ruby + , semantic-ruby diff --git a/semantic-ruby/test/Test.hs b/semantic-ruby/test/Test.hs index 77ea9e6f9..7cdbdc9e9 100644 --- a/semantic-ruby/test/Test.hs +++ b/semantic-ruby/test/Test.hs @@ -1,12 +1,12 @@ {-# LANGUAGE DisambiguateRecordFields, OverloadedStrings, TypeApplications #-} module Main (main) where -import qualified System.Path as Path -import Test.Tasty -import TreeSitter.Ruby -import qualified TreeSitter.Ruby.AST as Rb import AST.Test import AST.Unmarshal +import qualified Language.Ruby.AST as Rb +import Language.Ruby.Grammar +import qualified System.Path as Path +import Test.Tasty main :: IO () main diff --git a/semantic-tsx/semantic-tsx.cabal b/semantic-tsx/semantic-tsx.cabal index 63cd2ed52..514056fce 100644 --- a/semantic-tsx/semantic-tsx.cabal +++ b/semantic-tsx/semantic-tsx.cabal @@ -63,13 +63,12 @@ test-suite test hs-source-dirs: test main-is: Test.hs build-depends: base - , tree-sitter - , tree-sitter-tsx - , semantic-ast , bytestring ^>= 0.10.8.2 , hedgehog >= 0.6 && <2 , pathtype ^>= 0.8.1 - , text + , semantic-ast + , semantic-tsx , tasty , tasty-hedgehog , tasty-hunit + , text diff --git a/semantic-tsx/test/Test.hs b/semantic-tsx/test/Test.hs index d185b303a..9697f3cfe 100644 --- a/semantic-tsx/test/Test.hs +++ b/semantic-tsx/test/Test.hs @@ -1,19 +1,19 @@ {-# LANGUAGE DisambiguateRecordFields, OverloadedStrings, TypeApplications #-} module Main (main) where -import qualified System.Path as Path -import Test.Tasty -import TreeSitter.TSX -import qualified TreeSitter.TSX.AST as Ts import AST.Test import AST.Unmarshal +import qualified Language.TSX.AST as Tsx +import Language.TSX.Grammar +import qualified System.Path as Path +import Test.Tasty main :: IO () main = readCorpusFiles (Path.relDir "tree-sitter-tsx/vendor/tree-sitter-typescript/tsx/corpus") >>= traverse (testCorpus parse) >>= defaultMain . tests - where parse = parseByteString @Ts.Program @() tree_sitter_tsx + where parse = parseByteString @Tsx.Program @() tree_sitter_tsx tests :: [TestTree] -> TestTree tests = testGroup "tree-sitter-tsx corpus tests" diff --git a/semantic-typescript/semantic-typescript.cabal b/semantic-typescript/semantic-typescript.cabal index 7257d9ee1..80417af09 100644 --- a/semantic-typescript/semantic-typescript.cabal +++ b/semantic-typescript/semantic-typescript.cabal @@ -63,13 +63,12 @@ test-suite test hs-source-dirs: test main-is: Test.hs build-depends: base - , tree-sitter - , tree-sitter-typescript - , semantic-ast , bytestring ^>= 0.10.8.2 , hedgehog >= 0.6 && <2 , pathtype ^>= 0.8.1 - , text + , semantic-ast + , semantic-typescript , tasty , tasty-hedgehog , tasty-hunit + , text diff --git a/semantic-typescript/test/Test.hs b/semantic-typescript/test/Test.hs index 2c650ce3a..df417b4ba 100644 --- a/semantic-typescript/test/Test.hs +++ b/semantic-typescript/test/Test.hs @@ -1,12 +1,12 @@ {-# LANGUAGE DisambiguateRecordFields, OverloadedStrings, TypeApplications #-} module Main (main) where -import qualified System.Path as Path -import Test.Tasty -import TreeSitter.TypeScript -import qualified TreeSitter.TypeScript.AST as Ts import AST.Test import AST.Unmarshal +import qualified Language.TypeScript.AST as Ts +import Language.TypeScript.Grammar +import qualified System.Path as Path +import Test.Tasty main :: IO () main From 4566a59f4267203912d2f0e0e8c03376ab5a6687 Mon Sep 17 00:00:00 2001 From: Timothy Clem Date: Thu, 20 Feb 2020 10:38:22 -0800 Subject: [PATCH 03/11] No real bootstrapping todo anymore --- .github/workflows/haskell.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/haskell.yml b/.github/workflows/haskell.yml index c7df54809..1016153e2 100644 --- a/.github/workflows/haskell.yml +++ b/.github/workflows/haskell.yml @@ -52,7 +52,7 @@ jobs: - name: Install dependencies run: | - script/bootstrap + cabal v2-update cabal v2-configure --project-file=cabal.project.ci --disable-optimization --enable-benchmarks --enable-tests --write-ghc-environment-files=always -j2 cabal v2-build --project-file=cabal.project.ci all --only-dependencies From 68bc0f68d5466a74678736feccad4d0fd5e2cfeb Mon Sep 17 00:00:00 2001 From: Timothy Clem Date: Thu, 20 Feb 2020 10:38:52 -0800 Subject: [PATCH 04/11] Bring back tree-sitter corpus tests for go --- .github/workflows/haskell.yml | 1 + semantic-ast/src/AST/Test.hs | 10 ++++++++-- semantic-go/semantic-go.cabal | 2 +- semantic-go/src/Language/Go/AST.hs | 3 ++- semantic-go/test/Test.hs | 3 ++- 5 files changed, 14 insertions(+), 5 deletions(-) diff --git a/.github/workflows/haskell.yml b/.github/workflows/haskell.yml index 1016153e2..b98095165 100644 --- a/.github/workflows/haskell.yml +++ b/.github/workflows/haskell.yml @@ -61,6 +61,7 @@ jobs: cabal v2-build --project-file=cabal.project.ci cabal v2-run --project-file=cabal.project.ci semantic:test cabal v2-run --project-file=cabal.project.ci semantic-core:test + cabal v2-run --project-file=cabal.project.ci semantic-go:test cabal v2-run --project-file=cabal.project.ci semantic-python:test:compiling cabal v2-run --project-file=cabal.project.ci semantic-python:test:graphing cd semantic-source; cabal v2-run --project-file=cabal.project.ci semantic-source:test; cd .. diff --git a/semantic-ast/src/AST/Test.hs b/semantic-ast/src/AST/Test.hs index f673c442e..876e39823 100644 --- a/semantic-ast/src/AST/Test.hs +++ b/semantic-ast/src/AST/Test.hs @@ -2,6 +2,7 @@ module AST.Test ( CorpusExample(..) , readCorpusFiles + , readCorpusFiles' , parseCorpusFile , testCorpus ) where @@ -22,7 +23,7 @@ import System.FilePath.Glob import Test.Tasty import Test.Tasty.HUnit -testCorpus :: (ByteString -> IO (Either String (t a))) -> Path.RelFile -> IO TestTree +testCorpus :: (ByteString -> IO (Either String (t a))) -> Path.AbsRelFile -> IO TestTree testCorpus parse path = do xs <- parseCorpusFile path case xs of @@ -50,10 +51,15 @@ readCorpusFiles parent = do files <- globDir1 (compile "**/*.txt") (Path.toString dir) pure (Path.relPath <$> files) +readCorpusFiles' :: Path.AbsDir -> IO [Path.AbsRelFile] +readCorpusFiles' dir = do + files <- globDir1 (compile "**/*.txt") (Path.toString dir) + pure (Path.file <$> files) + data CorpusExample = CorpusExample { name :: String, code :: ByteString } deriving (Eq, Show) -parseCorpusFile :: Path.RelFile -> IO (Either String [CorpusExample]) +parseCorpusFile :: Path.AbsRelFile -> IO (Either String [CorpusExample]) parseCorpusFile path = do c <- Data.ByteString.readFile (Path.toString path) pure $ parseOnly corpusParser c diff --git a/semantic-go/semantic-go.cabal b/semantic-go/semantic-go.cabal index 03baa3119..6b25ca4cc 100644 --- a/semantic-go/semantic-go.cabal +++ b/semantic-go/semantic-go.cabal @@ -31,7 +31,7 @@ common haskell , template-haskell ^>= 2.15 , text ^>= 1.2.3 , tree-sitter ^>= 0.9 - , tree-sitter-go ^>= 0.5.0.0 + , tree-sitter-go ^>= 0.5.0.1 ghc-options: -Weverything diff --git a/semantic-go/src/Language/Go/AST.hs b/semantic-go/src/Language/Go/AST.hs index 361c3c4c4..98b48f9ea 100644 --- a/semantic-go/src/Language/Go/AST.hs +++ b/semantic-go/src/Language/Go/AST.hs @@ -12,11 +12,12 @@ module Language.Go.AST ( module Language.Go.AST +, Go.getTestCorpusDir ) where import Prelude hiding (False, Float, Integer, Rational, String, True) import AST.GenerateSyntax import Language.Haskell.TH.Syntax (runIO) -import qualified TreeSitter.Go as Go (getNodeTypesPath, tree_sitter_go) +import qualified TreeSitter.Go as Go (getNodeTypesPath, getTestCorpusDir, tree_sitter_go) runIO Go.getNodeTypesPath >>= astDeclarationsForLanguage Go.tree_sitter_go diff --git a/semantic-go/test/Test.hs b/semantic-go/test/Test.hs index 2c04d8cc4..9a205ba5b 100644 --- a/semantic-go/test/Test.hs +++ b/semantic-go/test/Test.hs @@ -10,7 +10,8 @@ import Test.Tasty main :: IO () main - = readCorpusFiles (Path.relDir "tree-sitter-go/vendor/tree-sitter-go/corpus") + = Path.absDir <$> Go.getTestCorpusDir + >>= readCorpusFiles' >>= traverse (testCorpus parse) >>= defaultMain . tests where parse = parseByteString @Go.SourceFile @() tree_sitter_go From b5a47e30bc4653f9d068f0923f4027a53e9f9209 Mon Sep 17 00:00:00 2001 From: Timothy Clem Date: Thu, 20 Feb 2020 13:46:36 -0800 Subject: [PATCH 05/11] Bring back tree-sitter corpus tests for java --- semantic-java/semantic-java.cabal | 2 +- semantic-java/src/Language/Java/AST.hs | 3 ++- semantic-java/test/Test.hs | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/semantic-java/semantic-java.cabal b/semantic-java/semantic-java.cabal index 0557e17ae..0fa946b89 100644 --- a/semantic-java/semantic-java.cabal +++ b/semantic-java/semantic-java.cabal @@ -31,7 +31,7 @@ common haskell , template-haskell ^>= 2.15 , text ^>= 1.2.3 , tree-sitter ^>= 0.9 - , tree-sitter-java ^>= 0.7.0.0 + , tree-sitter-java ^>= 0.7.0.1 ghc-options: -Weverything diff --git a/semantic-java/src/Language/Java/AST.hs b/semantic-java/src/Language/Java/AST.hs index 4f951d6f5..467dea0ce 100644 --- a/semantic-java/src/Language/Java/AST.hs +++ b/semantic-java/src/Language/Java/AST.hs @@ -12,11 +12,12 @@ module Language.Java.AST ( module Language.Java.AST +, Java.getTestCorpusDir ) where import AST.GenerateSyntax import AST.Token import Language.Haskell.TH.Syntax (runIO) -import qualified TreeSitter.Java as Java (getNodeTypesPath, tree_sitter_java) +import qualified TreeSitter.Java as Java (getNodeTypesPath, getTestCorpusDir, tree_sitter_java) runIO Java.getNodeTypesPath >>= astDeclarationsForLanguage Java.tree_sitter_java diff --git a/semantic-java/test/Test.hs b/semantic-java/test/Test.hs index 0cb815697..8d272beaa 100644 --- a/semantic-java/test/Test.hs +++ b/semantic-java/test/Test.hs @@ -10,7 +10,8 @@ import Test.Tasty main :: IO () main - = readCorpusFiles (Path.relDir "tree-sitter-java/vendor/tree-sitter-java/corpus") + = Path.absDir <$> Java.getTestCorpusDir + >>= readCorpusFiles' >>= traverse (testCorpus parse) >>= defaultMain . tests where From a5b2e9922a3f21b628b7de72af706ebbad092f7d Mon Sep 17 00:00:00 2001 From: Timothy Clem Date: Thu, 20 Feb 2020 13:46:47 -0800 Subject: [PATCH 06/11] Bring back tree-sitter corpus tests for json --- semantic-json/semantic-json.cabal | 2 +- semantic-json/src/Language/JSON/AST.hs | 3 ++- semantic-json/test/Test.hs | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/semantic-json/semantic-json.cabal b/semantic-json/semantic-json.cabal index 4a590382a..f44391a64 100644 --- a/semantic-json/semantic-json.cabal +++ b/semantic-json/semantic-json.cabal @@ -31,7 +31,7 @@ common haskell , template-haskell ^>= 2.15 , text ^>= 1.2.3 , tree-sitter ^>= 0.9 - , tree-sitter-json ^>= 0.7.0.0 + , tree-sitter-json ^>= 0.7.0.1 ghc-options: -Weverything diff --git a/semantic-json/src/Language/JSON/AST.hs b/semantic-json/src/Language/JSON/AST.hs index 7c1a697f5..37713a0df 100644 --- a/semantic-json/src/Language/JSON/AST.hs +++ b/semantic-json/src/Language/JSON/AST.hs @@ -11,11 +11,12 @@ {-# LANGUAGE TypeApplications #-} module Language.JSON.AST ( module Language.JSON.AST +, JSON.getTestCorpusDir ) where import Prelude hiding (String) import AST.GenerateSyntax import Language.Haskell.TH.Syntax (runIO) -import qualified TreeSitter.JSON as JSON (getNodeTypesPath, tree_sitter_json) +import qualified TreeSitter.JSON as JSON (getNodeTypesPath, getTestCorpusDir, tree_sitter_json) runIO JSON.getNodeTypesPath >>= astDeclarationsForLanguage JSON.tree_sitter_json diff --git a/semantic-json/test/Test.hs b/semantic-json/test/Test.hs index 592d4ea73..7e9ae80bb 100644 --- a/semantic-json/test/Test.hs +++ b/semantic-json/test/Test.hs @@ -10,7 +10,8 @@ import Test.Tasty main :: IO () main - = readCorpusFiles (Path.relDir "tree-sitter-json/vendor/tree-sitter-json/corpus") + = Path.absDir <$> JSON.getTestCorpusDir + >>= readCorpusFiles' >>= traverse (testCorpus parse) >>= defaultMain . tests where parse = parseByteString @JSON.Document @() tree_sitter_json From d358ff311e841641035bb76620614305061cfcc2 Mon Sep 17 00:00:00 2001 From: Timothy Clem Date: Thu, 20 Feb 2020 13:46:53 -0800 Subject: [PATCH 07/11] php cleanup --- semantic-php/vendor/tree-sitter-php | 1 - 1 file changed, 1 deletion(-) delete mode 160000 semantic-php/vendor/tree-sitter-php diff --git a/semantic-php/vendor/tree-sitter-php b/semantic-php/vendor/tree-sitter-php deleted file mode 160000 index 6bb7e0421..000000000 --- a/semantic-php/vendor/tree-sitter-php +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 6bb7e04216eae8314e0b893029590247418b5998 From f9845be5d89c4bdb79702aa1a2c848676b4f498e Mon Sep 17 00:00:00 2001 From: Timothy Clem Date: Thu, 20 Feb 2020 13:47:06 -0800 Subject: [PATCH 08/11] Bring back tree-sitter corpus tests for python --- semantic-python/semantic-python.cabal | 11 +++++------ semantic-python/src/Language/Python/AST.hs | 3 ++- semantic-python/test/PreciseTest.hs | 7 ++++--- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/semantic-python/semantic-python.cabal b/semantic-python/semantic-python.cabal index 42fc306b6..488ec043e 100644 --- a/semantic-python/semantic-python.cabal +++ b/semantic-python/semantic-python.cabal @@ -34,7 +34,7 @@ common haskell , template-haskell ^>= 2.15 , text ^>= 1.2.3 , tree-sitter ^>= 0.9 - , tree-sitter-python ^>= 0.9.0.1 + , tree-sitter-python ^>= 0.9.0.2 , containers ghc-options: -Weverything @@ -116,18 +116,17 @@ test-suite test import: haskell type: exitcode-stdio-1.0 hs-source-dirs: test - main-is: Test.hs + main-is: PreciseTest.hs build-depends: base - , tree-sitter - , tree-sitter-python - , semantic-ast , bytestring ^>= 0.10.8.2 , hedgehog >= 0.6 && <2 , pathtype ^>= 0.8.1 - , text + , semantic-ast + , semantic-python , tasty , tasty-hedgehog , tasty-hunit + , text executable benchmark import: haskell diff --git a/semantic-python/src/Language/Python/AST.hs b/semantic-python/src/Language/Python/AST.hs index 0db2bb614..fd88ab06e 100644 --- a/semantic-python/src/Language/Python/AST.hs +++ b/semantic-python/src/Language/Python/AST.hs @@ -12,11 +12,12 @@ module Language.Python.AST ( module Language.Python.AST +, Python.getTestCorpusDir ) where import Prelude hiding (False, Float, Integer, String, True) import AST.GenerateSyntax import Language.Haskell.TH.Syntax (runIO) -import qualified TreeSitter.Python as Python (getNodeTypesPath, tree_sitter_python) +import qualified TreeSitter.Python as Python (getNodeTypesPath, getTestCorpusDir, tree_sitter_python) runIO Python.getNodeTypesPath >>= astDeclarationsForLanguage Python.tree_sitter_python diff --git a/semantic-python/test/PreciseTest.hs b/semantic-python/test/PreciseTest.hs index 742db329f..9f33cf604 100644 --- a/semantic-python/test/PreciseTest.hs +++ b/semantic-python/test/PreciseTest.hs @@ -3,14 +3,15 @@ module Main (main) where import qualified System.Path as Path import Test.Tasty -import TreeSitter.Python -import qualified TreeSitter.Python.AST as Py +import qualified Language.Python.AST as Py +import Language.Python.Grammar import AST.Test import AST.Unmarshal main :: IO () main - = readCorpusFiles (Path.relDir "tree-sitter-python/vendor/tree-sitter-python/test/corpus") + = Path.absDir <$> Py.getTestCorpusDir + >>= readCorpusFiles' >>= traverse (testCorpus parse) >>= defaultMain . tests where parse = parseByteString @Py.Module @() tree_sitter_python From b3ed6c37f1d338d761a95e3cbe23181227ac646f Mon Sep 17 00:00:00 2001 From: Timothy Clem Date: Thu, 20 Feb 2020 13:47:23 -0800 Subject: [PATCH 09/11] Bring back tree-sitter corpus tests for ruby --- semantic-ruby/semantic-ruby.cabal | 2 +- semantic-ruby/src/Language/Ruby/AST.hs | 3 ++- semantic-ruby/test/Test.hs | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/semantic-ruby/semantic-ruby.cabal b/semantic-ruby/semantic-ruby.cabal index 293352647..df6a3daa3 100644 --- a/semantic-ruby/semantic-ruby.cabal +++ b/semantic-ruby/semantic-ruby.cabal @@ -31,7 +31,7 @@ common haskell , template-haskell ^>= 2.15 , text ^>= 1.2.3 , tree-sitter ^>= 0.9 - , tree-sitter-ruby ^>= 0.5.0.1 + , tree-sitter-ruby ^>= 0.5.0.2 ghc-options: -Weverything diff --git a/semantic-ruby/src/Language/Ruby/AST.hs b/semantic-ruby/src/Language/Ruby/AST.hs index 4e3ce2426..0e6cc0db2 100644 --- a/semantic-ruby/src/Language/Ruby/AST.hs +++ b/semantic-ruby/src/Language/Ruby/AST.hs @@ -12,11 +12,12 @@ module Language.Ruby.AST ( module Language.Ruby.AST +, Ruby.getTestCorpusDir ) where import Prelude hiding (False, Float, Integer, Rational, String, True) import AST.GenerateSyntax import Language.Haskell.TH.Syntax (runIO) -import qualified TreeSitter.Ruby as Ruby (getNodeTypesPath, tree_sitter_ruby) +import qualified TreeSitter.Ruby as Ruby (getNodeTypesPath, getTestCorpusDir, tree_sitter_ruby) runIO Ruby.getNodeTypesPath >>= astDeclarationsForLanguage Ruby.tree_sitter_ruby diff --git a/semantic-ruby/test/Test.hs b/semantic-ruby/test/Test.hs index 7cdbdc9e9..21789e8e8 100644 --- a/semantic-ruby/test/Test.hs +++ b/semantic-ruby/test/Test.hs @@ -10,7 +10,8 @@ import Test.Tasty main :: IO () main - = readCorpusFiles (Path.relDir "tree-sitter-ruby/vendor/tree-sitter-ruby/test/corpus") + = Path.absDir <$> Rb.getTestCorpusDir + >>= readCorpusFiles' >>= traverse (testCorpus parse) >>= defaultMain . tests where parse = parseByteString @Rb.Program @() tree_sitter_ruby From a135e887915020ae53ff73bfc150671ecd6f19b8 Mon Sep 17 00:00:00 2001 From: Timothy Clem Date: Thu, 20 Feb 2020 14:22:24 -0800 Subject: [PATCH 10/11] Bring back tree-sitter corpus for tsx and typescript --- semantic-tsx/semantic-tsx.cabal | 2 +- semantic-tsx/src/Language/TSX/AST.hs | 3 ++- semantic-tsx/test/Test.hs | 3 ++- semantic-typescript/semantic-typescript.cabal | 2 +- semantic-typescript/src/Language/TypeScript/AST.hs | 3 ++- semantic-typescript/test/Test.hs | 3 ++- 6 files changed, 10 insertions(+), 6 deletions(-) diff --git a/semantic-tsx/semantic-tsx.cabal b/semantic-tsx/semantic-tsx.cabal index 514056fce..0ea266659 100644 --- a/semantic-tsx/semantic-tsx.cabal +++ b/semantic-tsx/semantic-tsx.cabal @@ -31,7 +31,7 @@ common haskell , template-haskell ^>= 2.15 , text ^>= 1.2.3 , tree-sitter ^>= 0.9 - , tree-sitter-tsx ^>= 0.5.0.0 + , tree-sitter-tsx ^>= 0.5.0.1 ghc-options: -Weverything diff --git a/semantic-tsx/src/Language/TSX/AST.hs b/semantic-tsx/src/Language/TSX/AST.hs index 77bf7bda0..dda33df97 100644 --- a/semantic-tsx/src/Language/TSX/AST.hs +++ b/semantic-tsx/src/Language/TSX/AST.hs @@ -12,11 +12,12 @@ module Language.TSX.AST ( module Language.TSX.AST +, TSX.getTestCorpusDir ) where import Prelude hiding (False, Float, Integer, String, True) import AST.GenerateSyntax import Language.Haskell.TH.Syntax (runIO) -import qualified TreeSitter.TSX as TSX (getNodeTypesPath, tree_sitter_tsx) +import qualified TreeSitter.TSX as TSX (getNodeTypesPath, getTestCorpusDir, tree_sitter_tsx) runIO TSX.getNodeTypesPath >>= astDeclarationsForLanguage TSX.tree_sitter_tsx diff --git a/semantic-tsx/test/Test.hs b/semantic-tsx/test/Test.hs index 9697f3cfe..c8f1cc723 100644 --- a/semantic-tsx/test/Test.hs +++ b/semantic-tsx/test/Test.hs @@ -10,7 +10,8 @@ import Test.Tasty main :: IO () main - = readCorpusFiles (Path.relDir "tree-sitter-tsx/vendor/tree-sitter-typescript/tsx/corpus") + = Path.absDir <$> Tsx.getTestCorpusDir + >>= readCorpusFiles' >>= traverse (testCorpus parse) >>= defaultMain . tests where parse = parseByteString @Tsx.Program @() tree_sitter_tsx diff --git a/semantic-typescript/semantic-typescript.cabal b/semantic-typescript/semantic-typescript.cabal index 80417af09..7af558781 100644 --- a/semantic-typescript/semantic-typescript.cabal +++ b/semantic-typescript/semantic-typescript.cabal @@ -31,7 +31,7 @@ common haskell , template-haskell ^>= 2.15 , text ^>= 1.2.3 , tree-sitter ^>= 0.9.0.0 - , tree-sitter-typescript ^>= 0.5.0.0 + , tree-sitter-typescript ^>= 0.5.0.1 ghc-options: -Weverything diff --git a/semantic-typescript/src/Language/TypeScript/AST.hs b/semantic-typescript/src/Language/TypeScript/AST.hs index fd755b4e9..54077fb09 100644 --- a/semantic-typescript/src/Language/TypeScript/AST.hs +++ b/semantic-typescript/src/Language/TypeScript/AST.hs @@ -12,11 +12,12 @@ module Language.TypeScript.AST ( module Language.TypeScript.AST +, TypeScript.getTestCorpusDir ) where import Prelude hiding (False, Float, Integer, String, True) import AST.GenerateSyntax import Language.Haskell.TH.Syntax (runIO) -import qualified TreeSitter.TypeScript as TypeScript (getNodeTypesPath, tree_sitter_typescript) +import qualified TreeSitter.TypeScript as TypeScript (getNodeTypesPath, getTestCorpusDir, tree_sitter_typescript) runIO TypeScript.getNodeTypesPath >>= astDeclarationsForLanguage TypeScript.tree_sitter_typescript diff --git a/semantic-typescript/test/Test.hs b/semantic-typescript/test/Test.hs index df417b4ba..e461f1d21 100644 --- a/semantic-typescript/test/Test.hs +++ b/semantic-typescript/test/Test.hs @@ -10,7 +10,8 @@ import Test.Tasty main :: IO () main - = readCorpusFiles (Path.relDir "tree-sitter-typescript/vendor/tree-sitter-typescript/typescript/corpus") + = Path.absDir <$> Ts.getTestCorpusDir + >>= readCorpusFiles' >>= traverse (testCorpus parse) >>= defaultMain . tests where parse = parseByteString @Ts.Program @() tree_sitter_typescript From 797805cbcbd3a125ebbc9c896253ac5f5009d751 Mon Sep 17 00:00:00 2001 From: Timothy Clem Date: Thu, 20 Feb 2020 14:22:32 -0800 Subject: [PATCH 11/11] Turn on all this tests in CI --- .github/workflows/haskell.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/haskell.yml b/.github/workflows/haskell.yml index b98095165..941a53cb1 100644 --- a/.github/workflows/haskell.yml +++ b/.github/workflows/haskell.yml @@ -62,7 +62,13 @@ jobs: cabal v2-run --project-file=cabal.project.ci semantic:test cabal v2-run --project-file=cabal.project.ci semantic-core:test cabal v2-run --project-file=cabal.project.ci semantic-go:test + cabal v2-run --project-file=cabal.project.ci semantic-java:test + cabal v2-run --project-file=cabal.project.ci semantic-json:test + cabal v2-run --project-file=cabal.project.ci semantic-python:test cabal v2-run --project-file=cabal.project.ci semantic-python:test:compiling cabal v2-run --project-file=cabal.project.ci semantic-python:test:graphing + cabal v2-run --project-file=cabal.project.ci semantic-ruby:test + cabal v2-run --project-file=cabal.project.ci semantic-tsx:test + cabal v2-run --project-file=cabal.project.ci semantic-typescript:test cd semantic-source; cabal v2-run --project-file=cabal.project.ci semantic-source:test; cd .. cd semantic-source; cabal v2-run --project-file=cabal.project.ci semantic-source:doctest -- src; cd ..