github/semantic
1
1
Fork 0
mirror of https://github.com/github/semantic.git synced 2024-12-21 05:41:54 +03:00
Code Issues Projects Releases Wiki Activity

Merge pull request #1293 from github/simpler-tree-sitter-module-hierarchy

Browse Source 
Simpler tree-sitter module hierarchy
This commit is contained in:
Rob Rix 2017-08-15 08:56:23 -04:00 committed by GitHub
commit e49b18638d
60 changed files with 223 additions and 382 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
.gitmodules vendored
View File

@ -13,30 +13,15 @@
[submodule "vendor/effects"]
path = vendor/effects
url = https://github.com/joshvera/effects.git
[submodule "languages/ruby/vendor/tree-sitter-ruby"]
path = languages/ruby/vendor/tree-sitter-ruby
url = https://github.com/tree-sitter/tree-sitter-ruby.git
[submodule "languages/c/vendor/tree-sitter-c"]
path = languages/c/vendor/tree-sitter-c
url = https://github.com/tree-sitter/tree-sitter-c.git
[submodule "languages/go/vendor/tree-sitter-go"]
path = languages/go/vendor/tree-sitter-go
url = https://github.com/tree-sitter/tree-sitter-go.git
[submodule "languages/javascript/vendor/tree-sitter-javascript"]
path = languages/javascript/vendor/tree-sitter-javascript
url = https://github.com/tree-sitter/tree-sitter-javascript.git
[submodule "vendor/haskell-tree-sitter"]
path = vendor/haskell-tree-sitter
url = https://github.com/tree-sitter/haskell-tree-sitter.git
[submodule "languages/typescript/vendor/tree-sitter-typescript"]
path = languages/typescript/vendor/tree-sitter-typescript
url = https://github.com/tree-sitter/tree-sitter-typescript/
[submodule "languages/python/vendor/tree-sitter-python"]
path = languages/python/vendor/tree-sitter-python
url = https://github.com/tree-sitter/tree-sitter-python.git
[submodule "languages/json/vendor/tree-sitter-json"]
path = languages/json/vendor/tree-sitter-json
url = https://github.com/tree-sitter/tree-sitter-json
[submodule "vendor/freer-cofreer"]
path = vendor/freer-cofreer
url = https://github.com/robrix/freer-cofreer.git

3
cabal.project
View File

@ -1,4 +1,3 @@
packages: ./ languages/*/ tools/*/
optional-packages: vendor/*/
packages: ./ tools/*/ vendor/*/ vendor/haskell-tree-sitter/languages/*/
jobs: $ncpus

2
languages/go/Setup.hs
View File

@ -1,2 +0,0 @@
import Distribution.Simple
main = defaultMain

25
languages/go/go.cabal
View File

@ -1,25 +0,0 @@
name: go
version: 0.1.0
synopsis: tree-sitter go language bindings
description: Please see README.md
homepage: https://github.com/github/semantic-diff#readme
author: semantic-code
maintainer: tclem@github.com
copyright: 2017 GitHub
category: Web
build-type: Simple
-- extra-source-files:
cabal-version: >=1.10
library
hs-source-dirs: src
exposed-modules: Text.Parser.TreeSitter.Go
build-depends: base >= 4.7 && < 5
, haskell-tree-sitter
default-language: Haskell2010
default-extensions: FlexibleInstances, OverloadedStrings, NoImplicitPrelude, RecordWildCards
c-sources: vendor/tree-sitter-go/src/parser.c
source-repository head
type: git
location: https://github.com/github/semantic-diff

6
languages/go/src/Text/Parser/TreeSitter/Go.hs
View File

@ -1,6 +0,0 @@
module Text.Parser.TreeSitter.Go where
import Text.Parser.TreeSitter
import Foreign.Ptr
foreign import ccall unsafe "vendor/tree-sitter-go/src/parser.c tree_sitter_go" tree_sitter_go :: Ptr Language

1
languages/go/vendor/tree-sitter-go vendored

@ -1 +0,0 @@
Subproject commit 025ece6ddcdf3e3cbc6c865e2d2c010217fbf90f

2
languages/json/Setup.hs
View File

@ -1,2 +0,0 @@
import Distribution.Simple
main = defaultMain

27
languages/json/json.cabal
View File

@ -1,27 +0,0 @@
name: json
version: 0.1.0
synopsis: tree-sitter json language bindings
description: Please see README.md
homepage: https://github.com/github/semantic-diff#readme
author: semantic-code
maintainer: vera@github.com
copyright: 2017 GitHub
category: Web
build-type: Simple
-- extra-source-files:
cabal-version: >=1.10
library
hs-source-dirs: src
exposed-modules: Text.Parser.TreeSitter.JSON
build-depends: base >= 4.7 && < 5
, haskell-tree-sitter
default-language: Haskell2010
default-extensions: FlexibleInstances, OverloadedStrings, NoImplicitPrelude, RecordWildCards
c-sources: vendor/tree-sitter-json/src/parser.c
cc-options: -std=c99 -Os
source-repository head
type: git
location: https://github.com/github/semantic-diff

6
languages/json/src/Text/Parser/TreeSitter/JSON.hs
View File

@ -1,6 +0,0 @@
module Text.Parser.TreeSitter.JSON where
import Foreign.Ptr
import Text.Parser.TreeSitter
foreign import ccall unsafe "vendor/tree-sitter-json/src/parser.c tree_sitter_json" tree_sitter_json :: Ptr Language

1
languages/json/vendor/tree-sitter-json vendored

@ -1 +0,0 @@
Subproject commit 81b519e6ad7aa87c61851f1f6925b59fcc34d347

2
languages/python/Setup.hs
View File

@ -1,2 +0,0 @@
import Distribution.Simple
main = defaultMain

27
languages/python/python.cabal
View File

@ -1,27 +0,0 @@
name: python
version: 0.1.0
synopsis: tree-sitter python language bindings
homepage: https://github.com/github/semantic-diff#readme
description: Please see README.md
author: semantic-code
maintainer: tclem@github.com
copyright: 2017 GitHub
category: Web
build-type: Simple
-- extra-source-files:
cabal-version: >=1.10
library
hs-source-dirs: src
exposed-modules: Text.Parser.TreeSitter.Python
build-depends: base >= 4.7 && < 5
, haskell-tree-sitter
default-language: Haskell2010
default-extensions: FlexibleInstances, OverloadedStrings, NoImplicitPrelude, RecordWildCards
c-sources: vendor/tree-sitter-python/src/parser.c
, vendor/tree-sitter-python/src/scanner.cc
extra-libraries: stdc++
source-repository head
type: git
location: https://github.com/github/semantic-diff

8
languages/python/src/Text/Parser/TreeSitter/Python.hs
View File

@ -1,8 +0,0 @@
module Text.Parser.TreeSitter.Python
( tree_sitter_python
) where
import Foreign.Ptr
import Text.Parser.TreeSitter
foreign import ccall unsafe "vendor/tree-sitter-python/src/parser.c tree_sitter_python" tree_sitter_python :: Ptr Language

1
languages/python/vendor/tree-sitter-python vendored

@ -1 +0,0 @@
Subproject commit 24220e066fd65ccfa5dba65c0095705d559e97cb

2
languages/ruby/Setup.hs
View File

@ -1,2 +0,0 @@
import Distribution.Simple
main = defaultMain

27
languages/ruby/ruby.cabal
View File

@ -1,27 +0,0 @@
name: ruby
version: 0.1.0
synopsis: tree-sitter ruby language bindings
description: Please see README.md
homepage: https://github.com/github/semantic-diff#readme
author: semantic-code
maintainer: tclem@github.com
copyright: 2017 GitHub
category: Web
build-type: Simple
-- extra-source-files:
cabal-version: >=1.10
library
hs-source-dirs: src
exposed-modules: Text.Parser.TreeSitter.Ruby
build-depends: base >= 4.7 && < 5
, haskell-tree-sitter
default-language: Haskell2010
default-extensions: FlexibleInstances, OverloadedStrings, NoImplicitPrelude, RecordWildCards
c-sources: vendor/tree-sitter-ruby/src/parser.c
, vendor/tree-sitter-ruby/src/scanner.cc
extra-libraries: stdc++
source-repository head
type: git
location: https://github.com/github/semantic-diff

8
languages/ruby/src/Text/Parser/TreeSitter/Ruby.hs
View File

@ -1,8 +0,0 @@
module Text.Parser.TreeSitter.Ruby
( tree_sitter_ruby
) where
import Foreign.Ptr
import Text.Parser.TreeSitter
foreign import ccall unsafe "vendor/tree-sitter-ruby/src/parser.c tree_sitter_ruby" tree_sitter_ruby :: Ptr Language

1
languages/ruby/vendor/tree-sitter-ruby vendored

@ -1 +0,0 @@
Subproject commit 0fca7f41c7fdb44ca7fdecb9a94434e429b4843f

2
languages/typescript/Setup.hs
View File

@ -1,2 +0,0 @@
import Distribution.Simple
main = defaultMain

6
languages/typescript/src/Text/Parser/TreeSitter/TypeScript.hs
View File

@ -1,6 +0,0 @@
module Text.Parser.TreeSitter.TypeScript where
import Foreign.Ptr
import Text.Parser.TreeSitter
foreign import ccall unsafe "vendor/tree-sitter-typescript/src/parser.c tree_sitter_typescript" tree_sitter_typescript :: Ptr Language

28
languages/typescript/typescript.cabal
View File

@ -1,28 +0,0 @@
name: typescript
version: 0.1.0
synopsis: tree-sitter typescript language bindings
description: Please see README.md
homepage: https://github.com/github/semantic-diff#readme
author: semantic-code
maintainer: vera@github.com
copyright: 2017 GitHub
category: Web
build-type: Simple
-- extra-source-files:
cabal-version: >=1.10
library
hs-source-dirs: src
exposed-modules: Text.Parser.TreeSitter.TypeScript
build-depends: base >= 4.7 && < 5
, haskell-tree-sitter
default-language: Haskell2010
default-extensions: FlexibleInstances, OverloadedStrings, NoImplicitPrelude, RecordWildCards
c-sources: vendor/tree-sitter-typescript/src/parser.c
, vendor/tree-sitter-typescript/src/scanner.c
cc-options: -std=c99 -Os
source-repository head
type: git
location: https://github.com/github/semantic-diff

1
languages/typescript/vendor/tree-sitter-typescript vendored

@ -1 +0,0 @@
Subproject commit 6d44526e059ebf456c9059ee1717126d69e41018

10
semantic-diff.cabal
View File

@ -116,11 +116,11 @@ library
, time
, unix
, haskell-tree-sitter
, go
, ruby
, typescript
, python
, json
, tree-sitter-go
, tree-sitter-json
, tree-sitter-python
, tree-sitter-ruby
, tree-sitter-typescript
default-language: Haskell2010
default-extensions: DeriveFunctor, DeriveFoldable, DeriveTraversable, DeriveGeneric, FlexibleContexts, FlexibleInstances, OverloadedStrings, RecordWildCards, StrictData
ghc-options: -Wall -fno-warn-name-shadowing -O -j

2
src/Data/Syntax/Assignment.hs
View File

@ -115,7 +115,7 @@ import GHC.Stack
import qualified Info
import Prelude hiding (head, until)
import Text.Parser.Combinators as Parsers
import Text.Parser.TreeSitter.Language
import TreeSitter.Language
-- | Assignment from an AST with some set of 'symbol's onto some other value.
--

1
src/Files.hs
View File

@ -29,6 +29,7 @@ import Text.Read
-- | Read a utf8-encoded file to a 'Blob'.
readFile :: forall m. MonadIO m => FilePath -> Maybe Language -> m Blob.Blob
readFile path@"/dev/null" _ = pure (Blob.emptyBlob path)
readFile path language = do
raw <- liftIO $ (Just <$> B.readFile path) `catch` (const (pure Nothing) :: IOException -> IO (Maybe B.ByteString))
pure $ fromMaybe (Blob.emptyBlob path) (Blob.sourceBlob path language . fromBytes <$> raw)

4
src/Language/Go/Syntax.hs
View File

@ -2,8 +2,8 @@
module Language.Go.Syntax where
import Language.Haskell.TH
import Text.Parser.TreeSitter.Go
import Text.Parser.TreeSitter.Language
import TreeSitter.Go
import TreeSitter.Language
-- | Statically-known rules corresponding to symbols in the grammar.
mkSymbolDatatype (mkName "Grammar") tree_sitter_go

4
src/Language/JSON/Grammar.hs
View File

@ -2,8 +2,8 @@
module Language.JSON.Grammar where
import Language.Haskell.TH
import Text.Parser.TreeSitter.Language
import Text.Parser.TreeSitter.JSON
import TreeSitter.Language
import TreeSitter.JSON
-- | Statically-known rules corresponding to symbols in the grammar.
-- v2 - bump this to regenerate

2
src/Language/Markdown.hs
View File

@ -11,7 +11,7 @@ import Data.Ix
import Data.Source
import qualified Data.Syntax.Assignment as A (AST, Node(..))
import Info
import Text.Parser.TreeSitter.Language (Symbol(..), SymbolType(..))
import TreeSitter.Language (Symbol(..), SymbolType(..))
data Grammar
= Document

4
src/Language/Python/Grammar.hs
View File

@ -2,8 +2,8 @@
module Language.Python.Grammar where
import Language.Haskell.TH
import Text.Parser.TreeSitter.Language
import Text.Parser.TreeSitter.Python
import TreeSitter.Language
import TreeSitter.Python
-- | Statically-known rules corresponding to symbols in the grammar.
-- v3 - bump this to regenerate

4
src/Language/Ruby/Grammar.hs
View File

@ -2,8 +2,8 @@
module Language.Ruby.Grammar where
import Language.Haskell.TH
import Text.Parser.TreeSitter.Language
import Text.Parser.TreeSitter.Ruby
import TreeSitter.Language
import TreeSitter.Ruby
-- v2 - Bump to get file to change to force template haskell to regenerate.
-- | Statically-known rules corresponding to symbols in the grammar.

17
src/Language/TypeScript.hs
View File

@ -69,7 +69,7 @@ termAssignment _ category children =
| S.Indexed _ <- unwrap statements
-> Just $ S.Export Nothing (toList (unwrap statements))
| otherwise -> Just $ S.Export (Just statements) []
(For, _:_) -> Just $ S.For (init children) [last children]
(For, _:_) -> Just $ S.For (init children >>= flattenExpressionStatements) [last children]
(Function, children) -> case break ((== ExpressionStatements) . Info.category . extract) children of
(inits, [body]) -> case inits of
[id, callSignature] -> Just $ S.Function id (toList (unwrap callSignature)) (toList (unwrap body))
@ -79,6 +79,9 @@ termAssignment _ category children =
(Ty, children) -> Just $ S.Ty children
(Interface, children) -> toInterface children
_ -> Nothing
where flattenExpressionStatements term
| Info.category (extract term) `elem` [ExpressionStatements, CommaOperator] = toList (unwrap term) >>= flattenExpressionStatements
| otherwise = [term]
categoryForTypeScriptName :: Text -> Category
categoryForTypeScriptName category = case category of
@ -95,8 +98,10 @@ categoryForTypeScriptName category = case category of
"arrow_function" -> Function
"generator_function" -> Function
"math_op" -> MathOperator -- math operator, e.g. +, -, *, /.
"update_expression" -> MathOperator -- math operator, e.g. ++, --
"bool_op" -> BooleanOperator -- boolean operator, e.g. ||, &&.
"comma_op" -> CommaOperator -- comma operator, e.g. expr1, expr2.
"sequence_expression" -> CommaOperator -- comma operator, e.g. expr1, expr2.
"delete_op" -> Operator -- delete operator, e.g. delete x[2].
"type_op" -> Operator -- type operator, e.g. typeof Object.
"void_op" -> Operator -- void operator, e.g. void 2.
@ -109,12 +114,15 @@ categoryForTypeScriptName category = case category of
"new_expression" -> Constructor
"class" -> Class
"catch" -> Catch
"catch_clause" -> Catch
"finally" -> Finally
"finally_clause" -> Finally
"if_statement" -> If
"trailing_if_statement" -> If
"empty_statement" -> Empty
"program" -> Program
"function_call" -> FunctionCall
"call_expression" -> FunctionCall
"pair" -> Pair
"string" -> StringLiteral
"integer" -> IntegerLiteral
@ -128,17 +136,23 @@ categoryForTypeScriptName category = case category of
"arguments" -> Args
"statement_block" -> ExpressionStatements
"assignment" -> Assignment
"assignment_expression" -> Assignment
"member_access" -> MemberAccess
"member_expression" -> MemberAccess
"op" -> Operator
"subscript_access" -> SubscriptAccess
"subscript_expression" -> SubscriptAccess
"regex" -> Regex
"template_string" -> TemplateString
"switch_statement" -> Switch
"math_assignment" -> MathAssignment
"augmented_assignment_expression" -> MathAssignment
"case" -> Case
"switch_case" -> Case
"true" -> Boolean
"false" -> Boolean
"ternary" -> Ternary
"ternary_expression" -> Ternary
"while_statement" -> While
"trailing_while_statement" -> While
"do_statement" -> DoWhile
@ -164,4 +178,5 @@ categoryForTypeScriptName category = case category of
"module" -> Module
"internal_module" -> Namespace
"interface_declaration" -> Interface
"parenthesized_expression" -> ParenthesizedExpression
name -> Other name

4
src/Language/TypeScript/Syntax.hs
View File

@ -2,8 +2,8 @@
module Language.TypeScript.Syntax where
import Language.Haskell.TH
import Text.Parser.TreeSitter.Language
import Text.Parser.TreeSitter.TypeScript
import TreeSitter.Language
import TreeSitter.TypeScript
-- | Statically-known rules corresponding to symbols in the grammar.
mkSymbolDatatype (mkName "Grammar") tree_sitter_typescript

15
src/Parser.hs
View File

@ -30,20 +30,19 @@ import qualified Language.Python.Syntax as Python
import qualified Language.Ruby.Syntax as Ruby
import Syntax hiding (Go)
import Term
import qualified Text.Parser.TreeSitter as TS
import Text.Parser.TreeSitter.Language (Symbol)
import Text.Parser.TreeSitter.Go
import Text.Parser.TreeSitter.Python
import Text.Parser.TreeSitter.Ruby
import Text.Parser.TreeSitter.TypeScript
import Text.Parser.TreeSitter.JSON
import qualified TreeSitter.Language as TS (Language, Symbol)
import TreeSitter.Go
import TreeSitter.Python
import TreeSitter.Ruby
import TreeSitter.TypeScript
import TreeSitter.JSON
-- | A parser from 'Source' onto some term type.
data Parser term where
-- | A parser producing 'AST' using a 'TS.Language'.
ASTParser :: (Bounded grammar, Enum grammar) => Ptr TS.Language -> Parser (AST grammar)
-- | A parser producing an à la carte term given an 'AST'-producing parser and an 'Assignment' onto 'Term's in some syntax type.
AssignmentParser :: (Bounded grammar, Ix grammar, Show grammar, Symbol grammar, Syntax.Error :< fs, Apply1 Foldable fs, Apply1 Functor fs, Eq ast, Recursive ast, Foldable (Base ast))
AssignmentParser :: (Bounded grammar, Ix grammar, Show grammar, TS.Symbol grammar, Syntax.Error :< fs, Apply1 Foldable fs, Apply1 Functor fs, Eq ast, Recursive ast, Foldable (Base ast))
=> Parser ast -- ^ A parser producing AST.
-> (forall x. Base ast x -> Node grammar) -- ^ A function extracting the symbol and location.
-> Assignment ast grammar (Term (Union fs) (Record Location)) -- ^ An assignment from AST onto 'Term's.

140
src/TreeSitter.hs
View File

@ -2,15 +2,16 @@
module TreeSitter
( treeSitterParser
, parseToAST
, defaultTermAssignment
) where
import Category
import Control.Comonad (extract)
import Control.Comonad.Cofree (unwrap)
import Control.Exception
import Control.Monad ((<=<))
import Data.Blob
import Data.ByteString.Unsafe (unsafeUseAsCStringLen)
import Data.Foldable (toList)
import Data.Functor.Foldable hiding (Nil)
import Data.Range
import Data.Record
@ -27,42 +28,43 @@ import Foreign.C.String (peekCString)
import Foreign.Marshal.Array (allocaArray)
import qualified Syntax as S
import Term
import Text.Parser.TreeSitter hiding (Language(..))
import qualified Text.Parser.TreeSitter as TS
import qualified Text.Parser.TreeSitter.Go as TS
import qualified Text.Parser.TreeSitter.Ruby as TS
import qualified Text.Parser.TreeSitter.TypeScript as TS
import qualified TreeSitter.Document as TS
import qualified TreeSitter.Node as TS
import qualified TreeSitter.Language as TS
import qualified TreeSitter.Go as TS
import qualified TreeSitter.Ruby as TS
import qualified TreeSitter.TypeScript as TS
import Info
-- | Returns a TreeSitter parser for the given language and TreeSitter grammar.
treeSitterParser :: Ptr TS.Language -> Blob -> IO (SyntaxTerm DefaultFields)
treeSitterParser language blob = bracket ts_document_new ts_document_free $ \ document -> do
ts_document_set_language document language
treeSitterParser language blob = bracket TS.ts_document_new TS.ts_document_free $ \ document -> do
TS.ts_document_set_language document language
unsafeUseAsCStringLen (sourceBytes (blobSource blob)) $ \ (sourceBytes, len) -> do
ts_document_set_input_string_with_length document sourceBytes len
ts_document_parse_halt_on_error document
TS.ts_document_set_input_string_with_length document sourceBytes len
TS.ts_document_parse_halt_on_error document
term <- documentToTerm language document blob
pure term
-- | Parse 'Source' with the given 'TS.Language' and return its AST.
parseToAST :: (Bounded grammar, Enum grammar) => Ptr TS.Language -> Blob -> IO (A.AST grammar)
parseToAST language Blob{..} = bracket ts_document_new ts_document_free $ \ document -> do
ts_document_set_language document language
parseToAST language Blob{..} = bracket TS.ts_document_new TS.ts_document_free $ \ document -> do
TS.ts_document_set_language document language
root <- unsafeUseAsCStringLen (sourceBytes blobSource) $ \ (source, len) -> do
ts_document_set_input_string_with_length document source len
ts_document_parse_halt_on_error document
TS.ts_document_set_input_string_with_length document source len
TS.ts_document_parse_halt_on_error document
alloca (\ rootPtr -> do
ts_document_root_node_p document rootPtr
TS.ts_document_root_node_p document rootPtr
peek rootPtr)
anaM toAST root
toAST :: forall grammar . (Bounded grammar, Enum grammar) => Node -> IO (Base (A.AST grammar) Node)
toAST node@Node{..} = do
toAST :: forall grammar . (Bounded grammar, Enum grammar) => TS.Node -> IO (Base (A.AST grammar) TS.Node)
toAST node@TS.Node{..} = do
let count = fromIntegral nodeChildCount
children <- allocaArray count $ \ childNodesPtr -> do
_ <- with nodeTSNode (\ nodePtr -> ts_node_copy_child_nodes nullPtr nodePtr childNodesPtr (fromIntegral count))
_ <- with nodeTSNode (\ nodePtr -> TS.ts_node_copy_child_nodes nullPtr nodePtr childNodesPtr (fromIntegral count))
peekArray count childNodesPtr
pure $! A.Node (toEnum (min (fromIntegral nodeSymbol) (fromEnum (maxBound :: grammar)))) (nodeRange node) (nodeSpan node) :< children
@ -71,46 +73,46 @@ anaM g = a where a = pure . embed <=< traverse a <=< g
-- | Return a parser for a tree sitter language & document.
documentToTerm :: Ptr TS.Language -> Ptr Document -> Blob -> IO (SyntaxTerm DefaultFields)
documentToTerm :: Ptr TS.Language -> Ptr TS.Document -> Blob -> IO (SyntaxTerm DefaultFields)
documentToTerm language document Blob{..} = do
root <- alloca (\ rootPtr -> do
ts_document_root_node_p document rootPtr
TS.ts_document_root_node_p document rootPtr
peek rootPtr)
toTerm root
where toTerm :: Node -> IO (SyntaxTerm DefaultFields)
toTerm node = do
name <- peekCString (nodeType node)
where toTerm :: TS.Node -> IO (SyntaxTerm DefaultFields)
toTerm node@TS.Node{..} = do
name <- peekCString nodeType
children <- getChildren (fromIntegral (nodeNamedChildCount node)) copyNamed
let allChildren = getChildren (fromIntegral (nodeChildCount node)) copyAll
children <- getChildren (fromIntegral nodeNamedChildCount) copyNamed
let allChildren = getChildren (fromIntegral nodeChildCount) copyAll
let source = slice (nodeRange node) blobSource
assignTerm language source (range :. categoryForLanguageProductionName language (pack name) :. nodeSpan node :. Nil) children allChildren
where getChildren count copy = do
nodes <- allocaArray count $ \ childNodesPtr -> do
_ <- with (nodeTSNode node) (\ nodePtr -> copy nodePtr childNodesPtr (fromIntegral count))
_ <- with nodeTSNode (\ nodePtr -> copy nodePtr childNodesPtr (fromIntegral count))
peekArray count childNodesPtr
children <- traverse toTerm nodes
return $! filter isNonEmpty children
range = nodeRange node
copyNamed = ts_node_copy_named_child_nodes document
copyAll = ts_node_copy_child_nodes document
copyNamed = TS.ts_node_copy_named_child_nodes document
copyAll = TS.ts_node_copy_child_nodes document
isNonEmpty :: HasField fields Category => SyntaxTerm fields -> Bool
isNonEmpty = (/= Empty) . category . extract
nodeRange :: Node -> Range
nodeRange Node{..} = Range (fromIntegral nodeStartByte) (fromIntegral nodeEndByte)
nodeRange :: TS.Node -> Range
nodeRange TS.Node{..} = Range (fromIntegral nodeStartByte) (fromIntegral nodeEndByte)
nodeSpan :: Node -> Span
nodeSpan Node{..} = nodeStartPoint `seq` nodeEndPoint `seq` Span (pointPos nodeStartPoint) (pointPos nodeEndPoint)
where pointPos TSPoint{..} = pointRow `seq` pointColumn `seq` Pos (1 + fromIntegral pointRow) (1 + fromIntegral pointColumn)
nodeSpan :: TS.Node -> Span
nodeSpan TS.Node{..} = nodeStartPoint `seq` nodeEndPoint `seq` Span (pointPos nodeStartPoint) (pointPos nodeEndPoint)
where pointPos TS.TSPoint{..} = pointRow `seq` pointColumn `seq` Pos (1 + fromIntegral pointRow) (1 + fromIntegral pointColumn)
assignTerm :: Ptr TS.Language -> Source -> Record DefaultFields -> [ SyntaxTerm DefaultFields ] -> IO [ SyntaxTerm DefaultFields ] -> IO (SyntaxTerm DefaultFields)
assignTerm language source annotation children allChildren =
cofree . (annotation :<) <$> case assignTermByLanguage source (category annotation) children of
Just a -> pure a
_ -> defaultTermAssignment source (category annotation) children allChildren
case assignTermByLanguage source (category annotation) children of
Just a -> pure (cofree (annotation :< a))
_ -> defaultTermAssignment source annotation children allChildren
where assignTermByLanguage :: Source -> Category -> [ SyntaxTerm DefaultFields ] -> Maybe (S.Syntax (SyntaxTerm DefaultFields))
assignTermByLanguage = case languageForTSLanguage language of
Just Language.Go -> Go.termAssignment
@ -118,33 +120,58 @@ assignTerm language source annotation children allChildren =
Just TypeScript -> TS.termAssignment
_ -> \ _ _ _ -> Nothing
defaultTermAssignment :: Source -> Category -> [ SyntaxTerm DefaultFields ] -> IO [ SyntaxTerm DefaultFields ] -> IO (S.Syntax (SyntaxTerm DefaultFields))
defaultTermAssignment source category children allChildren
| category `elem` operatorCategories = S.Operator <$> allChildren
| otherwise = pure $! case (category, children) of
(ParseError, children) -> S.ParseError children
defaultTermAssignment :: Source -> Record DefaultFields -> [ SyntaxTerm DefaultFields ] -> IO [ SyntaxTerm DefaultFields ] -> IO (SyntaxTerm DefaultFields)
defaultTermAssignment source annotation children allChildren
| category annotation `elem` operatorCategories = cofree . (annotation :<) . S.Operator <$> allChildren
| otherwise = case (category annotation, children) of
(ParseError, children) -> toTerm $ S.ParseError children
(Comment, _) -> S.Comment (toText source)
(Comment, _) -> toTerm $ S.Comment (toText source)
(Pair, [key, value]) -> S.Pair key value
(Pair, [key, value]) -> toTerm $ S.Pair key value
-- Control flow statements
(If, condition : body) -> S.If condition body
(Switch, _) -> uncurry S.Switch (break ((== Case) . Info.category . extract) children)
(Case, expr : body) -> S.Case expr body
(While, expr : rest) -> S.While expr rest
(If, condition : body) -> toTerm $ S.If condition body
(Switch, _) -> let (subject, body) = break ((== Other "switch_body") . Info.category . extract) children in toTerm $ S.Switch subject (body >>= toList . unwrap)
(Case, expr : body) -> toTerm $ S.Case expr body
(While, expr : rest) -> toTerm $ S.While expr rest
-- Statements
(Return, _) -> S.Return children
(Yield, _) -> S.Yield children
(Throw, [expr]) -> S.Throw expr
(Break, [label]) -> S.Break (Just label)
(Break, []) -> S.Break Nothing
(Continue, [label]) -> S.Continue (Just label)
(Continue, []) -> S.Continue Nothing
(Return, _) -> toTerm $ S.Return children
(Yield, _) -> toTerm $ S.Yield children
(Throw, [expr]) -> toTerm $ S.Throw expr
(Break, [label]) -> toTerm $ S.Break (Just label)
(Break, []) -> toTerm $ S.Break Nothing
(Continue, [label]) -> toTerm $ S.Continue (Just label)
(Continue, []) -> toTerm $ S.Continue Nothing
(_, []) -> S.Leaf (toText source)
(_, children) -> S.Indexed children
(ParenthesizedExpression, [child]) -> pure child
(Other "unary_expression", _) -> do
cs <- allChildren
let c = case category . extract <$> cs of
[Other s, _]
| s `elem` ["-", "+", "++", "--"] -> MathOperator
| s == "~" -> BitwiseOperator
| s == "!" -> BooleanOperator
[_, Other t]
| t `elem` ["--", "++"] -> MathOperator
_ -> Operator
pure (cofree ((setCategory annotation c) :< S.Operator cs))
(Other "binary_expression", _) -> do
cs <- allChildren
let c = case category . extract <$> cs of
[_, Other s, _]
| s `elem` ["<=", "<", ">=", ">", "==", "===", "!=", "!=="] -> RelationalOperator
| s `elem` ["*", "+", "-", "/", "%"] -> MathOperator
| s `elem` ["&&", "||"] -> BooleanOperator
| s `elem` [">>", ">>=", ">>>", ">>>=", "<<", "<<=", "&", "^", "|"] -> BitwiseOperator
_ -> Operator
pure (cofree ((setCategory annotation c) :< S.Operator cs))
(_, []) -> toTerm $ S.Leaf (toText source)
(_, children) -> toTerm $ S.Indexed children
where operatorCategories =
[ Operator
, Binary
@ -156,6 +183,7 @@ defaultTermAssignment source category children allChildren
, RelationalOperator
, BitwiseOperator
]
toTerm = pure . cofree . (annotation :<)
categoryForLanguageProductionName :: Ptr TS.Language -> Text -> Category

2
test/Data/Syntax/Assignment/Spec.hs
View File

@ -14,7 +14,7 @@ import Data.Syntax.Assignment
import GHC.Stack (getCallStack)
import Prelude hiding (words)
import Test.Hspec
import Text.Parser.TreeSitter.Language (Symbol(..), SymbolType(..))
import TreeSitter.Language (Symbol(..), SymbolType(..))
spec :: Spec
spec = do

3
test/fixtures/go/constructors.diff+A.txt vendored
View File

@ -12,10 +12,9 @@
(Identifier)
(ChannelTy
(Identifier))
(ParenthesizedExpression
(RelationalOperator
(Identifier)
(Identifier))))
(Identifier)))
(FunctionCall
(Identifier)
(ChannelTy

3
test/fixtures/go/constructors.diff+B.txt vendored
View File

@ -12,10 +12,9 @@
(Identifier)
(ChannelTy
(Identifier))
(ParenthesizedExpression
(RelationalOperator
(Identifier)
(Identifier))))
(Identifier)))
(FunctionCall
(Identifier)
(ChannelTy

3
test/fixtures/go/constructors.diff-A.txt vendored
View File

@ -12,10 +12,9 @@
(Identifier)
(ChannelTy
(Identifier))
(ParenthesizedExpression
(RelationalOperator
(Identifier)
(Identifier))))
(Identifier)))
(FunctionCall
(Identifier)
(ChannelTy

3
test/fixtures/go/constructors.diff-B.txt vendored
View File

@ -12,10 +12,9 @@
(Identifier)
(ChannelTy
(Identifier))
(ParenthesizedExpression
(RelationalOperator
(Identifier)
(Identifier))))
(Identifier)))
(FunctionCall
(Identifier)
(ChannelTy

3
test/fixtures/go/constructors.diffA-B.txt vendored
View File

@ -14,10 +14,9 @@
(ChannelTy
{ (Identifier)
->(Identifier) })
(ParenthesizedExpression
(RelationalOperator
(Identifier)
(Identifier))))
(Identifier)))
(FunctionCall
(Identifier)
(ChannelTy

3
test/fixtures/go/constructors.diffB-A.txt vendored
View File

@ -14,10 +14,9 @@
(ChannelTy
{ (Identifier)
->(Identifier) })
(ParenthesizedExpression
(RelationalOperator
(Identifier)
(Identifier))))
(Identifier)))
(FunctionCall
(Identifier)
(ChannelTy

3
test/fixtures/go/constructors.parseA.txt vendored
View File

@ -12,10 +12,9 @@
(Identifier)
(ChannelTy
(Identifier))
(ParenthesizedExpression
(RelationalOperator
(Identifier)
(Identifier))))
(Identifier)))
(FunctionCall
(Identifier)
(ChannelTy

3
test/fixtures/go/constructors.parseB.txt vendored
View File

@ -12,10 +12,9 @@
(Identifier)
(ChannelTy
(Identifier))
(ParenthesizedExpression
(RelationalOperator
(Identifier)
(Identifier))))
(Identifier)))
(FunctionCall
(Identifier)
(ChannelTy

3
test/fixtures/go/type-conversion-expressions.diff+A.txt vendored
View File

@ -23,8 +23,7 @@
(Identifier))
(Identifier))
(FunctionCall
(ParenthesizedExpression
(SubscriptAccess
(Identifier)
(Identifier)))
(Identifier))
(Identifier))))+}

3
test/fixtures/go/type-conversion-expressions.diff+B.txt vendored
View File

@ -23,8 +23,7 @@
(Identifier))
(Identifier))
(FunctionCall
(ParenthesizedExpression
(SubscriptAccess
(Identifier)
(Identifier)))
(Identifier))
(Identifier))))+}

3
test/fixtures/go/type-conversion-expressions.diff-A.txt vendored
View File

@ -23,8 +23,7 @@
(Identifier))
(Identifier))
(FunctionCall
(ParenthesizedExpression
(SubscriptAccess
(Identifier)
(Identifier)))
(Identifier))
(Identifier))))-}

3
test/fixtures/go/type-conversion-expressions.diff-B.txt vendored
View File

@ -23,8 +23,7 @@
(Identifier))
(Identifier))
(FunctionCall
(ParenthesizedExpression
(SubscriptAccess
(Identifier)
(Identifier)))
(Identifier))
(Identifier))))-}

3
test/fixtures/go/type-conversion-expressions.diffA-B.txt vendored
View File

@ -32,11 +32,10 @@
{ (Identifier)
->(Identifier) })
(FunctionCall
(ParenthesizedExpression
(SubscriptAccess
{ (Identifier)
->(Identifier) }
{ (Identifier)
->(Identifier) }))
->(Identifier) })
{ (Identifier)
->(Identifier) })))

3
test/fixtures/go/type-conversion-expressions.diffB-A.txt vendored
View File

@ -32,11 +32,10 @@
{ (Identifier)
->(Identifier) })
(FunctionCall
(ParenthesizedExpression
(SubscriptAccess
{ (Identifier)
->(Identifier) }
{ (Identifier)
->(Identifier) }))
->(Identifier) })
{ (Identifier)
->(Identifier) })))

3
test/fixtures/go/type-conversion-expressions.parseA.txt vendored
View File

@ -23,8 +23,7 @@
(Identifier))
(Identifier))
(FunctionCall
(ParenthesizedExpression
(SubscriptAccess
(Identifier)
(Identifier)))
(Identifier))
(Identifier))))

3
test/fixtures/go/type-conversion-expressions.parseB.txt vendored
View File

@ -23,8 +23,7 @@
(Identifier))
(Identifier))
(FunctionCall
(ParenthesizedExpression
(SubscriptAccess
(Identifier)
(Identifier)))
(Identifier))
(Identifier))))

6
test/fixtures/ruby/for.diff+A.txt vendored
View File

@ -1,10 +1,12 @@
{+(Program
(For
(Identifier)
(Identifier)
(Other "in"
(Identifier))
(Identifier))
(For
(Identifier)
(Identifier)
(Identifier)
(Other "in"
(Identifier))
(Identifier)))+}

3
test/fixtures/ruby/for.diff+B.txt vendored
View File

@ -1,10 +1,11 @@
{+(Program
(For
(Identifier)
(Other "in"
(ArrayLiteral
(IntegerLiteral)
(IntegerLiteral)
(IntegerLiteral))
(IntegerLiteral)))
(MethodCall
(Identifier)
(Identifier))))+}

6
test/fixtures/ruby/for.diff-A.txt vendored
View File

@ -1,10 +1,12 @@
{-(Program
(For
(Identifier)
(Identifier)
(Other "in"
(Identifier))
(Identifier))
(For
(Identifier)
(Identifier)
(Identifier)
(Other "in"
(Identifier))
(Identifier)))-}

3
test/fixtures/ruby/for.diff-B.txt vendored
View File

@ -1,10 +1,11 @@
{-(Program
(For
(Identifier)
(Other "in"
(ArrayLiteral
(IntegerLiteral)
(IntegerLiteral)
(IntegerLiteral))
(IntegerLiteral)))
(MethodCall
(Identifier)
(Identifier))))-}

10
test/fixtures/ruby/for.diffA-B.txt vendored
View File

@ -2,11 +2,12 @@
(For
{ (Identifier)
->(Identifier) }
{ (Identifier)
->(ArrayLiteral
(Other "in"
{+(ArrayLiteral
(IntegerLiteral)
(IntegerLiteral)
(IntegerLiteral)) }
(IntegerLiteral))+}
{-(Identifier)-})
{ (Identifier)
->(MethodCall
(Identifier)
@ -14,5 +15,6 @@
{-(For
(Identifier)
(Identifier)
(Identifier)
(Other "in"
(Identifier))
(Identifier))-})

10
test/fixtures/ruby/for.diffB-A.txt vendored
View File

@ -2,11 +2,12 @@
(For
{ (Identifier)
->(Identifier) }
{ (ArrayLiteral
(Other "in"
{+(Identifier)+}
{-(ArrayLiteral
(IntegerLiteral)
(IntegerLiteral)
(IntegerLiteral))
->(Identifier) }
(IntegerLiteral))-})
{ (MethodCall
(Identifier)
(Identifier))
@ -14,5 +15,6 @@
{+(For
(Identifier)
(Identifier)
(Identifier)
(Other "in"
(Identifier))
(Identifier))+})

6
test/fixtures/ruby/for.parseA.txt vendored
View File

@ -1,10 +1,12 @@
(Program
(For
(Identifier)
(Identifier)
(Other "in"
(Identifier))
(Identifier))
(For
(Identifier)
(Identifier)
(Identifier)
(Other "in"
(Identifier))
(Identifier)))

3
test/fixtures/ruby/for.parseB.txt vendored
View File

@ -1,10 +1,11 @@
(Program
(For
(Identifier)
(Other "in"
(ArrayLiteral
(IntegerLiteral)
(IntegerLiteral)
(IntegerLiteral))
(IntegerLiteral)))
(MethodCall
(Identifier)
(Identifier))))

2
vendor/haskell-tree-sitter vendored

@ -1 +1 @@
Subproject commit 5a67480dcfdac091d0febc221d121bedc47b6a24
Subproject commit 45bbc5aff11ac5d4364b024d5b8693c297ee9ebb