github/semantic
1
1
Fork 0
mirror of https://github.com/github/semantic.git synced 2024-12-28 09:21:35 +03:00
Code Issues Projects Releases Wiki Activity

Merge pull request #414 from github/more-precise-parsing

Browse Source 
More precise parsing
This commit is contained in:
Timothy Clem 2020-01-06 13:38:35 -08:00 committed by GitHub
commit 9cd8717cc2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
68 changed files with 1192 additions and 408 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.stylish-haskell.yaml
View File

@ -173,7 +173,7 @@ steps:
# `{-#LANGUAGE #-}'.
#
# Default: vertical.
style: compact
style: vertical
# Align affects alignment of closing pragma brackets.
#

9
cabal.project
View File

@ -2,10 +2,13 @@ packages: .
semantic-analysis
semantic-ast
semantic-core
semantic-go
semantic-java
semantic-json
semantic-python
semantic-ruby
semantic-tsx
semantic-typescript
semantic-tags
jobs: $ncpus
@ -19,6 +22,9 @@ package semantic-analysis
package semantic-core
ghc-options: -Werror
package semantic-go
ghc-options: -Werror
package semantic-java
ghc-options: -Werror
@ -28,6 +34,9 @@ package semantic-json
package semantic-python
ghc-options: -Werror
package semantic-ruby
ghc-options: -Werror
package semantic-tags
ghc-options: -Werror

29
script/clone-example-repos
View File

@ -1,22 +1,15 @@
#!/bin/bash
#/ Usage: script/clone-example-repos
#/
#/ Clone some example repositories for smoke testing parsing and assignment
#/ Clone some example repositories for smoke testing parsing, assignment, and precise ASTs.
#/
#/ NOTES:
#/ - This script is intended to be called by `test/Examples.hs`
#/ - Go and Ruby examples are in submodules
#/ - PHP doesn't have any parse-examples
#/ - Java and Haskell have good examples, but they have assignment failures so currently aren't tested
set -e
cd $(dirname "$0")/..
# mkdir -p test/examplerepos || true
# git clone --single-branch --recurse-submodules https://github.com/tree-sitter/haskell-tree-sitter.git tmp/haskell-tree-sitter || true
mkdir -p tmp
# dir="tmp/haskell-tree-sitter"
# clone_repo LOCAL_PATH URL SHA
function clone_repo {
@ -36,6 +29,10 @@ function clone_repo {
popd > /dev/null
}
go_examples="tmp/go-examples"
clone_repo "$go_examples/go" golang/go 870e12d7bfaea70fb0d743842f5864eb059cb939
clone_repo "$go_examples/moby" moby/moby f57f260b49b6142366e6bc1274204ee0a1205945
python_examples="tmp/python-examples"
clone_repo "$python_examples/numpy" numpy/numpy 058851c5cfc98f50f11237b1c13d77cfd1f40475
clone_repo "$python_examples/thealgorithms" thealgorithms/python c6be53e1c43f870f5364eef1499ee1b411c966fb
@ -48,6 +45,9 @@ clone_repo "$python_examples/scrapy" scrapy/scrapy 65d631329a1434ec013f24341e4b8
clone_repo "$python_examples/pytorch" pytorch/pytorch c865d46736db4afff51690a712e35ed8e3899490
clone_repo "$python_examples/certbot" certbot/certbot bb8222200a8cbd39a3ce9584ce6dfed6c5d05228
ruby_examples="tmp/ruby-examples"
clone_repo "$ruby_examples/ruby_spec" ruby/spec c3e6b9017926f44a76e2b966c4dd35fa84c4cd3b
ts_examples="tmp/typescript-examples"
clone_repo "$ts_examples/desktop" desktop/desktop d1324f56d02dd9afca5d2e9da545905a7d41d671
clone_repo "$ts_examples/npm" npm/npm ee147fbbca6f2707d3b16f4fa78f4c4606b2d9b1
@ -58,13 +58,6 @@ clone_repo "$ts_examples/npm" npm/npm ee147fbbca6f2707d3b16f4fa78f4c4606b2d9b1
# clone_repo "$java_examples/RxJava" ReactiveX/RxJava 8a6bf14fc9a61f7c1c0016ca217be02ca86211d2
# haskell_examples="$dir/tree-sitter-haskell/vendor/tree-sitter-haskell/examples"
# # clone_repo "$haskell_examples/effects" joshvera/effects 08f5f36f2600362685af593f4b327e933b60bf97
# # clone_repo "$haskell_examples/postgrest" PostgRest/postgrest f80cfbf165f951a062b3cbedac4556019905ca49
# # clone_repo "$haskell_examples/ivory" GaloisInc/ivory 3d00324ad1c113c7e70957ff6a6d636d271d0fc4
# go_examples="$dir/tree-sitter-go/vendor/tree-sitter-go/examples"
# clone_repo "$go_examples/go" "golang/go" "870e12d7bfaea70fb0d743842f5864eb059cb939"
# clone_repo "$go_examples/moby" "moby/moby" "f57f260b49b6142366e6bc1274204ee0a1205945"
ruby_examples="tmp/ruby-examples"
clone_repo "$ruby_examples/ruby_spec" "ruby/spec" "c3e6b9017926f44a76e2b966c4dd35fa84c4cd3b"
# clone_repo "$haskell_examples/effects" joshvera/effects 08f5f36f2600362685af593f4b327e933b60bf97
# clone_repo "$haskell_examples/postgrest" PostgRest/postgrest f80cfbf165f951a062b3cbedac4556019905ca49
# clone_repo "$haskell_examples/ivory" GaloisInc/ivory 3d00324ad1c113c7e70957ff6a6d636d271d0fc4

3
script/ghci-flags
View File

@ -40,10 +40,13 @@ function flags {
echo "-isemantic-analysis/src"
echo "-isemantic-ast/src"
echo "-isemantic-core/src"
echo "-isemantic-go/src"
echo "-isemantic-java/src"
echo "-isemantic-json/src"
echo "-isemantic-python/src"
echo "-isemantic-ruby/src"
echo "-isemantic-tsx/src"
echo "-isemantic-typescript/src"
echo "-isemantic-tags/src"
echo "-iapp"
echo "-isrc"

5
script/ghci-flags-dependencies
View File

@ -11,8 +11,11 @@ echo "semantic.cabal"
echo "semantic-analysis/semantic-analysis.cabal"
echo "semantic-ast/semantic-ast.cabal"
echo "semantic-core/semantic-core.cabal"
echo "semantic-tags/semantic-tags.cabal"
echo "semantic-go/semantic-go.cabal"
echo "semantic-java/semantic-java.cabal"
echo "semantic-json/semantic-json.cabal"
echo "semantic-python/semantic-python.cabal"
echo "semantic-ruby/semantic-ruby.cabal"
echo "semantic-tags/semantic-tags.cabal"
echo "semantic-tsx/semantic-tsx.cabal"
echo "semantic-typescript/semantic-typescript.cabal"

21
semantic-go/LICENSE Normal file
View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2019 GitHub
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

3
semantic-go/README.md Normal file
View File

@ -0,0 +1,3 @@
# Semantic support for Go
This package implements `semantic` support for Go using the `semantic-core` intermediate language.

2
semantic-go/Setup.hs Normal file
View File

@ -0,0 +1,2 @@
import Distribution.Simple
main = defaultMain

54
semantic-go/semantic-go.cabal Normal file
View File

@ -0,0 +1,54 @@
cabal-version: 2.4
name: semantic-go
version: 0.0.0.0
synopsis: Semantic support for Go.
description: Semantic support for Go using the semantic-core intermediate language.
homepage: https://github.com/github/semantic/tree/master/semantic-go#readme
bug-reports: https://github.com/github/semantic/issues
license: MIT
license-file: LICENSE
author: The Semantic authors
maintainer: opensource+semantic@github.com
copyright: (c) 2019 GitHub, Inc.
category: Language
build-type: Simple
stability: alpha
extra-source-files: README.md
tested-with: GHC == 8.6.5
common haskell
default-language: Haskell2010
build-depends: base ^>= 4.13
, fused-effects ^>= 1.0
, fused-syntax
, parsers ^>= 0.12.10
, semantic-core ^>= 0.0
, semantic-source ^>= 0.0
, semantic-tags ^>= 0.0
, text ^>= 1.2.3
, tree-sitter ^>= 0.7.2
, tree-sitter-go ^>= 0.4
ghc-options:
-Weverything
-Wno-missing-local-signatures
-Wno-missing-import-lists
-Wno-implicit-prelude
-Wno-safe
-Wno-unsafe
-Wno-name-shadowing
-Wno-monomorphism-restriction
-Wno-missed-specialisations
-Wno-all-missed-specialisations
-Wno-star-is-type
if (impl(ghc >= 8.8))
ghc-options: -Wno-missing-deriving-strategies
library
import: haskell
exposed-modules:
Language.Go
Language.Go.Tags
hs-source-dirs: src

20
semantic-go/src/Language/Go.hs Normal file
View File

@ -0,0 +1,20 @@
-- | Semantic functionality for Go programs.
module Language.Go
( Term(..)
, TreeSitter.Go.tree_sitter_go
) where
import qualified Language.Go.Tags as GoTags
import qualified Tags.Tagging.Precise as Tags
import qualified TreeSitter.Go (tree_sitter_go)
import qualified TreeSitter.Go.AST as Go
import qualified TreeSitter.Unmarshal as TS
newtype Term a = Term { getTerm :: Go.SourceFile a }
instance TS.Unmarshal Term where
unmarshalNode node = Term <$> TS.unmarshalNode node
instance Tags.ToTags Term where
tags src = Tags.runTagging src . GoTags.tags . getTerm

108
semantic-go/src/Language/Go/Tags.hs Normal file
View File

@ -0,0 +1,108 @@
{-# LANGUAGE AllowAmbiguousTypes #-}
{-# LANGUAGE DataKinds #-}
{-# LANGUAGE DisambiguateRecordFields #-}
{-# LANGUAGE FlexibleContexts #-}
{-# LANGUAGE FlexibleInstances #-}
{-# LANGUAGE MultiParamTypeClasses #-}
{-# LANGUAGE NamedFieldPuns #-}
{-# LANGUAGE PartialTypeSignatures #-}
{-# LANGUAGE ScopedTypeVariables #-}
{-# LANGUAGE TypeApplications #-}
{-# LANGUAGE TypeFamilies #-}
{-# LANGUAGE TypeOperators #-}
{-# LANGUAGE UndecidableInstances #-}
module Language.Go.Tags
( ToTags(..)
) where
import AST.Element
import Control.Effect.Reader
import Control.Effect.Writer
import Data.Monoid (Ap (..))
import Data.Text as Text
import GHC.Generics
import Source.Loc
import Source.Source as Source
import Tags.Tag
import qualified Tags.Tagging.Precise as Tags
import qualified TreeSitter.Go.AST as Go
class ToTags t where
tags
:: ( Has (Reader Source) sig m
, Has (Writer Tags.Tags) sig m
)
=> t Loc
-> m ()
instance (ToTagsBy strategy t, strategy ~ ToTagsInstance t) => ToTags t where
tags = tags' @strategy
class ToTagsBy (strategy :: Strategy) t where
tags'
:: ( Has (Reader Source) sig m
, Has (Writer Tags.Tags) sig m
)
=> t Loc
-> m ()
data Strategy = Generic | Custom
type family ToTagsInstance t :: Strategy where
ToTagsInstance (_ :+: _) = 'Custom
ToTagsInstance Go.FunctionDeclaration = 'Custom
ToTagsInstance Go.MethodDeclaration = 'Custom
ToTagsInstance Go.CallExpression = 'Custom
ToTagsInstance _ = 'Generic
instance ToTagsBy 'Custom Go.FunctionDeclaration where
tags' t@Go.FunctionDeclaration
{ ann = loc@Loc { byteRange }
, name = Go.Identifier { text }
} = yieldTag text Function loc byteRange >> gtags t
instance ToTagsBy 'Custom Go.MethodDeclaration where
tags' t@Go.MethodDeclaration
{ ann = loc@Loc { byteRange }
, name = Go.FieldIdentifier { text }
} = yieldTag text Function loc byteRange >> gtags t
instance ToTagsBy 'Custom Go.CallExpression where
tags' t@Go.CallExpression
{ ann = loc@Loc { byteRange }
, function = Go.Expression expr
} = match expr
where
match expr = case expr of
Prj Go.SelectorExpression { field = Go.FieldIdentifier { text }} -> yield text
Prj Go.Identifier { text } -> yield text
Prj Go.CallExpression { function = Go.Expression e } -> match e
Prj Go.ParenthesizedExpression { extraChildren = Go.Expression e } -> match e
_ -> gtags t
yield name = yieldTag name Call loc byteRange >> gtags t
instance (ToTags l, ToTags r) => ToTagsBy 'Custom (l :+: r) where
tags' (L1 l) = tags l
tags' (R1 r) = tags r
gtags
:: ( Has (Reader Source) sig m
, Has (Writer Tags.Tags) sig m
, Generic1 t
, Tags.GFoldable1 ToTags (Rep1 t)
)
=> t Loc
-> m ()
gtags = getAp . Tags.gfoldMap1 @ToTags (Ap . tags) . from1
instance (Generic1 t, Tags.GFoldable1 ToTags (Rep1 t)) => ToTagsBy 'Generic t where
tags' = gtags
yieldTag :: (Has (Reader Source) sig m, Has (Writer Tags.Tags) sig m) => Text -> Kind -> Loc -> Range -> m ()
yieldTag name kind loc range = do
src <- ask @Source
let sliced = slice src range
Tags.yield (Tag name kind loc (Tags.firstLine sliced) Nothing)

29
semantic-python/src/Language/Python/Tags.hs
View File

@ -1,4 +1,17 @@
{-# LANGUAGE AllowAmbiguousTypes, DataKinds, DisambiguateRecordFields, FlexibleContexts, FlexibleInstances, MultiParamTypeClasses, NamedFieldPuns, OverloadedStrings, ScopedTypeVariables, TypeApplications, TypeFamilies, TypeOperators, UndecidableInstances #-}
{-# LANGUAGE AllowAmbiguousTypes #-}
{-# LANGUAGE DataKinds #-}
{-# LANGUAGE DisambiguateRecordFields #-}
{-# LANGUAGE FlexibleContexts #-}
{-# LANGUAGE FlexibleInstances #-}
{-# LANGUAGE MultiParamTypeClasses #-}
{-# LANGUAGE NamedFieldPuns #-}
{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE ScopedTypeVariables #-}
{-# LANGUAGE TypeApplications #-}
{-# LANGUAGE TypeFamilies #-}
{-# LANGUAGE TypeOperators #-}
{-# LANGUAGE UndecidableInstances #-}
module Language.Python.Tags
( ToTags(..)
) where
@ -77,25 +90,25 @@ keywordFunctionCall t loc range name = do
gtags t
instance ToTagsBy 'Custom Py.AssertStatement where
tags' t@Py.AssertStatement { ann = loc@Loc { byteRange = range } } = keywordFunctionCall t loc range "assert"
tags' t@Py.AssertStatement { ann = loc@Loc { byteRange } } = keywordFunctionCall t loc byteRange "assert"
instance ToTagsBy 'Custom Py.Await where
tags' t@Py.Await { ann = loc@Loc { byteRange = range } } = keywordFunctionCall t loc range "await"
tags' t@Py.Await { ann = loc@Loc { byteRange } } = keywordFunctionCall t loc byteRange "await"
instance ToTagsBy 'Custom Py.DeleteStatement where
tags' t@Py.DeleteStatement { ann = loc@Loc { byteRange = range } } = keywordFunctionCall t loc range "del"
tags' t@Py.DeleteStatement { ann = loc@Loc { byteRange } } = keywordFunctionCall t loc byteRange "del"
instance ToTagsBy 'Custom Py.ExecStatement where
tags' t@Py.ExecStatement { ann = loc@Loc { byteRange = range } } = keywordFunctionCall t loc range "exec"
tags' t@Py.ExecStatement { ann = loc@Loc { byteRange } } = keywordFunctionCall t loc byteRange "exec"
instance ToTagsBy 'Custom Py.GlobalStatement where
tags' t@Py.GlobalStatement { ann = loc@Loc { byteRange = range } } = keywordFunctionCall t loc range "global"
tags' t@Py.GlobalStatement { ann = loc@Loc { byteRange } } = keywordFunctionCall t loc byteRange "global"
instance ToTagsBy 'Custom Py.NonlocalStatement where
tags' t@Py.NonlocalStatement { ann = loc@Loc { byteRange = range } } = keywordFunctionCall t loc range "nonlocal"
tags' t@Py.NonlocalStatement { ann = loc@Loc { byteRange } } = keywordFunctionCall t loc byteRange "nonlocal"
instance ToTagsBy 'Custom Py.PrintStatement where
tags' t@Py.PrintStatement { ann = loc@Loc { byteRange = range } } = keywordFunctionCall t loc range "print"
tags' t@Py.PrintStatement { ann = loc@Loc { byteRange } } = keywordFunctionCall t loc byteRange "print"
instance ToTagsBy 'Custom Py.FunctionDefinition where
tags' t@Py.FunctionDefinition

16
semantic-ruby/src/Language/Ruby/Tags.hs
View File

@ -1,4 +1,18 @@
{-# LANGUAGE AllowAmbiguousTypes, DataKinds, DisambiguateRecordFields, FlexibleContexts, FlexibleInstances, MultiParamTypeClasses, NamedFieldPuns, OverloadedStrings, PartialTypeSignatures, ScopedTypeVariables, TypeApplications, TypeFamilies, TypeOperators, UndecidableInstances #-}
{-# LANGUAGE AllowAmbiguousTypes #-}
{-# LANGUAGE DataKinds #-}
{-# LANGUAGE DisambiguateRecordFields #-}
{-# LANGUAGE FlexibleContexts #-}
{-# LANGUAGE FlexibleInstances #-}
{-# LANGUAGE MultiParamTypeClasses #-}
{-# LANGUAGE NamedFieldPuns #-}
{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE PartialTypeSignatures #-}
{-# LANGUAGE ScopedTypeVariables #-}
{-# LANGUAGE TypeApplications #-}
{-# LANGUAGE TypeFamilies #-}
{-# LANGUAGE TypeOperators #-}
{-# LANGUAGE UndecidableInstances #-}
module Language.Ruby.Tags
( ToTags(..)
) where

21
semantic-tsx/LICENSE Normal file
View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2019 GitHub
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

3
semantic-tsx/README.md Normal file
View File

@ -0,0 +1,3 @@
# Semantic support for TSX
This package implements `semantic` support for TSX using the `semantic-core` intermediate language.

2
semantic-tsx/Setup.hs Normal file
View File

@ -0,0 +1,2 @@
import Distribution.Simple
main = defaultMain

54
semantic-tsx/semantic-tsx.cabal Normal file
View File

@ -0,0 +1,54 @@
cabal-version: 2.4
name: semantic-tsx
version: 0.0.0.0
synopsis: Semantic support for TSX.
description: Semantic support for TSX using the semantic-core intermediate language.
homepage: https://github.com/github/semantic/tree/master/semantic-tsx#readme
bug-reports: https://github.com/github/semantic/issues
license: MIT
license-file: LICENSE
author: The Semantic authors
maintainer: opensource+semantic@github.com
copyright: (c) 2019 GitHub, Inc.
category: Language
build-type: Simple
stability: alpha
extra-source-files: README.md
tested-with: GHC == 8.6.5
common haskell
default-language: Haskell2010
build-depends: base ^>= 4.13
, fused-effects ^>= 1.0
, fused-syntax
, parsers ^>= 0.12.10
, semantic-core ^>= 0.0
, semantic-source ^>= 0.0
, semantic-tags ^>= 0.0
, text ^>= 1.2.3
, tree-sitter ^>= 0.7.2
, tree-sitter-tsx ^>= 0.4
ghc-options:
-Weverything
-Wno-missing-local-signatures
-Wno-missing-import-lists
-Wno-implicit-prelude
-Wno-safe
-Wno-unsafe
-Wno-name-shadowing
-Wno-monomorphism-restriction
-Wno-missed-specialisations
-Wno-all-missed-specialisations
-Wno-star-is-type
if (impl(ghc >= 8.8))
ghc-options: -Wno-missing-deriving-strategies
library
import: haskell
exposed-modules:
Language.TSX
Language.TSX.Tags
hs-source-dirs: src

21
semantic-tsx/src/Language/TSX.hs Normal file
View File

@ -0,0 +1,21 @@
{-# OPTIONS_GHC -freduction-depth=0 #-}
-- | Semantic functionality for TSX programs.
module Language.TSX
( Term(..)
, TreeSitter.TSX.tree_sitter_tsx
) where
import qualified Language.TSX.Tags as TsxTags
import qualified Tags.Tagging.Precise as Tags
import qualified TreeSitter.TSX (tree_sitter_tsx)
import qualified TreeSitter.TSX.AST as TSX
import qualified TreeSitter.Unmarshal as TS
newtype Term a = Term { getTerm :: TSX.Program a }
instance TS.Unmarshal Term where
unmarshalNode node = Term <$> TS.unmarshalNode node
instance Tags.ToTags Term where
tags src = Tags.runTagging src . TsxTags.tags . getTerm

149
semantic-tsx/src/Language/TSX/Tags.hs Normal file
View File

@ -0,0 +1,149 @@
{-# LANGUAGE AllowAmbiguousTypes #-}
{-# LANGUAGE DataKinds #-}
{-# LANGUAGE DisambiguateRecordFields #-}
{-# LANGUAGE FlexibleContexts #-}
{-# LANGUAGE FlexibleInstances #-}
{-# LANGUAGE MultiParamTypeClasses #-}
{-# LANGUAGE NamedFieldPuns #-}
{-# LANGUAGE PartialTypeSignatures #-}
{-# LANGUAGE ScopedTypeVariables #-}
{-# LANGUAGE TypeApplications #-}
{-# LANGUAGE TypeFamilies #-}
{-# LANGUAGE TypeOperators #-}
{-# LANGUAGE UndecidableInstances #-}
{-# OPTIONS_GHC -freduction-depth=0 #-}
module Language.TSX.Tags
( ToTags(..)
) where
import AST.Element
import Control.Effect.Reader
import Control.Effect.Writer
import Data.Foldable
import Data.Monoid (Ap (..))
import Data.Text as Text
import GHC.Generics
import Source.Loc
import Source.Source as Source
import Tags.Tag
import qualified Tags.Tagging.Precise as Tags
import qualified TreeSitter.TSX.AST as Tsx
class ToTags t where
tags
:: ( Has (Reader Source) sig m
, Has (Writer Tags.Tags) sig m
)
=> t Loc
-> m ()
instance (ToTagsBy strategy t, strategy ~ ToTagsInstance t) => ToTags t where
tags = tags' @strategy
class ToTagsBy (strategy :: Strategy) t where
tags'
:: ( Has (Reader Source) sig m
, Has (Writer Tags.Tags) sig m
)
=> t Loc
-> m ()
data Strategy = Generic | Custom
type family ToTagsInstance t :: Strategy where
ToTagsInstance (_ :+: _) = 'Custom
ToTagsInstance Tsx.CallExpression = 'Custom
ToTagsInstance Tsx.Class = 'Custom
ToTagsInstance Tsx.ClassDeclaration = 'Custom
ToTagsInstance Tsx.Function = 'Custom
ToTagsInstance Tsx.FunctionDeclaration = 'Custom
ToTagsInstance Tsx.FunctionSignature = 'Custom
ToTagsInstance Tsx.MethodDefinition = 'Custom
ToTagsInstance _ = 'Generic
instance ToTagsBy 'Custom Tsx.Function where
tags' t@Tsx.Function
{ ann = loc@Loc { byteRange }
, name = Just Tsx.Identifier { text }
} = yieldTag text Function loc byteRange >> gtags t
tags' t = gtags t
instance ToTagsBy 'Custom Tsx.FunctionSignature where
tags' t@Tsx.FunctionSignature
{ ann = loc@Loc { byteRange }
, name = Tsx.Identifier { text }
} = yieldTag text Function loc byteRange >> gtags t
instance ToTagsBy 'Custom Tsx.FunctionDeclaration where
tags' t@Tsx.FunctionDeclaration
{ ann = loc@Loc { byteRange }
, name = Tsx.Identifier { text }
} = yieldTag text Function loc byteRange >> gtags t
instance ToTagsBy 'Custom Tsx.MethodDefinition where
tags' t@Tsx.MethodDefinition
{ ann = loc@Loc { byteRange }
, name
} = case name of
Prj Tsx.PropertyIdentifier { text } -> yield text
-- TODO: There are more here
_ -> gtags t
where
yield name = yieldTag name Call loc byteRange >> gtags t
instance ToTagsBy 'Custom Tsx.ClassDeclaration where
tags' t@Tsx.ClassDeclaration
{ ann = loc@Loc { byteRange }
, name = Tsx.TypeIdentifier { text }
} = yieldTag text Class loc byteRange >> gtags t
instance ToTagsBy 'Custom Tsx.CallExpression where
tags' t@Tsx.CallExpression
{ ann = loc@Loc { byteRange }
, function = Tsx.Expression expr
} = match expr
where
match expr = case expr of
Prj Tsx.Identifier { text } -> yield text
Prj Tsx.NewExpression { constructor = Prj Tsx.Identifier { text } } -> yield text
Prj Tsx.CallExpression { function = Tsx.Expression expr } -> match expr
Prj Tsx.MemberExpression { property = Tsx.PropertyIdentifier { text } } -> yield text
Prj Tsx.Function { name = Just Tsx.Identifier { text }} -> yield text
Prj Tsx.ParenthesizedExpression { extraChildren } -> for_ extraChildren $ \ x -> case x of
Prj (Tsx.Expression expr) -> match expr
_ -> tags x
_ -> gtags t
yield name = yieldTag name Call loc byteRange >> gtags t
instance ToTagsBy 'Custom Tsx.Class where
tags' t@Tsx.Class
{ ann = loc@Loc { byteRange }
, name = Just Tsx.TypeIdentifier { text }
} = yieldTag text Class loc byteRange >> gtags t
tags' t = gtags t
instance (ToTags l, ToTags r) => ToTagsBy 'Custom (l :+: r) where
tags' (L1 l) = tags l
tags' (R1 r) = tags r
gtags
:: ( Has (Reader Source) sig m
, Has (Writer Tags.Tags) sig m
, Generic1 t
, Tags.GFoldable1 ToTags (Rep1 t)
)
=> t Loc
-> m ()
gtags = getAp . Tags.gfoldMap1 @ToTags (Ap . tags) . from1
instance (Generic1 t, Tags.GFoldable1 ToTags (Rep1 t)) => ToTagsBy 'Generic t where
tags' = gtags
yieldTag :: (Has (Reader Source) sig m, Has (Writer Tags.Tags) sig m) => Text -> Kind -> Loc -> Range -> m ()
yieldTag name kind loc range = do
src <- ask @Source
let sliced = slice src range
Tags.yield (Tag name kind loc (Tags.firstLine sliced) Nothing)

21
semantic-typescript/LICENSE Normal file
View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2019 GitHub
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

3
semantic-typescript/README.md Normal file
View File

@ -0,0 +1,3 @@
# Semantic support for TypeScript
This package implements `semantic` support for TypeScript using the `semantic-core` intermediate language.

2
semantic-typescript/Setup.hs Normal file
View File

@ -0,0 +1,2 @@
import Distribution.Simple
main = defaultMain

54
semantic-typescript/semantic-typescript.cabal Normal file
View File

@ -0,0 +1,54 @@
cabal-version: 2.4
name: semantic-typescript
version: 0.0.0.0
synopsis: Semantic support for TypeScript.
description: Semantic support for TypeScript using the semantic-core intermediate language.
homepage: https://github.com/github/semantic/tree/master/semantic-typescript#readme
bug-reports: https://github.com/github/semantic/issues
license: MIT
license-file: LICENSE
author: The Semantic authors
maintainer: opensource+semantic@github.com
copyright: (c) 2019 GitHub, Inc.
category: Language
build-type: Simple
stability: alpha
extra-source-files: README.md
tested-with: GHC == 8.6.5
common haskell
default-language: Haskell2010
build-depends: base ^>= 4.13
, fused-effects ^>= 1.0
, fused-syntax
, parsers ^>= 0.12.10
, semantic-core ^>= 0.0
, semantic-source ^>= 0.0
, semantic-tags ^>= 0.0
, text ^>= 1.2.3
, tree-sitter ^>= 0.7.2
, tree-sitter-typescript ^>= 0.4
ghc-options:
-Weverything
-Wno-missing-local-signatures
-Wno-missing-import-lists
-Wno-implicit-prelude
-Wno-safe
-Wno-unsafe
-Wno-name-shadowing
-Wno-monomorphism-restriction
-Wno-missed-specialisations
-Wno-all-missed-specialisations
-Wno-star-is-type
if (impl(ghc >= 8.8))
ghc-options: -Wno-missing-deriving-strategies
library
import: haskell
exposed-modules:
Language.TypeScript
Language.TypeScript.Tags
hs-source-dirs: src

21
semantic-typescript/src/Language/TypeScript.hs Normal file
View File

@ -0,0 +1,21 @@
{-# OPTIONS_GHC -freduction-depth=0 #-}
-- | Semantic functionality for TypeScript programs.
module Language.TypeScript
( Term(..)
, TreeSitter.TypeScript.tree_sitter_typescript
) where
import qualified Language.TypeScript.Tags as TsTags
import qualified Tags.Tagging.Precise as Tags
import qualified TreeSitter.TypeScript (tree_sitter_typescript)
import qualified TreeSitter.TypeScript.AST as TypeScript
import qualified TreeSitter.Unmarshal as TS
newtype Term a = Term { getTerm :: TypeScript.Program a }
instance TS.Unmarshal Term where
unmarshalNode node = Term <$> TS.unmarshalNode node
instance Tags.ToTags Term where
tags src = Tags.runTagging src . TsTags.tags . getTerm

141
semantic-typescript/src/Language/TypeScript/Tags.hs Normal file
View File

@ -0,0 +1,141 @@
{-# LANGUAGE AllowAmbiguousTypes #-}
{-# LANGUAGE DataKinds #-}
{-# LANGUAGE DisambiguateRecordFields #-}
{-# LANGUAGE FlexibleContexts #-}
{-# LANGUAGE FlexibleInstances #-}
{-# LANGUAGE MultiParamTypeClasses #-}
{-# LANGUAGE NamedFieldPuns #-}
{-# LANGUAGE PartialTypeSignatures #-}
{-# LANGUAGE ScopedTypeVariables #-}
{-# LANGUAGE TypeApplications #-}
{-# LANGUAGE TypeFamilies #-}
{-# LANGUAGE TypeOperators #-}
{-# LANGUAGE UndecidableInstances #-}
{-# OPTIONS_GHC -freduction-depth=0 #-}
module Language.TypeScript.Tags
( ToTags(..)
) where
import AST.Element
import Control.Effect.Reader
import Control.Effect.Writer
import Data.Foldable
import Data.Monoid (Ap (..))
import Data.Text as Text
import GHC.Generics
import Source.Loc
import Source.Source as Source
import Tags.Tag
import qualified Tags.Tagging.Precise as Tags
import qualified TreeSitter.TypeScript.AST as Ts
class ToTags t where
tags
:: ( Has (Reader Source) sig m
, Has (Writer Tags.Tags) sig m
)
=> t Loc
-> m ()
instance (ToTagsBy strategy t, strategy ~ ToTagsInstance t) => ToTags t where
tags = tags' @strategy
class ToTagsBy (strategy :: Strategy) t where
tags'
:: ( Has (Reader Source) sig m
, Has (Writer Tags.Tags) sig m
)
=> t Loc
-> m ()
data Strategy = Generic | Custom
type family ToTagsInstance t :: Strategy where
ToTagsInstance (_ :+: _) = 'Custom
ToTagsInstance Ts.CallExpression = 'Custom
ToTagsInstance Ts.ClassDeclaration = 'Custom
ToTagsInstance Ts.Function = 'Custom
ToTagsInstance Ts.FunctionDeclaration = 'Custom
ToTagsInstance Ts.FunctionSignature = 'Custom
ToTagsInstance Ts.MethodDefinition = 'Custom
ToTagsInstance _ = 'Generic
instance ToTagsBy 'Custom Ts.Function where
tags' t@Ts.Function
{ ann = loc@Loc { byteRange }
, name = Just Ts.Identifier { text }
} = yieldTag text Function loc byteRange >> gtags t
tags' t = gtags t
instance ToTagsBy 'Custom Ts.FunctionSignature where
tags' t@Ts.FunctionSignature
{ ann = loc@Loc { byteRange }
, name = Ts.Identifier { text }
} = yieldTag text Function loc byteRange >> gtags t
instance ToTagsBy 'Custom Ts.FunctionDeclaration where
tags' t@Ts.FunctionDeclaration
{ ann = loc@Loc { byteRange }
, name = Ts.Identifier { text }
} = yieldTag text Function loc byteRange >> gtags t
instance ToTagsBy 'Custom Ts.MethodDefinition where
tags' t@Ts.MethodDefinition
{ ann = loc@Loc { byteRange }
, name
} = case name of
Prj Ts.PropertyIdentifier { text } -> yield text
-- TODO: There are more here
_ -> gtags t
where
yield name = yieldTag name Call loc byteRange >> gtags t
instance ToTagsBy 'Custom Ts.ClassDeclaration where
tags' t@Ts.ClassDeclaration
{ ann = loc@Loc { byteRange }
, name = Ts.TypeIdentifier { text }
} = yieldTag text Class loc byteRange >> gtags t
instance ToTagsBy 'Custom Ts.CallExpression where
tags' t@Ts.CallExpression
{ ann = loc@Loc { byteRange }
, function = Ts.Expression expr
} = match expr
where
match expr = case expr of
Prj Ts.Identifier { text } -> yield text
Prj Ts.NewExpression { constructor = Prj Ts.Identifier { text } } -> yield text
Prj Ts.CallExpression { function = Ts.Expression expr } -> match expr
Prj Ts.MemberExpression { property = Ts.PropertyIdentifier { text } } -> yield text
Prj Ts.Function { name = Just Ts.Identifier { text }} -> yield text
Prj Ts.ParenthesizedExpression { extraChildren } -> for_ extraChildren $ \ x -> case x of
Prj (Ts.Expression expr) -> match expr
_ -> tags x
_ -> gtags t
yield name = yieldTag name Call loc byteRange >> gtags t
instance (ToTags l, ToTags r) => ToTagsBy 'Custom (l :+: r) where
tags' (L1 l) = tags l
tags' (R1 r) = tags r
gtags
:: ( Has (Reader Source) sig m
, Has (Writer Tags.Tags) sig m
, Generic1 t
, Tags.GFoldable1 ToTags (Rep1 t)
)
=> t Loc
-> m ()
gtags = getAp . Tags.gfoldMap1 @ToTags (Ap . tags) . from1
instance (Generic1 t, Tags.GFoldable1 ToTags (Rep1 t)) => ToTagsBy 'Generic t where
tags' = gtags
yieldTag :: (Has (Reader Source) sig m, Has (Writer Tags.Tags) sig m) => Text -> Kind -> Loc -> Range -> m ()
yieldTag name kind loc range = do
src <- ask @Source
let sliced = slice src range
Tags.yield (Tag name kind loc (Tags.firstLine sliced) Nothing)

3
semantic.cabal
View File

@ -285,10 +285,13 @@ library
, proto-lens-jsonpb
, proto-lens-runtime >= 0.5 && <0.7
, reducers ^>= 3.12.3
, semantic-go ^>= 0
, semantic-java ^>= 0
, semantic-json ^>= 0
, semantic-python ^>= 0
, semantic-ruby ^>= 0
, semantic-tsx ^>= 0
, semantic-typescript ^>= 0
, semantic-tags ^>= 0
, semigroupoids ^>= 5.3.2
, split ^>= 0.2.3.3

12
src/Data/Language.hs
View File

@ -153,7 +153,11 @@ textToLanguage = \case
data PerLanguageModes = PerLanguageModes
{ pythonMode :: LanguageMode
, rubyMode :: LanguageMode
-- , typescriptMode :: LanguageMode
, goMode :: LanguageMode
, typescriptMode :: LanguageMode
, tsxMode :: LanguageMode
, javascriptMode :: LanguageMode
, jsxMode :: LanguageMode
}
deriving (Eq, Ord, Show)
@ -161,7 +165,11 @@ defaultLanguageModes :: PerLanguageModes
defaultLanguageModes = PerLanguageModes
{ pythonMode = ALaCarte
, rubyMode = ALaCarte
-- , typescriptMode = ALaCarte
, goMode = ALaCarte
, typescriptMode = ALaCarte
, tsxMode = ALaCarte
, javascriptMode = ALaCarte
, jsxMode = ALaCarte
}
data LanguageMode

2
src/Language/Go/Assignment.hs
View File

@ -454,7 +454,7 @@ assignment' = makeTerm' <$> symbol AssignmentStatement <*> children (infixTerm
assign l r = inject (Statement.Assignment [] l r)
augmentedAssign :: (f :< Go.Syntax) => (Term Loc -> Term Loc -> f (Term Loc)) -> Term Loc -> Term Loc -> Sum Go.Syntax (Term Loc)
augmentedAssign c l r = assign l (makeTerm1 (c l r))
augmentedAssign c l r = inject (Statement.AugmentedAssignment (makeTerm1 (c l r)))
invert cons a b = Expression.Not (makeTerm1 (cons a b))

1
src/Language/Go/Term.hs
View File

@ -104,6 +104,7 @@ type Syntax =
, Literal.Reference
, Literal.TextElement
, Statement.Assignment
, Statement.AugmentedAssignment
, Statement.Break
, Statement.Continue
, Statement.For

2
src/Language/TSX/Assignment.hs
View File

@ -104,7 +104,7 @@ augmentedAssignmentExpression = makeTerm' <$> symbol AugmentedAssignmentExpressi
, assign Expression.LShift <$ symbol AnonLAngleLAngleEqual
, assign Expression.BOr <$ symbol AnonPipeEqual ])
where assign :: (f :< TSX.Syntax) => (Term Loc -> Term Loc -> f (Term Loc)) -> Term Loc -> Term Loc -> Sum TSX.Syntax (Term Loc)
assign c l r = inject (Statement.Assignment [] l (makeTerm1 (c l r)))
assign c l r = inject (Statement.AugmentedAssignment (makeTerm1 (c l r)))
awaitExpression :: Assignment (Term Loc)

1
src/Language/TSX/Term.hs
View File

@ -93,6 +93,7 @@ type Syntax =
, Literal.TextElement
, Literal.Regex
, Statement.Assignment
, Statement.AugmentedAssignment
, Statement.Break
, Statement.Catch
, Statement.Continue

2
src/Language/TypeScript/Assignment.hs
View File

@ -103,7 +103,7 @@ augmentedAssignmentExpression = makeTerm' <$> symbol AugmentedAssignmentExpressi
, assign Expression.LShift <$ symbol AnonLAngleLAngleEqual
, assign Expression.BOr <$ symbol AnonPipeEqual ])
where assign :: (f :< TypeScript.Syntax) => (Term Loc -> Term Loc -> f (Term Loc)) -> Term Loc -> Term Loc -> Sum TypeScript.Syntax (Term Loc)
assign c l r = inject (Statement.Assignment [] l (makeTerm1 (c l r)))
assign c l r = inject (Statement.AugmentedAssignment (makeTerm1 (c l r)))
awaitExpression :: Assignment (Term Loc)

1
src/Language/TypeScript/Term.hs
View File

@ -93,6 +93,7 @@ type Syntax =
, Literal.TextElement
, Literal.Regex
, Statement.Assignment
, Statement.AugmentedAssignment
, Statement.Break
, Statement.Catch
, Statement.Continue

138
src/Parsing/Parser.hs
View File

@ -5,9 +5,15 @@ module Parsing.Parser
-- $abstract
, SomeParser(..)
, goParser
, goParserALaCarte
, goParserPrecise
, javaParser
, javascriptParserALaCarte
, javascriptParserPrecise
, javascriptParser
, jsonParser
, jsxParserALaCarte
, jsxParserPrecise
, jsxParser
, markdownParser
, phpParser
@ -17,7 +23,11 @@ module Parsing.Parser
, rubyParserALaCarte
, rubyParserPrecise
, rubyParser
, tsxParserALaCarte
, tsxParserPrecise
, tsxParser
, typescriptParserALaCarte
, typescriptParserPrecise
, typescriptParser
-- * Modes by term type
, TermMode
@ -35,7 +45,8 @@ import qualified Data.Map as Map
import qualified Data.Syntax as Syntax
import Data.Term
import Foreign.Ptr
import qualified Language.Go.Assignment as Go
import qualified Language.Go as GoPrecise
import qualified Language.Go.Assignment as GoALaCarte
import qualified Language.Java as Java
import qualified Language.JSON as JSON
import qualified Language.Markdown.Assignment as Markdown
@ -44,8 +55,10 @@ import qualified Language.Python as PythonPrecise
import qualified Language.Python.Assignment as PythonALaCarte
import qualified Language.Ruby as RubyPrecise
import qualified Language.Ruby.Assignment as RubyALaCarte
import qualified Language.TSX.Assignment as TSX
import qualified Language.TypeScript.Assignment as TypeScript
import qualified Language.TSX as TSXPrecise
import qualified Language.TSX.Assignment as TSXALaCarte
import qualified Language.TypeScript as TypeScriptPrecise
import qualified Language.TypeScript.Assignment as TypeScriptALaCarte
import Prelude hiding (fail)
import Prologue
import TreeSitter.Go
@ -103,20 +116,44 @@ data Parser term where
data SomeParser c a where
SomeParser :: c t => Parser (t a) -> SomeParser c a
goParser :: c Go.Term => (Language, SomeParser c Loc)
goParser = (Go, SomeParser (AssignmentParser (ASTParser tree_sitter_go) Go.assignment))
goParserALaCarte :: c GoALaCarte.Term => (Language, SomeParser c Loc)
goParserALaCarte = (Go, SomeParser (AssignmentParser (ASTParser tree_sitter_go) GoALaCarte.assignment))
goParserPrecise :: c GoPrecise.Term => (Language, SomeParser c Loc)
goParserPrecise = (Go, SomeParser (UnmarshalParser @GoPrecise.Term GoPrecise.tree_sitter_go))
goParser :: (c GoALaCarte.Term, c GoPrecise.Term) => PerLanguageModes -> (Language, SomeParser c Loc)
goParser modes = case goMode modes of
ALaCarte -> goParserALaCarte
Precise -> goParserPrecise
javaParser :: c Java.Term => (Language, SomeParser c Loc)
javaParser = (Java, SomeParser (UnmarshalParser @Java.Term Java.tree_sitter_java))
javascriptParser :: c TSX.Term => (Language, SomeParser c Loc)
javascriptParser = (JavaScript, SomeParser (AssignmentParser (ASTParser tree_sitter_tsx) TSX.assignment))
javascriptParserALaCarte :: c TSXALaCarte.Term => (Language, SomeParser c Loc)
javascriptParserALaCarte = (JavaScript, SomeParser (AssignmentParser (ASTParser tree_sitter_tsx) TSXALaCarte.assignment))
javascriptParserPrecise :: c TSXPrecise.Term => (Language, SomeParser c Loc)
javascriptParserPrecise = (JavaScript, SomeParser (UnmarshalParser @TSXPrecise.Term TSXPrecise.tree_sitter_tsx))
javascriptParser :: (c TSXALaCarte.Term, c TSXPrecise.Term) => PerLanguageModes -> (Language, SomeParser c Loc)
javascriptParser modes = case javascriptMode modes of
ALaCarte -> javascriptParserALaCarte
Precise -> javascriptParserPrecise
jsonParser :: c JSON.Term => (Language, SomeParser c Loc)
jsonParser = (JSON, SomeParser (UnmarshalParser @JSON.Term JSON.tree_sitter_json))
jsxParser :: c TSX.Term => (Language, SomeParser c Loc)
jsxParser = (JSX, SomeParser (AssignmentParser (ASTParser tree_sitter_tsx) TSX.assignment))
jsxParserALaCarte :: c TSXALaCarte.Term => (Language, SomeParser c Loc)
jsxParserALaCarte = (JSX, SomeParser (AssignmentParser (ASTParser tree_sitter_tsx) TSXALaCarte.assignment))
jsxParserPrecise :: c TSXPrecise.Term => (Language, SomeParser c Loc)
jsxParserPrecise = (JSX, SomeParser (UnmarshalParser @TSXPrecise.Term TSXPrecise.tree_sitter_tsx))
jsxParser :: (c TSXALaCarte.Term, c TSXPrecise.Term) => PerLanguageModes -> (Language, SomeParser c Loc)
jsxParser modes = case jsxMode modes of
ALaCarte -> jsxParserALaCarte
Precise -> jsxParserPrecise
markdownParser :: c Markdown.Term => (Language, SomeParser c Loc)
markdownParser = (Markdown, SomeParser (AssignmentParser MarkdownParser Markdown.assignment))
@ -146,43 +183,61 @@ rubyParser modes = case rubyMode modes of
ALaCarte -> rubyParserALaCarte
Precise -> rubyParserPrecise
tsxParser :: c TSX.Term => (Language, SomeParser c Loc)
tsxParser = (TSX, SomeParser (AssignmentParser (ASTParser tree_sitter_tsx) TSX.assignment))
tsxParserALaCarte :: c TSXALaCarte.Term => (Language, SomeParser c Loc)
tsxParserALaCarte = (TSX, SomeParser (AssignmentParser (ASTParser tree_sitter_tsx) TSXALaCarte.assignment))
typescriptParser :: c TypeScript.Term => (Language, SomeParser c Loc)
typescriptParser = (TypeScript, SomeParser (AssignmentParser (ASTParser tree_sitter_typescript) TypeScript.assignment))
tsxParserPrecise :: c TSXPrecise.Term => (Language, SomeParser c Loc)
tsxParserPrecise = (TSX, SomeParser (UnmarshalParser @TSXPrecise.Term TSXPrecise.tree_sitter_tsx))
tsxParser :: (c TSXALaCarte.Term, c TSXPrecise.Term) => PerLanguageModes -> (Language, SomeParser c Loc)
tsxParser modes = case tsxMode modes of
ALaCarte -> tsxParserALaCarte
Precise -> tsxParserPrecise
typescriptParserALaCarte :: c TypeScriptALaCarte.Term => (Language, SomeParser c Loc)
typescriptParserALaCarte = (TypeScript, SomeParser (AssignmentParser (ASTParser tree_sitter_typescript) TypeScriptALaCarte.assignment))
typescriptParserPrecise :: c TypeScriptPrecise.Term => (Language, SomeParser c Loc)
typescriptParserPrecise = (TypeScript, SomeParser (UnmarshalParser @TypeScriptPrecise.Term TypeScriptPrecise.tree_sitter_typescript))
typescriptParser :: (c TypeScriptALaCarte.Term, c TypeScriptPrecise.Term) => PerLanguageModes -> (Language, SomeParser c Loc)
typescriptParser modes = case typescriptMode modes of
ALaCarte -> typescriptParserALaCarte
Precise -> typescriptParserPrecise
-- | A type family selecting the language mode for a given term type.
type family TermMode term where
TermMode Java.Term = 'Precise
TermMode JSON.Term = 'Precise
TermMode PythonPrecise.Term = 'Precise
TermMode RubyPrecise.Term = 'Precise
TermMode _ = 'ALaCarte
TermMode GoPrecise.Term = 'Precise
TermMode Java.Term = 'Precise
TermMode JSON.Term = 'Precise
TermMode PythonPrecise.Term = 'Precise
TermMode RubyPrecise.Term = 'Precise
TermMode TypeScriptPrecise.Term = 'Precise
TermMode TSXPrecise.Term = 'Precise
TermMode _ = 'ALaCarte
-- | The canonical set of parsers producing à la carte terms.
aLaCarteParsers
:: ( c Go.Term
:: ( c GoALaCarte.Term
, c Markdown.Term
, c PHP.Term
, c PythonALaCarte.Term
, c RubyALaCarte.Term
, c TSX.Term
, c TypeScript.Term
, c TSXALaCarte.Term
, c TypeScriptALaCarte.Term
)
=> Map Language (SomeParser c Loc)
aLaCarteParsers = Map.fromList
[ goParser
, javascriptParser
, jsxParser
[ javascriptParserALaCarte
, jsxParserALaCarte
, markdownParser
, phpParser
, pythonParserALaCarte
, rubyParserALaCarte
, typescriptParser
, tsxParser
, tsxParserALaCarte
, typescriptParserALaCarte
, goParserALaCarte
]
-- | The canonical set of parsers producing precise terms.
@ -191,18 +246,27 @@ preciseParsers
, c JSON.Term
, c PythonPrecise.Term
, c RubyPrecise.Term
, c GoPrecise.Term
, c TypeScriptPrecise.Term
, c TSXPrecise.Term
)
=> Map Language (SomeParser c Loc)
preciseParsers = Map.fromList
[ javaParser
[ goParserPrecise
, javascriptParserPrecise
, jsonParser
, jsxParserPrecise
, pythonParserPrecise
, rubyParserPrecise
, tsxParserPrecise
, typescriptParserPrecise
, javaParser
]
-- | The canonical set of all parsers for the passed per-language modes.
allParsers
:: ( c Go.Term
:: ( c GoALaCarte.Term
, c GoPrecise.Term
, c Java.Term
, c JSON.Term
, c Markdown.Term
@ -211,21 +275,23 @@ allParsers
, c PythonPrecise.Term
, c RubyALaCarte.Term
, c RubyPrecise.Term
, c TSX.Term
, c TypeScript.Term
, c TSXALaCarte.Term
, c TSXPrecise.Term
, c TypeScriptALaCarte.Term
, c TypeScriptPrecise.Term
)
=> PerLanguageModes
-> Map Language (SomeParser c Loc)
allParsers modes = Map.fromList
[ goParser
[ goParser modes
, javaParser
, javascriptParser
, javascriptParser modes
, jsonParser
, jsxParser
, jsxParser modes
, markdownParser
, phpParser
, pythonParser modes
, rubyParser modes
, typescriptParser
, tsxParser
, tsxParser modes
, typescriptParser modes
]

22
src/Semantic/Api/Terms.hs
View File

@ -1,4 +1,5 @@
{-# LANGUAGE AllowAmbiguousTypes, DataKinds, FlexibleContexts, FlexibleInstances, MultiParamTypeClasses, RankNTypes, RecordWildCards, ScopedTypeVariables, TypeApplications, TypeFamilies, UndecidableInstances #-}
{-# OPTIONS_GHC -freduction-depth=0 #-}
module Semantic.Api.Terms
( termGraph
, parseTermBuilder
@ -40,8 +41,11 @@ import Source.Loc
import qualified Language.Java as Java
import qualified Language.JSON as JSON
import qualified Language.Go as GoPrecise
import qualified Language.Python as PythonPrecise
import qualified Language.Ruby as RubyPrecise
import qualified Language.TypeScript as TypeScriptPrecise
import qualified Language.TSX as TSXPrecise
termGraph :: (Traversable t, Has Distribute sig m, Has (Error SomeException) sig m, Has Parse sig m) => t Blob -> m ParseTreeGraphResponse
@ -108,6 +112,9 @@ instance (TermMode term ~ strategy, ShowTermBy strategy term) => ShowTerm term w
class ShowTermBy (strategy :: LanguageMode) term where
showTermBy :: (Has (Reader Config) sig m) => term Loc -> m Builder
instance ShowTermBy 'Precise GoPrecise.Term where
showTermBy = serialize Show . void . GoPrecise.getTerm
instance ShowTermBy 'Precise Java.Term where
showTermBy = serialize Show . void . Java.getTerm
@ -120,6 +127,12 @@ instance ShowTermBy 'Precise PythonPrecise.Term where
instance ShowTermBy 'Precise RubyPrecise.Term where
showTermBy = serialize Show . void . RubyPrecise.getTerm
instance ShowTermBy 'Precise TSXPrecise.Term where
showTermBy = serialize Show . void . TSXPrecise.getTerm
instance ShowTermBy 'Precise TypeScriptPrecise.Term where
showTermBy = serialize Show . void . TypeScriptPrecise.getTerm
instance (Recursive (term Loc), Show1 syntax, Base (term Loc) ~ TermF syntax Loc) => ShowTermBy 'ALaCarte term where
showTermBy = serialize Show . quieterm
@ -136,6 +149,9 @@ instance (TermMode term ~ strategy, SExprTermBy strategy term) => SExprTerm term
class SExprTermBy (strategy :: LanguageMode) term where
sexprTermBy :: term Loc -> Builder
instance SExprTermBy 'Precise GoPrecise.Term where
sexprTermBy = SExpr.Precise.serializeSExpression . GoPrecise.getTerm
instance SExprTermBy 'Precise Java.Term where
sexprTermBy = SExpr.Precise.serializeSExpression . Java.getTerm
@ -148,6 +164,12 @@ instance SExprTermBy 'Precise PythonPrecise.Term where
instance SExprTermBy 'Precise RubyPrecise.Term where
sexprTermBy = SExpr.Precise.serializeSExpression . RubyPrecise.getTerm
instance SExprTermBy 'Precise TSXPrecise.Term where
sexprTermBy = SExpr.Precise.serializeSExpression . TSXPrecise.getTerm
instance SExprTermBy 'Precise TypeScriptPrecise.Term where
sexprTermBy = SExpr.Precise.serializeSExpression . TypeScriptPrecise.getTerm
instance (Recursive (term Loc), SExpr.ToSExpression (Base (term Loc))) => SExprTermBy 'ALaCarte term where
sexprTermBy = SExpr.serializeSExpression ByConstructorName

24
src/Semantic/CLI.hs
View File

@ -168,16 +168,20 @@ graphCommand = command "graph" (info graphArgumentsParser (progDesc "Compute a g
languageModes :: Parser Language.PerLanguageModes
languageModes = Language.PerLanguageModes
<$> option auto ( long "python-mode"
<> help "The AST representation to use for Python sources"
<> metavar "ALaCarte|Precise"
<> value Language.ALaCarte
<> showDefault)
<*> option auto ( long "ruby-mode"
<> help "The AST representation to use for Ruby sources"
<> metavar "ALaCarte|Precise"
<> value Language.ALaCarte
<> showDefault)
<$> languageModeOption "python" "Python"
<*> languageModeOption "ruby" "Ruby"
<*> languageModeOption "go" "Go"
<*> languageModeOption "typescript" "TypeScript"
<*> languageModeOption "tsx" "TSX"
<*> languageModeOption "javascript" "JavaScript"
<*> languageModeOption "jsx" "JSX"
where
languageModeOption shortName fullName
= option auto ( long (shortName <> "-mode")
<> help ("The AST representation to use for " <> fullName <> " sources")
<> metavar "ALaCarte|Precise"
<> value Language.ALaCarte
<> showDefault)
filePathReader :: ReadM File
filePathReader = fileForPath <$> str

8
src/Semantic/Graph.hs
View File

@ -104,13 +104,13 @@ instance
analysisParsers :: Map Language (SomeParser AnalyzeTerm Loc)
analysisParsers = Map.fromList
[ goParser
, javascriptParser
[ goParserALaCarte
, javascriptParserALaCarte
, phpParser
, pythonParserALaCarte
, rubyParserALaCarte
, typescriptParser
, tsxParser
, typescriptParserALaCarte
, tsxParserALaCarte
]
runGraph :: ( Has Distribute sig m

143
test/Examples.hs
View File

@ -1,25 +1,19 @@
{-# LANGUAGE FlexibleContexts, RecordWildCards, OverloadedStrings, TypeApplications #-}
{-# OPTIONS_GHC -O1 #-}
module Main (main, knownFailuresForPath) where
module Main (main) where
import Control.Carrier.Parse.Measured
import Control.Carrier.Reader
import Control.Concurrent.Async (forConcurrently)
import Control.Exception (displayException)
import qualified Control.Foldl as Foldl
import Control.Lens
import Control.Monad
import Control.Monad.Trans.Resource (ResIO, runResourceT)
import Data.Blob
import qualified Data.ByteString.Lazy.Char8 as BLC
import qualified Data.ByteString.Streaming.Char8 as ByteStream
import Data.Foldable
import Data.Language (LanguageMode (..), PerLanguageModes (..))
import Data.List
import qualified Data.Text as Text
import Data.Set (Set)
import Data.Traversable
import qualified Streaming.Prelude as Stream
import System.FilePath.Glob
import System.Path ((</>))
import qualified System.Path as Path
@ -35,35 +29,66 @@ import Semantic.Config as Config
import Semantic.Task
import Semantic.Task.Files
data LanguageExample
= LanguageExample
{ languageName :: String
, languageExtension :: String
, languageSkips :: [Path.RelFile]
} deriving (Eq, Show)
data LanguageExample =
LanguageExample
{ languageName :: String
, languageExtension :: String
, languageSkips :: [Path.RelFile]
, languageDirSkips :: [Path.RelDir]
}
deriving (Eq, Show)
le :: String -> String -> [Path.RelFile] -> LanguageExample
le :: String -> String -> [Path.RelFile] -> [Path.RelDir] -> LanguageExample
le = LanguageExample
examples :: [LanguageExample]
examples =
[ le "python" "**/*.py" mempty
, le "ruby" "**/*.rb" rubySkips
-- , le "typescript" "**/*.[jt]s*" Nothing -- (Just $ Path.relFile "typescript/script/known_failures.txt")
-- , le "typescript" "**/*.tsx" Nothing
-- , le "javascript" ".js" examples Nothing -- parse JavaScript with TypeScript parser.
-- , le "go" ".go" examples (Just $ Path.relFile "script/known-failures.txt")
[ le "go" "**/*.go" goFileSkips goDirSkips
, le "python" "**/*.py" mempty mempty
, le "ruby" "**/*.rb" rubySkips mempty
, le "typescript" "**/*.[jt]s" typescriptSkips mempty
, le "typescript" "**/*.[jt]sx" tsxSkips mempty
]
-- TODO: Java assignment errors need to be investigated
-- , le "java" ".java" examples (Just $ Path.relFile "script/known_failures_guava.txt")
goFileSkips :: [Path.RelFile]
goFileSkips = Path.relPath <$>
[
-- Super slow
"go/src/vendor/golang_org/x/text/unicode/norm/tables.go"
, "go/src/vendor/golang_org/x/text/unicode/bidi/tables.go"
, "go/src/vendor/golang_org/x/net/idna/tables.go"
, "go/src/cmd/vendor/golang.org/x/arch/x86/x86asm/tables.go"
, "moby/vendor/golang.org/x/text/unicode/norm/tables9.0.0.go"
, "moby/vendor/golang.org/x/text/unicode/norm/tables10.0.0.go"
-- TODO: Haskell assignment errors need to be investigated
-- , le "haskell" ".hs" "examples/effects" (Just "script/known-failures-effects.txt")
-- , le "haskell" ".hs" "examples/postgrest" (Just "script/known-failures-postgrest.txt")
-- , le "haskell" ".hs" "examples/ivory" (Just "script/known-failures-ivory.txt")
-- Assignment timeouts
, "go/src/cmd/compile/internal/gc/constFold_test.go"
, "go/src/cmd/compile/internal/gc/testdata/arithConst.go"
, "moby/vendor/github.com/docker/swarmkit/api/types.pb.go"
, "moby/vendor/github.com/docker/swarmkit/api/control.pb.go"
-- , ("php", ".php") -- TODO: No parse-examples in tree-sitter yet
]-- where examples = Path.relDir "examples"
-- Parser timeouts
, "moby/vendor/github.com/ugorji/go/codec/fast-path.generated.go"
-- Parse errors
, "go/src/math/big/arith.go" -- Unhandled identifier character: 'ŝ'
, "go/src/cmd/vet/testdata/deadcode.go"
, "moby/vendor/github.com/beorn7/perks/quantile/stream.go" -- Unhandled identifier character: 'ƒ'
-- UTF8 encoding issues ("Cannot decode byte '\xe3': Data.Text.Internal.Encoding.decodeUtf8: Invalid UTF-8 stream")
, "go/src/text/template/exec_test.go"
, "go/src/bufio/bufio_test.go"
, "go/doc/progs/go1.go"
]
goDirSkips :: [Path.RelDir]
goDirSkips = Path.relDir <$>
[ "go/src/cmd/compile/internal/ssa"
, "go/test/fixedbugs"
, "go/test/syntax"
, "go/test/method4.dir"
, "go/test"
]
rubySkips :: [Path.RelFile]
rubySkips = Path.relFile <$>
@ -78,17 +103,51 @@ rubySkips = Path.relFile <$>
, "ruby_spec/core/enumerable/shared/inject.rb"
-- Doesn't parse
, "ruby_spec/language/string_spec.rb"
, "ruby_spec/language/fixtures/freeze_magic_comment_required_diff_enc.rb"
-- Can't detect method calls inside heredoc bodies with precise ASTs
, "ruby_spec/core/argf/readpartial_spec.rb"
, "ruby_spec/core/process/exec_spec.rb"
]
tsxSkips :: [Path.RelFile]
tsxSkips = Path.relFile <$>
[
-- Cannot decode byte '\xe2': Data.Text.Internal.Encoding.decodeUtf8: Invalid UTF-8 stream
"desktop/app/src/ui/clone-repository/clone-github-repository.tsx"
, "desktop/app/src/ui/toolbar/revert-progress.tsx"
]
typescriptSkips :: [Path.RelFile]
typescriptSkips = Path.relFile <$>
[
-- Assignment timeouts
"npm/node_modules/request/node_modules/http-signature/node_modules/sshpk/node_modules/tweetnacl/nacl-fast.js"
, "npm/node_modules/cli-table2/test/cell-test.js"
, "npm/node_modules/request/node_modules/har-validator/node_modules/ajv/dist/regenerator.min.js"
, "npm/node_modules/request/node_modules/har-validator/node_modules/ajv/dist/ajv.bundle.js"
, "npm/node_modules/request/node_modules/har-validator/node_modules/ajv/dist/ajv.min.js"
, "npm/node_modules/request/node_modules/har-validator/node_modules/ajv/dist/nodent.min.js"
, "npm/node_modules/bluebird/js/browser/bluebird.js"
, "npm/node_modules/bluebird/js/browser/bluebird.min.js"
, "npm/node_modules/bluebird/js/browser/bluebird.core.js"
, "npm/node_modules/cli-table2/node_modules/lodash/index.js"
, "npm/node_modules/cli-table2/node_modules/lodash/index.js"
-- Cannot decode byte '\xd0': Data.Text.Internal.Encoding.decodeUtf8: Invalid UTF-8 stream
, "npm/node_modules/npm-profile/node_modules/make-fetch-happen/node_modules/socks-proxy-agent/node_modules/socks/node_modules/smart-buffer/test/smart-buffer.test.js"
, "npm/node_modules/pacote/node_modules/make-fetch-happen/node_modules/socks-proxy-agent/node_modules/socks/node_modules/smart-buffer/test/smart-buffer.test.js"
, "npm/node_modules/archy/test/multi_line.js"
, "npm/node_modules/archy/test/beep.js"
, "npm/node_modules/cli-table2/test/cell-test.js"
]
buildExamples :: TaskSession -> LanguageExample -> Path.RelDir -> IO Tasty.TestTree
buildExamples session lang tsDir = do
let skips = fmap (tsDir </>) (languageSkips lang)
let fileSkips = fmap (tsDir </>) (languageSkips lang)
dirSkips = fmap (tsDir </>) (languageDirSkips lang)
files <- globDir1 (compile (languageExtension lang)) (Path.toString tsDir)
let paths = filter (`notElem` skips) $ Path.relFile <$> files
let paths = filter (\x -> Path.takeDirectory x `notElem` dirSkips) . filter (`notElem` fileSkips) $ Path.relFile <$> files
trees <- for paths $ \file -> do
pure . HUnit.testCaseSteps (Path.toString file) $ \step -> do
-- Use alacarte language mode
@ -143,7 +202,6 @@ buildExamples session lang tsDir = do
(_, Left e) -> HUnit.assertFailure ("Unable to parse (precise)" <> show (displayException e))
(Left e, _) -> HUnit.assertFailure ("Unable to parse (a la carte)" <> show (displayException e))
filterALaCarteSymbols :: String -> [Text.Text] -> [Text.Text]
filterALaCarteSymbols "ruby" symbols
= filterOutInstanceVariables
@ -168,12 +226,22 @@ aLaCarteLanguageModes :: PerLanguageModes
aLaCarteLanguageModes = PerLanguageModes
{ pythonMode = ALaCarte
, rubyMode = ALaCarte
, goMode = ALaCarte
, typescriptMode = ALaCarte
, tsxMode = ALaCarte
, javascriptMode = ALaCarte
, jsxMode = ALaCarte
}
preciseLanguageModes :: PerLanguageModes
preciseLanguageModes = PerLanguageModes
{ pythonMode = Precise
, rubyMode = Precise
, goMode = Precise
, typescriptMode = Precise
, tsxMode = Precise
, javascriptMode = Precise
, jsxMode = Precise
}
testOptions :: Config.Options
@ -194,19 +262,6 @@ main = withOptions testOptions $ \ config logger statter -> do
Tasty.defaultMain $ Tasty.testGroup "parse-examples" allTests
knownFailuresForPath :: Path.RelDir -> Maybe Path.RelFile -> IO (Set Path.RelFile)
knownFailuresForPath _ Nothing = pure mempty
knownFailuresForPath tsDir (Just path)
= runResourceT
( ByteStream.readFile @ResIO (Path.toString (tsDir </> path))
& ByteStream.lines
& ByteStream.denull
& Stream.mapped ByteStream.toLazy
& Stream.filter ((/= '#') . BLC.head)
& Stream.map (Path.relFile . BLC.unpack)
& Foldl.purely Stream.fold_ Foldl.set
)
parseSymbolsFilePath ::
( Has (Error SomeException) sig m
, Has Distribute sig m

2
test/Semantic/CLI/Spec.hs
View File

@ -63,7 +63,7 @@ parseFixtures =
path' = [File "test/fixtures/ruby/corpus/and-or.A.rb" Ruby, File "test/fixtures/ruby/corpus/and-or.B.rb" Ruby]
path'' = [File "test/fixtures/ruby/corpus/method-declaration.A.rb" Ruby]
prefix = Path.relDir "test/fixtures/cli"
run = runReader (PerLanguageModes ALaCarte ALaCarte)
run = runReader defaultLanguageModes
diffFixtures :: [(String, [BlobPair] -> ParseC TaskC Builder, [(File, File)], Path.RelFile)]
diffFixtures =

6
test/Semantic/Spec.hs
View File

@ -17,15 +17,15 @@ spec = do
let methodsBlob = makeBlob "def foo\nend\n" "methods.rb" Ruby mempty
it "returns error if given an unknown language (json)" $ do
output <- fmap runBuilder . runTaskOrDie . runReader (PerLanguageModes ALaCarte ALaCarte) $ parseTermBuilder TermJSONTree [ setBlobLanguage Unknown methodsBlob ]
output <- fmap runBuilder . runTaskOrDie . runReader defaultLanguageModes $ parseTermBuilder TermJSONTree [ setBlobLanguage Unknown methodsBlob ]
output `shouldBe` "{\"trees\":[{\"path\":\"methods.rb\",\"error\":\"NoLanguageForBlob \\\"methods.rb\\\"\",\"language\":\"Unknown\"}]}\n"
it "throws if given an unknown language for sexpression output" $ do
res <- runTaskWithOptions defaultOptions (runReader (PerLanguageModes ALaCarte ALaCarte) (runParseWithConfig (parseTermBuilder TermSExpression [setBlobLanguage Unknown methodsBlob])))
res <- runTaskWithOptions defaultOptions (runReader defaultLanguageModes (runParseWithConfig (parseTermBuilder TermSExpression [setBlobLanguage Unknown methodsBlob])))
case res of
Left exc -> fromException exc `shouldBe` Just (NoLanguageForBlob "methods.rb")
Right _bad -> fail "Expected parseTermBuilder to fail for an unknown language"
it "renders with the specified renderer" $ do
output <- fmap runBuilder . runTaskOrDie . runReader (PerLanguageModes ALaCarte ALaCarte) $ parseTermBuilder TermSExpression [methodsBlob]
output <- fmap runBuilder . runTaskOrDie . runReader defaultLanguageModes $ parseTermBuilder TermSExpression [methodsBlob]
output `shouldBe` "(Statements\n (Method\n (Empty)\n (Identifier)\n (Statements)))\n"

64
test/fixtures/go/corpus/assignment-statements.diffA-B.txt vendored
View File

@ -8,12 +8,7 @@
{ (Identifier)
->(Identifier) }
(Integer))
(Assignment
(Statements
{ (Identifier)
->(Identifier) }
{ (Identifier)
->(Identifier) })
(AugmentedAssignment
(Plus
(Statements
{ (Identifier)
@ -23,43 +18,38 @@
(Statements
(Integer)
(Integer))))
{+(Assignment
{+(Identifier)+}
{+(AugmentedAssignment
{+(Times
{+(Identifier)+}
{+(Integer)+})+})+}
{+(Assignment
{+(Identifier)+}
{+(AugmentedAssignment
{+(Plus
{+(Identifier)+}
{+(Integer)+})+})+}
{+(Assignment
{+(Identifier)+}
{+(AugmentedAssignment
{+(LShift
{+(Identifier)+}
{+(Integer)+})+})+}
{+(Assignment
{+(Identifier)+}
{+(RShift
(AugmentedAssignment
{ (Times
{-(Identifier)-}
{-(Integer)-})
->(RShift
{+(Identifier)+}
{+(Integer)+})+})+}
{+(Assignment
{+(Identifier)+}
{+(Integer)+}) })
{+(AugmentedAssignment
{+(DividedBy
{+(Identifier)+}
{+(Integer)+})+})+}
{+(Assignment
{+(Identifier)+}
{+(AugmentedAssignment
{+(BXOr
{+(Identifier)+}
{+(Integer)+})+})+}
{+(Assignment
{+(Identifier)+}
{+(AugmentedAssignment
{+(Modulo
{+(Identifier)+}
{+(Integer)+})+})+}
{+(Assignment
{+(Identifier)+}
{+(AugmentedAssignment
{+(Not
{+(BAnd
{+(Identifier)+}
@ -76,43 +66,31 @@
{+(KeyValue
{+(Identifier)+}
{+(Integer)+})+})+})+})+})+})+}
{-(Assignment
{-(Identifier)-}
{-(Times
{-(Identifier)-}
{-(Integer)-})-})-}
{-(Assignment
{-(Identifier)-}
{-(AugmentedAssignment
{-(Plus
{-(Identifier)-}
{-(Integer)-})-})-}
{-(Assignment
{-(Identifier)-}
{-(AugmentedAssignment
{-(LShift
{-(Identifier)-}
{-(Integer)-})-})-}
{-(Assignment
{-(Identifier)-}
{-(AugmentedAssignment
{-(RShift
{-(Identifier)-}
{-(Integer)-})-})-}
{-(Assignment
{-(Identifier)-}
{-(AugmentedAssignment
{-(DividedBy
{-(Identifier)-}
{-(Integer)-})-})-}
{-(Assignment
{-(Identifier)-}
{-(AugmentedAssignment
{-(BXOr
{-(Identifier)-}
{-(Integer)-})-})-}
{-(Assignment
{-(Identifier)-}
{-(AugmentedAssignment
{-(Modulo
{-(Identifier)-}
{-(Integer)-})-})-}
{-(Assignment
{-(Identifier)-}
{-(AugmentedAssignment
{-(Not
{-(BAnd
{-(Identifier)-}

64
test/fixtures/go/corpus/assignment-statements.diffB-A.txt vendored
View File

@ -8,12 +8,7 @@
{ (Identifier)
->(Identifier) }
(Integer))
(Assignment
(Statements
{ (Identifier)
->(Identifier) }
{ (Identifier)
->(Identifier) })
(AugmentedAssignment
(Plus
(Statements
{ (Identifier)
@ -23,43 +18,38 @@
(Statements
(Integer)
(Integer))))
{+(Assignment
{+(Identifier)+}
{+(AugmentedAssignment
{+(Times
{+(Identifier)+}
{+(Integer)+})+})+}
{+(Assignment
{+(Identifier)+}
{+(Plus
(AugmentedAssignment
{ (Times
{-(Identifier)-}
{-(Integer)-})
->(Plus
{+(Identifier)+}
{+(Integer)+})+})+}
{+(Assignment
{+(Identifier)+}
{+(Integer)+}) })
{+(AugmentedAssignment
{+(LShift
{+(Identifier)+}
{+(Integer)+})+})+}
{+(Assignment
{+(Identifier)+}
{+(AugmentedAssignment
{+(RShift
{+(Identifier)+}
{+(Integer)+})+})+}
{+(Assignment
{+(Identifier)+}
{+(AugmentedAssignment
{+(DividedBy
{+(Identifier)+}
{+(Integer)+})+})+}
{+(Assignment
{+(Identifier)+}
{+(AugmentedAssignment
{+(BXOr
{+(Identifier)+}
{+(Integer)+})+})+}
{+(Assignment
{+(Identifier)+}
{+(AugmentedAssignment
{+(Modulo
{+(Identifier)+}
{+(Integer)+})+})+}
{+(Assignment
{+(Identifier)+}
{+(AugmentedAssignment
{+(Not
{+(BAnd
{+(Identifier)+}
@ -76,43 +66,31 @@
{+(KeyValue
{+(Identifier)+}
{+(Integer)+})+})+})+})+})+})+}
{-(Assignment
{-(Identifier)-}
{-(Times
{-(Identifier)-}
{-(Integer)-})-})-}
{-(Assignment
{-(Identifier)-}
{-(AugmentedAssignment
{-(Plus
{-(Identifier)-}
{-(Integer)-})-})-}
{-(Assignment
{-(Identifier)-}
{-(AugmentedAssignment
{-(LShift
{-(Identifier)-}
{-(Integer)-})-})-}
{-(Assignment
{-(Identifier)-}
{-(AugmentedAssignment
{-(RShift
{-(Identifier)-}
{-(Integer)-})-})-}
{-(Assignment
{-(Identifier)-}
{-(AugmentedAssignment
{-(DividedBy
{-(Identifier)-}
{-(Integer)-})-})-}
{-(Assignment
{-(Identifier)-}
{-(AugmentedAssignment
{-(BXOr
{-(Identifier)-}
{-(Integer)-})-})-}
{-(Assignment
{-(Identifier)-}
{-(AugmentedAssignment
{-(Modulo
{-(Identifier)-}
{-(Integer)-})-})-}
{-(Assignment
{-(Identifier)-}
{-(AugmentedAssignment
{-(Not
{-(BAnd
{-(Identifier)-}

29
test/fixtures/go/corpus/assignment-statements.parseA.txt vendored
View File

@ -7,10 +7,7 @@
(Assignment
(Identifier)
(Integer))
(Assignment
(Statements
(Identifier)
(Identifier))
(AugmentedAssignment
(Plus
(Statements
(Identifier)
@ -18,43 +15,35 @@
(Statements
(Integer)
(Integer))))
(Assignment
(Identifier)
(AugmentedAssignment
(Times
(Identifier)
(Integer)))
(Assignment
(Identifier)
(AugmentedAssignment
(Plus
(Identifier)
(Integer)))
(Assignment
(Identifier)
(AugmentedAssignment
(LShift
(Identifier)
(Integer)))
(Assignment
(Identifier)
(AugmentedAssignment
(RShift
(Identifier)
(Integer)))
(Assignment
(Identifier)
(AugmentedAssignment
(DividedBy
(Identifier)
(Integer)))
(Assignment
(Identifier)
(AugmentedAssignment
(BXOr
(Identifier)
(Integer)))
(Assignment
(Identifier)
(AugmentedAssignment
(Modulo
(Identifier)
(Integer)))
(Assignment
(Identifier)
(AugmentedAssignment
(Not
(BAnd
(Identifier)

29
test/fixtures/go/corpus/assignment-statements.parseB.txt vendored
View File

@ -7,10 +7,7 @@
(Assignment
(Identifier)
(Integer))
(Assignment
(Statements
(Identifier)
(Identifier))
(AugmentedAssignment
(Plus
(Statements
(Identifier)
@ -18,43 +15,35 @@
(Statements
(Integer)
(Integer))))
(Assignment
(Identifier)
(AugmentedAssignment
(Times
(Identifier)
(Integer)))
(Assignment
(Identifier)
(AugmentedAssignment
(Plus
(Identifier)
(Integer)))
(Assignment
(Identifier)
(AugmentedAssignment
(LShift
(Identifier)
(Integer)))
(Assignment
(Identifier)
(AugmentedAssignment
(RShift
(Identifier)
(Integer)))
(Assignment
(Identifier)
(AugmentedAssignment
(DividedBy
(Identifier)
(Integer)))
(Assignment
(Identifier)
(AugmentedAssignment
(BXOr
(Identifier)
(Integer)))
(Assignment
(Identifier)
(AugmentedAssignment
(Modulo
(Identifier)
(Integer)))
(Assignment
(Identifier)
(AugmentedAssignment
(Not
(BAnd
(Identifier)

36
test/fixtures/go/corpus/binary-expressions.diffA-B.txt vendored
View File

@ -20,16 +20,22 @@
->(Identifier) }
{ (Identifier)
->(Identifier) }))
{+(Equal
{+(Identifier)+}
{+(Identifier)+})+}
{+(Not
{+(Equal
{+(Identifier)+}
{+(Identifier)+})+})+}
{+(LessThan
{+(Identifier)+}
{+(Identifier)+})+}
(Equal
{ (Identifier)
->(Identifier) }
{ (Identifier)
->(Identifier) })
(Not
(Equal
{ (Identifier)
->(Identifier) }
{ (Identifier)
->(Identifier) }))
(LessThan
{ (Identifier)
->(Identifier) }
{ (Identifier)
->(Identifier) })
{+(LessThanEqual
{+(Identifier)+}
{+(Identifier)+})+}
@ -72,16 +78,6 @@
{+(BAnd
{+(Identifier)+}
{+(Identifier)+})+}
{-(Equal
{-(Identifier)-}
{-(Identifier)-})-}
{-(Not
{-(Equal
{-(Identifier)-}
{-(Identifier)-})-})-}
{-(LessThan
{-(Identifier)-}
{-(Identifier)-})-}
{-(LessThanEqual
{-(Identifier)-}
{-(Identifier)-})-}

36
test/fixtures/go/corpus/binary-expressions.diffB-A.txt vendored
View File

@ -20,16 +20,22 @@
->(Identifier) }
{ (Identifier)
->(Identifier) }))
{+(Equal
{+(Identifier)+}
{+(Identifier)+})+}
{+(Not
{+(Equal
{+(Identifier)+}
{+(Identifier)+})+})+}
{+(LessThan
{+(Identifier)+}
{+(Identifier)+})+}
(Equal
{ (Identifier)
->(Identifier) }
{ (Identifier)
->(Identifier) })
(Not
(Equal
{ (Identifier)
->(Identifier) }
{ (Identifier)
->(Identifier) }))
(LessThan
{ (Identifier)
->(Identifier) }
{ (Identifier)
->(Identifier) })
{+(LessThanEqual
{+(Identifier)+}
{+(Identifier)+})+}
@ -72,16 +78,6 @@
{+(BAnd
{+(Identifier)+}
{+(Identifier)+})+}
{-(Equal
{-(Identifier)-}
{-(Identifier)-})-}
{-(Not
{-(Equal
{-(Identifier)-}
{-(Identifier)-})-})-}
{-(LessThan
{-(Identifier)-}
{-(Identifier)-})-}
{-(LessThanEqual
{-(Identifier)-}
{-(Identifier)-})-}

10
test/fixtures/go/corpus/method-declarations.diffA-B.txt vendored
View File

@ -81,19 +81,13 @@
(Empty))))
(Method
(Statements
(Assignment
(MemberAccess
(Identifier)
(Identifier))
(AugmentedAssignment
(Times
(MemberAccess
(Identifier)
(Identifier))
(Identifier)))
(Assignment
(MemberAccess
(Identifier)
(Identifier))
(AugmentedAssignment
(Times
(MemberAccess
(Identifier)

10
test/fixtures/go/corpus/method-declarations.diffB-A.txt vendored
View File

@ -81,19 +81,13 @@
(Empty))))
(Method
(Statements
(Assignment
(MemberAccess
(Identifier)
(Identifier))
(AugmentedAssignment
(Times
(MemberAccess
(Identifier)
(Identifier))
(Identifier)))
(Assignment
(MemberAccess
(Identifier)
(Identifier))
(AugmentedAssignment
(Times
(MemberAccess
(Identifier)

10
test/fixtures/go/corpus/method-declarations.parseA.txt vendored
View File

@ -49,19 +49,13 @@
(Empty))))
(Method
(Statements
(Assignment
(MemberAccess
(Identifier)
(Identifier))
(AugmentedAssignment
(Times
(MemberAccess
(Identifier)
(Identifier))
(Identifier)))
(Assignment
(MemberAccess
(Identifier)
(Identifier))
(AugmentedAssignment
(Times
(MemberAccess
(Identifier)

10
test/fixtures/go/corpus/method-declarations.parseB.txt vendored
View File

@ -63,19 +63,13 @@
(Empty))))
(Method
(Statements
(Assignment
(MemberAccess
(Identifier)
(Identifier))
(AugmentedAssignment
(Times
(MemberAccess
(Identifier)
(Identifier))
(Identifier)))
(Assignment
(MemberAccess
(Identifier)
(Identifier))
(AugmentedAssignment
(Times
(MemberAccess
(Identifier)

18
test/fixtures/go/corpus/slice-expressions.diffA-B.txt vendored
View File

@ -27,14 +27,16 @@
->(Integer) }
{ (Empty)
->(Integer) })
(Slice
{ (Identifier)
->(Identifier) }
{ (Empty)
->(Integer) }
{ (Integer)
->(Integer) }
(Empty))
{+(Slice
{+(Identifier)+}
{+(Integer)+}
{+(Integer)+}
{+(Empty)+})+}
{-(Slice
{-(Identifier)-}
{-(Empty)-}
{-(Integer)-}
{-(Empty)-})-}
{-(Slice
{-(Identifier)-}
{-(Empty)-}

18
test/fixtures/go/corpus/slice-expressions.diffB-A.txt vendored
View File

@ -19,23 +19,21 @@
{+(Empty)+}
{+(Empty)+}
{+(Empty)+})+}
{+(Slice
{+(Identifier)+}
{+(Integer)+}
{+(Integer)+}
{+(Integer)+})+}
(Slice
{ (Identifier)
->(Identifier) }
(Integer)
{ (Empty)
->(Integer) }
{ (Empty)
->(Integer) })
(Slice
{ (Identifier)
->(Identifier) }
{ (Empty)
->(Integer) }
(Integer)
(Empty))
{-(Slice
{-(Identifier)-}
{-(Empty)-}
{-(Integer)-}
{-(Empty)-})-}
{-(Slice
{-(Identifier)-}
{-(Integer)-}

10
test/fixtures/javascript/corpus/export.diffA-B.txt vendored
View File

@ -35,15 +35,15 @@
(Identifier)
{ (Empty)
->(Identifier) })
{+(Assignment
{+(Empty)+}
{+(Identifier)+}
{+(Empty)+})+}
(Assignment
(Empty)
{ (Identifier)
->(Identifier) }
(Empty))))
(Empty))
{+(Assignment
{+(Empty)+}
{+(Identifier)+}
{+(Empty)+})+}))
(DefaultExport
{ (Identifier)
->(Identifier) })

13
test/fixtures/javascript/corpus/export.diffB-A.txt vendored
View File

@ -35,14 +35,11 @@
(Identifier)
{ (Identifier)
->(Empty) })
{+(Assignment
{+(Empty)+}
{+(Identifier)+}
{+(Empty)+})+}
{-(Assignment
{-(Empty)-}
{-(Identifier)-}
{-(Empty)-})-}
(Assignment
(Empty)
{ (Identifier)
->(Identifier) }
(Empty))
{-(Assignment
{-(Empty)-}
{-(Identifier)-}

4
test/fixtures/javascript/corpus/import.diffA-B.txt vendored
View File

@ -2,7 +2,8 @@
{+(Import)+}
{+(QualifiedAliasedImport
{+(Identifier)+})+}
{+(Import)+}
{ (Import)
->(Import) }
{+(Import)+}
{+(Import)+}
{+(Statements
@ -13,7 +14,6 @@
{+(QualifiedAliasedImport
{+(Identifier)+})+})+}
{+(SideEffectImport)+}
{-(Import)-}
{-(QualifiedAliasedImport
{-(Identifier)-})-}
{-(Import)-}

4
test/fixtures/javascript/corpus/import.diffB-A.txt vendored
View File

@ -2,7 +2,8 @@
{+(Import)+}
{+(QualifiedAliasedImport
{+(Identifier)+})+}
{+(Import)+}
{ (Import)
->(Import) }
{+(Import)+}
{+(Import)+}
{+(Statements
@ -13,7 +14,6 @@
{+(QualifiedAliasedImport
{+(Identifier)+})+})+}
{+(SideEffectImport)+}
{-(Import)-}
{-(QualifiedAliasedImport
{-(Identifier)-})-}
{-(Import)-}

3
test/fixtures/javascript/corpus/math-assignment-operator.diffA-B.txt vendored
View File

@ -1,6 +1,5 @@
(Statements
(Assignment
(Identifier)
(AugmentedAssignment
(Plus
(Identifier)
{ (Float)

3
test/fixtures/javascript/corpus/math-assignment-operator.diffB-A.txt vendored
View File

@ -1,6 +1,5 @@
(Statements
(Assignment
(Identifier)
(AugmentedAssignment
(Plus
(Identifier)
{ (Float)

3
test/fixtures/javascript/corpus/math-assignment-operator.parseA.txt vendored
View File

@ -1,6 +1,5 @@
(Statements
(Assignment
(Identifier)
(AugmentedAssignment
(Plus
(Identifier)
(Float))))

3
test/fixtures/javascript/corpus/math-assignment-operator.parseB.txt vendored
View File

@ -1,6 +1,5 @@
(Statements
(Assignment
(Identifier)
(AugmentedAssignment
(Plus
(Identifier)
(Float))))

10
test/fixtures/typescript/corpus/export.diffA-B.txt vendored
View File

@ -35,15 +35,15 @@
(Identifier)
{ (Empty)
->(Identifier) })
{+(Assignment
{+(Empty)+}
{+(Identifier)+}
{+(Empty)+})+}
(Assignment
(Empty)
{ (Identifier)
->(Identifier) }
(Empty))))
(Empty))
{+(Assignment
{+(Empty)+}
{+(Identifier)+}
{+(Empty)+})+}))
(DefaultExport
{ (Identifier)
->(Identifier) })

13
test/fixtures/typescript/corpus/export.diffB-A.txt vendored
View File

@ -35,14 +35,11 @@
(Identifier)
{ (Identifier)
->(Empty) })
{+(Assignment
{+(Empty)+}
{+(Identifier)+}
{+(Empty)+})+}
{-(Assignment
{-(Empty)-}
{-(Identifier)-}
{-(Empty)-})-}
(Assignment
(Empty)
{ (Identifier)
->(Identifier) }
(Empty))
{-(Assignment
{-(Empty)-}
{-(Identifier)-}

12
test/fixtures/typescript/corpus/import.diffA-B.txt vendored
View File

@ -1,12 +1,10 @@
(Statements
{+(Import)+}
{+(QualifiedAliasedImport
{+(Identifier)+})+}
{+(Import)+}
{ (Import)
->(Import) }
{ (QualifiedAliasedImport
{-(Identifier)-})
->(QualifiedAliasedImport
{+(Identifier)+}) }
{+(Import)+}
{+(Import)+}
{+(Import)+}
{+(Statements
{+(Import)+}
@ -16,6 +14,8 @@
{+(QualifiedAliasedImport
{+(Identifier)+})+})+}
{+(SideEffectImport)+}
{-(QualifiedAliasedImport
{-(Identifier)-})-}
{-(Import)-}
{-(Import)-}
{-(Import)-}

16
test/fixtures/typescript/corpus/import.diffB-A.txt vendored
View File

@ -1,12 +1,10 @@
(Statements
{+(Import)+}
{+(QualifiedAliasedImport
{+(Identifier)+})+}
{+(Import)+}
{ (Import)
->(Import) }
{ (QualifiedAliasedImport
{-(Identifier)-})
->(QualifiedAliasedImport
{+(Identifier)+}) }
{+(Import)+}
{+(Import)+}
{+(Import)+}
{+(Statements
{+(Import)+}
@ -16,8 +14,10 @@
{+(QualifiedAliasedImport
{+(Identifier)+})+})+}
{+(SideEffectImport)+}
{+(QualifiedAliasedImport
{+(Identifier)+})+}
{ (QualifiedAliasedImport
{-(Identifier)-})
->(QualifiedAliasedImport
{+(Identifier)+}) }
{-(Import)-}
{-(Import)-}
{-(Import)-}

3
test/fixtures/typescript/corpus/math-assignment-operator.diffA-B.txt vendored
View File

@ -1,6 +1,5 @@
(Statements
(Assignment
(Identifier)
(AugmentedAssignment
(Plus
(Identifier)
{ (Float)

3
test/fixtures/typescript/corpus/math-assignment-operator.diffB-A.txt vendored
View File

@ -1,6 +1,5 @@
(Statements
(Assignment
(Identifier)
(AugmentedAssignment
(Plus
(Identifier)
{ (Float)

3
test/fixtures/typescript/corpus/math-assignment-operator.parseA.txt vendored
View File

@ -1,6 +1,5 @@
(Statements
(Assignment
(Identifier)
(AugmentedAssignment
(Plus
(Identifier)
(Float))))

3
test/fixtures/typescript/corpus/math-assignment-operator.parseB.txt vendored
View File

@ -1,6 +1,5 @@
(Statements
(Assignment
(Identifier)
(AugmentedAssignment
(Plus
(Identifier)
(Float))))