Merge branch 'master' into json-output

README.md

@@ -96,7 +96,7 @@ Available options:
 
 ## Development
 
-`semantic` requires at least GHC 8.6.4 and Cabal 2.4. We strongly recommend using [`ghcup`][ghcup] to sandbox GHC versions, as GHC packages installed through your OS's package manager may not install statically-linked versions of the GHC boot libraries.
+`semantic` requires at least GHC 8.6.4 and Cabal 2.4. We strongly recommend using [`ghcup`][ghcup] to sandbox GHC versions, as GHC packages installed through your OS's package manager may not install statically-linked versions of the GHC boot libraries. `semantic` currently builds only on Unix systems; users of other operating systems may wish to use the [Docker images](https://github.com/github/semantic/packages/11609).
 
 We use `cabal's` [Nix-style local builds][nix] for development. To get started quickly:
 

semantic-ast/semantic-ast.cabal

@@ -41,9 +41,9 @@ library
   -- other-modules:
   -- other-extensions:
   build-depends:       base ^>=4.12.0.0
-                     , tree-sitter ^>= 0.5
+                     , tree-sitter ^>= 0.7
                      , semantic-source ^>= 0.0
-                     , tree-sitter-python ^>= 0.6
+                     , tree-sitter-python ^>= 0.8
                      , bytestring ^>= 0.10.8.2
                      , optparse-applicative ^>= 0.14.3.0
                      , pretty-simple ^>= 3.1.0.0
@@ -60,9 +60,9 @@ executable semantic-ast
   -- other-extensions:
   build-depends:       base ^>=4.12.0.0
                      , semantic-ast
-                     , tree-sitter ^>= 0.5
+                     , tree-sitter ^>= 0.7
                      , semantic-source ^>= 0.0
-                     , tree-sitter-python ^>= 0.6
+                     , tree-sitter-python ^>= 0.8
                      , bytestring ^>= 0.10.8.2
                      , optparse-applicative ^>= 0.14.3.0
                      , pretty-simple ^>= 3.1.0.0

semantic-java/semantic-java.cabal

@@ -27,8 +27,8 @@ library
     , fused-effects    ^>= 0.5
     , semantic-source  ^>= 0.0
     , semantic-tags    ^>= 0.0
-    , tree-sitter      ^>= 0.5
-    , tree-sitter-java ^>= 0.4
+    , tree-sitter      ^>= 0.7
+    , tree-sitter-java ^>= 0.6
   hs-source-dirs:      src
   default-language:    Haskell2010
   ghc-options:

semantic-json/semantic-json.cabal

@@ -24,8 +24,8 @@ library
   build-depends:
       base              >= 4.12 && < 5
     , semantic-tags    ^>= 0.0
-    , tree-sitter      ^>= 0.5
-    , tree-sitter-json ^>= 0.3
+    , tree-sitter      ^>= 0.7
+    , tree-sitter-json ^>= 0.5
   hs-source-dirs:      src
   default-language:    Haskell2010
   ghc-options:

semantic-python/semantic-python.cabal

@@ -28,8 +28,8 @@ common haskell
                      , semantic-source ^>= 0.0
                      , semantic-tags ^>= 0.0
                      , text ^>= 1.2.3
-                     , tree-sitter ^>= 0.5
-                     , tree-sitter-python ^>= 0.6
+                     , tree-sitter ^>= 0.7
+                     , tree-sitter-python ^>= 0.8
 
   ghc-options:
     -Weverything

semantic-python/src/Language/Python/Core.hs

@@ -317,6 +317,7 @@ instance Compile Py.Integer
 instance Compile Py.Lambda
 instance Compile Py.List
 instance Compile Py.ListComprehension
+instance Compile Py.ListSplat
 
 instance Compile Py.Module where
   compile it@Py.Module { Py.extraChildren = stmts } _cc =

semantic.cabal

@@ -1,7 +1,7 @@
 cabal-version:       2.4
 
 name:                semantic
-version:             0.9.0.0
+version:             0.10.0.0
 synopsis:            Framework and executable for analyzing and diffing untrusted code.
 description:         Semantic is a library for parsing, analyzing, and comparing source code across many languages.
 homepage:            http://github.com/github/semantic#readme
@@ -58,7 +58,7 @@ common dependencies
                      , fused-effects ^>= 0.5.0.0
                      , fused-effects-exceptions ^>= 0.2.0.0
                      , hashable ^>= 1.2.7.0
-                     , tree-sitter ^>= 0.5
+                     , tree-sitter ^>= 0.7
                      , mtl ^>= 2.2.2
                      , network ^>= 2.8.0.0
                      , pathtype ^>= 0.8.1
@@ -296,12 +296,12 @@ library
                      , unliftio-core ^>= 0.1.2.0
                      , unordered-containers ^>= 0.2.9.0
                      , vector ^>= 0.12.0.2
-                     , tree-sitter-go ^>= 0.2
+                     , tree-sitter-go ^>= 0.3
                      , tree-sitter-php ^>= 0.2
-                     , tree-sitter-python ^>= 0.6
-                     , tree-sitter-ruby ^>= 0.2
-                     , tree-sitter-typescript ^>= 0.2.1
-                     , tree-sitter-tsx ^>= 0.2.1
+                     , tree-sitter-python ^>= 0.8
+                     , tree-sitter-ruby ^>= 0.3.1
+                     , tree-sitter-typescript ^>= 0.3
+                     , tree-sitter-tsx ^>= 0.3
 
 executable semantic
   import:              haskell, dependencies, executable-flags
@@ -351,7 +351,7 @@ test-suite test
                      , Generators
                      , Properties
   build-depends:       semantic
-                     , tree-sitter-json ^>= 0.3
+                     , tree-sitter-json ^>= 0.5
                      , Glob ^>= 0.10.0
                      , hedgehog ^>= 1
                      , hspec >= 2.6 && <3

src/Language/Python/Assignment.hs

@@ -137,7 +137,7 @@ keywordArgument :: Assignment (Term Loc)
 keywordArgument = makeTerm <$> symbol KeywordArgument <*> children (Statement.Assignment [] <$> term expression <*> term expression)
 
 parenthesizedExpression :: Assignment (Term Loc)
-parenthesizedExpression = symbol ParenthesizedExpression *> children expressions
+parenthesizedExpression = (symbol ParenthesizedExpression <|> symbol ParenthesizedExpression') *> children expressions
 
 parameter :: Assignment (Term Loc)
 parameter =  makeTerm <$> symbol DefaultParameter <*> children (Statement.Assignment [] <$> term expression <*> term expression)

src/Language/Ruby/Assignment.hs

@@ -70,11 +70,15 @@ expressionChoices =
   , method
   , methodCall
   , mk Break Statement.Break
+  , mk Break' Statement.Break
   , mk Next Statement.Continue
+  , mk Next' Statement.Continue
   , mk Redo Statement.Retry
   , mk Retry Statement.Retry
   , mk Return Statement.Return
+  , mk Return' Statement.Return
   , mk Yield Statement.Yield
+  , mk Yield' Statement.Yield
   , module'
   , pair
   , parenthesizedExpressions
@@ -92,7 +96,7 @@ expressionChoices =
   , while'
   ]
   where
-    mk s construct = makeTerm <$> symbol s <*> children ((construct .) . fromMaybe <$> emptyTerm <*> optional (symbol ArgumentList *> children expressions))
+    mk s construct = makeTerm <$> symbol s <*> children ((construct .) . fromMaybe <$> emptyTerm <*> optional ((symbol ArgumentList <|> symbol ArgumentList') *> children expressions))
 
 expressions :: Assignment (Term Loc)
 expressions = makeTerm'' <$> location <*> many expression
@@ -315,10 +319,10 @@ pair :: Assignment (Term Loc)
 pair =   makeTerm <$> symbol Pair <*> children (Literal.KeyValue <$> expression <*> (expression <|> emptyTerm))
 
 args :: Assignment [Term Loc]
-args = (symbol ArgumentList <|> symbol ArgumentListWithParens) *> children (many expression) <|> many expression
+args = (symbol ArgumentList <|> symbol ArgumentList') *> children (many expression) <|> many expression
 
 methodCall :: Assignment (Term Loc)
-methodCall = makeTerm' <$> symbol MethodCall <*> children (require <|> load <|> send)
+methodCall = makeTerm' <$> (symbol MethodCall <|> symbol MethodCall') <*> children (require <|> load <|> send)
   where
     send = inject <$> ((regularCall <|> funcCall <|> scopeCall <|> dotCall) <*> optional block)
 
@@ -335,8 +339,8 @@ methodCall = makeTerm' <$> symbol MethodCall <*> children (require <|> load <|>
     load = inject <$ symbol Identifier <*> do
       s <- rawSource
       guard (s == "load")
-      (symbol ArgumentList <|> symbol ArgumentListWithParens) *> children (Ruby.Syntax.Load <$> expression <*> optional expression)
-    nameExpression = (symbol ArgumentList <|> symbol ArgumentListWithParens) *> children expression
+      (symbol ArgumentList <|> symbol ArgumentList') *> children (Ruby.Syntax.Load <$> expression <*> optional expression)
+    nameExpression = (symbol ArgumentList <|> symbol ArgumentList') *> children expression
 
 methodSelector :: Assignment (Term Loc)
 methodSelector = makeTerm <$> symbols <*> (Syntax.Identifier <$> (name <$> source))
@@ -465,8 +469,10 @@ invert term = makeTerm <$> location <*> fmap Expression.Not term
 
 -- | Match a term optionally preceded by comment(s), or a sequence of comments if the term is not present.
 term :: Assignment (Term Loc) -> Assignment (Term Loc)
-term term = contextualize comment term <|> makeTerm1 <$> (Syntax.Context <$> some1 (comment <|> heredocEnd) <*> emptyTerm)
-  where heredocEnd = makeTerm <$> symbol HeredocEnd <*> (Literal.TextElement <$> source)
+term term = contextualize comment term <|> makeTerm1 <$> (Syntax.Context <$> some1 (comment <|> heredocBody) <*> emptyTerm)
+  where
+    heredocBody = makeTerm <$> symbol HeredocBody <*> children (some (interpolation <|> escapeSequence <|> heredocEnd))
+    heredocEnd = makeTerm <$> symbol HeredocEnd <*> (Literal.TextElement <$> source)
 
 -- | Match a series of terms or comments until a delimiter is matched.
 manyTermsTill :: Assignment (Term Loc) -> Assignment b -> Assignment [Term Loc]

src/Semantic/CLI.hs

@@ -6,11 +6,11 @@ import           Control.Effect.Reader
 import           Control.Exception as Exc (displayException)
 import           Data.Blob
 import           Data.Blob.IO
+import qualified Data.Flag as Flag
 import           Data.Handle
 import qualified Data.Language as Language
 import           Data.List (intercalate)
 import           Data.Project
-import qualified Data.Flag as Flag
 import           Options.Applicative hiding (style)
 import           Prologue
 import           Semantic.Api hiding (File)
@@ -28,10 +28,10 @@ import qualified System.Path as Path
 import qualified System.Path.PartClass as Path.PartClass
 
 import Control.Concurrent (mkWeakThreadId, myThreadId)
-import Control.Exception (Exception(..), throwTo)
-import System.Posix.Signals
+import Control.Exception (Exception (..), throwTo)
+import Proto.Semantic_JSON ()
 import System.Mem.Weak (deRefWeak)
-import Proto.Semantic_JSON()
+import System.Posix.Signals
 
 newtype SignalException = SignalException Signal
   deriving (Show)
@@ -157,8 +157,10 @@ graphCommand = command "graph" (info graphArgumentsParser (progDesc "Compute a g
       case paths of
         (x:_) -> pure $! Project (takeDirectory x) blobs (Language.languageForFilePath x) mempty
         _     -> pure $! Project "/" mempty Language.Unknown mempty
+
+    allLanguages = intercalate "|" . fmap show $ [Language.Go .. maxBound]
     readProjectRecursively = makeReadProjectRecursivelyTask
-      <$> option auto (long "language" <> help "The language for the analysis.")
+      <$> option auto (long "language" <> help "The language for the analysis." <> metavar allLanguages)
       <*> optional (pathOption (long "root" <> help "Root directory of project. Optional, defaults to entry file/directory." <> metavar "DIR"))
       <*> many (pathOption (long "exclude-dir" <> help "Exclude a directory (e.g. vendor)" <> metavar "DIR"))
       <*> argument path (metavar "PATH")

test/fixtures/ruby/corpus/heredoc.diffA-B.txt

@@ -1,9 +1,10 @@
 (Statements
 {+(Send
-  {+(Send
-    {+(Identifier)+}
-    {+(TextElement)+}
+  {+(Context
     {+(Statements
+      {+(TextElement)+})+}
+    {+(Send
+      {+(Identifier)+}
       {+(TextElement)+})+})+}
   {+(Identifier)+})+}
 {-(TextElement)-}

test/fixtures/ruby/corpus/heredoc.diffB-A.txt

@@ -3,9 +3,10 @@
 {+(Statements
   {+(TextElement)+})+}
 {-(Send
-  {-(Send
-    {-(Identifier)-}
-    {-(TextElement)-}
+  {-(Context
     {-(Statements
+      {-(TextElement)-})-}
+    {-(Send
+      {-(Identifier)-}
       {-(TextElement)-})-})-}
   {-(Identifier)-})-})

test/fixtures/ruby/corpus/heredoc.parseB.txt

@@ -1,8 +1,9 @@
 (Statements
   (Send
-    (Send
-      (Identifier)
-      (TextElement)
+    (Context
       (Statements
+        (TextElement))
+      (Send
+        (Identifier)
         (TextElement)))
     (Identifier)))