Merge branch 'master' into analysis-effects

.github/workflows/haskell.yml

@@ -19,7 +19,7 @@ jobs:
 
     steps:
     - uses: actions/checkout@master
-      if: github.event.action == 'opened' || github.event.action == 'synchronize'
+      if: github.event.action == 'opened' || github.event.action == 'synchronize' || github.event.ref == 'refs/heads/master'
 
     - uses: actions/setup-haskell@v1
       name: Setup Haskell
@@ -31,13 +31,13 @@ jobs:
       name: Cache ~/.cabal/packages
       with:
         path: ~/.cabal/packages
-        key: ${{ runner.os }}-${{ matrix.ghc }}-cabal-packages
+        key: ${{ runner.os }}-${{ matrix.ghc }}-v1-cabal-packages
 
     - uses: actions/cache@v1
       name: Cache ~/.cabal/store
       with:
         path: ~/.cabal/store
-        key: ${{ runner.os }}-${{ matrix.ghc }}-cabal-store
+        key: ${{ runner.os }}-${{ matrix.ghc }}-v1-cabal-store
 
     - uses: actions/cache@v1
       name: Cache dist-newstyle

cabal.project

@@ -5,6 +5,7 @@ packages: .
           semantic-java
           semantic-json
           semantic-python
+          semantic-ruby
           semantic-tags
 
 jobs: $ncpus
@@ -42,3 +43,8 @@ source-repository-package
   type: git
   location: https://github.com/antitypical/fused-syntax.git
   tag: d11e14581217590a5c67f79cbaeee35ac8acee6a
+
+source-repository-package
+  type: git
+  location: https://github.com/fused-effects/fused-effects-readline.git
+  tag: 7a96949c77c73c6e5975c8d6171ffb63eb76b467

script/clone-example-repos

@@ -13,10 +13,10 @@
 set -e
 cd $(dirname "$0")/..
 
-mkdir -p test/examplerepos || true
-git clone --single-branch --recurse-submodules https://github.com/tree-sitter/haskell-tree-sitter.git tmp/haskell-tree-sitter || true
-
-dir="tmp/haskell-tree-sitter"
+# mkdir -p test/examplerepos || true
+# git clone --single-branch --recurse-submodules https://github.com/tree-sitter/haskell-tree-sitter.git tmp/haskell-tree-sitter || true
+mkdir -p tmp
+# dir="tmp/haskell-tree-sitter"
 
 # clone_repo LOCAL_PATH URL SHA
 function clone_repo {
@@ -36,7 +36,7 @@ function clone_repo {
   popd > /dev/null
 }
 
-python_examples="$dir/tree-sitter-python/vendor/tree-sitter-python/examples"
+python_examples="tmp/python-examples"
 clone_repo "$python_examples/numpy" numpy/numpy 058851c5cfc98f50f11237b1c13d77cfd1f40475
 clone_repo "$python_examples/thealgorithms" thealgorithms/python c6be53e1c43f870f5364eef1499ee1b411c966fb
 clone_repo "$python_examples/flask" pallets/flask 0b5b4a66ef99c8b91569dd9b9b34911834689d3f
@@ -48,26 +48,23 @@ clone_repo "$python_examples/scrapy" scrapy/scrapy 65d631329a1434ec013f24341e4b8
 clone_repo "$python_examples/pytorch" pytorch/pytorch c865d46736db4afff51690a712e35ed8e3899490
 clone_repo "$python_examples/certbot" certbot/certbot bb8222200a8cbd39a3ce9584ce6dfed6c5d05228
 
-ts_examples="$dir/tree-sitter-typescript/vendor/tree-sitter-typescript/examples"
+ts_examples="tmp/typescript-examples"
 clone_repo "$ts_examples/desktop" desktop/desktop d1324f56d02dd9afca5d2e9da545905a7d41d671
 clone_repo "$ts_examples/npm" npm/npm ee147fbbca6f2707d3b16f4fa78f4c4606b2d9b1
 
-# Java examples are disabled because the assignment code is not yet
-# robust and for reasons of CI celerity.
-
-java_examples="$dir/tree-sitter-java/vendor/tree-sitter-java/examples"
+# java_examples="$dir/tree-sitter-java/vendor/tree-sitter-java/examples"
 # clone_repo "$java_examples/elasticsearch" elastic/elasticsearch 4d62640bf116af7e825d89c7319a39c3f2f325b4
 # clone_repo "$java_examples/guava" google/guava e24fddc5fff7fd36d33ea38737b6606a7e476845
 # clone_repo "$java_examples/RxJava" ReactiveX/RxJava 8a6bf14fc9a61f7c1c0016ca217be02ca86211d2
 
-haskell_examples="$dir/tree-sitter-haskell/vendor/tree-sitter-haskell/examples"
-# clone_repo "$haskell_examples/effects" joshvera/effects 08f5f36f2600362685af593f4b327e933b60bf97
-# clone_repo "$haskell_examples/postgrest" PostgRest/postgrest f80cfbf165f951a062b3cbedac4556019905ca49
-# clone_repo "$haskell_examples/ivory" GaloisInc/ivory 3d00324ad1c113c7e70957ff6a6d636d271d0fc4
+# haskell_examples="$dir/tree-sitter-haskell/vendor/tree-sitter-haskell/examples"
+# # clone_repo "$haskell_examples/effects" joshvera/effects 08f5f36f2600362685af593f4b327e933b60bf97
+# # clone_repo "$haskell_examples/postgrest" PostgRest/postgrest f80cfbf165f951a062b3cbedac4556019905ca49
+# # clone_repo "$haskell_examples/ivory" GaloisInc/ivory 3d00324ad1c113c7e70957ff6a6d636d271d0fc4
 
-go_examples="$dir/tree-sitter-go/vendor/tree-sitter-go/examples"
-clone_repo "$go_examples/go" "golang/go" "870e12d7bfaea70fb0d743842f5864eb059cb939"
-clone_repo "$go_examples/moby" "moby/moby" "f57f260b49b6142366e6bc1274204ee0a1205945"
+# go_examples="$dir/tree-sitter-go/vendor/tree-sitter-go/examples"
+# clone_repo "$go_examples/go" "golang/go" "870e12d7bfaea70fb0d743842f5864eb059cb939"
+# clone_repo "$go_examples/moby" "moby/moby" "f57f260b49b6142366e6bc1274204ee0a1205945"
 
-ruby_examples="$dir/tree-sitter-ruby/vendor/tree-sitter-ruby/examples"
+ruby_examples="tmp/ruby-examples"
 clone_repo "$ruby_examples/ruby_spec" "ruby/spec" "c3e6b9017926f44a76e2b966c4dd35fa84c4cd3b"

script/fix-broken-cabal-store

@@ -1,5 +1,7 @@
 #!/bin/bash
 
-rm -rf ~/.cabal/store/ghc-8.6.5/tr-sttr*
-rm -rf ~/.cabal/store/ghc-8.6.5/lib/libHStr-sttr*
-rm -rf ~/.cabal/store/ghc-8.6.5/package.db/tr-sttr*
+store_dir="$HOME/.cabal/store/ghc-$(ghc --numeric-version)"
+
+rm -rf "$store_dir"/tr-sttr*
+rm -rf "$store_dir"/lib/libHStr-sttr*
+rm -rf "$store_dir"/package.db/tr-sttr*

script/ghci-flags

@@ -43,6 +43,7 @@ function flags {
   echo "-isemantic-java/src"
   echo "-isemantic-json/src"
   echo "-isemantic-python/src"
+  echo "-isemantic-ruby/src"
   echo "-isemantic-tags/src"
   echo "-iapp"
   echo "-isrc"

script/ghci-flags-dependencies

@@ -14,4 +14,5 @@ echo "semantic-core/semantic-core.cabal"
 echo "semantic-java/semantic-java.cabal"
 echo "semantic-json/semantic-json.cabal"
 echo "semantic-python/semantic-python.cabal"
+echo "semantic-ruby/semantic-ruby.cabal"
 echo "semantic-tags/semantic-tags.cabal"

semantic-analysis/semantic-analysis.cabal

@@ -57,17 +57,16 @@ library
     Analysis.ScopeGraph
     Analysis.Typecheck
     Control.Carrier.Fail.WithLoc
-    Control.Carrier.Readline.Haskeline
-    Control.Effect.Readline
   build-depends:
       algebraic-graphs            ^>= 0.3
     , base                         >= 4.13 && < 5
     , containers                  ^>= 0.6
     , fused-effects               ^>= 1.0
+    , fused-effects-readline
     , fused-syntax
     , haskeline                   ^>= 0.7.5
     , pathtype                    ^>= 0.8.1
-    , prettyprinter                >= 1.2.1 && < 1.4
+    , prettyprinter                >= 1.2 && < 2
     , prettyprinter-ansi-terminal ^>= 1.1.1
     , semantic-source             ^>= 0
     , terminal-size               ^>= 0.3

semantic-analysis/src/Control/Carrier/Readline/Haskeline.hs

@@ -1,64 +0,0 @@
-{-# LANGUAGE FlexibleInstances, GeneralizedNewtypeDeriving, MultiParamTypeClasses, ScopedTypeVariables, TypeApplications, TypeOperators, UndecidableInstances #-}
-module Control.Carrier.Readline.Haskeline
-( -- * Readline carrier
-  runReadline
-, runReadlineWithHistory
-, ReadlineC (..)
-  -- * Readline effect
-, module Control.Effect.Readline
-, runM
-) where
-
-import Control.Algebra
-import Control.Carrier.Lift
-import Control.Carrier.Reader
-import Control.Effect.Readline
-import Control.Monad.Fix
-import Control.Monad.IO.Class
-import Data.Coerce
-import Data.Text.Prettyprint.Doc
-import Data.Text.Prettyprint.Doc.Render.Terminal (renderIO)
-import System.Console.Haskeline hiding (Handler, handle)
-import System.Console.Terminal.Size as Size
-import System.Path ((</>))
-import qualified System.Path as Path
-import System.Path.Directory
-import System.IO (stdout)
-
-runReadline :: MonadException m => Prefs -> Settings m -> ReadlineC m a -> m a
-runReadline prefs settings = runInputTWithPrefs prefs (coerce settings) . runM . runReader (Line 0) . runReadlineC
-
-runReadlineWithHistory :: MonadException m => ReadlineC m a -> m a
-runReadlineWithHistory block = do
-  homeDir <- liftIO getHomeDirectory
-  prefs <- liftIO $ readPrefs (Path.toString (homeDir </> Path.relFile ".haskeline"))
-  let settingsDir = homeDir </> Path.relDir ".local" </> Path.relDir "semantic-core"
-      settings = Settings
-        { complete = noCompletion
-        , historyFile = Just (Path.toString (settingsDir </> Path.relFile "repl_history"))
-        , autoAddHistory = True
-        }
-  liftIO $ createDirectoryIfMissing True settingsDir
-
-  runReadline prefs settings block
-
-newtype ReadlineC m a = ReadlineC { runReadlineC :: ReaderC Line (LiftC (InputT m)) a }
-  deriving (Applicative, Functor, Monad, MonadFix, MonadIO)
-
-instance MonadException m => Algebra Readline (ReadlineC m) where
-  alg (Prompt prompt k) = ReadlineC $ do
-    str <- sendM (getInputLine @m (cyan <> prompt <> plain))
-    Line line <- ask
-    local increment (runReadlineC (k line str))
-    where cyan = "\ESC[1;36m\STX"
-          plain = "\ESC[0m\STX"
-  alg (Print doc k) = do
-    s <- maybe 80 Size.width <$> liftIO size
-    liftIO (renderIO stdout (layoutSmart defaultLayoutOptions { layoutPageWidth = AvailablePerLine s 0.8 } (doc <> line)))
-    k
-
-
-newtype Line = Line Int
-
-increment :: Line -> Line
-increment (Line n) = Line (n + 1)

semantic-analysis/src/Control/Effect/Readline.hs

@@ -1,32 +0,0 @@
-{-# LANGUAGE DeriveFunctor, DeriveGeneric, FlexibleContexts, MultiParamTypeClasses #-}
-module Control.Effect.Readline
-( -- * Readline effect
-  Readline (..)
-, prompt
-, print
-  -- * Re-exports
-, Algebra
-, Has
-, run
-) where
-
-import Control.Algebra
-import Data.Text.Prettyprint.Doc
-import Data.Text.Prettyprint.Doc.Render.Terminal
-import GHC.Generics (Generic1)
-import Prelude hiding (print)
-
-data Readline m k
-  = Prompt String (Int -> Maybe String -> m k)
-  | Print (Doc AnsiStyle) (m k)
-  deriving (Functor, Generic1)
-
-instance HFunctor Readline
-instance Effect Readline
-
-
-prompt :: Has Readline sig m => String -> m (Int, Maybe String)
-prompt p = send (Prompt p (curry pure))
-
-print :: Has Readline sig m => Doc AnsiStyle -> m ()
-print s = send (Print s (pure ()))

semantic-ast/semantic-ast.cabal

@@ -41,7 +41,7 @@ library
   -- other-modules:
   -- other-extensions:
   build-depends:       base ^>= 4.13
-                     , tree-sitter ^>= 0.7.1
+                     , tree-sitter ^>= 0.7.2
                      , semantic-source ^>= 0.0
                      , tree-sitter-python ^>= 0.8
                      , bytestring ^>= 0.10.8.2

semantic-ast/src/Main.hs

@@ -10,9 +10,7 @@ import Source.Range
 import Source.Span
 import Data.ByteString.Char8
 import Data.ByteString (readFile)
-import System.IO (FilePath)
 import Options.Applicative hiding (style)
-import Data.Semigroup ((<>))
 import Text.Pretty.Simple (pPrint, pPrintNoColor)
 import Data.Foldable (traverse_)
 import Control.Monad ((>=>))

semantic-core/semantic-core.cabal

@@ -51,7 +51,7 @@ library
     , fused-syntax
     , parsers                     ^>= 0.12.10
     , pathtype                    ^>= 0.8.1
-    , prettyprinter                >= 1.2.1 && < 1.4
+    , prettyprinter                >= 1.2.1 && < 2
     , prettyprinter-ansi-terminal ^>= 1.1.1
     , semantic-analysis           ^>= 0
     , semantic-source             ^>= 0

semantic-java/semantic-java.cabal

@@ -27,7 +27,7 @@ library
     , fused-effects    ^>= 1.0
     , semantic-source  ^>= 0.0
     , semantic-tags    ^>= 0.0
-    , tree-sitter      ^>= 0.7.1
+    , tree-sitter      ^>= 0.7.2
     , tree-sitter-java ^>= 0.6
   hs-source-dirs:      src
   default-language:    Haskell2010

semantic-json/semantic-json.cabal

@@ -24,7 +24,7 @@ library
   build-depends:
       base              >= 4.13 && < 5
     , semantic-tags    ^>= 0.0
-    , tree-sitter      ^>= 0.7
+    , tree-sitter      ^>= 0.7.2
     , tree-sitter-json ^>= 0.5
   hs-source-dirs:      src
   default-language:    Haskell2010

semantic-python/semantic-python.cabal

@@ -28,7 +28,7 @@ common haskell
                      , semantic-source ^>= 0.0
                      , semantic-tags ^>= 0.0
                      , text ^>= 1.2.3
-                     , tree-sitter ^>= 0.7
+                     , tree-sitter ^>= 0.7.2
                      , tree-sitter-python ^>= 0.8
 
   ghc-options:

semantic-python/src/Language/Python/Tags.hs

@@ -1,4 +1,4 @@
-{-# LANGUAGE AllowAmbiguousTypes, DataKinds, DisambiguateRecordFields, FlexibleContexts, FlexibleInstances, MultiParamTypeClasses, NamedFieldPuns, ScopedTypeVariables, TypeApplications, TypeFamilies, TypeOperators, UndecidableInstances #-}
+{-# LANGUAGE AllowAmbiguousTypes, DataKinds, DisambiguateRecordFields, FlexibleContexts, FlexibleInstances, MultiParamTypeClasses, NamedFieldPuns, OverloadedStrings, ScopedTypeVariables, TypeApplications, TypeFamilies, TypeOperators, UndecidableInstances #-}
 module Language.Python.Tags
 ( ToTags(..)
 ) where
@@ -46,6 +46,16 @@ type family ToTagsInstance t :: Strategy where
   ToTagsInstance Py.FunctionDefinition = 'Custom
   ToTagsInstance Py.ClassDefinition    = 'Custom
   ToTagsInstance Py.Call               = 'Custom
+
+  -- These built-in functions all get handled as calls
+  ToTagsInstance Py.AssertStatement    = 'Custom
+  ToTagsInstance Py.Await              = 'Custom
+  ToTagsInstance Py.DeleteStatement    = 'Custom
+  ToTagsInstance Py.ExecStatement      = 'Custom
+  ToTagsInstance Py.GlobalStatement    = 'Custom
+  ToTagsInstance Py.NonlocalStatement  = 'Custom
+  ToTagsInstance Py.PrintStatement     = 'Custom
+
   ToTagsInstance _                     = 'Generic
 
 
@@ -53,6 +63,40 @@ instance (ToTags l, ToTags r) => ToTagsBy 'Custom (l :+: r) where
   tags' (L1 l) = tags l
   tags' (R1 r) = tags r
 
+keywordFunctionCall
+  :: ( Has (Reader Source) sig m
+     , Has (Writer Tags.Tags) sig m
+     , Generic1 t
+     , Tags.GFoldable1 ToTags (Rep1 t)
+     )
+  => t Loc -> Loc -> Range -> Text -> m ()
+keywordFunctionCall t loc range name = do
+  src <- ask @Source
+  let sliced = slice src range
+  Tags.yield (Tag name Function loc (Tags.firstLine sliced) Nothing)
+  gtags t
+
+instance ToTagsBy 'Custom Py.AssertStatement where
+  tags' t@Py.AssertStatement { ann = loc@Loc { byteRange = range } } = keywordFunctionCall t loc range "assert"
+
+instance ToTagsBy 'Custom Py.Await where
+  tags' t@Py.Await { ann = loc@Loc { byteRange = range } } = keywordFunctionCall t loc range "await"
+
+instance ToTagsBy 'Custom Py.DeleteStatement where
+  tags' t@Py.DeleteStatement { ann = loc@Loc { byteRange = range } } = keywordFunctionCall t loc range "del"
+
+instance ToTagsBy 'Custom Py.ExecStatement where
+  tags' t@Py.ExecStatement { ann = loc@Loc { byteRange = range } } = keywordFunctionCall t loc range "exec"
+
+instance ToTagsBy 'Custom Py.GlobalStatement where
+  tags' t@Py.GlobalStatement { ann = loc@Loc { byteRange = range } } = keywordFunctionCall t loc range "global"
+
+instance ToTagsBy 'Custom Py.NonlocalStatement where
+  tags' t@Py.NonlocalStatement { ann = loc@Loc { byteRange = range } } = keywordFunctionCall t loc range "nonlocal"
+
+instance ToTagsBy 'Custom Py.PrintStatement where
+  tags' t@Py.PrintStatement { ann = loc@Loc { byteRange = range } } = keywordFunctionCall t loc range "print"
+
 instance ToTagsBy 'Custom Py.FunctionDefinition where
   tags' t@Py.FunctionDefinition
     { ann = loc@Loc { byteRange = Range { start } }
@@ -80,13 +124,21 @@ instance ToTagsBy 'Custom Py.ClassDefinition where
 instance ToTagsBy 'Custom Py.Call where
   tags' t@Py.Call
     { ann = loc@Loc { byteRange = range }
-    , function = Py.PrimaryExpression (Prj Py.Identifier { text = name })
-    } = do
+    , function = Py.PrimaryExpression expr
+    } = match expr
+      where
+        match expr = case expr of
+          (Prj Py.Attribute { attribute = Py.Identifier _ name }) -> yield name
+          (Prj (Py.Identifier _ name)) -> yield name
+          (Prj Py.Call { function = Py.PrimaryExpression expr' }) -> match expr' -- Nested call expression like this in Python represent creating an instance of a class and calling it: e.g. AClass()()
+          (Prj (Py.ParenthesizedExpression _ (Prj (Py.Expression (Prj (Py.PrimaryExpression expr')))))) -> match expr' -- Parenthesized expressions
+          _ -> gtags t
+        yield name = do
           src <- ask @Source
           let sliced = slice src range
           Tags.yield (Tag name Call loc (Tags.firstLine sliced) Nothing)
           gtags t
-  tags' t@Py.Call{} = gtags t
+
 
 docComment :: Source -> (Py.CompoundStatement :+: Py.SimpleStatement) Loc -> Maybe Text
 docComment src (R1 (Py.SimpleStatement (Prj Py.ExpressionStatement { extraChildren = L1 (Prj (Py.Expression (Prj (Py.PrimaryExpression (Prj Py.String { ann }))))) :|_ }))) = Just (toText (slice src (byteRange ann)))

semantic-ruby/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 GitHub
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

semantic-ruby/README.md

@@ -0,0 +1,3 @@
+# Semantic support for Ruby
+
+This package implements `semantic` support for Ruby using the `semantic-core` intermediate language.

semantic-ruby/Setup.hs

@@ -0,0 +1,2 @@
+import Distribution.Simple
+main = defaultMain

semantic-ruby/semantic-ruby.cabal

@@ -0,0 +1,54 @@
+cabal-version:       2.4
+
+name:                semantic-ruby
+version:             0.0.0.0
+synopsis:            Semantic support for Ruby.
+description:         Semantic support for Ruby using the semantic-core intermediate language.
+homepage:            https://github.com/github/semantic/tree/master/semantic-ruby#readme
+bug-reports:         https://github.com/github/semantic/issues
+license:             MIT
+license-file:        LICENSE
+author:              The Semantic authors
+maintainer:          opensource+semantic@github.com
+copyright:           (c) 2019 GitHub, Inc.
+category:            Language
+build-type:          Simple
+stability:           alpha
+extra-source-files:  README.md
+
+tested-with:         GHC == 8.6.5
+
+common haskell
+  default-language:    Haskell2010
+  build-depends:       base ^>= 4.13
+                     , fused-effects ^>= 1.0
+                     , fused-syntax
+                     , parsers ^>= 0.12.10
+                     , semantic-core ^>= 0.0
+                     , semantic-source ^>= 0.0
+                     , semantic-tags ^>= 0.0
+                     , text ^>= 1.2.3
+                     , tree-sitter ^>= 0.7.2
+                     , tree-sitter-ruby ^>= 0.4
+
+  ghc-options:
+    -Weverything
+    -Wno-missing-local-signatures
+    -Wno-missing-import-lists
+    -Wno-implicit-prelude
+    -Wno-safe
+    -Wno-unsafe
+    -Wno-name-shadowing
+    -Wno-monomorphism-restriction
+    -Wno-missed-specialisations
+    -Wno-all-missed-specialisations
+    -Wno-star-is-type
+  if (impl(ghc >= 8.8))
+    ghc-options: -Wno-missing-deriving-strategies
+
+library
+  import: haskell
+  exposed-modules:
+    Language.Ruby
+    Language.Ruby.Tags
+  hs-source-dirs: src

semantic-ruby/src/Language/Ruby.hs

@@ -0,0 +1,24 @@
+{-# LANGUAGE TypeApplications #-}
+
+-- | Semantic functionality for Ruby programs.
+module Language.Ruby
+( Term(..)
+, TreeSitter.Ruby.tree_sitter_ruby
+) where
+
+
+import Control.Carrier.State.Strict
+import Data.Text (Text)
+import qualified Language.Ruby.Tags as PyTags
+import qualified Tags.Tagging.Precise as Tags
+import qualified TreeSitter.Ruby (tree_sitter_ruby)
+import qualified TreeSitter.Ruby.AST as Rb
+import qualified TreeSitter.Unmarshal as TS
+
+newtype Term a = Term { getTerm :: Rb.Program a }
+
+instance TS.Unmarshal Term where
+  unmarshalNode node = Term <$> TS.unmarshalNode node
+
+instance Tags.ToTags Term where
+  tags src = Tags.runTagging src . evalState @[Text] [] . PyTags.tags . getTerm

semantic-ruby/src/Language/Ruby/Tags.hs

@@ -0,0 +1,303 @@
+{-# LANGUAGE AllowAmbiguousTypes, DataKinds, DisambiguateRecordFields, FlexibleContexts, FlexibleInstances, MultiParamTypeClasses, NamedFieldPuns, OverloadedStrings, PartialTypeSignatures, ScopedTypeVariables, TypeApplications, TypeFamilies, TypeOperators, UndecidableInstances #-}
+module Language.Ruby.Tags
+( ToTags(..)
+) where
+
+import           AST.Element
+import           Control.Effect.Reader
+import           Control.Effect.State
+import           Control.Effect.Writer
+import           Control.Monad
+import           Data.Monoid (Ap (..))
+import           Data.Foldable
+import           Data.Text as Text
+import           GHC.Generics
+import           Source.Loc
+import           Source.Source as Source
+import           Tags.Tag
+import qualified Tags.Tagging.Precise as Tags
+import qualified TreeSitter.Ruby.AST as Rb
+
+class ToTags t where
+  tags
+    :: ( Has (Reader Source) sig m
+       , Has (Writer Tags.Tags) sig m
+       , Has (State [Text]) sig m
+       )
+    => t Loc
+    -> m ()
+
+instance (ToTagsBy strategy t, strategy ~ ToTagsInstance t) => ToTags t where
+  tags = tags' @strategy
+
+
+class ToTagsBy (strategy :: Strategy) t where
+  tags'
+    :: ( Has (Reader Source) sig m
+       , Has (Writer Tags.Tags) sig m
+       , Has (State [Text]) sig m
+       )
+    => t Loc
+    -> m ()
+
+
+data Strategy = Generic | Custom
+
+type family ToTagsInstance t :: Strategy where
+  ToTagsInstance (_ :+: _)             = 'Custom
+  ToTagsInstance Rb.Class              = 'Custom
+  ToTagsInstance Rb.SingletonClass     = 'Custom
+  ToTagsInstance Rb.Module             = 'Custom
+
+  ToTagsInstance Rb.Method             = 'Custom
+  ToTagsInstance Rb.SingletonMethod    = 'Custom
+
+  ToTagsInstance Rb.Call               = 'Custom
+  ToTagsInstance Rb.Lhs                = 'Custom
+  ToTagsInstance Rb.MethodCall         = 'Custom
+  ToTagsInstance Rb.Alias              = 'Custom
+  ToTagsInstance Rb.Undef              = 'Custom
+
+  -- Along with class, module, and method definitions, these introduce new lexical scopes for locals
+  ToTagsInstance Rb.Block              = 'Custom
+  ToTagsInstance Rb.DoBlock            = 'Custom
+  ToTagsInstance Rb.Lambda             = 'Custom
+
+  -- Parameters and assignment introduce locals
+  ToTagsInstance Rb.MethodParameters   = 'Custom
+  ToTagsInstance Rb.LambdaParameters   = 'Custom
+  ToTagsInstance Rb.BlockParameters    = 'Custom
+  ToTagsInstance Rb.Assignment         = 'Custom
+
+  ToTagsInstance _                     = 'Generic
+
+instance (ToTags l, ToTags r) => ToTagsBy 'Custom (l :+: r) where
+  tags' (L1 l) = tags l
+  tags' (R1 r) = tags r
+
+-- These are all valid, but point to methods in Kernel and other parts of the
+-- Ruby stdlib. A la carte displays some of these, but not others and since we
+-- have nothing to link to yet (can't jump-to-def), we hide them from the
+-- current tags output.
+nameBlacklist :: [Text]
+nameBlacklist =
+  [ "alias"
+  , "load"
+  , "require_relative"
+  , "require"
+  , "super"
+  , "undef"
+  , "__FILE__"
+  , "lambda"
+  ]
+
+yieldTag :: (Has (Reader Source) sig m, Has (Writer Tags.Tags) sig m) => Text -> Kind -> Loc -> Range -> m ()
+yieldTag name Call _ _ | name `elem` nameBlacklist = pure ()
+yieldTag name kind loc range = do
+  src <- ask @Source
+  let sliced = slice src range
+  Tags.yield (Tag name kind loc (Tags.firstLine sliced) Nothing)
+
+instance ToTagsBy 'Custom Rb.Class where
+  tags' t@Rb.Class
+    { ann = loc@Loc { byteRange = range }
+    , name = expr
+    } = enterScope True $ case expr of
+      Prj Rb.Constant { text } -> yield text
+      Prj Rb.ScopeResolution { name = Prj Rb.Constant { text } } -> yield text
+      Prj Rb.ScopeResolution { name = Prj Rb.Identifier { text } } -> yield text
+      _ -> gtags t
+    where
+      yield name = yieldTag name Class loc range >> gtags t
+
+instance ToTagsBy 'Custom Rb.SingletonClass where
+  tags' t@Rb.SingletonClass
+    { ann = loc@Loc { byteRange = range }
+    , value = Rb.Arg expr
+    } = enterScope True $ case expr of
+      Prj (Rb.Primary (Prj (Rb.Lhs (Prj (Rb.Variable (Prj Rb.Constant { text })))))) -> yield text
+      Prj (Rb.Primary (Prj (Rb.Lhs (Prj Rb.ScopeResolution { name = Prj Rb.Constant { text } })))) -> yield text
+      Prj (Rb.Primary (Prj (Rb.Lhs (Prj Rb.ScopeResolution { name = Prj Rb.Identifier { text } })))) -> yield text
+      _ -> gtags t
+    where
+      yield name = yieldTag name Class loc range >> gtags t
+
+instance ToTagsBy 'Custom Rb.Module where
+  tags' t@Rb.Module
+    { ann = loc@Loc { byteRange = range }
+    , name = expr
+    } = enterScope True $ case expr of
+      Prj Rb.Constant { text = name } -> yield name
+      Prj Rb.ScopeResolution { name = Prj Rb.Constant { text = name } } -> yield name
+      Prj Rb.ScopeResolution { name = Prj Rb.Identifier { text = name } } -> yield name
+      _ -> gtags t
+    where
+      yield name = yieldTag name Module loc range >> gtags t
+
+yieldMethodNameTag
+  :: ( Has (State [Text]) sig m
+     , Has (Reader Source) sig m
+     , Has (Writer Tags.Tags) sig m
+     , Generic1 t
+     , Tags.GFoldable1 ToTags (Rep1 t)
+     ) => t Loc -> Loc -> Range -> Rb.MethodName Loc -> m ()
+yieldMethodNameTag t loc range (Rb.MethodName expr) = enterScope True $ case expr of
+  Prj Rb.Identifier { text = name } -> yield name
+  Prj Rb.Constant { text = name } -> yield name
+  -- Prj Rb.ClassVariable { text = name } -> yield name
+  Prj Rb.Operator { text = name } -> yield name
+  -- Prj Rb.GlobalVariable { text = name } -> yield name
+  -- Prj Rb.InstanceVariable { text = name } -> yield name
+  Prj Rb.Setter { extraChildren = Rb.Identifier { text = name } } -> yield (name <> "=") -- NB: Matches existing tags output, TODO: Remove this.
+  -- TODO: Should we report symbol method names as tags?
+  -- Prj Rb.Symbol { extraChildren = [Prj Rb.EscapeSequence { text = name }] } -> yield name
+  _ -> gtags t
+  where
+    yield name = yieldTag name Function loc range >> gtags t
+
+enterScope :: (Has (State [Text]) sig m) => Bool -> m () -> m ()
+enterScope createNew m = do
+  locals <- get @[Text]
+  when createNew $ put @[Text] [] -- NB: Matches existing behavior in assignment, not necessarily correct
+  m
+  put locals
+
+instance ToTagsBy 'Custom Rb.Method where
+  tags' t@Rb.Method
+    { ann = loc@Loc { byteRange = range }
+    , name = expr
+    } = yieldMethodNameTag t loc range expr
+
+instance ToTagsBy 'Custom Rb.SingletonMethod where
+  tags' t@Rb.SingletonMethod
+    { ann = loc@Loc { byteRange = range }
+    , name = expr
+    } = yieldMethodNameTag t loc range expr
+
+instance ToTagsBy 'Custom Rb.Block where
+  tags' = enterScope False . gtags
+
+instance ToTagsBy 'Custom Rb.DoBlock where
+  tags' = enterScope False . gtags
+
+instance ToTagsBy 'Custom Rb.Lambda where
+  tags' = enterScope False . gtags
+
+instance ToTagsBy 'Custom Rb.Call where
+  tags' t@Rb.Call
+    { ann = loc@Loc { byteRange = range }
+    , method = expr
+    } = case expr of
+      Prj Rb.Identifier { text = name } -> yield name Call
+      Prj Rb.Constant { text = name } -> yield name Call -- TODO: Should be Constant
+      Prj Rb.Operator { text = name } -> yield name Call
+      _ -> gtags t
+    where
+      yield name kind = yieldTag name kind loc range >> gtags t
+
+instance ToTagsBy 'Custom Rb.Lhs where
+  tags' t@(Rb.Lhs expr) = case expr of
+    Prj (Rb.Variable (Prj Rb.Identifier { ann = loc@Loc { byteRange }, text })) -> yield text Call loc byteRange
+    Prj Rb.ScopeResolution { ann = loc@Loc { byteRange }, name = Prj Rb.Identifier { text } } -> yield text Call loc byteRange
+    -- TODO: These would be great to track, but doesn't match current a la carte tags output
+    -- Prj (Rb.Variable (Prj Rb.Constant { ann = loc@Loc { byteRange }, text })) -> yield text Constant loc byteRange
+    -- Prj Rb.ScopeResolution { ann = loc@Loc { byteRange }, name = Prj Rb.Constant { text } } -> yield text Constant loc byteRange
+    _ -> gtags t
+    where
+      yield name kind loc range = do
+        locals <- get @[Text]
+        unless (name `elem` locals) $ yieldTag name kind loc range
+        gtags t
+
+instance ToTagsBy 'Custom Rb.MethodCall where
+  tags' t@Rb.MethodCall
+    { ann = loc@Loc { byteRange = range }
+    , method = expr
+    } = case expr of
+      Prj (Rb.Variable (Prj Rb.Identifier { text = name })) -> yield name Call
+      Prj (Rb.Variable (Prj Rb.Constant { text = name })) -> yield name Call -- TODO: Should be Constant
+      Prj Rb.ScopeResolution { name = Prj Rb.Constant { text } } -> yield text Call
+      Prj Rb.ScopeResolution { name = Prj Rb.Identifier { text } } -> yield text Call -- TODO: Should be Constant
+      _ -> gtags t
+    where
+      yield name kind = yieldTag name kind loc range >> gtags t
+
+instance ToTagsBy 'Custom Rb.Alias where
+  tags' t@Rb.Alias
+    { ann = loc@Loc { byteRange = range }
+    , alias = Rb.MethodName aliasExpr
+    , name = Rb.MethodName nameExpr
+    } = do
+      case aliasExpr of
+        Prj Rb.Identifier { text } -> yieldTag text Function loc range
+        _ -> tags aliasExpr
+      case nameExpr of
+        Prj Rb.Identifier { text } -> yieldTag text Call loc range
+        _ -> tags nameExpr
+      gtags t
+
+instance ToTagsBy 'Custom Rb.Undef where
+  tags' t@Rb.Undef
+    { ann = loc@Loc { byteRange = range }
+    , extraChildren
+    } = for_ extraChildren $ \(Rb.MethodName expr) -> do
+      case expr of
+        Prj Rb.Identifier { text } -> yieldTag text Call loc range
+        _ -> tags expr
+      gtags t
+
+introduceLocals
+  :: ( Has (Reader Source) sig m
+     , Has (Writer Tags.Tags) sig m
+     , Has (State [Text]) sig m
+     )
+  => [((Rb.BlockParameter :+: Rb.DestructuredParameter :+: Rb.HashSplatParameter) :+:
+      ((Rb.Identifier :+: Rb.KeywordParameter) :+: (Rb.OptionalParameter :+: Rb.SplatParameter)))
+      Loc ]
+  -> m ()
+introduceLocals params = for_ params $ \param -> case param of
+  Prj Rb.BlockParameter { name = Rb.Identifier { text = lvar } } -> modify (lvar :)
+  Prj Rb.DestructuredParameter { extraChildren } -> introduceLocals extraChildren
+  Prj Rb.HashSplatParameter { name = Just Rb.Identifier { text = lvar } } -> modify (lvar :)
+  Prj Rb.Identifier { text = lvar } -> modify (lvar :)
+  Prj Rb.KeywordParameter { name = Rb.Identifier { text = lvar }} -> modify (lvar :)
+  Prj Rb.OptionalParameter { name = Rb.Identifier { text = lvar }} -> modify (lvar :)
+  Prj Rb.SplatParameter { name = Just Rb.Identifier { text = lvar } } -> modify (lvar :)
+  _ -> tags param
+
+instance ToTagsBy 'Custom Rb.MethodParameters where
+  tags' Rb.MethodParameters{ extraChildren } = introduceLocals extraChildren
+
+instance ToTagsBy 'Custom Rb.LambdaParameters where
+  tags' Rb.LambdaParameters{ extraChildren } = introduceLocals extraChildren
+
+instance ToTagsBy 'Custom Rb.BlockParameters where
+  tags' Rb.BlockParameters{ extraChildren } = introduceLocals extraChildren
+
+instance ToTagsBy 'Custom Rb.Assignment where
+  tags' t@Rb.Assignment{ left } = do
+    case left of
+      Prj (Rb.Lhs (Prj (Rb.Variable (Prj Rb.Identifier { text })))) -> modify (text :)
+      Prj Rb.LeftAssignmentList { extraChildren } -> introduceLhsLocals extraChildren
+      _ -> tags left
+    gtags t
+    where
+      introduceLhsLocals xs = for_ xs $ \x -> case x of
+        Prj (Rb.Lhs (Prj (Rb.Variable (Prj Rb.Identifier { text })))) -> modify (text :)
+        Prj Rb.DestructuredLeftAssignment { extraChildren } -> introduceLhsLocals extraChildren
+        Prj Rb.RestAssignment { extraChildren = Just (Rb.Lhs (Prj (Rb.Variable (Prj Rb.Identifier { text })))) } -> modify (text :)
+        _ -> tags x
+
+gtags
+  :: ( Has (Reader Source) sig m
+     , Has (Writer Tags.Tags) sig m
+     , Has (State [Text]) sig m
+     , Generic1 t
+     , Tags.GFoldable1 ToTags (Rep1 t)
+     )
+  => t Loc
+  -> m ()
+gtags = getAp . Tags.gfoldMap1 @ToTags (Ap . tags) . from1
+
+instance (Generic1 t, Tags.GFoldable1 ToTags (Rep1 t)) => ToTagsBy 'Generic t where
+  tags' = gtags

semantic-tags/src/Tags/Tag.hs

@@ -16,9 +16,12 @@ data Tag = Tag
   deriving (Eq, Show)
 
 data Kind
+  -- Definitions
   = Function
   | Method
   | Class
   | Module
+  -- References
   | Call
+  -- | Constant -- TODO: New kind for constant references
   deriving (Bounded, Enum, Eq, Show)

semantic.cabal

@@ -59,7 +59,7 @@ common dependencies
                      , fused-effects-exceptions ^>= 1
                      , fused-effects-resumable ^>= 0.1
                      , hashable >= 1.2.7 && < 1.4
-                     , tree-sitter ^>= 0.7.1
+                     , tree-sitter ^>= 0.7.2
                      , mtl ^>= 2.2.2
                      , network ^>= 2.8.0.0
                      , pathtype ^>= 0.8.1
@@ -278,7 +278,7 @@ library
                      , optparse-applicative >= 0.14.3 && < 0.16
                      , parallel ^>= 3.2.2.0
                      , parsers ^>= 0.12.9
-                     , prettyprinter >= 1.2.1 && < 1.4
+                     , prettyprinter >= 1.2 && < 2
                      , pretty-show ^>= 1.9.5
                      , profunctors ^>= 5.3
                      , proto-lens >= 0.5 && < 0.7
@@ -288,6 +288,7 @@ library
                      , semantic-java ^>= 0
                      , semantic-json ^>= 0
                      , semantic-python ^>= 0
+                     , semantic-ruby ^>= 0
                      , semantic-tags ^>= 0
                      , semigroupoids ^>= 5.3.2
                      , split ^>= 0.2.3.3
@@ -298,12 +299,12 @@ library
                      , unliftio-core ^>= 0.1.2.0
                      , unordered-containers ^>= 0.2.9.0
                      , vector ^>= 0.12.0.2
-                     , tree-sitter-go ^>= 0.3
+                     , tree-sitter-go ^>= 0.4
                      , tree-sitter-php ^>= 0.2
                      , tree-sitter-python ^>= 0.8
-                     , tree-sitter-ruby ^>= 0.3.1
-                     , tree-sitter-typescript ^>= 0.3
-                     , tree-sitter-tsx ^>= 0.3
+                     , tree-sitter-ruby ^>= 0.4
+                     , tree-sitter-typescript ^>= 0.4
+                     , tree-sitter-tsx ^>= 0.4
 
 executable semantic
   import:              haskell, dependencies, executable-flags
@@ -376,6 +377,7 @@ test-suite parse-examples
   build-depends:       semantic
                      , Glob
                      , foldl ^>= 1.4.5
+                     , lens >= 4.17 && < 4.19
                      , resourcet ^>= 1.2
                      , streaming
                      , streaming-bytestring ^>= 0.1.6

src/Assigning/Assignment.hs

@@ -145,9 +145,8 @@ location = tracing Loc `Then` pure
 getLocals :: HasCallStack => Assignment ast grammar [Text]
 getLocals = tracing GetLocals `Then` pure
 
-putLocals :: (HasCallStack, Enum grammar, Eq1 ast, Ix grammar) => [Text] -> Assignment ast grammar ()
-putLocals l = (tracing (PutLocals l) `Then` pure)
-          <|> (tracing End `Then` pure)
+putLocals :: HasCallStack => [Text] -> Assignment ast grammar ()
+putLocals l = tracing (PutLocals l) `Then` pure
 
 -- | Zero-width production of the current node.
 currentNode :: HasCallStack => Assignment ast grammar (TermF ast (Node grammar) ())
@@ -242,8 +241,6 @@ runAssignment source = \ assignment state -> go assignment state >>= requireExha
         run yield t initialState = state `seq` maybe (anywhere Nothing) atNode (listToMaybe stateNodes)
           where atNode (Term (In node f)) = case runTracing t of
                   Loc -> yield (nodeLocation node) state
-                  GetLocals -> yield stateLocals state
-                  PutLocals l -> yield () (state { stateLocals = l })
                   CurrentNode -> yield (In node (() <$ f)) state
                   Source -> yield (Source.bytes (Source.slice source (nodeByteRange node))) (advanceState state)
                   Children child -> do
@@ -253,6 +250,8 @@ runAssignment source = \ assignment state -> go assignment state >>= requireExha
                   _ -> anywhere (Just node)
 
                 anywhere node = case runTracing t of
+                  GetLocals -> yield stateLocals state
+                  PutLocals l -> yield () (state { stateLocals = l })
                   End -> requireExhaustive (tracingCallSite t) ((), state) >>= uncurry yield
                   Loc -> yield (L.Loc (Range stateOffset stateOffset) (Span statePos statePos)) state
                   Many rule -> fix (\ recur state -> (go rule state >>= \ (a, state') -> first (a:) <$> if state == state' then pure ([], state') else recur state') `catchError` const (pure ([], state))) state >>= uncurry yield

src/Data/Abstract/AccessControls/Instances.hs

@@ -117,6 +117,7 @@ instance AccessControls1 Literal.Null
 instance AccessControls1 Literal.KeyValue
 
 instance AccessControls1 Statement.Assignment
+instance AccessControls1 Statement.AugmentedAssignment
 instance AccessControls1 Statement.Break
 instance AccessControls1 Statement.Catch
 instance AccessControls1 Statement.Continue

src/Data/Language.hs

@@ -150,14 +150,18 @@ textToLanguage = \case
   _ -> Unknown
 
 
-newtype PerLanguageModes = PerLanguageModes
+data PerLanguageModes = PerLanguageModes
   { pythonMode :: LanguageMode
+  , rubyMode   :: LanguageMode
+  -- , typescriptMode :: LanguageMode
   }
   deriving (Eq, Ord, Show)
 
 defaultLanguageModes :: PerLanguageModes
 defaultLanguageModes = PerLanguageModes
   { pythonMode = ALaCarte
+  , rubyMode = ALaCarte
+  -- , typescriptMode = ALaCarte
   }
 
 data LanguageMode

src/Data/Syntax/Statement.hs

@@ -131,6 +131,21 @@ instance Evaluatable Let where
     unit
 
 
+-- AugmentedAssignment
+
+newtype AugmentedAssignment a = AugmentedAssignment { augmentedAssignmentTarget :: a }
+  deriving (Diffable, Foldable, FreeVariables1, Functor, Generic1, Hashable1, ToJSONFields1, Traversable)
+
+instance Eq1 AugmentedAssignment where liftEq = genericLiftEq
+instance Ord1 AugmentedAssignment where liftCompare = genericLiftCompare
+instance Show1 AugmentedAssignment where liftShowsPrec = genericLiftShowsPrec
+
+instance Declarations1 AugmentedAssignment where
+  liftDeclaredName declaredName AugmentedAssignment{..} = declaredName augmentedAssignmentTarget
+
+instance Evaluatable AugmentedAssignment
+
+
 -- Assignment
 
 -- | Assignment to a variable or other lvalue.

src/Language/Python/Assignment.hs

@@ -288,7 +288,7 @@ assignment' =  makeAssignment <$> symbol Assignment <*> children ((,,) <$> term
   where rvalue = expressionList <|> assignment' <|> yield <|> emptyTerm
         makeAssignment loc (lhs, maybeType, rhs) = makeTerm loc (Statement.Assignment (maybeToList maybeType) lhs rhs)
         assign :: (f :< Python.Syntax) => (Term Loc -> Term Loc -> f (Term Loc)) -> Term Loc -> Term Loc -> Sum Python.Syntax (Term Loc)
-        assign c l r = inject (Statement.Assignment [] l (makeTerm1 (c l r)))
+        assign c l r = inject (Statement.AugmentedAssignment (makeTerm1 (c l r)))
 
 yield :: Assignment (Term Loc)
 yield = makeTerm <$> symbol Yield <*> (Statement.Yield <$> children (term ( expression <|> emptyTerm )))

src/Language/Python/Term.hs

@@ -86,6 +86,7 @@ type Syntax =
   , Python.Syntax.QualifiedAliasedImport
   , Python.Syntax.Redirect
   , Statement.Assignment
+  , Statement.AugmentedAssignment
   , Statement.Break
   , Statement.Catch
   , Statement.Continue

src/Language/Ruby/Assignment.hs

@@ -32,6 +32,7 @@ import qualified Data.Syntax.Directive as Directive
 import qualified Data.Syntax.Expression as Expression
 import qualified Data.Syntax.Literal as Literal
 import qualified Data.Syntax.Statement as Statement
+import qualified Data.Text as Text
 import qualified Language.Ruby.Syntax as Ruby.Syntax
 import           Language.Ruby.Term as Ruby
 import           TreeSitter.Ruby as Grammar
@@ -93,6 +94,7 @@ expressionChoices =
   , unless
   , until'
   , while'
+  , do'
   ]
   where
     mk s construct = makeTerm <$> symbol s <*> children ((construct .) . fromMaybe <$> emptyTerm <*> optional ((symbol ArgumentList <|> symbol ArgumentList') *> children expressions))
@@ -128,13 +130,15 @@ identifier =
   <|> mk Setter
   <|> mk SplatArgument
   <|> mk HashSplatArgument
-  <|> mk BlockArgument
   <|> mk Uninterpreted
+  <|> symbol BlockArgument *> children expression
   where
     mk s = makeTerm <$> symbol s <*> (Syntax.Identifier . name <$> source)
     zsuper = makeTerm <$> symbol Super <*> (Ruby.Syntax.ZSuper <$ source)
     vcallOrLocal = do
-      (loc, ident, locals) <- identWithLocals
+      loc <- symbol Identifier
+      ident <- source
+      locals <- getLocals
       case ident of
         "__FILE__" -> pure $ makeTerm loc Directive.File
         "__LINE__" -> pure $ makeTerm loc Directive.Line
@@ -294,6 +298,9 @@ while' =
       makeTerm <$> symbol While         <*> children      (Statement.While <$> expression <*> expressions)
   <|> makeTerm <$> symbol WhileModifier <*> children (flip Statement.While <$> expression <*> expression)
 
+do' :: Assignment (Term Loc)
+do' = symbol Do *> children expressions
+
 until' :: Assignment (Term Loc)
 until' =
       makeTerm <$> symbol Until         <*> children      (Statement.While <$> invert expression <*> expressions)
@@ -388,26 +395,26 @@ assignment' = makeTerm  <$> symbol Assignment         <*> children (Ruby.Syntax.
                 ])
   where
     assign :: (f :< Ruby.Syntax) => (Term Loc -> Term Loc -> f (Term Loc)) -> Term Loc -> Term Loc -> Sum Ruby.Syntax (Term Loc)
-    assign c l r = inject (Ruby.Syntax.Assignment [] l (makeTerm1 (c l r)))
+    assign c l r = inject (Statement.AugmentedAssignment (makeTerm1 (c l r)))
 
     lhs  = makeTerm <$> symbol LeftAssignmentList  <*> children (many expr) <|> expr
     rhs  = makeTerm <$> symbol RightAssignmentList <*> children (many expr) <|> expr
-    expr = makeTerm <$> symbol RestAssignment      <*> (Syntax.Identifier . name <$> source)
+    expr = makeTerm <$> symbol RestAssignment      <*> restAssign
        <|> makeTerm <$> symbol DestructuredLeftAssignment <*> children (many expr)
        <|> lhsIdent
        <|> expression
 
-identWithLocals :: Assignment (Loc, Text, [Text])
-identWithLocals = do
-  loc <- symbol Identifier
-  -- source advances, so it's important we call getLocals first
+    restAssign = do
       locals <- getLocals
-  ident <- source
-  pure (loc, ident, locals)
+      ident <- Text.dropWhile (== '*') <$> source
+      putLocals (ident : locals)
+      pure $ Syntax.Identifier (name ident)
 
 lhsIdent :: Assignment (Term Loc)
 lhsIdent = do
-  (loc, ident, locals) <- identWithLocals
+  locals <- getLocals
+  loc <- symbol Identifier
+  ident <- source
   putLocals (ident : locals)
   pure $ makeTerm loc (Syntax.Identifier (name ident))
 

src/Language/Ruby/Term.hs

@@ -87,6 +87,7 @@ type Syntax =
   , Literal.SymbolElement
   , Literal.TextElement
   , Ruby.Syntax.Assignment
+  , Statement.AugmentedAssignment
   , Statement.Break
   , Statement.Catch
   , Statement.Continue

src/Parsing/Parser.hs

@@ -14,6 +14,8 @@ module Parsing.Parser
 , pythonParserALaCarte
 , pythonParserPrecise
 , pythonParser
+, rubyParserALaCarte
+, rubyParserPrecise
 , rubyParser
 , tsxParser
 , typescriptParser
@@ -40,7 +42,8 @@ import qualified Language.Markdown.Assignment as Markdown
 import qualified Language.PHP.Assignment as PHP
 import qualified Language.Python as PythonPrecise
 import qualified Language.Python.Assignment as PythonALaCarte
-import qualified Language.Ruby.Assignment as Ruby
+import qualified Language.Ruby as RubyPrecise
+import qualified Language.Ruby.Assignment as RubyALaCarte
 import qualified Language.TSX.Assignment as TSX
 import qualified Language.TypeScript.Assignment as TypeScript
 import           Prelude hiding (fail)
@@ -132,8 +135,16 @@ pythonParser modes = case pythonMode modes of
   ALaCarte -> pythonParserALaCarte
   Precise  -> pythonParserPrecise
 
-rubyParser :: c Ruby.Term => (Language, SomeParser c Loc)
-rubyParser = (Ruby, SomeParser (AssignmentParser (ASTParser tree_sitter_ruby) Ruby.assignment))
+rubyParserALaCarte :: c RubyALaCarte.Term => (Language, SomeParser c Loc)
+rubyParserALaCarte = (Ruby, SomeParser (AssignmentParser (ASTParser tree_sitter_ruby) RubyALaCarte.assignment))
+
+rubyParserPrecise :: c RubyPrecise.Term => (Language, SomeParser c Loc)
+rubyParserPrecise = (Ruby, SomeParser (UnmarshalParser @RubyPrecise.Term RubyPrecise.tree_sitter_ruby))
+
+rubyParser :: (c RubyALaCarte.Term, c RubyPrecise.Term) => PerLanguageModes -> (Language, SomeParser c Loc)
+rubyParser modes = case rubyMode modes of
+  ALaCarte -> rubyParserALaCarte
+  Precise  -> rubyParserPrecise
 
 tsxParser :: c TSX.Term => (Language, SomeParser c Loc)
 tsxParser = (TSX, SomeParser (AssignmentParser (ASTParser tree_sitter_tsx) TSX.assignment))
@@ -147,6 +158,7 @@ type family TermMode term where
   TermMode Java.Term          = 'Precise
   TermMode JSON.Term          = 'Precise
   TermMode PythonPrecise.Term = 'Precise
+  TermMode RubyPrecise.Term   = 'Precise
   TermMode _                  = 'ALaCarte
 
 
@@ -156,7 +168,7 @@ aLaCarteParsers
      , c Markdown.Term
      , c PHP.Term
      , c PythonALaCarte.Term
-     , c Ruby.Term
+     , c RubyALaCarte.Term
      , c TSX.Term
      , c TypeScript.Term
      )
@@ -168,7 +180,7 @@ aLaCarteParsers = Map.fromList
   , markdownParser
   , phpParser
   , pythonParserALaCarte
-  , rubyParser
+  , rubyParserALaCarte
   , typescriptParser
   , tsxParser
   ]
@@ -178,12 +190,14 @@ preciseParsers
   :: ( c Java.Term
      , c JSON.Term
      , c PythonPrecise.Term
+     , c RubyPrecise.Term
      )
   => Map Language (SomeParser c Loc)
 preciseParsers = Map.fromList
   [ javaParser
   , jsonParser
   , pythonParserPrecise
+  , rubyParserPrecise
   ]
 
 -- | The canonical set of all parsers for the passed per-language modes.
@@ -195,7 +209,8 @@ allParsers
      , c PHP.Term
      , c PythonALaCarte.Term
      , c PythonPrecise.Term
-     , c Ruby.Term
+     , c RubyALaCarte.Term
+     , c RubyPrecise.Term
      , c TSX.Term
      , c TypeScript.Term
      )
@@ -210,7 +225,7 @@ allParsers modes = Map.fromList
   , markdownParser
   , phpParser
   , pythonParser modes
-  , rubyParser
+  , rubyParser modes
   , typescriptParser
   , tsxParser
   ]

src/Semantic/Api/Terms.hs

@@ -41,6 +41,7 @@ import           Source.Loc
 import qualified Language.Java as Java
 import qualified Language.JSON as JSON
 import qualified Language.Python as PythonPrecise
+import qualified Language.Ruby as RubyPrecise
 
 
 termGraph :: (Traversable t, Has Distribute sig m, Has (Error SomeException) sig m, Has Parse sig m) => t Blob -> m ParseTreeGraphResponse
@@ -116,6 +117,9 @@ instance ShowTermBy 'Precise JSON.Term where
 instance ShowTermBy 'Precise PythonPrecise.Term where
   showTermBy = serialize Show . void . PythonPrecise.getTerm
 
+instance ShowTermBy 'Precise RubyPrecise.Term where
+  showTermBy = serialize Show . void . RubyPrecise.getTerm
+
 instance (Recursive (term Loc), Show1 syntax, Base (term Loc) ~ TermF syntax Loc) => ShowTermBy 'ALaCarte term where
   showTermBy = serialize Show . quieterm
 
@@ -141,6 +145,9 @@ instance SExprTermBy 'Precise JSON.Term where
 instance SExprTermBy 'Precise PythonPrecise.Term where
   sexprTermBy = SExpr.Precise.serializeSExpression . PythonPrecise.getTerm
 
+instance SExprTermBy 'Precise RubyPrecise.Term where
+  sexprTermBy = SExpr.Precise.serializeSExpression . RubyPrecise.getTerm
+
 instance (Recursive (term Loc), SExpr.ToSExpression (Base (term Loc))) => SExprTermBy 'ALaCarte term where
   sexprTermBy = SExpr.serializeSExpression ByConstructorName
 

src/Semantic/CLI.hs

@@ -173,6 +173,11 @@ languageModes = Language.PerLanguageModes
                   <> metavar "ALaCarte|Precise"
                   <> value Language.ALaCarte
                   <> showDefault)
+  <*> option auto (  long "ruby-mode"
+                  <> help "The AST representation to use for Ruby sources"
+                  <> metavar "ALaCarte|Precise"
+                  <> value Language.ALaCarte
+                  <> showDefault)
 
 filePathReader :: ReadM File
 filePathReader = fileForPath <$> str

src/Semantic/Graph.hs

@@ -108,7 +108,7 @@ analysisParsers = Map.fromList
   , javascriptParser
   , phpParser
   , pythonParserALaCarte
-  , rubyParser
+  , rubyParserALaCarte
   , typescriptParser
   , tsxParser
   ]

test/Examples.hs

@@ -1,62 +1,61 @@
-{-# LANGUAGE FlexibleContexts, RecordWildCards, TypeApplications #-}
+{-# LANGUAGE FlexibleContexts, RecordWildCards, OverloadedStrings, TypeApplications #-}
 {-# OPTIONS_GHC -O1 #-}
-module Main (main) where
+module Main (main, knownFailuresForPath) where
 
 import           Control.Carrier.Parse.Measured
 import           Control.Carrier.Reader
 import           Control.Concurrent.Async (forConcurrently)
 import           Control.Exception (displayException)
 import qualified Control.Foldl as Foldl
+import           Control.Lens
 import           Control.Monad
-import           Control.Monad.IO.Class
 import           Control.Monad.Trans.Resource (ResIO, runResourceT)
 import           Data.Blob
 import qualified Data.ByteString.Lazy.Char8 as BLC
 import qualified Data.ByteString.Streaming.Char8 as ByteStream
-import           Data.Either
-import           Data.Function ((&))
-import           Data.Language (defaultLanguageModes)
+import           Data.Foldable
+import           Data.Language (LanguageMode (..), PerLanguageModes (..))
+import           Data.List
+import qualified Data.Text as Text
 import           Data.Set (Set)
-import           Data.Typeable
+import           Data.Traversable
 import qualified Streaming.Prelude as Stream
 import           System.FilePath.Glob
 import           System.Path ((</>))
 import qualified System.Path as Path
 import qualified System.Process as Process
+import qualified Test.Tasty as Tasty
+import qualified Test.Tasty.HUnit as HUnit
 
 import Data.Flag
-import Semantic.Api (TermOutputFormat (..), parseTermBuilder)
+import Proto.Semantic as P hiding (Blob, BlobPair)
+import Proto.Semantic_Fields as P
+import Semantic.Api.Symbols (parseSymbols)
 import Semantic.Config as Config
 import Semantic.Task
 import Semantic.Task.Files
 
-import qualified Test.Tasty as Tasty
-import qualified Test.Tasty.HUnit as HUnit
-
 data LanguageExample
   = LanguageExample
   { languageName      :: String
   , languageExtension :: String
-  , languageExampleDir       :: Path.RelDir
-  , languageKnownFailuresTxt :: Maybe Path.RelFile
+  , languageSkips     :: [Path.RelFile]
   } deriving (Eq, Show)
 
-le :: String -> String -> Path.RelDir -> Maybe Path.RelFile -> LanguageExample
+le :: String -> String -> [Path.RelFile] -> LanguageExample
 le = LanguageExample
 
 examples :: [LanguageExample]
 examples =
-  [ le "python" ".py" examples (Just $ Path.relFile "script/known_failures.txt")
-  , le "ruby" ".rb" examples (Just $ Path.relFile "script/known_failures.txt")
-  , le "typescript" ".ts" examples (Just $ Path.relFile "typescript/script/known_failures.txt")
-  , le "typescript" ".tsx" examples (Just $ Path.relFile "typescript/script/known_failures.txt")
-  , le "typescript" ".js" examples Nothing -- parse JavaScript with TypeScript parser.
-  , le "go" ".go" examples (Just $ Path.relFile "script/known-failures.txt")
+  [ le "python" "**/*.py" mempty
+  , le "ruby" "**/*.rb" rubySkips
+  -- , le "typescript" "**/*.[jt]s*" Nothing -- (Just $ Path.relFile "typescript/script/known_failures.txt")
+  -- , le "typescript" "**/*.tsx" Nothing
+  -- , le "javascript" ".js" examples Nothing -- parse JavaScript with TypeScript parser.
+  -- , le "go" ".go" examples (Just $ Path.relFile "script/known-failures.txt")
 
   -- TODO: Java assignment errors need to be investigated
-  -- , le "java" ".java" "examples/guava" (Just "script/known_failures_guava.txt")
-  -- , le "java" ".java" "examples/elasticsearch" (Just "script/known_failures_elasticsearch.txt")
-  -- , le "java" ".java" "examples/RxJava" (Just "script/known_failures_RxJava.txt")
+  -- , le "java" ".java" examples (Just $ Path.relFile "script/known_failures_guava.txt")
 
   -- TODO: Haskell assignment errors need to be investigated
   -- , le "haskell" ".hs" "examples/effects" (Just "script/known-failures-effects.txt")
@@ -64,27 +63,119 @@ examples =
   -- , le "haskell" ".hs" "examples/ivory" (Just "script/known-failures-ivory.txt")
 
   -- , ("php", ".php") -- TODO: No parse-examples in tree-sitter yet
-  ] where examples = Path.relDir "examples"
+  ]-- where examples = Path.relDir "examples"
+
+rubySkips :: [Path.RelFile]
+rubySkips = Path.relFile <$>
+  [
+  -- UTF8 encoding issues ("Cannot decode byte '\xe3': Data.Text.Internal.Encoding.decodeUtf8: Invalid UTF-8 stream")
+  -- These are going to be hard to fix as Ruby allows non-utf8 character content in string literals
+    "ruby_spec/optional/capi/string_spec.rb"
+  , "ruby_spec/core/string/b_spec.rb"
+  , "ruby_spec/core/string/shared/encode.rb"
+
+  -- Doesn't parse b/c of issue with r<<i
+  , "ruby_spec/core/enumerable/shared/inject.rb"
+  -- Doesn't parse
+  , "ruby_spec/language/string_spec.rb"
+
+  -- Can't detect method calls inside heredoc bodies with precise ASTs
+  , "ruby_spec/core/argf/readpartial_spec.rb"
+  , "ruby_spec/core/process/exec_spec.rb"
+  ]
 
 buildExamples :: TaskSession -> LanguageExample -> Path.RelDir -> IO Tasty.TestTree
 buildExamples session lang tsDir = do
-  knownFailures <- knownFailuresForPath tsDir (languageKnownFailuresTxt lang)
-  files <- globDir1 (compile ("**/*" <> languageExtension lang)) (Path.toString (tsDir </> languageExampleDir lang))
-  let paths = Path.relFile <$> files
-  trees <- forConcurrently paths $ \file -> pure $ HUnit.testCase (Path.toString file) $ do
-    res <- runTask session (runParse (parseFilePath file))
-    case res of
-      Left (SomeException e) -> case cast e of
-        -- We have a number of known assignment timeouts, consider these pending specs instead of failing the build.
-        Just AssignmentTimedOut -> pure ()
-        Just ParserTimedOut     ->     pure ()
-        -- Other exceptions are true failures
-        _                       -> HUnit.assertFailure (show (displayException e))
-      _ -> if file `elem` knownFailures
-              then pure ()
-              else (isRight res) HUnit.@? ("Error: " <> either show show res)
+  let skips = fmap (tsDir </>) (languageSkips lang)
+  files <- globDir1 (compile (languageExtension lang)) (Path.toString tsDir)
+  let paths = filter (`notElem` skips) $ Path.relFile <$> files
+  trees <- for paths $ \file -> do
+    pure . HUnit.testCaseSteps (Path.toString file) $ \step -> do
+      -- Use alacarte language mode
+      step "a la carte"
+      alacarte <- runTask session (runParse (parseSymbolsFilePath aLaCarteLanguageModes file))
+      assertOK "a la carte" alacarte
+
+      -- Test out precise language mode
+      step "precise"
+      precise <- runTask session (runParse (parseSymbolsFilePath preciseLanguageModes file))
+      assertOK "precise" precise
+
+      -- Compare the two
+      step "compare"
+      assertMatch alacarte precise
+
   pure (Tasty.testGroup (languageName lang) trees)
 
+  where
+    assertOK msg = either (\e -> HUnit.assertFailure (msg <> " failed to parse" <> show e)) (refuteErrors msg)
+    refuteErrors msg a = case toList (a^.files) of
+      [x] | (e:_) <- toList (x^.errors) -> HUnit.assertFailure (msg <> " parse errors " <> show e)
+      _ -> pure ()
+
+    assertMatch a b = case (a, b) of
+      (Right a, Right b) -> case (toList (a^.files), toList (b^.files)) of
+        ([x], [y]) | e1:_ <- toList (x^.errors)
+                   , e2:_ <- toList (y^.errors)
+                   -> HUnit.assertFailure ("Parse errors (both) " <> show e1 <> show e2)
+        (_, [y])   | e:_ <- toList (y^.errors)
+                   -> HUnit.assertFailure ("Parse errors (precise) " <> show e)
+        ([x], _)   | e:_ <- toList (x^.errors)
+                   -> HUnit.assertFailure ("Parse errors (a la carte) " <> show e)
+        ([x], [y]) -> do
+          HUnit.assertEqual "Expected paths to be equal" (x^.path) (y^.path)
+          let aLaCarteSymbols = sort . filterALaCarteSymbols (languageName lang) $ toListOf (symbols . traverse . symbol) x
+              preciseSymbols = sort $ toListOf (symbols . traverse . symbol) y
+              delta = aLaCarteSymbols \\ preciseSymbols
+              msg = "Found in a la carte, but not precise: "
+                  <> show delta
+                  <> "\n"
+                  <> "Found in precise but not a la carte: "
+                  <> show (preciseSymbols \\ aLaCarteSymbols)
+                  <> "\n"
+                  <> "Expected: " <> show aLaCarteSymbols <> "\n"
+                  <> "But got:" <> show preciseSymbols
+
+          HUnit.assertBool ("Expected symbols to be equal.\n" <> msg) (null delta)
+          pure ()
+        _          -> HUnit.assertFailure "Expected 1 file in each response"
+      (Left e1, Left e2) -> HUnit.assertFailure ("Unable to parse (both)" <> show (displayException e1) <> show (displayException e2))
+      (_, Left e)        -> HUnit.assertFailure ("Unable to parse (precise)" <> show (displayException e))
+      (Left e, _)        -> HUnit.assertFailure ("Unable to parse (a la carte)" <> show (displayException e))
+
+
+filterALaCarteSymbols :: String -> [Text.Text] -> [Text.Text]
+filterALaCarteSymbols "ruby" symbols
+  = filterOutInstanceVariables
+  . filterOutBuiltInMethods
+  $ symbols
+  where
+    filterOutInstanceVariables = filter (not . Text.isPrefixOf "@")
+    filterOutBuiltInMethods = filter (`notElem` blacklist)
+    blacklist =
+      [ "alias"
+      , "load"
+      , "require_relative"
+      , "require"
+      , "super"
+      , "undef"
+      , "defined?"
+      , "lambda"
+      ]
+filterALaCarteSymbols _      symbols = symbols
+
+aLaCarteLanguageModes :: PerLanguageModes
+aLaCarteLanguageModes = PerLanguageModes
+  { pythonMode = ALaCarte
+  , rubyMode = ALaCarte
+  }
+
+preciseLanguageModes :: PerLanguageModes
+preciseLanguageModes = PerLanguageModes
+  { pythonMode = Precise
+  , rubyMode = Precise
+  }
+
 testOptions :: Config.Options
 testOptions = defaultOptions
   { optionsFailOnWarning = flag FailOnWarning True
@@ -98,8 +189,7 @@ main = withOptions testOptions $ \ config logger statter -> do
   let session = TaskSession config "-" False logger statter
 
   allTests <- forConcurrently examples $ \lang@LanguageExample{..} -> do
-    let tsLang = Path.relDir ("tree-sitter-" <> languageName)
-    let tsDir = Path.relDir "tmp/haskell-tree-sitter" </> tsLang </> Path.relDir "vendor" </> tsLang
+    let tsDir = Path.relDir "tmp" </> Path.relDir (languageName <> "-examples")
     buildExamples session lang tsDir
 
   Tasty.defaultMain $ Tasty.testGroup "parse-examples" allTests
@@ -117,6 +207,13 @@ knownFailuresForPath tsDir (Just path)
   & Foldl.purely Stream.fold_ Foldl.set
   )
 
-
-parseFilePath :: (Has (Error SomeException) sig m, Has Distribute sig m, Has Parse sig m, Has Files sig m, Has (Reader Config) sig m, MonadIO m) => Path.RelFile -> m Bool
-parseFilePath path = readBlob (fileForTypedPath path) >>= runReader defaultLanguageModes . parseTermBuilder @[] TermShow . pure >>= const (pure True)
+parseSymbolsFilePath ::
+  ( Has (Error SomeException) sig m
+  , Has Distribute sig m
+  , Has Parse sig m
+  , Has Files sig m
+  )
+  => PerLanguageModes
+  -> Path.RelFile
+  -> m ParseTreeSymbolResponse
+parseSymbolsFilePath languageModes path = readBlob (fileForTypedPath path) >>= runReader languageModes . parseSymbols . pure @[]

test/Semantic/CLI/Spec.hs

@@ -63,7 +63,7 @@ parseFixtures =
         path' = [File "test/fixtures/ruby/corpus/and-or.A.rb" Ruby, File "test/fixtures/ruby/corpus/and-or.B.rb" Ruby]
         path'' = [File "test/fixtures/ruby/corpus/method-declaration.A.rb" Ruby]
         prefix = Path.relDir "test/fixtures/cli"
-        run = runReader (PerLanguageModes ALaCarte)
+        run = runReader (PerLanguageModes ALaCarte ALaCarte)
 
 diffFixtures :: [(String, [BlobPair] -> ParseC TaskC Builder, [(File, File)], Path.RelFile)]
 diffFixtures =

test/Semantic/Spec.hs

@@ -17,15 +17,15 @@ spec = do
     let methodsBlob = makeBlob "def foo\nend\n" "methods.rb" Ruby mempty
 
     it "returns error if given an unknown language (json)" $ do
-      output <- fmap runBuilder . runTaskOrDie . runReader (PerLanguageModes ALaCarte) $ parseTermBuilder TermJSONTree [ setBlobLanguage Unknown methodsBlob ]
+      output <- fmap runBuilder . runTaskOrDie . runReader (PerLanguageModes ALaCarte ALaCarte) $ parseTermBuilder TermJSONTree [ setBlobLanguage Unknown methodsBlob ]
       output `shouldBe` "{\"trees\":[{\"path\":\"methods.rb\",\"error\":\"NoLanguageForBlob \\\"methods.rb\\\"\",\"language\":\"Unknown\"}]}\n"
 
     it "throws if given an unknown language for sexpression output" $ do
-      res <- runTaskWithOptions defaultOptions (runReader (PerLanguageModes ALaCarte) (runParseWithConfig (parseTermBuilder TermSExpression [setBlobLanguage Unknown methodsBlob])))
+      res <- runTaskWithOptions defaultOptions (runReader (PerLanguageModes ALaCarte ALaCarte) (runParseWithConfig (parseTermBuilder TermSExpression [setBlobLanguage Unknown methodsBlob])))
       case res of
         Left exc    -> fromException exc `shouldBe` Just (NoLanguageForBlob "methods.rb")
         Right _bad  -> fail "Expected parseTermBuilder to fail for an unknown language"
 
     it "renders with the specified renderer" $ do
-      output <- fmap runBuilder . runTaskOrDie . runReader (PerLanguageModes ALaCarte) $ parseTermBuilder TermSExpression [methodsBlob]
+      output <- fmap runBuilder . runTaskOrDie . runReader (PerLanguageModes ALaCarte ALaCarte) $ parseTermBuilder TermSExpression [methodsBlob]
       output `shouldBe` "(Statements\n  (Method\n    (Empty)\n    (Identifier)\n    (Statements)))\n"

test/fixtures/python/corpus/assignment.diffA-B.txt

@@ -10,18 +10,14 @@
   { (Identifier)
   ->(Identifier) }
     (Integer))
-{+(Assignment
-  {+(Identifier)+}
-  {+(Statements
-    {+(Integer)+}
-    {+(Integer)+})+})+}
-{-(Assignment
-  {-(Statements
+  (Assignment
+  { (Statements
     {-(Identifier)-}
-    {-(Identifier)-})-}
-  {-(Statements
-    {-(Integer)-}
-    {-(Integer)-})-})-}
+    {-(Identifier)-})
+  ->(Identifier) }
+    (Statements
+      (Integer)
+      (Integer)))
 {-(Assignment
   {-(Identifier)-}
   {-(Statements

test/fixtures/python/corpus/augmented-assignment.diffA-B.txt

@@ -1,27 +1,23 @@
 (Statements
-  (Assignment
-    (Identifier)
+  (AugmentedAssignment
   { (Plus
     {-(Identifier)-}
     {-(Integer)-})
   ->(RShift
     {+(Identifier)+}
     {+(Integer)+}) })
-  (Assignment
-  { (Identifier)
-  ->(Identifier) }
-  { (RShift
-    {-(Identifier)-}
-    {-(Integer)-})
-  ->(DividedBy
+{+(AugmentedAssignment
+  {+(DividedBy
     {+(Identifier)+}
-    {+(Integer)+}) })
-  (Assignment
-  { (Identifier)
-  ->(Identifier) }
-  { (DividedBy
+    {+(Integer)+})+})+}
+  (AugmentedAssignment
+  { (RShift
     {-(Identifier)-}
     {-(Integer)-})
   ->(Plus
     {+(Identifier)+}
-    {+(Integer)+}) }))
+    {+(Integer)+}) })
+{-(AugmentedAssignment
+  {-(DividedBy
+    {-(Identifier)-}
+    {-(Integer)-})-})-})

test/fixtures/python/corpus/augmented-assignment.diffB-A.txt

@@ -1,27 +1,24 @@
 (Statements
-  (Assignment
-    (Identifier)
+  (AugmentedAssignment
   { (RShift
     {-(Identifier)-}
     {-(Integer)-})
   ->(Plus
     {+(Identifier)+}
     {+(Integer)+}) })
-  (Assignment
-  { (Identifier)
-  ->(Identifier) }
-  { (DividedBy
-    {-(Identifier)-}
-    {-(Integer)-})
-  ->(RShift
+{+(AugmentedAssignment
+  {+(RShift
     {+(Identifier)+}
-    {+(Integer)+}) })
-  (Assignment
-  { (Identifier)
-  ->(Identifier) }
-  { (Plus
-    {-(Identifier)-}
-    {-(Integer)-})
-  ->(DividedBy
+    {+(Integer)+})+})+}
+{+(AugmentedAssignment
+  {+(DividedBy
     {+(Identifier)+}
-    {+(Integer)+}) }))
+    {+(Integer)+})+})+}
+{-(AugmentedAssignment
+  {-(DividedBy
+    {-(Identifier)-}
+    {-(Integer)-})-})-}
+{-(AugmentedAssignment
+  {-(Plus
+    {-(Identifier)-}
+    {-(Integer)-})-})-})

test/fixtures/python/corpus/augmented-assignment.parseA.txt

@@ -1,16 +1,13 @@
 (Statements
-  (Assignment
-    (Identifier)
+  (AugmentedAssignment
     (Plus
       (Identifier)
       (Integer)))
-  (Assignment
-    (Identifier)
+  (AugmentedAssignment
     (RShift
       (Identifier)
       (Integer)))
-  (Assignment
-    (Identifier)
+  (AugmentedAssignment
     (DividedBy
       (Identifier)
       (Integer))))

test/fixtures/python/corpus/augmented-assignment.parseB.txt

@@ -1,16 +1,13 @@
 (Statements
-  (Assignment
-    (Identifier)
+  (AugmentedAssignment
     (RShift
       (Identifier)
       (Integer)))
-  (Assignment
-    (Identifier)
+  (AugmentedAssignment
     (DividedBy
       (Identifier)
       (Integer)))
-  (Assignment
-    (Identifier)
+  (AugmentedAssignment
     (Plus
       (Identifier)
       (Integer))))

test/fixtures/python/corpus/comparison-operator.diffB-A.txt

@@ -29,14 +29,12 @@
   {+(Member
     {+(Identifier)+}
     {+(Identifier)+})+})+}
-{+(Not
-  {+(Equal
-    {+(Identifier)+}
-    {+(Identifier)+})+})+}
-{-(Not
-  {-(Equal
-    {-(Identifier)-}
-    {-(Identifier)-})-})-}
+  (Not
+    (Equal
+    { (Identifier)
+    ->(Identifier) }
+    { (Identifier)
+    ->(Identifier) }))
 {-(Not
   {-(Member
     {-(Identifier)-}

test/fixtures/python/corpus/import-from-statement.diffA-B.txt

@@ -1,23 +1,4 @@
 (Statements
-{+(Import
-  {+(Alias
-    {+(Identifier)+}
-    {+(Identifier)+})+})+}
-{+(Import
-  {+(Alias
-    {+(Identifier)+}
-    {+(Identifier)+})+}
-  {+(Alias
-    {+(Identifier)+}
-    {+(Identifier)+})+})+}
-{+(Import)+}
-{+(Import
-  {+(Alias
-    {+(Identifier)+}
-    {+(Identifier)+})+}
-  {+(Alias
-    {+(Identifier)+}
-    {+(Identifier)+})+})+}
 { (Import
   {-(Alias
     {-(Identifier)-}
@@ -26,18 +7,37 @@
   {+(Alias
     {+(Identifier)+}
     {+(Identifier)+})+}) }
-{+(Import
-  {+(Alias
-    {+(Identifier)+}
-    {+(Identifier)+})+})+}
-{-(Import
+{ (Import
   {-(Alias
     {-(Identifier)-}
     {-(Identifier)-})-}
   {-(Alias
     {-(Identifier)-}
-    {-(Identifier)-})-})-}
-{-(Import)-}
+    {-(Identifier)-})-})
+->(Import
+  {+(Alias
+    {+(Identifier)+}
+    {+(Identifier)+})+}
+  {+(Alias
+    {+(Identifier)+}
+    {+(Identifier)+})+}) }
+{ (Import)
+->(Import) }
+{+(Import
+  {+(Alias
+    {+(Identifier)+}
+    {+(Identifier)+})+}
+  {+(Alias
+    {+(Identifier)+}
+    {+(Identifier)+})+})+}
+{+(Import
+  {+(Alias
+    {+(Identifier)+}
+    {+(Identifier)+})+})+}
+{+(Import
+  {+(Alias
+    {+(Identifier)+}
+    {+(Identifier)+})+})+}
 {-(Import
   {-(Alias
     {-(Identifier)-}

test/fixtures/python/corpus/import-from-statement.diffB-A.txt

@@ -1,16 +1,4 @@
 (Statements
-{+(Import
-  {+(Alias
-    {+(Identifier)+}
-    {+(Identifier)+})+})+}
-{+(Import
-  {+(Alias
-    {+(Identifier)+}
-    {+(Identifier)+})+}
-  {+(Alias
-    {+(Identifier)+}
-    {+(Identifier)+})+})+}
-{+(Import)+}
 { (Import
   {-(Alias
     {-(Identifier)-}
@@ -19,18 +7,30 @@
   {+(Alias
     {+(Identifier)+}
     {+(Identifier)+})+}) }
-{+(Import
-  {+(Alias
-    {+(Identifier)+}
-    {+(Identifier)+})+})+}
-{-(Import
+{ (Import
   {-(Alias
     {-(Identifier)-}
     {-(Identifier)-})-}
   {-(Alias
     {-(Identifier)-}
-    {-(Identifier)-})-})-}
-{-(Import)-}
+    {-(Identifier)-})-})
+->(Import
+  {+(Alias
+    {+(Identifier)+}
+    {+(Identifier)+})+}
+  {+(Alias
+    {+(Identifier)+}
+    {+(Identifier)+})+}) }
+{ (Import)
+->(Import) }
+{+(Import
+  {+(Alias
+    {+(Identifier)+}
+    {+(Identifier)+})+})+}
+{+(Import
+  {+(Alias
+    {+(Identifier)+}
+    {+(Identifier)+})+})+}
 {-(Import
   {-(Alias
     {-(Identifier)-}

test/fixtures/python/corpus/unary-operator.diffB-A.txt

@@ -2,9 +2,10 @@
 {+(Negate
   {+(Identifier)+})+}
 {+(Identifier)+}
-  (Complement
-  { (Identifier)
-  ->(Identifier) })
+{+(Complement
+  {+(Identifier)+})+}
+{-(Complement
+  {-(Identifier)-})-}
 {-(Negate
   {-(Identifier)-})-}
 {-(Identifier)-})

test/fixtures/ruby/analysis/break.rb

@@ -1,6 +1,6 @@
 ii = 0
 while ii < 5
-  ii += 1
+  ii = ii + 1
   if ii == 3
     break
   end

test/fixtures/ruby/analysis/next.rb

@@ -1,8 +1,8 @@
 ii = 0
 jj = 0
 while ii < 5
-  ii += 1
-  jj += 1
+  ii = ii + 1
+  jj = jj + 1
   if (ii == 3) && (jj == 3)
     ii = 0
     next

test/fixtures/ruby/corpus/calls.parseA.txt

@@ -23,7 +23,8 @@
       (Identifier)))
   (Send
     (Identifier)
-    (Identifier))
+    (Send
+      (Identifier)))
   (Send
     (Identifier)
     (Identifier))

test/fixtures/ruby/corpus/conditional-assignment.diffA-B.txt

@@ -1,6 +1,5 @@
 (Statements
-  (Assignment
-    (Identifier)
+  (AugmentedAssignment
   { (Or
     {-(Identifier)-}
     {-(Integer)-})

test/fixtures/ruby/corpus/conditional-assignment.diffB-A.txt

@@ -1,6 +1,5 @@
 (Statements
-  (Assignment
-    (Identifier)
+  (AugmentedAssignment
   { (And
     {-(Identifier)-}
     {-(Integer)-})

test/fixtures/ruby/corpus/conditional-assignment.parseA.txt

@@ -1,6 +1,5 @@
 (Statements
-  (Assignment
-    (Identifier)
+  (AugmentedAssignment
     (Or
       (Identifier)
       (Integer))))

test/fixtures/ruby/corpus/conditional-assignment.parseB.txt

@@ -1,6 +1,5 @@
 (Statements
-  (Assignment
-    (Identifier)
+  (AugmentedAssignment
     (And
       (Identifier)
       (Integer))))

test/fixtures/ruby/corpus/for.diffB-A.txt

@@ -7,33 +7,34 @@
     {+(Identifier)+})+}
   {+(Send
     {+(Identifier)+})+})+}
+{+(ForEach
+  {+(Statements
+    {+(Send
+      {+(Identifier)+})+}
+    {+(Send
+      {+(Identifier)+})+})+}
+  {+(Send
+    {+(Identifier)+})+}
+  {+(Send
+    {+(Identifier)+})+})+}
   (ForEach
     (Statements
       (Send
       { (Identifier)
-      ->(Identifier) })
-    {+(Send
-      {+(Identifier)+})+})
+      ->(Identifier) }))
   { (Array
     {-(Integer)-}
     {-(Integer)-}
     {-(Integer)-})
-  ->(Send
-    {+(Identifier)+}) }
-    (Send
-    { (Identifier)
-    ->(Identifier) }
+  ->(Enumeration
+    {+(Integer)+}
+    {+(Integer)+}
+    {+(Empty)+}) }
+  { (Send
+    {-(Identifier)-}
     {-(Send
-      {-(Identifier)-})-}))
-{+(ForEach
-  {+(Statements
-    {+(Send
-      {+(Identifier)+})+})+}
-  {+(Enumeration
-    {+(Integer)+}
-    {+(Integer)+}
-    {+(Empty)+})+}
-  {+(Boolean)+})+}
+      {-(Identifier)-})-})
+  ->(Boolean) })
 {+(ForEach
   {+(Statements
     {+(Send

test/fixtures/ruby/corpus/math-assignment.diffA-B.txt

@@ -1,27 +1,22 @@
 (Statements
-  (Assignment
-    (Identifier)
+  (AugmentedAssignment
     (Plus
       (Identifier)
     { (Integer)
     ->(Integer) }))
-  (Assignment
-    (Identifier)
+  (AugmentedAssignment
     (Minus
       (Identifier)
       (Integer)))
-  (Assignment
-    (Identifier)
+  (AugmentedAssignment
     (Times
       (Identifier)
       (Integer)))
-  (Assignment
-    (Identifier)
+  (AugmentedAssignment
     (DividedBy
       (Identifier)
       (Integer)))
-  (Assignment
-    (Identifier)
+  (AugmentedAssignment
     (Power
       (Identifier)
       (Integer))))

test/fixtures/ruby/corpus/math-assignment.diffB-A.txt

@@ -1,27 +1,22 @@
 (Statements
-  (Assignment
-    (Identifier)
+  (AugmentedAssignment
     (Plus
       (Identifier)
     { (Integer)
     ->(Integer) }))
-  (Assignment
-    (Identifier)
+  (AugmentedAssignment
     (Minus
       (Identifier)
       (Integer)))
-  (Assignment
-    (Identifier)
+  (AugmentedAssignment
     (Times
       (Identifier)
       (Integer)))
-  (Assignment
-    (Identifier)
+  (AugmentedAssignment
     (DividedBy
       (Identifier)
       (Integer)))
-  (Assignment
-    (Identifier)
+  (AugmentedAssignment
     (Power
       (Identifier)
       (Integer))))

test/fixtures/ruby/corpus/math-assignment.parseA.txt

@@ -1,26 +1,21 @@
 (Statements
-  (Assignment
-    (Identifier)
+  (AugmentedAssignment
     (Plus
       (Identifier)
       (Integer)))
-  (Assignment
-    (Identifier)
+  (AugmentedAssignment
     (Minus
       (Identifier)
       (Integer)))
-  (Assignment
-    (Identifier)
+  (AugmentedAssignment
     (Times
       (Identifier)
       (Integer)))
-  (Assignment
-    (Identifier)
+  (AugmentedAssignment
     (DividedBy
       (Identifier)
       (Integer)))
-  (Assignment
-    (Identifier)
+  (AugmentedAssignment
     (Power
       (Identifier)
       (Integer))))

test/fixtures/ruby/corpus/math-assignment.parseB.txt

@@ -1,26 +1,21 @@
 (Statements
-  (Assignment
-    (Identifier)
+  (AugmentedAssignment
     (Plus
       (Identifier)
       (Integer)))
-  (Assignment
-    (Identifier)
+  (AugmentedAssignment
     (Minus
       (Identifier)
       (Integer)))
-  (Assignment
-    (Identifier)
+  (AugmentedAssignment
     (Times
       (Identifier)
       (Integer)))
-  (Assignment
-    (Identifier)
+  (AugmentedAssignment
     (DividedBy
       (Identifier)
       (Integer)))
-  (Assignment
-    (Identifier)
+  (AugmentedAssignment
     (Power
       (Identifier)
       (Integer))))

test/fixtures/ruby/corpus/method-calls.diffA-B.txt

@@ -28,7 +28,8 @@
     {-(Array)-}
     {-(Integer)-})-}
   {-(Identifier)-}
-  {-(Identifier)-}
+  {-(Send
+    {-(Identifier)-})-}
   {-(Function
     {-(Empty)-}
     {-(Identifier)-}

test/fixtures/ruby/corpus/method-calls.diffB-A.txt

@@ -28,7 +28,8 @@
     {+(Array)+}
     {+(Integer)+})+}
   {+(Identifier)+}
-  {+(Identifier)+}
+  {+(Send
+    {+(Identifier)+})+}
   {+(Function
     {+(Empty)+}
     {+(Identifier)+}

test/fixtures/ruby/corpus/method-calls.parseA.txt

@@ -27,7 +27,8 @@
       (Array)
       (Integer))
     (Identifier)
-    (Identifier)
+    (Send
+      (Identifier))
     (Function
       (Empty)
       (Identifier)

test/fixtures/ruby/corpus/multiple-assignments.diffA-B.txt

@@ -2,8 +2,7 @@
   (Assignment
     (Statements
       (Identifier)
-    { (Identifier)
-    ->(Identifier) }
+      (Identifier)
     {-(Identifier)-})
     (Array
       (Integer)

test/fixtures/ruby/corpus/multiple-assignments.diffB-A.txt

@@ -2,8 +2,7 @@
   (Assignment
     (Statements
       (Identifier)
-    { (Identifier)
-    ->(Identifier) }
+      (Identifier)
     {+(Identifier)+})
     (Array
       (Integer)