Merge remote-tracking branch 'origin/master' into typescript-assignment

src/Data/Syntax/Literal.hs

@@ -44,6 +44,20 @@ newtype Float a = Float { floatContent :: ByteString }
 instance Eq1 Data.Syntax.Literal.Float where liftEq = genericLiftEq
 instance Show1 Data.Syntax.Literal.Float where liftShowsPrec = genericLiftShowsPrec
 
+-- Rational literals e.g. `2/3r`
+newtype Rational a = Rational ByteString
+  deriving (Diffable, Eq, Foldable, Functor, GAlign, Generic1, Show, Traversable)
+
+instance Eq1 Data.Syntax.Literal.Rational where liftEq = genericLiftEq
+instance Show1 Data.Syntax.Literal.Rational where liftShowsPrec = genericLiftShowsPrec
+
+-- Complex literals e.g. `3 + 2i`
+newtype Complex a = Complex ByteString
+  deriving (Diffable, Eq, Foldable, Functor, GAlign, Generic1, Show, Traversable)
+
+instance Eq1 Data.Syntax.Literal.Complex where liftEq = genericLiftEq
+instance Show1 Data.Syntax.Literal.Complex where liftShowsPrec = genericLiftShowsPrec
+
 -- Strings, symbols
 
 newtype String a = String { stringElements :: [a] }

src/Language/Ruby/Syntax.hs

@@ -1,4 +1,4 @@
-{-# LANGUAGE DataKinds, RankNTypes, TypeOperators #-}
+{-# LANGUAGE DataKinds, DeriveAnyClass, RankNTypes, TypeOperators #-}
 module Language.Ruby.Syntax
 ( assignment
 , Syntax
@@ -9,7 +9,8 @@ module Language.Ruby.Syntax
 import Data.Maybe (fromMaybe)
 import Data.Record
 import Data.Functor (void)
-import Data.Syntax (emptyTerm, makeTerm, parseError, handleError)
+import Data.List.NonEmpty (some1)
+import Data.Syntax (contextualize, emptyTerm, parseError, handleError, infixContext, makeTerm, makeTerm', makeTerm1)
 import qualified Data.Syntax as Syntax
 import Data.Syntax.Assignment hiding (Assignment, Error)
 import qualified Data.Syntax.Assignment as Assignment
@@ -41,11 +42,13 @@ type Syntax = '[
   , Expression.Subscript
   , Literal.Array
   , Literal.Boolean
+  , Literal.Complex
   , Literal.Float
   , Literal.Hash
   , Literal.Integer
   , Literal.KeyValue
   , Literal.Null
+  , Literal.Rational
   , Literal.String
   , Literal.Symbol
   , Literal.TextElement
@@ -66,6 +69,7 @@ type Syntax = '[
   , Statement.Try
   , Statement.While
   , Statement.Yield
+  , Syntax.Context
   , Syntax.Empty
   , Syntax.Error
   , Syntax.Identifier
@@ -76,13 +80,12 @@ type Syntax = '[
 type Term = Term.Term (Union Syntax) (Record Location)
 type Assignment = HasCallStack => Assignment.Assignment (AST Grammar) Grammar Term
 
-
 -- | Assignment from AST in Ruby’s grammar onto a program in Ruby’s syntax.
 assignment :: Assignment
 assignment = handleError $ makeTerm <$> symbol Program <*> children (Syntax.Program <$> many expression)
 
 expression :: Assignment
-expression = handleError $
+expression = handleError . term $
       alias
   <|> assignment'
   <|> begin
@@ -92,7 +95,6 @@ expression = handleError $
   <|> call
   <|> case'
   <|> class'
-  <|> comment
   <|> conditional
   <|> emptyStatement
   <|> endBlock
@@ -152,26 +154,24 @@ identifier =
   <|> mk Uninterpreted
   where mk s = makeTerm <$> symbol s <*> (Syntax.Identifier <$> source)
 
+-- TODO: Handle interpolation in all literals that support it (strings, regexes, symbols, subshells, etc).
 literal :: Assignment
 literal =
-      makeTerm <$> symbol Grammar.True <*> (Literal.true <$ source)
+      makeTerm <$> token  Grammar.True     <*> pure Literal.true
-  <|> makeTerm <$> symbol Grammar.False <*> (Literal.false <$ source)
+  <|> makeTerm <$> token  Grammar.False    <*> pure Literal.false
-  <|> makeTerm <$> symbol Grammar.Integer <*> (Literal.Integer <$> source)
+  <|> makeTerm <$> token  Grammar.Nil      <*> pure Literal.Null
-  <|> makeTerm <$> symbol Grammar.Float <*> (Literal.Float <$> source)
+  <|> makeTerm <$> symbol Grammar.Integer  <*> (Literal.Integer <$> source)
-  <|> makeTerm <$> symbol Grammar.Nil <*> (Literal.Null <$ source)
+  <|> makeTerm <$> symbol Grammar.Float    <*> (Literal.Float <$> source)
+  <|> makeTerm <$> symbol Grammar.Rational <*> (Literal.Rational <$> source)
+  <|> makeTerm <$> symbol Grammar.Complex  <*> (Literal.Complex <$> source)
    -- TODO: Do we want to represent the difference between .. and ...
   <|> makeTerm <$> symbol Range <*> children (Expression.Enumeration <$> expression <*> expression <*> emptyTerm)
   <|> makeTerm <$> symbol Array <*> children (Literal.Array <$> many expression)
-  <|> makeTerm <$> symbol Hash <*> children (Literal.Hash <$> many pair)
+  <|> makeTerm <$> symbol Hash  <*> children (Literal.Hash <$> (many . term) pair)
-  -- TODO: Give subshell it's own literal and allow interpolation
   <|> makeTerm <$> symbol Subshell <*> (Literal.TextElement <$> source)
-  -- TODO: Handle interpolation
   <|> makeTerm <$> symbol String <*> (Literal.TextElement <$> source)
-  -- TODO: this isn't quite right `"a" "b"` ends up as TextElement {textElementContent = "\"a\"\"b\""}
+  <|> makeTerm <$> symbol ChainedString <*> children (many (term (makeTerm <$> symbol String <*> (Literal.TextElement <$> source))))
-  <|> makeTerm <$> symbol ChainedString <*> children (Literal.TextElement . mconcat <$> many (symbol String *> source))
-  -- TODO: Handle interpolation, dedicated literal?
   <|> makeTerm <$> symbol Regex <*> (Literal.TextElement <$> source)
-  -- TODO: Handle interpolation
   <|> makeTerm <$> symbol Symbol <*> (Literal.Symbol <$> source)
 
 heredoc :: Assignment
@@ -289,14 +289,13 @@ subscript = makeTerm <$> symbol ElementReference <*> children (Expression.Subscr
 
 pair :: Assignment
 pair = makeTerm <$> symbol Pair <*> children (Literal.KeyValue <$> expression <*> expression)
-   <|> makeTerm <$> symbol Pair <*> (Syntax.Empty <$ source)
+   <|> makeTerm <$> token  Pair <*> pure Syntax.Empty
 
 methodCall :: Assignment
 methodCall = makeTerm <$> symbol MethodCall <*> children (Expression.Call <$> expression <*> args <*> (block <|> emptyTerm))
   where
     args = (symbol ArgumentList <|> symbol ArgumentListWithParens) *> children (many expression) <|> pure []
 
-
 call :: Assignment
 call = makeTerm <$> symbol Call <*> children (Expression.MemberAccess <$> expression <*> (expression <|> args))
   where
@@ -317,29 +316,31 @@ begin :: Assignment
 begin = makeTerm <$> symbol Begin <*> children (Statement.Try <$> expressions <*> many rescue)
 
 assignment' :: Assignment
-assignment'
+assignment' = makeTerm  <$> symbol Assignment         <*> children (Statement.Assignment <$> lhs <*> rhs)
-   =  makeTerm <$> symbol Assignment <*> children (Statement.Assignment <$> lhs <*> rhs)
+          <|> makeTerm' <$> symbol OperatorAssignment <*> children (infixTerm lhs expression
-  <|> makeTerm <$> symbol OperatorAssignment <*> children (lhs >>= \ var -> Statement.Assignment var <$>
+                [ assign Expression.Plus      <$ symbol AnonPlusEqual
-         (makeTerm <$> symbol AnonPlusEqual               <*> (Expression.Plus var      <$> expression)
+                , assign Expression.Minus     <$ symbol AnonMinusEqual
-      <|> makeTerm <$> symbol AnonMinusEqual              <*> (Expression.Minus var     <$> expression)
+                , assign Expression.Times     <$ symbol AnonStarEqual
-      <|> makeTerm <$> symbol AnonStarEqual               <*> (Expression.Times var     <$> expression)
+                , assign Expression.Power     <$ symbol AnonStarStarEqual
-      <|> makeTerm <$> symbol AnonStarStarEqual           <*> (Expression.Power var     <$> expression)
+                , assign Expression.DividedBy <$ symbol AnonSlashEqual
-      <|> makeTerm <$> symbol AnonSlashEqual              <*> (Expression.DividedBy var <$> expression)
+                , assign Expression.And       <$ symbol AnonPipePipeEqual
-      <|> makeTerm <$> symbol AnonPipePipeEqual           <*> (Expression.And var       <$> expression)
+                , assign Expression.BOr       <$ symbol AnonPipeEqual
-      <|> makeTerm <$> symbol AnonPipeEqual               <*> (Expression.BOr var       <$> expression)
+                , assign Expression.And       <$ symbol AnonAmpersandAmpersandEqual
-      <|> makeTerm <$> symbol AnonAmpersandAmpersandEqual <*> (Expression.And var       <$> expression)
+                , assign Expression.BAnd      <$ symbol AnonAmpersandEqual
-      <|> makeTerm <$> symbol AnonAmpersandEqual          <*> (Expression.BAnd var      <$> expression)
+                , assign Expression.Modulo    <$ symbol AnonPercentEqual
-      <|> makeTerm <$> symbol AnonPercentEqual            <*> (Expression.Modulo var    <$> expression)
+                , assign Expression.RShift    <$ symbol AnonRAngleRAngleEqual
-      <|> makeTerm <$> symbol AnonRAngleRAngleEqual       <*> (Expression.RShift var    <$> expression)
+                , assign Expression.LShift    <$ symbol AnonLAngleLAngleEqual
-      <|> makeTerm <$> symbol AnonLAngleLAngleEqual       <*> (Expression.LShift var    <$> expression)
+                , assign Expression.BXOr      <$ symbol AnonCaretEqual
-      <|> makeTerm <$> symbol AnonCaretEqual              <*> (Expression.BXOr var      <$> expression)))
+                ])
   where
-    lhs = makeTerm <$> symbol LeftAssignmentList <*> children (many expr) <|> expr
+    assign :: f :< Syntax => (Term -> Term -> f Term) -> Term -> Term -> Union Syntax Term
-    rhs = makeTerm <$> symbol RightAssignmentList <*> children (many expr) <|> expr
+    assign c l r = inj (Statement.Assignment l (makeTerm1 (c l r)))
-    expr =
+
-          makeTerm <$> symbol RestAssignment <*> (Syntax.Identifier <$> source)
+    lhs  = makeTerm <$> symbol LeftAssignmentList  <*> children (many expr) <|> expr
-      <|> makeTerm <$> symbol DestructuredLeftAssignment <*> children (many expr)
+    rhs  = makeTerm <$> symbol RightAssignmentList <*> children (many expr) <|> expr
-      <|> expression
+    expr = makeTerm <$> symbol RestAssignment      <*> (Syntax.Identifier <$> source)
+       <|> makeTerm <$> symbol DestructuredLeftAssignment <*> children (many expr)
+       <|> expression
 
 unary :: Assignment
 unary = symbol Unary >>= \ location ->
@@ -350,42 +351,32 @@ unary = symbol Unary >>= \ location ->
   <|> makeTerm location . Expression.Negate <$> children ( symbol AnonMinus' *> expression )
   <|> children ( symbol AnonPlus *> expression )
 
+-- TODO: Distinguish `===` from `==` ?
+-- TODO: Distinuish `=~` and `!~` ?
 binary  :: Assignment
-binary = symbol Binary >>= \ loc -> children $ (expression <|> emptyTerm) >>= \ lexpression -> go loc lexpression
+binary = makeTerm' <$> symbol Binary <*> children (infixTerm expression expression
-  where
+  [ (inj .) . Expression.Plus             <$ symbol AnonPlus
-    go loc lexpression
+  , (inj .) . Expression.Minus            <$ symbol AnonMinus'
-       =  mk AnonAnd Expression.And
+  , (inj .) . Expression.Times            <$ symbol AnonStar'
-      <|> mk AnonAmpersandAmpersand Expression.And
+  , (inj .) . Expression.Power            <$ symbol AnonStarStar
-      <|> mk AnonOr Expression.Or
+  , (inj .) . Expression.DividedBy        <$ symbol AnonSlash
-      <|> mk AnonPipePipe Expression.Or
+  , (inj .) . Expression.Modulo           <$ symbol AnonPercent
-      <|> mk AnonLAngleLAngle Expression.LShift
+  , (inj .) . Expression.And              <$ (symbol AnonAnd <|> symbol AnonAmpersandAmpersand)
-      <|> mk AnonRAngleRAngle Expression.RShift
+  , (inj .) . Expression.BAnd             <$ symbol AnonAmpersand
-      <|> mk AnonEqualEqual Expression.Equal
+  , (inj .) . Expression.Or               <$ (symbol AnonOr <|> symbol AnonPipePipe)
-      <|> mkNot AnonBangEqual Expression.Equal
+  , (inj .) . Expression.BOr              <$ symbol AnonPipe
-       -- TODO: Distinguish `===` from `==` ?
+  , (inj .) . Expression.BXOr             <$ symbol AnonCaret
-      <|> mk AnonEqualEqualEqual Expression.Equal
+  , (inj .) . Expression.Equal            <$ (symbol AnonEqualEqual <|> symbol AnonEqualEqualEqual <|> symbol AnonEqualTilde)
-      <|> mk AnonLAngleEqualRAngle Expression.Comparison
+  , (inj .) . invert Expression.Equal     <$ (symbol AnonBangEqual <|> symbol AnonBangTilde)
-      -- TODO: Distinuish `=~` and `!~` ?
+  , (inj .) . Expression.LShift           <$ symbol AnonLAngleLAngle
-      <|> mk AnonEqualTilde Expression.Equal
+  , (inj .) . Expression.RShift           <$ symbol AnonRAngleRAngle
-      <|> mkNot AnonBangTilde Expression.Equal
+  , (inj .) . Expression.Comparison       <$ symbol AnonLAngleEqualRAngle
-      <|> mk AnonLAngle Expression.LessThan
+  , (inj .) . Expression.LessThan         <$ symbol AnonLAngle
-      <|> mk AnonLAngleEqual Expression.LessThanEqual
+  , (inj .) . Expression.GreaterThan      <$ symbol AnonRAngle
-      <|> mk AnonRAngle Expression.GreaterThan
+  , (inj .) . Expression.LessThanEqual    <$ symbol AnonLAngleEqual
-      <|> mk AnonRAngleEqual Expression.GreaterThanEqual
+  , (inj .) . Expression.GreaterThanEqual <$ symbol AnonRAngleEqual
-      <|> mk AnonAmpersand Expression.BAnd
+  ])
-      <|> mk AnonCaret Expression.BXOr
+  where invert cons a b = Expression.Not (makeTerm1 (cons a b))
-      <|> mk AnonPipe Expression.BOr
-      <|> mk AnonPlus Expression.Plus
-      <|> mk AnonSlash Expression.DividedBy
-      <|> mk AnonPercent Expression.Modulo
-      <|> mk AnonStarStar Expression.Power
-      -- TODO: binary minus and binary star (hidden nodes). Doesn't work b/c we
-      -- can't match hidden nodes (they aren't in the tree).
-      -- <|> mk HiddenBinaryMinus Expression.Minus
-      -- FIXME: This falls through to always assign binary as minus, which isn't correct.
-      <|> makeTerm loc <$> (Expression.Minus lexpression <$> (expression <|> emptyTerm))
-      where mk s constr = makeTerm loc <$> (symbol s *> (constr lexpression <$> expression))
-            mkNot s constr = makeTerm loc <$ symbol s <*> (Expression.Not <$> (makeTerm <$> location <*> (constr lexpression <$> expression)))
 
 conditional :: Assignment
 conditional = makeTerm <$> symbol Conditional <*> children (Statement.If <$> expression <*> expression <*> expression)
@@ -398,3 +389,15 @@ emptyStatement = makeTerm <$> symbol EmptyStatement <*> (Syntax.Empty <$ source
 
 invert :: Assignment -> Assignment
 invert term = makeTerm <$> location <*> fmap Expression.Not term
+
+-- | Match a term optionally preceded by comment(s), or a sequence of comments if the term is not present.
+term :: Assignment -> Assignment
+term term = contextualize comment term <|> makeTerm1 <$> (Syntax.Context <$> some1 comment <*> emptyTerm)
+
+-- | Match infix terms separated by any of a list of operators, assigning any comments following each operand.
+infixTerm :: HasCallStack
+          => Assignment
+          -> Assignment
+          -> [Assignment.Assignment (AST Grammar) Grammar (Term -> Term -> Union Syntax Term)]
+          -> Assignment.Assignment (AST Grammar) Grammar (Union Syntax Term)
+infixTerm = infixContext comment

test/fixtures/ruby/hash.A.rb

@@ -1 +1,12 @@
 { :key1 => "value", :key2 => 1, "key3" => false, :"symbol_key" => 10 }
+{}
+{
+  #comment
+  a: 1, # comment
+  #comment
+  'a' => 1
+  #comment
+}
+{
+  #foo
+}

test/fixtures/ruby/hash.diffA-B.txt

@@ -17,4 +17,18 @@
     ->(Boolean) })
   {-(Pair
       (SymbolLiteral)
-      (IntegerLiteral))-}))
+      (IntegerLiteral))-})
+{-(Object)-}
+{-(Object
+    (Comment)
+    (Pair
+      (Identifier)
+      (IntegerLiteral))
+    (Comment)
+    (Comment)
+    (Pair
+      (StringLiteral)
+      (IntegerLiteral))
+    (Comment))-}
+{-(Object
+    (Comment))-})

test/fixtures/ruby/hash.diffB-A.txt

@@ -17,4 +17,18 @@
     ->(Boolean) })
   {+(Pair
       (SymbolLiteral)
-      (IntegerLiteral))+}))
+      (IntegerLiteral))+})
+{+(Object)+}
+{+(Object
+    (Comment)
+    (Pair
+      (Identifier)
+      (IntegerLiteral))
+    (Comment)
+    (Comment)
+    (Pair
+      (StringLiteral)
+      (IntegerLiteral))
+    (Comment))+}
+{+(Object
+    (Comment))+})

test/fixtures/ruby/hash.parseA.txt

@@ -11,4 +11,18 @@
       (Boolean))
     (Pair
       (SymbolLiteral)
-      (IntegerLiteral))))
+      (IntegerLiteral)))
+  (Object)
+  (Object
+    (Comment)
+    (Pair
+      (Identifier)
+      (IntegerLiteral))
+    (Comment)
+    (Comment)
+    (Pair
+      (StringLiteral)
+      (IntegerLiteral))
+    (Comment))
+  (Object
+    (Comment)))