Merge branch 'master' into bump-grpc-haskell

.gitmodules

@@ -9,7 +9,7 @@
 	url = https://github.com/robrix/freer-cofreer.git
 [submodule "vendor/fastsum"]
 	path = vendor/fastsum
-	url = git@github.com:patrickt/fastsum.git
+	url = https://github.com/patrickt/fastsum.git
 [submodule "vendor/semilattices"]
 	path = vendor/semilattices
 	url = https://github.com/robrix/semilattices.git

src/Data/Reprinting/Operator.hs

@@ -1,7 +1,13 @@
 module Data.Reprinting.Operator
   ( Operator (..)
+  , Direction (..)
   ) where
 
+data Direction
+  = Less
+  | Greater
+    deriving (Show, Eq)
+
 -- | A sum type representing every concievable infix operator a
 -- language can define. These are handled by instances of 'Concrete'
 -- and given appropriate precedence.
@@ -9,4 +15,26 @@ data Operator
   = Add
   | Multiply
   | Subtract
+  | Divide
+  | Modulus
+  | Raise
+  | FloorDivide
+  | RegexMatch
+  | RegexNotMatch
+  | LogicalOr
+  | LogicalAnd
+  | LogicalNot
+  | LogicalXor
+  | BinaryOr
+  | BinaryAnd
+  | BinaryXor
+  | BinaryComplement
+  | NumericNegate
+  | LeftShift
+  | RightShift
+  | Eql
+  | StrictEql
+  | Compare Direction
+  | CompareEql Direction
+  | Spaceship
     deriving (Show, Eq)

src/Data/Reprinting/Scope.hs

@@ -6,22 +6,37 @@ module Data.Reprinting.Scope
 
 import Data.Reprinting.Operator
 
+
 -- | A 'Scope' represents a scope in which other tokens can be
--- interpreted. For example, in the 'Imperative' context a 'TSep'
+-- interpreted. For example, in the 'Imperative' context a 'Sep'
 -- could be a semicolon or newline, whereas in a 'List' context a
--- 'TSep' is probably going to be a comma.
+-- 'Sep' is probably going to be a comma.
+-- TODO: look into sharing control-flow constructs with 'Flow'
 data Scope
-  = List
-  | Hash
-  | Pair
-  | Method
-  | Function
-  | Call
-  | Params
-  | Return
-  | If
-  | InfixL Operator Int
-  | Imperative
+  = List                -- ^ List literals (usually comma-separated, in square brackets)
+  | Hash                -- ^ Hashes (key-value pairs, in curly brackets)
+  | Pair                -- ^ Colon-separated key-value pairs
+  | Slice               -- ^ Range-selection context, as in Go or Python
+  | Method              -- ^ Member-function declaration
+  | Atom                -- ^ Quoted symbols, e.g. Ruby Symbol
+  | Function            -- ^ Function declaration
+  | Namespace           -- ^ Namespace/module context
+  | Call                -- ^ Function call (usually comma-separated arguments)
+  | Params              -- ^ Function parameters (ibid.)
+  | Return              -- ^ Zero or more values
+  | Loop                -- ^ @for@, @while@, @foreach@ loops
+  | If                  -- ^ Conditionals
+  | Case                -- ^ @case@ or @switch@ context
+  | InfixL Operator Int -- ^ Left-associative operators, with context
+  | Prefix Operator     -- ^ Prefix operators
+  | Indexing            -- ^ Single-element array/list indexing
+  | Imperative          -- ^ ALGOL-style top-to-bottom int
+  | Interpolation       -- ^ String interpolation
+  | Catch               -- ^ @try@
+  | Finally             -- ^ @except@
+  | BeginBlock          -- ^ Ruby-specific: @BEGIN@
+  | EndBlock            -- ^ Ruby-specific: @END@
+  | Class               -- ^ Class definition
     deriving (Show, Eq)
 
 precedenceOf :: [Scope] -> Int

src/Data/Reprinting/Token.hs

@@ -4,6 +4,7 @@ module Data.Reprinting.Token
   , isControl
   , Element (..)
   , Control (..)
+  , Flow (..)
   ) where
 
 import Data.Text (Text)
@@ -31,15 +32,39 @@ isControl _ = False
 -- and are interpreted into language-specific representations at a
 -- later point in the reprinting pipeline.
 data Element
-  = Run Text   -- ^ A literal chunk of text.
-  | Truth Bool -- ^ A boolean value.
-  | Nullity    -- ^ @null@ or @nil@ or some other zero value.
-  | Sep        -- ^ Some sort of delimiter, interpreted in some 'Context'.
-  | Sym        -- ^ Some sort of symbol, interpreted in some 'Context'.
-  | Then
+  = Run Text         -- ^ A literal chunk of text.
+  | Truth Bool       -- ^ A boolean value.
+  | Glyph Text       -- ^ A glyph like 'a' or #a.
+  | Nullity          -- ^ @null@ or @nil@ or some other zero value.
+  | Sep              -- ^ Some sort of delimiter, interpreted in some 'Context'.
+  | Sym              -- ^ Some sort of symbol, interpreted in some 'Context'.
+  | Open             -- ^ The beginning of some 'Context', such as an @[@ or @{@.
+  | Close            -- ^ The opposite of 'Open'.
+  | Access           -- ^ Member/method access
+  | Resolve          -- ^ Namespace/package resolution
+  | Assign           -- ^ Variable binding
+  | Self             -- ^ @self@ or @this@
+  | Superclass       -- ^ @super@
+  | Flow Flow        -- ^ Control-flow token (@if@, @else@, @for@...)
+  | Extends          -- ^ Subclassing indicator (syntax varies)
+    deriving (Eq, Show)
+
+-- | Helper datum to corral control-flow entities like @while@, @for@,
+-- etc. Usually corresponds to a keyword in a given language.
+data Flow
+  = Break
+  | Continue
   | Else
-  | Open       -- ^ The beginning of some 'Context', such as an @[@ or @{@.
-  | Close      -- ^ The opposite of 'TOpen'.
+  | For
+  | Foreach
+  | In     -- ^ Usually associated with 'Foreach' loops
+  | Rescue -- ^ AKA @catch@ in most languages
+  | Retry
+  | Switch -- ^ AKA @case@
+  | Then   -- ^ The true-branch of @if@-statements
+  | Try
+  | While
+  | Yield
     deriving (Eq, Show)
 
 -- | 'Control' tokens describe information about some AST's context.

src/Data/Syntax.hs

@@ -11,6 +11,7 @@ import Data.Location
 import qualified Data.Set as Set
 import Data.Sum
 import Data.Term
+import qualified Data.Reprinting.Token as Token
 import GHC.Types (Constraint)
 import GHC.TypeLits
 import Diffing.Algorithm
@@ -165,7 +166,7 @@ instance Evaluatable Identifier where
   eval _ (Identifier name) = pure (LvalLocal name)
 
 instance Tokenize Identifier where
-  tokenize = yield . Run . formatName . Data.Syntax.name
+  tokenize = yield . Token.Run . formatName . Data.Syntax.name
 
 instance FreeVariables1 Identifier where
   liftFreeVariables _ (Identifier x) = Set.singleton x

src/Data/Syntax/Directive.hs

@@ -2,15 +2,18 @@
 {-# OPTIONS_GHC -Wno-missing-export-lists #-}
 module Data.Syntax.Directive where
 
-import           Data.Abstract.Evaluatable
-import           Data.Abstract.Module (ModuleInfo (..))
-import           Data.JSON.Fields
-import           Data.Span
+import Prologue
+
 import qualified Data.Text as T
-import           Diffing.Algorithm
-import           Prologue
 import           Proto3.Suite.Class
 
+import Data.Abstract.Evaluatable
+import Data.Abstract.Module (ModuleInfo (..))
+import Data.JSON.Fields
+import Data.Span
+import Diffing.Algorithm
+import Reprinting.Tokenize
+
 -- A file directive like the Ruby constant `__FILE__`.
 data File a = File
   deriving (Declarations1, Diffable, Eq, Foldable, FreeVariables1, Functor, Generic1, Hashable1, Ord, Show, ToJSONFields1, Traversable, Named1, Message1, NFData1)
@@ -22,6 +25,10 @@ instance Show1 File where liftShowsPrec = genericLiftShowsPrec
 instance Evaluatable File where
   eval _ File = rvalBox =<< (string . T.pack . modulePath <$> currentModule)
 
+-- We may need a separate token class for these given additional languages
+instance Tokenize File where
+  tokenize _ = yield . Run $ "__FILE__"
+
 
 -- A line directive like the Ruby constant `__LINE__`.
 data Line a = Line
@@ -33,3 +40,7 @@ instance Show1 Line where liftShowsPrec = genericLiftShowsPrec
 
 instance Evaluatable Line where
   eval _ Line = rvalBox =<< (integer . fromIntegral . posLine . spanStart <$> currentSpan)
+
+-- PT TODO: proper token for this
+instance Tokenize Line where
+  tokenize _ = yield . Run $ "__FILE__"

src/Data/Syntax/Expression.hs

@@ -6,6 +6,7 @@ import Prelude hiding (null)
 import Prologue hiding (This, index, null)
 
 import Data.Fixed
+import Data.List (intersperse)
 import Proto3.Suite.Class
 
 import           Control.Abstract.ScopeGraph as ScopeGraph
@@ -15,6 +16,7 @@ import           Data.JSON.Fields
 import qualified Data.Reprinting.Scope as Scope
 import           Diffing.Algorithm hiding (Delete)
 import           Reprinting.Tokenize
+import           Data.Reprinting.Token (Element (..))
 import qualified Data.Reprinting.Token as Token
 import qualified Data.List.NonEmpty as NonEmpty
 
@@ -52,6 +54,9 @@ instance Evaluatable LessThan where
     go x = case x of
       (LessThan a b)         -> liftComparison (Concrete (<)) a b
 
+instance Tokenize LessThan where
+  tokenize LessThan{..} = within' (Scope.InfixL (Compare Less) 4) $ lhs *> yield Token.Sym <* rhs
+
 data LessThanEqual a = LessThanEqual { lhs :: a, rhs :: a }
   deriving (Declarations1, Diffable, Eq, Foldable, FreeVariables1, Functor, Generic1, Hashable1, Ord, Show, ToJSONFields1, Traversable, Named1, Message1, NFData1)
 
@@ -64,6 +69,9 @@ instance Evaluatable LessThanEqual where
     go x = case x of
       (LessThanEqual a b)         -> liftComparison (Concrete (<=)) a b
 
+instance Tokenize LessThanEqual where
+  tokenize LessThanEqual{..} = within' (Scope.InfixL (CompareEql Less) 4) $ lhs *> yield Token.Sym <* rhs
+
 data GreaterThan a = GreaterThan { lhs :: a, rhs :: a }
   deriving (Declarations1, Diffable, Eq, Foldable, FreeVariables1, Functor, Generic1, Hashable1, Ord, Show, ToJSONFields1, Traversable, Named1, Message1, NFData1)
 
@@ -76,6 +84,9 @@ instance Evaluatable GreaterThan where
     go x = case x of
       (GreaterThan a b)         -> liftComparison (Concrete (>)) a b
 
+instance Tokenize GreaterThan where
+  tokenize GreaterThan{..} = within' (Scope.InfixL (Compare Greater) 4) $ lhs *> yield Token.Sym <* rhs
+
 data GreaterThanEqual a = GreaterThanEqual { lhs :: a, rhs :: a }
   deriving (Declarations1, Diffable, Eq, Foldable, FreeVariables1, Functor, Generic1, Hashable1, Ord, Show, ToJSONFields1, Traversable, Named1, Message1, NFData1)
 
@@ -88,6 +99,9 @@ instance Evaluatable GreaterThanEqual where
     go x = case x of
       (GreaterThanEqual a b)         -> liftComparison (Concrete (>=)) a b
 
+instance Tokenize GreaterThanEqual where
+  tokenize GreaterThanEqual{..} = within' (Scope.InfixL (CompareEql Greater) 4) $ lhs *> yield Token.Sym <* rhs
+
 data Equal a = Equal { lhs :: a, rhs :: a }
   deriving (Declarations1, Diffable, Eq, Foldable, FreeVariables1, Functor, Generic1, Hashable1, Ord, Show, ToJSONFields1, Traversable, Named1, Message1, NFData1)
 
@@ -102,6 +116,9 @@ instance Evaluatable Equal where
       -- We need some mechanism to customize this behavior per-language.
       (Equal a b)         -> liftComparison (Concrete (==)) a b
 
+instance Tokenize Equal where
+  tokenize Equal{..} = within' (Scope.InfixL Eql 4) $ lhs *> yield Token.Sym <* rhs
+
 data StrictEqual a = StrictEqual { lhs :: a, rhs :: a }
   deriving (Declarations1, Diffable, Eq, Foldable, FreeVariables1, Functor, Generic1, Hashable1, Ord, Show, ToJSONFields1, Traversable, Named1, Message1, NFData1)
 
@@ -116,6 +133,9 @@ instance Evaluatable StrictEqual where
       -- We need some mechanism to customize this behavior per-language.
       (StrictEqual a b)         -> liftComparison (Concrete (==)) a b
 
+instance Tokenize StrictEqual where
+  tokenize StrictEqual{..} = within' (Scope.InfixL StrictEql 4) $ lhs *> yield Token.Sym <* rhs
+
 data Comparison a = Comparison { lhs :: a, rhs :: a }
   deriving (Declarations1, Diffable, Eq, Foldable, FreeVariables1, Functor, Generic1, Hashable1, Ord, Show, ToJSONFields1, Traversable, Named1, Message1, NFData1)
 
@@ -128,6 +148,9 @@ instance Evaluatable Comparison where
     go x = case x of
       (Comparison a b)         -> liftComparison (Concrete (==)) a b
 
+instance Tokenize Comparison where
+  tokenize Comparison{..} = within' (Scope.InfixL Spaceship 4) $ lhs *> yield Token.Sym <* rhs
+
 data Plus a = Plus { lhs :: a, rhs :: a }
   deriving (Declarations1, Diffable, Eq, Foldable, FreeVariables1, Functor, Generic1, Hashable1, Ord, Show, ToJSONFields1, Traversable, Named1, Message1, NFData1)
 
@@ -181,6 +204,9 @@ instance Evaluatable DividedBy where
   eval eval t = rvalBox =<< (traverse (eval >=> Abstract.value) t >>= go) where
     go (DividedBy a b)     = liftNumeric2 div' a b where div'   = liftIntegralFrac div (/)
 
+instance Tokenize DividedBy where
+  tokenize DividedBy{..} = within' (Scope.InfixL Divide 7) $ lhs *> yield Token.Sym <* rhs
+
 data Modulo a = Modulo { lhs :: a, rhs :: a }
   deriving (Declarations1, Diffable, Eq, Foldable, FreeVariables1, Functor, Generic1, Hashable1, Ord, Show, ToJSONFields1, Traversable, Named1, Message1, NFData1)
 
@@ -192,6 +218,9 @@ instance Evaluatable Modulo where
   eval eval t = rvalBox =<< (traverse (eval >=> Abstract.value) t >>= go) where
     go (Modulo a b)        = liftNumeric2 mod'' a b where mod'' = liftIntegralFrac mod mod'
 
+instance Tokenize Modulo where
+  tokenize Modulo{..} = within' (Scope.InfixL Modulus 7) $ lhs *> yield Token.Sym <* rhs
+
 data Power a = Power { lhs :: a, rhs :: a }
   deriving (Declarations1, Diffable, Eq, Foldable, FreeVariables1, Functor, Generic1, Hashable1, Ord, Show, ToJSONFields1, Traversable, Named1, Message1, NFData1)
 
@@ -203,6 +232,9 @@ instance Evaluatable Power where
   eval eval t = rvalBox =<< (traverse (eval >=> Abstract.value) t >>= go) where
     go (Power a b)         = liftNumeric2 liftedExponent a b
 
+instance Tokenize Power where
+  tokenize Power{..} = within' (Scope.InfixL Raise 9) $ lhs *> yield Token.Sym <* rhs
+
 newtype Negate a = Negate { value :: a }
   deriving (Declarations1, Diffable, Eq, Foldable, FreeVariables1, Functor, Generic1, Hashable1, Ord, Show, ToJSONFields1, Traversable, Named1, Message1, NFData1)
 
@@ -214,6 +246,9 @@ instance Evaluatable Negate where
   eval eval t = rvalBox =<< (traverse (eval >=> Abstract.value) t >>= go) where
     go (Negate a)          = liftNumeric negate a
 
+instance Tokenize Negate where
+  tokenize Negate{..} = within' (Scope.Prefix NumericNegate) $ yield Token.Sym <* value
+
 data FloorDivision a = FloorDivision { lhs :: a, rhs :: a }
   deriving (Declarations1, Diffable, Eq, Foldable, FreeVariables1, Functor, Generic1, Hashable1, Ord, Show, ToJSONFields1, Traversable, Named1, Message1, NFData1)
 
@@ -225,6 +260,9 @@ instance Evaluatable FloorDivision where
   eval eval t = rvalBox =<< (traverse (eval >=> Abstract.value) t >>= go) where
     go (FloorDivision a b) = liftNumeric2 liftedFloorDiv a b
 
+instance Tokenize FloorDivision where
+  tokenize FloorDivision{..} = within' (Scope.InfixL FloorDivide 7) $ lhs *> yield Token.Sym <* rhs
+
 -- | Regex matching operators (Ruby's =~ and ~!)
 data Matches a = Matches { lhs :: a, rhs :: a }
   deriving (Declarations1, Diffable, Eq, Foldable, FreeVariables1, Functor, Generic1, Hashable1, Ord, Show, ToJSONFields1, Traversable, Named1, Message1, NFData1)
@@ -234,6 +272,9 @@ instance Ord1 Matches where liftCompare = genericLiftCompare
 instance Show1 Matches where liftShowsPrec = genericLiftShowsPrec
 instance Evaluatable Matches
 
+instance Tokenize Matches where
+  tokenize Matches{..} = within' (Scope.InfixL RegexMatch 1) $ lhs *> yield Token.Sym <* rhs
+
 data NotMatches a = NotMatches { lhs :: a, rhs :: a }
   deriving (Declarations1, Diffable, Eq, Foldable, FreeVariables1, Functor, Generic1, Hashable1, Ord, Show, ToJSONFields1, Traversable, Named1, Message1, NFData1)
 
@@ -242,6 +283,9 @@ instance Ord1 NotMatches where liftCompare = genericLiftCompare
 instance Show1 NotMatches where liftShowsPrec = genericLiftShowsPrec
 instance Evaluatable NotMatches
 
+instance Tokenize NotMatches where
+  tokenize NotMatches{..} = within' (Scope.InfixL RegexNotMatch 1) $ lhs *> yield Token.Sym <* rhs
+
 data Or a = Or { lhs :: a, rhs :: a }
   deriving (Declarations1, Diffable, Eq, Foldable, FreeVariables1, Functor, Generic1, Hashable1, Ord, Show, ToJSONFields1, Traversable, Named1, Message1, NFData1)
 
@@ -254,6 +298,9 @@ instance Evaluatable Or where
     a' <- eval a >>= Abstract.value
     ifthenelse a' (rvalBox a') (eval b)
 
+instance Tokenize Or where
+  tokenize Or{..} = within' (Scope.InfixL LogicalOr 2) $ lhs *> yield Token.Sym <* rhs
+
 data And a = And { lhs :: a, rhs :: a }
   deriving (Declarations1, Diffable, Eq, Foldable, FreeVariables1, Functor, Generic1, Hashable1, Ord, Show, ToJSONFields1, Traversable, Named1, Message1, NFData1)
 
@@ -266,6 +313,9 @@ instance Evaluatable And where
       cond <- a
       ifthenelse cond b (pure cond)
 
+instance Tokenize And where
+  tokenize And{..} = within' (Scope.InfixL LogicalAnd 2) $ lhs *> yield Token.Sym <* rhs
+
 newtype Not a = Not { value :: a }
   deriving (Declarations1, Diffable, Eq, Foldable, FreeVariables1, Functor, Generic1, Hashable1, Ord, Show, ToJSONFields1, Traversable, Named1, Message1, NFData1)
 
@@ -277,6 +327,9 @@ instance Evaluatable Not where
   eval eval t = rvalBox =<< go (fmap (eval >=> Abstract.value) t) where
     go (Not a) = a >>= asBool >>= boolean . not
 
+instance Tokenize Not where
+  tokenize Not{..} = within' (Scope.Prefix LogicalNot) $ yield Token.Sym <* value
+
 data XOr a = XOr { lhs :: a, rhs :: a }
   deriving (Declarations1, Diffable, Eq, Foldable, FreeVariables1, Functor, Generic1, Hashable1, Ord, Show, ToJSONFields1, Traversable, Named1, Message1, NFData1)
 
@@ -289,6 +342,9 @@ instance Evaluatable XOr where
   eval eval t = rvalBox =<< go (fmap (eval >=> Abstract.value) t) where
     go (XOr a b) = liftA2 (/=) (a >>= asBool) (b >>= asBool) >>= boolean
 
+instance Tokenize XOr where
+  tokenize XOr{..} = within' (Scope.InfixL LogicalXor 2) $ lhs *> yield Token.Sym <* rhs
+
 -- | Javascript delete operator
 newtype Delete a = Delete { value :: a }
   deriving (Declarations1, Diffable, Eq, Foldable, FreeVariables1, Functor, Generic1, Hashable1, Ord, Show, ToJSONFields1, Traversable, Named1, Message1, NFData1)
@@ -340,7 +396,7 @@ instance Show1 Typeof where liftShowsPrec = genericLiftShowsPrec
 instance Evaluatable Typeof
 
 -- | Bitwise operators.
-data BOr a = BOr { left :: a, right :: a }
+data BOr a = BOr { lhs :: a, rhs :: a }
   deriving (Declarations1, Diffable, Eq, Foldable, FreeVariables1, Functor, Generic1, Hashable1, Ord, Show, ToJSONFields1, Traversable, Named1, Message1, NFData1)
 
 instance Eq1 BOr where liftEq = genericLiftEq
@@ -352,7 +408,10 @@ instance Evaluatable BOr where
     b' <- eval b >>= Abstract.value >>= castToInteger
     liftBitwise2 (.|.) a' b' >>= rvalBox
 
-data BAnd a = BAnd { left :: a, right :: a }
+instance Tokenize BOr where
+  tokenize BOr{..} = within' (Scope.InfixL BinaryOr 4) $ lhs *> yield Token.Sym <* rhs
+
+data BAnd a = BAnd { lhs :: a, rhs :: a }
   deriving (Declarations1, Diffable, Eq, Foldable, FreeVariables1, Functor, Generic1, Hashable1, Ord, Show, ToJSONFields1, Traversable, Named1, Message1, NFData1)
 
 instance Eq1 BAnd where liftEq = genericLiftEq
@@ -364,8 +423,10 @@ instance Evaluatable BAnd where
     b' <- eval b >>= Abstract.value >>= castToInteger
     liftBitwise2 (.&.) a' b' >>= rvalBox
 
+instance Tokenize BAnd where
+  tokenize BAnd{..} = within' (Scope.InfixL BinaryAnd 5) $ lhs *> yield Token.Sym <* rhs
 
-data BXOr a = BXOr { left :: a, right :: a }
+data BXOr a = BXOr { lhs :: a, rhs :: a }
   deriving (Declarations1, Diffable, Eq, Foldable, FreeVariables1, Functor, Generic1, Hashable1, Ord, Show, ToJSONFields1, Traversable, Named1, Message1, NFData1)
 
 instance Eq1 BXOr where liftEq = genericLiftEq
@@ -377,7 +438,10 @@ instance Evaluatable BXOr where
     b' <- eval b >>= Abstract.value >>= castToInteger
     liftBitwise2 xor a' b' >>= rvalBox
 
-data LShift a = LShift { left :: a, right :: a }
+instance Tokenize BXOr where
+  tokenize BXOr{..} = within' (Scope.InfixL BinaryXor 5) $ lhs *> yield Token.Sym <* rhs
+
+data LShift a = LShift { lhs :: a, rhs :: a }
   deriving (Declarations1, Diffable, Eq, Foldable, FreeVariables1, Functor, Generic1, Hashable1, Ord, Show, ToJSONFields1, Traversable, Named1, Message1, NFData1)
 
 instance Eq1 LShift where liftEq = genericLiftEq
@@ -391,7 +455,10 @@ instance Evaluatable LShift where
     where
       shiftL' a b = shiftL a (fromIntegral (toInteger b))
 
-data RShift a = RShift { left :: a, right :: a }
+instance Tokenize LShift where
+  tokenize LShift{..} = within' (Scope.InfixL LeftShift 4) $ lhs *> yield Token.Sym <* rhs
+
+data RShift a = RShift { lhs :: a, rhs :: a }
   deriving (Declarations1, Diffable, Eq, Foldable, FreeVariables1, Functor, Generic1, Hashable1, Ord, Show, ToJSONFields1, Traversable, Named1, Message1, NFData1)
 
 instance Eq1 RShift where liftEq = genericLiftEq
@@ -405,7 +472,10 @@ instance Evaluatable RShift where
     where
       shiftR' a b = shiftR a (fromIntegral (toInteger b))
 
-data UnsignedRShift a = UnsignedRShift { left :: a, right :: a }
+instance Tokenize RShift where
+  tokenize RShift{..} = within' (Scope.InfixL RightShift 4) $ lhs *> yield Token.Sym <* rhs
+
+data UnsignedRShift a = UnsignedRShift { lhs :: a, rhs :: a }
   deriving (Declarations1, Diffable, Eq, Foldable, FreeVariables1, Functor, Generic1, Hashable1, Ord, Show, ToJSONFields1, Traversable, Named1, Message1, NFData1)
 
 instance Eq1 UnsignedRShift where liftEq = genericLiftEq
@@ -429,6 +499,9 @@ instance Evaluatable Complement where
     a' <- eval a >>= Abstract.value >>= castToInteger
     liftBitwise complement a' >>= rvalBox
 
+instance Tokenize Complement where
+  tokenize Complement{..} = within' (Scope.Prefix BinaryComplement) $ yield Token.Sym <* value
+
 -- | Member Access (e.g. a.b)
 data MemberAccess a = MemberAccess { lhs :: a, rhs :: Name }
   deriving (Diffable, Eq, Foldable, FreeVariables1, Functor, Generic1, Hashable1, Ord, Show, ToJSONFields1, Traversable, Named1, Message1, NFData1)
@@ -455,6 +528,9 @@ instance Evaluatable MemberAccess where
 
     pure $! LvalMember ptr propName
 
+instance Tokenize MemberAccess where
+  tokenize MemberAccess{..} = lhs *> yield Access *> yield (Run (formatName rhs))
+
 -- | Subscript (e.g a[1])
 data Subscript a = Subscript { lhs :: a, rhs :: [a] }
   deriving (Declarations1, Diffable, Eq, Foldable, FreeVariables1, Functor, Generic1, Hashable1, Ord, Show, ToJSONFields1, Traversable, Named1, Message1, NFData1)
@@ -469,6 +545,9 @@ instance Evaluatable Subscript where
   eval eval (Subscript l [r]) = Rval <$> join (index <$> (eval l >>= Abstract.value) <*> (eval r >>= Abstract.value))
   eval _ (Subscript _ _)   = rvalBox =<< throwUnspecializedError (UnspecializedError "Eval unspecialized for subscript with slices")
 
+instance Tokenize Subscript where
+  tokenize Subscript{..} = lhs *> within' Scope.Indexing (sequenceA_ (intersperse (yield Token.Sep) rhs))
+
 data Member a = Member { lhs :: a, rhs :: a }
   deriving (Declarations1, Diffable, Eq, Foldable, FreeVariables1, Functor, Generic1, Hashable1, Ord, Show, ToJSONFields1, Traversable, Named1, Message1, NFData1)
 
@@ -478,6 +557,9 @@ instance Show1 Member where liftShowsPrec = genericLiftShowsPrec
 
 instance Evaluatable Member where
 
+instance Tokenize Member where
+  tokenize Member{..} = lhs *> yield Token.Access <* rhs
+
 -- | Enumeration (e.g. a[1:10:1] in Python (start at index 1, stop at index 10, step 1 element from start to stop))
 data Enumeration a = Enumeration { enumerationStart :: !a, enumerationEnd :: !a, enumerationStep :: !a }
   deriving (Declarations1, Diffable, Eq, Foldable, FreeVariables1, Functor, Generic1, Hashable1, Ord, Show, ToJSONFields1, Traversable, Named1, Message1, NFData1)
@@ -489,6 +571,8 @@ instance Show1 Enumeration where liftShowsPrec = genericLiftShowsPrec
 -- TODO: Implement Eval instance for Enumeration
 instance Evaluatable Enumeration
 
+instance Tokenize Enumeration where
+  tokenize Enumeration{..} = within Scope.Slice $ enumerationStart *> enumerationEnd *> enumerationStep
 
 -- | InstanceOf (e.g. a instanceof b in JavaScript
 data InstanceOf a = InstanceOf { instanceOfSubject :: !a, instanceOfObject :: !a }
@@ -515,6 +599,10 @@ instance Evaluatable ScopeResolution where
   eval eval (ScopeResolution xs) = Rval <$> foldl1 f (fmap (eval >=> address) xs)
     where f ns id = ns >>= flip evaluateInScopedEnv id
 
+instance Tokenize ScopeResolution where
+  tokenize (ScopeResolution (a :| rest)) = 
+    a *> for_ rest (yield Token.Resolve *>)
+
 instance Declarations1 ScopeResolution where
   liftDeclaredName declaredName = declaredName . NonEmpty.last . scopes
 
@@ -585,9 +673,15 @@ instance Show1 Super where liftShowsPrec = genericLiftShowsPrec
 instance Evaluatable Super where
   eval _ Super = Rval <$> (maybeM (box unit) =<< self)
 
+instance Tokenize Super where
+  tokenize _ = yield Superclass
+
 data This a = This
   deriving (Diffable, Eq, Foldable, Functor,  Generic1, Ord, Show, Traversable, FreeVariables1, Declarations1, ToJSONFields1, Hashable1, Named1, Message1, NFData1)
 
+instance Tokenize This where
+  tokenize _ = yield Self
+
 instance Eq1 This where liftEq = genericLiftEq
 instance Ord1 This where liftCompare = genericLiftCompare
 instance Show1 This where liftShowsPrec = genericLiftShowsPrec

src/Data/Syntax/Literal.hs

@@ -13,6 +13,7 @@ import           Diffing.Algorithm
 import           Numeric.Exts
 import           Proto3.Suite.Class
 import           Reprinting.Tokenize as Tok
+import qualified Data.Reprinting.Scope as Scope
 import           Text.Read (readMaybe)
 
 -- Boolean
@@ -85,6 +86,9 @@ instance Evaluatable Data.Syntax.Literal.Rational where
       parsed = readMaybe @Prelude.Integer (T.unpack trimmed)
     in rvalBox =<< (rational <$> maybe (throwEvalError (RationalFormatError r)) (pure . toRational) parsed)
 
+instance Tokenize Data.Syntax.Literal.Rational where
+  tokenize (Rational t) = yield . Run $ t
+
 -- Complex literals e.g. `3 + 2i`
 newtype Complex a = Complex { value :: Text }
   deriving (Diffable, Eq, Foldable, Functor, Generic1, Hashable1, Ord, Show, Traversable, FreeVariables1, Declarations1, ToJSONFields1, Named1, Message1, NFData1)
@@ -96,6 +100,9 @@ instance Show1 Data.Syntax.Literal.Complex where liftShowsPrec = genericLiftShow
 -- TODO: Implement Eval instance for Complex
 instance Evaluatable Complex
 
+instance Tokenize Complex where
+  tokenize (Complex v) = yield . Run $ v
+
 -- Strings, symbols
 
 newtype String a = String { stringElements :: [a] }
@@ -110,6 +117,9 @@ instance Show1 Data.Syntax.Literal.String where liftShowsPrec = genericLiftShows
 -- TODO: Implement Eval instance for String
 instance Evaluatable Data.Syntax.Literal.String
 
+instance Tokenize Data.Syntax.Literal.String where
+  tokenize = sequenceA_
+
 newtype Character a = Character { characterContent :: Text }
   deriving (Declarations1, Diffable, Eq, Foldable, FreeVariables1, Functor, Generic1, Hashable1, Ord, Show, ToJSONFields1, Traversable, Named1, Message1, NFData1)
 
@@ -119,6 +129,9 @@ instance Show1 Data.Syntax.Literal.Character where liftShowsPrec = genericLiftSh
 
 instance Evaluatable Data.Syntax.Literal.Character
 
+instance Tokenize Character where
+  tokenize = yield . Glyph . characterContent
+
 -- | An interpolation element within a string literal.
 newtype InterpolationElement a = InterpolationElement { interpolationBody :: a }
   deriving (Declarations1, Diffable, Eq, Foldable, FreeVariables1, Functor, Generic1, Hashable1, Ord, Show, ToJSONFields1, Traversable, Named1, Message1, NFData1)
@@ -130,6 +143,9 @@ instance Show1 InterpolationElement where liftShowsPrec = genericLiftShowsPrec
 -- TODO: Implement Eval instance for InterpolationElement
 instance Evaluatable InterpolationElement
 
+instance Tokenize InterpolationElement where
+  tokenize = sequenceA_
+
 -- | A sequence of textual contents within a string literal.
 newtype TextElement a = TextElement { textElementContent :: Text }
   deriving (Diffable, Eq, Foldable, Functor, Generic1, Hashable1, Ord, Show, Traversable, FreeVariables1, Declarations1, ToJSONFields1, Named1, Message1, NFData1)
@@ -149,6 +165,9 @@ isTripleQuoted (TextElement t) =
   let trip = "\"\"\""
   in  T.take 3 t == trip && T.takeEnd 3 t == trip
 
+quoted :: Text -> TextElement a
+quoted t = TextElement ("\"" <> t <> "\"")
+
 -- | A sequence of textual contents within a string literal.
 newtype EscapeSequence a = EscapeSequence { value :: Text }
   deriving (Diffable, Eq, Foldable, Functor, Generic1, Hashable1, Ord, Show, Traversable, FreeVariables1, Declarations1, ToJSONFields1, Named1, Message1, NFData1)
@@ -160,6 +179,9 @@ instance Show1 EscapeSequence where liftShowsPrec = genericLiftShowsPrec
 -- TODO: Implement Eval instance for EscapeSequence
 instance Evaluatable EscapeSequence
 
+instance Tokenize EscapeSequence where
+  tokenize (EscapeSequence e) = yield . Run $ e
+
 data Null a = Null
   deriving (Eq, Ord, Show, Foldable, Traversable, Functor, Generic1, Hashable1, Diffable, FreeVariables1, Declarations1, ToJSONFields1, Named1, Message1, NFData1)
 
@@ -182,6 +204,9 @@ instance Show1 Symbol where liftShowsPrec = genericLiftShowsPrec
 -- TODO: Implement Eval instance for Symbol
 instance Evaluatable Symbol
 
+instance Tokenize Symbol where
+  tokenize s = within Scope.Atom (yield Sym *> sequenceA_ s)
+
 newtype SymbolElement a = SymbolElement { symbolContent :: Text }
   deriving (Declarations1, Diffable, Eq, Foldable, FreeVariables1, Functor, Generic1, Hashable1, Ord, Show, ToJSONFields1, Traversable, Named1, Message1, NFData1)
 
@@ -192,6 +217,9 @@ instance Show1 SymbolElement where liftShowsPrec = genericLiftShowsPrec
 instance Evaluatable SymbolElement where
   eval _ (SymbolElement s) = rvalBox (symbol s)
 
+instance Tokenize SymbolElement where
+  tokenize = yield . Run . symbolContent
+
 newtype Regex a = Regex { regexContent :: Text }
   deriving (Diffable, Eq, Foldable, Functor, Generic1, Hashable1, Ord, Show, Traversable, FreeVariables1, Declarations1, ToJSONFields1, Named1, Message1, NFData1)
 
@@ -205,6 +233,9 @@ instance Show1 Regex where liftShowsPrec = genericLiftShowsPrec
 instance Evaluatable Regex where
   eval _ (Regex x) = rvalBox (regex x)
 
+instance Tokenize Regex where
+  tokenize = yield . Run . regexContent
+
 -- Collections
 
 newtype Array a = Array { arrayElements :: [a] }

src/Data/Syntax/Statement.hs

@@ -14,7 +14,7 @@ import Data.Abstract.Evaluatable as Abstract
 import Control.Abstract.ScopeGraph
 import Data.JSON.Fields
 import Diffing.Algorithm
-import Reprinting.Tokenize
+import Reprinting.Tokenize (Tokenize (..), imperative, within', yield)
 import qualified Data.Reprinting.Token as Token
 import qualified Data.Reprinting.Scope as Scope
 
@@ -57,9 +57,9 @@ instance Evaluatable If where
 instance Tokenize If where
   tokenize If{..} = within' Scope.If $ do
     ifCondition
-    yield Token.Then
+    yield (Token.Flow Token.Then)
     ifThenBody
-    yield Token.Else
+    yield (Token.Flow Token.Else)
     ifElseBody
 
 -- | Else statement. The else condition is any term, that upon successful completion, continues evaluation to the elseBody, e.g. `for ... else` in Python.
@@ -73,6 +73,9 @@ instance Show1 Else where liftShowsPrec = genericLiftShowsPrec
 -- TODO: Implement Eval instance for Else
 instance Evaluatable Else
 
+instance Tokenize Else where
+  tokenize Else{..} = within' Scope.If (yield (Token.Flow Token.Else) *> elseCondition *> yield Token.Sep *> elseBody)
+
 -- TODO: Alternative definition would flatten if/else if/else chains: data If a = If ![(a, a)] !(Maybe a)
 
 -- | Goto statement (e.g. `goto a` in Go).
@@ -98,6 +101,12 @@ instance Show1 Match where liftShowsPrec = genericLiftShowsPrec
 -- TODO: Implement Eval instance for Match
 instance Evaluatable Match
 
+instance Tokenize Match where
+  tokenize Match{..} = do
+    yield (Token.Flow Token.Switch)
+    matchSubject
+    yield (Token.Flow Token.In) -- This may need further refinement
+    matchPatterns
 
 -- | A pattern in a pattern-matching or computed jump control-flow statement, like 'case' in C or JavaScript, 'when' in Ruby, or the left-hand side of '->' in the body of Haskell 'case' expressions.
 data Pattern a = Pattern { value :: !a, patternBody :: !a }
@@ -110,6 +119,8 @@ instance Show1 Pattern where liftShowsPrec = genericLiftShowsPrec
 -- TODO: Implement Eval instance for Pattern
 instance Evaluatable Pattern
 
+instance Tokenize Pattern where
+  tokenize Pattern{..} = within' Scope.Case (value *> patternBody)
 
 -- | A let statement or local binding, like 'a as b' or 'let a = b'.
 data Let a  = Let { letVariable :: !a, letValue :: !a, letBody :: !a }
@@ -167,6 +178,10 @@ instance Evaluatable Assignment where
 
     pure (Rval rhs)
 
+instance Tokenize Assignment where
+  -- Should we be using 'assignmentContext' in here?
+  tokenize Assignment{..} = assignmentTarget *> yield Token.Assign <* assignmentValue
+
 -- | Post increment operator (e.g. 1++ in Go, or i++ in C).
 newtype PostIncrement a = PostIncrement { value :: a }
   deriving (Declarations1, Diffable, Eq, Foldable, FreeVariables1, Functor, Generic1, Hashable1, Ord, Show, ToJSONFields1, Traversable, Named1, Message1, NFData1)
@@ -239,6 +254,9 @@ instance Show1 Yield where liftShowsPrec = genericLiftShowsPrec
 -- TODO: Implement Eval instance for Yield
 instance Evaluatable Yield
 
+instance Tokenize Yield where
+  tokenize (Yield y) = yield (Token.Flow Token.Yield) *> y
+
 
 newtype Break a = Break { value :: a }
   deriving (Declarations1, Diffable, Eq, Foldable, FreeVariables1, Functor, Generic1, Hashable1, Ord, Show, ToJSONFields1, Traversable, Named1, Message1, NFData1)
@@ -250,6 +268,9 @@ instance Show1 Break where liftShowsPrec = genericLiftShowsPrec
 instance Evaluatable Break where
   eval eval (Break x) = Rval <$> (eval x >>= address >>= throwBreak)
 
+instance Tokenize Break where
+  tokenize (Break b) = yield (Token.Flow Token.Break) *> b
+
 newtype Continue a = Continue { value :: a }
   deriving (Declarations1, Diffable, Eq, Foldable, FreeVariables1, Functor, Generic1, Hashable1, Ord, Show, ToJSONFields1, Traversable, Named1, Message1, NFData1)
 
@@ -260,6 +281,9 @@ instance Show1 Continue where liftShowsPrec = genericLiftShowsPrec
 instance Evaluatable Continue where
   eval eval (Continue x) = Rval <$> (eval x >>= address >>= throwContinue)
 
+instance Tokenize Continue where
+  tokenize (Continue c) = yield (Token.Flow Token.Continue) *> c
+
 newtype Retry a = Retry { value :: a }
   deriving (Declarations1, Diffable, Eq, Foldable, FreeVariables1, Functor, Generic1, Hashable1, Ord, Show, ToJSONFields1, Traversable, Named1, Message1, NFData1)
 
@@ -270,6 +294,8 @@ instance Show1 Retry where liftShowsPrec = genericLiftShowsPrec
 -- TODO: Implement Eval instance for Retry
 instance Evaluatable Retry
 
+instance Tokenize Retry where
+  tokenize (Retry r) = yield (Token.Flow Token.Retry) *> r
 
 newtype NoOp a = NoOp { value :: a }
   deriving (Declarations1, Diffable, Eq, Foldable, FreeVariables1, Functor, Generic1, Hashable1, Ord, Show, ToJSONFields1, Traversable, Named1, Message1, NFData1)
@@ -304,6 +330,13 @@ instance Show1 ForEach where liftShowsPrec = genericLiftShowsPrec
 -- TODO: Implement Eval instance for ForEach
 instance Evaluatable ForEach
 
+instance Tokenize ForEach where
+  tokenize ForEach{..} = within' Scope.Loop $ do
+    yield (Token.Flow Token.Foreach)
+    forEachBinding
+    yield (Token.Flow Token.In)
+    forEachSubject
+    forEachBody
 
 data While a = While { whileCondition :: !a, whileBody :: !a }
   deriving (Declarations1, Diffable, Eq, Foldable, FreeVariables1, Functor, Generic1, Hashable1, Ord, Show, ToJSONFields1, Traversable, Named1, Message1, NFData1)
@@ -315,6 +348,12 @@ instance Show1 While where liftShowsPrec = genericLiftShowsPrec
 instance Evaluatable While where
   eval eval While{..} = rvalBox =<< while (eval whileCondition >>= Abstract.value) (eval whileBody >>= Abstract.value)
 
+instance Tokenize While where
+  tokenize While{..} = within' Scope.Loop $ do
+    yield (Token.Flow Token.While)
+    whileCondition
+    whileBody
+
 data DoWhile a = DoWhile { doWhileCondition :: !a, doWhileBody :: !a }
   deriving (Declarations1, Diffable, Eq, Foldable, FreeVariables1, Functor, Generic1, Hashable1, Ord, Show, ToJSONFields1, Traversable, Named1, Message1, NFData1)
 
@@ -348,6 +387,12 @@ instance Show1 Try where liftShowsPrec = genericLiftShowsPrec
 -- TODO: Implement Eval instance for Try
 instance Evaluatable Try
 
+instance Tokenize Try where
+  tokenize Try{..} = do
+    yield (Token.Flow Token.Try)
+    tryBody
+    yield (Token.Flow Token.Rescue)
+    sequenceA_ tryCatch
 
 data Catch a = Catch { catchException :: !a, catchBody :: !a }
   deriving (Declarations1, Diffable, Eq, Foldable, FreeVariables1, Functor, Generic1, Hashable1, Ord, Show, ToJSONFields1, Traversable, Named1, Message1, NFData1)
@@ -359,6 +404,8 @@ instance Show1 Catch where liftShowsPrec = genericLiftShowsPrec
 -- TODO: Implement Eval instance for Catch
 instance Evaluatable Catch
 
+instance Tokenize Catch where
+  tokenize Data.Syntax.Statement.Catch{..} = within' Scope.Catch $ catchException *> catchBody
 
 newtype Finally a = Finally { value :: a }
   deriving (Declarations1, Diffable, Eq, Foldable, FreeVariables1, Functor, Generic1, Hashable1, Ord, Show, ToJSONFields1, Traversable, Named1, Message1, NFData1)
@@ -370,6 +417,8 @@ instance Show1 Finally where liftShowsPrec = genericLiftShowsPrec
 -- TODO: Implement Eval instance for Finally
 instance Evaluatable Finally
 
+instance Tokenize Finally where
+  tokenize (Finally f) = within' Scope.Finally f
 
 -- Scoping
 
@@ -384,6 +433,9 @@ instance Show1 ScopeEntry where liftShowsPrec = genericLiftShowsPrec
 -- TODO: Implement Eval instance for ScopeEntry
 instance Evaluatable ScopeEntry
 
+instance Tokenize ScopeEntry where
+  tokenize (ScopeEntry t) = within' Scope.BeginBlock (sequenceA_ t)
+
 
 -- | ScopeExit (e.g. `END {}` block in Ruby or Perl).
 newtype ScopeExit a = ScopeExit { terms :: [a] }
@@ -395,3 +447,6 @@ instance Show1 ScopeExit where liftShowsPrec = genericLiftShowsPrec
 
 -- TODO: Implement Eval instance for ScopeExit
 instance Evaluatable ScopeExit
+
+instance Tokenize ScopeExit where
+  tokenize (ScopeExit t) = within' Scope.EndBlock (sequenceA_ t)

src/Language/Python/PrettyPrint.hs

@@ -35,10 +35,10 @@ step (Defer el cs)  = case (el, cs) of
   (Close, Imperative:Return:_) -> pure () -- Don't hardwarp or indent for return statements
 
   -- If statements
-  (Open,  If:_)                -> emit "if" *> space
-  (Then,  If:_)                -> emit ":"
-  (Else,  If:xs)               -> endContext (imperativeDepth xs) *> emit "else:"
-  (Close, If:_)                -> pure ()
+  (Open,       If:_)           -> emit "if" *> space
+  (Flow Then,  If:_)           -> emit ":"
+  (Flow Else,  If:xs)          -> endContext (imperativeDepth xs) *> emit "else:"
+  (Close,      If:_)           -> pure ()
 
   -- Booleans
   (Truth True,  _)               -> emit "True"

src/Language/Ruby/Syntax.hs

@@ -12,9 +12,10 @@ import           Data.Abstract.BaseError
 import           Data.Abstract.Evaluatable
 import qualified Data.Abstract.Module as M
 import           Data.Abstract.Path
-import qualified Data.Reprinting.Scope as Scope
 import           Data.JSON.Fields
 import qualified Data.Language as Language
+import qualified Data.Reprinting.Scope as Scope
+import           Data.Reprinting.Token as Token
 import           Diffing.Algorithm
 import           Proto3.Suite.Class
 import           Reprinting.Tokenize
@@ -93,6 +94,13 @@ instance Evaluatable Require where
     bindAll importedEnv
     rvalBox v -- Returns True if the file was loaded, False if it was already loaded. http://ruby-doc.org/core-2.5.0/Kernel.html#method-i-require
 
+instance Tokenize Require where
+  tokenize Require{..} = do
+    yield . Run $ if requireRelative
+                     then "require_relative"
+                     else "require"
+    within' Scope.Params requirePath
+
 doRequire :: ( Member (Boolean value) sig
              , Member (Modules address) sig
              , Carrier sig m
@@ -113,6 +121,11 @@ instance Eq1 Load where liftEq = genericLiftEq
 instance Ord1 Load where liftCompare = genericLiftCompare
 instance Show1 Load where liftShowsPrec = genericLiftShowsPrec
 
+instance Tokenize Load where
+  tokenize Load{..} = do
+    yield (Run "load")
+    within' Scope.Params $ loadPath *> fromMaybe (pure ()) loadWrap
+
 instance Evaluatable Load where
   eval eval (Load x Nothing) = do
     path <- eval x >>= value >>= asString
@@ -163,6 +176,15 @@ instance Evaluatable Class where
 instance Declarations1 Class where
   liftDeclaredName declaredName = declaredName . classIdentifier
 
+instance Tokenize Class where
+  tokenize Class{..} = within' Scope.Class $ do
+    classIdentifier
+    case classSuperClass of
+      Just a -> yield Token.Extends *> a
+      Nothing -> pure ()
+    classBody
+
+
 data Module a = Module { moduleIdentifier :: !a, moduleStatements :: ![a] }
   deriving (Diffable, Eq, Foldable, FreeVariables1, Functor, Generic1, Hashable1, Ord, Show, ToJSONFields1, Traversable, Named1, Message1, NFData1)
 
@@ -179,6 +201,13 @@ instance Evaluatable Module where
 instance Declarations1 Module where
   liftDeclaredName declaredName = declaredName . moduleIdentifier
 
+instance Tokenize Module where
+  tokenize Module{..} = do
+    yield (Run "module")
+    moduleIdentifier
+    within' Scope.Namespace $ sequenceA_ moduleStatements
+
+
 data LowPrecedenceAnd a = LowPrecedenceAnd { lhs :: a, rhs :: a }
   deriving (Declarations1, Diffable, Eq, Foldable, FreeVariables1, Functor, Generic1, Hashable1, Ord, Show, ToJSONFields1, Traversable, Named1, Message1, NFData1)
 
@@ -193,6 +222,13 @@ instance Eq1 LowPrecedenceAnd where liftEq = genericLiftEq
 instance Ord1 LowPrecedenceAnd where liftCompare = genericLiftCompare
 instance Show1 LowPrecedenceAnd where liftShowsPrec = genericLiftShowsPrec
 
+-- TODO: These should probably be expressed with a new context/token,
+-- rather than a literal run, and need to take surrounding precedence
+-- into account.
+instance Tokenize LowPrecedenceAnd where
+  tokenize LowPrecedenceAnd{..} = lhs *> yield (Token.Run "and") <* rhs
+
+
 data LowPrecedenceOr a = LowPrecedenceOr { lhs :: a, rhs :: a }
   deriving (Declarations1, Diffable, Eq, Foldable, FreeVariables1, Functor, Generic1, Hashable1, Ord, Show, ToJSONFields1, Traversable, Named1, Message1, NFData1)
 
@@ -206,3 +242,6 @@ instance Evaluatable LowPrecedenceOr where
 instance Eq1 LowPrecedenceOr where liftEq = genericLiftEq
 instance Ord1 LowPrecedenceOr where liftCompare = genericLiftCompare
 instance Show1 LowPrecedenceOr where liftShowsPrec = genericLiftShowsPrec
+
+instance Tokenize LowPrecedenceOr where
+  tokenize LowPrecedenceOr{..} = lhs *> yield (Token.Run "or") <* rhs