Merge branch 'master' into auto-generate-test-cases

semantic-diff.cabal

@@ -21,6 +21,8 @@ library
                      , Data.Functor.Both
                      , Data.RandomWalkSimilarity
                      , Data.Record
+                     , Data.Mergeable
+                     , Data.Mergeable.Generic
                      , Data.These.Arbitrary
                      , Diff
                      , Diff.Arbitrary
@@ -28,7 +30,6 @@ library
                      , Info
                      , Interpreter
                      , Language
-                     , Operation
                      , Parser
                      , Patch
                      , Patch.Arbitrary
@@ -40,6 +41,7 @@ library
                      , Renderer.Summary
                      , SES
                      , Source
+                     , SourceSpan
                      , SplitDiff
                      , Syntax
                      , Term
@@ -75,6 +77,7 @@ library
                      , comonad
                      , protolude
                      , wl-pprint-text
+                     , quickcheck-instances
   default-language:    Haskell2010
   default-extensions:  DeriveFunctor, DeriveFoldable, DeriveTraversable, DeriveGeneric, FlexibleContexts, FlexibleInstances, OverloadedStrings, NoImplicitPrelude, RecordWildCards, LambdaCase
   ghc-options:        -Wall -fno-warn-name-shadowing -O2 -threaded -fprof-auto "-with-rtsopts=-N -p -s -h -i0.1" -j
@@ -101,6 +104,7 @@ test-suite semantic-diff-test
   main-is:             Spec.hs
   other-modules:       AlignmentSpec
                      , CorpusSpec
+                     , Data.Mergeable.Spec
                      , Data.RandomWalkSimilarity.Spec
                      , Diff.Spec
                      , DiffSummarySpec

src/Algorithm.hs

@@ -1,7 +1,36 @@
 module Algorithm where
 
-import Control.Monad.Trans.Free
-import Operation
+import Control.Monad.Free.Church
+import Prologue
 
--- | A lazily-produced AST for diffing.
-type Algorithm a annotation = Free (Operation a annotation)
+-- | A single step in a diffing algorithm.
+--
+-- 'term' is the type of terms.
+-- 'diff' is the type of diffs.
+-- 'f' represents the continuation after diffing. Often 'Algorithm'.
+data AlgorithmF term diff f
+  -- | Recursively diff two terms and pass the result to the continuation.
+  = Recursive term term (diff -> f)
+  -- | Diff two lists by each element’s position, and pass the resulting list of diffs to the continuation.
+  | ByIndex [term] [term] ([diff] -> f)
+  -- | Diff two lists by each element’s similarity and pass the resulting list of diffs to the continuation.
+  | BySimilarity [term] [term] ([diff] -> f)
+  deriving Functor
+
+-- | The free monad for 'AlgorithmF'. This enables us to construct diff values using do-notation. We use the Church-encoded free monad 'F' for efficiency.
+type Algorithm term diff = F (AlgorithmF term diff)
+
+
+-- DSL
+
+-- | Constructs a 'Recursive' diff of two terms.
+recursively :: term -> term -> Algorithm term diff diff
+recursively a b = wrap (Recursive a b pure)
+
+-- | Constructs a 'ByIndex' diff of two lists of terms.
+byIndex :: [term] -> [term] -> Algorithm term diff [diff]
+byIndex a b = wrap (ByIndex a b pure)
+
+-- | Constructs a 'BySimilarity' diff of two lists of terms.
+bySimilarity :: [term] -> [term] -> Algorithm term diff [diff]
+bySimilarity a b = wrap (BySimilarity a b pure)

src/Alignment.hs

@@ -61,40 +61,11 @@ alignPatch sources patch = case patch of
 
 -- | The Applicative instance f is either Identity or Both. Identity is for Terms in Patches, Both is for Diffs in unchanged portions of the diff.
 alignSyntax :: (Applicative f, HasField fields Range) => (forall a. f a -> Join These a) -> (CofreeF (Syntax leaf) (Record fields) term -> term) -> (term -> Range) -> f (Source Char) -> CofreeF (Syntax leaf) (f (Record fields)) [Join These term] -> [Join These term]
-alignSyntax toJoinThese toNode getRange sources (infos :< syntax) = case syntax of
-  Leaf s -> catMaybes $ wrapInBranch (const (Leaf s)) . fmap (flip (,) []) <$> (Join <$> bisequenceL (runJoin lineRanges))
-  Comment a -> catMaybes $ wrapInBranch (const (Comment a)) . fmap (flip (,) []) <$> (Join <$> bisequenceL (runJoin lineRanges))
-  Indexed children ->
-    catMaybes $ wrapInBranch Indexed <$> alignBranch getRange (join children) bothRanges
-  Syntax.Function id params body -> catMaybes $ wrapInBranch Indexed <$> alignBranch getRange (fromMaybe [] id <> fromMaybe []  params <> body) bothRanges
-  -- Align FunctionCalls like Indexed nodes by appending identifier to its children.
-  Syntax.FunctionCall identifier children ->
-    catMaybes $ wrapInBranch Indexed <$> alignBranch getRange (join (identifier : children)) bothRanges
-  Syntax.Assignment assignmentId value ->
-    catMaybes $ wrapInBranch Indexed <$> alignBranch getRange (assignmentId <> value) bothRanges
-  Syntax.MemberAccess memberId property ->
-    catMaybes $ wrapInBranch Indexed <$> alignBranch getRange (memberId <> property) bothRanges
-  Syntax.MethodCall targetId methodId args ->
-    catMaybes $ wrapInBranch Indexed <$> alignBranch getRange (targetId <> methodId <> args) bothRanges
-  Syntax.Args children ->
-    catMaybes $ wrapInBranch Indexed <$> alignBranch getRange (join children) bothRanges
-  Syntax.VarDecl decl ->
-    catMaybes $ wrapInBranch Indexed <$> alignBranch getRange decl bothRanges
-  Syntax.VarAssignment id value ->
-    catMaybes $ wrapInBranch Indexed <$> alignBranch getRange (id <> value) bothRanges
-  Switch expr cases ->
-    catMaybes $ wrapInBranch Indexed <$> alignBranch getRange (expr <> join cases) bothRanges
-  Case expr body ->
-    catMaybes $ wrapInBranch Indexed <$> alignBranch getRange (expr <> body) bothRanges
-  Fixed children ->
-    catMaybes $ wrapInBranch Fixed <$> alignBranch getRange (join children) bothRanges
-  Pair a b -> catMaybes $ wrapInBranch Indexed <$> alignBranch getRange (a <> b) bothRanges
-  Object children -> catMaybes $ wrapInBranch Indexed <$> alignBranch getRange (join children) bothRanges
-  Commented cs expr -> catMaybes $ wrapInBranch Indexed <$> alignBranch getRange (join cs <> join (maybeToList expr)) bothRanges
-  Ternary expr cases -> catMaybes $ wrapInBranch Indexed <$> alignBranch getRange (expr <> join cases) bothRanges
-  Operator cases -> catMaybes $ wrapInBranch Indexed <$> alignBranch getRange (join cases) bothRanges
-  MathAssignment key value -> catMaybes $ wrapInBranch Indexed <$> alignBranch getRange (key <> value) bothRanges
-  SubscriptAccess key value -> catMaybes $ wrapInBranch Indexed <$> alignBranch getRange (key <> value) bothRanges
+alignSyntax toJoinThese toNode getRange sources (infos :< syntax) = catMaybes $ case syntax of
+  Leaf s -> wrapInBranch (const (Leaf s)) <$> alignBranch getRange [] bothRanges
+  Comment a -> wrapInBranch (const (Comment a)) <$> alignBranch getRange [] bothRanges
+  Fixed children -> wrapInBranch Fixed <$> alignBranch getRange (join children) bothRanges
+  _ -> wrapInBranch Indexed <$> alignBranch getRange (join (toList syntax)) bothRanges
   where bothRanges = modifyJoin (fromThese [] []) lineRanges
         lineRanges = toJoinThese $ actualLineRanges <$> (characterRange <$> infos) <*> sources
         wrapInBranch constructor = applyThese $ toJoinThese ((\ info (range, children) -> toNode (setCharacterRange info range :< constructor children)) <$> infos)

src/Category.hs

@@ -2,7 +2,7 @@ module Category where
 
 import Prologue
 import Data.Hashable
-import Test.QuickCheck
+import Test.QuickCheck hiding (Args)
 import Data.Text.Arbitrary()
 
 -- | A standardized category of AST node. Used to determine the semantics for
@@ -60,6 +60,12 @@ data Category
   | VarDecl
   -- | A switch expression.
   | Switch
+  -- | A for expression.
+  | For
+  -- | A while expression.
+  | While
+  -- | A do/while expression.
+  | DoWhile
   -- | A ternary expression.
   | Ternary
   -- | A case expression.
@@ -68,27 +74,65 @@ data Category
   | Operator
   -- | An object/dictionary/hash literal.
   | Object
+  -- | A return statement.
+  | Return
+  -- | A throw statement.
+  | Throw
+  -- | A constructor statement, e.g. new Foo;
+  | Constructor
+  -- | A try statement.
+  | Try
+  -- | A catch statement.
+  | Catch
+  -- | A finally statement.
+  | Finally
+  -- | A class declaration.
+  | Class
+  -- | A class method declaration.
+  | Method
   -- | A non-standard category, which can be used for comparability.
   | Other Text
   deriving (Eq, Generic, Ord, Show)
 
-
 -- Instances
 
 instance Hashable Category
 
 instance Arbitrary Category where
-  arbitrary = oneof
-    [ pure Program
+  arbitrary = oneof [
+      pure Program
     , pure Error
+    , pure Boolean
     , pure BinaryOperator
     , pure DictionaryLiteral
     , pure Pair
     , pure FunctionCall
+    , pure Function
+    , pure Identifier
+    , pure Params
+    , pure ExpressionStatements
+    , pure MethodCall
+    , pure Args
     , pure StringLiteral
     , pure IntegerLiteral
+    , pure Regex
     , pure SymbolLiteral
+    , pure TemplateString
     , pure ArrayLiteral
+    , pure Assignment
+    , pure MathAssignment
+    , pure MemberAccess
+    , pure SubscriptAccess
+    , pure VarAssignment
+    , pure VarDecl
+    , pure For
+    , pure DoWhile
+    , pure While
+    , pure Switch
+    , pure Ternary
+    , pure Case
+    , pure Operator
+    , pure Object
     , Other <$> arbitrary
     ]
 

src/Data/Mergeable.hs

@@ -0,0 +1,37 @@
+{-# LANGUAGE DefaultSignatures #-}
+module Data.Mergeable where
+
+import Data.Functor.Identity
+import Data.Mergeable.Generic
+import GHC.Generics
+import Prologue
+
+-- Classes
+
+-- | A 'Mergeable' functor is one which supports pushing itself through an 'Alternative' functor. Note the similarities with 'Traversable' & 'Crosswalk'.
+--
+-- This is a kind of distributive law which produces (at least) the union of the two functors’ shapes; i.e. unlike 'Traversable', an 'empty' value in the inner functor does not produce an 'empty' result, and unlike 'Crosswalk', an 'empty' value in the outer functor does not produce an 'empty' result.
+--
+-- For example, we can use 'merge' to select one side or the other of a diff node in 'Syntax', while correctly handling the fact that some patches don’t have any content for that side:
+--
+-- @
+-- let before = iter (\ (a :< s) -> cofree . (fst a :<) <$> sequenceAlt syntax) . fmap (maybeFst . unPatch)
+-- @
+class Functor t => Mergeable t where
+  -- | Merge a functor by mapping its elements into an 'Alternative' functor, combining them, and pushing the 'Mergeable' functor inside.
+  merge :: Alternative f => (a -> f b) -> t a -> f (t b)
+  default merge :: (Generic1 t, GMergeable (Rep1 t), Alternative f) => (a -> f b) -> t a -> f (t b)
+  merge = genericMerge
+
+  -- | Sequnce a 'Mergeable' functor by 'merge'ing the 'Alternative' values.
+  sequenceAlt :: Alternative f => t (f a) -> f (t a)
+  sequenceAlt = merge identity
+
+
+-- Instances
+
+instance Mergeable [] where merge = gmerge
+
+instance Mergeable Maybe
+
+instance Mergeable Identity where merge f = fmap Identity . f . runIdentity

src/Data/Mergeable/Generic.hs

@@ -0,0 +1,46 @@
+{-# LANGUAGE TypeOperators #-}
+module Data.Mergeable.Generic where
+
+import GHC.Generics
+import Prologue
+
+-- Classes
+
+class GMergeable t where
+  gmerge :: Alternative f => (a -> f b) -> t a -> f (t b)
+
+genericMerge :: (Generic1 t, GMergeable (Rep1 t), Alternative f) => (a -> f b) -> t a -> f (t b)
+genericMerge f = fmap to1 . gmerge f . from1
+
+
+-- Instances
+
+instance GMergeable U1 where
+  gmerge _ _ = pure U1
+
+instance GMergeable Par1 where
+  gmerge f (Par1 a) = Par1 <$> f a
+
+instance GMergeable (K1 i c) where
+  gmerge _ (K1 a) = pure (K1 a)
+
+instance GMergeable f => GMergeable (Rec1 f) where
+  gmerge f (Rec1 a) = Rec1 <$> gmerge f a
+
+instance GMergeable f => GMergeable (M1 i c f) where
+  gmerge f (M1 a) = M1 <$> gmerge f a
+
+instance (GMergeable f, GMergeable g) => GMergeable (f :+: g) where
+  gmerge f (L1 a) = L1 <$> gmerge f a
+  gmerge f (R1 b) = R1 <$> gmerge f b
+
+instance (GMergeable f, GMergeable g) => GMergeable (f :*: g) where
+  gmerge f (a :*: b) = (:*:) <$> gmerge f a <*> gmerge f b
+
+instance GMergeable [] where
+  gmerge f (x:xs) = ((:) <$> f x <|> pure identity) <*> gmerge f xs
+  gmerge _ [] = pure []
+
+instance GMergeable Maybe where
+  gmerge f (Just a) = Just <$> f a
+  gmerge _ Nothing = pure empty

src/Data/RandomWalkSimilarity.hs

@@ -18,24 +18,18 @@ import Test.QuickCheck hiding (Fixed)
 import Test.QuickCheck.Random
 
 -- | Given a function comparing two terms recursively, and a function to compute a Hashable label from an unpacked term, compute the diff of a pair of lists of terms using a random walk similarity metric, which completes in log-linear time. This implementation is based on the paper [_RWS-Diff—Flexible and Efficient Change Detection in Hierarchical Data_](https://github.com/github/semantic-diff/files/325837/RWS-Diff.Flexible.and.Efficient.Change.Detection.in.Hierarchical.Data.pdf).
-rws :: (Hashable label, Eq annotation, Prologue.Foldable f, Functor f, Eq (f (Cofree f annotation))) =>
-  -- | A function which comapres a pair of terms recursively, returning 'Just' their diffed value if appropriate, or 'Nothing' if they should not be compared.
-  (Cofree f annotation -> Cofree f annotation -> Maybe (Free (CofreeF f (Both annotation)) (Patch (Cofree f annotation)))) ->
-  -- | A function to compute a label for an unpacked term.
-  (forall b. CofreeF f annotation b -> label) ->
-  -- | The old list of terms.
-  [Cofree f annotation] ->
-  -- | The new list of terms.
-  [Cofree f annotation] ->
-  [Free (CofreeF f (Both annotation)) (Patch (Cofree f annotation))]
+rws :: (Hashable label, Eq annotation, Prologue.Foldable f, Functor f, Eq (f (Cofree f annotation)))
+  => (Cofree f annotation -> Cofree f annotation -> Maybe (Free (CofreeF f (Both annotation)) (Patch (Cofree f annotation)))) -- ^ A function which comapres a pair of terms recursively, returning 'Just' their diffed value if appropriate, or 'Nothing' if they should not be compared.
+  -> (forall b. CofreeF f annotation b -> label) -- ^ A function to compute a label for an unpacked term.
+  -> [Cofree f annotation] -- ^ The list of old terms.
+  -> [Cofree f annotation] -- ^ The list of new terms.
+  -> [Free (CofreeF f (Both annotation)) (Patch (Cofree f annotation))]
 rws compare getLabel as bs
   | null as, null bs = []
-  | null as = insert <$> bs
-  | null bs = delete <$> as
+  | null as = inserting <$> bs
+  | null bs = deleting <$> as
   | otherwise = fmap snd . uncurry deleteRemaining . (`runState` (negate 1, fas)) $ traverse findNearestNeighbourTo fbs
-  where insert = pure . Insert
-        delete = pure . Delete
-        (p, q, d) = (2, 2, 15)
+  where (p, q, d) = (2, 2, 15)
         fas = zipWith featurize [0..] as
         fbs = zipWith featurize [0..] bs
         kdas = KdTree.build (Vector.toList . feature) fas
@@ -43,14 +37,14 @@ rws compare getLabel as bs
         findNearestNeighbourTo kv@(UnmappedTerm _ _ v) = do
           (previous, unmapped) <- get
           let (UnmappedTerm i _ _) = KdTree.nearest kdas kv
-          fromMaybe (pure (negate 1, insert v)) $ do
+          fromMaybe (pure (negate 1, inserting v)) $ do
             found <- find ((== i) . termIndex) unmapped
             guard (i >= previous)
             compared <- compare (term found) v
             pure $! do
               put (i, List.delete found unmapped)
               pure (i, compared)
-        deleteRemaining diffs (_, unmapped) = foldl' (flip (List.insertBy (comparing fst))) diffs ((termIndex &&& delete . term) <$> unmapped)
+        deleteRemaining diffs (_, unmapped) = foldl' (flip (List.insertBy (comparing fst))) diffs ((termIndex &&& deleting . term) <$> unmapped)
 
 -- | A term which has not yet been mapped by `rws`, along with its feature vector summary & index.
 data UnmappedTerm a = UnmappedTerm { termIndex :: {-# UNPACK #-} !Int, feature :: !(Vector.Vector Double), term :: !a }

src/Diff.hs

@@ -5,6 +5,7 @@ module Diff where
 import Prologue
 import Data.Functor.Foldable as Foldable
 import Data.Functor.Both as Both
+import Data.Mergeable
 import Patch
 import Syntax
 import Term
@@ -28,10 +29,7 @@ diffCost = diffSum $ patchSum termSize
 mergeMaybe :: (Patch (Term leaf annotation) -> Maybe (Term leaf annotation)) -> Diff leaf annotation -> Maybe (Term leaf annotation)
 mergeMaybe transform = iter algebra . fmap transform
   where algebra :: CofreeF (Syntax leaf) (Both annotation) (Maybe (Term leaf annotation)) -> Maybe (Term leaf annotation)
-        algebra (annotations :< syntax) = Just . cofree $ Both.fst annotations :< case syntax of
-          Leaf s -> Leaf s
-          Indexed i -> Indexed (catMaybes i)
-          Fixed i -> Fixed (catMaybes i)
+        algebra (annotations :< syntax) = cofree . (Both.fst annotations :<) <$> sequenceAlt syntax
 
 -- | Recover the before state of a diff.
 beforeTerm :: Diff leaf annotation -> Maybe (Term leaf annotation)

src/DiffSummary.hs

@@ -1,59 +1,85 @@
 {-# LANGUAGE DataKinds, TypeFamilies, ScopedTypeVariables #-}
-module DiffSummary (DiffSummary(..), diffSummary, DiffInfo(..)) where
+
+module DiffSummary (DiffSummary(..), diffSummaries, DiffInfo(..), annotatedSummaries) where
 
 import Prologue hiding (snd, intercalate)
 import Diff
-import Info (category)
 import Patch
 import Term
-import Syntax
-import Category
+import Info (category, characterRange)
+import Range
+import Syntax as S
+import Category as C
 import Data.Functor.Foldable as Foldable
 import Data.Functor.Both
-import Data.Record
 import Data.Text as Text (intercalate)
-import Text.PrettyPrint.Leijen.Text ((<+>), squotes, space, string)
+import Test.QuickCheck hiding (Fixed)
+import Patch.Arbitrary()
+import Data.Record
+import Text.PrettyPrint.Leijen.Text ((<+>), squotes, space, string, Doc, punctuate, pretty)
 import qualified Text.PrettyPrint.Leijen.Text as P
+import SourceSpan
+import Source
 
-data DiffInfo = DiffInfo { categoryName :: Text, termName :: Text } deriving (Eq, Show)
+data DiffInfo = LeafInfo { categoryName :: Text, termName :: Text }
+ | BranchInfo { branches :: [ DiffInfo ], categoryName :: Text, branchType :: Branch }
+ | ErrorInfo { errorSpan :: SourceSpan, categoryName :: Text }
+ deriving (Eq, Show)
 
-toTermName :: (HasCategory leaf, HasField fields Category) => Term leaf (Record fields) -> Text
-toTermName term = case unwrap term of
-  Fixed children -> fromMaybe "EmptyFixedNode" $ (toCategoryName . category) . extract <$> head children
-  Indexed children -> fromMaybe "EmptyIndexedNode" $ (toCategoryName . category) . extract <$> head children
+toTermName :: (HasCategory leaf, HasField fields Category, HasField fields Range) => Source Char -> Term leaf (Record fields) -> Text
+toTermName source term = case unwrap term of
+  S.Fixed children -> fromMaybe "branch" $ (toCategoryName . category) . extract <$> head children
+  S.Indexed children -> fromMaybe "branch" $ (toCategoryName . category) . extract <$> head children
   Leaf leaf -> toCategoryName leaf
-  Syntax.Assignment identifier value -> toTermName identifier <> toTermName value
-  Syntax.Function identifier _ _ -> (maybe "anonymous" toTermName identifier)
-  Syntax.FunctionCall i _ -> toTermName i
-  Syntax.MemberAccess base property -> case (unwrap base, unwrap property) of
-    (Syntax.FunctionCall{}, Syntax.FunctionCall{}) -> toTermName base <> "()." <> toTermName property <> "()"
-    (Syntax.FunctionCall{}, _) -> toTermName base <> "()." <> toTermName property
-    (_, Syntax.FunctionCall{}) -> toTermName base <> "." <> toTermName property <> "()"
-    (_, _) -> toTermName base <> "." <> toTermName property
-  Syntax.MethodCall targetId methodId _ -> toTermName targetId <> sep <> toTermName methodId <> "()"
+  S.Assignment identifier value -> toTermName' identifier <> toTermName' value
+  S.Function identifier _ _ -> (maybe "anonymous" toTermName' identifier)
+  S.FunctionCall i _ -> toTermName' i
+  S.MemberAccess base property -> case (unwrap base, unwrap property) of
+    (S.FunctionCall{}, S.FunctionCall{}) -> toTermName' base <> "()." <> toTermName' property <> "()"
+    (S.FunctionCall{}, _) -> toTermName' base <> "()." <> toTermName' property
+    (_, S.FunctionCall{}) -> toTermName' base <> "." <> toTermName' property <> "()"
+    (_, _) -> toTermName' base <> "." <> toTermName' property
+  S.MethodCall targetId methodId _ -> toTermName' targetId <> sep <> toTermName' methodId <> "()"
     where sep = case unwrap targetId of
-            Syntax.FunctionCall{} -> "()."
+            S.FunctionCall{} -> "()."
             _ -> "."
-  Syntax.SubscriptAccess base element -> case (unwrap base, unwrap element) of
-    (Syntax.FunctionCall{}, Syntax.FunctionCall{}) -> toTermName base <> "()." <> toTermName element <> "()"
-    (Syntax.FunctionCall{}, _) -> toTermName base <> "()." <> toTermName element
-    (_, Syntax.FunctionCall{}) -> toTermName base <> "[" <> toTermName element <> "()" <> "]"
-    (_, _) -> toTermName base <> "[" <> toTermName element <> "]"
-  Syntax.VarAssignment varId _ -> toTermName varId
-  Syntax.VarDecl decl -> toTermName decl
+  S.SubscriptAccess base element -> case (unwrap base, unwrap element) of
+    (S.FunctionCall{}, S.FunctionCall{}) -> toTermName' base <> "()." <> toTermName' element <> "()"
+    (S.FunctionCall{}, _) -> toTermName' base <> "()." <> toTermName' element
+    (_, S.FunctionCall{}) -> toTermName' base <> "[" <> toTermName' element <> "()" <> "]"
+    (_, _) -> toTermName' base <> "[" <> toTermName' element <> "]"
+  S.VarAssignment varId _ -> toTermName' varId
+  S.VarDecl decl -> toTermName' decl
   -- TODO: We should remove Args from Syntax since I don't think we should ever
   -- evaluate Args as a single toTermName Text - joshvera
-  Syntax.Args args -> mconcat $ toTermName <$> args
+  S.Args args -> mconcat $ toTermName' <$> args
   -- TODO: We should remove Case from Syntax since I don't think we should ever
   -- evaluate Case as a single toTermName Text - joshvera
-  Syntax.Case expr _ -> toTermName expr
-  Syntax.Switch expr _ -> toTermName expr
-  Syntax.Ternary expr _ -> toTermName expr
-  Syntax.MathAssignment id _ -> toTermName id
-  Syntax.Operator syntaxes -> mconcat $ toTermName <$> syntaxes
-  Syntax.Object kvs -> "{" <> intercalate ", " (toTermName <$> kvs) <> "}"
-  Syntax.Pair a b -> toTermName a <> ": " <> toTermName b
+  S.Case expr _ -> toTermName' expr
+  S.Switch expr _ -> toTermName' expr
+  S.Ternary expr _ -> toTermName' expr
+  S.MathAssignment id _ -> toTermName' id
+  S.Operator _ -> termNameFromSource term
+  S.Object kvs -> "{" <> intercalate ", " (toTermName' <$> kvs) <> "}"
+  S.Pair a b -> toTermName' a <> ": " <> toTermName' b
+  S.Return expr -> maybe "empty" toTermName' expr
+  S.Error span _ -> displayStartEndPos span
+  S.For _ _ -> termNameFromChildren term
+  S.While expr _ -> toTermName' expr
+  S.DoWhile _ expr -> toTermName' expr
+  S.Throw expr -> termNameFromSource expr
+  S.Constructor expr -> toTermName' expr
+  S.Try expr _ _ -> termNameFromSource expr
+  S.Array _ -> termNameFromSource term
+  S.Class identifier _ _ -> toTermName' identifier
+  S.Method identifier _ _ -> toTermName' identifier
   Comment a -> toCategoryName a
+  S.Commented _ _ -> termNameFromChildren term
+  where toTermName' = toTermName source
+        termNameFromChildren term = termNameFromRange (unionRangesFrom (range term) (range <$> toList (unwrap term)))
+        termNameFromSource term = termNameFromRange (range term)
+        termNameFromRange range = toText $ Source.slice range source
+        range = characterRange . extract
 
 class HasCategory a where
   toCategoryName :: a -> Text
@@ -67,97 +93,148 @@ instance HasCategory Category where
     BinaryOperator -> "binary operator"
     Boolean -> "boolean"
     DictionaryLiteral -> "dictionary"
-    Error -> "error"
+    C.Error -> "error"
     ExpressionStatements -> "expression statements"
-    Category.Assignment -> "assignment"
-    Category.Function -> "function"
-    Category.FunctionCall -> "function call"
-    Category.MemberAccess -> "member access"
-    Category.MethodCall -> "method call"
-    Category.Args -> "arguments"
-    Category.VarAssignment -> "var assignment"
-    Category.VarDecl -> "variable"
-    Category.Switch -> "switch statement"
-    Category.Case -> "case statement"
-    Category.SubscriptAccess -> "subscript access"
-    Category.MathAssignment -> "math assignment"
-    Category.Ternary -> "ternary"
-    Category.Operator -> "operator"
+    C.Assignment -> "assignment"
+    C.Function -> "function"
+    C.FunctionCall -> "function call"
+    C.MemberAccess -> "member access"
+    C.MethodCall -> "method call"
+    C.Args -> "arguments"
+    C.VarAssignment -> "var assignment"
+    C.VarDecl -> "variable"
+    C.Switch -> "switch statement"
+    C.Case -> "case statement"
+    C.SubscriptAccess -> "subscript access"
+    C.MathAssignment -> "math assignment"
+    C.Ternary -> "ternary"
+    C.Operator -> "operator"
     Identifier -> "identifier"
     IntegerLiteral -> "integer"
     Other s -> s
-    Category.Pair -> "pair"
+    C.Pair -> "pair"
     Params -> "params"
     Program -> "top level"
     Regex -> "regex"
     StringLiteral -> "string"
     SymbolLiteral -> "symbol"
     TemplateString -> "template string"
-    Category.Object -> "object"
+    C.For -> "for statement"
+    C.While -> "while statement"
+    C.DoWhile -> "do/while statement"
+    C.Object -> "object"
+    C.Return -> "return statement"
+    C.Throw -> "throw statement"
+    C.Constructor -> "constructor"
+    C.Catch -> "catch statement"
+    C.Try -> "try statement"
+    C.Finally -> "finally statement"
+    C.Class -> "class"
+    C.Method -> "method"
 
 instance (HasCategory leaf, HasField fields Category) => HasCategory (Term leaf (Record fields)) where
   toCategoryName = toCategoryName . category . extract
 
+data Branch = BIndexed | BFixed | BCommented deriving (Show, Eq, Generic)
+instance Arbitrary Branch where
+  arbitrary = oneof [ pure BIndexed, pure BFixed ]
+  shrink = genericShrink
+
 data DiffSummary a = DiffSummary {
   patch :: Patch a,
   parentAnnotations :: [Category]
-} deriving (Eq, Functor, Show)
+} deriving (Eq, Functor, Show, Generic)
 
-instance P.Pretty (DiffSummary DiffInfo) where
-  pretty DiffSummary{..} = case patch of
-    Insert diffInfo -> "Added the" <+> squotes (toDoc $ termName diffInfo) <+> (toDoc $ categoryName diffInfo) P.<> maybeParentContext parentAnnotations
-    Delete diffInfo -> "Deleted the" <+> squotes (toDoc $ termName diffInfo) <+> (toDoc $ categoryName diffInfo) P.<> maybeParentContext parentAnnotations
-    Replace t1 t2 -> "Replaced the" <+> squotes (toDoc $ termName t1) <+> (toDoc $ categoryName t1) <+> "with the" <+> P.squotes (toDoc $ termName t2) <+> (toDoc $ categoryName t2) P.<> maybeParentContext parentAnnotations
-    where
-      maybeParentContext annotations = if null annotations
-        then ""
-        else space <> "in the" <+> (toDoc . intercalate "/" $ toCategoryName <$> annotations) <+> "context"
-      toDoc = string . toS
+instance (Eq a, Arbitrary a) => Arbitrary (DiffSummary a) where
+  arbitrary = DiffSummary <$> arbitrary <*> arbitrary
+  shrink = genericShrink
 
-diffSummary :: (HasCategory leaf, HasField fields Category) => Diff leaf (Record fields) -> [DiffSummary DiffInfo]
-diffSummary = cata $ \case
+instance P.Pretty DiffInfo where
+  pretty LeafInfo{..} = squotes (string $ toSL termName) <+> (string $ toSL categoryName)
+  pretty BranchInfo{..} = mconcat $ punctuate (string "," <> space) (pretty <$> branches)
+  pretty ErrorInfo{..} = "syntax error at" <+> (string . toSL $ displayStartEndPos errorSpan) <+> "in" <+> (string . toSL $ spanName errorSpan)
+
+annotatedSummaries :: DiffSummary DiffInfo -> [Text]
+annotatedSummaries DiffSummary{..} = show . (P.<> maybeParentContext parentAnnotations) <$> summaries patch
+
+summaries :: Patch DiffInfo -> [P.Doc]
+summaries (Insert info) = (("Added" <+> "the") <+>) <$> toLeafInfos info
+summaries (Delete info) = (("Deleted" <+> "the") <+>) <$> toLeafInfos info
+summaries (Replace i1 i2) = zipWith (\a b -> "Replaced" <+> "the" <+> a <+> "with the" <+> b) (toLeafInfos i1) (toLeafInfos i2)
+
+toLeafInfos :: DiffInfo -> [Doc]
+toLeafInfos LeafInfo{..} = pure $ squotes (toDoc termName) <+> (toDoc categoryName)
+toLeafInfos BranchInfo{..} = pretty <$> branches
+toLeafInfos err@ErrorInfo{} = pure $ pretty err
+
+maybeParentContext :: [Category] -> Doc
+maybeParentContext annotations = if null annotations
+  then ""
+  else space <> "in the" <+> (toDoc . intercalate "/" $ toCategoryName <$> annotations) <+> "context"
+toDoc :: Text -> Doc
+toDoc = string . toS
+
+diffSummaries :: (HasCategory leaf, HasField fields Category, HasField fields Range) => Both (Source Char) -> Diff leaf (Record fields) -> [DiffSummary DiffInfo]
+diffSummaries sources = cata $ \case
   -- Skip comments and leaves since they don't have any changes
   (Free (_ :< Leaf _)) -> []
-  Free (_ :< (Syntax.Comment _)) -> []
-  (Free (infos :< Indexed children)) -> prependSummary (category $ snd infos) <$> join children
-  (Free (infos :< Fixed children)) -> prependSummary (category $ snd infos) <$> join children
-  (Free (infos :< Syntax.FunctionCall identifier children)) -> prependSummary (category $ snd infos) <$> join (Prologue.toList (identifier : children))
-  (Free (infos :< Syntax.Function id ps body)) -> prependSummary (category $ snd infos) <$> (fromMaybe [] id) <> (fromMaybe [] ps) <> body
-  (Free (infos :< Syntax.Assignment id value)) -> prependSummary (category $ snd infos) <$> id <> value
-  (Free (infos :< Syntax.MemberAccess base property)) -> prependSummary (category $ snd infos) <$> base <> property
-  (Free (infos :< Syntax.SubscriptAccess base property)) -> prependSummary (category $ snd infos) <$> base <> property
-  (Free (infos :< Syntax.MethodCall targetId methodId ps)) -> prependSummary (category $ snd infos) <$> targetId <> methodId <> ps
-  (Free (infos :< Syntax.VarAssignment varId value)) -> prependSummary (category $ snd infos) <$> varId <> value
-  (Free (infos :< Syntax.VarDecl decl)) -> prependSummary (category $ snd infos) <$> decl
-  (Free (infos :< Syntax.Args args)) -> prependSummary (category $ snd infos) <$> join args
-  (Free (infos :< Syntax.Switch expr cases)) -> prependSummary (category $ snd infos) <$> expr <> join cases
-  (Free (infos :< Syntax.Case expr body)) -> prependSummary (category $ snd infos) <$> expr <> body
-  Free (infos :< (Syntax.Ternary expr cases)) -> prependSummary (category $ snd infos) <$> expr <> join cases
-  Free (infos :< (Syntax.MathAssignment id value)) -> prependSummary (category $ snd infos) <$> id <> value
-  Free (infos :< (Syntax.Operator syntaxes)) -> prependSummary (category $ snd infos) <$> join syntaxes
-  Free (infos :< (Syntax.Object kvs)) -> prependSummary (category $ snd infos) <$> join kvs
-  Free (infos :< (Syntax.Pair a b)) -> prependSummary (category $ snd infos) <$> a <> b
-  Free (infos :< (Syntax.Commented cs leaf)) -> prependSummary (category $ snd infos) <$> join cs <> fromMaybe [] leaf
-  (Pure (Insert term)) -> (\info -> DiffSummary (Insert info) []) <$> termToDiffInfo term
-  (Pure (Delete term)) -> (\info -> DiffSummary (Delete info) []) <$> termToDiffInfo term
-  (Pure (Replace t1 t2)) -> (\(info1, info2) -> DiffSummary (Replace info1 info2) []) <$> zip (termToDiffInfo t1) (termToDiffInfo t2)
+  Free (_ :< (S.Comment _)) -> []
+  (Free (infos :< S.Indexed children)) -> annotateWithCategory infos <$> join children
+  (Free (infos :< S.Fixed children)) -> annotateWithCategory infos <$> join children
+  (Free (infos :< S.FunctionCall identifier children)) -> annotateWithCategory infos <$> join (Prologue.toList (identifier : children))
+  (Free (infos :< S.Function id ps body)) -> annotateWithCategory infos <$> (fromMaybe [] id) <> (fromMaybe [] ps) <> body
+  (Free (infos :< S.Assignment id value)) -> annotateWithCategory infos <$> id <> value
+  (Free (infos :< S.MemberAccess base property)) -> annotateWithCategory infos <$> base <> property
+  (Free (infos :< S.SubscriptAccess base property)) -> annotateWithCategory infos <$> base <> property
+  (Free (infos :< S.MethodCall targetId methodId ps)) -> annotateWithCategory infos <$> targetId <> methodId <> ps
+  (Free (infos :< S.VarAssignment varId value)) -> annotateWithCategory infos <$> varId <> value
+  (Free (infos :< S.VarDecl decl)) -> annotateWithCategory infos <$> decl
+  (Free (infos :< S.Args args)) -> annotateWithCategory infos <$> join args
+  (Free (infos :< S.Switch expr cases)) -> annotateWithCategory infos <$> expr <> join cases
+  (Free (infos :< S.Case expr body)) -> annotateWithCategory infos <$> expr <> body
+  Free (infos :< (S.Ternary expr cases)) -> annotateWithCategory infos <$> expr <> join cases
+  Free (infos :< (S.MathAssignment id value)) -> annotateWithCategory infos <$> id <> value
+  Free (infos :< (S.Operator syntaxes)) -> annotateWithCategory infos <$> join syntaxes
+  Free (infos :< (S.Object kvs)) -> annotateWithCategory infos <$> join kvs
+  Free (infos :< (S.Return expr)) -> annotateWithCategory infos <$> fromMaybe [] expr
+  Free (infos :< (S.Pair a b)) -> annotateWithCategory infos <$> a <> b
+  Free (infos :< (S.Commented cs leaf)) -> annotateWithCategory infos <$> join cs <> fromMaybe [] leaf
+  Free (infos :< (S.Error _ children)) -> annotateWithCategory infos <$> join children
+  (Free (infos :< S.For exprs body)) -> annotateWithCategory infos <$> join exprs <> body
+  (Free (infos :< S.While expr body)) -> annotateWithCategory infos <$> expr <> body
+  (Free (infos :< S.DoWhile expr body)) -> annotateWithCategory infos <$> expr <> body
+  (Free (infos :< S.Throw expr)) -> annotateWithCategory infos <$> expr
+  (Free (infos :< S.Constructor expr)) -> annotateWithCategory infos <$> expr
+  (Free (infos :< S.Try expr catch finally)) -> annotateWithCategory infos <$> expr <> fromMaybe [] catch <> fromMaybe [] finally
+  (Free (infos :< S.Array children)) -> annotateWithCategory infos <$> join children
+  (Free (infos :< S.Class identifier superclass definitions)) -> annotateWithCategory infos <$> identifier <> fromMaybe [] superclass <> join definitions
+  (Free (infos :< S.Method identifier params definitions)) -> annotateWithCategory infos <$> identifier <> join params <> join definitions
+  (Pure (Insert term)) -> [ DiffSummary (Insert $ termToDiffInfo afterSource term) [] ]
+  (Pure (Delete term)) -> [ DiffSummary (Delete $ termToDiffInfo beforeSource term) [] ]
+  (Pure (Replace t1 t2)) -> [ DiffSummary (Replace (termToDiffInfo beforeSource t1) (termToDiffInfo afterSource t2)) [] ]
+  where
+    (beforeSource, afterSource) = runJoin sources
+    annotateWithCategory infos = prependSummary (category $ snd infos)
 
-termToDiffInfo :: (HasCategory leaf, HasField fields Category) => Term leaf (Record fields) -> [DiffInfo]
-termToDiffInfo term = case unwrap term of
-  Leaf _ -> [ DiffInfo (toCategoryName term) (toTermName term) ]
-  Indexed children -> join $ termToDiffInfo <$> children
-  Fixed children -> join $ termToDiffInfo <$> children
-  Syntax.FunctionCall identifier _ -> [ DiffInfo (toCategoryName term) (toTermName identifier) ]
-  Syntax.Ternary ternaryCondition _ -> [ DiffInfo (toCategoryName term) (toTermName ternaryCondition) ]
-  Syntax.Function identifier _ _ -> [ DiffInfo (toCategoryName term) (maybe "anonymous" toTermName identifier) ]
-  Syntax.Assignment identifier _ -> [ DiffInfo (toCategoryName term) (toTermName identifier) ]
-  Syntax.MathAssignment identifier _ -> [ DiffInfo (toCategoryName term) (toTermName identifier) ]
+
+termToDiffInfo :: (HasCategory leaf, HasField fields Category, HasField fields Range) => Source Char -> Term leaf (Record fields) -> DiffInfo
+termToDiffInfo blob term = case unwrap term of
+  Leaf _ -> LeafInfo (toCategoryName term) (toTermName' term)
+  S.Indexed children -> BranchInfo (termToDiffInfo' <$> children) (toCategoryName term) BIndexed
+  S.Fixed children -> BranchInfo (termToDiffInfo' <$> children) (toCategoryName term) BFixed
+  S.FunctionCall identifier _ -> LeafInfo (toCategoryName term) (toTermName' identifier)
+  S.Ternary ternaryCondition _ -> LeafInfo (toCategoryName term) (toTermName' ternaryCondition)
+  S.Function identifier _ _ -> LeafInfo (toCategoryName term) (maybe "anonymous" toTermName' identifier)
+  S.Assignment identifier _ -> LeafInfo (toCategoryName term) (toTermName' identifier)
+  S.MathAssignment identifier _ -> LeafInfo (toCategoryName term) (toTermName' identifier)
   -- Currently we cannot express the operator for an operator production from TreeSitter. Eventually we should be able to
   -- use the term name of the operator identifier when we have that production value. Until then, I'm using a placeholder value
   -- to indicate where that value should be when constructing DiffInfos.
-  Syntax.Operator _ -> [DiffInfo (toCategoryName term) "x"]
-  Commented cs leaf -> join (termToDiffInfo <$> cs) <> maybe [] (\leaf -> [ DiffInfo (toCategoryName term) (toTermName leaf) ]) leaf
-  _ -> [ DiffInfo (toCategoryName term) (toTermName term) ]
+  Commented cs leaf -> BranchInfo (termToDiffInfo' <$> cs <> maybeToList leaf) (toCategoryName term) BCommented
+  S.Error sourceSpan _ -> ErrorInfo sourceSpan (toCategoryName term)
+  _ -> LeafInfo (toCategoryName term) (toTermName' term)
+  where toTermName' = toTermName blob
+        termToDiffInfo' = termToDiffInfo blob
 
 prependSummary :: Category -> DiffSummary DiffInfo -> DiffSummary DiffInfo
 prependSummary annotation summary = summary { parentAnnotations = annotation : parentAnnotations summary }

src/Diffing.hs

@@ -12,7 +12,6 @@ import qualified Data.Text.ICU.Convert as Convert
 import Data.These
 import Diff
 import Info
-import Category
 import Interpreter
 import Language
 import Parser
@@ -33,6 +32,32 @@ import TreeSitter
 import Text.Parser.TreeSitter.Language
 import qualified Data.Text as T
 
+-- | Given a parser and renderer, diff two sources and return the rendered
+-- | result.
+-- | Returns the rendered result strictly, so it's always fully evaluated
+-- | with respect to other IO actions.
+diffFiles :: (HasField fields Category, HasField fields Cost, HasField fields Range, Eq (Record fields)) => Parser fields -> Renderer (Record fields) -> Both SourceBlob -> IO Text
+diffFiles parser renderer sourceBlobs = do
+  let sources = source <$> sourceBlobs
+  terms <- sequence $ parser <$> sourceBlobs
+
+  let replaceLeaves = breakDownLeavesByWord <$> sources
+  let areNullOids = runBothWith (\a b -> (oid a == nullOid || null (source a), oid b == nullOid || null (source b))) sourceBlobs
+  let textDiff = case areNullOids of
+        (True, False) -> pure $ Insert (snd terms)
+        (False, True) -> pure $ Delete (fst terms)
+        (_, _) ->
+          runBothWith (diffTerms construct compareCategoryEq diffCostWithCachedTermCosts) (replaceLeaves <*> terms)
+
+  pure $! renderer sourceBlobs textDiff
+
+  where construct (info :< syntax) = free (Free ((setCost <$> info <*> sumCost syntax) :< syntax))
+        sumCost = fmap getSum . foldMap (fmap Sum . getCost)
+        getCost diff = case runFree diff of
+          Free (info :< _) -> cost <$> info
+          Pure patch -> uncurry both (fromThese 0 0 (unPatch (cost . extract <$> patch)))
+
+
 -- | Return a parser based on the file extension (including the ".").
 parserForType :: Text -> Parser '[Range, Category, Cost]
 parserForType mediaType = case languageForType mediaType of
@@ -43,9 +68,10 @@ parserForType mediaType = case languageForType mediaType of
 
 -- | A fallback parser that treats a file simply as rows of strings.
 lineByLineParser :: Parser '[Range, Category, Cost]
-lineByLineParser input = pure . cofree . root $ case foldl' annotateLeaves ([], 0) lines of
+lineByLineParser blob = pure . cofree . root $ case foldl' annotateLeaves ([], 0) lines of
   (leaves, _) -> cofree <$> leaves
   where
+    input = source blob
     lines = actualLines input
     root children = let cost = 1 + fromIntegral (length children) in
       ((Range 0 $ length input) .: Other "program" .: cost .: RNil) :< Indexed children
@@ -83,30 +109,8 @@ readAndTranscodeFile path = do
   text <- B1.readFile path
   transcode text
 
--- | Given a parser and renderer, diff two sources and return the rendered
--- | result.
--- | Returns the rendered result strictly, so it's always fully evaluated
--- | with respect to other IO actions.
-diffFiles :: (HasField fields Category, HasField fields Cost, HasField fields Range, Eq (Record fields)) => Parser fields -> Renderer (Record fields) -> Both SourceBlob -> IO Text
-diffFiles parser renderer sourceBlobs = do
-  let sources = source <$> sourceBlobs
-  terms <- sequence $ parser <$> sources
-
-  let replaceLeaves = breakDownLeavesByWord <$> sources
-  let areNullOids = runBothWith (\a b -> (oid a == nullOid || null (source a), oid b == nullOid || null (source b))) sourceBlobs
-  let textDiff = case areNullOids of
-        (True, False) -> pure $ Insert (snd terms)
-        (False, True) -> pure $ Delete (fst terms)
-        (_, _) ->
-          runBothWith (diffTerms construct shouldCompareTerms diffCostWithCachedTermCosts) (replaceLeaves <*> terms)
-
-  pure $! renderer textDiff sourceBlobs
-  where construct (info :< syntax) = free (Free ((setCost <$> info <*> sumCost syntax) :< syntax))
-        sumCost = fmap getSum . foldMap (fmap Sum . getCost)
-        getCost diff = case runFree diff of
-          Free (info :< _) -> cost <$> info
-          Pure patch -> uncurry both (fromThese 0 0 (unPatch (cost . extract <$> patch)))
-        shouldCompareTerms = (==) `on` category . extract
+compareCategoryEq :: HasField fields Category => Term leaf (Record fields) -> Term leaf (Record fields) -> Bool
+compareCategoryEq = (==) `on` category . extract
 
 -- | The sum of the node count of the diff’s patches.
 diffCostWithCachedTermCosts :: HasField fields Cost => Diff leaf (Record fields) -> Integer
@@ -114,7 +118,6 @@ diffCostWithCachedTermCosts diff = unCost $ case runFree diff of
   Free (info :< _) -> sum (cost <$> info)
   Pure patch -> sum (cost . extract <$> patch)
 
-
 -- | Returns a rendered diff given a parser, diff arguments and two source blobs.
 textDiff :: (Eq (Record fields), HasField fields Category, HasField fields Cost, HasField fields Range) => Parser fields -> DiffArguments -> Both SourceBlob -> IO Text
 textDiff parser arguments sources = case format arguments of

src/Info.hs

@@ -1,5 +1,5 @@
 {-# LANGUAGE DataKinds, GeneralizedNewtypeDeriving #-}
-module Info where
+module Info (Range(..), characterRange, setCharacterRange, Category(..), category, setCategory, Cost(..), cost, setCost) where
 
 import Data.Record
 import Prologue

src/Interpreter.hs

@@ -1,7 +1,7 @@
+{-# LANGUAGE RankNTypes #-}
 module Interpreter (Comparable, DiffConstructor, diffTerms) where
 
 import Algorithm
-import Category
 import Data.Align.Generic
 import Data.Functor.Foldable
 import Data.Functor.Both
@@ -10,12 +10,12 @@ import Data.RandomWalkSimilarity
 import Data.Record
 import Data.These
 import Diff
+import qualified Control.Monad.Free.Church as F
 import Info
-import Operation
 import Patch
 import Prologue hiding (lookup)
 import SES
-import Syntax
+import Syntax as S
 import Term
 
 -- | Returns whether two terms are comparable
@@ -24,38 +24,66 @@ type Comparable leaf annotation = Term leaf annotation -> Term leaf annotation -
 -- | Constructs a diff from the CofreeF containing its annotation and syntax. This function has the opportunity to, for example, cache properties in the annotation.
 type DiffConstructor leaf annotation = CofreeF (Syntax leaf) (Both annotation) (Diff leaf annotation) -> Diff leaf annotation
 
--- | Diff two terms, given a function that determines whether two terms can be compared and a cost function.
-diffTerms :: (Eq leaf, Hashable leaf, Eq (Record fields), HasField fields Category) => DiffConstructor leaf (Record fields) -> Comparable leaf (Record fields) -> SES.Cost (Diff leaf (Record fields)) -> Term leaf (Record fields) -> Term leaf (Record fields) -> Diff leaf (Record fields)
-diffTerms construct comparable cost a b = fromMaybe (pure $ Replace a b) $ constructAndRun construct comparable cost a b
+-- | Diff two terms recursively, given functions characterizing the diffing.
+diffTerms :: (Eq leaf, Hashable leaf, Eq (Record fields), HasField fields Category)
+  => DiffConstructor leaf (Record fields) -- ^ A function to wrap up & possibly annotate every produced diff.
+  -> Comparable leaf (Record fields) -- ^ A function to determine whether or not two terms should even be compared.
+  -> SES.Cost (Diff leaf (Record fields)) -- ^ A function to compute the cost of a given diff node.
+  -> Term leaf (Record fields) -- ^ A term representing the old state.
+  -> Term leaf (Record fields) -- ^ A term representing the new state.
+  -> Diff leaf (Record fields)
+diffTerms construct comparable cost a b = fromMaybe (replacing a b) $ diffComparableTerms construct comparable cost a b
 
--- | Constructs an algorithm and runs it
-constructAndRun :: (Eq leaf, Hashable leaf, Eq (Record fields), HasField fields Category) => DiffConstructor leaf (Record fields) -> Comparable leaf (Record fields) -> SES.Cost (Diff leaf (Record fields)) -> Term leaf (Record fields) -> Term leaf (Record fields) -> Maybe (Diff leaf (Record fields))
-constructAndRun construct comparable cost t1 t2
-  | not $ comparable t1 t2 = Nothing
-  | (category <$> t1) == (category <$> t2) = hylo construct runCofree <$> zipTerms t1 t2
-  | otherwise =
-  run construct comparable cost $ algorithm a b where
-    algorithm (Indexed a') (Indexed b') = wrap $! ByIndex a' b' (annotate . Indexed)
-    algorithm (Leaf a') (Leaf b') | a' == b' = annotate $ Leaf b'
-    algorithm a' b' = wrap $! Recursive (cofree (annotation1 :< a')) (cofree (annotation2 :< b')) pure
-    (annotation1 :< a, annotation2 :< b) = (runCofree t1, runCofree t2)
-    annotate = pure . construct . (both annotation1 annotation2 :<)
+-- | Diff two terms recursively, given functions characterizing the diffing. If the terms are incomparable, returns 'Nothing'.
+diffComparableTerms :: (Eq leaf, Hashable leaf, Eq (Record fields), HasField fields Category) => DiffConstructor leaf (Record fields) -> Comparable leaf (Record fields) -> SES.Cost (Diff leaf (Record fields)) -> Term leaf (Record fields) -> Term leaf (Record fields) -> Maybe (Diff leaf (Record fields))
+diffComparableTerms construct comparable cost = recur
+  where recur a b
+          | (category <$> a) == (category <$> b) = hylo construct runCofree <$> zipTerms a b
+          | comparable a b = runAlgorithm construct recur cost getLabel (Just <$> algorithmWithTerms construct a b)
+          | otherwise = Nothing
+        getLabel (h :< t) = (category h, case t of
+          Leaf s -> Just s
+          _ -> Nothing)
 
--- | Runs the diff algorithm
-run :: (Eq leaf, Hashable leaf, Eq (Record fields), HasField fields Category) => DiffConstructor leaf (Record fields) -> Comparable leaf (Record fields) -> SES.Cost (Diff leaf (Record fields)) -> Algorithm leaf (Record fields) (Diff leaf (Record fields)) -> Maybe (Diff leaf (Record fields))
-run construct comparable cost algorithm = case runFree algorithm of
-  Pure diff -> Just diff
-  Free (Recursive t1 t2 f) -> run construct comparable cost . f $ recur a b where
-    (annotation1 :< a, annotation2 :< b) = (runCofree t1, runCofree t2)
-    annotate = construct . (both annotation1 annotation2 :<)
+-- | Construct an algorithm to diff a pair of terms.
+algorithmWithTerms :: (TermF leaf (Both a) diff -> diff) -> Term leaf a -> Term leaf a -> Algorithm (Term leaf a) diff diff
+algorithmWithTerms construct t1 t2 = case (unwrap t1, unwrap t2) of
+  (Indexed a, Indexed b) -> byIndex Indexed a b
+  (S.FunctionCall identifierA argsA, S.FunctionCall identifierB argsB) -> do
+    identifier <- recursively identifierA identifierB
+    byIndex (S.FunctionCall identifier) argsA argsB
+  (S.Switch exprA casesA, S.Switch exprB casesB) -> do
+    expr <- recursively exprA exprB
+    byIndex (S.Switch expr) casesA casesB
+  (S.Object a, S.Object b) -> byIndex S.Object a b
+  (Commented commentsA a, Commented commentsB b) -> do
+    wrapped <- sequenceA (recursively <$> a <*> b)
+    byIndex (`Commented` wrapped) commentsA commentsB
+  (Array a, Array b) -> byIndex Array a b
+  (S.Class identifierA paramsA expressionsA, S.Class identifierB paramsB expressionsB) -> do
+    identifier <- recursively identifierA identifierB
+    params <- sequenceA (recursively <$> paramsA <*> paramsB)
+    byIndex (S.Class identifier params) expressionsA expressionsB
+  (S.Method identifierA paramsA expressionsA, S.Method identifierB paramsB expressionsB) -> do
+    identifier <- recursively identifierA identifierB
+    params <- Algorithm.byIndex paramsA paramsB
+    expressions <- Algorithm.byIndex expressionsA expressionsB
+    annotate $! S.Method identifier params expressions
+  _ -> recursively t1 t2
+  where annotate = pure . construct . (both (extract t1) (extract t2) :<)
+        byIndex constructor a b = Algorithm.byIndex a b >>= annotate . constructor
 
-    recur a b = maybe (pure (Replace t1 t2)) (annotate . fmap diffThese) (galign a b)
-
-    diffThese = these (pure . Delete) (pure . Insert) (diffTerms construct comparable cost)
-
-  Free (ByIndex a b f) -> run construct comparable cost . f $ ses (constructAndRun construct comparable cost) cost a b
-
-  Free (ByRandomWalkSimilarity a b f) -> run construct comparable cost . f $ rws (constructAndRun construct comparable cost) getLabel a b
-    where getLabel (h :< t) = (category h, case t of
-            Leaf s -> Just s
-            _ -> Nothing)
+-- | Run an algorithm, given functions characterizing the evaluation.
+runAlgorithm :: (Functor f, GAlign f, Eq a, Eq annotation, Eq (f (Cofree f annotation)), Prologue.Foldable f, Traversable f, Hashable label)
+  => (CofreeF f (Both annotation) (Free (CofreeF f (Both annotation)) (Patch (Cofree f annotation))) -> Free (CofreeF f (Both annotation)) (Patch (Cofree f annotation))) -- ^ A function to wrap up & possibly annotate every produced diff.
+  -> (Cofree f annotation -> Cofree f annotation -> Maybe (Free (CofreeF f (Both annotation)) (Patch (Cofree f annotation)))) -- ^ A function to diff two subterms recursively, if they are comparable, or else return 'Nothing'.
+  -> SES.Cost (Free (CofreeF f (Both annotation)) (Patch (Cofree f annotation))) -- ^ A function to compute the cost of a given diff node.
+  -> (forall b. CofreeF f annotation b -> label) -- ^ A function to compute a label for a given term.
+  -> Algorithm (Cofree f annotation) (Free (CofreeF f (Both annotation)) (Patch (Cofree f annotation))) a -- ^ The algorithm to run.
+  -> a
+runAlgorithm construct recur cost getLabel = F.iter $ \case
+  Recursive a b f -> f (maybe (replacing a b) (construct . (both (extract a) (extract b) :<)) $ do
+    aligned <- galign (unwrap a) (unwrap b)
+    traverse (these (Just . deleting) (Just . inserting) recur) aligned)
+  ByIndex as bs f -> f (ses recur cost as bs)
+  BySimilarity as bs f -> f (rws recur getLabel as bs)

src/Operation.hs

@@ -1,18 +0,0 @@
-module Operation where
-
-import Prologue
-import Diff
-import Term
-
--- | A single step in a diffing algorithm.
-data Operation
-  a -- ^ The type of leaves in the syntax tree, typically String, but possibly some datatype representing different leaves more precisely.
-  annotation -- ^ The type of annotations.
-  f -- ^ The type representing another level of the diffing algorithm. Often Algorithm.
-  -- | Recursively diff two terms and pass the result to the continuation.
-  = Recursive (Term a annotation) (Term a annotation) (Diff a annotation -> f)
-  -- | Diff two dictionaries and pass the result to the continuation.
-  -- | Diff two arrays and pass the result to the continuation.
-  | ByIndex [Term a annotation] [Term a annotation] ([Diff a annotation] -> f)
-  | ByRandomWalkSimilarity [Term a annotation] [Term a annotation] ([Diff a annotation] -> f)
-  deriving Functor

src/Parser.hs

@@ -1,64 +1,71 @@
+{-# LANGUAGE ScopedTypeVariables #-}
 module Parser where
 
 import Prologue hiding (Constructor)
 import Data.Record
 import Data.Text (pack)
-import Category
+import Category as C
 import Info
-import Range
 import qualified Syntax as S
 import Term
 import qualified Data.Set as Set
 import Source
+import SourceSpan
 
 -- | A function that takes a source file and returns an annotated AST.
 -- | The return is in the IO monad because some of the parsers are written in C
 -- | and aren't pure.
-type Parser fields = Source Char -> IO (Term Text (Record fields))
+type Parser fields = SourceBlob -> IO (Term Text (Record fields))
 
--- | Categories that are treated as fixed nodes.
-fixedCategories :: Set.Set Category
-fixedCategories = Set.fromList [ BinaryOperator, Pair ]
-
--- | Should these categories be treated as fixed nodes?
-isFixed :: Category -> Bool
-isFixed = flip Set.member fixedCategories
+-- | Whether a category is an Operator Category
+isOperator :: Category -> Bool
+isOperator = flip Set.member (Set.fromList [ Operator, BinaryOperator ])
 
 -- | Given a function that maps production names to sets of categories, produce
 -- | a Constructor.
-termConstructor :: (Show (Record fields), HasField fields Category, HasField fields Range) => Source Char -> (Record fields) -> [Term Text (Record fields)] -> Term Text (Record fields)
-termConstructor source info = cofree . construct
+termConstructor :: forall fields. (Show (Record fields), HasField fields Category, HasField fields Range) => Source Char -> SourceSpan -> (Record fields) -> [Term Text (Record fields)] -> Term Text (Record fields)
+termConstructor source sourceSpan info = cofree . construct
   where
     withDefaultInfo syntax = (info :< syntax)
-    construct [] = withDefaultInfo . S.Leaf . pack . toString $ slice (characterRange info) source
+    construct :: (Show (Record fields), HasField fields Category, HasField fields Range) => [Term Text (Record fields)] -> CofreeF (S.Syntax Text) (Record fields) (Term Text (Record fields))
+    construct [] = case category info of
+      Return -> withDefaultInfo $ S.Return Nothing -- Map empty return statements to Return Nothing
+      _ -> withDefaultInfo . S.Leaf . pack . toString $ slice (characterRange info) source
+    construct children | Return == category info =
+      withDefaultInfo $ S.Return (listToMaybe children)
     construct children | Assignment == category info = case children of
       (identifier:value:[]) -> withDefaultInfo $ S.Assignment identifier value
+      children -> withDefaultInfo $ S.Error sourceSpan children
     construct children | MathAssignment == category info = case children of
       (identifier:value:[]) -> withDefaultInfo $ S.MathAssignment identifier value
+      children -> withDefaultInfo $ S.Error sourceSpan children
     construct children | MemberAccess == category info = case children of
       (base:property:[]) -> withDefaultInfo $ S.MemberAccess base property
+      children -> withDefaultInfo $ S.Error sourceSpan children
     construct children | SubscriptAccess == category info = case children of
       (base:element:[]) -> withDefaultInfo $ S.SubscriptAccess base element
-    construct children | Operator == category info = withDefaultInfo $ S.Operator children
-    construct children | Function == category info = withDefaultInfo $ case children of
-      (body:[]) -> S.Function Nothing Nothing body
+      _ -> withDefaultInfo $ S.Error sourceSpan children
+    construct children | isOperator (category info) = withDefaultInfo $ S.Operator children
+    construct children | Function == category info = case children of
+      (body:[]) -> withDefaultInfo $ S.Function Nothing Nothing body
       (params:body:[]) | (info :< _) <- runCofree params, Params == category info ->
-        S.Function Nothing (Just params) body
+        withDefaultInfo $ S.Function Nothing (Just params) body
       (id:body:[]) | (info :< _) <- runCofree id, Identifier == category info ->
-        S.Function (Just id) Nothing body
+        withDefaultInfo $ S.Function (Just id) Nothing body
       (id:params:body:[]) | (info :< _) <- runCofree id, Identifier == category info ->
-        S.Function (Just id) (Just params) body
-      x -> error $ "Expected a function declaration but got: " <> show x
+        withDefaultInfo $ S.Function (Just id) (Just params) body
+      _ -> withDefaultInfo $ S.Error sourceSpan children
 
     construct children | FunctionCall == category info = case runCofree <$> children of
       [ (_ :< S.MemberAccess{..}), params@(_ :< S.Args{}) ] ->
         setCategory info MethodCall :< S.MethodCall memberId property (cofree params)
       (x:xs) ->
         withDefaultInfo $ S.FunctionCall (cofree x) (cofree <$> xs)
+      _ -> withDefaultInfo $ S.Error sourceSpan children
 
     construct children | Ternary == category info = case children of
       (condition:cases) -> withDefaultInfo $ S.Ternary condition cases
-
+      _ -> withDefaultInfo $ S.Error sourceSpan children
     construct children | Args == category info = withDefaultInfo $ S.Args children
     construct children | VarAssignment == category info
                          , [x, y] <- children = withDefaultInfo $ S.VarAssignment x y
@@ -79,7 +86,59 @@ termConstructor source info = cofree . construct
         toTuple child | S.Indexed [key,value] <- unwrap child = [cofree (extract child :< S.Pair key value)]
         toTuple child | S.Fixed [key,value] <- unwrap child = [cofree (extract child :< S.Pair key value)]
         toTuple child | S.Leaf c <- unwrap child = [cofree (extract child :< S.Comment c)]
+        toTuple child = pure child
 
-    construct children | isFixed (category info) = withDefaultInfo $ S.Fixed children
+    construct children | Pair == (category info) = withDefaultInfo $ S.Fixed children
+    construct children | C.Error == category info =
+      withDefaultInfo $ S.Error sourceSpan children
+    construct children | For == category info, Just (exprs, body) <- unsnoc children =
+      withDefaultInfo $ S.For exprs body
+    construct children | While == category info, [expr, body] <- children =
+      withDefaultInfo $ S.While expr body
+    construct children | DoWhile == category info, [expr, body] <- children =
+      withDefaultInfo $ S.DoWhile expr body
+    construct children | Throw == category info, [expr] <- children =
+      withDefaultInfo $ S.Throw expr
+    construct children | Constructor == category info, [expr] <- children =
+      withDefaultInfo $ S.Constructor expr
+    construct children | Try == category info = case children of
+      [body] -> withDefaultInfo $ S.Try body Nothing Nothing
+      [body, catch] | Catch <- category (extract catch) -> withDefaultInfo $ S.Try body (Just catch) Nothing
+      [body, finally] | Finally <- category (extract finally) -> withDefaultInfo $ S.Try body Nothing (Just finally)
+      [body, catch, finally] | Catch <- category (extract catch),
+                               Finally <- category (extract finally) ->
+        withDefaultInfo $ S.Try body (Just catch) (Just finally)
+      _ -> withDefaultInfo $ S.Error sourceSpan children
+    construct children | ArrayLiteral == category info =
+      withDefaultInfo $ S.Array children
+    construct children | Method == category info = case children of
+      [identifier, params, exprs] |
+        Params == category (extract params),
+        S.Indexed params' <- unwrap params,
+        exprs' <- expressionStatements exprs ->
+          withDefaultInfo $ S.Method identifier params' exprs'
+      [identifier, exprs] | exprs' <- expressionStatements exprs ->
+        withDefaultInfo $ S.Method identifier mempty exprs'
+      _ ->
+        withDefaultInfo $ S.Error sourceSpan children
+    construct children | Class == category info = case children of
+      [identifier, superclass, definitions] | definitions' <- methodDefinitions definitions ->
+        withDefaultInfo $ S.Class identifier (Just superclass) definitions'
+      [identifier, definitions] | definitions' <- methodDefinitions definitions ->
+        withDefaultInfo $ S.Class identifier Nothing definitions'
+      _ ->
+        withDefaultInfo $ S.Error sourceSpan children
     construct children =
       withDefaultInfo $ S.Indexed children
+
+expressionStatements :: HasField fields Category => Term Text (Record fields) -> [Term Text (Record fields)]
+expressionStatements exprs |
+  Other "statement_block" == category (extract exprs),
+  S.Indexed exprs' <- unwrap exprs = exprs'
+expressionStatements _ = mempty
+
+methodDefinitions :: HasField fields Category => Term Text (Record fields) -> [Term Text (Record fields)]
+methodDefinitions definitions |
+  Other "class_body" == category (extract definitions),
+  S.Indexed definitions' <- unwrap definitions = definitions'
+methodDefinitions _ = mempty

src/Patch.hs

@@ -1,5 +1,8 @@
 module Patch
 ( Patch(..)
+, replacing
+, inserting
+, deleting
 , after
 , before
 , unPatch
@@ -18,6 +21,22 @@ data Patch a
   | Delete a
   deriving (Eq, Foldable, Functor, Generic, Ord, Show, Traversable)
 
+
+-- DSL
+
+-- | Constructs the replacement of one value by another in an Applicative context.
+replacing :: Applicative f => a -> a -> f (Patch a)
+replacing = (pure .) . Replace
+
+-- | Constructs the insertion of a value in an Applicative context.
+inserting :: Applicative f => a -> f (Patch a)
+inserting = pure . Insert
+
+-- | Constructs the deletion of a value in an Applicative context.
+deleting :: Applicative f => a -> f (Patch a)
+deleting = pure . Delete
+
+
 -- | Return the item from the after side of the patch.
 after :: Patch a -> Maybe a
 after = maybeSnd . unPatch

src/Prologue.hs

@@ -2,12 +2,10 @@ module Prologue
 ( module X
 , lookup
 , traceShowId
-, FilePath
 ) where
 
 import Protolude as X
 import Data.List (lookup)
-import System.IO (FilePath)
 
 import Control.Comonad.Trans.Cofree as X
 import Control.Monad.Trans.Free as X

src/Renderer.hs

@@ -5,8 +5,8 @@ import Data.Functor.Both
 import Diff
 import Source
 
--- | A function that will render a diff, given the two source files.
-type Renderer annotation = Diff Text annotation -> Both SourceBlob -> Text
+-- | A function that will render a diff, given the two source blobs.
+type Renderer annotation = Both SourceBlob -> Diff Text annotation -> Text
 
 data DiffArguments = DiffArguments { format :: Format, output :: Maybe FilePath, outputPath :: FilePath }
  deriving (Show)

src/Renderer/JSON.hs

@@ -7,7 +7,7 @@ module Renderer.JSON (
 import Prologue hiding (toList)
 import Alignment
 import Category
-import Data.Aeson hiding (json)
+import Data.Aeson as A hiding (json)
 import Data.Bifunctor.Join
 import Data.ByteString.Builder
 import Data.Record
@@ -15,7 +15,6 @@ import qualified Data.Text as T
 import Data.These
 import Data.Vector hiding (toList)
 import Info
-import Range
 import Renderer
 import Source hiding (fromList)
 import SplitDiff
@@ -24,7 +23,7 @@ import Term
 
 -- | Render a diff to a string representing its JSON.
 json :: (HasField fields Category, HasField fields Range) => Renderer (Record fields)
-json diff sources = toS . toLazyByteString . fromEncoding . pairs $ "rows" .= annotateRows (alignDiff (source <$> sources) diff) <> "oids" .= (oid <$> sources) <> "paths" .= (path <$> sources)
+json blobs diff = toS . toLazyByteString . fromEncoding . pairs $ "rows" .= annotateRows (alignDiff (source <$> blobs) diff) <> "oids" .= (oid <$> blobs) <> "paths" .= (path <$> blobs)
   where annotateRows = fmap (fmap NumberedLine) . numberedRows
 
 newtype NumberedLine a = NumberedLine (Int, a)
@@ -36,13 +35,13 @@ instance ToJSON Category where
   toJSON (Other s) = String s
   toJSON s = String . T.pack $ show s
 instance ToJSON Range where
-  toJSON (Range start end) = Array . fromList $ toJSON <$> [ start, end ]
+  toJSON (Range start end) = A.Array . fromList $ toJSON <$> [ start, end ]
   toEncoding (Range start end) = foldable [ start,  end ]
 instance ToJSON a => ToJSON (Join These a) where
-  toJSON (Join vs) = Array . fromList $ toJSON <$> these pure pure (\ a b -> [ a, b ]) vs
+  toJSON (Join vs) = A.Array . fromList $ toJSON <$> these pure pure (\ a b -> [ a, b ]) vs
   toEncoding = foldable
 instance ToJSON a => ToJSON (Join (,) a) where
-  toJSON (Join (a, b)) = Array . fromList $ toJSON <$> [ a, b ]
+  toJSON (Join (a, b)) = A.Array . fromList $ toJSON <$> [ a, b ]
   toEncoding = foldable
 instance (HasField fields Category, HasField fields Range) => ToJSON (SplitDiff leaf (Record fields)) where
   toJSON splitDiff = case runFree splitDiff of
@@ -73,6 +72,9 @@ termFields info syntax = "range" .= characterRange info : "category" .= category
   S.Args c -> childrenFields c
   S.Assignment assignmentId property -> [ "assignmentIdentifier" .= assignmentId ] <> [ "property" .= property ]
   S.MemberAccess memberId value -> [ "memberIdentifier" .= memberId ] <> [ "value" .= value ]
+  S.For exprs body -> [ "forExpressions" .= exprs ] <> [ "forBody" .= body ]
+  S.While expr body -> [ "whileExpr" .= expr ]  <> [ "whileBody" .= body ]
+  S.DoWhile expr body -> [ "doWhileExpr" .= expr ]  <> [ "doWhileBody" .= body ]
   S.Switch expr cases -> [ "switchExpression" .= expr ] <> [ "cases" .= cases ]
   S.Case expr body -> [ "caseExpression" .= expr ] <> [ "caseStatement" .= body ]
   S.VarDecl decl -> [ "variableDeclaration" .= decl ]
@@ -83,8 +85,16 @@ termFields info syntax = "range" .= characterRange info : "category" .= category
   S.SubscriptAccess id property -> [ "subscriptId" .= id ] <> [ "property" .= property ]
   S.Object pairs -> childrenFields pairs
   S.Pair a b -> childrenFields [a, b]
+  S.Return expr -> [ "returnExpression" .= expr ]
+  S.Constructor expr -> [ "constructorExpression" .= expr ]
   S.Comment _ -> []
   S.Commented comments child -> childrenFields (comments <> maybeToList child)
+  S.Error sourceSpan c -> [ "sourceSpan" .= sourceSpan ] <> childrenFields c
+  S.Throw c -> [ "throwExpression" .= c ]
+  S.Try body catch finally -> [ "tryBody" .= body ] <> [ "tryCatch" .= catch ] <> [ "tryFinally" .= finally ]
+  S.Array c -> childrenFields c
+  S.Class identifier superclass definitions -> [ "classIdentifier" .= identifier ] <> [ "superclass" .= superclass ] <> [ "definitions" .= definitions ]
+  S.Method identifier params definitions -> [ "methodIdentifier" .= identifier ] <> [ "params" .= params ] <> [ "definitions" .= definitions ]
   where childrenFields c = [ "children" .= c ]
 
 patchFields :: (KeyValue kv, HasField fields Category, HasField fields Range) => SplitPatch (Term leaf (Record fields)) -> [kv]

src/Renderer/Patch.hs

@@ -27,7 +27,7 @@ truncatePatch _ blobs = pack $ header blobs <> "#timed_out\nTruncating diff: tim
 
 -- | Render a diff in the traditional patch format.
 patch :: HasField fields Range => Renderer (Record fields)
-patch diff blobs = pack $ case getLast (foldMap (Last . Just) string) of
+patch blobs diff = pack $ case getLast (foldMap (Last . Just) string) of
   Just c | c /= '\n' -> string <> "\n\\ No newline at end of file\n"
   _ -> string
   where string = header blobs <> mconcat (showHunk blobs <$> hunks diff blobs)

src/Renderer/Split.hs

@@ -12,7 +12,6 @@ import Data.These
 import Info
 import Prologue hiding (div, head, fst, snd, link)
 import qualified Prologue
-import Range
 import Renderer
 import Source
 import SplitDiff
@@ -33,7 +32,7 @@ classifyMarkup category element = (element !) . A.class_ . textValue $ styleName
 styleName :: Category -> Text
 styleName category = "category-" <> case category of
   Program -> "program"
-  Error -> "error"
+  C.Error -> "error"
   BinaryOperator -> "binary-operator"
   Boolean -> "boolean"
   DictionaryLiteral -> "dictionary"
@@ -41,7 +40,6 @@ styleName category = "category-" <> case category of
   StringLiteral -> "string"
   SymbolLiteral -> "symbol"
   IntegerLiteral -> "integer"
-  ArrayLiteral -> "array"
   C.FunctionCall -> "function_call"
   C.Function -> "function"
   C.MethodCall -> "method_call"
@@ -62,6 +60,18 @@ styleName category = "category-" <> case category of
   C.Ternary -> "ternary"
   C.Operator -> "operator"
   C.Object -> "object"
+  C.For -> "for"
+  C.While -> "while"
+  C.DoWhile -> "do_while"
+  C.Return -> "return_statement"
+  C.Throw -> "throw_statement"
+  C.Constructor -> "constructor"
+  C.Try -> "try_statement"
+  C.Catch -> "catch_statement"
+  C.Finally -> "finally_statement"
+  ArrayLiteral -> "array"
+  C.Class -> "class_statement"
+  C.Method -> "method"
   Other string -> string
 
 -- | Pick the class name for a split patch.
@@ -73,7 +83,7 @@ splitPatchToClassName patch = stringValue $ "patch " <> case patch of
 
 -- | Render a diff as an HTML split diff.
 split :: (HasField fields Category, HasField fields Cost, HasField fields Range) => Renderer (Record fields)
-split diff blobs = TL.toStrict . renderHtml
+split blobs diff = TL.toStrict . renderHtml
   . docTypeHtml
     . ((head $ link ! A.rel "stylesheet" ! A.href "style.css") <>)
     . body

src/Renderer/Summary.hs

@@ -7,8 +7,8 @@ import Data.Aeson
 import Data.Record
 import Range
 import DiffSummary
-import Data.Text (pack)
-import Text.PrettyPrint.Leijen.Text (pretty)
+import Source
 
 summary :: (HasField fields Category, HasField fields Range) => Renderer (Record fields)
-summary diff _ = toS . encode $ pack . show . pretty <$> diffSummary diff
+summary blobs diff = toS . encode $ summaries >>= annotatedSummaries
+  where summaries = diffSummaries (source <$> blobs) diff

src/SES.hs

@@ -38,8 +38,8 @@ diffAt diffTerms cost (i, j) as bs
   | null bs = pure $ foldr delete [] as
   | otherwise = pure []
   where
-    delete = consWithCost cost . pure . Delete
-    insert = consWithCost cost . pure . Insert
+    delete = consWithCost cost . deleting
+    insert = consWithCost cost . inserting
     costOf [] = 0
     costOf ((_, c) : _) = c
     best = minimumBy (comparing costOf)

src/Source.hs

@@ -8,6 +8,15 @@ import qualified Data.Vector as Vector
 import Numeric
 import Range
 
+-- | The source, oid, path, and Maybe SourceKind of a blob in a Git repo.
+data SourceBlob = SourceBlob { source :: Source Char, oid :: String, path :: FilePath, blobKind :: Maybe SourceKind }
+  deriving (Show, Eq)
+
+-- | The contents of a source file, backed by a vector for efficient slicing.
+newtype Source a = Source { getVector :: Vector.Vector a  }
+  deriving (Eq, Show, Foldable, Functor, Traversable)
+
+-- | The kind of a blob, along with it's file mode.
 data SourceKind = PlainBlob Word32  | ExecutableBlob Word32 | SymlinkBlob Word32
   deriving (Show, Eq)
 
@@ -16,17 +25,11 @@ modeToDigits (PlainBlob mode) = showOct mode ""
 modeToDigits (ExecutableBlob mode) = showOct mode ""
 modeToDigits (SymlinkBlob mode) = showOct mode ""
 
-data SourceBlob = SourceBlob { source :: Source Char, oid :: String, path :: FilePath, blobKind :: Maybe SourceKind }
-  deriving (Show, Eq)
 
 -- | The default plain blob mode
 defaultPlainBlob :: SourceKind
 defaultPlainBlob = PlainBlob 0o100644
 
--- | The contents of a source file, backed by a vector for efficient slicing.
-newtype Source a = Source { getVector :: Vector.Vector a  }
-  deriving (Eq, Show, Foldable, Functor, Traversable)
-
 
 -- | Map blobs with Nothing blobKind to empty blobs.
 idOrEmptySourceBlob :: SourceBlob -> SourceBlob

src/SourceSpan.hs

@@ -0,0 +1,82 @@
+-- |
+-- Source position and span information
+-- Mostly taken from purescript's SourcePos definition.
+--
+module SourceSpan where
+
+import Prologue
+import Data.Aeson ((.=), (.:))
+import qualified Data.Aeson as A
+import Test.QuickCheck
+import Data.Text.Arbitrary()
+
+-- |
+-- Source position information
+--
+data SourcePos = SourcePos
+  { -- |
+    -- Line number
+    --
+    line :: !Int
+    -- |
+    -- Column number
+    --
+  , column :: !Int
+  } deriving (Show, Read, Eq, Ord, Generic)
+
+displaySourcePos :: SourcePos -> Text
+displaySourcePos sp =
+  "line " <> show (line sp) <> ", column " <> show (column sp)
+
+instance A.ToJSON SourcePos where
+  toJSON SourcePos{..} =
+    A.toJSON [line, column]
+
+instance A.FromJSON SourcePos where
+  parseJSON arr = do
+    [line, col] <- A.parseJSON arr
+    pure $ SourcePos line col
+
+data SourceSpan = SourceSpan
+  { -- |
+    -- Source name
+    --
+    spanName :: !Text
+    -- |
+    -- Start of the span
+    --
+  , spanStart :: !SourcePos
+    -- End of the span
+    --
+  , spanEnd :: !SourcePos
+  } deriving (Show, Read, Eq, Ord, Generic)
+
+displayStartEndPos :: SourceSpan -> Text
+displayStartEndPos sp =
+  displaySourcePos (spanStart sp) <> " - " <> displaySourcePos (spanEnd sp)
+
+displaySourceSpan :: SourceSpan -> Text
+displaySourceSpan sp =
+  spanName sp <> " " <> displayStartEndPos sp
+
+instance A.ToJSON SourceSpan where
+  toJSON SourceSpan{..} =
+    A.object [ "name"  .= spanName
+             , "start" .= spanStart
+             , "end"   .= spanEnd
+             ]
+
+instance A.FromJSON SourceSpan where
+  parseJSON = A.withObject "SourceSpan" $ \o ->
+    SourceSpan     <$>
+      o .: "name"  <*>
+      o .: "start" <*>
+      o .: "end"
+
+instance Arbitrary SourcePos where
+  arbitrary = SourcePos <$> arbitrary <*> arbitrary
+  shrink = genericShrink
+
+instance Arbitrary SourceSpan where
+  arbitrary = SourceSpan <$> arbitrary <*> arbitrary <*> arbitrary
+  shrink = genericShrink

src/SplitDiff.hs

@@ -2,7 +2,6 @@ module SplitDiff where
 
 import Data.Record
 import Info
-import Range
 import Prologue
 import Syntax
 import Term (Term)

src/Syntax.hs

@@ -1,16 +1,19 @@
+{-# LANGUAGE DeriveAnyClass #-}
 module Syntax where
 
-import Prologue
+import Data.Mergeable
 import GHC.Generics
+import Prologue
 import Test.QuickCheck hiding (Fixed)
+import SourceSpan
 
 -- | A node in an abstract syntax tree.
-data Syntax
-  a -- ^ The type of leaves in the syntax tree, typically String, but possibly some datatype representing different leaves more precisely.
-  f -- ^ The type representing another level of the tree, e.g. the children of branches. Often Cofree or Fix or similar.
-  =
+--
+-- 'a' is the type of leaves in the syntax tree, typically 'Text', but possibly some datatype representing different leaves more precisely.
+-- 'f' is the type representing another level of the tree, e.g. the children of branches. Often 'Cofree', 'Free' or similar.
+data Syntax a f
   -- | A terminal syntax node, e.g. an identifier, or atomic literal.
-  Leaf a
+  = Leaf a
   -- | An ordered branch of child nodes, expected to be variadic in the grammar, e.g. a list of statements or uncurried function parameters.
   | Indexed [f]
   -- | An ordered branch of child nodes, expected to be of fixed length in the grammar, e.g. a binary operator & its operands.
@@ -46,14 +49,36 @@ data Syntax
   | Switch { switchExpr :: f, cases :: [f] }
   | Case { caseExpr :: f, caseStatements :: f }
   | Object { keyValues :: [f] }
+  -- | A pair in an Object. e.g. foo: bar or foo => bar
   | Pair f f
+  -- | A comment.
   | Comment a
+  -- | A term preceded or followed by any number of comments.
   | Commented [f] (Maybe f)
-  deriving (Eq, Foldable, Functor, Generic, Generic1, Ord, Show, Traversable)
+  | Error SourceSpan [f]
+  | For [f] f
+  | DoWhile { doWhileBody :: f, doWhileExpr :: f }
+  | While { whileExpr :: f, whileBody :: f }
+  | Return (Maybe f)
+  | Throw f
+  | Constructor f
+  | Try f (Maybe f) (Maybe f)
+  -- | An array literal with list of children.
+  | Array [f]
+  -- | A class with an identifier, superclass, and a list of definitions.
+  | Class f (Maybe f) [f]
+  -- | A method definition with an identifier, params, and a list of expressions.
+  | Method f [f] [f]
+  deriving (Eq, Foldable, Functor, Generic, Generic1, Mergeable, Ord, Show, Traversable)
 
 
 -- Instances
 
+instance (Arbitrary leaf, Arbitrary f) => Arbitrary (Syntax leaf f) where
+  arbitrary = sized (syntaxOfSize (`resize` arbitrary) )
+
+  shrink = genericShrink
+
 syntaxOfSize :: Arbitrary leaf => (Int -> Gen f) -> Int -> Gen (Syntax leaf f)
 syntaxOfSize recur n | n <= 1 = oneof $ (Leaf <$> arbitrary) : branchGeneratorsOfSize n
                      | otherwise = oneof $ branchGeneratorsOfSize n
@@ -67,8 +92,3 @@ syntaxOfSize recur n | n <= 1 = oneof $ (Leaf <$> arbitrary) : branchGeneratorsO
           first <- recur m
           rest <- childrenOfSize (n - m)
           pure $! first : rest
-
-instance (Arbitrary leaf, Arbitrary f) => Arbitrary (Syntax leaf f) where
-  arbitrary = sized (syntaxOfSize (`resize` arbitrary) )
-
-  shrink = genericShrink

src/TreeSitter.hs

@@ -7,22 +7,22 @@ import Category
 import Info
 import Language
 import Parser
-import Range
 import Source
 import Foreign
 import Foreign.C.String
 import Text.Parser.TreeSitter hiding (Language(..))
 import qualified Text.Parser.TreeSitter as TS
+import SourceSpan
 
 -- | Returns a TreeSitter parser for the given language and TreeSitter grammar.
 treeSitterParser :: Language -> Ptr TS.Language -> Parser '[Range, Category, Cost]
-treeSitterParser language grammar contents = do
+treeSitterParser language grammar blob = do
   document <- ts_document_make
   ts_document_set_language document grammar
-  withCString (toString contents) (\source -> do
+  withCString (toString $ source blob) (\source -> do
     ts_document_set_input_string document source
     ts_document_parse document
-    term <- documentToTerm language document contents
+    term <- documentToTerm language document blob
     ts_document_free document
     pure term)
 
@@ -30,15 +30,26 @@ treeSitterParser language grammar contents = do
 categoriesForLanguage :: Language -> Text -> Category
 categoriesForLanguage language name = case (language, name) of
   (JavaScript, "object") -> Object
-  (JavaScript, "rel_op") -> BinaryOperator -- relational operator, e.g. >, <, <=, >=, ==, !=
+  (JavaScript, "expression_statement") -> ExpressionStatements
   (JavaScript, "this_expression") -> Identifier
   (JavaScript, "null") -> Identifier
   (JavaScript, "undefined") -> Identifier
   (JavaScript, "arrow_function") -> Function
   (JavaScript, "generator_function") -> Function
-  (JavaScript, "delete_op") -> Operator
-  (JavaScript, "type_op") -> Operator
-  (JavaScript, "void_op") -> Operator
+  (JavaScript, "math_op") -> BinaryOperator -- bitwise operator, e.g. +, -, *, /.
+  (JavaScript, "bool_op") -> BinaryOperator -- boolean operator, e.g. ||, &&.
+  (JavaScript, "bitwise_op") -> BinaryOperator -- bitwise operator, e.g. ^, &, etc.
+  (JavaScript, "rel_op") -> BinaryOperator -- relational operator, e.g. >, <, <=, >=, ==, !=.
+  (JavaScript, "comma_op") -> Operator -- comma operator, e.g. expr1, expr2.
+  (JavaScript, "delete_op") -> Operator -- delete operator, e.g. delete x[2].
+  (JavaScript, "type_op") -> Operator -- type operator, e.g. typeof Object.
+  (JavaScript, "void_op") -> Operator -- void operator, e.g. void 2.
+  (JavaScript, "for_in_statement") -> For
+  (JavaScript, "for_of_statement") -> For
+  (JavaScript, "new_expression") -> Constructor
+  (JavaScript, "class")  -> Class
+  (JavaScript, "catch") -> Catch
+  (JavaScript, "finally") -> Finally
 
   (Ruby, "hash") -> Object
   _ -> defaultCategoryForNodeName name
@@ -73,11 +84,18 @@ defaultCategoryForNodeName name = case name of
   "true" -> Boolean
   "false" -> Boolean
   "ternary" -> Ternary
+  "for_statement" -> For
+  "while_statement" -> While
+  "do_statement" -> DoWhile
+  "return_statement" -> Return
+  "throw_statement" -> Throw
+  "try_statement" -> Try
+  "method_definition" -> Method
   _ -> Other name
 
 -- | Return a parser for a tree sitter language & document.
 documentToTerm :: Language -> Ptr Document -> Parser '[Range, Category, Cost]
-documentToTerm language document contents = alloca $ \ root -> do
+documentToTerm language document blob = alloca $ \ root -> do
   ts_document_root_node_p document root
   toTerm root
   where toTerm node = do
@@ -88,9 +106,13 @@ documentToTerm language document contents = alloca $ \ root -> do
           -- Note: The strict application here is semantically important. Without it, we may not evaluate the range until after we’ve exited the scope that `node` was allocated within, meaning `alloca` will free it & other stack data may overwrite it.
           range <- pure $! Range { start = fromIntegral $ ts_node_p_start_char node, end = fromIntegral $ ts_node_p_end_char node }
 
+          sourceSpan <- pure $! SourceSpan { spanName = toS (path blob)
+            , spanStart = SourcePos (fromIntegral $ ts_node_p_start_point_row node) (fromIntegral $ ts_node_p_start_point_column node)
+            , spanEnd = SourcePos (fromIntegral $ ts_node_p_end_point_row node) (fromIntegral $ ts_node_p_end_point_column node) }
+
           let cost' = 1 + sum (cost . extract <$> children)
           let info = range .: (categoriesForLanguage language (toS name)) .: cost' .: RNil
-          pure $! termConstructor contents info children
+          pure $! termConstructor (source blob) sourceSpan info children
         getChild node n out = do
           _ <- ts_node_p_named_child node n out
           toTerm out

test/AlignmentSpec.hs

@@ -26,6 +26,7 @@ import Term
 import Test.Hspec
 import Test.Hspec.QuickCheck
 import Test.QuickCheck
+import GHC.Show (Show(..))
 
 spec :: Spec
 spec = parallel $ do

test/CorpusSpec.hs

@@ -1,7 +1,7 @@
 {-# LANGUAGE DataKinds, FlexibleContexts, GeneralizedNewtypeDeriving #-}
 module CorpusSpec where
 
-import System.IO
+import Data.String
 import Diffing
 import Renderer
 import qualified Renderer.JSON as J
@@ -23,6 +23,7 @@ import qualified Source as S
 import System.FilePath
 import System.FilePath.Glob
 import Test.Hspec
+import GHC.Show (Show(..))
 
 spec :: Spec
 spec = parallel $ do
@@ -77,7 +78,7 @@ testDiff renderer paths diff matcher = do
   case diff of
     Nothing -> matcher actual actual
     Just file -> do
-      expected <- Verbatim . T.pack <$> readFile file
+      expected <- Verbatim <$> readFile file
       matcher actual expected
   where parser = parserForFilepath (fst paths)
         sourceBlobs sources = pure S.SourceBlob <*> sources <*> pure mempty <*> paths <*> pure (Just S.defaultPlainBlob)

test/Data/Mergeable/Spec.hs

@@ -0,0 +1,53 @@
+{-# LANGUAGE RankNTypes, ScopedTypeVariables #-}
+module Data.Mergeable.Spec where
+
+import Data.Functor.Identity
+import Data.Mergeable
+import Prologue
+import Syntax
+import Test.Hspec
+import Test.Hspec.QuickCheck
+import Test.QuickCheck
+
+spec :: Spec
+spec = parallel $ do
+  describe "[]" $ do
+    let gen = scale (`div` 25) arbitrary :: Gen [Char]
+    withAlternativeInstances sequenceAltLaws gen
+    withAlternativeInstances mergeLaws gen
+  describe "Maybe" $ do
+    withAlternativeInstances sequenceAltLaws (arbitrary :: Gen (Maybe Char))
+    withAlternativeInstances mergeLaws (arbitrary :: Gen (Maybe Char))
+  describe "Identity" $ do
+    withAlternativeInstances sequenceAltLaws (Identity <$> arbitrary :: Gen (Identity Char))
+    withAlternativeInstances mergeLaws (Identity <$> arbitrary :: Gen (Identity Char))
+  describe "Syntax" $ do
+    withAlternativeInstances sequenceAltLaws (sized (syntaxOfSize (const arbitrary)) :: Gen (Syntax Char Char))
+    withAlternativeInstances mergeLaws (sized (syntaxOfSize (const arbitrary)) :: Gen (Syntax Char Char))
+
+  prop "subsumes catMaybes/Just" $ do
+    \ a -> sequenceAlt a `shouldBe` pure (catMaybes (a :: [Maybe Char]))
+
+mergeLaws :: forall f g a. (Mergeable f, Alternative g, Eq (g (f a)), Show (f a), Show (g (f a))) => Gen (f a) -> Gen (Blind (a -> g a)) -> Spec
+mergeLaws value function = describe "merge" $ do
+  prop "identity" . forAll value $
+    \ a -> merge pure a `shouldNotBe` (empty :: g (f a))
+
+  let pair = (,) <$> value <*> function
+  prop "relationship with sequenceAlt" . forAll pair $
+    \ (a, f) -> merge (getBlind f) a `shouldBe` sequenceAlt (fmap (getBlind f) a)
+
+sequenceAltLaws :: forall f g a. (Mergeable f, Alternative g, Eq (g (f a)), Show (f a), Show (g (f a))) => Gen (f a) -> Gen (Blind (a -> g a)) -> Spec
+sequenceAltLaws value function = do
+  describe "sequenceAlt" $ do
+    prop "identity" . forAll value $
+      \ a -> sequenceAlt (pure <$> a) `shouldNotBe` (empty :: g (f a))
+
+    prop "relationship with merge" . forAll (Blind <$> (fmap . getBlind <$> function <*> value) :: Gen (Blind (f (g a)))) $
+      \ a -> sequenceAlt (getBlind a) `shouldBe` merge identity (getBlind a)
+
+
+withAlternativeInstances :: forall f a. (Arbitrary a, CoArbitrary a, Eq (f a), Show (f a)) => (forall g. (Alternative g, Eq (g (f a)), Show (g (f a))) => Gen (f a) -> Gen (Blind (a -> g a)) -> Spec) -> Gen (f a) -> Spec
+withAlternativeInstances laws gen = do
+  describe "[]" $ laws gen (scale (`div` 25) (arbitrary :: Gen (Blind (a -> [a]))))
+  describe "Maybe" $ laws gen (arbitrary :: Gen (Blind (a -> Maybe a)))

test/DiffSummarySpec.hs

@@ -7,40 +7,99 @@ import Test.Hspec
 import Test.Hspec.QuickCheck
 import Diff
 import Syntax
+import Term
 import Patch
 import Category
 import DiffSummary
-import Text.PrettyPrint.Leijen.Text
+import Text.PrettyPrint.Leijen.Text (pretty)
+import Test.Hspec.QuickCheck
+import Diff.Arbitrary
+import Data.List (partition)
 import Term.Arbitrary
 import Interpreter
+import Info
+import Source
+import Data.Functor.Both
 
-arrayInfo :: Record '[Category]
-arrayInfo = ArrayLiteral .: RNil
+arrayInfo :: Record '[Category, Range]
+arrayInfo = ArrayLiteral .: Range 0 3 .: RNil
 
-literalInfo :: Record '[Category]
-literalInfo = StringLiteral .: RNil
+literalInfo :: Record '[Category, Range]
+literalInfo = StringLiteral .: Range 1 2 .: RNil
 
-testDiff :: Diff Text (Record '[Category])
+testDiff :: Diff Text (Record '[Category, Range])
 testDiff = free $ Free (pure arrayInfo :< Indexed [ free $ Pure (Insert (cofree $ literalInfo :< Leaf "a")) ])
 
 testSummary :: DiffSummary DiffInfo
-testSummary = DiffSummary { patch = Insert (DiffInfo "string" "a"), parentAnnotations = [] }
+testSummary = DiffSummary { patch = Insert (LeafInfo "string" "a"), parentAnnotations = [] }
 
 replacementSummary :: DiffSummary DiffInfo
-replacementSummary = DiffSummary { patch = Replace (DiffInfo "string" "a") (DiffInfo "symbol" "b"), parentAnnotations = [ ArrayLiteral ] }
+replacementSummary = DiffSummary { patch = Replace (LeafInfo "string" "a") (LeafInfo "symbol" "b"), parentAnnotations = [ ArrayLiteral ] }
+
+sources :: Both (Source Char)
+sources = both (fromText "[]") (fromText "[a]")
 
 spec :: Spec
 spec = parallel $ do
-  describe "diffSummary" $ do
+  describe "diffSummaries" $ do
     it "outputs a diff summary" $ do
-      diffSummary testDiff `shouldBe` [ DiffSummary { patch = Insert (DiffInfo "string" "a"), parentAnnotations = [ ArrayLiteral ] } ]
+      diffSummaries sources testDiff `shouldBe` [ DiffSummary { patch = Insert (LeafInfo "string" "a"), parentAnnotations = [ ArrayLiteral ] } ]
 
     prop "equal terms produce identity diffs" $
-      \ a -> let term = toTerm (a :: ArbitraryTerm Text (Record '[Category])) in
-        diffSummary (diffTerms wrap (==) diffCost term term) `shouldBe` []
+      \ a -> let term = toTerm (a :: ArbitraryTerm Text (Record '[Category, Range])) in
+        diffSummaries sources (diffTerms wrap (==) diffCost term term) `shouldBe` []
 
-  describe "show" $ do
+  describe "annotatedSummaries" $ do
     it "should print adds" $
-      show (pretty testSummary) `shouldBe` ("Added the 'a' string" :: Text)
+      annotatedSummaries testSummary `shouldBe` ["Added the 'a' string"]
     it "prints a replacement" $ do
-      show (pretty replacementSummary) `shouldBe` ("Replaced the 'a' string with the 'b' symbol in the array context" :: Text)
+      annotatedSummaries replacementSummary `shouldBe` ["Replaced the 'a' string with the 'b' symbol in the array context"]
+  describe "DiffInfo" $ do
+    prop "patches in summaries match the patches in diffs" $
+      \a -> let
+        diff = (toDiff (a :: ArbitraryDiff Text (Record '[Category, Cost, Range])))
+        summaries = diffSummaries sources diff
+        patches = toList diff
+        in
+          case (partition isBranchNode (patch <$> summaries), partition isIndexedOrFixed patches) of
+            ((branchPatches, otherPatches), (branchDiffPatches, otherDiffPatches)) ->
+              (() <$ branchPatches, () <$ otherPatches) `shouldBe` (() <$ branchDiffPatches, () <$ otherDiffPatches)
+    prop "generates one LeafInfo for each child in an arbitrary branch patch" $
+      \a -> let
+        diff = (toDiff (a :: ArbitraryDiff Text (Record '[Category, Range])))
+        diffInfoPatches = patch <$> diffSummaries sources diff
+        syntaxPatches = toList diff
+        extractLeaves :: DiffInfo -> [DiffInfo]
+        extractLeaves (BranchInfo children _ _) = join $ extractLeaves <$> children
+        extractLeaves leaf = [ leaf ]
+
+        extractDiffLeaves :: Term Text (Record '[Category, Range]) -> [ Term Text (Record '[Category, Range]) ]
+        extractDiffLeaves term = case unwrap term of
+          (Indexed children) -> join $ extractDiffLeaves <$> children
+          (Fixed children) -> join $ extractDiffLeaves <$> children
+          Commented children leaf -> children <> maybeToList leaf >>= extractDiffLeaves
+          _ -> [ term ]
+        in
+          case (partition isBranchNode diffInfoPatches, partition isIndexedOrFixed syntaxPatches) of
+            ((branchPatches, _), (diffPatches, _)) ->
+              let listOfLeaves = foldMap extractLeaves (join $ toList <$> branchPatches)
+                  listOfDiffLeaves = foldMap extractDiffLeaves (diffPatches >>= toList)
+               in
+                length listOfLeaves `shouldBe` length listOfDiffLeaves
+
+isIndexedOrFixed :: Patch (Term a annotation) -> Bool
+isIndexedOrFixed = any (isIndexedOrFixed' . unwrap)
+
+isIndexedOrFixed' :: Syntax a f -> Bool
+isIndexedOrFixed' syntax = case syntax of
+  (Indexed _) -> True
+  (Fixed _) -> True
+  _ -> False
+
+isBranchInfo :: DiffInfo -> Bool
+isBranchInfo info = case info of
+  (BranchInfo _ _ _) -> True
+  (LeafInfo _ _) -> False
+
+isBranchNode :: Patch DiffInfo -> Bool
+isBranchNode = any isBranchInfo

test/Spec.hs

@@ -3,6 +3,7 @@ module Main where
 import Prologue
 import qualified AlignmentSpec
 import qualified CorpusSpec
+import qualified Data.Mergeable.Spec
 import qualified Data.RandomWalkSimilarity.Spec
 import qualified Diff.Spec
 import qualified DiffSummarySpec
@@ -15,6 +16,7 @@ main :: IO ()
 main = hspec . parallel $ do
   describe "Alignment" AlignmentSpec.spec
   describe "Corpus" CorpusSpec.spec
+  describe "Data.Mergeable" Data.Mergeable.Spec.spec
   describe "Data.RandomWalkSimilarity" Data.RandomWalkSimilarity.Spec.spec
   describe "Diff.Spec" Diff.Spec.spec
   describe "DiffSummary" DiffSummarySpec.spec

test/diffs/asymmetrical-context.split.js

@@ -1,7 +1,7 @@
 <!DOCTYPE HTML>
-<html><head><link rel="stylesheet" href="style.css"></head><body><table class="diff"><colgroup><col width="40"><col><col width="40"><col></colgroup><tr><td class="blob-num">1</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statement"><li><ul class="category-method_call"><li><span class="category-identifier">console</span></li>.<li><span class="category-identifier">log</span></li>(<li><ul class="category-arguments"><li><ul class="category-string"><li><span class="category-string">&#39;</span></li><li><span class="category-string">hello</span></li><li><span class="category-string">&#39;</span></li></ul></li></ul></li>)</ul></li>;</ul></li>
+<html><head><link rel="stylesheet" href="style.css"></head><body><table class="diff"><colgroup><col width="40"><col><col width="40"><col></colgroup><tr><td class="blob-num">1</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statements"><li><ul class="category-method_call"><li><span class="category-identifier">console</span></li>.<li><span class="category-identifier">log</span></li>(<li><ul class="category-arguments"><li><ul class="category-string"><li><span class="category-string">&#39;</span></li><li><span class="category-string">hello</span></li><li><span class="category-string">&#39;</span></li></ul></li></ul></li>)</ul></li>;</ul></li>
 </ul></td>
-<td class="blob-num">1</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statement"><li><ul class="category-method_call"><li><span class="category-identifier">console</span></li>.<li><span class="category-identifier">log</span></li>(<li><ul class="category-arguments"><li><ul class="category-string"><li><span class="category-string">&#39;</span></li><li><span class="category-string">hello</span></li><li><span class="category-string">&#39;</span></li></ul></li></ul></li>)</ul></li>;</ul></li>
+<td class="blob-num">1</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statements"><li><ul class="category-method_call"><li><span class="category-identifier">console</span></li>.<li><span class="category-identifier">log</span></li>(<li><ul class="category-arguments"><li><ul class="category-string"><li><span class="category-string">&#39;</span></li><li><span class="category-string">hello</span></li><li><span class="category-string">&#39;</span></li></ul></li></ul></li>)</ul></li>;</ul></li>
 </ul></td>
 
 </tr><tr><td class="blob-num blob-num-empty empty-cell"></td><td class="blob-code blob-code-empty empty-cell"></td>
@@ -20,9 +20,9 @@
 <td class="blob-num">5</td><td class="blob-code"><ul class="category-program">
 </ul></td>
 
-</tr><tr><td class="blob-num">2</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statement"><li><ul class="category-method_call"><li><span class="category-identifier">console</span></li>.<li><span class="category-identifier">log</span></li>(<li><ul class="category-arguments"><li><ul class="category-string"><li><span class="category-string">&#39;</span></li><li><span class="category-string">world</span></li><li><span class="category-string">&#39;</span></li></ul></li></ul></li>)</ul></li>;</ul></li>
+</tr><tr><td class="blob-num">2</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statements"><li><ul class="category-method_call"><li><span class="category-identifier">console</span></li>.<li><span class="category-identifier">log</span></li>(<li><ul class="category-arguments"><li><ul class="category-string"><li><span class="category-string">&#39;</span></li><li><span class="category-string">world</span></li><li><span class="category-string">&#39;</span></li></ul></li></ul></li>)</ul></li>;</ul></li>
 </ul></td>
-<td class="blob-num">6</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statement"><li><ul class="category-method_call"><li><span class="category-identifier">console</span></li>.<li><span class="category-identifier">log</span></li>(<li><ul class="category-arguments"><li><ul class="category-string"><li><span class="category-string">&#39;</span></li><li><span class="category-string">world</span></li><li><span class="category-string">&#39;</span></li></ul></li></ul></li>)</ul></li>;</ul></li>
+<td class="blob-num">6</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statements"><li><ul class="category-method_call"><li><span class="category-identifier">console</span></li>.<li><span class="category-identifier">log</span></li>(<li><ul class="category-arguments"><li><ul class="category-string"><li><span class="category-string">&#39;</span></li><li><span class="category-string">world</span></li><li><span class="category-string">&#39;</span></li></ul></li></ul></li>)</ul></li>;</ul></li>
 </ul></td>
 
 </tr><tr><td class="blob-num">3</td><td class="blob-code"><ul class="category-program"></ul></td>

test/diffs/dictionary.json.js

@@ -1 +1 @@
-{"rows":[[{"number":1,"terms":[{"range":[0,2],"category":"Program","children":[{"range":[0,2],"category":"expression_statement","children":[{"range":[0,2],"category":"Object","children":[]}]}]}],"range":[0,2],"hasChanges":false},{"number":1,"terms":[{"range":[0,2],"category":"Program","children":[{"range":[0,2],"category":"expression_statement","children":[{"range":[0,2],"category":"Object","children":[]}]}]}],"range":[0,2],"hasChanges":false}],[{"number":2,"terms":[{"range":[2,12],"category":"Program","children":[{"range":[2,12],"category":"expression_statement","children":[{"range":[2,12],"category":"Object","children":[{"range":[4,10],"category":"Pair","children":[{"range":[4,7],"category":"StringLiteral","children":[{"range":[4,5],"category":"StringLiteral"},{"range":[5,6],"category":"StringLiteral"},{"range":[6,7],"category":"StringLiteral"}]},{"patch":"replace","range":[9,10],"category":"number"}]}]}]}]}],"range":[2,12],"hasChanges":true},{"number":2,"terms":[{"range":[2,12],"category":"Program","children":[{"range":[2,12],"category":"expression_statement","children":[{"range":[2,12],"category":"Object","children":[{"range":[4,10],"category":"Pair","children":[{"range":[4,7],"category":"StringLiteral","children":[{"range":[4,5],"category":"StringLiteral"},{"range":[5,6],"category":"StringLiteral"},{"range":[6,7],"category":"StringLiteral"}]},{"patch":"replace","range":[9,10],"category":"number"}]}]}]}]}],"range":[2,12],"hasChanges":true}],[{"number":3,"terms":[{"range":[12,21],"category":"Program","children":[{"range":[12,21],"category":"expression_statement","children":[{"range":[12,21],"category":"Object","children":[{"range":[14,20],"category":"Pair","children":[{"range":[14,17],"category":"StringLiteral","children":[{"range":[14,15],"category":"StringLiteral"},{"range":[15,16],"category":"StringLiteral"},{"range":[16,17],"category":"StringLiteral"}]},{"range":[19,20],"category":"number"}]}]}]}]}],"range":[12,21],"hasChanges":false},{"number":3,"terms":[{"range":[12,21],"category":"Program","children":[{"range":[12,21],"category":"expression_statement","children":[{"range":[12,21],"category":"Object","children":[{"range":[14,20],"category":"Pair","children":[{"range":[14,17],"category":"StringLiteral","children":[{"range":[14,15],"category":"StringLiteral"},{"range":[15,16],"category":"StringLiteral"},{"range":[16,17],"category":"StringLiteral"}]},{"range":[19,20],"category":"number"}]}]}]}]}],"range":[12,21],"hasChanges":false}],[{"number":4,"terms":[{"range":[21,23],"category":"Program","children":[{"range":[21,23],"category":"expression_statement","children":[{"range":[21,22],"category":"Object","children":[]}]}]}],"range":[21,23],"hasChanges":false},{"number":4,"terms":[{"range":[21,23],"category":"Program","children":[{"range":[21,23],"category":"expression_statement","children":[{"range":[21,22],"category":"Object","children":[]}]}]}],"range":[21,23],"hasChanges":false}],[{"number":5,"terms":[{"range":[23,23],"category":"Program","children":[]}],"range":[23,23],"hasChanges":false},{"number":5,"terms":[{"range":[23,23],"category":"Program","children":[]}],"range":[23,23],"hasChanges":false}]],"oids":["",""],"paths":["test/diffs/dictionary.A.js","test/diffs/dictionary.B.js"]}
+{"rows":[[{"number":1,"terms":[{"range":[0,2],"category":"Program","children":[{"range":[0,2],"category":"ExpressionStatements","children":[{"range":[0,2],"category":"Object","children":[]}]}]}],"range":[0,2],"hasChanges":false},{"number":1,"terms":[{"range":[0,2],"category":"Program","children":[{"range":[0,2],"category":"ExpressionStatements","children":[{"range":[0,2],"category":"Object","children":[]}]}]}],"range":[0,2],"hasChanges":false}],[{"number":2,"terms":[{"range":[2,12],"category":"Program","children":[{"range":[2,12],"category":"ExpressionStatements","children":[{"range":[2,12],"category":"Object","children":[{"range":[4,10],"category":"Pair","children":[{"range":[4,7],"category":"StringLiteral","children":[{"range":[4,5],"category":"StringLiteral"},{"range":[5,6],"category":"StringLiteral"},{"range":[6,7],"category":"StringLiteral"}]},{"patch":"replace","range":[9,10],"category":"number"}]}]}]}]}],"range":[2,12],"hasChanges":true},{"number":2,"terms":[{"range":[2,12],"category":"Program","children":[{"range":[2,12],"category":"ExpressionStatements","children":[{"range":[2,12],"category":"Object","children":[{"range":[4,10],"category":"Pair","children":[{"range":[4,7],"category":"StringLiteral","children":[{"range":[4,5],"category":"StringLiteral"},{"range":[5,6],"category":"StringLiteral"},{"range":[6,7],"category":"StringLiteral"}]},{"patch":"replace","range":[9,10],"category":"number"}]}]}]}]}],"range":[2,12],"hasChanges":true}],[{"number":3,"terms":[{"range":[12,21],"category":"Program","children":[{"range":[12,21],"category":"ExpressionStatements","children":[{"range":[12,21],"category":"Object","children":[{"range":[14,20],"category":"Pair","children":[{"range":[14,17],"category":"StringLiteral","children":[{"range":[14,15],"category":"StringLiteral"},{"range":[15,16],"category":"StringLiteral"},{"range":[16,17],"category":"StringLiteral"}]},{"range":[19,20],"category":"number"}]}]}]}]}],"range":[12,21],"hasChanges":false},{"number":3,"terms":[{"range":[12,21],"category":"Program","children":[{"range":[12,21],"category":"ExpressionStatements","children":[{"range":[12,21],"category":"Object","children":[{"range":[14,20],"category":"Pair","children":[{"range":[14,17],"category":"StringLiteral","children":[{"range":[14,15],"category":"StringLiteral"},{"range":[15,16],"category":"StringLiteral"},{"range":[16,17],"category":"StringLiteral"}]},{"range":[19,20],"category":"number"}]}]}]}]}],"range":[12,21],"hasChanges":false}],[{"number":4,"terms":[{"range":[21,23],"category":"Program","children":[{"range":[21,23],"category":"ExpressionStatements","children":[{"range":[21,22],"category":"Object","children":[]}]}]}],"range":[21,23],"hasChanges":false},{"number":4,"terms":[{"range":[21,23],"category":"Program","children":[{"range":[21,23],"category":"ExpressionStatements","children":[{"range":[21,22],"category":"Object","children":[]}]}]}],"range":[21,23],"hasChanges":false}],[{"number":5,"terms":[{"range":[23,23],"category":"Program","children":[]}],"range":[23,23],"hasChanges":false},{"number":5,"terms":[{"range":[23,23],"category":"Program","children":[]}],"range":[23,23],"hasChanges":false}]],"oids":["",""],"paths":["test/diffs/dictionary.A.js","test/diffs/dictionary.B.js"]}

test/diffs/dictionary.split.js

@@ -1,22 +1,22 @@
 <!DOCTYPE HTML>
-<html><head><link rel="stylesheet" href="style.css"></head><body><table class="diff"><colgroup><col width="40"><col><col width="40"><col></colgroup><tr><td class="blob-num">1</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statement"><li><ul class="category-object">{
+<html><head><link rel="stylesheet" href="style.css"></head><body><table class="diff"><colgroup><col width="40"><col><col width="40"><col></colgroup><tr><td class="blob-num">1</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statements"><li><ul class="category-object">{
 </ul></li></ul></li></ul></td>
-<td class="blob-num">1</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statement"><li><ul class="category-object">{
+<td class="blob-num">1</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statements"><li><ul class="category-object">{
 </ul></li></ul></li></ul></td>
 
-</tr><tr><td class="blob-num blob-num-replacement">2</td><td class="blob-code blob-code-replacement"><ul class="category-program"><li><ul class="category-expression_statement"><li><ul class="category-object">  <li><ul class="category-pair"><li><ul class="category-string"><li><span class="category-string">&quot;</span></li><li><span class="category-string">b</span></li><li><span class="category-string">&quot;</span></li></ul></li>: <li><div class="patch replace" data="1"><span class="category-number">4</span></div></li></ul></li>,
+</tr><tr><td class="blob-num blob-num-replacement">2</td><td class="blob-code blob-code-replacement"><ul class="category-program"><li><ul class="category-expression_statements"><li><ul class="category-object">  <li><ul class="category-pair"><li><ul class="category-string"><li><span class="category-string">&quot;</span></li><li><span class="category-string">b</span></li><li><span class="category-string">&quot;</span></li></ul></li>: <li><div class="patch replace" data="1"><span class="category-number">4</span></div></li></ul></li>,
 </ul></li></ul></li></ul></td>
-<td class="blob-num blob-num-replacement">2</td><td class="blob-code blob-code-replacement"><ul class="category-program"><li><ul class="category-expression_statement"><li><ul class="category-object">  <li><ul class="category-pair"><li><ul class="category-string"><li><span class="category-string">&quot;</span></li><li><span class="category-string">b</span></li><li><span class="category-string">&quot;</span></li></ul></li>: <li><div class="patch replace" data="1"><span class="category-number">5</span></div></li></ul></li>,
+<td class="blob-num blob-num-replacement">2</td><td class="blob-code blob-code-replacement"><ul class="category-program"><li><ul class="category-expression_statements"><li><ul class="category-object">  <li><ul class="category-pair"><li><ul class="category-string"><li><span class="category-string">&quot;</span></li><li><span class="category-string">b</span></li><li><span class="category-string">&quot;</span></li></ul></li>: <li><div class="patch replace" data="1"><span class="category-number">5</span></div></li></ul></li>,
 </ul></li></ul></li></ul></td>
 
-</tr><tr><td class="blob-num">3</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statement"><li><ul class="category-object">  <li><ul class="category-pair"><li><ul class="category-string"><li><span class="category-string">&quot;</span></li><li><span class="category-string">a</span></li><li><span class="category-string">&quot;</span></li></ul></li>: <li><span class="category-number">5</span></li></ul></li>
+</tr><tr><td class="blob-num">3</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statements"><li><ul class="category-object">  <li><ul class="category-pair"><li><ul class="category-string"><li><span class="category-string">&quot;</span></li><li><span class="category-string">a</span></li><li><span class="category-string">&quot;</span></li></ul></li>: <li><span class="category-number">5</span></li></ul></li>
 </ul></li></ul></li></ul></td>
-<td class="blob-num">3</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statement"><li><ul class="category-object">  <li><ul class="category-pair"><li><ul class="category-string"><li><span class="category-string">&quot;</span></li><li><span class="category-string">a</span></li><li><span class="category-string">&quot;</span></li></ul></li>: <li><span class="category-number">5</span></li></ul></li>
+<td class="blob-num">3</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statements"><li><ul class="category-object">  <li><ul class="category-pair"><li><ul class="category-string"><li><span class="category-string">&quot;</span></li><li><span class="category-string">a</span></li><li><span class="category-string">&quot;</span></li></ul></li>: <li><span class="category-number">5</span></li></ul></li>
 </ul></li></ul></li></ul></td>
 
-</tr><tr><td class="blob-num">4</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statement"><li><ul class="category-object">}</ul></li>
+</tr><tr><td class="blob-num">4</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statements"><li><ul class="category-object">}</ul></li>
 </ul></li></ul></td>
-<td class="blob-num">4</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statement"><li><ul class="category-object">}</ul></li>
+<td class="blob-num">4</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statements"><li><ul class="category-object">}</ul></li>
 </ul></li></ul></td>
 
 </tr><tr><td class="blob-num">5</td><td class="blob-code"><ul class="category-program"></ul></td>

test/diffs/insert.split.js

@@ -1,11 +1,11 @@
 <!DOCTYPE HTML>
-<html><head><link rel="stylesheet" href="style.css"></head><body><table class="diff"><colgroup><col width="40"><col><col width="40"><col></colgroup><tr><td class="blob-num">1</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statement"><li><ul class="category-method_call"><li><span class="category-identifier">console</span></li>.<li><span class="category-identifier">log</span></li>(<li><ul class="category-arguments"><li><ul class="category-string"><li><span class="category-string">&#39;</span></li><li><span class="category-string">hello</span></li><li><span class="category-string">&#39;</span></li></ul></li></ul></li>)</ul></li>;</ul></li>
+<html><head><link rel="stylesheet" href="style.css"></head><body><table class="diff"><colgroup><col width="40"><col><col width="40"><col></colgroup><tr><td class="blob-num">1</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statements"><li><ul class="category-method_call"><li><span class="category-identifier">console</span></li>.<li><span class="category-identifier">log</span></li>(<li><ul class="category-arguments"><li><ul class="category-string"><li><span class="category-string">&#39;</span></li><li><span class="category-string">hello</span></li><li><span class="category-string">&#39;</span></li></ul></li></ul></li>)</ul></li>;</ul></li>
 </ul></td>
-<td class="blob-num">1</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statement"><li><ul class="category-method_call"><li><span class="category-identifier">console</span></li>.<li><span class="category-identifier">log</span></li>(<li><ul class="category-arguments"><li><ul class="category-string"><li><span class="category-string">&#39;</span></li><li><span class="category-string">hello</span></li><li><span class="category-string">&#39;</span></li></ul></li></ul></li>)</ul></li>;</ul></li>
+<td class="blob-num">1</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statements"><li><ul class="category-method_call"><li><span class="category-identifier">console</span></li>.<li><span class="category-identifier">log</span></li>(<li><ul class="category-arguments"><li><ul class="category-string"><li><span class="category-string">&#39;</span></li><li><span class="category-string">hello</span></li><li><span class="category-string">&#39;</span></li></ul></li></ul></li>)</ul></li>;</ul></li>
 </ul></td>
 
 </tr><tr><td class="blob-num blob-num-empty empty-cell"></td><td class="blob-code blob-code-empty empty-cell"></td>
-<td class="blob-num blob-num-replacement">2</td><td class="blob-code blob-code-replacement"><ul class="category-program"><li><div class="patch insert" data="9"><ul class="category-expression_statement"><li><ul class="category-method_call"><li><span class="category-identifier">console</span></li>.<li><span class="category-identifier">log</span></li>(<li><ul class="category-arguments"><li><ul class="category-string"><li><span class="category-string">&#39;</span></li><li><span class="category-string">world</span></li><li><span class="category-string">&#39;</span></li></ul></li></ul></li>)</ul></li>;</ul></div></li>
+<td class="blob-num blob-num-replacement">2</td><td class="blob-code blob-code-replacement"><ul class="category-program"><li><div class="patch insert" data="9"><ul class="category-expression_statements"><li><ul class="category-method_call"><li><span class="category-identifier">console</span></li>.<li><span class="category-identifier">log</span></li>(<li><ul class="category-arguments"><li><ul class="category-string"><li><span class="category-string">&#39;</span></li><li><span class="category-string">world</span></li><li><span class="category-string">&#39;</span></li></ul></li></ul></li>)</ul></li>;</ul></div></li>
 </ul></td>
 
 </tr><tr><td class="blob-num">2</td><td class="blob-code"><ul class="category-program"></ul></td>

test/diffs/multiline-insert.split.js

@@ -1,7 +1,7 @@
 <!DOCTYPE HTML>
-<html><head><link rel="stylesheet" href="style.css"></head><body><table class="diff"><colgroup><col width="40"><col><col width="40"><col></colgroup><tr><td class="blob-num">1</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statement"><li><ul class="category-method_call"><li><span class="category-identifier">console</span></li>.<li><span class="category-identifier">log</span></li>(<li><ul class="category-arguments"><li><ul class="category-string"><li><span class="category-string">&#39;</span></li><li><span class="category-string">hello</span></li><li><span class="category-string">&#39;</span></li></ul></li></ul></li>)</ul></li>;</ul></li>
+<html><head><link rel="stylesheet" href="style.css"></head><body><table class="diff"><colgroup><col width="40"><col><col width="40"><col></colgroup><tr><td class="blob-num">1</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statements"><li><ul class="category-method_call"><li><span class="category-identifier">console</span></li>.<li><span class="category-identifier">log</span></li>(<li><ul class="category-arguments"><li><ul class="category-string"><li><span class="category-string">&#39;</span></li><li><span class="category-string">hello</span></li><li><span class="category-string">&#39;</span></li></ul></li></ul></li>)</ul></li>;</ul></li>
 </ul></td>
-<td class="blob-num">1</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statement"><li><ul class="category-method_call"><li><span class="category-identifier">console</span></li>.<li><span class="category-identifier">log</span></li>(<li><ul class="category-arguments"><li><ul class="category-string"><li><span class="category-string">&#39;</span></li><li><span class="category-string">hello</span></li><li><span class="category-string">&#39;</span></li></ul></li></ul></li>)</ul></li>;</ul></li>
+<td class="blob-num">1</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statements"><li><ul class="category-method_call"><li><span class="category-identifier">console</span></li>.<li><span class="category-identifier">log</span></li>(<li><ul class="category-arguments"><li><ul class="category-string"><li><span class="category-string">&#39;</span></li><li><span class="category-string">hello</span></li><li><span class="category-string">&#39;</span></li></ul></li></ul></li>)</ul></li>;</ul></li>
 </ul></td>
 
 </tr><tr><td class="blob-num blob-num-empty empty-cell"></td><td class="blob-code blob-code-empty empty-cell"></td>
@@ -9,16 +9,16 @@
 </ul></li></ul></div></li></ul></td>
 
 </tr><tr><td class="blob-num blob-num-empty empty-cell"></td><td class="blob-code blob-code-empty empty-cell"></td>
-<td class="blob-num blob-num-replacement">3</td><td class="blob-code blob-code-replacement"><ul class="category-program"><li><div class="patch insert" data="12"><ul class="category-if_statement"><li><ul class="category-expression_statements">  <li><ul class="category-expression_statement"><li><ul class="category-method_call"><li><span class="category-identifier">console</span></li>.<li><span class="category-identifier">log</span></li>(<li><ul class="category-arguments"><li><ul class="category-string"><li><span class="category-string">&#39;</span></li><li><span class="category-string">cruel</span></li><li><span class="category-string">&#39;</span></li></ul></li></ul></li>)</ul></li>;</ul></li>
+<td class="blob-num blob-num-replacement">3</td><td class="blob-code blob-code-replacement"><ul class="category-program"><li><div class="patch insert" data="12"><ul class="category-if_statement"><li><ul class="category-expression_statements">  <li><ul class="category-expression_statements"><li><ul class="category-method_call"><li><span class="category-identifier">console</span></li>.<li><span class="category-identifier">log</span></li>(<li><ul class="category-arguments"><li><ul class="category-string"><li><span class="category-string">&#39;</span></li><li><span class="category-string">cruel</span></li><li><span class="category-string">&#39;</span></li></ul></li></ul></li>)</ul></li>;</ul></li>
 </ul></li></ul></div></li></ul></td>
 
 </tr><tr><td class="blob-num blob-num-empty empty-cell"></td><td class="blob-code blob-code-empty empty-cell"></td>
 <td class="blob-num blob-num-replacement">4</td><td class="blob-code blob-code-replacement"><ul class="category-program"><li><div class="patch insert" data="12"><ul class="category-if_statement"><li><ul class="category-expression_statements">}</ul></li></ul></div></li>
 </ul></td>
 
-</tr><tr><td class="blob-num">2</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statement"><li><ul class="category-method_call"><li><span class="category-identifier">console</span></li>.<li><span class="category-identifier">log</span></li>(<li><ul class="category-arguments"><li><ul class="category-string"><li><span class="category-string">&#39;</span></li><li><span class="category-string">world</span></li><li><span class="category-string">&#39;</span></li></ul></li></ul></li>)</ul></li>;</ul></li>
+</tr><tr><td class="blob-num">2</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statements"><li><ul class="category-method_call"><li><span class="category-identifier">console</span></li>.<li><span class="category-identifier">log</span></li>(<li><ul class="category-arguments"><li><ul class="category-string"><li><span class="category-string">&#39;</span></li><li><span class="category-string">world</span></li><li><span class="category-string">&#39;</span></li></ul></li></ul></li>)</ul></li>;</ul></li>
 </ul></td>
-<td class="blob-num">5</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statement"><li><ul class="category-method_call"><li><span class="category-identifier">console</span></li>.<li><span class="category-identifier">log</span></li>(<li><ul class="category-arguments"><li><ul class="category-string"><li><span class="category-string">&#39;</span></li><li><span class="category-string">world</span></li><li><span class="category-string">&#39;</span></li></ul></li></ul></li>)</ul></li>;</ul></li>
+<td class="blob-num">5</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statements"><li><ul class="category-method_call"><li><span class="category-identifier">console</span></li>.<li><span class="category-identifier">log</span></li>(<li><ul class="category-arguments"><li><ul class="category-string"><li><span class="category-string">&#39;</span></li><li><span class="category-string">world</span></li><li><span class="category-string">&#39;</span></li></ul></li></ul></li>)</ul></li>;</ul></li>
 </ul></td>
 
 </tr><tr><td class="blob-num">3</td><td class="blob-code"><ul class="category-program"></ul></td>

test/diffs/nested-insert.split.js

@@ -4,13 +4,13 @@
 <td class="blob-num">1</td><td class="blob-code"><ul class="category-program"><li><ul class="category-if_statement">if (<li><span class="category-boolean">true</span></li>) <li><ul class="category-expression_statements">{
 </ul></li></ul></li></ul></td>
 
-</tr><tr><td class="blob-num">2</td><td class="blob-code"><ul class="category-program"><li><ul class="category-if_statement"><li><ul class="category-expression_statements">  <li><ul class="category-expression_statement"><li><ul class="category-method_call"><li><span class="category-identifier">console</span></li>.<li><span class="category-identifier">log</span></li>(<li><ul class="category-arguments"><li><ul class="category-string"><li><span class="category-string">&#39;</span></li><li><span class="category-string">hello</span></li><li><span class="category-string">&#39;</span></li></ul></li></ul></li>)</ul></li>;</ul></li>
+</tr><tr><td class="blob-num">2</td><td class="blob-code"><ul class="category-program"><li><ul class="category-if_statement"><li><ul class="category-expression_statements">  <li><ul class="category-expression_statements"><li><ul class="category-method_call"><li><span class="category-identifier">console</span></li>.<li><span class="category-identifier">log</span></li>(<li><ul class="category-arguments"><li><ul class="category-string"><li><span class="category-string">&#39;</span></li><li><span class="category-string">hello</span></li><li><span class="category-string">&#39;</span></li></ul></li></ul></li>)</ul></li>;</ul></li>
 </ul></li></ul></li></ul></td>
-<td class="blob-num">2</td><td class="blob-code"><ul class="category-program"><li><ul class="category-if_statement"><li><ul class="category-expression_statements">  <li><ul class="category-expression_statement"><li><ul class="category-method_call"><li><span class="category-identifier">console</span></li>.<li><span class="category-identifier">log</span></li>(<li><ul class="category-arguments"><li><ul class="category-string"><li><span class="category-string">&#39;</span></li><li><span class="category-string">hello</span></li><li><span class="category-string">&#39;</span></li></ul></li></ul></li>)</ul></li>;</ul></li>
+<td class="blob-num">2</td><td class="blob-code"><ul class="category-program"><li><ul class="category-if_statement"><li><ul class="category-expression_statements">  <li><ul class="category-expression_statements"><li><ul class="category-method_call"><li><span class="category-identifier">console</span></li>.<li><span class="category-identifier">log</span></li>(<li><ul class="category-arguments"><li><ul class="category-string"><li><span class="category-string">&#39;</span></li><li><span class="category-string">hello</span></li><li><span class="category-string">&#39;</span></li></ul></li></ul></li>)</ul></li>;</ul></li>
 </ul></li></ul></li></ul></td>
 
 </tr><tr><td class="blob-num blob-num-empty empty-cell"></td><td class="blob-code blob-code-empty empty-cell"></td>
-<td class="blob-num blob-num-replacement">3</td><td class="blob-code blob-code-replacement"><ul class="category-program"><li><ul class="category-if_statement"><li><ul class="category-expression_statements">  <li><div class="patch insert" data="9"><ul class="category-expression_statement"><li><ul class="category-method_call"><li><span class="category-identifier">console</span></li>.<li><span class="category-identifier">log</span></li>(<li><ul class="category-arguments"><li><ul class="category-string"><li><span class="category-string">&#39;</span></li><li><span class="category-string">world</span></li><li><span class="category-string">&#39;</span></li></ul></li></ul></li>)</ul></li>;</ul></div></li>
+<td class="blob-num blob-num-replacement">3</td><td class="blob-code blob-code-replacement"><ul class="category-program"><li><ul class="category-if_statement"><li><ul class="category-expression_statements">  <li><div class="patch insert" data="9"><ul class="category-expression_statements"><li><ul class="category-method_call"><li><span class="category-identifier">console</span></li>.<li><span class="category-identifier">log</span></li>(<li><ul class="category-arguments"><li><ul class="category-string"><li><span class="category-string">&#39;</span></li><li><span class="category-string">world</span></li><li><span class="category-string">&#39;</span></li></ul></li></ul></li>)</ul></li>;</ul></div></li>
 </ul></li></ul></li></ul></td>
 
 </tr><tr><td class="blob-num">3</td><td class="blob-code"><ul class="category-program"><li><ul class="category-if_statement"><li><ul class="category-expression_statements">}</ul></li></ul></li>

test/diffs/newline-at-eof.json.js

@@ -1 +1 @@
-{"rows":[[{"number":1,"terms":[{"range":[0,29],"category":"Program","children":[{"range":[0,28],"category":"expression_statement","children":[{"range":[0,27],"category":"MethodCall","children":[{"range":[0,7],"category":"Identifier"},{"range":[8,11],"category":"Identifier"},{"range":[12,26],"category":"Args","children":[{"range":[12,26],"category":"StringLiteral","children":[{"range":[12,13],"category":"StringLiteral"},{"range":[13,18],"category":"StringLiteral"},{"range":[18,19],"category":"StringLiteral"},{"range":[20,25],"category":"StringLiteral"},{"range":[25,26],"category":"StringLiteral"}]}]}]}]}]}],"range":[0,29],"hasChanges":false},{"number":1,"terms":[{"range":[0,29],"category":"Program","children":[{"range":[0,28],"category":"expression_statement","children":[{"range":[0,27],"category":"MethodCall","children":[{"range":[0,7],"category":"Identifier"},{"range":[8,11],"category":"Identifier"},{"range":[12,26],"category":"Args","children":[{"range":[12,26],"category":"StringLiteral","children":[{"range":[12,13],"category":"StringLiteral"},{"range":[13,18],"category":"StringLiteral"},{"range":[18,19],"category":"StringLiteral"},{"range":[20,25],"category":"StringLiteral"},{"range":[25,26],"category":"StringLiteral"}]}]}]}]}]}],"range":[0,29],"hasChanges":false}],[{"number":2,"terms":[{"range":[29,29],"category":"Program","children":[]}],"range":[29,29],"hasChanges":false},{"number":2,"terms":[{"range":[29,30],"category":"Program","children":[]}],"range":[29,30],"hasChanges":false}],[{"number":3,"terms":[{"range":[30,56],"category":"Program","children":[{"patch":"insert","range":[30,55],"category":"expression_statement","children":[{"range":[30,54],"category":"MethodCall","children":[{"range":[30,37],"category":"Identifier"},{"range":[38,41],"category":"Identifier"},{"range":[42,53],"category":"Args","children":[{"range":[42,53],"category":"StringLiteral","children":[{"range":[42,43],"category":"StringLiteral"},{"range":[43,52],"category":"StringLiteral"},{"range":[52,53],"category":"StringLiteral"}]}]}]}]}]}],"range":[30,56],"hasChanges":true}],[{"number":4,"terms":[{"range":[56,56],"category":"Program","children":[]}],"range":[56,56],"hasChanges":false}]],"oids":["",""],"paths":["test/diffs/newline-at-eof.A.js","test/diffs/newline-at-eof.B.js"]}
+{"rows":[[{"number":1,"terms":[{"range":[0,29],"category":"Program","children":[{"range":[0,28],"category":"ExpressionStatements","children":[{"range":[0,27],"category":"MethodCall","children":[{"range":[0,7],"category":"Identifier"},{"range":[8,11],"category":"Identifier"},{"range":[12,26],"category":"Args","children":[{"range":[12,26],"category":"StringLiteral","children":[{"range":[12,13],"category":"StringLiteral"},{"range":[13,18],"category":"StringLiteral"},{"range":[18,19],"category":"StringLiteral"},{"range":[20,25],"category":"StringLiteral"},{"range":[25,26],"category":"StringLiteral"}]}]}]}]}]}],"range":[0,29],"hasChanges":false},{"number":1,"terms":[{"range":[0,29],"category":"Program","children":[{"range":[0,28],"category":"ExpressionStatements","children":[{"range":[0,27],"category":"MethodCall","children":[{"range":[0,7],"category":"Identifier"},{"range":[8,11],"category":"Identifier"},{"range":[12,26],"category":"Args","children":[{"range":[12,26],"category":"StringLiteral","children":[{"range":[12,13],"category":"StringLiteral"},{"range":[13,18],"category":"StringLiteral"},{"range":[18,19],"category":"StringLiteral"},{"range":[20,25],"category":"StringLiteral"},{"range":[25,26],"category":"StringLiteral"}]}]}]}]}]}],"range":[0,29],"hasChanges":false}],[{"number":2,"terms":[{"range":[29,29],"category":"Program","children":[]}],"range":[29,29],"hasChanges":false},{"number":2,"terms":[{"range":[29,30],"category":"Program","children":[]}],"range":[29,30],"hasChanges":false}],[{"number":3,"terms":[{"range":[30,56],"category":"Program","children":[{"patch":"insert","range":[30,55],"category":"ExpressionStatements","children":[{"range":[30,54],"category":"MethodCall","children":[{"range":[30,37],"category":"Identifier"},{"range":[38,41],"category":"Identifier"},{"range":[42,53],"category":"Args","children":[{"range":[42,53],"category":"StringLiteral","children":[{"range":[42,43],"category":"StringLiteral"},{"range":[43,52],"category":"StringLiteral"},{"range":[52,53],"category":"StringLiteral"}]}]}]}]}]}],"range":[30,56],"hasChanges":true}],[{"number":4,"terms":[{"range":[56,56],"category":"Program","children":[]}],"range":[56,56],"hasChanges":false}]],"oids":["",""],"paths":["test/diffs/newline-at-eof.A.js","test/diffs/newline-at-eof.B.js"]}

test/diffs/newline-at-eof.split.js

@@ -1,7 +1,7 @@
 <!DOCTYPE HTML>
-<html><head><link rel="stylesheet" href="style.css"></head><body><table class="diff"><colgroup><col width="40"><col><col width="40"><col></colgroup><tr><td class="blob-num">1</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statement"><li><ul class="category-method_call"><li><span class="category-identifier">console</span></li>.<li><span class="category-identifier">log</span></li>(<li><ul class="category-arguments"><li><ul class="category-string"><li><span class="category-string">&quot;</span></li><li><span class="category-string">hello</span></li><li><span class="category-string">,</span></li> <li><span class="category-string">world</span></li><li><span class="category-string">&quot;</span></li></ul></li></ul></li>)</ul></li>;</ul></li>
+<html><head><link rel="stylesheet" href="style.css"></head><body><table class="diff"><colgroup><col width="40"><col><col width="40"><col></colgroup><tr><td class="blob-num">1</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statements"><li><ul class="category-method_call"><li><span class="category-identifier">console</span></li>.<li><span class="category-identifier">log</span></li>(<li><ul class="category-arguments"><li><ul class="category-string"><li><span class="category-string">&quot;</span></li><li><span class="category-string">hello</span></li><li><span class="category-string">,</span></li> <li><span class="category-string">world</span></li><li><span class="category-string">&quot;</span></li></ul></li></ul></li>)</ul></li>;</ul></li>
 </ul></td>
-<td class="blob-num">1</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statement"><li><ul class="category-method_call"><li><span class="category-identifier">console</span></li>.<li><span class="category-identifier">log</span></li>(<li><ul class="category-arguments"><li><ul class="category-string"><li><span class="category-string">&quot;</span></li><li><span class="category-string">hello</span></li><li><span class="category-string">,</span></li> <li><span class="category-string">world</span></li><li><span class="category-string">&quot;</span></li></ul></li></ul></li>)</ul></li>;</ul></li>
+<td class="blob-num">1</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statements"><li><ul class="category-method_call"><li><span class="category-identifier">console</span></li>.<li><span class="category-identifier">log</span></li>(<li><ul class="category-arguments"><li><ul class="category-string"><li><span class="category-string">&quot;</span></li><li><span class="category-string">hello</span></li><li><span class="category-string">,</span></li> <li><span class="category-string">world</span></li><li><span class="category-string">&quot;</span></li></ul></li></ul></li>)</ul></li>;</ul></li>
 </ul></td>
 
 </tr><tr><td class="blob-num">2</td><td class="blob-code"><ul class="category-program"></ul></td>
@@ -9,7 +9,7 @@
 </ul></td>
 
 </tr><tr><td class="blob-num blob-num-empty empty-cell"></td><td class="blob-code blob-code-empty empty-cell"></td>
-<td class="blob-num blob-num-replacement">3</td><td class="blob-code blob-code-replacement"><ul class="category-program"><li><div class="patch insert" data="9"><ul class="category-expression_statement"><li><ul class="category-method_call"><li><span class="category-identifier">console</span></li>.<li><span class="category-identifier">log</span></li>(<li><ul class="category-arguments"><li><ul class="category-string"><li><span class="category-string">&quot;</span></li><li><span class="category-string">insertion</span></li><li><span class="category-string">&quot;</span></li></ul></li></ul></li>)</ul></li>;</ul></div></li>
+<td class="blob-num blob-num-replacement">3</td><td class="blob-code blob-code-replacement"><ul class="category-program"><li><div class="patch insert" data="9"><ul class="category-expression_statements"><li><ul class="category-method_call"><li><span class="category-identifier">console</span></li>.<li><span class="category-identifier">log</span></li>(<li><ul class="category-arguments"><li><ul class="category-string"><li><span class="category-string">&quot;</span></li><li><span class="category-string">insertion</span></li><li><span class="category-string">&quot;</span></li></ul></li></ul></li>)</ul></li>;</ul></div></li>
 </ul></td>
 
 </tr><tr><td class="blob-num blob-num-empty empty-cell"></td><td class="blob-code blob-code-empty empty-cell"></td>

test/diffs/no-newline-at-eof.json.js

@@ -1 +1 @@
-{"rows":[[{"number":1,"terms":[{"range":[0,28],"category":"Program","children":[{"range":[0,28],"category":"expression_statement","children":[{"range":[0,27],"category":"MethodCall","children":[{"range":[0,7],"category":"Identifier"},{"range":[8,11],"category":"Identifier"},{"range":[12,26],"category":"Args","children":[{"range":[12,26],"category":"StringLiteral","children":[{"range":[12,13],"category":"StringLiteral"},{"range":[13,18],"category":"StringLiteral"},{"range":[18,19],"category":"StringLiteral"},{"range":[20,25],"category":"StringLiteral"},{"range":[25,26],"category":"StringLiteral"}]}]}]}]}]}],"range":[0,28],"hasChanges":false},{"number":1,"terms":[{"range":[0,29],"category":"Program","children":[{"range":[0,28],"category":"expression_statement","children":[{"range":[0,27],"category":"MethodCall","children":[{"range":[0,7],"category":"Identifier"},{"range":[8,11],"category":"Identifier"},{"range":[12,26],"category":"Args","children":[{"range":[12,26],"category":"StringLiteral","children":[{"range":[12,13],"category":"StringLiteral"},{"range":[13,18],"category":"StringLiteral"},{"range":[18,19],"category":"StringLiteral"},{"range":[20,25],"category":"StringLiteral"},{"range":[25,26],"category":"StringLiteral"}]}]}]}]}]}],"range":[0,29],"hasChanges":false}],[{"number":2,"terms":[{"range":[29,30],"category":"Program","children":[]}],"range":[29,30],"hasChanges":false}],[{"number":3,"terms":[{"range":[30,55],"category":"Program","children":[{"patch":"insert","range":[30,55],"category":"expression_statement","children":[{"range":[30,54],"category":"MethodCall","children":[{"range":[30,37],"category":"Identifier"},{"range":[38,41],"category":"Identifier"},{"range":[42,53],"category":"Args","children":[{"range":[42,53],"category":"StringLiteral","children":[{"range":[42,43],"category":"StringLiteral"},{"range":[43,52],"category":"StringLiteral"},{"range":[52,53],"category":"StringLiteral"}]}]}]}]}]}],"range":[30,55],"hasChanges":true}]],"oids":["",""],"paths":["test/diffs/no-newline-at-eof.A.js","test/diffs/no-newline-at-eof.B.js"]}
+{"rows":[[{"number":1,"terms":[{"range":[0,28],"category":"Program","children":[{"range":[0,28],"category":"ExpressionStatements","children":[{"range":[0,27],"category":"MethodCall","children":[{"range":[0,7],"category":"Identifier"},{"range":[8,11],"category":"Identifier"},{"range":[12,26],"category":"Args","children":[{"range":[12,26],"category":"StringLiteral","children":[{"range":[12,13],"category":"StringLiteral"},{"range":[13,18],"category":"StringLiteral"},{"range":[18,19],"category":"StringLiteral"},{"range":[20,25],"category":"StringLiteral"},{"range":[25,26],"category":"StringLiteral"}]}]}]}]}]}],"range":[0,28],"hasChanges":false},{"number":1,"terms":[{"range":[0,29],"category":"Program","children":[{"range":[0,28],"category":"ExpressionStatements","children":[{"range":[0,27],"category":"MethodCall","children":[{"range":[0,7],"category":"Identifier"},{"range":[8,11],"category":"Identifier"},{"range":[12,26],"category":"Args","children":[{"range":[12,26],"category":"StringLiteral","children":[{"range":[12,13],"category":"StringLiteral"},{"range":[13,18],"category":"StringLiteral"},{"range":[18,19],"category":"StringLiteral"},{"range":[20,25],"category":"StringLiteral"},{"range":[25,26],"category":"StringLiteral"}]}]}]}]}]}],"range":[0,29],"hasChanges":false}],[{"number":2,"terms":[{"range":[29,30],"category":"Program","children":[]}],"range":[29,30],"hasChanges":false}],[{"number":3,"terms":[{"range":[30,55],"category":"Program","children":[{"patch":"insert","range":[30,55],"category":"ExpressionStatements","children":[{"range":[30,54],"category":"MethodCall","children":[{"range":[30,37],"category":"Identifier"},{"range":[38,41],"category":"Identifier"},{"range":[42,53],"category":"Args","children":[{"range":[42,53],"category":"StringLiteral","children":[{"range":[42,43],"category":"StringLiteral"},{"range":[43,52],"category":"StringLiteral"},{"range":[52,53],"category":"StringLiteral"}]}]}]}]}]}],"range":[30,55],"hasChanges":true}]],"oids":["",""],"paths":["test/diffs/no-newline-at-eof.A.js","test/diffs/no-newline-at-eof.B.js"]}

test/diffs/no-newline-at-eof.split.js

@@ -1,6 +1,6 @@
 <!DOCTYPE HTML>
-<html><head><link rel="stylesheet" href="style.css"></head><body><table class="diff"><colgroup><col width="40"><col><col width="40"><col></colgroup><tr><td class="blob-num">1</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statement"><li><ul class="category-method_call"><li><span class="category-identifier">console</span></li>.<li><span class="category-identifier">log</span></li>(<li><ul class="category-arguments"><li><ul class="category-string"><li><span class="category-string">&quot;</span></li><li><span class="category-string">hello</span></li><li><span class="category-string">,</span></li> <li><span class="category-string">world</span></li><li><span class="category-string">&quot;</span></li></ul></li></ul></li>)</ul></li>;</ul></li></ul></td>
-<td class="blob-num">1</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statement"><li><ul class="category-method_call"><li><span class="category-identifier">console</span></li>.<li><span class="category-identifier">log</span></li>(<li><ul class="category-arguments"><li><ul class="category-string"><li><span class="category-string">&quot;</span></li><li><span class="category-string">hello</span></li><li><span class="category-string">,</span></li> <li><span class="category-string">world</span></li><li><span class="category-string">&quot;</span></li></ul></li></ul></li>)</ul></li>;</ul></li>
+<html><head><link rel="stylesheet" href="style.css"></head><body><table class="diff"><colgroup><col width="40"><col><col width="40"><col></colgroup><tr><td class="blob-num">1</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statements"><li><ul class="category-method_call"><li><span class="category-identifier">console</span></li>.<li><span class="category-identifier">log</span></li>(<li><ul class="category-arguments"><li><ul class="category-string"><li><span class="category-string">&quot;</span></li><li><span class="category-string">hello</span></li><li><span class="category-string">,</span></li> <li><span class="category-string">world</span></li><li><span class="category-string">&quot;</span></li></ul></li></ul></li>)</ul></li>;</ul></li></ul></td>
+<td class="blob-num">1</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statements"><li><ul class="category-method_call"><li><span class="category-identifier">console</span></li>.<li><span class="category-identifier">log</span></li>(<li><ul class="category-arguments"><li><ul class="category-string"><li><span class="category-string">&quot;</span></li><li><span class="category-string">hello</span></li><li><span class="category-string">,</span></li> <li><span class="category-string">world</span></li><li><span class="category-string">&quot;</span></li></ul></li></ul></li>)</ul></li>;</ul></li>
 </ul></td>
 
 </tr><tr><td class="blob-num blob-num-empty empty-cell"></td><td class="blob-code blob-code-empty empty-cell"></td>
@@ -8,6 +8,6 @@
 </ul></td>
 
 </tr><tr><td class="blob-num blob-num-empty empty-cell"></td><td class="blob-code blob-code-empty empty-cell"></td>
-<td class="blob-num blob-num-replacement">3</td><td class="blob-code blob-code-replacement"><ul class="category-program"><li><div class="patch insert" data="9"><ul class="category-expression_statement"><li><ul class="category-method_call"><li><span class="category-identifier">console</span></li>.<li><span class="category-identifier">log</span></li>(<li><ul class="category-arguments"><li><ul class="category-string"><li><span class="category-string">&quot;</span></li><li><span class="category-string">insertion</span></li><li><span class="category-string">&quot;</span></li></ul></li></ul></li>)</ul></li>;</ul></div></li></ul></td>
+<td class="blob-num blob-num-replacement">3</td><td class="blob-code blob-code-replacement"><ul class="category-program"><li><div class="patch insert" data="9"><ul class="category-expression_statements"><li><ul class="category-method_call"><li><span class="category-identifier">console</span></li>.<li><span class="category-identifier">log</span></li>(<li><ul class="category-arguments"><li><ul class="category-string"><li><span class="category-string">&quot;</span></li><li><span class="category-string">insertion</span></li><li><span class="category-string">&quot;</span></li></ul></li></ul></li>)</ul></li>;</ul></div></li></ul></td>
 
 </tr></table></body></html>

test/diffs/reformat.split.js

@@ -1,13 +1,13 @@
 <!DOCTYPE HTML>
 <html><head><link rel="stylesheet" href="style.css"></head><body><table class="diff"><colgroup><col width="40"><col><col width="40"><col></colgroup><tr><td class="blob-num blob-num-empty empty-cell"></td><td class="blob-code blob-code-empty empty-cell"></td>
-<td class="blob-num">1</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statement"><li><ul class="category-array">[
+<td class="blob-num">1</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statements"><li><ul class="category-array">[
 </ul></li></ul></li></ul></td>
 
-</tr><tr><td class="blob-num">1</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statement"><li><ul class="category-array">[ <li><span class="category-identifier">bar</span></li> ]</ul></li>;</ul></li></ul></td>
-<td class="blob-num">2</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statement"><li><ul class="category-array">  <li><span class="category-identifier">bar</span></li>
+</tr><tr><td class="blob-num">1</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statements"><li><ul class="category-array">[ <li><span class="category-identifier">bar</span></li> ]</ul></li>;</ul></li></ul></td>
+<td class="blob-num">2</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statements"><li><ul class="category-array">  <li><span class="category-identifier">bar</span></li>
 </ul></li></ul></li></ul></td>
 
 </tr><tr><td class="blob-num blob-num-empty empty-cell"></td><td class="blob-code blob-code-empty empty-cell"></td>
-<td class="blob-num">3</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statement"><li><ul class="category-array">]</ul></li>;</ul></li></ul></td>
+<td class="blob-num">3</td><td class="blob-code"><ul class="category-program"><li><ul class="category-expression_statements"><li><ul class="category-array">]</ul></li>;</ul></li></ul></td>
 
 </tr></table></body></html>

test/repos/js-test

@@ -1 +1 @@
-Subproject commit 8747ef2483104c72b468cbb2eede87956014c70e
+Subproject commit 3af143b53c8f9a2a6761d8e9ea91d47982cc3bc0

vendor/text-icu

@@ -1 +1 @@
-Subproject commit a660194e2358ffcc86a9fe11a67fc3223c316007
+Subproject commit 6d07c2b2034f2bfdcd038de0d6a3ceca445f0221

vendor/tree-sitter-parsers

@@ -1 +1 @@
-Subproject commit 36a5a0fc1ff4990f34842a27bc9c6cb0d181a50e
+Subproject commit b32d12c13da744d1b0edc2901988f5e29eae2e49

weekly/2016-07-26.md

@@ -0,0 +1,30 @@
+# July 26, 2016 weekly
+
+Last week was Summit.
+
+## What went well?
+
+@rewinfrey - Pairing w/ @joshvera. Made significant progress on diff summary property tests. And the PR got merged!
+
+@joshvera - Pairing. Summit!
+
+@robrix - Summit! @joshvera & @rewinfrey really came together on the diff summary stuff, too ❤️
+
+
+
+## What was challenging?
+
+@rewinfrey - How to add more test cases in a non-manual way.
+
+@joshvera - Figuring out how much the JS parser covers. If we improved our error handling to cover more cases that might get us partway to staff shipping.
+
+@robrix - **After the fact:** I don’t remember what I said, now, nor what was challenging that week.
+
+
+## What did you learn?
+
+@rewinfrey - All the things at Summit! Learned a lot about property testing while pairing with @joshvera.
+
+@joshvera - Learned about property tests. Been reading about probabilities through this machine learning book. Onto continuous probability.
+
+@robrix - Some stuff about derivative parsers, probably?

weekly/2016-08-02.md

@@ -0,0 +1,45 @@
+## What went well?
+
+@rewinfrey
+
+* Made a lot of progress on auto generation of test cases.
+
+@robrix
+
+* Mergeable PR went well. A pretty big step for maintenance costs.
+Self-assessment forms are really streamlined compared to last time.
+
+@joshvera
+
+* Adding remaining cases to Syntax is going pretty well.
+
+## What went less well?
+
+@rewinfrey
+
+* Initial confusion for being on platform support was confusing. By the end of the week we knocked out a production bug that was affecting customers.
+
+@robrix
+
+* Though the self-assessment training was more streamlined, there was a lot of training involved.
+
+@joshvera
+
+* Realized mapping C into Syntax will be more trouble than anticipated.
+
+## What did you learn?
+
+@rewinfrey
+
+* Learned a lot about Haskell’s shell interactions.
+* Specifically creating process and how Haskell abstracts from communicating with the shell.
+* Learned about Yesod to do web programming in Haskell.
+
+@robrix
+
+* Reinforced knowledge of generic programing and property tests for laws governing new type classes. In the case of Mergeable, it’s difficult to describe the powerset behavior Mergeable has, and property tests helped with that.
+
+@joshvera
+
+* Learned about probability distributions and recalled the `suchThat` is kind of like filter for Arbitrary types in quickcheck.
+