diff --git a/src/Diffing.hs b/src/Diffing.hs
index 42d8088f2..b589315c2 100644
--- a/src/Diffing.hs
+++ b/src/Diffing.hs
@@ -8,7 +8,6 @@ import Data.Functor.Both
import Data.Functor.Foldable
import Data.RandomWalkSimilarity
import Data.Record
-import qualified Data.Set as Set
import qualified Data.Text.IO as TextIO
import qualified Data.Text.ICU.Detect as Detect
import qualified Data.Text.ICU.Convert as Convert
@@ -21,7 +20,6 @@ import Language
import Language.Markdown
import Parser
import Patch
-import Range
import Renderer
import Renderer.JSON
import Renderer.Patch
@@ -89,25 +87,7 @@ lineByLineParser blob = pure . cofree . root $ case foldl' annotateLeaves ([], 0
-- | Return the parser that should be used for a given path.
parserForFilepath :: FilePath -> Parser (Syntax Text) (Record '[Cost, Range, Category])
-parserForFilepath path blob = decorateTerm termCostDecorator <$> do
- parsed <- parserForType (toS (takeExtension path)) blob
- pure $! breakDownLeavesByWord (source blob) parsed
-
--- | Replace every string leaf with leaves of the words in the string.
-breakDownLeavesByWord :: (HasField fields Category, HasField fields Range) => Source Char -> Term (Syntax Text) (Record fields) -> Term (Syntax Text) (Record fields)
-breakDownLeavesByWord source = cata replaceIn
- where
- replaceIn (info :< syntax) = cofree $ info :< syntax'
- where syntax' = case (ranges, syntax) of
- (_:_:_, Leaf _) | Set.notMember (category info) preserveSyntax -> Indexed (makeLeaf info <$> ranges)
- _ -> syntax
- ranges = rangesAndWordsInSource (characterRange info)
- rangesAndWordsInSource range = rangesAndWordsFrom (start range) (toString $ slice range source)
- makeLeaf info (range, substring) = cofree $ setCharacterRange info range :< Leaf (toS substring)
- -- Some Category constructors should retain their original structure, and not be sliced
- -- into words. This Set represents those Category constructors for which we want to
- -- preserve the original Syntax.
- preserveSyntax = Set.fromList [Regex, Category.Comment, Category.TemplateString]
+parserForFilepath path blob = decorateTerm termCostDecorator <$> parserForType (toS (takeExtension path)) blob
-- | Transcode a file to a unicode source.
transcode :: B1.ByteString -> IO (Source Char)
diff --git a/test/diffs/asymmetrical-context.split.js b/test/diffs/asymmetrical-context.split.js
index 10de89c0c..d389f8f30 100644
--- a/test/diffs/asymmetrical-context.split.js
+++ b/test/diffs/asymmetrical-context.split.js
@@ -1,7 +1,7 @@
-
1
console
.
log
(
'
hello
'
)
;
+
1
console
.
log
(
'hello'
)
;
-
1
console
.
log
(
'
hello
'
)
;
+
1
console
.
log
(
'hello'
)
;
@@ -20,9 +20,9 @@
5
-
2
console
.
log
(
'
world
'
)
;
+
2
console
.
log
(
'world'
)
;
-
6
console
.
log
(
'
world
'
)
;
+
6
console
.
log
(
'world'
)
;
3
diff --git a/test/diffs/dictionary.json.js b/test/diffs/dictionary.json.js
index 0aad658a8..63dbcb304 100644
--- a/test/diffs/dictionary.json.js
+++ b/test/diffs/dictionary.json.js
@@ -1 +1 @@
-{"oids":["0000000000000000000000000000000000000000","0000000000000000000000000000000000000000"],"paths":["test/diffs/dictionary.A.js","test/diffs/dictionary.B.js"],"rows":[[{"terms":[{"category":"Program","children":[{"category":"ExpressionStatements","children":[{"category":"Object","children":[],"range":[0,2]}],"range":[0,2]}],"range":[0,2]}],"hasChanges":false,"range":[0,2],"number":1},{"terms":[{"category":"Program","children":[{"category":"ExpressionStatements","children":[{"category":"Object","children":[],"range":[0,2]}],"range":[0,2]}],"range":[0,2]}],"hasChanges":false,"range":[0,2],"number":1}],[{"terms":[{"category":"Program","children":[{"category":"ExpressionStatements","children":[{"category":"Object","children":[{"category":"Pair","children":[{"category":"StringLiteral","children":[{"category":"StringLiteral","range":[4,5]},{"category":"StringLiteral","range":[5,6]},{"category":"StringLiteral","range":[6,7]}],"range":[4,7]},{"category":"number","patch":"replace","range":[9,10]}],"range":[4,10]}],"range":[2,12]}],"range":[2,12]}],"range":[2,12]}],"hasChanges":true,"range":[2,12],"number":2},{"terms":[{"category":"Program","children":[{"category":"ExpressionStatements","children":[{"category":"Object","children":[{"category":"Pair","children":[{"category":"StringLiteral","children":[{"category":"StringLiteral","range":[4,5]},{"category":"StringLiteral","range":[5,6]},{"category":"StringLiteral","range":[6,7]}],"range":[4,7]},{"category":"number","patch":"replace","range":[9,10]}],"range":[4,10]}],"range":[2,12]}],"range":[2,12]}],"range":[2,12]}],"hasChanges":true,"range":[2,12],"number":2}],[{"terms":[{"category":"Program","children":[{"category":"ExpressionStatements","children":[{"category":"Object","children":[{"category":"Pair","children":[{"category":"StringLiteral","children":[{"category":"StringLiteral","range":[14,15]},{"category":"StringLiteral","range":[15,16]},{"category":"StringLiteral","range":[16,17]}],"range":[14,17]},{"category":"number","range":[19,20]}],"range":[14,20]}],"range":[12,21]}],"range":[12,21]}],"range":[12,21]}],"hasChanges":false,"range":[12,21],"number":3},{"terms":[{"category":"Program","children":[{"category":"ExpressionStatements","children":[{"category":"Object","children":[{"category":"Pair","children":[{"category":"StringLiteral","children":[{"category":"StringLiteral","range":[14,15]},{"category":"StringLiteral","range":[15,16]},{"category":"StringLiteral","range":[16,17]}],"range":[14,17]},{"category":"number","range":[19,20]}],"range":[14,20]}],"range":[12,21]}],"range":[12,21]}],"range":[12,21]}],"hasChanges":false,"range":[12,21],"number":3}],[{"terms":[{"category":"Program","children":[{"category":"ExpressionStatements","children":[{"category":"Object","children":[],"range":[21,22]}],"range":[21,23]}],"range":[21,23]}],"hasChanges":false,"range":[21,23],"number":4},{"terms":[{"category":"Program","children":[{"category":"ExpressionStatements","children":[{"category":"Object","children":[],"range":[21,22]}],"range":[21,23]}],"range":[21,23]}],"hasChanges":false,"range":[21,23],"number":4}],[{"terms":[{"category":"Program","children":[],"range":[23,23]}],"hasChanges":false,"range":[23,23],"number":5},{"terms":[{"category":"Program","children":[],"range":[23,23]}],"hasChanges":false,"range":[23,23],"number":5}]]}
\ No newline at end of file
+{"oids":["0000000000000000000000000000000000000000","0000000000000000000000000000000000000000"],"paths":["test/diffs/dictionary.A.js","test/diffs/dictionary.B.js"],"rows":[[{"terms":[{"category":"Program","children":[{"category":"ExpressionStatements","children":[{"category":"Object","children":[],"range":[0,2]}],"range":[0,2]}],"range":[0,2]}],"hasChanges":false,"range":[0,2],"number":1},{"terms":[{"category":"Program","children":[{"category":"ExpressionStatements","children":[{"category":"Object","children":[],"range":[0,2]}],"range":[0,2]}],"range":[0,2]}],"hasChanges":false,"range":[0,2],"number":1}],[{"terms":[{"category":"Program","children":[{"category":"ExpressionStatements","children":[{"category":"Object","children":[{"category":"Pair","children":[{"category":"StringLiteral","range":[4,7]},{"category":"number","patch":"replace","range":[9,10]}],"range":[4,10]}],"range":[2,12]}],"range":[2,12]}],"range":[2,12]}],"hasChanges":true,"range":[2,12],"number":2},{"terms":[{"category":"Program","children":[{"category":"ExpressionStatements","children":[{"category":"Object","children":[{"category":"Pair","children":[{"category":"StringLiteral","range":[4,7]},{"category":"number","patch":"replace","range":[9,10]}],"range":[4,10]}],"range":[2,12]}],"range":[2,12]}],"range":[2,12]}],"hasChanges":true,"range":[2,12],"number":2}],[{"terms":[{"category":"Program","children":[{"category":"ExpressionStatements","children":[{"category":"Object","children":[{"category":"Pair","children":[{"category":"StringLiteral","range":[14,17]},{"category":"number","range":[19,20]}],"range":[14,20]}],"range":[12,21]}],"range":[12,21]}],"range":[12,21]}],"hasChanges":false,"range":[12,21],"number":3},{"terms":[{"category":"Program","children":[{"category":"ExpressionStatements","children":[{"category":"Object","children":[{"category":"Pair","children":[{"category":"StringLiteral","range":[14,17]},{"category":"number","range":[19,20]}],"range":[14,20]}],"range":[12,21]}],"range":[12,21]}],"range":[12,21]}],"hasChanges":false,"range":[12,21],"number":3}],[{"terms":[{"category":"Program","children":[{"category":"ExpressionStatements","children":[{"category":"Object","children":[],"range":[21,22]}],"range":[21,23]}],"range":[21,23]}],"hasChanges":false,"range":[21,23],"number":4},{"terms":[{"category":"Program","children":[{"category":"ExpressionStatements","children":[{"category":"Object","children":[],"range":[21,22]}],"range":[21,23]}],"range":[21,23]}],"hasChanges":false,"range":[21,23],"number":4}],[{"terms":[{"category":"Program","children":[],"range":[23,23]}],"hasChanges":false,"range":[23,23],"number":5},{"terms":[{"category":"Program","children":[],"range":[23,23]}],"hasChanges":false,"range":[23,23],"number":5}]]}
\ No newline at end of file
diff --git a/test/diffs/dictionary.split.js b/test/diffs/dictionary.split.js
index 74480d548..a2e6497e3 100644
--- a/test/diffs/dictionary.split.js
+++ b/test/diffs/dictionary.split.js
@@ -4,14 +4,14 @@
diff --git a/test/diffs/newline-at-eof.json.js b/test/diffs/newline-at-eof.json.js
index ca833cba4..8e1174f59 100644
--- a/test/diffs/newline-at-eof.json.js
+++ b/test/diffs/newline-at-eof.json.js
@@ -1 +1 @@
-{"oids":["0000000000000000000000000000000000000000","0000000000000000000000000000000000000000"],"paths":["test/diffs/newline-at-eof.A.js","test/diffs/newline-at-eof.B.js"],"rows":[[{"terms":[{"category":"Program","children":[{"category":"ExpressionStatements","children":[{"category":"MethodCall","children":[{"category":"Identifier","range":[0,7]},{"category":"Identifier","range":[8,11]},{"category":"StringLiteral","children":[{"category":"StringLiteral","range":[12,13]},{"category":"StringLiteral","range":[13,18]},{"category":"StringLiteral","range":[18,19]},{"category":"StringLiteral","range":[20,25]},{"category":"StringLiteral","range":[25,26]}],"range":[12,26]}],"range":[0,27]}],"range":[0,28]}],"range":[0,29]}],"hasChanges":false,"range":[0,29],"number":1},{"terms":[{"category":"Program","children":[{"category":"ExpressionStatements","children":[{"category":"MethodCall","children":[{"category":"Identifier","range":[0,7]},{"category":"Identifier","range":[8,11]},{"category":"StringLiteral","children":[{"category":"StringLiteral","range":[12,13]},{"category":"StringLiteral","range":[13,18]},{"category":"StringLiteral","range":[18,19]},{"category":"StringLiteral","range":[20,25]},{"category":"StringLiteral","range":[25,26]}],"range":[12,26]}],"range":[0,27]}],"range":[0,28]}],"range":[0,29]}],"hasChanges":false,"range":[0,29],"number":1}],[{"terms":[{"category":"Program","children":[],"range":[29,29]}],"hasChanges":false,"range":[29,29],"number":2},{"terms":[{"category":"Program","children":[],"range":[29,30]}],"hasChanges":false,"range":[29,30],"number":2}],[{"terms":[{"category":"Program","children":[{"category":"ExpressionStatements","children":[{"category":"MethodCall","children":[{"category":"Identifier","range":[30,37]},{"category":"Identifier","range":[38,41]},{"category":"StringLiteral","children":[{"category":"StringLiteral","range":[42,43]},{"category":"StringLiteral","range":[43,52]},{"category":"StringLiteral","range":[52,53]}],"range":[42,53]}],"range":[30,54]}],"patch":"insert","range":[30,55]}],"range":[30,56]}],"hasChanges":true,"range":[30,56],"number":3}],[{"terms":[{"category":"Program","children":[],"range":[56,56]}],"hasChanges":false,"range":[56,56],"number":4}]]}
\ No newline at end of file
+{"oids":["0000000000000000000000000000000000000000","0000000000000000000000000000000000000000"],"paths":["test/diffs/newline-at-eof.A.js","test/diffs/newline-at-eof.B.js"],"rows":[[{"terms":[{"category":"Program","children":[{"category":"ExpressionStatements","children":[{"category":"MethodCall","children":[{"category":"Identifier","range":[0,7]},{"category":"Identifier","range":[8,11]},{"category":"StringLiteral","range":[12,26]}],"range":[0,27]}],"range":[0,28]}],"range":[0,29]}],"hasChanges":false,"range":[0,29],"number":1},{"terms":[{"category":"Program","children":[{"category":"ExpressionStatements","children":[{"category":"MethodCall","children":[{"category":"Identifier","range":[0,7]},{"category":"Identifier","range":[8,11]},{"category":"StringLiteral","range":[12,26]}],"range":[0,27]}],"range":[0,28]}],"range":[0,29]}],"hasChanges":false,"range":[0,29],"number":1}],[{"terms":[{"category":"Program","children":[],"range":[29,29]}],"hasChanges":false,"range":[29,29],"number":2},{"terms":[{"category":"Program","children":[],"range":[29,30]}],"hasChanges":false,"range":[29,30],"number":2}],[{"terms":[{"category":"Program","children":[{"category":"ExpressionStatements","children":[{"category":"MethodCall","children":[{"category":"Identifier","range":[30,37]},{"category":"Identifier","range":[38,41]},{"category":"StringLiteral","range":[42,53]}],"range":[30,54]}],"patch":"insert","range":[30,55]}],"range":[30,56]}],"hasChanges":true,"range":[30,56],"number":3}],[{"terms":[{"category":"Program","children":[],"range":[56,56]}],"hasChanges":false,"range":[56,56],"number":4}]]}
\ No newline at end of file
diff --git a/test/diffs/newline-at-eof.split.js b/test/diffs/newline-at-eof.split.js
index e3f5295b4..432c466f7 100644
--- a/test/diffs/newline-at-eof.split.js
+++ b/test/diffs/newline-at-eof.split.js
@@ -1,7 +1,7 @@
-
1
console
.
log
(
"
hello
,
world
"
)
;
+
1
console
.
log
(
"hello, world"
)
;
-
1
console
.
log
(
"
hello
,
world
"
)
;
+
1
console
.
log
(
"hello, world"
)
;
2
@@ -9,7 +9,7 @@
-
3
console
.
log
(
"
insertion
"
)
;
+
3
console
.
log
(
"insertion"
)
;
diff --git a/test/diffs/no-newline-at-eof.json.js b/test/diffs/no-newline-at-eof.json.js
index 2fb8bd7b6..544b3796d 100644
--- a/test/diffs/no-newline-at-eof.json.js
+++ b/test/diffs/no-newline-at-eof.json.js
@@ -1 +1 @@
-{"oids":["0000000000000000000000000000000000000000","0000000000000000000000000000000000000000"],"paths":["test/diffs/no-newline-at-eof.A.js","test/diffs/no-newline-at-eof.B.js"],"rows":[[{"terms":[{"category":"Program","children":[{"category":"ExpressionStatements","children":[{"category":"MethodCall","children":[{"category":"Identifier","range":[0,7]},{"category":"Identifier","range":[8,11]},{"category":"StringLiteral","children":[{"category":"StringLiteral","range":[12,13]},{"category":"StringLiteral","range":[13,18]},{"category":"StringLiteral","range":[18,19]},{"category":"StringLiteral","range":[20,25]},{"category":"StringLiteral","range":[25,26]}],"range":[12,26]}],"range":[0,27]}],"range":[0,28]}],"range":[0,28]}],"hasChanges":false,"range":[0,28],"number":1},{"terms":[{"category":"Program","children":[{"category":"ExpressionStatements","children":[{"category":"MethodCall","children":[{"category":"Identifier","range":[0,7]},{"category":"Identifier","range":[8,11]},{"category":"StringLiteral","children":[{"category":"StringLiteral","range":[12,13]},{"category":"StringLiteral","range":[13,18]},{"category":"StringLiteral","range":[18,19]},{"category":"StringLiteral","range":[20,25]},{"category":"StringLiteral","range":[25,26]}],"range":[12,26]}],"range":[0,27]}],"range":[0,28]}],"range":[0,29]}],"hasChanges":false,"range":[0,29],"number":1}],[{"terms":[{"category":"Program","children":[],"range":[29,30]}],"hasChanges":false,"range":[29,30],"number":2}],[{"terms":[{"category":"Program","children":[{"category":"ExpressionStatements","children":[{"category":"MethodCall","children":[{"category":"Identifier","range":[30,37]},{"category":"Identifier","range":[38,41]},{"category":"StringLiteral","children":[{"category":"StringLiteral","range":[42,43]},{"category":"StringLiteral","range":[43,52]},{"category":"StringLiteral","range":[52,53]}],"range":[42,53]}],"range":[30,54]}],"patch":"insert","range":[30,55]}],"range":[30,55]}],"hasChanges":true,"range":[30,55],"number":3}]]}
\ No newline at end of file
+{"oids":["0000000000000000000000000000000000000000","0000000000000000000000000000000000000000"],"paths":["test/diffs/no-newline-at-eof.A.js","test/diffs/no-newline-at-eof.B.js"],"rows":[[{"terms":[{"category":"Program","children":[{"category":"ExpressionStatements","children":[{"category":"MethodCall","children":[{"category":"Identifier","range":[0,7]},{"category":"Identifier","range":[8,11]},{"category":"StringLiteral","range":[12,26]}],"range":[0,27]}],"range":[0,28]}],"range":[0,28]}],"hasChanges":false,"range":[0,28],"number":1},{"terms":[{"category":"Program","children":[{"category":"ExpressionStatements","children":[{"category":"MethodCall","children":[{"category":"Identifier","range":[0,7]},{"category":"Identifier","range":[8,11]},{"category":"StringLiteral","range":[12,26]}],"range":[0,27]}],"range":[0,28]}],"range":[0,29]}],"hasChanges":false,"range":[0,29],"number":1}],[{"terms":[{"category":"Program","children":[],"range":[29,30]}],"hasChanges":false,"range":[29,30],"number":2}],[{"terms":[{"category":"Program","children":[{"category":"ExpressionStatements","children":[{"category":"MethodCall","children":[{"category":"Identifier","range":[30,37]},{"category":"Identifier","range":[38,41]},{"category":"StringLiteral","range":[42,53]}],"range":[30,54]}],"patch":"insert","range":[30,55]}],"range":[30,55]}],"hasChanges":true,"range":[30,55],"number":3}]]}
\ No newline at end of file
diff --git a/test/diffs/no-newline-at-eof.split.js b/test/diffs/no-newline-at-eof.split.js
index c6d2dd59b..f877070d3 100644
--- a/test/diffs/no-newline-at-eof.split.js
+++ b/test/diffs/no-newline-at-eof.split.js
@@ -1,6 +1,6 @@
-