Merge branch 'master' into benchmark-alignment

src/Category.hs

@@ -5,9 +5,13 @@ import Data.String
 
 -- | A standardized category of AST node. Used to determine the semantics for
 -- | semantic diffing and define comparability of nodes.
-data Category =
+data Category
+  -- | The top-level branch node.
+  = Program
+  -- | A node indicating syntax errors.
+  | Error
   -- | An operator with 2 operands.
-  BinaryOperator
+  | BinaryOperator
   -- | A literal key-value data structure.
   | DictionaryLiteral
   -- | A pair, e.g. of a key & value

src/DiffSummary.hs

@@ -35,6 +35,8 @@ instance HasCategory Text where
 
 instance HasCategory Category where
   toCategoryName category = case category of
+    Program -> "top level"
+    Error -> "error"
     BinaryOperator -> "binary operator"
     DictionaryLiteral -> "dictionary"
     Pair -> "pair"
@@ -43,7 +45,7 @@ instance HasCategory Category where
     IntegerLiteral -> "integer"
     SymbolLiteral -> "symbol"
     ArrayLiteral -> "array"
-    (Other s) -> s
+    Other s -> s
 
 instance HasCategory leaf => HasCategory (Term leaf Info) where
   toCategoryName = toCategoryName . category . extract

src/Diffing.hs

@@ -90,7 +90,7 @@ diffFiles parser renderer sourceBlobs = do
   let textDiff = case areNullOids of
         (True, False) -> pure $ Insert (snd terms)
         (False, True) -> pure $ Delete (fst terms)
-        (_, _) -> runBothWith (diffTerms construct ((==) `on` category . extract) diffCostWithCachedTermSizes) $ replaceLeaves <*> terms
+        (_, _) -> runBothWith (diffTerms construct shouldCompareTerms diffCostWithCachedTermSizes) $ replaceLeaves <*> terms
 
   pure $! renderer textDiff sourceBlobs
   where construct :: CofreeF (Syntax Text) (Both Info) (Diff Text Info) -> Diff Text Info
@@ -100,6 +100,7 @@ diffFiles parser renderer sourceBlobs = do
         getCost diff = case runFree diff of
           Free (info :< _) -> cost <$> info
           Pure patch -> uncurry both (fromThese 0 0 (unPatch (cost . extract <$> patch)))
+        shouldCompareTerms = (==) `on` category . extract
 
 -- | The sum of the node count of the diff’s patches.
 diffCostWithCachedTermSizes :: Diff a Info -> Integer

src/Renderer/Split.hs

@@ -36,6 +36,8 @@ classifyMarkup category element = (element !) . A.class_ . stringValue $ styleNa
 -- | Return the appropriate style name for the given category.
 styleName :: Category -> String
 styleName category = "category-" ++ case category of
+  Program -> "program"
+  Error -> "error"
   BinaryOperator -> "binary-operator"
   DictionaryLiteral -> "dictionary"
   Pair -> "pair"

src/TreeSitter.hs

@@ -37,13 +37,15 @@ categoriesForLanguage language name = case (language, name) of
 -- | Given a node name from TreeSitter, return the correct categories.
 defaultCategoryForNodeName :: String -> Category
 defaultCategoryForNodeName name = case name of
+  "program" -> Program
+  "ERROR" -> Error
   "function_call" -> FunctionCall
   "pair" -> Pair
   "string" -> StringLiteral
   "integer" -> IntegerLiteral
   "symbol" -> SymbolLiteral
   "array" -> ArrayLiteral
-  _ -> (Other name)
+  _ -> Other name
 
 -- | Return a parser for a tree sitter language & document.
 documentToTerm :: Language -> Ptr Document -> Parser

test/diffs/dictionary.json.js

@@ -1 +1 @@
-{"rows":[[{"number":1,"terms":[{"range":[0,2],"category":"program","children":[{"range":[0,2],"category":"expression_statement","children":[{"range":[0,2],"category":"DictionaryLiteral","children":{}}]}]}],"range":[0,2],"hasChanges":false},{"number":1,"terms":[{"range":[0,2],"category":"program","children":[{"range":[0,2],"category":"expression_statement","children":[{"range":[0,2],"category":"DictionaryLiteral","children":{}}]}]}],"range":[0,2],"hasChanges":false}],[{"number":2,"terms":[{"range":[2,12],"category":"program","children":[{"range":[2,12],"category":"expression_statement","children":[{"range":[2,12],"category":"DictionaryLiteral","children":{"\"b\"":{"range":[4,10],"category":"Pair","children":[{"range":[4,7],"category":"StringLiteral","children":[{"range":[4,5],"category":"StringLiteral"},{"range":[5,6],"category":"StringLiteral"},{"range":[6,7],"category":"StringLiteral"}]},{"patch":"replace","range":[9,10],"category":"number"}]}}}]}]}],"range":[2,12],"hasChanges":true},{"number":2,"terms":[{"range":[2,12],"category":"program","children":[{"range":[2,12],"category":"expression_statement","children":[{"range":[2,12],"category":"DictionaryLiteral","children":{"\"b\"":{"range":[4,10],"category":"Pair","children":[{"range":[4,7],"category":"StringLiteral","children":[{"range":[4,5],"category":"StringLiteral"},{"range":[5,6],"category":"StringLiteral"},{"range":[6,7],"category":"StringLiteral"}]},{"patch":"replace","range":[9,10],"category":"number"}]}}}]}]}],"range":[2,12],"hasChanges":true}],[{"number":3,"terms":[{"range":[12,21],"category":"program","children":[{"range":[12,21],"category":"expression_statement","children":[{"range":[12,21],"category":"DictionaryLiteral","children":{"\"a\"":{"range":[14,20],"category":"Pair","children":[{"range":[14,17],"category":"StringLiteral","children":[{"range":[14,15],"category":"StringLiteral"},{"range":[15,16],"category":"StringLiteral"},{"range":[16,17],"category":"StringLiteral"}]},{"range":[19,20],"category":"number"}]}}}]}]}],"range":[12,21],"hasChanges":false},{"number":3,"terms":[{"range":[12,21],"category":"program","children":[{"range":[12,21],"category":"expression_statement","children":[{"range":[12,21],"category":"DictionaryLiteral","children":{"\"a\"":{"range":[14,20],"category":"Pair","children":[{"range":[14,17],"category":"StringLiteral","children":[{"range":[14,15],"category":"StringLiteral"},{"range":[15,16],"category":"StringLiteral"},{"range":[16,17],"category":"StringLiteral"}]},{"range":[19,20],"category":"number"}]}}}]}]}],"range":[12,21],"hasChanges":false}],[{"number":4,"terms":[{"range":[21,23],"category":"program","children":[{"range":[21,23],"category":"expression_statement","children":[{"range":[21,22],"category":"DictionaryLiteral","children":{}}]}]}],"range":[21,23],"hasChanges":false},{"number":4,"terms":[{"range":[21,23],"category":"program","children":[{"range":[21,23],"category":"expression_statement","children":[{"range":[21,22],"category":"DictionaryLiteral","children":{}}]}]}],"range":[21,23],"hasChanges":false}],[{"number":5,"terms":[{"range":[23,23],"category":"program","children":[]}],"range":[23,23],"hasChanges":false},{"number":5,"terms":[{"range":[23,23],"category":"program","children":[]}],"range":[23,23],"hasChanges":false}]],"oids":["",""],"paths":["test/diffs/dictionary.A.js","test/diffs/dictionary.B.js"]}
+{"rows":[[{"number":1,"terms":[{"range":[0,2],"category":"Program","children":[{"range":[0,2],"category":"expression_statement","children":[{"range":[0,2],"category":"DictionaryLiteral","children":{}}]}]}],"range":[0,2],"hasChanges":false},{"number":1,"terms":[{"range":[0,2],"category":"Program","children":[{"range":[0,2],"category":"expression_statement","children":[{"range":[0,2],"category":"DictionaryLiteral","children":{}}]}]}],"range":[0,2],"hasChanges":false}],[{"number":2,"terms":[{"range":[2,12],"category":"Program","children":[{"range":[2,12],"category":"expression_statement","children":[{"range":[2,12],"category":"DictionaryLiteral","children":{"\"b\"":{"range":[4,10],"category":"Pair","children":[{"range":[4,7],"category":"StringLiteral","children":[{"range":[4,5],"category":"StringLiteral"},{"range":[5,6],"category":"StringLiteral"},{"range":[6,7],"category":"StringLiteral"}]},{"patch":"replace","range":[9,10],"category":"number"}]}}}]}]}],"range":[2,12],"hasChanges":true},{"number":2,"terms":[{"range":[2,12],"category":"Program","children":[{"range":[2,12],"category":"expression_statement","children":[{"range":[2,12],"category":"DictionaryLiteral","children":{"\"b\"":{"range":[4,10],"category":"Pair","children":[{"range":[4,7],"category":"StringLiteral","children":[{"range":[4,5],"category":"StringLiteral"},{"range":[5,6],"category":"StringLiteral"},{"range":[6,7],"category":"StringLiteral"}]},{"patch":"replace","range":[9,10],"category":"number"}]}}}]}]}],"range":[2,12],"hasChanges":true}],[{"number":3,"terms":[{"range":[12,21],"category":"Program","children":[{"range":[12,21],"category":"expression_statement","children":[{"range":[12,21],"category":"DictionaryLiteral","children":{"\"a\"":{"range":[14,20],"category":"Pair","children":[{"range":[14,17],"category":"StringLiteral","children":[{"range":[14,15],"category":"StringLiteral"},{"range":[15,16],"category":"StringLiteral"},{"range":[16,17],"category":"StringLiteral"}]},{"range":[19,20],"category":"number"}]}}}]}]}],"range":[12,21],"hasChanges":false},{"number":3,"terms":[{"range":[12,21],"category":"Program","children":[{"range":[12,21],"category":"expression_statement","children":[{"range":[12,21],"category":"DictionaryLiteral","children":{"\"a\"":{"range":[14,20],"category":"Pair","children":[{"range":[14,17],"category":"StringLiteral","children":[{"range":[14,15],"category":"StringLiteral"},{"range":[15,16],"category":"StringLiteral"},{"range":[16,17],"category":"StringLiteral"}]},{"range":[19,20],"category":"number"}]}}}]}]}],"range":[12,21],"hasChanges":false}],[{"number":4,"terms":[{"range":[21,23],"category":"Program","children":[{"range":[21,23],"category":"expression_statement","children":[{"range":[21,22],"category":"DictionaryLiteral","children":{}}]}]}],"range":[21,23],"hasChanges":false},{"number":4,"terms":[{"range":[21,23],"category":"Program","children":[{"range":[21,23],"category":"expression_statement","children":[{"range":[21,22],"category":"DictionaryLiteral","children":{}}]}]}],"range":[21,23],"hasChanges":false}],[{"number":5,"terms":[{"range":[23,23],"category":"Program","children":[]}],"range":[23,23],"hasChanges":false},{"number":5,"terms":[{"range":[23,23],"category":"Program","children":[]}],"range":[23,23],"hasChanges":false}]],"oids":["",""],"paths":["test/diffs/dictionary.A.js","test/diffs/dictionary.B.js"]}

test/diffs/newline-at-eof.json.js

@@ -1 +1 @@
-{"rows":[[{"number":1,"terms":[{"range":[0,29],"category":"program","children":[{"range":[0,28],"category":"expression_statement","children":[{"range":[0,27],"category":"FunctionCall","children":[{"range":[0,11],"category":"member_access","children":[{"range":[0,7],"category":"identifier"},{"range":[8,11],"category":"identifier"}]},{"range":[12,26],"category":"arguments","children":[{"range":[12,26],"category":"StringLiteral","children":[{"range":[12,13],"category":"StringLiteral"},{"range":[13,18],"category":"StringLiteral"},{"range":[18,19],"category":"StringLiteral"},{"range":[20,25],"category":"StringLiteral"},{"range":[25,26],"category":"StringLiteral"}]}]}]}]}]}],"range":[0,29],"hasChanges":false},{"number":1,"terms":[{"range":[0,29],"category":"program","children":[{"range":[0,28],"category":"expression_statement","children":[{"range":[0,27],"category":"FunctionCall","children":[{"range":[0,11],"category":"member_access","children":[{"range":[0,7],"category":"identifier"},{"range":[8,11],"category":"identifier"}]},{"range":[12,26],"category":"arguments","children":[{"range":[12,26],"category":"StringLiteral","children":[{"range":[12,13],"category":"StringLiteral"},{"range":[13,18],"category":"StringLiteral"},{"range":[18,19],"category":"StringLiteral"},{"range":[20,25],"category":"StringLiteral"},{"range":[25,26],"category":"StringLiteral"}]}]}]}]}]}],"range":[0,29],"hasChanges":false}],[{"number":2,"terms":[{"range":[29,29],"category":"program","children":[]}],"range":[29,29],"hasChanges":false},{"number":2,"terms":[{"range":[29,30],"category":"program","children":[]}],"range":[29,30],"hasChanges":false}],[{"number":3,"terms":[{"range":[30,56],"category":"program","children":[{"patch":"insert","range":[30,55],"category":"expression_statement","children":[{"range":[30,54],"category":"FunctionCall","children":[{"range":[30,41],"category":"member_access","children":[{"range":[30,37],"category":"identifier"},{"range":[38,41],"category":"identifier"}]},{"range":[42,53],"category":"arguments","children":[{"range":[42,53],"category":"StringLiteral","children":[{"range":[42,43],"category":"StringLiteral"},{"range":[43,52],"category":"StringLiteral"},{"range":[52,53],"category":"StringLiteral"}]}]}]}]}]}],"range":[30,56],"hasChanges":true}],[{"number":4,"terms":[{"range":[56,56],"category":"program","children":[]}],"range":[56,56],"hasChanges":false}]],"oids":["",""],"paths":["test/diffs/newline-at-eof.A.js","test/diffs/newline-at-eof.B.js"]}
+{"rows":[[{"number":1,"terms":[{"range":[0,29],"category":"Program","children":[{"range":[0,28],"category":"expression_statement","children":[{"range":[0,27],"category":"FunctionCall","children":[{"range":[0,11],"category":"member_access","children":[{"range":[0,7],"category":"identifier"},{"range":[8,11],"category":"identifier"}]},{"range":[12,26],"category":"arguments","children":[{"range":[12,26],"category":"StringLiteral","children":[{"range":[12,13],"category":"StringLiteral"},{"range":[13,18],"category":"StringLiteral"},{"range":[18,19],"category":"StringLiteral"},{"range":[20,25],"category":"StringLiteral"},{"range":[25,26],"category":"StringLiteral"}]}]}]}]}]}],"range":[0,29],"hasChanges":false},{"number":1,"terms":[{"range":[0,29],"category":"Program","children":[{"range":[0,28],"category":"expression_statement","children":[{"range":[0,27],"category":"FunctionCall","children":[{"range":[0,11],"category":"member_access","children":[{"range":[0,7],"category":"identifier"},{"range":[8,11],"category":"identifier"}]},{"range":[12,26],"category":"arguments","children":[{"range":[12,26],"category":"StringLiteral","children":[{"range":[12,13],"category":"StringLiteral"},{"range":[13,18],"category":"StringLiteral"},{"range":[18,19],"category":"StringLiteral"},{"range":[20,25],"category":"StringLiteral"},{"range":[25,26],"category":"StringLiteral"}]}]}]}]}]}],"range":[0,29],"hasChanges":false}],[{"number":2,"terms":[{"range":[29,29],"category":"Program","children":[]}],"range":[29,29],"hasChanges":false},{"number":2,"terms":[{"range":[29,30],"category":"Program","children":[]}],"range":[29,30],"hasChanges":false}],[{"number":3,"terms":[{"range":[30,56],"category":"Program","children":[{"patch":"insert","range":[30,55],"category":"expression_statement","children":[{"range":[30,54],"category":"FunctionCall","children":[{"range":[30,41],"category":"member_access","children":[{"range":[30,37],"category":"identifier"},{"range":[38,41],"category":"identifier"}]},{"range":[42,53],"category":"arguments","children":[{"range":[42,53],"category":"StringLiteral","children":[{"range":[42,43],"category":"StringLiteral"},{"range":[43,52],"category":"StringLiteral"},{"range":[52,53],"category":"StringLiteral"}]}]}]}]}]}],"range":[30,56],"hasChanges":true}],[{"number":4,"terms":[{"range":[56,56],"category":"Program","children":[]}],"range":[56,56],"hasChanges":false}]],"oids":["",""],"paths":["test/diffs/newline-at-eof.A.js","test/diffs/newline-at-eof.B.js"]}

test/diffs/no-newline-at-eof.json.js

@@ -1 +1 @@
-{"rows":[[{"number":1,"terms":[{"range":[0,28],"category":"program","children":[{"range":[0,28],"category":"expression_statement","children":[{"range":[0,27],"category":"FunctionCall","children":[{"range":[0,11],"category":"member_access","children":[{"range":[0,7],"category":"identifier"},{"range":[8,11],"category":"identifier"}]},{"range":[12,26],"category":"arguments","children":[{"range":[12,26],"category":"StringLiteral","children":[{"range":[12,13],"category":"StringLiteral"},{"range":[13,18],"category":"StringLiteral"},{"range":[18,19],"category":"StringLiteral"},{"range":[20,25],"category":"StringLiteral"},{"range":[25,26],"category":"StringLiteral"}]}]}]}]}]}],"range":[0,28],"hasChanges":false},{"number":1,"terms":[{"range":[0,29],"category":"program","children":[{"range":[0,28],"category":"expression_statement","children":[{"range":[0,27],"category":"FunctionCall","children":[{"range":[0,11],"category":"member_access","children":[{"range":[0,7],"category":"identifier"},{"range":[8,11],"category":"identifier"}]},{"range":[12,26],"category":"arguments","children":[{"range":[12,26],"category":"StringLiteral","children":[{"range":[12,13],"category":"StringLiteral"},{"range":[13,18],"category":"StringLiteral"},{"range":[18,19],"category":"StringLiteral"},{"range":[20,25],"category":"StringLiteral"},{"range":[25,26],"category":"StringLiteral"}]}]}]}]}]}],"range":[0,29],"hasChanges":false}],[{"number":2,"terms":[{"range":[29,30],"category":"program","children":[]}],"range":[29,30],"hasChanges":false}],[{"number":3,"terms":[{"range":[30,55],"category":"program","children":[{"patch":"insert","range":[30,55],"category":"expression_statement","children":[{"range":[30,54],"category":"FunctionCall","children":[{"range":[30,41],"category":"member_access","children":[{"range":[30,37],"category":"identifier"},{"range":[38,41],"category":"identifier"}]},{"range":[42,53],"category":"arguments","children":[{"range":[42,53],"category":"StringLiteral","children":[{"range":[42,43],"category":"StringLiteral"},{"range":[43,52],"category":"StringLiteral"},{"range":[52,53],"category":"StringLiteral"}]}]}]}]}]}],"range":[30,55],"hasChanges":true}]],"oids":["",""],"paths":["test/diffs/no-newline-at-eof.A.js","test/diffs/no-newline-at-eof.B.js"]}
+{"rows":[[{"number":1,"terms":[{"range":[0,28],"category":"Program","children":[{"range":[0,28],"category":"expression_statement","children":[{"range":[0,27],"category":"FunctionCall","children":[{"range":[0,11],"category":"member_access","children":[{"range":[0,7],"category":"identifier"},{"range":[8,11],"category":"identifier"}]},{"range":[12,26],"category":"arguments","children":[{"range":[12,26],"category":"StringLiteral","children":[{"range":[12,13],"category":"StringLiteral"},{"range":[13,18],"category":"StringLiteral"},{"range":[18,19],"category":"StringLiteral"},{"range":[20,25],"category":"StringLiteral"},{"range":[25,26],"category":"StringLiteral"}]}]}]}]}]}],"range":[0,28],"hasChanges":false},{"number":1,"terms":[{"range":[0,29],"category":"Program","children":[{"range":[0,28],"category":"expression_statement","children":[{"range":[0,27],"category":"FunctionCall","children":[{"range":[0,11],"category":"member_access","children":[{"range":[0,7],"category":"identifier"},{"range":[8,11],"category":"identifier"}]},{"range":[12,26],"category":"arguments","children":[{"range":[12,26],"category":"StringLiteral","children":[{"range":[12,13],"category":"StringLiteral"},{"range":[13,18],"category":"StringLiteral"},{"range":[18,19],"category":"StringLiteral"},{"range":[20,25],"category":"StringLiteral"},{"range":[25,26],"category":"StringLiteral"}]}]}]}]}]}],"range":[0,29],"hasChanges":false}],[{"number":2,"terms":[{"range":[29,30],"category":"Program","children":[]}],"range":[29,30],"hasChanges":false}],[{"number":3,"terms":[{"range":[30,55],"category":"Program","children":[{"patch":"insert","range":[30,55],"category":"expression_statement","children":[{"range":[30,54],"category":"FunctionCall","children":[{"range":[30,41],"category":"member_access","children":[{"range":[30,37],"category":"identifier"},{"range":[38,41],"category":"identifier"}]},{"range":[42,53],"category":"arguments","children":[{"range":[42,53],"category":"StringLiteral","children":[{"range":[42,43],"category":"StringLiteral"},{"range":[43,52],"category":"StringLiteral"},{"range":[52,53],"category":"StringLiteral"}]}]}]}]}]}],"range":[30,55],"hasChanges":true}]],"oids":["",""],"paths":["test/diffs/no-newline-at-eof.A.js","test/diffs/no-newline-at-eof.B.js"]}

weekly/2016-06-06.md

@@ -0,0 +1,63 @@
+# May 31th, 2016
+
+## Agenda
+
+1. Retrospective on last week:
+	- What went well?
+	- What was challenging?
+	- What did you learn?
+
+
+## What went well?
+
+@robrix:
+
+- As of this morning we’re pushing data to graphite.
+- There's a 2 hour window of when data is pulled in to graphite.
+- We have a much more thorough understanding of the shape of the SES problems.
+
+@rewinfrey:
+- Came away with a much more thorough understanding of SES.
+- Pairing with Rob on the profiling issues.
+
+@joshvera
+- Exploring the current productions of tree-sitter output.
+- Got an airbnb for the Mini-Summit.
+
+
+## What was challenging?
+
+@robrix:
+- SES performance has been a problem. Briefly confused into thinking we had solved the problem. We’re profiling semantic-diff-tool and running the tests on semantic-diff. If we don’t clean semantic-diff-tool then it doesn’t know that semantic-diff has been rebuilt and it doesn’t try to relink it. Happens specifically when changing branches. 
+- SES performance depends on O(n) cost function.
+
+@rewinfrey:
+- Heartbreaking to discover that a huge performance win was a bad build.
+- Felt like it was hard to contribute to the deployment and build process.
+- Do we have a fallback in place in case S3 fails?
+
+@joshvera:
+- Error productions from tree-sitter are difficult to debug and obscure diff summary output.
+- Understanding and communicating how our deployment process works to other people. Maybe this means we need better documentation?
+
+
+## What did you learn?
+
+@robrix:
+- Learning about parallelism because we have large asymptotic factors in SES.
+- Developed a stronger intuition for why cost has to be linear with respect to the size of the diff tree.
+
+@rewinfrey:
+- Learned a lot about profiling in Haskell.
+- Learned how to use Profiteur to visualize the space and time costs for a given computation.
+- Trying to use the Eval monad to parallelize the Minimax algorithm.
+
+@joshvera:
+- Learned about designing CRISPR proteins that can be edited into bacteria to defend against viruses and plasmids.
+- Read up on GHC's Core language in order to understand some of the optimizations GHC performs.
+
+
+## Other Items
+
+- Mini-Summit plans set for the week of June 20th.
+- Rob on vacation starting Tuesday June 7th! :sunglasses: