Handle changes in the valid region of a partially tokenized buffer

spec/app/tokenized-buffer-spec.coffee

@@ -67,8 +67,39 @@ fdescribe "TokenizedBuffer", ->
       beforeEach ->
         # tokenize chunk 1 only
         advanceClock()
+        changeHandler.reset()
 
-      describe "when there is a buffer change inside a tokenized region", ->
+      describe "when there is a buffer change inside the tokenized region", ->
+        describe "when lines are added", ->
+          it "pushes the invalid rows down", ->
+            expect(tokenizedBuffer.firstInvalidRow()).toBe 5
+            buffer.insert([1, 0], '\n\n')
+            changeHandler.reset()
+
+            expect(tokenizedBuffer.firstInvalidRow()).toBe 7
+            advanceClock()
+            expect(changeHandler).toHaveBeenCalledWith(start: 7, end: 11, delta: 0)
+
+        describe "when lines are removed", ->
+          it "pulls the invalid rows up", ->
+            expect(tokenizedBuffer.firstInvalidRow()).toBe 5
+            buffer.delete([[1, 0], [3, 0]])
+            changeHandler.reset()
+
+            expect(tokenizedBuffer.firstInvalidRow()).toBe 3
+            advanceClock()
+            expect(changeHandler).toHaveBeenCalledWith(start: 3, end: 7, delta: 0)
+
+        describe "when the change invalidates all the lines before the current invalid region", ->
+          it "retokenizes the invalidated lines and continues into the valid region", ->
+            expect(tokenizedBuffer.firstInvalidRow()).toBe 5
+            buffer.insert([2, 0], '/*')
+            changeHandler.reset()
+            expect(tokenizedBuffer.firstInvalidRow()).toBe 3
+
+            advanceClock()
+            expect(changeHandler).toHaveBeenCalledWith(start: 3, end: 7, delta: 0)
+            expect(tokenizedBuffer.firstInvalidRow()).toBe 8
 
       describe "when there is a buffer change surrounding an invalid row", ->
 

src/app/tokenized-buffer.coffee

@@ -31,14 +31,13 @@ class TokenizedBuffer
     end = oldRange.end.row
     delta = newRange.end.row - oldRange.end.row
 
+    @updateInvalidRows(start, end, delta)
+
     previousStack = @stackForRow(end) # used in spill detection below
-
     stack = @stackForRow(start - 1)
-
     @screenLines[start..end] = @buildTokenizedScreenLinesForRows(start, end + delta, stack)
 
     unless _.isEqual(@stackForRow(end + delta), previousStack)
-      console.log "spill"
       @invalidateRow(end + delta + 1)
 
     @trigger "change", { start, end, delta, bufferChange: e }
@@ -72,6 +71,7 @@ class TokenizedBuffer
       loop
         previousStack = @stackForRow(row)
         @screenLines[row] = @buildTokenizedScreenLineForRow(row, @stackForRow(row - 1))
+        @validateRow(row)
         if --rowsRemaining == 0
           break
         if row == lastRow or _.isEqual(@stackForRow(row), previousStack)
@@ -192,6 +192,15 @@ class TokenizedBuffer
     @invalidRows.sort()
     @tokenizeInBackground()
 
+  validateRow: (row) ->
+    @invalidRows.shift() if @invalidRows[0] == row
+
+  updateInvalidRows: (start, end, delta) ->
+    updatedRows = []
+    for row in @invalidRows
+      updatedRows.push(row + delta)
+    @invalidRows = updatedRows
+
   logLines: (start=0, end=@buffer.getLastRow()) ->
     for row in [start..end]
       line = @lineForScreenRow(row).text