From dc60017c024cbfbf5fa586992ea5ab925d0ac732 Mon Sep 17 00:00:00 2001
From: Corey Johnson & Nathan Sobo <cj+nathan@github.com>
Date: Tue, 11 Dec 2012 16:51:05 -0800
Subject: [PATCH] Async-tokenize remaining buffer changes that exceed chunk
 size

---
 spec/app/tokenized-buffer-spec.coffee | 12 ++++++++++++
 src/app/tokenized-buffer.coffee       | 17 ++++++++++++++---
 2 files changed, 26 insertions(+), 3 deletions(-)

diff --git a/spec/app/tokenized-buffer-spec.coffee b/spec/app/tokenized-buffer-spec.coffee
index a874e43f0..ce5e37b76 100644
--- a/spec/app/tokenized-buffer-spec.coffee
+++ b/spec/app/tokenized-buffer-spec.coffee
@@ -262,6 +262,18 @@ describe "TokenizedBuffer", ->
             delete event.bufferChange
             expect(event).toEqual(start: 5, end: 7, delta: 0)
 
+      describe "when there is an insertion that is larger than the chunk size", ->
+        it "tokenizes the initial chunk synchronously, then tokenizes the remaining lines in the background", ->
+          commentBlock = _.multiplyString("// a comment\n", tokenizedBuffer.chunkSize + 2)
+          buffer.insert([0,0], commentBlock)
+          expect(tokenizedBuffer.lineForScreenRow(0).ruleStack?).toBeTruthy()
+          expect(tokenizedBuffer.lineForScreenRow(4).ruleStack?).toBeTruthy()
+          expect(tokenizedBuffer.lineForScreenRow(5).ruleStack?).toBeFalsy()
+
+          advanceClock()
+          expect(tokenizedBuffer.lineForScreenRow(5).ruleStack?).toBeTruthy()
+          expect(tokenizedBuffer.lineForScreenRow(6).ruleStack?).toBeTruthy()
+
       describe ".findOpeningBracket(closingBufferPosition)", ->
         it "returns the position of the matching bracket, skipping any nested brackets", ->
           expect(tokenizedBuffer.findOpeningBracket([9, 2])).toEqual [1, 29]
diff --git a/src/app/tokenized-buffer.coffee b/src/app/tokenized-buffer.coffee
index d1c0cb742..fa144540f 100644
--- a/src/app/tokenized-buffer.coffee
+++ b/src/app/tokenized-buffer.coffee
@@ -107,7 +107,7 @@ class TokenizedBuffer
     else
       @screenLines[start..end] = @buildPlaceholderScreenLinesForRows(start, end + delta, stack)
 
-    unless _.isEqual(@stackForRow(end + delta), previousStack)
+    if @stackForRow(end + delta) and not _.isEqual(@stackForRow(end + delta), previousStack)
       @invalidateRow(end + delta + 1)
 
     @trigger "change", { start, end, delta, bufferChange: e }
@@ -122,11 +122,22 @@ class TokenizedBuffer
 
   buildTokenizedScreenLinesForRows: (startRow, endRow, startingStack) ->
     ruleStack = startingStack
-    for row in [startRow..endRow]
-      screenLine = @buildTokenizedScreenLineForRow(row, ruleStack)
+    lastRowToTokenize = startRow + @chunkSize - 1
+    screenLines = for row in [startRow..endRow]
+      if row <= lastRowToTokenize
+        screenLine = @buildTokenizedScreenLineForRow(row, ruleStack)
+      else
+        screenLine = @buildPlaceholderScreenLineForRow(row)
+
       ruleStack = screenLine.ruleStack
       screenLine
 
+    if endRow > lastRowToTokenize
+      @invalidateRow(lastRowToTokenize + 1)
+      @tokenizeInBackground()
+
+    screenLines
+
   buildTokenizedScreenLineForRow: (row, ruleStack) ->
     line = @buffer.lineForRow(row)
     { tokens, ruleStack } = @languageMode.tokenizeLine(line, ruleStack)