Clear invalid rows when short-circuiting for null grammar

spec/tokenized-buffer-spec.coffee

@@ -878,3 +878,25 @@ describe "TokenizedBuffer", ->
       expect(tokenizedBuffer.tokenizedLineForRow(6).foldable).toBe true
       expect(tokenizedBuffer.tokenizedLineForRow(7).foldable).toBe false
       expect(tokenizedBuffer.tokenizedLineForRow(8).foldable).toBe false
+
+  describe "when the buffer is configured with the null grammar", ->
+    it "uses the placeholder tokens and does not actually tokenize using the grammar", ->
+      spyOn(atom.grammars.nullGrammar, 'tokenizeLine').andCallThrough()
+      buffer = atom.project.bufferForPathSync('sample.will-use-the-null-grammar')
+      buffer.setText('a\nb\nc')
+
+      tokenizedBuffer = new TokenizedBuffer({buffer})
+      tokenizeCallback = jasmine.createSpy('onDidTokenize')
+      tokenizedBuffer.onDidTokenize(tokenizeCallback)
+
+      fullyTokenize(tokenizedBuffer)
+
+      expect(tokenizeCallback.callCount).toBe 1
+      expect(atom.grammars.nullGrammar.tokenizeLine.callCount).toBe 0
+
+      expect(tokenizedBuffer.tokenizedLineForRow(0).tokens.length).toBe 1
+      expect(tokenizedBuffer.tokenizedLineForRow(0).tokens[0].value).toBe 'a'
+      expect(tokenizedBuffer.tokenizedLineForRow(1).tokens.length).toBe 1
+      expect(tokenizedBuffer.tokenizedLineForRow(1).tokens[0].value).toBe 'b'
+      expect(tokenizedBuffer.tokenizedLineForRow(2).tokens.length).toBe 1
+      expect(tokenizedBuffer.tokenizedLineForRow(2).tokens[0].value).toBe 'c'

src/tokenized-buffer.coffee

@@ -138,13 +138,19 @@ class TokenizedBuffer extends Model
 
   tokenizeInBackground: ->
     return if not @visible or @pendingChunk or not @isAlive()
-    return if @grammar is @grammar.registry.nullGrammar
+
     @pendingChunk = true
     _.defer =>
       @pendingChunk = false
       @tokenizeNextChunk() if @isAlive() and @buffer.isAlive()
 
   tokenizeNextChunk: ->
+    # Short circuit null grammar which can just use the placeholder tokens
+    if @grammar is atom.grammars.nullGrammar and @firstInvalidRow()?
+      @invalidRows = []
+      @markTokenizationComplete()
+      return
+
     rowsRemaining = @chunkSize
 
     while @firstInvalidRow()? and rowsRemaining > 0
@@ -178,10 +184,13 @@ class TokenizedBuffer extends Model
     if @firstInvalidRow()?
       @tokenizeInBackground()
     else
-      unless @fullyTokenized
-        @emit 'tokenized'
-        @emitter.emit 'did-tokenize'
-      @fullyTokenized = true
+      @markTokenizationComplete()
+
+  markTokenizationComplete: ->
+    unless @fullyTokenized
+      @emit 'tokenized'
+      @emitter.emit 'did-tokenize'
+    @fullyTokenized = true
 
   firstInvalidRow: ->
     @invalidRows[0]