Simplify on-demand placeholder line creation and add test coverage

spec/tokenized-buffer-spec.coffee

@@ -568,6 +568,42 @@ describe "TokenizedBuffer", ->
       expect(tokenizedBuffer.isFoldableAtRow(7)).toBe false
       expect(tokenizedBuffer.isFoldableAtRow(8)).toBe false
 
+  describe "::tokenizedLineForRow(row)", ->
+    it "returns the tokenized line for a row, or a placeholder line if it hasn't been tokenized yet", ->
+      buffer = atom.project.bufferForPathSync('sample.js')
+      grammar = atom.grammars.grammarForScopeName('source.js')
+      tokenizedBuffer = new TokenizedBuffer({buffer, grammar, tabLength: 2})
+      line0 = buffer.lineForRow(0)
+
+      jsScopeStartId = grammar.startIdForScope(grammar.scopeName)
+      jsScopeEndId = grammar.endIdForScope(grammar.scopeName)
+      startTokenizing(tokenizedBuffer)
+      expect(tokenizedBuffer.tokenizedLines[0]).toBeUndefined()
+      expect(tokenizedBuffer.tokenizedLineForRow(0).text).toBe(line0)
+      expect(tokenizedBuffer.tokenizedLineForRow(0).tags).toEqual([jsScopeStartId, line0.length, jsScopeEndId])
+      advanceClock(1)
+      expect(tokenizedBuffer.tokenizedLines[0]).not.toBeUndefined()
+      expect(tokenizedBuffer.tokenizedLineForRow(0).text).toBe(line0)
+      expect(tokenizedBuffer.tokenizedLineForRow(0).tags).not.toEqual([jsScopeStartId, line0.length, jsScopeEndId])
+
+      nullScopeStartId = NullGrammar.startIdForScope(NullGrammar.scopeName)
+      nullScopeEndId = NullGrammar.endIdForScope(NullGrammar.scopeName)
+      tokenizedBuffer.setGrammar(NullGrammar)
+      startTokenizing(tokenizedBuffer)
+      expect(tokenizedBuffer.tokenizedLines[0]).toBeUndefined()
+      expect(tokenizedBuffer.tokenizedLineForRow(0).text).toBe(line0)
+      expect(tokenizedBuffer.tokenizedLineForRow(0).tags).toEqual([nullScopeStartId, line0.length, nullScopeEndId])
+      advanceClock(1)
+      expect(tokenizedBuffer.tokenizedLineForRow(0).text).toBe(line0)
+      expect(tokenizedBuffer.tokenizedLineForRow(0).tags).toEqual([nullScopeStartId, line0.length, nullScopeEndId])
+
+    it "returns undefined if the requested row is outside the buffer range", ->
+      buffer = atom.project.bufferForPathSync('sample.js')
+      grammar = atom.grammars.grammarForScopeName('source.js')
+      tokenizedBuffer = new TokenizedBuffer({buffer, grammar, tabLength: 2})
+      fullyTokenize(tokenizedBuffer)
+      expect(tokenizedBuffer.tokenizedLineForRow(999)).toBeUndefined()
+
   describe "when the buffer is configured with the null grammar", ->
     it "does not actually tokenize using the grammar", ->
       spyOn(NullGrammar, 'tokenizeLine').andCallThrough()

src/language-mode.coffee

@@ -247,9 +247,6 @@ class LanguageMode
 
   suggestedIndentForLineAtBufferRow: (bufferRow, line, options) ->
     tokenizedLine = @editor.tokenizedBuffer.buildTokenizedLineForRowWithText(bufferRow, line)
-    iterator = tokenizedLine.getTokenIterator()
-    iterator.next()
-    scopeDescriptor = new ScopeDescriptor(scopes: iterator.getScopes())
     @suggestedIndentForTokenizedLineAtBufferRow(bufferRow, line, tokenizedLine, options)
 
   suggestedIndentForTokenizedLineAtBufferRow: (bufferRow, line, tokenizedLine, options) ->

src/tokenized-buffer.coffee

@@ -269,14 +269,8 @@ class TokenizedBuffer extends Model
       else
         text = @buffer.lineForRow(bufferRow)
         lineEnding = @buffer.lineEndingForRow(bufferRow)
-        tags = [
-          @grammar.startIdForScope(@grammar.scopeName),
-          text.length,
-          @grammar.endIdForScope(@grammar.scopeName)
-        ]
+        tags = [@grammar.startIdForScope(@grammar.scopeName), text.length, @grammar.endIdForScope(@grammar.scopeName)]
         @tokenizedLines[bufferRow] = new TokenizedLine({openScopes: [], text, tags, lineEnding, @tokenIterator})
-    else
-      null
 
   tokenizedLinesForRows: (startRow, endRow) ->
     for row in [startRow..endRow] by 1