Parse until position equals line length including trailing newline

spec/app/tokenized-buffer-spec.coffee

@@ -352,3 +352,34 @@ describe "TokenizedBuffer", ->
 
       expect(tokens[0].value).toBe "#"
       expect(tokens[0].scopes).toEqual ["text.git-commit", "meta.scope.metadata.git-commit", "comment.line.number-sign.git-commit", "punctuation.definition.comment.git-commit"]
+
+  describe "when a C++ source file is tokenized", ->
+    beforeEach ->
+      editSession =  fixturesProject.buildEditSessionForPath('includes.cc', autoIndent: false)
+      buffer = editSession.buffer
+      tokenizedBuffer = editSession.displayBuffer.tokenizedBuffer
+      editSession.setVisible(true)
+      fullyTokenize(tokenizedBuffer)
+
+    afterEach ->
+      editSession.destroy()
+
+    it "correctly parses the first include line", ->
+      longLine = tokenizedBuffer.lineForScreenRow(0)
+      expect(longLine.text).toBe '#include "a.h"'
+      { tokens } = longLine
+
+      expect(tokens[0].value).toBe "#"
+      expect(tokens[0].scopes).toEqual ["source.c++", "meta.preprocessor.c.include"]
+      expect(tokens[1].value).toBe 'include'
+      expect(tokens[1].scopes).toEqual ["source.c++", "meta.preprocessor.c.include", "keyword.control.import.include.c"]
+
+    it "correctly parses the second include line", ->
+      commentLine = tokenizedBuffer.lineForScreenRow(1)
+      expect(commentLine.text).toBe '#include "b.h"'
+      { tokens } = commentLine
+
+      expect(tokens[0].value).toBe "#"
+      expect(tokens[0].scopes).toEqual ["source.c++", "meta.preprocessor.c.include"]
+      expect(tokens[1].value).toBe 'include'
+      expect(tokens[1].scopes).toEqual ["source.c++", "meta.preprocessor.c.include", "keyword.control.import.include.c"]

spec/fixtures/includes.cc

@@ -0,0 +1,2 @@
+#include "a.h"
+#include "b.h"

src/app/text-mate-grammar.coffee

@@ -41,7 +41,7 @@ class TextMateGrammar
         tokens = [new Token(value: "", scopes: scopes)]
         return { tokens, ruleStack }
 
-      break if position == line.length
+      break if position == line.length + 1 # include trailing newline position
 
       if match = _.last(ruleStack).getNextTokens(ruleStack, line, position, firstLine)
         { nextTokens, tokensStartPosition, tokensEndPosition } = match
@@ -55,10 +55,11 @@ class TextMateGrammar
         position = tokensEndPosition
 
       else # push filler token for unmatched text at end of line
-        tokens.push(new Token(
-          value: line[position...line.length]
-          scopes: scopes
-        ))
+        if position < line.length
+          tokens.push(new Token(
+            value: line[position...line.length]
+            scopes: scopes
+          ))
         break
 
     ruleStack.forEach (rule) -> rule.clearAnchorPosition()