From 559b9132f92b180db6322e4a1731f0e0865e739b Mon Sep 17 00:00:00 2001
From: Kevin Sawicki <kevin@github.com>
Date: Fri, 21 Dec 2012 09:52:45 -0800
Subject: [PATCH] Parse until position equals line length including trailing
 newline

---
 spec/app/tokenized-buffer-spec.coffee | 31 +++++++++++++++++++++++++++
 spec/fixtures/includes.cc             |  2 ++
 src/app/text-mate-grammar.coffee      | 11 +++++-----
 3 files changed, 39 insertions(+), 5 deletions(-)
 create mode 100644 spec/fixtures/includes.cc

diff --git a/spec/app/tokenized-buffer-spec.coffee b/spec/app/tokenized-buffer-spec.coffee
index 800243ce6..2eb7b74fb 100644
--- a/spec/app/tokenized-buffer-spec.coffee
+++ b/spec/app/tokenized-buffer-spec.coffee
@@ -352,3 +352,34 @@ describe "TokenizedBuffer", ->
 
       expect(tokens[0].value).toBe "#"
       expect(tokens[0].scopes).toEqual ["text.git-commit", "meta.scope.metadata.git-commit", "comment.line.number-sign.git-commit", "punctuation.definition.comment.git-commit"]
+
+  describe "when a C++ source file is tokenized", ->
+    beforeEach ->
+      editSession =  fixturesProject.buildEditSessionForPath('includes.cc', autoIndent: false)
+      buffer = editSession.buffer
+      tokenizedBuffer = editSession.displayBuffer.tokenizedBuffer
+      editSession.setVisible(true)
+      fullyTokenize(tokenizedBuffer)
+
+    afterEach ->
+      editSession.destroy()
+
+    it "correctly parses the first include line", ->
+      longLine = tokenizedBuffer.lineForScreenRow(0)
+      expect(longLine.text).toBe '#include "a.h"'
+      { tokens } = longLine
+
+      expect(tokens[0].value).toBe "#"
+      expect(tokens[0].scopes).toEqual ["source.c++", "meta.preprocessor.c.include"]
+      expect(tokens[1].value).toBe 'include'
+      expect(tokens[1].scopes).toEqual ["source.c++", "meta.preprocessor.c.include", "keyword.control.import.include.c"]
+
+    it "correctly parses the second include line", ->
+      commentLine = tokenizedBuffer.lineForScreenRow(1)
+      expect(commentLine.text).toBe '#include "b.h"'
+      { tokens } = commentLine
+
+      expect(tokens[0].value).toBe "#"
+      expect(tokens[0].scopes).toEqual ["source.c++", "meta.preprocessor.c.include"]
+      expect(tokens[1].value).toBe 'include'
+      expect(tokens[1].scopes).toEqual ["source.c++", "meta.preprocessor.c.include", "keyword.control.import.include.c"]
diff --git a/spec/fixtures/includes.cc b/spec/fixtures/includes.cc
new file mode 100644
index 000000000..600af314e
--- /dev/null
+++ b/spec/fixtures/includes.cc
@@ -0,0 +1,2 @@
+#include "a.h"
+#include "b.h"
diff --git a/src/app/text-mate-grammar.coffee b/src/app/text-mate-grammar.coffee
index d132d6738..dbba03344 100644
--- a/src/app/text-mate-grammar.coffee
+++ b/src/app/text-mate-grammar.coffee
@@ -41,7 +41,7 @@ class TextMateGrammar
         tokens = [new Token(value: "", scopes: scopes)]
         return { tokens, ruleStack }
 
-      break if position == line.length
+      break if position == line.length + 1 # include trailing newline position
 
       if match = _.last(ruleStack).getNextTokens(ruleStack, line, position, firstLine)
         { nextTokens, tokensStartPosition, tokensEndPosition } = match
@@ -55,10 +55,11 @@ class TextMateGrammar
         position = tokensEndPosition
 
       else # push filler token for unmatched text at end of line
-        tokens.push(new Token(
-          value: line[position...line.length]
-          scopes: scopes
-        ))
+        if position < line.length
+          tokens.push(new Token(
+            value: line[position...line.length]
+            scopes: scopes
+          ))
         break
 
     ruleStack.forEach (rule) -> rule.clearAnchorPosition()