Add support for TextMate grammars with newlines in their regexes

spec/app/editor-spec.coffee

@@ -998,7 +998,7 @@ describe "Editor", ->
         line0 = editor.renderedLines.find('.line:first')
         span0 = line0.children('span:eq(0)')
         expect(span0).toMatchSelector '.source.js'
-        expect(span0.children('span:eq(0)')).toMatchSelector '.storage.type.js'
+        expect(span0.children('span:eq(0)')).toMatchSelector '.storage.modifier.js'
         expect(span0.children('span:eq(0)').text()).toBe 'var'
 
         span0_1 = span0.children('span:eq(1)')
@@ -1023,9 +1023,9 @@ describe "Editor", ->
 
       describe "when lines are updated in the buffer", ->
         it "syntax highlights the updated lines", ->
-          expect(editor.renderedLines.find('.line:eq(0) > span:first > span:first')).toMatchSelector '.storage.type.js'
+          expect(editor.renderedLines.find('.line:eq(0) > span:first > span:first')).toMatchSelector '.storage.modifier.js'
           buffer.insert([0, 0], "q")
-          expect(editor.renderedLines.find('.line:eq(0) > span:first > span:first')).not.toMatchSelector '.storage.type.js'
+          expect(editor.renderedLines.find('.line:eq(0) > span:first > span:first')).not.toMatchSelector '.storage.modifier.js'
 
           # verify that re-highlighting can occur below the changed line
           buffer.insert([5,0], "/* */")

spec/app/text-mate-grammar-spec.coffee

@@ -216,3 +216,26 @@ describe "TextMateGrammar", ->
           expect(tokens[18]).toEqual value: '</', scopes: ["text.html.ruby","meta.tag.block.any.html","punctuation.definition.tag.begin.html"]
           expect(tokens[19]).toEqual value: 'div', scopes: ["text.html.ruby","meta.tag.block.any.html","entity.name.tag.block.any.html"]
           expect(tokens[20]).toEqual value: '>', scopes: ["text.html.ruby","meta.tag.block.any.html","punctuation.definition.tag.end.html"]
+
+    it "can parse a grammar with newline charachters in its regular expressions (regression)", ->
+      grammar = new TextMateGrammar
+        name: "test"
+        scopeName: "source.imaginaryLanguage"
+        repository: {}
+        patterns: [
+          {
+            name: "comment-body";
+            begin: "//";
+            end: "\\n";
+            beginCaptures:
+              "0": { name: "comment-start" }
+          }
+        ]
+
+      {tokens, stack} = grammar.getLineTokens("// a singleLineComment")
+      expect(stack.length).toBe 1
+      expect(stack[0].scopeName).toBe "source.imaginaryLanguage"
+
+      expect(tokens.length).toBe 2
+      expect(tokens[0].value).toBe "//"
+      expect(tokens[1].value).toBe " a singleLineComment"

spec/app/token-spec.coffee

@@ -13,4 +13,4 @@ describe "Token", ->
     token = _.last(screenLine.tokens)
 
   afterEach ->
-    editSession.destroy()
+    editSession.destroy()

spec/app/tokenized-buffer-spec.coffee

@@ -23,7 +23,7 @@ describe "TokenizedBuffer", ->
 
   describe "tokenization", ->
     it "tokenizes all the lines in the buffer on construction", ->
-      expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual(value: 'var', scopes: ['source.js', 'storage.type.js'])
+      expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual(value: 'var', scopes: ['source.js', 'storage.modifier.js'])
       expect(tokenizedBuffer.lineForScreenRow(11).tokens[1]).toEqual(value: 'return', scopes: ['source.js', 'keyword.control.js'])
 
     describe "when the buffer changes", ->
@@ -77,7 +77,7 @@ describe "TokenizedBuffer", ->
           buffer.change(range, "foo()")
 
           # previous line 0 remains
-          expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual(value: 'var', scopes: ['source.js', 'storage.type.js'])
+          expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual(value: 'var', scopes: ['source.js', 'storage.modifier.js'])
 
           # previous line 3 should be combined with input to form line 1
           expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: 'foo', scopes: ['source.js'])
@@ -113,7 +113,7 @@ describe "TokenizedBuffer", ->
           buffer.change(range, "foo()\nbar()\nbaz()\nquux()")
 
           # previous line 0 remains
-          expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual( value: 'var', scopes: ['source.js', 'storage.type.js'])
+          expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual( value: 'var', scopes: ['source.js', 'storage.modifier.js'])
 
           # 3 new lines inserted
           expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: 'foo', scopes: ['source.js'])

src/app/text-mate-grammar.coffee

@@ -93,9 +93,14 @@ class Rule
   getNextTokens: (stack, line, position) ->
     patterns = @getIncludedPatterns()
 
-    return null unless result = @getScanner().findNextMatch(line, position)
+    return null unless result = @getScanner().findNextMatch(line + "\n", position)
     { index, captureIndices } = result
 
+    lineLength = line.length
+    captureIndices = captureIndices.map (value, index) ->
+      value = lineLength if index % 3 != 0 and value > lineLength
+      value
+
     [firstCaptureIndex, firstCaptureStart, firstCaptureEnd] = captureIndices
     nextTokens = patterns[index].handleMatch(stack, line, captureIndices)
     { nextTokens, tokensStartPosition: firstCaptureStart, tokensEndPosition: firstCaptureEnd }