Parser handles patterns w/ begin/end regexes (if begin/end are on the same line)

spec/app/parser-spec.coffee

@@ -3,7 +3,7 @@ plist = require 'plist'
 fs = require 'fs'
 _ = require 'underscore'
 
-describe "Parser", ->
+fdescribe "Parser", ->
   parser = null
 
   beforeEach ->
@@ -44,3 +44,13 @@ describe "Parser", ->
           expect(tokens[4]).toEqual value: ' ', scopes: ['source.coffee', 'meta.class.instance.constructor']
           expect(tokens[5]).toEqual value: 'foo.bar.Baz', scopes: ['source.coffee', 'meta.class.instance.constructor', 'entity.name.type.instance.coffee']
           expect(tokens[6]).toEqual value: ' ', scopes: ['source.coffee']
+
+      describe "when the line matches a begin and end pattern", ->
+        it "returns tokens based on the beginCaptures, endCaptures and the child scope", ->
+          {tokens, state} = parser.getLineTokens("'''single-quoted heredoc'''")
+
+          expect(tokens.length).toBe 3
+
+          expect(tokens[0]).toEqual value: "'''", scopes: ['source.coffee', 'string.quoted.heredoc.coffee', 'punctuation.definition.string.begin.coffee']
+          expect(tokens[1]).toEqual value: "single-quoted heredoc", scopes: ['source.coffee', 'string.quoted.heredoc.coffee']
+          expect(tokens[2]).toEqual value: "'''", scopes: ['source.coffee', 'string.quoted.heredoc.coffee', 'punctuation.definition.string.end.coffee']

src/app/parser.coffee

@@ -5,48 +5,59 @@ class Parser
   constructor: (@grammar) ->
 
   getLineTokens: (line, stateStack=@initialStateStack()) ->
-    tokens = []
-    currentScopes = _.pluck(stateStack, 'scopeName')
-    state = _.last(stateStack)
+    lineTokens = []
 
     startPosition = 0
     loop
-      { match, pattern } = @findNextMatch(line, state.patterns, startPosition)
+      { match, pattern } = @findNextMatch(line, _.last(stateStack).patterns, startPosition)
+      currentScopes = _.pluck(stateStack, 'scopeName')
 
       if not match or match.index > startPosition
         nextPosition = match?.index ? line.length
         if nextPosition > startPosition
-          tokens.push
+          lineTokens.push
             value: line[startPosition...nextPosition]
             scopes: new Array(currentScopes...)
           startPosition = nextPosition
 
       break unless match
 
-      tokens.push(@tokensForMatch(match, pattern, startPosition, currentScopes)...)
+      { tokens, stateStack } = @tokensForMatch(match, pattern, startPosition, currentScopes, stateStack)
+
+      lineTokens.push(tokens...)
       startPosition += match[0].length
 
-    { state, tokens }
+    { state: stateStack, tokens: lineTokens }
 
   findNextMatch: (line, patterns, startPosition) ->
     firstMatch = null
     matchedPattern = null
     for pattern in patterns
-      if match = pattern.match.search(line, startPosition)
+      continue unless regex = pattern.begin or pattern.match
+      if match = regex.search(line, startPosition)
         if !firstMatch or match.index < firstMatch.index
           firstMatch = match
           matchedPattern = pattern
 
     { match: firstMatch, pattern: matchedPattern }
 
-  tokensForMatch: (match, pattern, matchStartPosition, scopes) ->
+  tokensForMatch: (match, pattern, matchStartPosition, scopes, stateStack) ->
     tokens = []
-    scopes = scopes.concat(pattern.name)
-    if captures = pattern.captures
+    scopes = scopes.concat(pattern.name) if pattern.name
+
+    captures = pattern.captures
+    if pattern.begin
+      captures ?= pattern.beginCaptures
+      stateStack = stateStack.concat(ParserState.forPattern(pattern))
+    else if pattern.popStateStack
+      stateStack = stateStack[0...-1]
+
+    if captures
       tokens.push(@tokensForMatchWithCaptures(match, captures, matchStartPosition, scopes)...)
     else
       tokens.push(value: match[0], scopes: scopes)
-    tokens
+
+    { tokens, stateStack }
 
   tokensForMatchWithCaptures: (match, captures, matchStartPosition, scopes) ->
     tokens = []
@@ -75,6 +86,20 @@ class ParserState
   scopeName: null
   patterns: null
 
+  @forPattern: (pattern) ->
+    endPattern =
+      popStateStack: true
+      match: pattern.endSource
+      captures: pattern.endCaptures
+    new ParserState(scopeName: pattern.name, patterns: [endPattern])
+
   constructor: ({@scopeName, @patterns}) ->
     for pattern in @patterns
-      pattern.match = new OnigRegExp(pattern.match)
+      if pattern.match
+        pattern.matchSource = pattern.match
+        pattern.match = new OnigRegExp(pattern.match)
+      else if pattern.begin
+        pattern.beginSource = pattern.begin
+        pattern.begin = new OnigRegExp(pattern.begin)
+        pattern.endSource = pattern.end
+        pattern.end = new OnigRegExp(pattern.end)