Tokenized buffer uses TextMate grammar

spec/app/tokenized-buffer-spec.coffee

@@ -13,14 +13,14 @@ describe "TokenizedBuffer", ->
   afterEach ->
     editSession.destroy()
 
-  describe ".findClosingBracket(startBufferPosition)", ->
-    it "returns the position of the matching bracket, skipping any nested brackets", ->
-      expect(tokenizedBuffer.findClosingBracket([1, 29])).toEqual [9, 2]
-
   describe ".findOpeningBracket(closingBufferPosition)", ->
     it "returns the position of the matching bracket, skipping any nested brackets", ->
       expect(tokenizedBuffer.findOpeningBracket([9, 2])).toEqual [1, 29]
 
+  describe ".findClosingBracket(startBufferPosition)", ->
+    it "returns the position of the matching bracket, skipping any nested brackets", ->
+      expect(tokenizedBuffer.findClosingBracket([1, 29])).toEqual [9, 2]
+
   describe "tokenization", ->
     it "tokenizes all the lines in the buffer on construction", ->
       expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual(value: 'var', scopes: ['source.js', 'storage.type.js'])
@@ -34,30 +34,30 @@ describe "TokenizedBuffer", ->
         tokenizedBuffer.on "change", changeHandler
 
       describe "when lines are updated, but none are added or removed", ->
-        fit "updates tokens for each of the changed lines", ->
+        it "updates tokens for each of the changed lines", ->
           range = new Range([0, 0], [2, 0])
           buffer.change(range, "foo()\n7\n")
 
           expect(tokenizedBuffer.lineForScreenRow(0).tokens[1]).toEqual(value: '(', scopes: ['source.js', 'meta.brace.round.js'])
           expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: '7', scopes: ['source.js', 'constant.numeric.js'])
 
-          # line 2 is unchanged
-          expect(tokenizedBuffer.lineForScreenRow(2).tokens[1]).toEqual(value: 'if', scopes: ['source.js'])
-
           expect(changeHandler).toHaveBeenCalled()
           [event] = changeHandler.argsForCall[0]
 
           expect(event.oldRange).toEqual range
           expect(event.newRange).toEqual new Range([0, 0], [2,0])
 
+          # line 2 is unchanged
+          expect(tokenizedBuffer.lineForScreenRow(2).tokens[1]).toEqual(value: 'if', scopes: ['source.js', 'keyword.control.js'])
+
         it "updates tokens for lines beyond the changed lines if needed", ->
           buffer.insert([5, 30], '/* */')
           changeHandler.reset()
 
           buffer.insert([2, 0], '/*')
-          expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].type).toBe 'comment'
+          expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
-          expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].type).toBe 'comment'
+          expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
-          expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].type).toBe 'comment'
+          expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
 
           expect(changeHandler).toHaveBeenCalled()
           [event] = changeHandler.argsForCall[0]
@@ -69,7 +69,7 @@ describe "TokenizedBuffer", ->
           buffer.insert([5, 0], '*/')
 
           buffer.insert([1, 0], 'var ')
-          expect(tokenizedBuffer.lineForScreenRow(1).tokens[0].type).toBe 'comment'
+          expect(tokenizedBuffer.lineForScreenRow(1).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
 
       describe "when lines are both updated and removed", ->
         it "updates tokens to reflect the removed lines", ->
@@ -77,16 +77,16 @@ describe "TokenizedBuffer", ->
           buffer.change(range, "foo()")
 
           # previous line 0 remains
-          expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual(type: 'keyword.definition', value: 'var')
+          expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual(value: 'var', scopes: ['source.js', 'storage.type.js'])
 
           # previous line 3 should be combined with input to form line 1
-          expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(type: 'identifier', value: 'foo')
+          expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: 'foo', scopes: ['source.js'])
-          expect(tokenizedBuffer.lineForScreenRow(1).tokens[6]).toEqual(type: 'identifier', value: 'pivot')
+          expect(tokenizedBuffer.lineForScreenRow(1).tokens[6]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js'])
 
           # lines below deleted regions should be shifted upward
-          expect(tokenizedBuffer.lineForScreenRow(2).tokens[1]).toEqual(type: 'keyword', value: 'while')
+          expect(tokenizedBuffer.lineForScreenRow(2).tokens[1]).toEqual(value: 'while', scopes: ['source.js', 'keyword.control.js'])
-          expect(tokenizedBuffer.lineForScreenRow(3).tokens[1]).toEqual(type: 'identifier', value: 'current')
+          expect(tokenizedBuffer.lineForScreenRow(3).tokens[1]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js'])
-          expect(tokenizedBuffer.lineForScreenRow(4).tokens[3]).toEqual(type: 'keyword.operator', value: '<')
+          expect(tokenizedBuffer.lineForScreenRow(4).tokens[1]).toEqual(value: '<', scopes: ['source.js', 'keyword.operator.js'])
 
           expect(changeHandler).toHaveBeenCalled()
           [event] = changeHandler.argsForCall[0]
@@ -98,9 +98,9 @@ describe "TokenizedBuffer", ->
           changeHandler.reset()
 
           buffer.change(new Range([2, 0], [3, 0]), '/*')
-          expect(tokenizedBuffer.lineForScreenRow(2).tokens[0].type).toBe 'comment'
+          expect(tokenizedBuffer.lineForScreenRow(2).tokens[0].scopes).toEqual ['source.js', 'comment.block.js', 'punctuation.definition.comment.js']
-          expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].type).toBe 'comment'
+          expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
-          expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].type).toBe 'comment'
+          expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
 
           expect(changeHandler).toHaveBeenCalled()
           [event] = changeHandler.argsForCall[0]
@@ -113,19 +113,19 @@ describe "TokenizedBuffer", ->
           buffer.change(range, "foo()\nbar()\nbaz()\nquux()")
 
           # previous line 0 remains
-          expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual(type: 'keyword.definition', value: 'var')
+          expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual( value: 'var', scopes: ['source.js', 'storage.type.js'])
 
           # 3 new lines inserted
-          expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(type: 'identifier', value: 'foo')
+          expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: 'foo', scopes: ['source.js'])
-          expect(tokenizedBuffer.lineForScreenRow(2).tokens[0]).toEqual(type: 'identifier', value: 'bar')
+          expect(tokenizedBuffer.lineForScreenRow(2).tokens[0]).toEqual(value: 'bar', scopes: ['source.js'])
-          expect(tokenizedBuffer.lineForScreenRow(3).tokens[0]).toEqual(type: 'identifier', value: 'baz')
+          expect(tokenizedBuffer.lineForScreenRow(3).tokens[0]).toEqual(value: 'baz', scopes: ['source.js'])
 
           # previous line 2 is joined with quux() on line 4
-          expect(tokenizedBuffer.lineForScreenRow(4).tokens[0]).toEqual(type: 'identifier', value: 'quux')
+          expect(tokenizedBuffer.lineForScreenRow(4).tokens[0]).toEqual(value: 'quux', scopes: ['source.js'])
-          expect(tokenizedBuffer.lineForScreenRow(4).tokens[4]).toEqual(type: 'keyword', value: 'if')
+          expect(tokenizedBuffer.lineForScreenRow(4).tokens[4]).toEqual(value: 'if', scopes: ['source.js', 'keyword.control.js'])
 
           # previous line 3 is pushed down to become line 5
-          expect(tokenizedBuffer.lineForScreenRow(5).tokens[3]).toEqual(type: 'identifier', value: 'pivot')
+          expect(tokenizedBuffer.lineForScreenRow(5).tokens[3]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js'])
 
           expect(changeHandler).toHaveBeenCalled()
           [event] = changeHandler.argsForCall[0]
@@ -137,13 +137,13 @@ describe "TokenizedBuffer", ->
           changeHandler.reset()
 
           buffer.insert([2, 0], '/*\nabcde\nabcder')
-          expect(tokenizedBuffer.lineForScreenRow(2).tokens[0].type).toBe 'comment'
+          expect(tokenizedBuffer.lineForScreenRow(2).tokens[0].scopes).toEqual ['source.js', 'comment.block.js', 'punctuation.definition.comment.js']
-          expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].type).toBe 'comment'
+          expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
-          expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].type).toBe 'comment'
+          expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
-          expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].type).toBe 'comment'
+          expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
-          expect(tokenizedBuffer.lineForScreenRow(6).tokens[0].type).toBe 'comment'
+          expect(tokenizedBuffer.lineForScreenRow(6).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
-          expect(tokenizedBuffer.lineForScreenRow(7).tokens[0].type).toBe 'comment'
+          expect(tokenizedBuffer.lineForScreenRow(7).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
-          expect(tokenizedBuffer.lineForScreenRow(8).tokens[0].type).not.toBe 'comment'
+          expect(tokenizedBuffer.lineForScreenRow(8).tokens[0].scopes).not.toBe ['source.js', 'comment.block.js']
 
           expect(changeHandler).toHaveBeenCalled()
           [event] = changeHandler.argsForCall[0]
@@ -165,10 +165,11 @@ describe "TokenizedBuffer", ->
         screenLine0 = tokenizedBuffer.lineForScreenRow(0)
         expect(screenLine0.text).toBe "# Econ 101#{tabText}"
         { tokens } = screenLine0
-        expect(tokens.length).toBe 2
+        expect(tokens.length).toBe 3
-        expect(tokens[0].value).toBe "# Econ 101"
+        expect(tokens[0].value).toBe "#"
-        expect(tokens[1].value).toBe tabText
+        expect(tokens[1].value).toBe " Econ 101"
-        expect(tokens[1].type).toBe tokens[0].type
+        expect(tokens[2].value).toBe tabText
-        expect(tokens[1].isAtomic).toBeTruthy()
+        expect(tokens[2].scopes).toEqual tokens[1].scopes
+        expect(tokens[2].isAtomic).toBeTruthy()
 
         expect(tokenizedBuffer.lineForScreenRow(2).text).toBe "#{tabText} buy()#{tabText}while supply > demand"

src/app/display-buffer.coffee

@@ -179,7 +179,7 @@ class DisplayBuffer
     @lineMap.bufferPositionForScreenPosition(position, options)
 
   stateForScreenRow: (screenRow) ->
-    @tokenizedBuffer.stateForRow(screenRow)
+    @tokenizedBuffer.stackForRow(screenRow)
 
   clipScreenPosition: (position, options) ->
     @lineMap.clipScreenPosition(position, options)

src/app/language-mode.coffee

@@ -60,7 +60,7 @@ class LanguageMode
 
   toggleLineCommentsInRange: (range) ->
     range = Range.fromObject(range)
-    @aceMode.toggleCommentLines(@tokenizedBuffer.stateForRow(range.start.row), @aceAdaptor, range.start.row, range.end.row)
+    @aceMode.toggleCommentLines(@tokenizedBuffer.stackForRow(range.start.row), @aceAdaptor, range.start.row, range.end.row)
 
   isBufferRowFoldable: (bufferRow) ->
     @aceMode.foldingRules?.getFoldWidget(@aceAdaptor, null, bufferRow) == "start"
@@ -72,13 +72,13 @@ class LanguageMode
       null
 
   indentationForRow: (row) ->
-    state = @tokenizedBuffer.stateForRow(row)
+    state = @tokenizedBuffer.stackForRow(row)
     previousRowText = @buffer.lineForRow(row - 1)
     @aceMode.getNextLineIndent(state, previousRowText, @editSession.tabText)
 
   autoIndentTextAfterBufferPosition: (text, bufferPosition) ->
     { row, column} = bufferPosition
-    state = @tokenizedBuffer.stateForRow(row)
+    state = @tokenizedBuffer.stackForRow(row)
     lineBeforeCursor = @buffer.lineForRow(row)[0...column]
     if text[0] == "\n"
       indent = @aceMode.getNextLineIndent(state, lineBeforeCursor, @editSession.tabText)
@@ -89,7 +89,7 @@ class LanguageMode
     {text, shouldOutdent}
 
   autoOutdentBufferRow: (bufferRow) ->
-    state = @tokenizedBuffer.stateForRow(bufferRow)
+    state = @tokenizedBuffer.stackForRow(bufferRow)
     @aceMode.autoOutdent(state, @aceAdaptor, bufferRow)
 
   getLineTokens: (line, stack) ->

src/app/token.coffee

@@ -13,6 +13,9 @@ class Token
   isEqual: (other) ->
     @value == other.value and _.isEqual(@scopes, other.scopes) and !!@isAtomic == !!other.isAtomic
 
+  isBracket: ->
+    /^meta\.brace\b/.test(_.last(@scopes))
+
   splitAt: (splitIndex) ->
     value1 = @value.substring(0, splitIndex)
     value2 = @value.substring(splitIndex)

src/app/tokenized-buffer.coffee

@@ -23,9 +23,9 @@ class TokenizedBuffer
   handleBufferChange: (e) ->
     oldRange = e.oldRange.copy()
     newRange = e.newRange.copy()
-    previousState = @stateForRow(oldRange.end.row) # used in spill detection below
+    previousStack = @stackForRow(oldRange.end.row) # used in spill detection below
 
-    stack = @stateForRow(newRange.start.row - 1)
+    stack = @stackForRow(newRange.start.row - 1)
     @screenLines[oldRange.start.row..oldRange.end.row] =
       @buildScreenLinesForRows(newRange.start.row, newRange.end.row, stack)
 
@@ -35,10 +35,10 @@ class TokenizedBuffer
     # each line until the line's new state matches the previous state. this covers
     # cases like inserting a /* needing to comment out lines below until we see a */
     for row in [newRange.end.row...@buffer.getLastRow()]
-      break if @stateForRow(row) == previousState
+      break if _.isEqual(@stackForRow(row), previousStack)
       nextRow = row + 1
-      previousState = @stateForRow(nextRow)
+      previousStack = @stackForRow(nextRow)
-      @screenLines[nextRow] = @buildScreenLineForRow(nextRow, @stateForRow(row))
+      @screenLines[nextRow] = @buildScreenLineForRow(nextRow, @stackForRow(row))
 
     # if highlighting spilled beyond the bounds of the textual change, update
     # the pre and post range to reflect area of highlight changes
@@ -74,7 +74,7 @@ class TokenizedBuffer
   linesForScreenRows: (startRow, endRow) ->
     @screenLines[startRow..endRow]
 
-  stateForRow: (row) ->
+  stackForRow: (row) ->
     @screenLines[row]?.stack
 
   destroy: ->
@@ -115,7 +115,7 @@ class TokenizedBuffer
     position = null
     depth = 0
     @backwardsIterateTokensInBufferRange range, (token, startPosition, { stop }) ->
-      if token.type.match /lparen|rparen/
+      if token.isBracket()
         if token.value == '}'
           depth++
         else if token.value == '{'
@@ -130,7 +130,7 @@ class TokenizedBuffer
     position = null
     depth = 0
     @iterateTokensInBufferRange range, (token, startPosition, { stop }) ->
-      if token.type.match /lparen|rparen/
+      if token.isBracket()
         if token.value == '{'
           depth++
         else if token.value == '}'