wip: hooking up tokenized buffer to textmate grammars

spec/app/tokenized-buffer-spec.coffee

@@ -23,8 +23,8 @@ describe "TokenizedBuffer", ->
 
   describe "tokenization", ->
     it "tokenizes all the lines in the buffer on construction", ->
-      expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual(type: 'keyword.definition', value: 'var')
-      expect(tokenizedBuffer.lineForScreenRow(11).tokens[1]).toEqual(type: 'keyword', value: 'return')
+      expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual(value: 'var', scopes: ['source.js', 'storage.type.js'])
+      expect(tokenizedBuffer.lineForScreenRow(11).tokens[1]).toEqual(value: 'return', scopes: ['source.js', 'keyword.control.js'])
 
     describe "when the buffer changes", ->
       changeHandler = null
@@ -34,15 +34,15 @@ describe "TokenizedBuffer", ->
         tokenizedBuffer.on "change", changeHandler
 
       describe "when lines are updated, but none are added or removed", ->
-        it "updates tokens for each of the changed lines", ->
+        fit "updates tokens for each of the changed lines", ->
           range = new Range([0, 0], [2, 0])
-          buffer.change(range, "foo()\nbar()\n")
+          buffer.change(range, "foo()\n7\n")
 
-          expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual(type: 'identifier', value: 'foo')
-          expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(type: 'identifier', value: 'bar')
+          expect(tokenizedBuffer.lineForScreenRow(0).tokens[1]).toEqual(value: '(', scopes: ['source.js', 'meta.brace.round.js'])
+          expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: '7', scopes: ['source.js', 'constant.numeric.js'])
 
           # line 2 is unchanged
-          expect(tokenizedBuffer.lineForScreenRow(2).tokens[1]).toEqual(type: 'keyword', value: 'if')
+          expect(tokenizedBuffer.lineForScreenRow(2).tokens[1]).toEqual(value: 'if', scopes: ['source.js'])
 
           expect(changeHandler).toHaveBeenCalled()
           [event] = changeHandler.argsForCall[0]

src/app/language-mode.coffee

@@ -1,5 +1,6 @@
 AceAdaptor = require 'ace-adaptor'
 Range = require 'range'
+TextMateGrammar = require 'text-mate-grammar'
 _ = require 'underscore'
 
 module.exports =
@@ -14,6 +15,7 @@ class LanguageMode
   constructor: (@editSession) ->
     @buffer = @editSession.buffer
     @aceMode = @requireAceMode()
+    @grammar = TextMateGrammar.grammarForExtension(@editSession.buffer.getExtension())
     @aceAdaptor = new AceAdaptor(@editSession)
 
     _.adviseBefore @editSession, 'insertText', (text) =>
@@ -90,6 +92,6 @@ class LanguageMode
     state = @tokenizedBuffer.stateForRow(bufferRow)
     @aceMode.autoOutdent(state, @aceAdaptor, bufferRow)
 
-  getLineTokens: (line, state) ->
-    {tokens, state} = @aceMode.getTokenizer().getLineTokens(line, state)
+  getLineTokens: (line, stack) ->
+    {tokens, stack} = @grammar.getLineTokens(line, stack)
 

src/app/screen-line.coffee

@@ -3,7 +3,7 @@ Point = require 'point'
 
 module.exports =
 class ScreenLine
-  state: null
+  stack: null
   text: null
   tokens: null
   screenDelta: null
@@ -16,7 +16,7 @@ class ScreenLine
     _.extend(this, extraFields)
 
   copy: ->
-    new ScreenLine(@tokens, @text, @screenDelta, @bufferDelta, { @state, @foldable })
+    new ScreenLine(@tokens, @text, @screenDelta, @bufferDelta, { @stack, @foldable })
 
   splitAt: (column) ->
     return [new ScreenLine([], '', [0, 0], [0, 0]), this] if column == 0
@@ -37,8 +37,8 @@ class ScreenLine
     [leftScreenDelta, rightScreenDelta] = @screenDelta.splitAt(column)
     [leftBufferDelta, rightBufferDelta] = @bufferDelta.splitAt(column)
 
-    leftFragment = new ScreenLine(leftTokens, leftText, leftScreenDelta, leftBufferDelta, {@state, @foldable})
-    rightFragment = new ScreenLine(rightTokens, rightText, rightScreenDelta, rightBufferDelta, {@state})
+    leftFragment = new ScreenLine(leftTokens, leftText, leftScreenDelta, leftBufferDelta, {@stack, @foldable})
+    rightFragment = new ScreenLine(rightTokens, rightText, rightScreenDelta, rightBufferDelta, {@stack})
     [leftFragment, rightFragment]
 
   concat: (other) ->
@@ -46,7 +46,7 @@ class ScreenLine
     text = @text + other.text
     screenDelta = @screenDelta.add(other.screenDelta)
     bufferDelta = @bufferDelta.add(other.bufferDelta)
-    new ScreenLine(tokens, text, screenDelta, bufferDelta, {state: other.state})
+    new ScreenLine(tokens, text, screenDelta, bufferDelta, {stack: other.stack})
 
   translateColumn: (sourceDeltaType, targetDeltaType, sourceColumn, options={}) ->
     { skipAtomicTokens } = options

src/app/token.coffee

@@ -1,20 +1,22 @@
+_ = require 'underscore'
+
 module.exports =
 class Token
   value: null
-  type: null
+  scopes: null
   isAtomic: null
 
-  constructor: ({@value, @type, @isAtomic, @bufferDelta, @fold}) ->
+  constructor: ({@value, @scopes, @isAtomic, @bufferDelta, @fold}) ->
     @screenDelta = @value.length
     @bufferDelta ?= @screenDelta
 
   isEqual: (other) ->
-    @value == other.value and @type == other.type and !!@isAtomic == !!other.isAtomic
+    @value == other.value and _.isEqual(@scopes, other.scopes) and !!@isAtomic == !!other.isAtomic
 
   splitAt: (splitIndex) ->
     value1 = @value.substring(0, splitIndex)
     value2 = @value.substring(splitIndex)
-    [new Token(value: value1, type: @type), new Token(value: value2, type: @type)]
+    [new Token(value: value1, scopes: @scopes), new Token(value: value2, scopes: @scopes)]
 
   breakOutTabCharacters: (tabText) ->
     return [this] unless /\t/.test(@value)
@@ -23,7 +25,7 @@ class Token
       if substring == '\t'
         @buildTabToken(tabText)
       else
-        new Token(value: substring, type: @type)
+        new Token(value: substring, scopes: @scopes)
 
   buildTabToken: (tabText) ->
-    new Token(value: tabText, type: @type, bufferDelta: 1, isAtomic: true)
+    new Token(value: tabText, scopes: @scopes, bufferDelta: 1, isAtomic: true)

src/app/tokenized-buffer.coffee

@@ -17,7 +17,7 @@ class TokenizedBuffer
   constructor: (@buffer, { @languageMode, @tabText }) ->
     @languageMode.tokenizedBuffer = this
     @id = @constructor.idCounter++
-    @screenLines = @buildScreenLinesForRows('start', 0, @buffer.getLastRow())
+    @screenLines = @buildScreenLinesForRows(0, @buffer.getLastRow())
     @buffer.on "change.tokenized-buffer#{@id}", (e) => @handleBufferChange(e)
 
   handleBufferChange: (e) ->
@@ -25,9 +25,9 @@ class TokenizedBuffer
     newRange = e.newRange.copy()
     previousState = @stateForRow(oldRange.end.row) # used in spill detection below
 
-    startState = @stateForRow(newRange.start.row - 1)
+    stack = @stateForRow(newRange.start.row - 1)
     @screenLines[oldRange.start.row..oldRange.end.row] =
-      @buildScreenLinesForRows(startState, newRange.start.row, newRange.end.row)
+      @buildScreenLinesForRows(newRange.start.row, newRange.end.row, stack)
 
     # spill detection
     # compare scanner state of last re-highlighted line with its previous state.
@@ -38,7 +38,7 @@ class TokenizedBuffer
       break if @stateForRow(row) == previousState
       nextRow = row + 1
       previousState = @stateForRow(nextRow)
-      @screenLines[nextRow] = @buildScreenLineForRow(@stateForRow(row), nextRow)
+      @screenLines[nextRow] = @buildScreenLineForRow(nextRow, @stateForRow(row))
 
     # if highlighting spilled beyond the bounds of the textual change, update
     # the pre and post range to reflect area of highlight changes
@@ -51,22 +51,22 @@ class TokenizedBuffer
 
     @trigger("change", {oldRange, newRange})
 
-  buildScreenLinesForRows: (startState, startRow, endRow) ->
-    state = startState
+  buildScreenLinesForRows: (startRow, endRow, startingStack) ->
+    stack = startingStack
     for row in [startRow..endRow]
-      screenLine = @buildScreenLineForRow(state, row)
-      state = screenLine.state
+      screenLine = @buildScreenLineForRow(row, stack)
+      stack = screenLine.stack
       screenLine
 
-  buildScreenLineForRow: (state, row) ->
+  buildScreenLineForRow: (row, stack) ->
     line = @buffer.lineForRow(row)
-    {tokens, state} = @languageMode.getLineTokens(line, state)
+    {tokens, stack} = @languageMode.getLineTokens(line, stack)
     tokenObjects = []
     for tokenProperties in tokens
       token = new Token(tokenProperties)
       tokenObjects.push(token.breakOutTabCharacters(@tabText)...)
     text = _.pluck(tokenObjects, 'value').join('')
-    new ScreenLine(tokenObjects, text, [1, 0], [1, 0], { state })
+    new ScreenLine(tokenObjects, text, [1, 0], [1, 0], { stack })
 
   lineForScreenRow: (row) ->
     @screenLines[row]
@@ -75,7 +75,7 @@ class TokenizedBuffer
     @screenLines[startRow..endRow]
 
   stateForRow: (row) ->
-    @screenLines[row]?.state ? 'start'
+    @screenLines[row]?.stack
 
   destroy: ->
     @buffer.off ".tokenized-buffer#{@id}"