Tokenized buffer uses TextMate grammar

This commit is contained in:
Corey Johnson & Nathan Sobo 2012-08-02 10:05:09 -07:00
parent b50b8eacca
commit 0a97cb0590
5 changed files with 57 additions and 53 deletions

View File

@ -13,14 +13,14 @@ describe "TokenizedBuffer", ->
afterEach ->
editSession.destroy()
describe ".findClosingBracket(startBufferPosition)", ->
it "returns the position of the matching bracket, skipping any nested brackets", ->
expect(tokenizedBuffer.findClosingBracket([1, 29])).toEqual [9, 2]
describe ".findOpeningBracket(closingBufferPosition)", ->
it "returns the position of the matching bracket, skipping any nested brackets", ->
expect(tokenizedBuffer.findOpeningBracket([9, 2])).toEqual [1, 29]
describe ".findClosingBracket(startBufferPosition)", ->
it "returns the position of the matching bracket, skipping any nested brackets", ->
expect(tokenizedBuffer.findClosingBracket([1, 29])).toEqual [9, 2]
describe "tokenization", ->
it "tokenizes all the lines in the buffer on construction", ->
expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual(value: 'var', scopes: ['source.js', 'storage.type.js'])
@ -34,30 +34,30 @@ describe "TokenizedBuffer", ->
tokenizedBuffer.on "change", changeHandler
describe "when lines are updated, but none are added or removed", ->
fit "updates tokens for each of the changed lines", ->
it "updates tokens for each of the changed lines", ->
range = new Range([0, 0], [2, 0])
buffer.change(range, "foo()\n7\n")
expect(tokenizedBuffer.lineForScreenRow(0).tokens[1]).toEqual(value: '(', scopes: ['source.js', 'meta.brace.round.js'])
expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: '7', scopes: ['source.js', 'constant.numeric.js'])
# line 2 is unchanged
expect(tokenizedBuffer.lineForScreenRow(2).tokens[1]).toEqual(value: 'if', scopes: ['source.js'])
expect(changeHandler).toHaveBeenCalled()
[event] = changeHandler.argsForCall[0]
expect(event.oldRange).toEqual range
expect(event.newRange).toEqual new Range([0, 0], [2,0])
# line 2 is unchanged
expect(tokenizedBuffer.lineForScreenRow(2).tokens[1]).toEqual(value: 'if', scopes: ['source.js', 'keyword.control.js'])
it "updates tokens for lines beyond the changed lines if needed", ->
buffer.insert([5, 30], '/* */')
changeHandler.reset()
buffer.insert([2, 0], '/*')
expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].type).toBe 'comment'
expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].type).toBe 'comment'
expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].type).toBe 'comment'
expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
expect(changeHandler).toHaveBeenCalled()
[event] = changeHandler.argsForCall[0]
@ -69,7 +69,7 @@ describe "TokenizedBuffer", ->
buffer.insert([5, 0], '*/')
buffer.insert([1, 0], 'var ')
expect(tokenizedBuffer.lineForScreenRow(1).tokens[0].type).toBe 'comment'
expect(tokenizedBuffer.lineForScreenRow(1).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
describe "when lines are both updated and removed", ->
it "updates tokens to reflect the removed lines", ->
@ -77,16 +77,16 @@ describe "TokenizedBuffer", ->
buffer.change(range, "foo()")
# previous line 0 remains
expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual(type: 'keyword.definition', value: 'var')
expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual(value: 'var', scopes: ['source.js', 'storage.type.js'])
# previous line 3 should be combined with input to form line 1
expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(type: 'identifier', value: 'foo')
expect(tokenizedBuffer.lineForScreenRow(1).tokens[6]).toEqual(type: 'identifier', value: 'pivot')
expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: 'foo', scopes: ['source.js'])
expect(tokenizedBuffer.lineForScreenRow(1).tokens[6]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js'])
# lines below deleted regions should be shifted upward
expect(tokenizedBuffer.lineForScreenRow(2).tokens[1]).toEqual(type: 'keyword', value: 'while')
expect(tokenizedBuffer.lineForScreenRow(3).tokens[1]).toEqual(type: 'identifier', value: 'current')
expect(tokenizedBuffer.lineForScreenRow(4).tokens[3]).toEqual(type: 'keyword.operator', value: '<')
expect(tokenizedBuffer.lineForScreenRow(2).tokens[1]).toEqual(value: 'while', scopes: ['source.js', 'keyword.control.js'])
expect(tokenizedBuffer.lineForScreenRow(3).tokens[1]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js'])
expect(tokenizedBuffer.lineForScreenRow(4).tokens[1]).toEqual(value: '<', scopes: ['source.js', 'keyword.operator.js'])
expect(changeHandler).toHaveBeenCalled()
[event] = changeHandler.argsForCall[0]
@ -98,9 +98,9 @@ describe "TokenizedBuffer", ->
changeHandler.reset()
buffer.change(new Range([2, 0], [3, 0]), '/*')
expect(tokenizedBuffer.lineForScreenRow(2).tokens[0].type).toBe 'comment'
expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].type).toBe 'comment'
expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].type).toBe 'comment'
expect(tokenizedBuffer.lineForScreenRow(2).tokens[0].scopes).toEqual ['source.js', 'comment.block.js', 'punctuation.definition.comment.js']
expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
expect(changeHandler).toHaveBeenCalled()
[event] = changeHandler.argsForCall[0]
@ -113,19 +113,19 @@ describe "TokenizedBuffer", ->
buffer.change(range, "foo()\nbar()\nbaz()\nquux()")
# previous line 0 remains
expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual(type: 'keyword.definition', value: 'var')
expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual( value: 'var', scopes: ['source.js', 'storage.type.js'])
# 3 new lines inserted
expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(type: 'identifier', value: 'foo')
expect(tokenizedBuffer.lineForScreenRow(2).tokens[0]).toEqual(type: 'identifier', value: 'bar')
expect(tokenizedBuffer.lineForScreenRow(3).tokens[0]).toEqual(type: 'identifier', value: 'baz')
expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: 'foo', scopes: ['source.js'])
expect(tokenizedBuffer.lineForScreenRow(2).tokens[0]).toEqual(value: 'bar', scopes: ['source.js'])
expect(tokenizedBuffer.lineForScreenRow(3).tokens[0]).toEqual(value: 'baz', scopes: ['source.js'])
# previous line 2 is joined with quux() on line 4
expect(tokenizedBuffer.lineForScreenRow(4).tokens[0]).toEqual(type: 'identifier', value: 'quux')
expect(tokenizedBuffer.lineForScreenRow(4).tokens[4]).toEqual(type: 'keyword', value: 'if')
expect(tokenizedBuffer.lineForScreenRow(4).tokens[0]).toEqual(value: 'quux', scopes: ['source.js'])
expect(tokenizedBuffer.lineForScreenRow(4).tokens[4]).toEqual(value: 'if', scopes: ['source.js', 'keyword.control.js'])
# previous line 3 is pushed down to become line 5
expect(tokenizedBuffer.lineForScreenRow(5).tokens[3]).toEqual(type: 'identifier', value: 'pivot')
expect(tokenizedBuffer.lineForScreenRow(5).tokens[3]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js'])
expect(changeHandler).toHaveBeenCalled()
[event] = changeHandler.argsForCall[0]
@ -137,13 +137,13 @@ describe "TokenizedBuffer", ->
changeHandler.reset()
buffer.insert([2, 0], '/*\nabcde\nabcder')
expect(tokenizedBuffer.lineForScreenRow(2).tokens[0].type).toBe 'comment'
expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].type).toBe 'comment'
expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].type).toBe 'comment'
expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].type).toBe 'comment'
expect(tokenizedBuffer.lineForScreenRow(6).tokens[0].type).toBe 'comment'
expect(tokenizedBuffer.lineForScreenRow(7).tokens[0].type).toBe 'comment'
expect(tokenizedBuffer.lineForScreenRow(8).tokens[0].type).not.toBe 'comment'
expect(tokenizedBuffer.lineForScreenRow(2).tokens[0].scopes).toEqual ['source.js', 'comment.block.js', 'punctuation.definition.comment.js']
expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
expect(tokenizedBuffer.lineForScreenRow(6).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
expect(tokenizedBuffer.lineForScreenRow(7).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
expect(tokenizedBuffer.lineForScreenRow(8).tokens[0].scopes).not.toBe ['source.js', 'comment.block.js']
expect(changeHandler).toHaveBeenCalled()
[event] = changeHandler.argsForCall[0]
@ -165,10 +165,11 @@ describe "TokenizedBuffer", ->
screenLine0 = tokenizedBuffer.lineForScreenRow(0)
expect(screenLine0.text).toBe "# Econ 101#{tabText}"
{ tokens } = screenLine0
expect(tokens.length).toBe 2
expect(tokens[0].value).toBe "# Econ 101"
expect(tokens[1].value).toBe tabText
expect(tokens[1].type).toBe tokens[0].type
expect(tokens[1].isAtomic).toBeTruthy()
expect(tokens.length).toBe 3
expect(tokens[0].value).toBe "#"
expect(tokens[1].value).toBe " Econ 101"
expect(tokens[2].value).toBe tabText
expect(tokens[2].scopes).toEqual tokens[1].scopes
expect(tokens[2].isAtomic).toBeTruthy()
expect(tokenizedBuffer.lineForScreenRow(2).text).toBe "#{tabText} buy()#{tabText}while supply > demand"

View File

@ -179,7 +179,7 @@ class DisplayBuffer
@lineMap.bufferPositionForScreenPosition(position, options)
stateForScreenRow: (screenRow) ->
@tokenizedBuffer.stateForRow(screenRow)
@tokenizedBuffer.stackForRow(screenRow)
clipScreenPosition: (position, options) ->
@lineMap.clipScreenPosition(position, options)

View File

@ -60,7 +60,7 @@ class LanguageMode
toggleLineCommentsInRange: (range) ->
range = Range.fromObject(range)
@aceMode.toggleCommentLines(@tokenizedBuffer.stateForRow(range.start.row), @aceAdaptor, range.start.row, range.end.row)
@aceMode.toggleCommentLines(@tokenizedBuffer.stackForRow(range.start.row), @aceAdaptor, range.start.row, range.end.row)
isBufferRowFoldable: (bufferRow) ->
@aceMode.foldingRules?.getFoldWidget(@aceAdaptor, null, bufferRow) == "start"
@ -72,13 +72,13 @@ class LanguageMode
null
indentationForRow: (row) ->
state = @tokenizedBuffer.stateForRow(row)
state = @tokenizedBuffer.stackForRow(row)
previousRowText = @buffer.lineForRow(row - 1)
@aceMode.getNextLineIndent(state, previousRowText, @editSession.tabText)
autoIndentTextAfterBufferPosition: (text, bufferPosition) ->
{ row, column} = bufferPosition
state = @tokenizedBuffer.stateForRow(row)
state = @tokenizedBuffer.stackForRow(row)
lineBeforeCursor = @buffer.lineForRow(row)[0...column]
if text[0] == "\n"
indent = @aceMode.getNextLineIndent(state, lineBeforeCursor, @editSession.tabText)
@ -89,7 +89,7 @@ class LanguageMode
{text, shouldOutdent}
autoOutdentBufferRow: (bufferRow) ->
state = @tokenizedBuffer.stateForRow(bufferRow)
state = @tokenizedBuffer.stackForRow(bufferRow)
@aceMode.autoOutdent(state, @aceAdaptor, bufferRow)
getLineTokens: (line, stack) ->

View File

@ -13,6 +13,9 @@ class Token
isEqual: (other) ->
@value == other.value and _.isEqual(@scopes, other.scopes) and !!@isAtomic == !!other.isAtomic
isBracket: ->
/^meta\.brace\b/.test(_.last(@scopes))
splitAt: (splitIndex) ->
value1 = @value.substring(0, splitIndex)
value2 = @value.substring(splitIndex)

View File

@ -23,9 +23,9 @@ class TokenizedBuffer
handleBufferChange: (e) ->
oldRange = e.oldRange.copy()
newRange = e.newRange.copy()
previousState = @stateForRow(oldRange.end.row) # used in spill detection below
previousStack = @stackForRow(oldRange.end.row) # used in spill detection below
stack = @stateForRow(newRange.start.row - 1)
stack = @stackForRow(newRange.start.row - 1)
@screenLines[oldRange.start.row..oldRange.end.row] =
@buildScreenLinesForRows(newRange.start.row, newRange.end.row, stack)
@ -35,10 +35,10 @@ class TokenizedBuffer
# each line until the line's new state matches the previous state. this covers
# cases like inserting a /* needing to comment out lines below until we see a */
for row in [newRange.end.row...@buffer.getLastRow()]
break if @stateForRow(row) == previousState
break if _.isEqual(@stackForRow(row), previousStack)
nextRow = row + 1
previousState = @stateForRow(nextRow)
@screenLines[nextRow] = @buildScreenLineForRow(nextRow, @stateForRow(row))
previousStack = @stackForRow(nextRow)
@screenLines[nextRow] = @buildScreenLineForRow(nextRow, @stackForRow(row))
# if highlighting spilled beyond the bounds of the textual change, update
# the pre and post range to reflect area of highlight changes
@ -74,7 +74,7 @@ class TokenizedBuffer
linesForScreenRows: (startRow, endRow) ->
@screenLines[startRow..endRow]
stateForRow: (row) ->
stackForRow: (row) ->
@screenLines[row]?.stack
destroy: ->
@ -115,7 +115,7 @@ class TokenizedBuffer
position = null
depth = 0
@backwardsIterateTokensInBufferRange range, (token, startPosition, { stop }) ->
if token.type.match /lparen|rparen/
if token.isBracket()
if token.value == '}'
depth++
else if token.value == '{'
@ -130,7 +130,7 @@ class TokenizedBuffer
position = null
depth = 0
@iterateTokensInBufferRange range, (token, startPosition, { stop }) ->
if token.type.match /lparen|rparen/
if token.isBracket()
if token.value == '{'
depth++
else if token.value == '}'