diff --git a/spec/app/tokenized-buffer-spec.coffee b/spec/app/tokenized-buffer-spec.coffee index 09835a1db..8eec05c09 100644 --- a/spec/app/tokenized-buffer-spec.coffee +++ b/spec/app/tokenized-buffer-spec.coffee @@ -13,14 +13,14 @@ describe "TokenizedBuffer", -> afterEach -> editSession.destroy() - describe ".findClosingBracket(startBufferPosition)", -> - it "returns the position of the matching bracket, skipping any nested brackets", -> - expect(tokenizedBuffer.findClosingBracket([1, 29])).toEqual [9, 2] - describe ".findOpeningBracket(closingBufferPosition)", -> it "returns the position of the matching bracket, skipping any nested brackets", -> expect(tokenizedBuffer.findOpeningBracket([9, 2])).toEqual [1, 29] + describe ".findClosingBracket(startBufferPosition)", -> + it "returns the position of the matching bracket, skipping any nested brackets", -> + expect(tokenizedBuffer.findClosingBracket([1, 29])).toEqual [9, 2] + describe "tokenization", -> it "tokenizes all the lines in the buffer on construction", -> expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual(value: 'var', scopes: ['source.js', 'storage.type.js']) @@ -34,30 +34,30 @@ describe "TokenizedBuffer", -> tokenizedBuffer.on "change", changeHandler describe "when lines are updated, but none are added or removed", -> - fit "updates tokens for each of the changed lines", -> + it "updates tokens for each of the changed lines", -> range = new Range([0, 0], [2, 0]) buffer.change(range, "foo()\n7\n") expect(tokenizedBuffer.lineForScreenRow(0).tokens[1]).toEqual(value: '(', scopes: ['source.js', 'meta.brace.round.js']) expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: '7', scopes: ['source.js', 'constant.numeric.js']) - # line 2 is unchanged - expect(tokenizedBuffer.lineForScreenRow(2).tokens[1]).toEqual(value: 'if', scopes: ['source.js']) - expect(changeHandler).toHaveBeenCalled() [event] = changeHandler.argsForCall[0] expect(event.oldRange).toEqual range expect(event.newRange).toEqual new Range([0, 0], [2,0]) + # line 2 is unchanged + expect(tokenizedBuffer.lineForScreenRow(2).tokens[1]).toEqual(value: 'if', scopes: ['source.js', 'keyword.control.js']) + it "updates tokens for lines beyond the changed lines if needed", -> buffer.insert([5, 30], '/* */') changeHandler.reset() buffer.insert([2, 0], '/*') - expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].type).toBe 'comment' - expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].type).toBe 'comment' - expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].type).toBe 'comment' + expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] expect(changeHandler).toHaveBeenCalled() [event] = changeHandler.argsForCall[0] @@ -69,7 +69,7 @@ describe "TokenizedBuffer", -> buffer.insert([5, 0], '*/') buffer.insert([1, 0], 'var ') - expect(tokenizedBuffer.lineForScreenRow(1).tokens[0].type).toBe 'comment' + expect(tokenizedBuffer.lineForScreenRow(1).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] describe "when lines are both updated and removed", -> it "updates tokens to reflect the removed lines", -> @@ -77,16 +77,16 @@ describe "TokenizedBuffer", -> buffer.change(range, "foo()") # previous line 0 remains - expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual(type: 'keyword.definition', value: 'var') + expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual(value: 'var', scopes: ['source.js', 'storage.type.js']) # previous line 3 should be combined with input to form line 1 - expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(type: 'identifier', value: 'foo') - expect(tokenizedBuffer.lineForScreenRow(1).tokens[6]).toEqual(type: 'identifier', value: 'pivot') + expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: 'foo', scopes: ['source.js']) + expect(tokenizedBuffer.lineForScreenRow(1).tokens[6]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js']) # lines below deleted regions should be shifted upward - expect(tokenizedBuffer.lineForScreenRow(2).tokens[1]).toEqual(type: 'keyword', value: 'while') - expect(tokenizedBuffer.lineForScreenRow(3).tokens[1]).toEqual(type: 'identifier', value: 'current') - expect(tokenizedBuffer.lineForScreenRow(4).tokens[3]).toEqual(type: 'keyword.operator', value: '<') + expect(tokenizedBuffer.lineForScreenRow(2).tokens[1]).toEqual(value: 'while', scopes: ['source.js', 'keyword.control.js']) + expect(tokenizedBuffer.lineForScreenRow(3).tokens[1]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js']) + expect(tokenizedBuffer.lineForScreenRow(4).tokens[1]).toEqual(value: '<', scopes: ['source.js', 'keyword.operator.js']) expect(changeHandler).toHaveBeenCalled() [event] = changeHandler.argsForCall[0] @@ -98,9 +98,9 @@ describe "TokenizedBuffer", -> changeHandler.reset() buffer.change(new Range([2, 0], [3, 0]), '/*') - expect(tokenizedBuffer.lineForScreenRow(2).tokens[0].type).toBe 'comment' - expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].type).toBe 'comment' - expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].type).toBe 'comment' + expect(tokenizedBuffer.lineForScreenRow(2).tokens[0].scopes).toEqual ['source.js', 'comment.block.js', 'punctuation.definition.comment.js'] + expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] expect(changeHandler).toHaveBeenCalled() [event] = changeHandler.argsForCall[0] @@ -113,19 +113,19 @@ describe "TokenizedBuffer", -> buffer.change(range, "foo()\nbar()\nbaz()\nquux()") # previous line 0 remains - expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual(type: 'keyword.definition', value: 'var') + expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual( value: 'var', scopes: ['source.js', 'storage.type.js']) # 3 new lines inserted - expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(type: 'identifier', value: 'foo') - expect(tokenizedBuffer.lineForScreenRow(2).tokens[0]).toEqual(type: 'identifier', value: 'bar') - expect(tokenizedBuffer.lineForScreenRow(3).tokens[0]).toEqual(type: 'identifier', value: 'baz') + expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: 'foo', scopes: ['source.js']) + expect(tokenizedBuffer.lineForScreenRow(2).tokens[0]).toEqual(value: 'bar', scopes: ['source.js']) + expect(tokenizedBuffer.lineForScreenRow(3).tokens[0]).toEqual(value: 'baz', scopes: ['source.js']) # previous line 2 is joined with quux() on line 4 - expect(tokenizedBuffer.lineForScreenRow(4).tokens[0]).toEqual(type: 'identifier', value: 'quux') - expect(tokenizedBuffer.lineForScreenRow(4).tokens[4]).toEqual(type: 'keyword', value: 'if') + expect(tokenizedBuffer.lineForScreenRow(4).tokens[0]).toEqual(value: 'quux', scopes: ['source.js']) + expect(tokenizedBuffer.lineForScreenRow(4).tokens[4]).toEqual(value: 'if', scopes: ['source.js', 'keyword.control.js']) # previous line 3 is pushed down to become line 5 - expect(tokenizedBuffer.lineForScreenRow(5).tokens[3]).toEqual(type: 'identifier', value: 'pivot') + expect(tokenizedBuffer.lineForScreenRow(5).tokens[3]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js']) expect(changeHandler).toHaveBeenCalled() [event] = changeHandler.argsForCall[0] @@ -137,13 +137,13 @@ describe "TokenizedBuffer", -> changeHandler.reset() buffer.insert([2, 0], '/*\nabcde\nabcder') - expect(tokenizedBuffer.lineForScreenRow(2).tokens[0].type).toBe 'comment' - expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].type).toBe 'comment' - expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].type).toBe 'comment' - expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].type).toBe 'comment' - expect(tokenizedBuffer.lineForScreenRow(6).tokens[0].type).toBe 'comment' - expect(tokenizedBuffer.lineForScreenRow(7).tokens[0].type).toBe 'comment' - expect(tokenizedBuffer.lineForScreenRow(8).tokens[0].type).not.toBe 'comment' + expect(tokenizedBuffer.lineForScreenRow(2).tokens[0].scopes).toEqual ['source.js', 'comment.block.js', 'punctuation.definition.comment.js'] + expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + expect(tokenizedBuffer.lineForScreenRow(6).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + expect(tokenizedBuffer.lineForScreenRow(7).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + expect(tokenizedBuffer.lineForScreenRow(8).tokens[0].scopes).not.toBe ['source.js', 'comment.block.js'] expect(changeHandler).toHaveBeenCalled() [event] = changeHandler.argsForCall[0] @@ -165,10 +165,11 @@ describe "TokenizedBuffer", -> screenLine0 = tokenizedBuffer.lineForScreenRow(0) expect(screenLine0.text).toBe "# Econ 101#{tabText}" { tokens } = screenLine0 - expect(tokens.length).toBe 2 - expect(tokens[0].value).toBe "# Econ 101" - expect(tokens[1].value).toBe tabText - expect(tokens[1].type).toBe tokens[0].type - expect(tokens[1].isAtomic).toBeTruthy() + expect(tokens.length).toBe 3 + expect(tokens[0].value).toBe "#" + expect(tokens[1].value).toBe " Econ 101" + expect(tokens[2].value).toBe tabText + expect(tokens[2].scopes).toEqual tokens[1].scopes + expect(tokens[2].isAtomic).toBeTruthy() expect(tokenizedBuffer.lineForScreenRow(2).text).toBe "#{tabText} buy()#{tabText}while supply > demand" diff --git a/src/app/display-buffer.coffee b/src/app/display-buffer.coffee index d4d92e560..0bda7addc 100644 --- a/src/app/display-buffer.coffee +++ b/src/app/display-buffer.coffee @@ -179,7 +179,7 @@ class DisplayBuffer @lineMap.bufferPositionForScreenPosition(position, options) stateForScreenRow: (screenRow) -> - @tokenizedBuffer.stateForRow(screenRow) + @tokenizedBuffer.stackForRow(screenRow) clipScreenPosition: (position, options) -> @lineMap.clipScreenPosition(position, options) diff --git a/src/app/language-mode.coffee b/src/app/language-mode.coffee index 1022105a1..52ddcbd61 100644 --- a/src/app/language-mode.coffee +++ b/src/app/language-mode.coffee @@ -60,7 +60,7 @@ class LanguageMode toggleLineCommentsInRange: (range) -> range = Range.fromObject(range) - @aceMode.toggleCommentLines(@tokenizedBuffer.stateForRow(range.start.row), @aceAdaptor, range.start.row, range.end.row) + @aceMode.toggleCommentLines(@tokenizedBuffer.stackForRow(range.start.row), @aceAdaptor, range.start.row, range.end.row) isBufferRowFoldable: (bufferRow) -> @aceMode.foldingRules?.getFoldWidget(@aceAdaptor, null, bufferRow) == "start" @@ -72,13 +72,13 @@ class LanguageMode null indentationForRow: (row) -> - state = @tokenizedBuffer.stateForRow(row) + state = @tokenizedBuffer.stackForRow(row) previousRowText = @buffer.lineForRow(row - 1) @aceMode.getNextLineIndent(state, previousRowText, @editSession.tabText) autoIndentTextAfterBufferPosition: (text, bufferPosition) -> { row, column} = bufferPosition - state = @tokenizedBuffer.stateForRow(row) + state = @tokenizedBuffer.stackForRow(row) lineBeforeCursor = @buffer.lineForRow(row)[0...column] if text[0] == "\n" indent = @aceMode.getNextLineIndent(state, lineBeforeCursor, @editSession.tabText) @@ -89,7 +89,7 @@ class LanguageMode {text, shouldOutdent} autoOutdentBufferRow: (bufferRow) -> - state = @tokenizedBuffer.stateForRow(bufferRow) + state = @tokenizedBuffer.stackForRow(bufferRow) @aceMode.autoOutdent(state, @aceAdaptor, bufferRow) getLineTokens: (line, stack) -> diff --git a/src/app/token.coffee b/src/app/token.coffee index 935d584b3..80710ce9d 100644 --- a/src/app/token.coffee +++ b/src/app/token.coffee @@ -13,6 +13,9 @@ class Token isEqual: (other) -> @value == other.value and _.isEqual(@scopes, other.scopes) and !!@isAtomic == !!other.isAtomic + isBracket: -> + /^meta\.brace\b/.test(_.last(@scopes)) + splitAt: (splitIndex) -> value1 = @value.substring(0, splitIndex) value2 = @value.substring(splitIndex) diff --git a/src/app/tokenized-buffer.coffee b/src/app/tokenized-buffer.coffee index 2e8939f5e..c4d62fe56 100644 --- a/src/app/tokenized-buffer.coffee +++ b/src/app/tokenized-buffer.coffee @@ -23,9 +23,9 @@ class TokenizedBuffer handleBufferChange: (e) -> oldRange = e.oldRange.copy() newRange = e.newRange.copy() - previousState = @stateForRow(oldRange.end.row) # used in spill detection below + previousStack = @stackForRow(oldRange.end.row) # used in spill detection below - stack = @stateForRow(newRange.start.row - 1) + stack = @stackForRow(newRange.start.row - 1) @screenLines[oldRange.start.row..oldRange.end.row] = @buildScreenLinesForRows(newRange.start.row, newRange.end.row, stack) @@ -35,10 +35,10 @@ class TokenizedBuffer # each line until the line's new state matches the previous state. this covers # cases like inserting a /* needing to comment out lines below until we see a */ for row in [newRange.end.row...@buffer.getLastRow()] - break if @stateForRow(row) == previousState + break if _.isEqual(@stackForRow(row), previousStack) nextRow = row + 1 - previousState = @stateForRow(nextRow) - @screenLines[nextRow] = @buildScreenLineForRow(nextRow, @stateForRow(row)) + previousStack = @stackForRow(nextRow) + @screenLines[nextRow] = @buildScreenLineForRow(nextRow, @stackForRow(row)) # if highlighting spilled beyond the bounds of the textual change, update # the pre and post range to reflect area of highlight changes @@ -74,7 +74,7 @@ class TokenizedBuffer linesForScreenRows: (startRow, endRow) -> @screenLines[startRow..endRow] - stateForRow: (row) -> + stackForRow: (row) -> @screenLines[row]?.stack destroy: -> @@ -115,7 +115,7 @@ class TokenizedBuffer position = null depth = 0 @backwardsIterateTokensInBufferRange range, (token, startPosition, { stop }) -> - if token.type.match /lparen|rparen/ + if token.isBracket() if token.value == '}' depth++ else if token.value == '{' @@ -130,7 +130,7 @@ class TokenizedBuffer position = null depth = 0 @iterateTokensInBufferRange range, (token, startPosition, { stop }) -> - if token.type.match /lparen|rparen/ + if token.isBracket() if token.value == '{' depth++ else if token.value == '}'