mirror of
https://github.com/pulsar-edit/pulsar.git
synced 2025-01-02 20:07:25 +03:00
WIP: Fleshing out specs for async tokenization
This commit is contained in:
Corey Johnson & Nathan Sobo
parent
e52dbb682e
commit
53c7ccb297
@ -67,121 +67,134 @@ describe "TokenizedBuffer", ->
|
||||
expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual(value: 'var', scopes: ['source.js', 'storage.modifier.js'])
|
||||
expect(tokenizedBuffer.lineForScreenRow(11).tokens[1]).toEqual(value: 'return', scopes: ['source.js', 'keyword.control.js'])
|
||||
|
||||
describe "when the buffer changes", ->
|
||||
describe "when lines are updated, but none are added or removed", ->
|
||||
it "updates tokens for each of the changed lines", ->
|
||||
range = new Range([0, 0], [2, 0])
|
||||
buffer.change(range, "foo()\n7\n")
|
||||
describe "when there is a buffer change starting in the tokenized region", ->
|
||||
describe "when the change causes fewer than @chunkSize lines to be retokenized", ->
|
||||
describe "when lines are updated, but none are added or removed", ->
|
||||
it "updates tokens for each of the changed lines", ->
|
||||
range = new Range([0, 0], [2, 0])
|
||||
buffer.change(range, "foo()\n7\n")
|
||||
|
||||
expect(tokenizedBuffer.lineForScreenRow(0).tokens[1]).toEqual(value: '(', scopes: ['source.js', 'meta.brace.round.js'])
|
||||
expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: '7', scopes: ['source.js', 'constant.numeric.js'])
|
||||
# line 2 is unchanged
|
||||
expect(tokenizedBuffer.lineForScreenRow(2).tokens[2]).toEqual(value: 'if', scopes: ['source.js', 'keyword.control.js'])
|
||||
expect(tokenizedBuffer.lineForScreenRow(0).tokens[1]).toEqual(value: '(', scopes: ['source.js', 'meta.brace.round.js'])
|
||||
expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: '7', scopes: ['source.js', 'constant.numeric.js'])
|
||||
# line 2 is unchanged
|
||||
expect(tokenizedBuffer.lineForScreenRow(2).tokens[2]).toEqual(value: 'if', scopes: ['source.js', 'keyword.control.js'])
|
||||
|
||||
expect(changeHandler).toHaveBeenCalled()
|
||||
[event] = changeHandler.argsForCall[0]
|
||||
delete event.bufferChange
|
||||
expect(event).toEqual(start: 0, end: 2, delta: 0)
|
||||
expect(changeHandler).toHaveBeenCalled()
|
||||
[event] = changeHandler.argsForCall[0]
|
||||
delete event.bufferChange
|
||||
expect(event).toEqual(start: 0, end: 2, delta: 0)
|
||||
|
||||
it "updates tokens for lines beyond the changed lines if needed", ->
|
||||
buffer.insert([5, 30], '/* */')
|
||||
changeHandler.reset()
|
||||
it "updates tokens for lines beyond the changed lines if needed", ->
|
||||
buffer.insert([5, 30], '/* */')
|
||||
changeHandler.reset()
|
||||
|
||||
buffer.insert([2, 0], '/*')
|
||||
expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
|
||||
expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
|
||||
expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
|
||||
buffer.insert([2, 0], '/*')
|
||||
expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
|
||||
expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
|
||||
expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
|
||||
|
||||
expect(changeHandler).toHaveBeenCalled()
|
||||
[event] = changeHandler.argsForCall[0]
|
||||
delete event.bufferChange
|
||||
expect(event).toEqual(start: 2, end: 5, delta: 0)
|
||||
expect(changeHandler).toHaveBeenCalled()
|
||||
[event] = changeHandler.argsForCall[0]
|
||||
delete event.bufferChange
|
||||
expect(event).toEqual(start: 2, end: 5, delta: 0)
|
||||
|
||||
it "resumes highlighting with the state of the previous line", ->
|
||||
buffer.insert([0, 0], '/*')
|
||||
buffer.insert([5, 0], '*/')
|
||||
it "resumes highlighting with the state of the previous line", ->
|
||||
buffer.insert([0, 0], '/*')
|
||||
buffer.insert([5, 0], '*/')
|
||||
|
||||
buffer.insert([1, 0], 'var ')
|
||||
expect(tokenizedBuffer.lineForScreenRow(1).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
|
||||
buffer.insert([1, 0], 'var ')
|
||||
expect(tokenizedBuffer.lineForScreenRow(1).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
|
||||
|
||||
describe "when lines are both updated and removed", ->
|
||||
it "updates tokens to reflect the removed lines", ->
|
||||
range = new Range([1, 0], [3, 0])
|
||||
buffer.change(range, "foo()")
|
||||
describe "when lines are both updated and removed", ->
|
||||
it "updates tokens to reflect the removed lines", ->
|
||||
range = new Range([1, 0], [3, 0])
|
||||
buffer.change(range, "foo()")
|
||||
|
||||
# previous line 0 remains
|
||||
expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual(value: 'var', scopes: ['source.js', 'storage.modifier.js'])
|
||||
# previous line 0 remains
|
||||
expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual(value: 'var', scopes: ['source.js', 'storage.modifier.js'])
|
||||
|
||||
# previous line 3 should be combined with input to form line 1
|
||||
expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: 'foo', scopes: ['source.js'])
|
||||
expect(tokenizedBuffer.lineForScreenRow(1).tokens[6]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js'])
|
||||
# previous line 3 should be combined with input to form line 1
|
||||
expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: 'foo', scopes: ['source.js'])
|
||||
expect(tokenizedBuffer.lineForScreenRow(1).tokens[6]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js'])
|
||||
|
||||
# lines below deleted regions should be shifted upward
|
||||
expect(tokenizedBuffer.lineForScreenRow(2).tokens[2]).toEqual(value: 'while', scopes: ['source.js', 'keyword.control.js'])
|
||||
expect(tokenizedBuffer.lineForScreenRow(3).tokens[4]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js'])
|
||||
expect(tokenizedBuffer.lineForScreenRow(4).tokens[4]).toEqual(value: '<', scopes: ['source.js', 'keyword.operator.js'])
|
||||
# lines below deleted regions should be shifted upward
|
||||
expect(tokenizedBuffer.lineForScreenRow(2).tokens[2]).toEqual(value: 'while', scopes: ['source.js', 'keyword.control.js'])
|
||||
expect(tokenizedBuffer.lineForScreenRow(3).tokens[4]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js'])
|
||||
expect(tokenizedBuffer.lineForScreenRow(4).tokens[4]).toEqual(value: '<', scopes: ['source.js', 'keyword.operator.js'])
|
||||
|
||||
expect(changeHandler).toHaveBeenCalled()
|
||||
[event] = changeHandler.argsForCall[0]
|
||||
delete event.bufferChange
|
||||
expect(event).toEqual(start: 1, end: 3, delta: -2)
|
||||
expect(changeHandler).toHaveBeenCalled()
|
||||
[event] = changeHandler.argsForCall[0]
|
||||
delete event.bufferChange
|
||||
expect(event).toEqual(start: 1, end: 3, delta: -2)
|
||||
|
||||
it "updates tokens for lines beyond the changed lines if needed", ->
|
||||
buffer.insert([5, 30], '/* */')
|
||||
changeHandler.reset()
|
||||
it "updates tokens for lines beyond the changed lines if needed", ->
|
||||
buffer.insert([5, 30], '/* */')
|
||||
changeHandler.reset()
|
||||
|
||||
buffer.change(new Range([2, 0], [3, 0]), '/*')
|
||||
expect(tokenizedBuffer.lineForScreenRow(2).tokens[0].scopes).toEqual ['source.js', 'comment.block.js', 'punctuation.definition.comment.js']
|
||||
expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
|
||||
expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
|
||||
buffer.change(new Range([2, 0], [3, 0]), '/*')
|
||||
expect(tokenizedBuffer.lineForScreenRow(2).tokens[0].scopes).toEqual ['source.js', 'comment.block.js', 'punctuation.definition.comment.js']
|
||||
expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
|
||||
expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
|
||||
|
||||
expect(changeHandler).toHaveBeenCalled()
|
||||
[event] = changeHandler.argsForCall[0]
|
||||
delete event.bufferChange
|
||||
expect(event).toEqual(start: 2, end: 5, delta: -1)
|
||||
expect(changeHandler).toHaveBeenCalled()
|
||||
[event] = changeHandler.argsForCall[0]
|
||||
delete event.bufferChange
|
||||
expect(event).toEqual(start: 2, end: 5, delta: -1)
|
||||
|
||||
describe "when lines are both updated and inserted", ->
|
||||
it "updates tokens to reflect the inserted lines", ->
|
||||
range = new Range([1, 0], [2, 0])
|
||||
buffer.change(range, "foo()\nbar()\nbaz()\nquux()")
|
||||
describe "when lines are both updated and inserted", ->
|
||||
it "updates tokens to reflect the inserted lines", ->
|
||||
range = new Range([1, 0], [2, 0])
|
||||
buffer.change(range, "foo()\nbar()\nbaz()\nquux()")
|
||||
|
||||
# previous line 0 remains
|
||||
expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual( value: 'var', scopes: ['source.js', 'storage.modifier.js'])
|
||||
# previous line 0 remains
|
||||
expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual( value: 'var', scopes: ['source.js', 'storage.modifier.js'])
|
||||
|
||||
# 3 new lines inserted
|
||||
expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: 'foo', scopes: ['source.js'])
|
||||
expect(tokenizedBuffer.lineForScreenRow(2).tokens[0]).toEqual(value: 'bar', scopes: ['source.js'])
|
||||
expect(tokenizedBuffer.lineForScreenRow(3).tokens[0]).toEqual(value: 'baz', scopes: ['source.js'])
|
||||
# 3 new lines inserted
|
||||
expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: 'foo', scopes: ['source.js'])
|
||||
expect(tokenizedBuffer.lineForScreenRow(2).tokens[0]).toEqual(value: 'bar', scopes: ['source.js'])
|
||||
expect(tokenizedBuffer.lineForScreenRow(3).tokens[0]).toEqual(value: 'baz', scopes: ['source.js'])
|
||||
|
||||
# previous line 2 is joined with quux() on line 4
|
||||
expect(tokenizedBuffer.lineForScreenRow(4).tokens[0]).toEqual(value: 'quux', scopes: ['source.js'])
|
||||
expect(tokenizedBuffer.lineForScreenRow(4).tokens[4]).toEqual(value: 'if', scopes: ['source.js', 'keyword.control.js'])
|
||||
# previous line 2 is joined with quux() on line 4
|
||||
expect(tokenizedBuffer.lineForScreenRow(4).tokens[0]).toEqual(value: 'quux', scopes: ['source.js'])
|
||||
expect(tokenizedBuffer.lineForScreenRow(4).tokens[4]).toEqual(value: 'if', scopes: ['source.js', 'keyword.control.js'])
|
||||
|
||||
# previous line 3 is pushed down to become line 5
|
||||
expect(tokenizedBuffer.lineForScreenRow(5).tokens[4]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js'])
|
||||
# previous line 3 is pushed down to become line 5
|
||||
expect(tokenizedBuffer.lineForScreenRow(5).tokens[4]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js'])
|
||||
|
||||
expect(changeHandler).toHaveBeenCalled()
|
||||
[event] = changeHandler.argsForCall[0]
|
||||
delete event.bufferChange
|
||||
expect(event).toEqual(start: 1, end: 2, delta: 2)
|
||||
expect(changeHandler).toHaveBeenCalled()
|
||||
[event] = changeHandler.argsForCall[0]
|
||||
delete event.bufferChange
|
||||
expect(event).toEqual(start: 1, end: 2, delta: 2)
|
||||
|
||||
it "updates tokens for lines beyond the changed lines if needed", ->
|
||||
buffer.insert([5, 30], '/* */')
|
||||
changeHandler.reset()
|
||||
it "updates tokens for lines beyond the changed lines if needed", ->
|
||||
buffer.insert([5, 30], '/* */')
|
||||
changeHandler.reset()
|
||||
|
||||
buffer.insert([2, 0], '/*\nabcde\nabcder')
|
||||
expect(tokenizedBuffer.lineForScreenRow(2).tokens[0].scopes).toEqual ['source.js', 'comment.block.js', 'punctuation.definition.comment.js']
|
||||
expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
|
||||
expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
|
||||
expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
|
||||
expect(tokenizedBuffer.lineForScreenRow(6).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
|
||||
expect(tokenizedBuffer.lineForScreenRow(7).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
|
||||
expect(tokenizedBuffer.lineForScreenRow(8).tokens[0].scopes).not.toBe ['source.js', 'comment.block.js']
|
||||
buffer.insert([2, 0], '/*\nabcde\nabcder')
|
||||
expect(tokenizedBuffer.lineForScreenRow(2).tokens[0].scopes).toEqual ['source.js', 'comment.block.js', 'punctuation.definition.comment.js']
|
||||
expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
|
||||
expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
|
||||
expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
|
||||
expect(tokenizedBuffer.lineForScreenRow(6).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
|
||||
expect(tokenizedBuffer.lineForScreenRow(7).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
|
||||
expect(tokenizedBuffer.lineForScreenRow(8).tokens[0].scopes).not.toBe ['source.js', 'comment.block.js']
|
||||
|
||||
expect(changeHandler).toHaveBeenCalled()
|
||||
[event] = changeHandler.argsForCall[0]
|
||||
delete event.bufferChange
|
||||
expect(event).toEqual(start: 2, end: 5, delta: 2)
|
||||
expect(changeHandler).toHaveBeenCalled()
|
||||
[event] = changeHandler.argsForCall[0]
|
||||
delete event.bufferChange
|
||||
expect(event).toEqual(start: 2, end: 5, delta: 2)
|
||||
|
||||
describe "when the change causes more than @chunkSize lines to be retokenized", ->
|
||||
describe "when the buffer change itself is larger than the chunk size", ->
|
||||
|
||||
describe "when the buffer change is smaller than the chunk size, but the scope of rehighlighting is larger", ->
|
||||
|
||||
describe "when the end of rehighlighted region is outside the tokenized region", ->
|
||||
it "extends the boundary of the tokenized region and resumes tokenizing after it", ->
|
||||
|
||||
|
||||
describe "when there is a buffer change in the untokenized region", ->
|
||||
it "updates the untokenized screen lines to match the buffer but does not tokenize the lines", ->
|
||||
|
||||
|
||||
describe "when the buffer contains tab characters", ->
|
||||
|
||||
Loading…
Reference in New Issue
Block a user