mirror of
https://github.com/pulsar-edit/pulsar.git
synced 2024-09-21 07:58:04 +03:00
Merge feature/modernize-tree-sitter
This commit is contained in:
commit
8637a75b19
43
packages/language-clojure/grammars/tree-sitter-clojure.cson
Normal file
43
packages/language-clojure/grammars/tree-sitter-clojure.cson
Normal file
@ -0,0 +1,43 @@
|
||||
'name': 'Clojure'
|
||||
'scopeName': 'source.clojure'
|
||||
'type': 'tree-sitter-2'
|
||||
'fileTypes': [
|
||||
'boot'
|
||||
'clj'
|
||||
'clj.hl'
|
||||
'cljc'
|
||||
'cljs'
|
||||
'cljs.hl'
|
||||
'cljx'
|
||||
'clojure'
|
||||
'edn'
|
||||
'org'
|
||||
'bb'
|
||||
'joke'
|
||||
'joker'
|
||||
]
|
||||
'firstLineMatch': '''(?x)
|
||||
# Hashbang
|
||||
^\\#!.*(?:\\s|\\/)
|
||||
boot
|
||||
(?:$|\\s)
|
||||
|
|
||||
# Modeline
|
||||
(?i:
|
||||
# Emacs
|
||||
-\\*-(?:\\s*(?=[^:;\\s]+\\s*-\\*-)|(?:.*?[;\\s]|(?<=-\\*-))mode\\s*:\\s*)
|
||||
clojure(script)?
|
||||
(?=[\\s;]|(?<![-*])-\\*-).*?-\\*-
|
||||
|
|
||||
# Vim
|
||||
(?:(?:\\s|^)vi(?:m[<=>]?\\d+|m)?|\\sex)(?=:(?=\\s*set?\\s[^\\n:]+:)|:(?!\\s*set?\\s))(?:(?:\\s|\\s*:\\s*)\\w*(?:\\s*=(?:[^\\n\\\\\\s]|\\\\.)*)?)*[\\s:](?:filetype|ft|syntax)\\s*=
|
||||
clojure
|
||||
(?=\\s|:|$)
|
||||
)
|
||||
'''
|
||||
treeSitter:
|
||||
grammar: 'ts/grammar.wasm'
|
||||
syntaxQuery: 'ts/highlights.scm'
|
||||
# localsQuery: 'ts/locals.scm'
|
||||
foldsQuery: 'ts/folds.scm'
|
||||
indentsQuery: 'ts/indents.scm'
|
7
packages/language-clojure/grammars/ts/folds.scm
Normal file
7
packages/language-clojure/grammars/ts/folds.scm
Normal file
@ -0,0 +1,7 @@
|
||||
[
|
||||
(list_lit)
|
||||
(vec_lit)
|
||||
(map_lit)
|
||||
(anon_fn_lit)
|
||||
(set_lit)
|
||||
] @fold
|
BIN
packages/language-clojure/grammars/ts/grammar.wasm
Executable file
BIN
packages/language-clojure/grammars/ts/grammar.wasm
Executable file
Binary file not shown.
58
packages/language-clojure/grammars/ts/highlights.scm
Normal file
58
packages/language-clojure/grammars/ts/highlights.scm
Normal file
@ -0,0 +1,58 @@
|
||||
(quoting_lit
|
||||
value: (list_lit
|
||||
"(" @punctuation.section.expression.begin
|
||||
.
|
||||
(sym_lit) @meta.symbol (#set! final "true")))
|
||||
|
||||
;; Collections
|
||||
(anon_fn_lit
|
||||
"(" @punctuation.section.expression.begin
|
||||
.
|
||||
(sym_lit) @entity.name.function @meta.expression
|
||||
")" @punctuation.section.expression.end)
|
||||
|
||||
(list_lit
|
||||
"(" @punctuation.section.expression.begin
|
||||
.
|
||||
(sym_lit) @entity.name.function @meta.expression
|
||||
")" @punctuation.section.expression.end)
|
||||
|
||||
(vec_lit
|
||||
"[" @punctuation.section.vector.begin
|
||||
"]" @punctuation.section.vector.end) @meta.vector
|
||||
|
||||
(map_lit
|
||||
"{" @punctuation.section.map.begin
|
||||
"}" @punctuation.section.map.end) @meta.map
|
||||
|
||||
(set_lit
|
||||
("#" "{") @punctuation.section.set.begin
|
||||
"}" @punctuation.section.set.end) @meta.map
|
||||
|
||||
; Includes
|
||||
((sym_name) @meta.symbol (#eq? @meta.symbol "import")) @keyword.control
|
||||
((sym_name) @meta.symbol (#eq? @meta.symbol "require")) @keyword.control
|
||||
((sym_name) @meta.symbol (#eq? @meta.symbol "use")) @keyword.control
|
||||
|
||||
(list_lit
|
||||
"(" @punctuation.section.expression.begin
|
||||
.
|
||||
((sym_lit) @meta.symbol (#eq? @meta.symbol "ns")) @keyword.control @meta.definition.global
|
||||
.
|
||||
(sym_lit) @meta.definition.global @entity.global
|
||||
")" @punctuation.section.expression.end)
|
||||
|
||||
(list_lit
|
||||
"(" @punctuation.section.expression.begin
|
||||
.
|
||||
((sym_lit) @meta.symbol (#match? @meta.symbol "^def")) @keyword.control
|
||||
.
|
||||
(sym_lit) @meta.definition.global @entity.global
|
||||
")" @punctuation.section.expression.end)
|
||||
|
||||
(sym_lit) @meta.symbol
|
||||
(kwd_lit) @constant.keyword
|
||||
(str_lit) @string.quoted.double
|
||||
(num_lit) @constant.numeric
|
||||
(comment) @comment.line.semicolon
|
||||
(dis_expr) @comment.block.clojure
|
0
packages/language-clojure/grammars/ts/indents.scm
Normal file
0
packages/language-clojure/grammars/ts/indents.scm
Normal file
34
packages/language-clojure/spec/fixtures/tokens.clj
vendored
Normal file
34
packages/language-clojure/spec/fixtures/tokens.clj
vendored
Normal file
@ -0,0 +1,34 @@
|
||||
(ns foobar)
|
||||
; <- punctuation.section.expression.begin
|
||||
; ^ meta.definition.global
|
||||
; ^ entity.global
|
||||
; ^ punctuation.section.expression.end
|
||||
|
||||
(defn foobar [a b]
|
||||
; <- keyword.control
|
||||
; ^ entity.global
|
||||
; ^ meta.definition.global
|
||||
; ^ punctuation.section.vector.begin
|
||||
; ^ meta.vector
|
||||
; ^ meta.symbol
|
||||
; ^ punctuation.section.vector.end
|
||||
(+ a b 10 20))
|
||||
;^ meta.expression
|
||||
;^ entity.name.function
|
||||
; ^ constant.numeric
|
||||
|
||||
(def a "A STRING")
|
||||
; <- keyword.control
|
||||
; ^ entity.global
|
||||
; ^ string.quoted.double
|
||||
|
||||
#{'asd}
|
||||
; <- punctuation.section.set.begin
|
||||
; ^ meta.symbol
|
||||
; ^ punctuation.section.set.end
|
||||
|
||||
{:key "value"}
|
||||
; <- punctuation.section.map.begin
|
||||
; ^ constant.keyword
|
||||
; ^ meta.map
|
||||
; ^ punctuation.section.map.end
|
34
packages/language-clojure/spec/fixtures/tree-sitter-folds.clj
vendored
Normal file
34
packages/language-clojure/spec/fixtures/tree-sitter-folds.clj
vendored
Normal file
@ -0,0 +1,34 @@
|
||||
(defn a [a b]
|
||||
; <- fold_begin.paren
|
||||
; ^ fold_new_position.paren
|
||||
(+ a b)
|
||||
[1
|
||||
; <- fold_begin.vector
|
||||
; ^ fold_new_position.vector
|
||||
2
|
||||
3]
|
||||
; ^ fold_end.vector
|
||||
{:a 10
|
||||
; <- fold_begin.map
|
||||
; ^ fold_new_position.map
|
||||
:b 20
|
||||
:c [1
|
||||
; ^ fold_begin.inner_vector
|
||||
; ^ fold_new_position.inner_vector
|
||||
2
|
||||
3]})
|
||||
; ^ fold_end.paren
|
||||
; ^ fold_end.map
|
||||
; ^ fold_end.inner_vector
|
||||
|
||||
#(inner
|
||||
; <- fold_begin.anon
|
||||
; ^ fold_new_position.anon
|
||||
"function"
|
||||
#{:with
|
||||
; <- fold_begin.set
|
||||
; ^ fold_new_position.set
|
||||
:inner
|
||||
:set})
|
||||
; ^ fold_end.anon
|
||||
; ^ fold_end.set
|
6
packages/language-clojure/spec/fixtures/tree-sitter-tokens.clj
vendored
Normal file
6
packages/language-clojure/spec/fixtures/tree-sitter-tokens.clj
vendored
Normal file
@ -0,0 +1,6 @@
|
||||
#_
|
||||
(+ 1 2)
|
||||
; <- comment.block
|
||||
|
||||
'(a b 1 2)
|
||||
; ^ !entity.name.function
|
28
packages/language-clojure/spec/tokenizer-spec.js
Normal file
28
packages/language-clojure/spec/tokenizer-spec.js
Normal file
@ -0,0 +1,28 @@
|
||||
const path = require('path');
|
||||
|
||||
describe('Clojure grammars', () => {
|
||||
|
||||
beforeEach(async () => {
|
||||
await atom.packages.activatePackage('language-clojure');
|
||||
});
|
||||
|
||||
it('tokenizes the editor using TextMate parser', async () => {
|
||||
atom.config.set('core.languageParser', 'textmate');
|
||||
await runGrammarTests(path.join(__dirname, 'fixtures', 'tokens.clj'), /;/)
|
||||
});
|
||||
|
||||
it('tokenizes the editor using node tree-sitter parser the same as TextMate', async () => {
|
||||
atom.config.set('core.languageParser', 'wasm-tree-sitter');
|
||||
await runGrammarTests(path.join(__dirname, 'fixtures', 'tokens.clj'), /;/)
|
||||
});
|
||||
|
||||
it('tokenizes the editor using node tree-sitter parser (specific rules)', async () => {
|
||||
atom.config.set('core.languageParser', 'wasm-tree-sitter');
|
||||
await runGrammarTests(path.join(__dirname, 'fixtures', 'tree-sitter-tokens.clj'), /;/)
|
||||
});
|
||||
|
||||
it('folds Clojure code', async () => {
|
||||
atom.config.set('core.languageParser', 'wasm-tree-sitter');
|
||||
await runFoldsTests(path.join(__dirname, 'fixtures', 'tree-sitter-folds.clj'), /;/)
|
||||
});
|
||||
});
|
@ -23,55 +23,8 @@ describe('WASM Tree-sitter Ruby grammar', () => {
|
||||
});
|
||||
|
||||
it('folds code', async () => {
|
||||
const editor = await openDocument('folds.rb');
|
||||
let grouped = {}
|
||||
normalized = normalizeTreeSitterTextData(editor, /#/).forEach(test => {
|
||||
const [kind, id] = test.expected.split('.')
|
||||
if(!kind || !id) {
|
||||
throw new Error(dedent`Folds must be in the format fold_end.some-id
|
||||
at ${test.testPosition.row+1}:${test.testPosition.column+1}`)
|
||||
}
|
||||
grouped[id] ||= {}
|
||||
grouped[id][kind] = test
|
||||
})
|
||||
for(const k in grouped) {
|
||||
const v = grouped[k]
|
||||
const keys = Object.keys(v)
|
||||
if(keys.indexOf('fold_begin') === -1)
|
||||
throw new Error(`Fold ${k} must contain fold_begin`)
|
||||
if(keys.indexOf('fold_end') === -1)
|
||||
throw new Error(`Fold ${k} must contain fold_end`)
|
||||
if(keys.indexOf('fold_new_position') === -1)
|
||||
throw new Error(`Fold ${k} must contain fold_new_position`)
|
||||
}
|
||||
|
||||
for(const k in grouped) {
|
||||
const fold = grouped[k]
|
||||
const begin = fold['fold_begin']
|
||||
const end = fold['fold_end']
|
||||
const newPos = fold['fold_new_position']
|
||||
|
||||
expect(editor.isFoldableAtBufferRow(begin.editorPosition.row))
|
||||
.toSatisfy((foldable, reason) => {
|
||||
reason(dedent`Editor is not foldable at row ${begin.editorPosition.row+1}
|
||||
at fixtures/folds.rb:${begin.testPosition.row+1}:${begin.testPosition.column+1}`)
|
||||
return foldable
|
||||
})
|
||||
editor.foldBufferRow(begin.editorPosition.row)
|
||||
|
||||
expect(editor.screenPositionForBufferPosition(end.editorPosition))
|
||||
.toSatisfy((screenPosition, reason) => {
|
||||
const {row,column} = newPos.editorPosition
|
||||
reason(`At row ${begin.editorPosition.row+1}, editor should fold ` +
|
||||
`up to the ${end.editorPosition.row+1}:${end.editorPosition.column+1}\n` +
|
||||
` into the new position ${row+1}:${column+1}\n`+
|
||||
` but folded to position ${screenPosition.row+1}:${screenPosition.column+1}\n`+
|
||||
` at fixtures/folds.rb:${newPos.testPosition.row+1}:${newPos.testPosition.column+1}\n` +
|
||||
` at fixtures/folds.rb:${end.testPosition.row+1}:${end.testPosition.column+1}`)
|
||||
return row === screenPosition.row && column === screenPosition.column
|
||||
})
|
||||
editor.unfoldAll()
|
||||
}
|
||||
const fullPath = path.join(__dirname, 'fixtures', 'folds.rb')
|
||||
await runFoldsTests(fullPath, /#/)
|
||||
});
|
||||
});
|
||||
|
||||
@ -81,36 +34,3 @@ async function openDocument(fileName) {
|
||||
await editor.languageMode.ready
|
||||
return editor
|
||||
}
|
||||
|
||||
// function normalizeTestData(editor, commentRegex) {
|
||||
// let allMatches = [], lastNonComment = 0
|
||||
// editor.getBuffer().getLines().forEach((row, i) => {
|
||||
// const m = row.match(commentRegex)
|
||||
// if(m) {
|
||||
// const scope = editor.scopeDescriptorForBufferPosition([i, m.index])
|
||||
// if(scope.scopes.find(s => s.match(/comment/))) {
|
||||
// allMatches.push({row: lastNonComment, text: row, col: m.index, testRow: i})
|
||||
// return
|
||||
// }
|
||||
// }
|
||||
// lastNonComment = i
|
||||
// })
|
||||
// return allMatches.map(({text, row, col, testRow}) => {
|
||||
// const exactPos = text.match(/\^\s+(.*)/)
|
||||
// if(exactPos) {
|
||||
// const expected = exactPos[1]
|
||||
// return {
|
||||
// expected,
|
||||
// editorPosition: {row, column: exactPos.index},
|
||||
// testPosition: {row: testRow, column: col}
|
||||
// }
|
||||
// } else {
|
||||
// const pos = text.match(/\<-\s+(.*)/)
|
||||
// return {
|
||||
// expected: pos[1],
|
||||
// editorPosition: {row, column: col},
|
||||
// testPosition: {row: testRow, column: col}
|
||||
// }
|
||||
// }
|
||||
// })
|
||||
// }
|
||||
|
@ -153,7 +153,7 @@ class WASMTreeSitterLanguageMode {
|
||||
.then(language => {
|
||||
this.rootLanguage = language;
|
||||
this.rootLanguageLayer = new LanguageLayer(null, this, grammar, 0);
|
||||
this.getOrCreateParserForLanguage(language);
|
||||
return this.getOrCreateParserForLanguage(language);
|
||||
})
|
||||
.then(() => this.rootLanguageLayer.update(null))
|
||||
.then(() => this.emitter.emit('did-tokenize'));
|
||||
@ -221,7 +221,6 @@ class WASMTreeSitterLanguageMode {
|
||||
|
||||
bufferDidFinishTransaction({ changes }) {
|
||||
if (!this.rootLanguageLayer) { return; }
|
||||
|
||||
for (let i = 0, { length } = changes; i < length; i++) {
|
||||
const { oldRange, newRange } = changes[i];
|
||||
spliceArray(
|
||||
|
82
vendor/jasmine.js
vendored
82
vendor/jasmine.js
vendored
@ -2718,9 +2718,9 @@ jasmine.Matchers.prototype.toSatisfy = function(fn) {
|
||||
// to construct an error showing where EXACTLY was the assertion that failed
|
||||
function normalizeTreeSitterTextData(editor, commentRegex) {
|
||||
let allMatches = [], lastNonComment = 0
|
||||
const checkAssert = new RegExp('^' + commentRegex.source + '\\s*[\\<\\-|\\^]')
|
||||
const checkAssert = new RegExp('^\\s*' + commentRegex.source + '\\s*[\\<\\-|\\^]')
|
||||
editor.getBuffer().getLines().forEach((row, i) => {
|
||||
const m = row.trim().match(commentRegex)
|
||||
const m = row.match(commentRegex)
|
||||
if(m) {
|
||||
// const scope = editor.scopeDescriptorForBufferPosition([i, m.index])
|
||||
// FIXME: use editor.scopeDescriptorForBufferPosition when it works
|
||||
@ -2757,8 +2757,7 @@ if (isCommonJS) exports.normalizeTreeSitterTextData = normalizeTreeSitterTextDat
|
||||
|
||||
async function openDocument(fullPath) {
|
||||
const editor = await atom.workspace.open(fullPath);
|
||||
const mode = editor.languageMode;
|
||||
await mode.ready;
|
||||
await editor.languageMode.ready;
|
||||
return editor;
|
||||
}
|
||||
|
||||
@ -2772,13 +2771,76 @@ async function runGrammarTests(fullPath, commentRegex) {
|
||||
})
|
||||
normalized.forEach(({expected, editorPosition, testPosition}) => {
|
||||
expect(editor.scopeDescriptorForBufferPosition(editorPosition).scopes).toSatisfy((scopes, reason) => {
|
||||
reason(`Expected to find scope "${expected}" but found "${scopes}"\n` +
|
||||
` at ${fullPath}:${testPosition.row+1}:${testPosition.column+1}`
|
||||
);
|
||||
const normalized = expected.replace(/([\.\-])/g, '\\$1')
|
||||
const scopeRegex = new RegExp('^' + normalized + '(\\..+)?$')
|
||||
return scopes.find(e => e.match(scopeRegex)) !== undefined;
|
||||
const dontFindScope = expected.startsWith("!");
|
||||
expected = expected.replace(/^!/, "")
|
||||
if(dontFindScope) {
|
||||
reason(`Expected to NOT find scope "${expected}" but found it\n` +
|
||||
` at ${fullPath}:${testPosition.row+1}:${testPosition.column+1}`
|
||||
);
|
||||
} else {
|
||||
reason(`Expected to find scope "${expected}" but found "${scopes}"\n` +
|
||||
` at ${fullPath}:${testPosition.row+1}:${testPosition.column+1}`
|
||||
);
|
||||
}
|
||||
const normalized = expected.replace(/([\.\-])/g, '\\$1');
|
||||
const scopeRegex = new RegExp('^' + normalized + '(\\..+)?$');
|
||||
let result = scopes.find(e => e.match(scopeRegex)) !== undefined;
|
||||
if(dontFindScope) result = !result;
|
||||
return result
|
||||
})
|
||||
})
|
||||
}
|
||||
if (isCommonJS) exports.runGrammarTests = runGrammarTests;
|
||||
|
||||
async function runFoldsTests(fullPath, commentRegex) {
|
||||
const editor = await openDocument(fullPath);
|
||||
let grouped = {}
|
||||
const normalized = normalizeTreeSitterTextData(editor, commentRegex).forEach(test => {
|
||||
const [kind, id] = test.expected.split('.')
|
||||
if(!kind || !id) {
|
||||
throw new Error(`Folds must be in the format fold_end.some-id\n` +
|
||||
` at ${test.testPosition.row+1}:${test.testPosition.column+1}`)
|
||||
}
|
||||
grouped[id] ||= {}
|
||||
grouped[id][kind] = test
|
||||
})
|
||||
for(const k in grouped) {
|
||||
const v = grouped[k]
|
||||
const keys = Object.keys(v)
|
||||
if(keys.indexOf('fold_begin') === -1)
|
||||
throw new Error(`Fold ${k} must contain fold_begin`)
|
||||
if(keys.indexOf('fold_end') === -1)
|
||||
throw new Error(`Fold ${k} must contain fold_end`)
|
||||
if(keys.indexOf('fold_new_position') === -1)
|
||||
throw new Error(`Fold ${k} must contain fold_new_position`)
|
||||
}
|
||||
|
||||
for(const k in grouped) {
|
||||
const fold = grouped[k]
|
||||
const begin = fold['fold_begin']
|
||||
const end = fold['fold_end']
|
||||
const newPos = fold['fold_new_position']
|
||||
|
||||
expect(editor.isFoldableAtBufferRow(begin.editorPosition.row))
|
||||
.toSatisfy((foldable, reason) => {
|
||||
reason(`Editor is not foldable at row ${begin.editorPosition.row+1}\n` +
|
||||
` at ${fullPath}:${begin.testPosition.row+1}:${begin.testPosition.column+1}`)
|
||||
return foldable
|
||||
})
|
||||
editor.foldBufferRow(begin.editorPosition.row)
|
||||
|
||||
expect(editor.screenPositionForBufferPosition(end.editorPosition))
|
||||
.toSatisfy((screenPosition, reason) => {
|
||||
const {row,column} = newPos.editorPosition
|
||||
reason(`At row ${begin.editorPosition.row+1}, editor should fold ` +
|
||||
`up to the ${end.editorPosition.row+1}:${end.editorPosition.column+1}\n` +
|
||||
` into the new position ${row+1}:${column+1}\n`+
|
||||
` but folded to position ${screenPosition.row+1}:${screenPosition.column+1}\n`+
|
||||
` at ${fullPath}:${newPos.testPosition.row+1}:${newPos.testPosition.column+1}\n` +
|
||||
` at ${fullPath}:${end.testPosition.row+1}:${end.testPosition.column+1}`)
|
||||
return row === screenPosition.row && column === screenPosition.column
|
||||
})
|
||||
editor.unfoldAll()
|
||||
}
|
||||
}
|
||||
if (isCommonJS) exports.runFoldsTests = runFoldsTests;
|
||||
|
Loading…
Reference in New Issue
Block a user