diff --git a/packages/language-clojure/grammars/tree-sitter-clojure.cson b/packages/language-clojure/grammars/tree-sitter-clojure.cson new file mode 100644 index 000000000..e6bb9771b --- /dev/null +++ b/packages/language-clojure/grammars/tree-sitter-clojure.cson @@ -0,0 +1,43 @@ +'name': 'Clojure' +'scopeName': 'source.clojure' +'type': 'tree-sitter-2' +'fileTypes': [ + 'boot' + 'clj' + 'clj.hl' + 'cljc' + 'cljs' + 'cljs.hl' + 'cljx' + 'clojure' + 'edn' + 'org' + 'bb' + 'joke' + 'joker' +] +'firstLineMatch': '''(?x) + # Hashbang + ^\\#!.*(?:\\s|\\/) + boot + (?:$|\\s) + | + # Modeline + (?i: + # Emacs + -\\*-(?:\\s*(?=[^:;\\s]+\\s*-\\*-)|(?:.*?[;\\s]|(?<=-\\*-))mode\\s*:\\s*) + clojure(script)? + (?=[\\s;]|(?]?\\d+|m)?|\\sex)(?=:(?=\\s*set?\\s[^\\n:]+:)|:(?!\\s*set?\\s))(?:(?:\\s|\\s*:\\s*)\\w*(?:\\s*=(?:[^\\n\\\\\\s]|\\\\.)*)?)*[\\s:](?:filetype|ft|syntax)\\s*= + clojure + (?=\\s|:|$) + ) +''' +treeSitter: + grammar: 'ts/grammar.wasm' + syntaxQuery: 'ts/highlights.scm' + # localsQuery: 'ts/locals.scm' + foldsQuery: 'ts/folds.scm' + indentsQuery: 'ts/indents.scm' diff --git a/packages/language-clojure/grammars/ts/folds.scm b/packages/language-clojure/grammars/ts/folds.scm new file mode 100644 index 000000000..bd70d34b4 --- /dev/null +++ b/packages/language-clojure/grammars/ts/folds.scm @@ -0,0 +1,7 @@ +[ + (list_lit) + (vec_lit) + (map_lit) + (anon_fn_lit) + (set_lit) +] @fold diff --git a/packages/language-clojure/grammars/ts/grammar.wasm b/packages/language-clojure/grammars/ts/grammar.wasm new file mode 100755 index 000000000..e50647edd Binary files /dev/null and b/packages/language-clojure/grammars/ts/grammar.wasm differ diff --git a/packages/language-clojure/grammars/ts/highlights.scm b/packages/language-clojure/grammars/ts/highlights.scm new file mode 100644 index 000000000..ac54f2f42 --- /dev/null +++ b/packages/language-clojure/grammars/ts/highlights.scm @@ -0,0 +1,58 @@ +(quoting_lit + value: (list_lit + "(" @punctuation.section.expression.begin + . + (sym_lit) @meta.symbol (#set! final "true"))) + +;; Collections +(anon_fn_lit + "(" @punctuation.section.expression.begin + . + (sym_lit) @entity.name.function @meta.expression + ")" @punctuation.section.expression.end) + +(list_lit + "(" @punctuation.section.expression.begin + . + (sym_lit) @entity.name.function @meta.expression + ")" @punctuation.section.expression.end) + +(vec_lit + "[" @punctuation.section.vector.begin + "]" @punctuation.section.vector.end) @meta.vector + +(map_lit + "{" @punctuation.section.map.begin + "}" @punctuation.section.map.end) @meta.map + +(set_lit + ("#" "{") @punctuation.section.set.begin + "}" @punctuation.section.set.end) @meta.map + +; Includes +((sym_name) @meta.symbol (#eq? @meta.symbol "import")) @keyword.control +((sym_name) @meta.symbol (#eq? @meta.symbol "require")) @keyword.control +((sym_name) @meta.symbol (#eq? @meta.symbol "use")) @keyword.control + +(list_lit + "(" @punctuation.section.expression.begin + . + ((sym_lit) @meta.symbol (#eq? @meta.symbol "ns")) @keyword.control @meta.definition.global + . + (sym_lit) @meta.definition.global @entity.global + ")" @punctuation.section.expression.end) + +(list_lit + "(" @punctuation.section.expression.begin + . + ((sym_lit) @meta.symbol (#match? @meta.symbol "^def")) @keyword.control + . + (sym_lit) @meta.definition.global @entity.global + ")" @punctuation.section.expression.end) + +(sym_lit) @meta.symbol +(kwd_lit) @constant.keyword +(str_lit) @string.quoted.double +(num_lit) @constant.numeric +(comment) @comment.line.semicolon +(dis_expr) @comment.block.clojure diff --git a/packages/language-clojure/grammars/ts/indents.scm b/packages/language-clojure/grammars/ts/indents.scm new file mode 100644 index 000000000..e69de29bb diff --git a/packages/language-clojure/spec/fixtures/tokens.clj b/packages/language-clojure/spec/fixtures/tokens.clj new file mode 100644 index 000000000..e9f69a385 --- /dev/null +++ b/packages/language-clojure/spec/fixtures/tokens.clj @@ -0,0 +1,34 @@ +(ns foobar) +; <- punctuation.section.expression.begin +; ^ meta.definition.global +; ^ entity.global +; ^ punctuation.section.expression.end + +(defn foobar [a b] + ; <- keyword.control + ; ^ entity.global + ; ^ meta.definition.global + ; ^ punctuation.section.vector.begin + ; ^ meta.vector + ; ^ meta.symbol + ; ^ punctuation.section.vector.end + (+ a b 10 20)) + ;^ meta.expression + ;^ entity.name.function + ; ^ constant.numeric + +(def a "A STRING") + ; <- keyword.control + ; ^ entity.global + ; ^ string.quoted.double + +#{'asd} +; <- punctuation.section.set.begin +; ^ meta.symbol +; ^ punctuation.section.set.end + +{:key "value"} +; <- punctuation.section.map.begin +; ^ constant.keyword +; ^ meta.map +; ^ punctuation.section.map.end diff --git a/packages/language-clojure/spec/fixtures/tree-sitter-folds.clj b/packages/language-clojure/spec/fixtures/tree-sitter-folds.clj new file mode 100644 index 000000000..83db030b0 --- /dev/null +++ b/packages/language-clojure/spec/fixtures/tree-sitter-folds.clj @@ -0,0 +1,34 @@ +(defn a [a b] +; <- fold_begin.paren +; ^ fold_new_position.paren + (+ a b) + [1 + ; <- fold_begin.vector + ; ^ fold_new_position.vector + 2 + 3] + ; ^ fold_end.vector + {:a 10 + ; <- fold_begin.map + ; ^ fold_new_position.map + :b 20 + :c [1 + ; ^ fold_begin.inner_vector + ; ^ fold_new_position.inner_vector + 2 + 3]}) +; ^ fold_end.paren +; ^ fold_end.map +; ^ fold_end.inner_vector + +#(inner +; <- fold_begin.anon + ; ^ fold_new_position.anon + "function" + #{:with + ; <- fold_begin.set + ; ^ fold_new_position.set + :inner + :set}) +; ^ fold_end.anon +; ^ fold_end.set diff --git a/packages/language-clojure/spec/fixtures/tree-sitter-tokens.clj b/packages/language-clojure/spec/fixtures/tree-sitter-tokens.clj new file mode 100644 index 000000000..771e37c1f --- /dev/null +++ b/packages/language-clojure/spec/fixtures/tree-sitter-tokens.clj @@ -0,0 +1,6 @@ +#_ +(+ 1 2) +; <- comment.block + +'(a b 1 2) +; ^ !entity.name.function diff --git a/packages/language-clojure/spec/tokenizer-spec.js b/packages/language-clojure/spec/tokenizer-spec.js new file mode 100644 index 000000000..4c3c845c8 --- /dev/null +++ b/packages/language-clojure/spec/tokenizer-spec.js @@ -0,0 +1,28 @@ +const path = require('path'); + +describe('Clojure grammars', () => { + + beforeEach(async () => { + await atom.packages.activatePackage('language-clojure'); + }); + + it('tokenizes the editor using TextMate parser', async () => { + atom.config.set('core.languageParser', 'textmate'); + await runGrammarTests(path.join(__dirname, 'fixtures', 'tokens.clj'), /;/) + }); + + it('tokenizes the editor using node tree-sitter parser the same as TextMate', async () => { + atom.config.set('core.languageParser', 'wasm-tree-sitter'); + await runGrammarTests(path.join(__dirname, 'fixtures', 'tokens.clj'), /;/) + }); + + it('tokenizes the editor using node tree-sitter parser (specific rules)', async () => { + atom.config.set('core.languageParser', 'wasm-tree-sitter'); + await runGrammarTests(path.join(__dirname, 'fixtures', 'tree-sitter-tokens.clj'), /;/) + }); + + it('folds Clojure code', async () => { + atom.config.set('core.languageParser', 'wasm-tree-sitter'); + await runFoldsTests(path.join(__dirname, 'fixtures', 'tree-sitter-folds.clj'), /;/) + }); +}); diff --git a/packages/language-ruby/spec/wasm-tree-sitter-spec.js b/packages/language-ruby/spec/wasm-tree-sitter-spec.js index 3ecfc52b1..58a982071 100644 --- a/packages/language-ruby/spec/wasm-tree-sitter-spec.js +++ b/packages/language-ruby/spec/wasm-tree-sitter-spec.js @@ -23,55 +23,8 @@ describe('WASM Tree-sitter Ruby grammar', () => { }); it('folds code', async () => { - const editor = await openDocument('folds.rb'); - let grouped = {} - normalized = normalizeTreeSitterTextData(editor, /#/).forEach(test => { - const [kind, id] = test.expected.split('.') - if(!kind || !id) { - throw new Error(dedent`Folds must be in the format fold_end.some-id - at ${test.testPosition.row+1}:${test.testPosition.column+1}`) - } - grouped[id] ||= {} - grouped[id][kind] = test - }) - for(const k in grouped) { - const v = grouped[k] - const keys = Object.keys(v) - if(keys.indexOf('fold_begin') === -1) - throw new Error(`Fold ${k} must contain fold_begin`) - if(keys.indexOf('fold_end') === -1) - throw new Error(`Fold ${k} must contain fold_end`) - if(keys.indexOf('fold_new_position') === -1) - throw new Error(`Fold ${k} must contain fold_new_position`) - } - - for(const k in grouped) { - const fold = grouped[k] - const begin = fold['fold_begin'] - const end = fold['fold_end'] - const newPos = fold['fold_new_position'] - - expect(editor.isFoldableAtBufferRow(begin.editorPosition.row)) - .toSatisfy((foldable, reason) => { - reason(dedent`Editor is not foldable at row ${begin.editorPosition.row+1} - at fixtures/folds.rb:${begin.testPosition.row+1}:${begin.testPosition.column+1}`) - return foldable - }) - editor.foldBufferRow(begin.editorPosition.row) - - expect(editor.screenPositionForBufferPosition(end.editorPosition)) - .toSatisfy((screenPosition, reason) => { - const {row,column} = newPos.editorPosition - reason(`At row ${begin.editorPosition.row+1}, editor should fold ` + - `up to the ${end.editorPosition.row+1}:${end.editorPosition.column+1}\n` + - ` into the new position ${row+1}:${column+1}\n`+ - ` but folded to position ${screenPosition.row+1}:${screenPosition.column+1}\n`+ - ` at fixtures/folds.rb:${newPos.testPosition.row+1}:${newPos.testPosition.column+1}\n` + - ` at fixtures/folds.rb:${end.testPosition.row+1}:${end.testPosition.column+1}`) - return row === screenPosition.row && column === screenPosition.column - }) - editor.unfoldAll() - } + const fullPath = path.join(__dirname, 'fixtures', 'folds.rb') + await runFoldsTests(fullPath, /#/) }); }); @@ -81,36 +34,3 @@ async function openDocument(fileName) { await editor.languageMode.ready return editor } - -// function normalizeTestData(editor, commentRegex) { -// let allMatches = [], lastNonComment = 0 -// editor.getBuffer().getLines().forEach((row, i) => { -// const m = row.match(commentRegex) -// if(m) { -// const scope = editor.scopeDescriptorForBufferPosition([i, m.index]) -// if(scope.scopes.find(s => s.match(/comment/))) { -// allMatches.push({row: lastNonComment, text: row, col: m.index, testRow: i}) -// return -// } -// } -// lastNonComment = i -// }) -// return allMatches.map(({text, row, col, testRow}) => { -// const exactPos = text.match(/\^\s+(.*)/) -// if(exactPos) { -// const expected = exactPos[1] -// return { -// expected, -// editorPosition: {row, column: exactPos.index}, -// testPosition: {row: testRow, column: col} -// } -// } else { -// const pos = text.match(/\<-\s+(.*)/) -// return { -// expected: pos[1], -// editorPosition: {row, column: col}, -// testPosition: {row: testRow, column: col} -// } -// } -// }) -// } diff --git a/src/wasm-tree-sitter-language-mode.js b/src/wasm-tree-sitter-language-mode.js index e3ff620bd..b821822c0 100644 --- a/src/wasm-tree-sitter-language-mode.js +++ b/src/wasm-tree-sitter-language-mode.js @@ -153,7 +153,7 @@ class WASMTreeSitterLanguageMode { .then(language => { this.rootLanguage = language; this.rootLanguageLayer = new LanguageLayer(null, this, grammar, 0); - this.getOrCreateParserForLanguage(language); + return this.getOrCreateParserForLanguage(language); }) .then(() => this.rootLanguageLayer.update(null)) .then(() => this.emitter.emit('did-tokenize')); @@ -221,7 +221,6 @@ class WASMTreeSitterLanguageMode { bufferDidFinishTransaction({ changes }) { if (!this.rootLanguageLayer) { return; } - for (let i = 0, { length } = changes; i < length; i++) { const { oldRange, newRange } = changes[i]; spliceArray( diff --git a/vendor/jasmine.js b/vendor/jasmine.js index 23f172e07..d69b33d2d 100644 --- a/vendor/jasmine.js +++ b/vendor/jasmine.js @@ -2718,9 +2718,9 @@ jasmine.Matchers.prototype.toSatisfy = function(fn) { // to construct an error showing where EXACTLY was the assertion that failed function normalizeTreeSitterTextData(editor, commentRegex) { let allMatches = [], lastNonComment = 0 - const checkAssert = new RegExp('^' + commentRegex.source + '\\s*[\\<\\-|\\^]') + const checkAssert = new RegExp('^\\s*' + commentRegex.source + '\\s*[\\<\\-|\\^]') editor.getBuffer().getLines().forEach((row, i) => { - const m = row.trim().match(commentRegex) + const m = row.match(commentRegex) if(m) { // const scope = editor.scopeDescriptorForBufferPosition([i, m.index]) // FIXME: use editor.scopeDescriptorForBufferPosition when it works @@ -2757,8 +2757,7 @@ if (isCommonJS) exports.normalizeTreeSitterTextData = normalizeTreeSitterTextDat async function openDocument(fullPath) { const editor = await atom.workspace.open(fullPath); - const mode = editor.languageMode; - await mode.ready; + await editor.languageMode.ready; return editor; } @@ -2772,13 +2771,76 @@ async function runGrammarTests(fullPath, commentRegex) { }) normalized.forEach(({expected, editorPosition, testPosition}) => { expect(editor.scopeDescriptorForBufferPosition(editorPosition).scopes).toSatisfy((scopes, reason) => { - reason(`Expected to find scope "${expected}" but found "${scopes}"\n` + - ` at ${fullPath}:${testPosition.row+1}:${testPosition.column+1}` - ); - const normalized = expected.replace(/([\.\-])/g, '\\$1') - const scopeRegex = new RegExp('^' + normalized + '(\\..+)?$') - return scopes.find(e => e.match(scopeRegex)) !== undefined; + const dontFindScope = expected.startsWith("!"); + expected = expected.replace(/^!/, "") + if(dontFindScope) { + reason(`Expected to NOT find scope "${expected}" but found it\n` + + ` at ${fullPath}:${testPosition.row+1}:${testPosition.column+1}` + ); + } else { + reason(`Expected to find scope "${expected}" but found "${scopes}"\n` + + ` at ${fullPath}:${testPosition.row+1}:${testPosition.column+1}` + ); + } + const normalized = expected.replace(/([\.\-])/g, '\\$1'); + const scopeRegex = new RegExp('^' + normalized + '(\\..+)?$'); + let result = scopes.find(e => e.match(scopeRegex)) !== undefined; + if(dontFindScope) result = !result; + return result }) }) } if (isCommonJS) exports.runGrammarTests = runGrammarTests; + +async function runFoldsTests(fullPath, commentRegex) { + const editor = await openDocument(fullPath); + let grouped = {} + const normalized = normalizeTreeSitterTextData(editor, commentRegex).forEach(test => { + const [kind, id] = test.expected.split('.') + if(!kind || !id) { + throw new Error(`Folds must be in the format fold_end.some-id\n` + + ` at ${test.testPosition.row+1}:${test.testPosition.column+1}`) + } + grouped[id] ||= {} + grouped[id][kind] = test + }) + for(const k in grouped) { + const v = grouped[k] + const keys = Object.keys(v) + if(keys.indexOf('fold_begin') === -1) + throw new Error(`Fold ${k} must contain fold_begin`) + if(keys.indexOf('fold_end') === -1) + throw new Error(`Fold ${k} must contain fold_end`) + if(keys.indexOf('fold_new_position') === -1) + throw new Error(`Fold ${k} must contain fold_new_position`) + } + + for(const k in grouped) { + const fold = grouped[k] + const begin = fold['fold_begin'] + const end = fold['fold_end'] + const newPos = fold['fold_new_position'] + + expect(editor.isFoldableAtBufferRow(begin.editorPosition.row)) + .toSatisfy((foldable, reason) => { + reason(`Editor is not foldable at row ${begin.editorPosition.row+1}\n` + + ` at ${fullPath}:${begin.testPosition.row+1}:${begin.testPosition.column+1}`) + return foldable + }) + editor.foldBufferRow(begin.editorPosition.row) + + expect(editor.screenPositionForBufferPosition(end.editorPosition)) + .toSatisfy((screenPosition, reason) => { + const {row,column} = newPos.editorPosition + reason(`At row ${begin.editorPosition.row+1}, editor should fold ` + + `up to the ${end.editorPosition.row+1}:${end.editorPosition.column+1}\n` + + ` into the new position ${row+1}:${column+1}\n`+ + ` but folded to position ${screenPosition.row+1}:${screenPosition.column+1}\n`+ + ` at ${fullPath}:${newPos.testPosition.row+1}:${newPos.testPosition.column+1}\n` + + ` at ${fullPath}:${end.testPosition.row+1}:${end.testPosition.column+1}`) + return row === screenPosition.row && column === screenPosition.column + }) + editor.unfoldAll() + } +} +if (isCommonJS) exports.runFoldsTests = runFoldsTests;