Merge feature/modernize-tree-sitter

packages/language-clojure/grammars/tree-sitter-clojure.cson

@@ -0,0 +1,43 @@
+'name': 'Clojure'
+'scopeName': 'source.clojure'
+'type': 'tree-sitter-2'
+'fileTypes': [
+  'boot'
+  'clj'
+  'clj.hl'
+  'cljc'
+  'cljs'
+  'cljs.hl'
+  'cljx'
+  'clojure'
+  'edn'
+  'org'
+  'bb'
+  'joke'
+  'joker'
+]
+'firstLineMatch': '''(?x)
+  # Hashbang
+  ^\\#!.*(?:\\s|\\/)
+    boot
+  (?:$|\\s)
+  |
+  # Modeline
+  (?i:
+    # Emacs
+    -\\*-(?:\\s*(?=[^:;\\s]+\\s*-\\*-)|(?:.*?[;\\s]|(?<=-\\*-))mode\\s*:\\s*)
+      clojure(script)?
+    (?=[\\s;]|(?<![-*])-\\*-).*?-\\*-
+    |
+    # Vim
+    (?:(?:\\s|^)vi(?:m[<=>]?\\d+|m)?|\\sex)(?=:(?=\\s*set?\\s[^\\n:]+:)|:(?!\\s*set?\\s))(?:(?:\\s|\\s*:\\s*)\\w*(?:\\s*=(?:[^\\n\\\\\\s]|\\\\.)*)?)*[\\s:](?:filetype|ft|syntax)\\s*=
+      clojure
+    (?=\\s|:|$)
+  )
+'''
+treeSitter:
+  grammar: 'ts/grammar.wasm'
+  syntaxQuery: 'ts/highlights.scm'
+  # localsQuery: 'ts/locals.scm'
+  foldsQuery: 'ts/folds.scm'
+  indentsQuery: 'ts/indents.scm'

packages/language-clojure/grammars/ts/folds.scm

@@ -0,0 +1,7 @@
+[
+  (list_lit)
+  (vec_lit)
+  (map_lit)
+  (anon_fn_lit)
+  (set_lit)
+] @fold

packages/language-clojure/grammars/ts/highlights.scm

@@ -0,0 +1,58 @@
+(quoting_lit
+  value: (list_lit
+          "(" @punctuation.section.expression.begin
+          .
+          (sym_lit) @meta.symbol (#set! final "true")))
+
+;; Collections
+(anon_fn_lit
+ "(" @punctuation.section.expression.begin
+ .
+ (sym_lit) @entity.name.function @meta.expression
+ ")" @punctuation.section.expression.end)
+
+(list_lit
+ "(" @punctuation.section.expression.begin
+ .
+ (sym_lit) @entity.name.function @meta.expression
+ ")" @punctuation.section.expression.end)
+
+(vec_lit
+ "[" @punctuation.section.vector.begin
+ "]" @punctuation.section.vector.end) @meta.vector
+
+(map_lit
+ "{" @punctuation.section.map.begin
+ "}" @punctuation.section.map.end) @meta.map
+
+(set_lit
+ ("#" "{") @punctuation.section.set.begin
+ "}" @punctuation.section.set.end) @meta.map
+
+; Includes
+((sym_name) @meta.symbol (#eq? @meta.symbol "import")) @keyword.control
+((sym_name) @meta.symbol (#eq? @meta.symbol "require")) @keyword.control
+((sym_name) @meta.symbol (#eq? @meta.symbol "use")) @keyword.control
+
+(list_lit
+ "(" @punctuation.section.expression.begin
+ .
+ ((sym_lit) @meta.symbol (#eq? @meta.symbol "ns")) @keyword.control @meta.definition.global
+ .
+ (sym_lit) @meta.definition.global @entity.global
+ ")" @punctuation.section.expression.end)
+
+(list_lit
+ "(" @punctuation.section.expression.begin
+ .
+ ((sym_lit) @meta.symbol (#match? @meta.symbol "^def")) @keyword.control
+ .
+ (sym_lit) @meta.definition.global @entity.global
+ ")" @punctuation.section.expression.end)
+
+(sym_lit) @meta.symbol
+(kwd_lit) @constant.keyword
+(str_lit) @string.quoted.double
+(num_lit) @constant.numeric
+(comment) @comment.line.semicolon
+(dis_expr) @comment.block.clojure

packages/language-clojure/spec/fixtures/tokens.clj

@@ -0,0 +1,34 @@
+(ns foobar)
+; <- punctuation.section.expression.begin
+; ^ meta.definition.global
+;   ^ entity.global
+;         ^ punctuation.section.expression.end
+
+(defn foobar [a b]
+  ; <- keyword.control
+  ;   ^ entity.global
+  ;   ^ meta.definition.global
+  ;          ^ punctuation.section.vector.begin
+  ;           ^ meta.vector
+  ;           ^ meta.symbol
+  ;              ^ punctuation.section.vector.end
+  (+ a b 10 20))
+  ;^ meta.expression
+  ;^ entity.name.function
+  ;      ^ constant.numeric
+
+(def a "A STRING")
+  ; <- keyword.control
+  ;  ^ entity.global
+  ;      ^ string.quoted.double
+
+#{'asd}
+; <- punctuation.section.set.begin
+;  ^ meta.symbol
+;     ^ punctuation.section.set.end
+
+{:key "value"}
+; <- punctuation.section.map.begin
+; ^ constant.keyword
+;    ^ meta.map
+;            ^ punctuation.section.map.end

packages/language-clojure/spec/fixtures/tree-sitter-folds.clj

@@ -0,0 +1,34 @@
+(defn a [a b]
+; <- fold_begin.paren
+;             ^ fold_new_position.paren
+  (+ a b)
+  [1
+  ; <- fold_begin.vector
+  ;  ^ fold_new_position.vector
+   2
+   3]
+  ; ^ fold_end.vector
+  {:a 10
+  ; <- fold_begin.map
+  ;      ^ fold_new_position.map
+   :b 20
+   :c [1
+   ;  ^ fold_begin.inner_vector
+   ;     ^ fold_new_position.inner_vector
+       2
+       3]})
+;         ^ fold_end.paren
+;        ^ fold_end.map
+;       ^ fold_end.inner_vector
+
+#(inner
+; <- fold_begin.anon
+  ;     ^ fold_new_position.anon
+  "function"
+  #{:with
+  ; <- fold_begin.set
+    ;     ^ fold_new_position.set
+    :inner
+    :set})
+;        ^ fold_end.anon
+;       ^ fold_end.set

packages/language-clojure/spec/fixtures/tree-sitter-tokens.clj

@@ -0,0 +1,6 @@
+#_
+(+ 1 2)
+; <- comment.block
+
+'(a b 1 2)
+; ^ !entity.name.function

packages/language-clojure/spec/tokenizer-spec.js

@@ -0,0 +1,28 @@
+const path = require('path');
+
+describe('Clojure grammars', () => {
+
+  beforeEach(async () => {
+    await atom.packages.activatePackage('language-clojure');
+  });
+
+  it('tokenizes the editor using TextMate parser', async () => {
+    atom.config.set('core.languageParser', 'textmate');
+    await runGrammarTests(path.join(__dirname, 'fixtures', 'tokens.clj'), /;/)
+  });
+
+  it('tokenizes the editor using node tree-sitter parser the same as TextMate', async () => {
+    atom.config.set('core.languageParser', 'wasm-tree-sitter');
+    await runGrammarTests(path.join(__dirname, 'fixtures', 'tokens.clj'), /;/)
+  });
+
+  it('tokenizes the editor using node tree-sitter parser (specific rules)', async () => {
+    atom.config.set('core.languageParser', 'wasm-tree-sitter');
+    await runGrammarTests(path.join(__dirname, 'fixtures', 'tree-sitter-tokens.clj'), /;/)
+  });
+
+  it('folds Clojure code', async () => {
+    atom.config.set('core.languageParser', 'wasm-tree-sitter');
+    await runFoldsTests(path.join(__dirname, 'fixtures', 'tree-sitter-folds.clj'), /;/)
+  });
+});

packages/language-ruby/spec/wasm-tree-sitter-spec.js

@@ -23,55 +23,8 @@ describe('WASM Tree-sitter Ruby grammar', () => {
   });
 
   it('folds code', async () => {
-    const editor = await openDocument('folds.rb');
-    let grouped = {}
-    normalized = normalizeTreeSitterTextData(editor, /#/).forEach(test => {
-      const [kind, id] = test.expected.split('.')
-      if(!kind || !id) {
-        throw new Error(dedent`Folds must be in the format fold_end.some-id
-          at ${test.testPosition.row+1}:${test.testPosition.column+1}`)
-      }
-      grouped[id] ||= {}
-      grouped[id][kind] = test
-    })
-    for(const k in grouped) {
-      const v = grouped[k]
-      const keys = Object.keys(v)
-      if(keys.indexOf('fold_begin') === -1)
-        throw new Error(`Fold ${k} must contain fold_begin`)
-      if(keys.indexOf('fold_end') === -1)
-        throw new Error(`Fold ${k} must contain fold_end`)
-      if(keys.indexOf('fold_new_position') === -1)
-        throw new Error(`Fold ${k} must contain fold_new_position`)
-    }
-
-    for(const k in grouped) {
-      const fold = grouped[k]
-      const begin = fold['fold_begin']
-      const end = fold['fold_end']
-      const newPos = fold['fold_new_position']
-
-      expect(editor.isFoldableAtBufferRow(begin.editorPosition.row))
-        .toSatisfy((foldable, reason) => {
-          reason(dedent`Editor is not foldable at row ${begin.editorPosition.row+1}
-            at fixtures/folds.rb:${begin.testPosition.row+1}:${begin.testPosition.column+1}`)
-          return foldable
-        })
-        editor.foldBufferRow(begin.editorPosition.row)
-
-      expect(editor.screenPositionForBufferPosition(end.editorPosition))
-        .toSatisfy((screenPosition, reason) => {
-          const {row,column} = newPos.editorPosition
-          reason(`At row ${begin.editorPosition.row+1}, editor should fold ` +
-            `up to the ${end.editorPosition.row+1}:${end.editorPosition.column+1}\n` +
-            `    into the new position  ${row+1}:${column+1}\n`+
-            `    but folded to position ${screenPosition.row+1}:${screenPosition.column+1}\n`+
-            `      at fixtures/folds.rb:${newPos.testPosition.row+1}:${newPos.testPosition.column+1}\n` +
-            `      at fixtures/folds.rb:${end.testPosition.row+1}:${end.testPosition.column+1}`)
-          return row === screenPosition.row && column === screenPosition.column
-        })
-      editor.unfoldAll()
-    }
+    const fullPath = path.join(__dirname, 'fixtures', 'folds.rb')
+    await runFoldsTests(fullPath, /#/)
   });
 });
 
@@ -81,36 +34,3 @@ async function openDocument(fileName) {
   await editor.languageMode.ready
   return editor
 }
-
-// function normalizeTestData(editor, commentRegex) {
-//   let allMatches = [], lastNonComment = 0
-//   editor.getBuffer().getLines().forEach((row, i) => {
-//     const m = row.match(commentRegex)
-//     if(m) {
-//       const scope = editor.scopeDescriptorForBufferPosition([i, m.index])
-//       if(scope.scopes.find(s => s.match(/comment/))) {
-//         allMatches.push({row: lastNonComment, text: row, col: m.index, testRow: i})
-//         return
-//       }
-//     }
-//     lastNonComment = i
-//   })
-//   return allMatches.map(({text, row, col, testRow}) => {
-//     const exactPos = text.match(/\^\s+(.*)/)
-//     if(exactPos) {
-//       const expected = exactPos[1]
-//       return {
-//         expected,
-//         editorPosition: {row, column: exactPos.index},
-//         testPosition: {row: testRow, column: col}
-//       }
-//     } else {
-//       const pos = text.match(/\<-\s+(.*)/)
-//       return {
-//         expected: pos[1],
-//         editorPosition: {row, column: col},
-//         testPosition: {row: testRow, column: col}
-//       }
-//     }
-//   })
-// }

src/wasm-tree-sitter-language-mode.js

@@ -153,7 +153,7 @@ class WASMTreeSitterLanguageMode {
       .then(language => {
         this.rootLanguage = language;
         this.rootLanguageLayer = new LanguageLayer(null, this, grammar, 0);
-        this.getOrCreateParserForLanguage(language);
+        return this.getOrCreateParserForLanguage(language);
       })
       .then(() => this.rootLanguageLayer.update(null))
       .then(() => this.emitter.emit('did-tokenize'));
@@ -221,7 +221,6 @@ class WASMTreeSitterLanguageMode {
 
   bufferDidFinishTransaction({ changes }) {
     if (!this.rootLanguageLayer) { return; }
-
     for (let i = 0, { length } = changes; i < length; i++) {
       const { oldRange, newRange } = changes[i];
       spliceArray(

vendor/jasmine.js

@@ -2718,9 +2718,9 @@ jasmine.Matchers.prototype.toSatisfy = function(fn) {
 // to construct an error showing where EXACTLY was the assertion that failed
 function normalizeTreeSitterTextData(editor, commentRegex) {
   let allMatches = [], lastNonComment = 0
-  const checkAssert = new RegExp('^' + commentRegex.source + '\\s*[\\<\\-|\\^]')
+  const checkAssert = new RegExp('^\\s*' + commentRegex.source + '\\s*[\\<\\-|\\^]')
   editor.getBuffer().getLines().forEach((row, i) => {
-    const m = row.trim().match(commentRegex)
+    const m = row.match(commentRegex)
     if(m) {
       // const scope = editor.scopeDescriptorForBufferPosition([i, m.index])
       // FIXME: use editor.scopeDescriptorForBufferPosition when it works
@@ -2757,8 +2757,7 @@ if (isCommonJS) exports.normalizeTreeSitterTextData = normalizeTreeSitterTextDat
 
 async function openDocument(fullPath) {
   const editor = await atom.workspace.open(fullPath);
-  const mode = editor.languageMode;
-  await mode.ready;
+  await editor.languageMode.ready;
   return editor;
 }
 
@@ -2772,13 +2771,76 @@ async function runGrammarTests(fullPath, commentRegex) {
   })
   normalized.forEach(({expected, editorPosition, testPosition}) => {
     expect(editor.scopeDescriptorForBufferPosition(editorPosition).scopes).toSatisfy((scopes, reason) => {
-      reason(`Expected to find scope "${expected}" but found "${scopes}"\n` +
-        `      at ${fullPath}:${testPosition.row+1}:${testPosition.column+1}`
-      );
-      const normalized = expected.replace(/([\.\-])/g, '\\$1')
-      const scopeRegex = new RegExp('^' + normalized + '(\\..+)?$')
-      return scopes.find(e => e.match(scopeRegex)) !== undefined;
+      const dontFindScope = expected.startsWith("!");
+      expected = expected.replace(/^!/, "")
+      if(dontFindScope) {
+        reason(`Expected to NOT find scope "${expected}" but found it\n` +
+          `      at ${fullPath}:${testPosition.row+1}:${testPosition.column+1}`
+        );
+      } else {
+        reason(`Expected to find scope "${expected}" but found "${scopes}"\n` +
+          `      at ${fullPath}:${testPosition.row+1}:${testPosition.column+1}`
+        );
+      }
+      const normalized = expected.replace(/([\.\-])/g, '\\$1');
+      const scopeRegex = new RegExp('^' + normalized + '(\\..+)?$');
+      let result = scopes.find(e => e.match(scopeRegex)) !== undefined;
+      if(dontFindScope) result = !result;
+      return result
     })
   })
 }
 if (isCommonJS) exports.runGrammarTests = runGrammarTests;
+
+async function runFoldsTests(fullPath, commentRegex) {
+  const editor = await openDocument(fullPath);
+  let grouped = {}
+  const normalized = normalizeTreeSitterTextData(editor, commentRegex).forEach(test => {
+    const [kind, id] = test.expected.split('.')
+    if(!kind || !id) {
+      throw new Error(`Folds must be in the format fold_end.some-id\n` +
+        `      at ${test.testPosition.row+1}:${test.testPosition.column+1}`)
+    }
+    grouped[id] ||= {}
+    grouped[id][kind] = test
+  })
+  for(const k in grouped) {
+    const v = grouped[k]
+    const keys = Object.keys(v)
+    if(keys.indexOf('fold_begin') === -1)
+      throw new Error(`Fold ${k} must contain fold_begin`)
+    if(keys.indexOf('fold_end') === -1)
+      throw new Error(`Fold ${k} must contain fold_end`)
+    if(keys.indexOf('fold_new_position') === -1)
+      throw new Error(`Fold ${k} must contain fold_new_position`)
+  }
+
+  for(const k in grouped) {
+    const fold = grouped[k]
+    const begin = fold['fold_begin']
+    const end = fold['fold_end']
+    const newPos = fold['fold_new_position']
+
+    expect(editor.isFoldableAtBufferRow(begin.editorPosition.row))
+      .toSatisfy((foldable, reason) => {
+        reason(`Editor is not foldable at row ${begin.editorPosition.row+1}\n` +
+          `      at ${fullPath}:${begin.testPosition.row+1}:${begin.testPosition.column+1}`)
+        return foldable
+      })
+      editor.foldBufferRow(begin.editorPosition.row)
+
+    expect(editor.screenPositionForBufferPosition(end.editorPosition))
+      .toSatisfy((screenPosition, reason) => {
+        const {row,column} = newPos.editorPosition
+        reason(`At row ${begin.editorPosition.row+1}, editor should fold ` +
+          `up to the ${end.editorPosition.row+1}:${end.editorPosition.column+1}\n` +
+          `    into the new position  ${row+1}:${column+1}\n`+
+          `    but folded to position ${screenPosition.row+1}:${screenPosition.column+1}\n`+
+          `      at ${fullPath}:${newPos.testPosition.row+1}:${newPos.testPosition.column+1}\n` +
+          `      at ${fullPath}:${end.testPosition.row+1}:${end.testPosition.column+1}`)
+        return row === screenPosition.row && column === screenPosition.column
+      })
+    editor.unfoldAll()
+  }
+}
+if (isCommonJS) exports.runFoldsTests = runFoldsTests;