Merge pull request #553 from pulsar-edit/fixes-on-clojure-tree-sitter

CHANGELOG.md

@@ -6,6 +6,7 @@
 
 ## [Unreleased]
 
+- Improved the Clojure language support by migrating it to tree-sitter and support block comments, quoting, and other advanced features on modern tree-sitter implementation
 - Added a modern implementation of Tree-sitter grammars behind an experimental flag. Enable the “Use Modern Tree-Sitter Implementation” in the Core settings to try it out.
 
 ## 1.105.0

packages/language-clojure/.gitignore

@@ -1 +0,0 @@
-node_modules

packages/language-clojure/grammars/clojure.cson

@@ -126,7 +126,7 @@
         'name': 'constant.numeric.octal.clojure'
       }
       {
-        # The decimal separator is optional only when followed by e, E or M! 
+        # The decimal separator is optional only when followed by e, E or M!
         'match': '([-+]?[0-9]+(?:(\\.|(?=[eEM]))[0-9]*([eE][-+]?[0-9]+)?)M?)'
         'name': 'constant.numeric.double.clojure'
       }

packages/language-clojure/grammars/tree-sitter-clojure.cson

@@ -1,6 +1,9 @@
-'name': 'Clojure'
-'scopeName': 'source.clojure'
-'type': 'modern-tree-sitter'
+name: 'Clojure'
+scopeName: 'source.clojure'
+type: 'modern-tree-sitter'
+injectionRegExp: '^source-clojure$'
+parser: 'tree-sitter-clojure'
+
 'fileTypes': [
   'boot'
   'clj'
@@ -10,7 +13,6 @@
   'cljs.hl'
   'cljx'
   'clojure'
-  'edn'
   'org'
   'bb'
   'joke'

packages/language-clojure/grammars/tree-sitter-edn.cson

@@ -0,0 +1,15 @@
+name: 'Clojure EDN'
+scopeName: 'source.edn'
+type: 'modern-tree-sitter'
+injectionRegExp: '^source-edn$'
+parser: 'tree-sitter-clojure'
+
+fileTypes: [
+  'edn'
+]
+treeSitter:
+  grammar: 'ts/grammar.wasm'
+  highlightsQuery: 'ts/edn-highlights.scm'
+  # localsQuery: 'ts/locals.scm'
+  foldsQuery: 'ts/folds.scm'
+  indentsQuery: 'ts/indents.scm'

packages/language-clojure/grammars/tree-sitter-quoted.cson

@@ -0,0 +1,11 @@
+scopeName: 'source.quoted.clojure'
+type: 'modern-tree-sitter'
+injectionRegExp: '^source-quoted-clojure$'
+parser: 'tree-sitter-clojure'
+
+treeSitter:
+  grammar: 'ts/grammar.wasm'
+  highlightsQuery: 'ts/edn-highlights.scm'
+  # localsQuery: 'ts/locals.scm'
+  foldsQuery: 'ts/folds.scm'
+  indentsQuery: 'ts/indents.scm'

packages/language-clojure/grammars/ts/edn-highlights.scm

@@ -0,0 +1,36 @@
+;; Collections
+(list_lit
+ "(" @punctuation.section.list.begin (#set! test.onlyIfNotDescendantOfNodeWithData "clojure.dismissTag")
+ ")" @punctuation.section.list.end)
+@meta.list
+
+(vec_lit
+ "[" @punctuation.section.vector.begin (#set! test.onlyIfNotDescendantOfNodeWithData "clojure.dismissTag")
+ "]" @punctuation.section.vector.end)
+@meta.vector
+
+(map_lit
+ "{" @punctuation.section.map.begin (#set! test.onlyIfNotDescendantOfNodeWithData "clojure.dismissTag")
+ "}" @punctuation.section.map.end)
+@meta.map
+
+(set_lit
+ ("#" "{") @punctuation.section.set.begin (#set! test.onlyIfNotDescendantOfNodeWithData "clojure.dismissTag")
+ "}" @punctuation.section.set.end)
+@meta.set
+
+((regex_lit) @string.regexp (#set! test.onlyIfNotDescendantOfNodeWithData "clojure.dismissTag"))
+((sym_lit) @meta.symbol (#set! test.onlyIfNotDescendantOfNodeWithData "clojure.dismissTag"))
+((kwd_lit) @constant.keyword (#set! test.onlyIfNotDescendantOfNodeWithData "clojure.dismissTag"))
+((str_lit) @string.quoted.double (#set! test.onlyIfNotDescendantOfNodeWithData "clojure.dismissTag"))
+((num_lit) @constant.numeric (#set! test.onlyIfNotDescendantOfNodeWithData "clojure.dismissTag"))
+((nil_lit) @constant.language (#set! test.onlyIfNotDescendantOfNodeWithData "clojure.dismissTag"))
+((bool_lit) @constant.language (#set! test.onlyIfNotDescendantOfNodeWithData clojure.dismissTag))
+(comment) @comment.line.semicolon
+((dis_expr)
+ @comment.block.clojure
+ (#set! test.onlyIfConfig language-clojure.dismissTag)
+ (#set! clojure.dismissTag true)
+ (#set! test.final true))
+
+("ERROR" @invalid.illegal)

packages/language-clojure/grammars/ts/highlights.scm

@@ -1,58 +1,111 @@
-(quoting_lit
-  value: (list_lit
-          "(" @punctuation.section.expression.begin
-          .
-          (sym_lit) @meta.symbol (#set! final "true")))
+;; Function calls
+(anon_fn_lit
+ "(" @punctuation.section.expression.begin (#set! test.onlyIfNotDescendantOfNodeWithData "clojure.dismissTag")
+ .
+ (sym_lit) @entity.name.function @meta.expression
+ ")" @punctuation.section.expression.end)
+
+(list_lit
+ "(" @punctuation.section.expression.begin (#set! test.onlyIfNotDescendantOfNodeWithData "clojure.dismissTag")
+ .
+ (sym_lit) @entity.name.function @meta.expression
+ ")" @punctuation.section.expression.end)
+
+; NS things like require
+((sym_name) @meta.symbol (#eq? @meta.symbol "import") (#set! test.onlyIfNotDescendantOfNodeWithData "clojure.dismissTag")) @keyword.control
+((sym_name) @meta.symbol (#eq? @meta.symbol "require") (#set! test.onlyIfNotDescendantOfNodeWithData "clojure.dismissTag")) @keyword.control
+
+;; USE
+((sym_name)
+ @meta.symbol
+ (#eq? @meta.symbol "use")
+ (#set! test.onlyIfConfig language-clojure.markDeprecations)
+ (#set! test.onlyIfNotDescendantOfNodeWithData clojure.dismissTag))
+@invalid.deprecated
+
+((sym_name)
+ @meta.symbol
+ (#eq? @meta.symbol "use")
+ (#set! test.onlyIfNotConfig language-clojure.markDeprecations)
+ (#set! test.onlyIfNotDescendantOfNodeWithData clojure.dismissTag))
+@keyword.control
+
+;; Namespace declaration
+((list_lit
+  "(" @punctuation.section.expression.begin (#set! test.onlyIfNotDescendantOfNodeWithData "clojure.dismissTag")
+  .
+  (sym_lit) @meta.definition.global @keyword.control (#eq? @meta.definition.global "ns")
+  .
+  (sym_lit) @meta.definition.global @entity.global
+  ")" @punctuation.section.expression.end)
+ @meta.namespace.clojure
+ (#set! isNamespace true))
+
+(list_lit
+  "("
+  .
+  (kwd_lit) @invalid.deprecated (#eq? @invalid.deprecated ":use")
+  (#set! test.onlyIfDescendantOfNodeWithData isNamespace)
+  (#set! test.onlyIfConfig language-clojure.markDeprecations))
+
+;; Definition
+(list_lit
+ "(" @punctuation.section.expression.begin (#set! test.onlyIfNotDescendantOfNodeWithData "clojure.dismissTag")
+ .
+ (sym_lit) @keyword.control (#match? @keyword.control "^def")
+ .
+ (sym_lit) @meta.definition.global @entity.global
+ ")" @punctuation.section.expression.end)
+
+;; Comment form ("Rich" comments)
+((list_lit
+  "(" @punctuation.section.expression.begin
+  .
+  (sym_lit) @meta.definition.global @keyword.control (#eq? @keyword.control "comment")
+  ")" @punctuation.section.expression.end)
+ @comment.block.clojure
+ (#set! test.onlyIfConfig language-clojure.commentTag)
+ (#set! clojure.dismissTag true))
+
+(list_lit
+ "(" @punctuation.section.expression.begin
+ .
+ (sym_lit) @keyword.control (#eq? @keyword.control "comment")
+ (#set! test.onlyIfNotConfig language-clojure.commentTag)
+ ")" @punctuation.section.expression.end)
+
+;;; COPY-PASTED from edn-highlights.
+;; IF you need to add something here, add to edn-highlights
+;; and then paste here, but DON'T PASTE the first `list_lit`
 
 ;; Collections
-(anon_fn_lit
- "(" @punctuation.section.expression.begin
- .
- (sym_lit) @entity.name.function @meta.expression
- ")" @punctuation.section.expression.end)
-
-(list_lit
- "(" @punctuation.section.expression.begin
- .
- (sym_lit) @entity.name.function @meta.expression
- ")" @punctuation.section.expression.end)
-
 (vec_lit
- "[" @punctuation.section.vector.begin
- "]" @punctuation.section.vector.end) @meta.vector
+ "[" @punctuation.section.vector.begin (#set! test.onlyIfNotDescendantOfNodeWithData "clojure.dismissTag")
+ "]" @punctuation.section.vector.end)
+@meta.vector
 
 (map_lit
- "{" @punctuation.section.map.begin
- "}" @punctuation.section.map.end) @meta.map
+ "{" @punctuation.section.map.begin (#set! test.onlyIfNotDescendantOfNodeWithData "clojure.dismissTag")
+ "}" @punctuation.section.map.end)
+@meta.map
 
 (set_lit
- ("#" "{") @punctuation.section.set.begin
- "}" @punctuation.section.set.end) @meta.map
+ ("#" "{") @punctuation.section.set.begin (#set! test.onlyIfNotDescendantOfNodeWithData "clojure.dismissTag")
+ "}" @punctuation.section.set.end)
+@meta.set
 
-; Includes
-((sym_name) @meta.symbol (#eq? @meta.symbol "import")) @keyword.control
-((sym_name) @meta.symbol (#eq? @meta.symbol "require")) @keyword.control
-((sym_name) @meta.symbol (#eq? @meta.symbol "use")) @keyword.control
-
-(list_lit
- "(" @punctuation.section.expression.begin
- .
- ((sym_lit) @meta.symbol (#eq? @meta.symbol "ns")) @keyword.control @meta.definition.global
- .
- (sym_lit) @meta.definition.global @entity.global
- ")" @punctuation.section.expression.end)
-
-(list_lit
- "(" @punctuation.section.expression.begin
- .
- ((sym_lit) @meta.symbol (#match? @meta.symbol "^def")) @keyword.control
- .
- (sym_lit) @meta.definition.global @entity.global
- ")" @punctuation.section.expression.end)
-
-(sym_lit) @meta.symbol
-(kwd_lit) @constant.keyword
-(str_lit) @string.quoted.double
-(num_lit) @constant.numeric
+((regex_lit) @string.regexp (#set! test.onlyIfNotDescendantOfNodeWithData "clojure.dismissTag"))
+((sym_lit) @meta.symbol (#set! test.onlyIfNotDescendantOfNodeWithData "clojure.dismissTag"))
+((kwd_lit) @constant.keyword (#set! test.onlyIfNotDescendantOfNodeWithData "clojure.dismissTag"))
+((str_lit) @string.quoted.double (#set! test.onlyIfNotDescendantOfNodeWithData "clojure.dismissTag"))
+((num_lit) @constant.numeric (#set! test.onlyIfNotDescendantOfNodeWithData "clojure.dismissTag"))
+((nil_lit) @constant.language (#set! test.onlyIfNotDescendantOfNodeWithData "clojure.dismissTag"))
+((bool_lit) @constant.language (#set! test.onlyIfNotDescendantOfNodeWithData clojure.dismissTag))
 (comment) @comment.line.semicolon
-(dis_expr) @comment.block.clojure
+((dis_expr)
+ @comment.block.clojure
+ (#set! test.onlyIfConfig language-clojure.dismissTag)
+ (#set! clojure.dismissTag true)
+ (#set! test.final true))
+
+("ERROR" @invalid.illegal)

packages/language-clojure/lib/main.js

@@ -0,0 +1,32 @@
+exports.activate = function() {
+  if (!atom.grammars.addInjectionPoint) return;
+
+  atom.grammars.addInjectionPoint('source.clojure', {
+    type: 'quoting_lit',
+    language: () => 'source-edn',
+    content: (node) => node,
+    includeChildren: true,
+    languageScope: 'source.edn',
+    coverShallowerScopes: true
+  });
+
+  atom.grammars.addInjectionPoint('source.clojure', {
+    type: 'syn_quoting_lit',
+    language: () => 'source-quoted-clojure',
+    content: (node) => node,
+    includeChildren: true,
+    languageScope: 'source.quoted.clojure',
+    coverShallowerScopes: true
+  });
+
+  ['unquoting_lit', 'unquote_splicing_lit'].forEach(scope => {
+    atom.grammars.addInjectionPoint('source.quoted.clojure', {
+      type: scope,
+      language: () => 'source-clojure',
+      content: (node) => node,
+      includeChildren: true,
+      languageScope: 'source.clojure',
+      coverShallowerScopes: true
+    });
+  })
+}

packages/language-clojure/package.json

@@ -2,10 +2,28 @@
   "name": "language-clojure",
   "version": "0.22.8",
   "description": "Clojure language support in Atom",
+  "main": "lib/main",
   "engines": {
     "atom": "*",
     "node": "*"
   },
   "repository": "https://github.com/pulsar-edit/pulsar",
-  "license": "MIT"
+  "license": "MIT",
+  "configSchema": {
+    "dismissTag": {
+      "type": "boolean",
+      "default": true,
+      "description": "Highlights forms after #_ as comments"
+    },
+    "commentTag": {
+      "type": "boolean",
+      "default": false,
+      "description": "Highlights forms like (comment ...) as comments"
+    },
+    "markDeprecations": {
+      "type": "boolean",
+      "default": true,
+      "description": "Marks deprecations like `use`"
+    }
+  }
 }

packages/language-clojure/spec/clojure-spec.coffee

@@ -2,6 +2,9 @@ describe "Clojure grammar", ->
   grammar = null
 
   beforeEach ->
+    atom.config.set('core.useTreeSitterParsers', false)
+    atom.config.set('core.useExperimentalModernTreeSitter', false)
+
     waitsForPromise ->
       atom.packages.activatePackage("language-clojure")
 

packages/language-clojure/spec/fixtures/config-toggle.clj

@@ -0,0 +1,20 @@
+#_
+(+ 1 2 3 (+ 4 5))
+;  ^ constant.numeric
+;         ^ entity.name.function
+
+(comment 1 2 3)
+;  ^ keyword.control
+;        ^ comment.block
+
+;; Deprecations
+(use '[foo.bar])
+; ^ keyword.control
+; ^ !invalid.deprecation
+
+(:use [foo.bar])
+; ^ !invalid.deprecated
+
+(ns other.namespace
+  (:use [foo.bar]))
+;   ^ !invalid.deprecated

packages/language-clojure/spec/fixtures/textmate-tokens.clj

@@ -0,0 +1,64 @@
+(ns foobar)
+; <- punctuation.section.expression.begin
+; ^ meta.definition.global
+;   ^ entity.global
+;         ^ punctuation.section.expression.end
+
+(defn foobar [a b]
+  ; <- keyword.control
+  ;   ^ entity.global
+  ;   ^ meta.definition.global
+  ;          ^ punctuation.section.vector.begin
+  ;           ^ meta.vector
+  ;           ^ meta.symbol
+  ;              ^ punctuation.section.vector.end
+  (+ a b 10 20))
+  ;^ meta.expression
+  ;^ entity.name.function
+  ;  ^ !entity.name.function
+  ;      ^ constant.numeric
+
+(def a "A STRING")
+  ; <- keyword.control
+  ;  ^ entity.global
+  ;      ^ string.quoted.double
+
+#{'foo}
+; <- punctuation.section.set.begin
+;   ^ meta.symbol
+;     ^ punctuation.section.set.end
+
+{:key "value"}
+; <- punctuation.section.map.begin
+; ^ constant.keyword
+;    ^ meta.map
+;            ^ punctuation.section.map.end
+
+;; Primitives
+10
+; <- constant.numeric
+10.2
+; <- constant.numeric
+10M
+; <- constant.numeric
+10N
+; <- constant.numeric
+10/2
+; <- constant.numeric
+:key
+; ^ meta.symbol
+symbol
+; <- meta.symbol
+"A string"
+; <- string.quoted.double
+#"A regular expression"
+; <- string.regexp
+nil
+; <- constant.language
+true
+; <- constant.language
+false
+; <- constant.language
+
+;; Comments
+;   ^ comment.line.semicolon

packages/language-clojure/spec/fixtures/tokens.clj

@@ -15,6 +15,7 @@
   (+ a b 10 20))
   ;^ meta.expression
   ;^ entity.name.function
+  ;  ^ !entity.name.function
   ;      ^ constant.numeric
 
 (def a "A STRING")
@@ -22,9 +23,9 @@
   ;  ^ entity.global
   ;      ^ string.quoted.double
 
-#{'asd}
+#{'foo}
 ; <- punctuation.section.set.begin
-;  ^ meta.symbol
+;   ^ meta.symbol
 ;     ^ punctuation.section.set.end
 
 {:key "value"}
@@ -32,3 +33,72 @@
 ; ^ constant.keyword
 ;    ^ meta.map
 ;            ^ punctuation.section.map.end
+
+;; Primitives
+10
+; <- constant.numeric
+10.2
+; <- constant.numeric
+10M
+; <- constant.numeric
+10N
+; <- constant.numeric
+10/2
+; <- constant.numeric
+:key
+; <- constant.keyword
+symbol
+; <- meta.symbol
+"A string"
+; <- string.quoted.double
+#"A regular expression"
+; <- string.regexp
+nil
+; <- constant.language
+true
+; <- constant.language
+false
+; <- constant.language
+error/
+; <- meta.symbol
+;    ^ invalid.illegal
+
+;; Quoting
+'(call param ~(call))
+;  ^ meta.symbol
+;  ^ !entity.name.function
+;       ^ meta.symbol
+;       ^ !entity.name.function
+;               ^ meta.symbol
+;               ^ !entity.name.function
+
+`(call param ~(call))
+;  ^ meta.symbol
+;  ^ !entity.name.function
+;       ^ meta.symbol
+;       ^ !entity.name.function
+;               ^ entity.name.function
+
+;; Comments
+;   ^ comment.line.semicolon
+
+#_
+(+ 1 2 3 (+ 4 5))
+;  ^ comment.block
+;           ^ comment.block
+
+(comment 1 2 3)
+;  ^ keyword.control
+;        ^ constant.numeric
+
+;; Deprecations
+(use '[foo.bar])
+; ^ invalid.deprecated
+
+
+(:use [foo.bar])
+; ^ !invalid.deprecated
+
+(ns other.namespace
+  (:use [foo.bar]))
+;   ^ invalid.deprecated

packages/language-clojure/spec/fixtures/tree-sitter-tokens.clj

@@ -1,6 +0,0 @@
-#_
-(+ 1 2)
-; <- comment.block
-
-'(a b 1 2)
-; ^ !entity.name.function

packages/language-clojure/spec/tokenizer-spec.js

@@ -3,7 +3,7 @@ const path = require('path');
 
 function setConfigForLanguageMode(mode) {
   let useTreeSitterParsers = mode !== 'textmate';
-  let useExperimentalModernTreeSitter = mode === 'wasm-tree-sitter';
+  let useExperimentalModernTreeSitter = mode === 'modern-tree-sitter';
   atom.config.set('core.useTreeSitterParsers', useTreeSitterParsers);
   atom.config.set('core.useExperimentalModernTreeSitter', useExperimentalModernTreeSitter);
 }
@@ -16,21 +16,27 @@ describe('Clojure grammars', () => {
 
   it('tokenizes the editor using TextMate parser', async () => {
     setConfigForLanguageMode('textmate');
+    await runGrammarTests(path.join(__dirname, 'fixtures', 'textmate-tokens.clj'), /;/)
+  });
+
+  it('tokenizes the editor using modern tree-sitter parser', async () => {
+    setConfigForLanguageMode('modern-tree-sitter');
+    atom.config.set('language-clojure.dismissTag', true);
+    atom.config.set('language-clojure.commentTag', false);
+    atom.config.set('language-clojure.markDeprecations', true);
     await runGrammarTests(path.join(__dirname, 'fixtures', 'tokens.clj'), /;/)
   });
 
-  it('tokenizes the editor using node tree-sitter parser the same as TextMate', async () => {
-    setConfigForLanguageMode('wasm-tree-sitter');
-    await runGrammarTests(path.join(__dirname, 'fixtures', 'tokens.clj'), /;/)
-  });
-
-  it('tokenizes the editor using node tree-sitter parser (specific rules)', async () => {
-    setConfigForLanguageMode('wasm-tree-sitter');
-    await runGrammarTests(path.join(__dirname, 'fixtures', 'tree-sitter-tokens.clj'), /;/)
+  it('tokenizes the editor using modern tree-sitter, but with all default configs toggled', async () => {
+    setConfigForLanguageMode('modern-tree-sitter');
+    atom.config.set('language-clojure.dismissTag', false);
+    atom.config.set('language-clojure.commentTag', true);
+    atom.config.set('language-clojure.markDeprecations', false);
+    await runGrammarTests(path.join(__dirname, 'fixtures', 'config-toggle.clj'), /;/)
   });
 
   it('folds Clojure code', async () => {
-    setConfigForLanguageMode('wasm-tree-sitter');
+    setConfigForLanguageMode('modern-tree-sitter');
     await runFoldsTests(path.join(__dirname, 'fixtures', 'tree-sitter-folds.clj'), /;/)
   });
 });