Make useExperimentalModernTreeSitter the default…

…and create `useLegacyTreeSitter` for those who want to opt into the previous default behavior.

(Legacy Tree-sitter grammars will soon be removed, but this is a step toward that future!)

packages/autocomplete-css/spec/provider-spec.js

@@ -33,7 +33,7 @@ const packagesToTest = {
 
 const wait = (ms) => new Promise(resolve => setTimeout(resolve, ms));
 
-const whenEditorReady = function(editor) {
+const whenEditorReady = function (editor) {
   const languageMode = editor.getBuffer().getLanguageMode();
   if (!languageMode.constructor.name.includes('TreeSitter')) {
     return Promise.resolve();
@@ -105,8 +105,6 @@ describe("CSS property name and value autocompletions", async () => {
         await atom.workspace.open(packagesToTest[packageLabel].file);
         editor = atom.workspace.getActiveTextEditor();
         await whenEditorReady(editor);
-        console.warn('USING TREE SITTER?!?', packageLabel, meta.useTreeSitter);
-        atom.config.set('core.useExperimentalModernTreeSitter', meta.useTreeSitter ?? false);
         atom.config.set('core.useTreeSitterParsers', meta.useTreeSitter ?? false);
       });
 
@@ -738,7 +736,7 @@ div:nth {
     })
   );
 
-  Object.keys(packagesToTest).forEach(function(packageLabel) {
+  Object.keys(packagesToTest).forEach(function (packageLabel) {
     if (packagesToTest[packageLabel].name !== 'language-css') {
       describe(`${packageLabel} files`, async () => {
         beforeEach(async () => {

packages/autocomplete-html/spec/.eslintrc.js

@@ -0,0 +1,12 @@
+module.exports = {
+  env: { jasmine: true },
+  globals: {
+    "waitsForPromise": true
+  },
+  rules: {
+    "node/no-unpublished-require": "off",
+    "node/no-extraneous-require": "off",
+    "no-unused-vars": "off",
+    "no-empty": "off"
+  }
+};

packages/grammar-selector/lib/grammar-list-view.js

@@ -26,10 +26,10 @@ module.exports = class GrammarListView {
           let badgeColor = 'badge-success';
           let badgeText = 'Tree-sitter';
 
-          if (isExperimentalTreeSitterMode()) {
+          if (isLegacyTreeSitterMode()) {
-            badgeColor = isModernTreeSitter(grammar) ?
+            badgeColor = isLegacyTreeSitter(grammar) ?
               'badge-success' : 'badge-warning';
-            badgeText = isModernTreeSitter(grammar) ?
+            badgeText = isLegacyTreeSitter(grammar) ?
               'Tree-sitter' : 'Legacy Tree-sitter';
           }
 
@@ -118,13 +118,15 @@ module.exports = class GrammarListView {
           return grammar !== atom.grammars.nullGrammar && grammar.name;
         });
 
-      // Don't show modern tree-sitter grammars in the selector unless the user
+      // Don't show legacy Tree-sitter grammars in the selector unless the user
       // has opted into it.
-      if (!isExperimentalTreeSitterMode()) {
+      if (!isLegacyTreeSitterMode()) {
-        grammars = grammars.filter(grammar => !isModernTreeSitter(grammar));
+        grammars = grammars.filter(grammar => !isLegacyTreeSitter(grammar));
       }
 
       if (atom.config.get('grammar-selector.hideDuplicateTextMateGrammars')) {
+        // Filter out all TextMate grammars for which there is a Tree-sitter
+        // grammar with the exact same name.
         const blacklist = new Set();
         grammars.forEach(grammar => {
           if (isTreeSitter(grammar)) {
@@ -155,24 +157,24 @@ module.exports = class GrammarListView {
 
 function getLanguageModeConfig() {
   let isTreeSitterMode = atom.config.get('core.useTreeSitterParsers');
-  let isExperimental = atom.config.get('core.useExperimentalModernTreeSitter');
+  let isLegacy = atom.config.get('core.useLegacyTreeSitter');
   if (!isTreeSitterMode) return 'textmate';
-  return isExperimental ? 'wasm-tree-sitter' : 'node-tree-sitter';
+  return isLegacy ? 'node-tree-sitter' : 'wasm-tree-sitter';
 }
 
-function isExperimentalTreeSitterMode() {
+function isLegacyTreeSitterMode() {
-  return getLanguageModeConfig() === 'wasm-tree-sitter';
+  return getLanguageModeConfig() === 'node-tree-sitter';
 }
 
 function isTreeSitter(grammar) {
-  return isOldTreeSitter(grammar) || isModernTreeSitter(grammar);
+  return isLegacyTreeSitter(grammar) || isModernTreeSitter(grammar);
 }
 
 function isModernTreeSitter(grammar) {
   return grammar.constructor.name === 'WASMTreeSitterGrammar';
 }
 
-function isOldTreeSitter(grammar) {
+function isLegacyTreeSitter(grammar) {
   return grammar.constructor.name === 'TreeSitterGrammar';
 }
 
@@ -183,6 +185,6 @@ function compareGrammarType(a, b) {
 function getGrammarScore(grammar) {
   let languageParser = getLanguageModeConfig();
   if (isModernTreeSitter(grammar)) { return -2; }
-  if (isOldTreeSitter(grammar)) { return -1; }
+  if (isLegacyTreeSitter(grammar)) { return -1; }
   return languageParser === 'textmate' ? -3 : 0;
 }

packages/grammar-selector/spec/.eslintrc.js

@@ -0,0 +1,8 @@
+module.exports = {
+  env: { jasmine: true },
+  rules: {
+    "node/no-unpublished-require": "off",
+    "no-unused-vars": "off",
+    "no-empty": "off"
+  }
+};

packages/grammar-selector/spec/grammar-selector-spec.js

@@ -3,9 +3,9 @@ const SelectListView = require('atom-select-list');
 
 function setConfigForLanguageMode(mode) {
   let useTreeSitterParsers = mode !== 'textmate';
-  let useExperimentalModernTreeSitter = mode === 'wasm-tree-sitter';
+  let useLegacyTreeSitter = mode === 'node-tree-sitter';
   atom.config.set('core.useTreeSitterParsers', useTreeSitterParsers);
-  atom.config.set('core.useExperimentalModernTreeSitter', useExperimentalModernTreeSitter);
+  atom.config.set('core.useLegacyTreeSitter', useLegacyTreeSitter);
 }
 
 describe('GrammarSelector', () => {

packages/language-clojure/spec/.eslintrc.js

@@ -0,0 +1,9 @@
+module.exports = {
+  env: { jasmine: true },
+  rules: {
+    "node/no-unpublished-require": "off",
+    "node/no-extraneous-require": "off",
+    "no-unused-vars": "off",
+    "no-empty": "off"
+  }
+};

packages/language-clojure/spec/clojure-spec.js

@@ -4,7 +4,6 @@ describe("Clojure grammar", function() {
 
   beforeEach(function() {
     atom.config.set('core.useTreeSitterParsers', false);
-    atom.config.set('core.useExperimentalModernTreeSitter', false);
 
     waitsForPromise(() => atom.packages.activatePackage("language-clojure"));
 

packages/language-clojure/spec/tokenizer-spec.js

@@ -3,9 +3,7 @@ const path = require('path');
 
 function setConfigForLanguageMode(mode) {
   let useTreeSitterParsers = mode !== 'textmate';
-  let useExperimentalModernTreeSitter = mode === 'modern-tree-sitter';
   atom.config.set('core.useTreeSitterParsers', useTreeSitterParsers);
-  atom.config.set('core.useExperimentalModernTreeSitter', useExperimentalModernTreeSitter);
 }
 
 describe('Clojure grammars', () => {

packages/language-html/spec/tree-sitter-spec.js

@@ -3,7 +3,7 @@ const dedent = require('dedent');
 describe('Tree-sitter HTML grammar', () => {
   beforeEach(async () => {
     atom.config.set('core.useTreeSitterParsers', true);
-    atom.config.set('core.useExperimentalModernTreeSitter', false);
+    atom.config.set('core.useLegacyTreeSitter', true);
     await atom.packages.activatePackage('language-html');
   });
 

packages/language-java/spec/.eslintrc.js

@@ -0,0 +1,12 @@
+module.exports = {
+  env: { jasmine: true },
+  globals: {
+    waitsForPromise: true
+  },
+  rules: {
+    "node/no-unpublished-require": "off",
+    "node/no-extraneous-require": "off",
+    "no-unused-vars": "off",
+    "no-empty": "off"
+  }
+};

packages/language-java/spec/tree-sitter-java-spec.js

@@ -8,7 +8,7 @@ describe('Tree-sitter based Java grammar', function() {
 
   beforeEach(function() {
     atom.config.set('core.useTreeSitterParsers', true);
-    atom.config.set('core.useExperimentalModernTreeSitter', false);
+    atom.config.set('core.useLegacyTreeSitter', true);
 
     waitsForPromise(() => atom.packages.activatePackage('language-java'));
 

packages/language-ruby/spec/.eslintrc.js

@@ -0,0 +1,15 @@
+module.exports = {
+  env: { jasmine: true },
+  globals: {
+    waitsForPromise: true,
+    runGrammarTests: true,
+    runFoldsTests: true,
+    normalizeTreeSitterTextData: true
+  },
+  rules: {
+    "node/no-unpublished-require": "off",
+    "node/no-extraneous-require": "off",
+    "no-unused-vars": "off",
+    "no-empty": "off"
+  }
+};

packages/language-ruby/spec/tokenizer-spec.js

@@ -16,7 +16,7 @@ describe('Ruby grammars', () => {
 
   xit('tokenizes the editor using node tree-sitter parser', async () => {
     atom.config.set('core.useTreeSitterParsers', true);
-    atom.config.set('core.useExperimentalModernTreeSitter', false);
+    atom.config.set('core.useLegacyTreeSitter', true);
     await runGrammarTests(path.join(__dirname, 'fixtures', 'textmate-grammar.rb'), /#/)
   });
 });

packages/language-ruby/spec/tree-sitter-spec.js

@@ -3,7 +3,7 @@ const dedent = require('dedent');
 describe('Tree-sitter Ruby grammar', () => {
   beforeEach(async () => {
     atom.config.set('core.useTreeSitterParsers', true);
-    atom.config.set('core.useExperimentalModernTreeSitter', false);
+    atom.config.set('core.useLegacyTreeSitter', true);
     await atom.packages.activatePackage('language-ruby');
   });
 

packages/language-ruby/spec/wasm-tree-sitter-spec.js

@@ -7,7 +7,6 @@ xdescribe('WASM Tree-sitter Ruby grammar', () => {
   beforeEach(async () => {
     await atom.packages.activatePackage('language-ruby');
     atom.config.set('core.useTreeSitterParsers', true);
-    atom.config.set('core.useExperimentalModernTreeSitter', true);
   });
 
   it('tokenizes symbols', async () => {

spec/grammar-registry-spec.js

@@ -11,9 +11,9 @@ const { OnigScanner } = SecondMate;
 // Expects one of `textmate`, `node-tree-sitter`, or `wasm-tree-sitter`.
 function setConfigForLanguageMode(mode, options = {}) {
   let useTreeSitterParsers = mode !== 'textmate';
-  let useExperimentalModernTreeSitter = mode === 'wasm-tree-sitter';
+  let useLegacyTreeSitter = mode === 'node-tree-sitter';
   atom.config.set('core.useTreeSitterParsers', useTreeSitterParsers, options);
-  atom.config.set('core.useExperimentalModernTreeSitter', useExperimentalModernTreeSitter, options);
+  atom.config.set('core.useLegacyTreeSitter', useLegacyTreeSitter, options);
 }
 
 describe('GrammarRegistry', () => {

spec/scope-resolver-spec.js

@@ -93,7 +93,7 @@ function rangeFromDescriptor(rawRange) {
 }
 
 describe('ScopeResolver', () => {
-  let editor, buffer, grammar, scopeResolver;
+  let editor, buffer, grammar;
 
   beforeEach(async () => {
     grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
@@ -101,7 +101,6 @@ describe('ScopeResolver', () => {
     buffer = editor.getBuffer();
     atom.grammars.addGrammar(grammar);
     atom.config.set('core.useTreeSitterParsers', true);
-    atom.config.set('core.useExperimentalModernTreeSitter', true);
   });
 
   afterEach(() => {
@@ -126,7 +125,7 @@ describe('ScopeResolver', () => {
     let { scopeResolver, captures } = await getAllCaptures(grammar, languageMode);
 
     for (let capture of captures) {
-      let { node, name } = capture;
+      let { node } = capture;
       let range = scopeResolver.store(capture);
       expect(stringForNodeRange(range))
         .toBe(stringForNodeRange(node));

spec/tree-sitter-language-mode-spec.js

@@ -42,7 +42,7 @@ describe('TreeSitterLanguageMode', () => {
     buffer = editor.getBuffer();
     editor.displayLayer.reset({ foldCharacter: '…' });
     atom.config.set('core.useTreeSitterParsers', true);
-    atom.config.set('core.useExperimentalModernTreeSitter', false);
+    atom.config.set('core.useLegacyTreeSitter', true);
   });
 
   describe('highlighting', () => {

spec/wasm-tree-sitter-language-mode-spec.js

@@ -66,7 +66,6 @@ describe('WASMTreeSitterLanguageMode', () => {
     buffer = editor.getBuffer();
     editor.displayLayer.reset({ foldCharacter: '…' });
     atom.config.set('core.useTreeSitterParsers', true);
-    atom.config.set('core.useExperimentalModernTreeSitter', true);
   });
 
   afterEach(() => {

src/config-schema.js

@@ -360,13 +360,14 @@ const configSchema = {
       useTreeSitterParsers: {
         type: 'boolean',
         default: true,
+        title: 'Use Tree-sitter Parsers',
         description: 'Use Tree-sitter parsers for supported languages.'
       },
-      useExperimentalModernTreeSitter: {
+      useLegacyTreeSitter: {
         type: 'boolean',
         default: false,
-        title: 'Use Modern Tree-Sitter Implementation',
+        title: 'Use legacy Tree-sitter Implementation',
-        description: 'Experimental: Use the new query-file-based Tree-sitter system instead of the legacy system from Atom. (This system will eventually replace the legacy system.) Has no effect unless "Use Tree Sitter Parsers" is also checked.'
+        description: 'Opt into the legacy Atom Tree-sitter system instead of the modern system added by Pulsar. (This legacy system will soon be removed.) Has no effect unless “Use Tree-sitter Parsers” is also checked.'
       },
       colorProfile: {
         description:

src/grammar-registry.js

@@ -46,7 +46,7 @@ module.exports = class GrammarRegistry {
     this.textmateRegistry.onDidUpdateGrammar(grammarAddedOrUpdated);
 
     let onLanguageModeChange = () => {
-      this.grammarScoresByBuffer.forEach((score, buffer) => {
+      this.grammarScoresByBuffer.forEach((_score, buffer) => {
         if (!this.languageOverridesByBufferId.has(buffer.id)) {
           this.autoAssignLanguageMode(buffer);
         }
@@ -55,7 +55,7 @@ module.exports = class GrammarRegistry {
 
     this.subscriptions.add(
       this.config.onDidChange('core.useTreeSitterParsers', onLanguageModeChange),
-      this.config.onDidChange('core.useExperimentalModernTreeSitter', onLanguageModeChange)
+      this.config.onDidChange('core.useLegacyTreeSitter', onLanguageModeChange)
     );
   }
 
@@ -253,10 +253,10 @@ module.exports = class GrammarRegistry {
       scope = new ScopeDescriptor({ scopes: [scope] })
     }
     let useTreeSitterParsers = this.config.get('core.useTreeSitterParsers', { scope });
-    let useExperimentalModernTreeSitter = this.config.get('core.useExperimentalModernTreeSitter', { scope });
+    let useLegacyTreeSitter = this.config.get('core.useLegacyTreeSitter', { scope });
 
     if (!useTreeSitterParsers) return 'textmate';
-    return useExperimentalModernTreeSitter ? 'wasm-tree-sitter' : 'node-tree-sitter';
+    return useLegacyTreeSitter ? 'node-tree-sitter' : 'wasm-tree-sitter';
   }
 
   // Extended: Returns a {Number} representing how well the grammar matches the
@@ -288,19 +288,15 @@ module.exports = class GrammarRegistry {
       if (isNewTreeSitter) {
         if (parserConfig === 'wasm-tree-sitter') {
           score += 0.1;
-        } else {
-          // Never automatically switch to a new Tree-sitter grammar unless the
-          // user has opted into the experimental setting.
-          score = -Infinity;
         }
       } else if (isOldTreeSitter) {
         if (parserConfig === 'node-tree-sitter') {
           score += 0.1;
         } else if (parserConfig === 'wasm-tree-sitter') {
-          // In experimental new-tree-sitter mode, we still would rather fall
+          // If `useLegacyTreeSitter` isn't checked, we probably still prefer a
-          // back to an old-tree-sitter grammar than a TM grammar. Bump the
+          // legacy Tree-sitter grammar over a TextMate-style grammar. Bump the
           // score, but just a bit less than we'd bump it if this were a
-          // new-tree-sitter grammar.
+          // modern Tree-sitter grammar.
           score += 0.09;
         }
       }
@@ -770,17 +766,17 @@ module.exports = class GrammarRegistry {
     let result = this.textmateRegistry.getGrammars();
     if (!(params && params.includeTreeSitter)) return result;
 
-    let includeModernTreeSitterGrammars =
+    let includeLegacyTreeSitterGrammars =
-      atom.config.get('core.useExperimentalModernTreeSitter') === true;
+      atom.config.get('core.useLegacyTreeSitter') === true;
 
-    const tsGrammars = Object.values(this.treeSitterGrammarsById)
+    let modernTsGrammars = Object.values(this.wasmTreeSitterGrammarsById)
       .filter(g => g.scopeName);
-    result = result.concat(tsGrammars);
+    result = result.concat(modernTsGrammars);
 
-    if (includeModernTreeSitterGrammars) {
+    if (includeLegacyTreeSitterGrammars) {
-      let modernTsGrammars = Object.values(this.wasmTreeSitterGrammarsById)
+      const legacyTsGrammars = Object.values(this.treeSitterGrammarsById)
         .filter(g => g.scopeName);
-      result = result.concat(modernTsGrammars);
+      result = result.concat(legacyTsGrammars);
     }
 
     return result;
@@ -790,8 +786,10 @@ module.exports = class GrammarRegistry {
     return this.textmateRegistry.scopeForId(id);
   }
 
-  // TODO: why is this being used? Can we remove it soon?
+  // Match up a language string (of the sort generated by an injection point)
-  treeSitterGrammarForLanguageString(languageString, type = 'original') {
+  // with a grammar. Checks the `injectionRegex` property on grammars and
+  // returns the one with the longest match.
+  treeSitterGrammarForLanguageString(languageString, type = 'wasm') {
     let longestMatchLength = 0;
     let grammarWithLongestMatch = null;
     let table = type === 'original' ? this.treeSitterGrammarsById : this.wasmTreeSitterGrammarsById;