From 76ac2cf81ca29f6203c9854a19e906fe6c5471c0 Mon Sep 17 00:00:00 2001 From: Andrew Dupont Date: Sat, 6 Jan 2024 15:46:53 -0800 Subject: [PATCH] =?UTF-8?q?Make=20`useExperimentalModernTreeSitter`=20the?= =?UTF-8?q?=20default=E2=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …and create `useLegacyTreeSitter` for those who want to opt into the previous default behavior. (Legacy Tree-sitter grammars will soon be removed, but this is a step toward that future!) --- .../autocomplete-css/spec/provider-spec.js | 6 +-- packages/autocomplete-html/spec/.eslintrc.js | 12 ++++++ .../grammar-selector/lib/grammar-list-view.js | 28 +++++++------- packages/grammar-selector/spec/.eslintrc.js | 8 ++++ .../spec/grammar-selector-spec.js | 4 +- packages/language-clojure/spec/.eslintrc.js | 9 +++++ .../language-clojure/spec/clojure-spec.js | 1 - .../language-clojure/spec/tokenizer-spec.js | 2 - .../language-html/spec/tree-sitter-spec.js | 2 +- packages/language-java/spec/.eslintrc.js | 12 ++++++ .../spec/tree-sitter-java-spec.js | 2 +- packages/language-ruby/spec/.eslintrc.js | 15 ++++++++ packages/language-ruby/spec/tokenizer-spec.js | 2 +- .../language-ruby/spec/tree-sitter-spec.js | 2 +- .../spec/wasm-tree-sitter-spec.js | 1 - spec/grammar-registry-spec.js | 4 +- spec/scope-resolver-spec.js | 5 +-- spec/tree-sitter-language-mode-spec.js | 2 +- spec/wasm-tree-sitter-language-mode-spec.js | 1 - src/config-schema.js | 7 ++-- src/grammar-registry.js | 38 +++++++++---------- 21 files changed, 106 insertions(+), 57 deletions(-) create mode 100644 packages/autocomplete-html/spec/.eslintrc.js create mode 100644 packages/grammar-selector/spec/.eslintrc.js create mode 100644 packages/language-clojure/spec/.eslintrc.js create mode 100644 packages/language-java/spec/.eslintrc.js create mode 100644 packages/language-ruby/spec/.eslintrc.js diff --git a/packages/autocomplete-css/spec/provider-spec.js b/packages/autocomplete-css/spec/provider-spec.js index 1249ee72c..cd85ef374 100644 --- a/packages/autocomplete-css/spec/provider-spec.js +++ b/packages/autocomplete-css/spec/provider-spec.js @@ -33,7 +33,7 @@ const packagesToTest = { const wait = (ms) => new Promise(resolve => setTimeout(resolve, ms)); -const whenEditorReady = function(editor) { +const whenEditorReady = function (editor) { const languageMode = editor.getBuffer().getLanguageMode(); if (!languageMode.constructor.name.includes('TreeSitter')) { return Promise.resolve(); @@ -105,8 +105,6 @@ describe("CSS property name and value autocompletions", async () => { await atom.workspace.open(packagesToTest[packageLabel].file); editor = atom.workspace.getActiveTextEditor(); await whenEditorReady(editor); - console.warn('USING TREE SITTER?!?', packageLabel, meta.useTreeSitter); - atom.config.set('core.useExperimentalModernTreeSitter', meta.useTreeSitter ?? false); atom.config.set('core.useTreeSitterParsers', meta.useTreeSitter ?? false); }); @@ -738,7 +736,7 @@ div:nth { }) ); - Object.keys(packagesToTest).forEach(function(packageLabel) { + Object.keys(packagesToTest).forEach(function (packageLabel) { if (packagesToTest[packageLabel].name !== 'language-css') { describe(`${packageLabel} files`, async () => { beforeEach(async () => { diff --git a/packages/autocomplete-html/spec/.eslintrc.js b/packages/autocomplete-html/spec/.eslintrc.js new file mode 100644 index 000000000..b62afc93c --- /dev/null +++ b/packages/autocomplete-html/spec/.eslintrc.js @@ -0,0 +1,12 @@ +module.exports = { + env: { jasmine: true }, + globals: { + "waitsForPromise": true + }, + rules: { + "node/no-unpublished-require": "off", + "node/no-extraneous-require": "off", + "no-unused-vars": "off", + "no-empty": "off" + } +}; diff --git a/packages/grammar-selector/lib/grammar-list-view.js b/packages/grammar-selector/lib/grammar-list-view.js index c54397375..8980a18ce 100644 --- a/packages/grammar-selector/lib/grammar-list-view.js +++ b/packages/grammar-selector/lib/grammar-list-view.js @@ -26,10 +26,10 @@ module.exports = class GrammarListView { let badgeColor = 'badge-success'; let badgeText = 'Tree-sitter'; - if (isExperimentalTreeSitterMode()) { - badgeColor = isModernTreeSitter(grammar) ? + if (isLegacyTreeSitterMode()) { + badgeColor = isLegacyTreeSitter(grammar) ? 'badge-success' : 'badge-warning'; - badgeText = isModernTreeSitter(grammar) ? + badgeText = isLegacyTreeSitter(grammar) ? 'Tree-sitter' : 'Legacy Tree-sitter'; } @@ -118,13 +118,15 @@ module.exports = class GrammarListView { return grammar !== atom.grammars.nullGrammar && grammar.name; }); - // Don't show modern tree-sitter grammars in the selector unless the user + // Don't show legacy Tree-sitter grammars in the selector unless the user // has opted into it. - if (!isExperimentalTreeSitterMode()) { - grammars = grammars.filter(grammar => !isModernTreeSitter(grammar)); + if (!isLegacyTreeSitterMode()) { + grammars = grammars.filter(grammar => !isLegacyTreeSitter(grammar)); } if (atom.config.get('grammar-selector.hideDuplicateTextMateGrammars')) { + // Filter out all TextMate grammars for which there is a Tree-sitter + // grammar with the exact same name. const blacklist = new Set(); grammars.forEach(grammar => { if (isTreeSitter(grammar)) { @@ -155,24 +157,24 @@ module.exports = class GrammarListView { function getLanguageModeConfig() { let isTreeSitterMode = atom.config.get('core.useTreeSitterParsers'); - let isExperimental = atom.config.get('core.useExperimentalModernTreeSitter'); + let isLegacy = atom.config.get('core.useLegacyTreeSitter'); if (!isTreeSitterMode) return 'textmate'; - return isExperimental ? 'wasm-tree-sitter' : 'node-tree-sitter'; + return isLegacy ? 'node-tree-sitter' : 'wasm-tree-sitter'; } -function isExperimentalTreeSitterMode() { - return getLanguageModeConfig() === 'wasm-tree-sitter'; +function isLegacyTreeSitterMode() { + return getLanguageModeConfig() === 'node-tree-sitter'; } function isTreeSitter(grammar) { - return isOldTreeSitter(grammar) || isModernTreeSitter(grammar); + return isLegacyTreeSitter(grammar) || isModernTreeSitter(grammar); } function isModernTreeSitter(grammar) { return grammar.constructor.name === 'WASMTreeSitterGrammar'; } -function isOldTreeSitter(grammar) { +function isLegacyTreeSitter(grammar) { return grammar.constructor.name === 'TreeSitterGrammar'; } @@ -183,6 +185,6 @@ function compareGrammarType(a, b) { function getGrammarScore(grammar) { let languageParser = getLanguageModeConfig(); if (isModernTreeSitter(grammar)) { return -2; } - if (isOldTreeSitter(grammar)) { return -1; } + if (isLegacyTreeSitter(grammar)) { return -1; } return languageParser === 'textmate' ? -3 : 0; } diff --git a/packages/grammar-selector/spec/.eslintrc.js b/packages/grammar-selector/spec/.eslintrc.js new file mode 100644 index 000000000..746f7afb7 --- /dev/null +++ b/packages/grammar-selector/spec/.eslintrc.js @@ -0,0 +1,8 @@ +module.exports = { + env: { jasmine: true }, + rules: { + "node/no-unpublished-require": "off", + "no-unused-vars": "off", + "no-empty": "off" + } +}; diff --git a/packages/grammar-selector/spec/grammar-selector-spec.js b/packages/grammar-selector/spec/grammar-selector-spec.js index 4f13eadbe..118c09dd1 100644 --- a/packages/grammar-selector/spec/grammar-selector-spec.js +++ b/packages/grammar-selector/spec/grammar-selector-spec.js @@ -3,9 +3,9 @@ const SelectListView = require('atom-select-list'); function setConfigForLanguageMode(mode) { let useTreeSitterParsers = mode !== 'textmate'; - let useExperimentalModernTreeSitter = mode === 'wasm-tree-sitter'; + let useLegacyTreeSitter = mode === 'node-tree-sitter'; atom.config.set('core.useTreeSitterParsers', useTreeSitterParsers); - atom.config.set('core.useExperimentalModernTreeSitter', useExperimentalModernTreeSitter); + atom.config.set('core.useLegacyTreeSitter', useLegacyTreeSitter); } describe('GrammarSelector', () => { diff --git a/packages/language-clojure/spec/.eslintrc.js b/packages/language-clojure/spec/.eslintrc.js new file mode 100644 index 000000000..5226d6921 --- /dev/null +++ b/packages/language-clojure/spec/.eslintrc.js @@ -0,0 +1,9 @@ +module.exports = { + env: { jasmine: true }, + rules: { + "node/no-unpublished-require": "off", + "node/no-extraneous-require": "off", + "no-unused-vars": "off", + "no-empty": "off" + } +}; diff --git a/packages/language-clojure/spec/clojure-spec.js b/packages/language-clojure/spec/clojure-spec.js index bf7cb0a46..8f58b9a3a 100644 --- a/packages/language-clojure/spec/clojure-spec.js +++ b/packages/language-clojure/spec/clojure-spec.js @@ -4,7 +4,6 @@ describe("Clojure grammar", function() { beforeEach(function() { atom.config.set('core.useTreeSitterParsers', false); - atom.config.set('core.useExperimentalModernTreeSitter', false); waitsForPromise(() => atom.packages.activatePackage("language-clojure")); diff --git a/packages/language-clojure/spec/tokenizer-spec.js b/packages/language-clojure/spec/tokenizer-spec.js index 969966320..a12e60851 100644 --- a/packages/language-clojure/spec/tokenizer-spec.js +++ b/packages/language-clojure/spec/tokenizer-spec.js @@ -3,9 +3,7 @@ const path = require('path'); function setConfigForLanguageMode(mode) { let useTreeSitterParsers = mode !== 'textmate'; - let useExperimentalModernTreeSitter = mode === 'modern-tree-sitter'; atom.config.set('core.useTreeSitterParsers', useTreeSitterParsers); - atom.config.set('core.useExperimentalModernTreeSitter', useExperimentalModernTreeSitter); } describe('Clojure grammars', () => { diff --git a/packages/language-html/spec/tree-sitter-spec.js b/packages/language-html/spec/tree-sitter-spec.js index 8fd99ae7b..973dd4597 100644 --- a/packages/language-html/spec/tree-sitter-spec.js +++ b/packages/language-html/spec/tree-sitter-spec.js @@ -3,7 +3,7 @@ const dedent = require('dedent'); describe('Tree-sitter HTML grammar', () => { beforeEach(async () => { atom.config.set('core.useTreeSitterParsers', true); - atom.config.set('core.useExperimentalModernTreeSitter', false); + atom.config.set('core.useLegacyTreeSitter', true); await atom.packages.activatePackage('language-html'); }); diff --git a/packages/language-java/spec/.eslintrc.js b/packages/language-java/spec/.eslintrc.js new file mode 100644 index 000000000..0e6313258 --- /dev/null +++ b/packages/language-java/spec/.eslintrc.js @@ -0,0 +1,12 @@ +module.exports = { + env: { jasmine: true }, + globals: { + waitsForPromise: true + }, + rules: { + "node/no-unpublished-require": "off", + "node/no-extraneous-require": "off", + "no-unused-vars": "off", + "no-empty": "off" + } +}; diff --git a/packages/language-java/spec/tree-sitter-java-spec.js b/packages/language-java/spec/tree-sitter-java-spec.js index b187082fd..af12e6a04 100644 --- a/packages/language-java/spec/tree-sitter-java-spec.js +++ b/packages/language-java/spec/tree-sitter-java-spec.js @@ -8,7 +8,7 @@ describe('Tree-sitter based Java grammar', function() { beforeEach(function() { atom.config.set('core.useTreeSitterParsers', true); - atom.config.set('core.useExperimentalModernTreeSitter', false); + atom.config.set('core.useLegacyTreeSitter', true); waitsForPromise(() => atom.packages.activatePackage('language-java')); diff --git a/packages/language-ruby/spec/.eslintrc.js b/packages/language-ruby/spec/.eslintrc.js new file mode 100644 index 000000000..36c4dcd82 --- /dev/null +++ b/packages/language-ruby/spec/.eslintrc.js @@ -0,0 +1,15 @@ +module.exports = { + env: { jasmine: true }, + globals: { + waitsForPromise: true, + runGrammarTests: true, + runFoldsTests: true, + normalizeTreeSitterTextData: true + }, + rules: { + "node/no-unpublished-require": "off", + "node/no-extraneous-require": "off", + "no-unused-vars": "off", + "no-empty": "off" + } +}; diff --git a/packages/language-ruby/spec/tokenizer-spec.js b/packages/language-ruby/spec/tokenizer-spec.js index 5960207d3..7cf8d71fd 100644 --- a/packages/language-ruby/spec/tokenizer-spec.js +++ b/packages/language-ruby/spec/tokenizer-spec.js @@ -16,7 +16,7 @@ describe('Ruby grammars', () => { xit('tokenizes the editor using node tree-sitter parser', async () => { atom.config.set('core.useTreeSitterParsers', true); - atom.config.set('core.useExperimentalModernTreeSitter', false); + atom.config.set('core.useLegacyTreeSitter', true); await runGrammarTests(path.join(__dirname, 'fixtures', 'textmate-grammar.rb'), /#/) }); }); diff --git a/packages/language-ruby/spec/tree-sitter-spec.js b/packages/language-ruby/spec/tree-sitter-spec.js index 7fce8a75f..01d243dc5 100644 --- a/packages/language-ruby/spec/tree-sitter-spec.js +++ b/packages/language-ruby/spec/tree-sitter-spec.js @@ -3,7 +3,7 @@ const dedent = require('dedent'); describe('Tree-sitter Ruby grammar', () => { beforeEach(async () => { atom.config.set('core.useTreeSitterParsers', true); - atom.config.set('core.useExperimentalModernTreeSitter', false); + atom.config.set('core.useLegacyTreeSitter', true); await atom.packages.activatePackage('language-ruby'); }); diff --git a/packages/language-ruby/spec/wasm-tree-sitter-spec.js b/packages/language-ruby/spec/wasm-tree-sitter-spec.js index 711b5aec1..10755671a 100644 --- a/packages/language-ruby/spec/wasm-tree-sitter-spec.js +++ b/packages/language-ruby/spec/wasm-tree-sitter-spec.js @@ -7,7 +7,6 @@ xdescribe('WASM Tree-sitter Ruby grammar', () => { beforeEach(async () => { await atom.packages.activatePackage('language-ruby'); atom.config.set('core.useTreeSitterParsers', true); - atom.config.set('core.useExperimentalModernTreeSitter', true); }); it('tokenizes symbols', async () => { diff --git a/spec/grammar-registry-spec.js b/spec/grammar-registry-spec.js index 66b3a78cb..fb5c33b42 100644 --- a/spec/grammar-registry-spec.js +++ b/spec/grammar-registry-spec.js @@ -11,9 +11,9 @@ const { OnigScanner } = SecondMate; // Expects one of `textmate`, `node-tree-sitter`, or `wasm-tree-sitter`. function setConfigForLanguageMode(mode, options = {}) { let useTreeSitterParsers = mode !== 'textmate'; - let useExperimentalModernTreeSitter = mode === 'wasm-tree-sitter'; + let useLegacyTreeSitter = mode === 'node-tree-sitter'; atom.config.set('core.useTreeSitterParsers', useTreeSitterParsers, options); - atom.config.set('core.useExperimentalModernTreeSitter', useExperimentalModernTreeSitter, options); + atom.config.set('core.useLegacyTreeSitter', useLegacyTreeSitter, options); } describe('GrammarRegistry', () => { diff --git a/spec/scope-resolver-spec.js b/spec/scope-resolver-spec.js index f3701241d..ec6fc7f7a 100644 --- a/spec/scope-resolver-spec.js +++ b/spec/scope-resolver-spec.js @@ -93,7 +93,7 @@ function rangeFromDescriptor(rawRange) { } describe('ScopeResolver', () => { - let editor, buffer, grammar, scopeResolver; + let editor, buffer, grammar; beforeEach(async () => { grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig); @@ -101,7 +101,6 @@ describe('ScopeResolver', () => { buffer = editor.getBuffer(); atom.grammars.addGrammar(grammar); atom.config.set('core.useTreeSitterParsers', true); - atom.config.set('core.useExperimentalModernTreeSitter', true); }); afterEach(() => { @@ -126,7 +125,7 @@ describe('ScopeResolver', () => { let { scopeResolver, captures } = await getAllCaptures(grammar, languageMode); for (let capture of captures) { - let { node, name } = capture; + let { node } = capture; let range = scopeResolver.store(capture); expect(stringForNodeRange(range)) .toBe(stringForNodeRange(node)); diff --git a/spec/tree-sitter-language-mode-spec.js b/spec/tree-sitter-language-mode-spec.js index 32a1e2c12..889e7e30a 100644 --- a/spec/tree-sitter-language-mode-spec.js +++ b/spec/tree-sitter-language-mode-spec.js @@ -42,7 +42,7 @@ describe('TreeSitterLanguageMode', () => { buffer = editor.getBuffer(); editor.displayLayer.reset({ foldCharacter: '…' }); atom.config.set('core.useTreeSitterParsers', true); - atom.config.set('core.useExperimentalModernTreeSitter', false); + atom.config.set('core.useLegacyTreeSitter', true); }); describe('highlighting', () => { diff --git a/spec/wasm-tree-sitter-language-mode-spec.js b/spec/wasm-tree-sitter-language-mode-spec.js index 9937880c8..dcc78a60e 100644 --- a/spec/wasm-tree-sitter-language-mode-spec.js +++ b/spec/wasm-tree-sitter-language-mode-spec.js @@ -66,7 +66,6 @@ describe('WASMTreeSitterLanguageMode', () => { buffer = editor.getBuffer(); editor.displayLayer.reset({ foldCharacter: '…' }); atom.config.set('core.useTreeSitterParsers', true); - atom.config.set('core.useExperimentalModernTreeSitter', true); }); afterEach(() => { diff --git a/src/config-schema.js b/src/config-schema.js index 29108d6a9..52ed1c965 100644 --- a/src/config-schema.js +++ b/src/config-schema.js @@ -360,13 +360,14 @@ const configSchema = { useTreeSitterParsers: { type: 'boolean', default: true, + title: 'Use Tree-sitter Parsers', description: 'Use Tree-sitter parsers for supported languages.' }, - useExperimentalModernTreeSitter: { + useLegacyTreeSitter: { type: 'boolean', default: false, - title: 'Use Modern Tree-Sitter Implementation', - description: 'Experimental: Use the new query-file-based Tree-sitter system instead of the legacy system from Atom. (This system will eventually replace the legacy system.) Has no effect unless "Use Tree Sitter Parsers" is also checked.' + title: 'Use legacy Tree-sitter Implementation', + description: 'Opt into the legacy Atom Tree-sitter system instead of the modern system added by Pulsar. (This legacy system will soon be removed.) Has no effect unless “Use Tree-sitter Parsers” is also checked.' }, colorProfile: { description: diff --git a/src/grammar-registry.js b/src/grammar-registry.js index f4852d70e..c5eeeb8d2 100644 --- a/src/grammar-registry.js +++ b/src/grammar-registry.js @@ -46,7 +46,7 @@ module.exports = class GrammarRegistry { this.textmateRegistry.onDidUpdateGrammar(grammarAddedOrUpdated); let onLanguageModeChange = () => { - this.grammarScoresByBuffer.forEach((score, buffer) => { + this.grammarScoresByBuffer.forEach((_score, buffer) => { if (!this.languageOverridesByBufferId.has(buffer.id)) { this.autoAssignLanguageMode(buffer); } @@ -55,7 +55,7 @@ module.exports = class GrammarRegistry { this.subscriptions.add( this.config.onDidChange('core.useTreeSitterParsers', onLanguageModeChange), - this.config.onDidChange('core.useExperimentalModernTreeSitter', onLanguageModeChange) + this.config.onDidChange('core.useLegacyTreeSitter', onLanguageModeChange) ); } @@ -253,10 +253,10 @@ module.exports = class GrammarRegistry { scope = new ScopeDescriptor({ scopes: [scope] }) } let useTreeSitterParsers = this.config.get('core.useTreeSitterParsers', { scope }); - let useExperimentalModernTreeSitter = this.config.get('core.useExperimentalModernTreeSitter', { scope }); + let useLegacyTreeSitter = this.config.get('core.useLegacyTreeSitter', { scope }); if (!useTreeSitterParsers) return 'textmate'; - return useExperimentalModernTreeSitter ? 'wasm-tree-sitter' : 'node-tree-sitter'; + return useLegacyTreeSitter ? 'node-tree-sitter' : 'wasm-tree-sitter'; } // Extended: Returns a {Number} representing how well the grammar matches the @@ -288,19 +288,15 @@ module.exports = class GrammarRegistry { if (isNewTreeSitter) { if (parserConfig === 'wasm-tree-sitter') { score += 0.1; - } else { - // Never automatically switch to a new Tree-sitter grammar unless the - // user has opted into the experimental setting. - score = -Infinity; } } else if (isOldTreeSitter) { if (parserConfig === 'node-tree-sitter') { score += 0.1; } else if (parserConfig === 'wasm-tree-sitter') { - // In experimental new-tree-sitter mode, we still would rather fall - // back to an old-tree-sitter grammar than a TM grammar. Bump the + // If `useLegacyTreeSitter` isn't checked, we probably still prefer a + // legacy Tree-sitter grammar over a TextMate-style grammar. Bump the // score, but just a bit less than we'd bump it if this were a - // new-tree-sitter grammar. + // modern Tree-sitter grammar. score += 0.09; } } @@ -770,17 +766,17 @@ module.exports = class GrammarRegistry { let result = this.textmateRegistry.getGrammars(); if (!(params && params.includeTreeSitter)) return result; - let includeModernTreeSitterGrammars = - atom.config.get('core.useExperimentalModernTreeSitter') === true; + let includeLegacyTreeSitterGrammars = + atom.config.get('core.useLegacyTreeSitter') === true; - const tsGrammars = Object.values(this.treeSitterGrammarsById) + let modernTsGrammars = Object.values(this.wasmTreeSitterGrammarsById) .filter(g => g.scopeName); - result = result.concat(tsGrammars); + result = result.concat(modernTsGrammars); - if (includeModernTreeSitterGrammars) { - let modernTsGrammars = Object.values(this.wasmTreeSitterGrammarsById) + if (includeLegacyTreeSitterGrammars) { + const legacyTsGrammars = Object.values(this.treeSitterGrammarsById) .filter(g => g.scopeName); - result = result.concat(modernTsGrammars); + result = result.concat(legacyTsGrammars); } return result; @@ -790,8 +786,10 @@ module.exports = class GrammarRegistry { return this.textmateRegistry.scopeForId(id); } - // TODO: why is this being used? Can we remove it soon? - treeSitterGrammarForLanguageString(languageString, type = 'original') { + // Match up a language string (of the sort generated by an injection point) + // with a grammar. Checks the `injectionRegex` property on grammars and + // returns the one with the longest match. + treeSitterGrammarForLanguageString(languageString, type = 'wasm') { let longestMatchLength = 0; let grammarWithLongestMatch = null; let table = type === 'original' ? this.treeSitterGrammarsById : this.wasmTreeSitterGrammarsById;