mirror of
https://github.com/pulsar-edit/pulsar.git
synced 2024-08-16 14:40:23 +03:00
Make useExperimentalModernTreeSitter
the default…
…and create `useLegacyTreeSitter` for those who want to opt into the previous default behavior. (Legacy Tree-sitter grammars will soon be removed, but this is a step toward that future!)
This commit is contained in:
parent
fd908ca0b7
commit
76ac2cf81c
@ -33,7 +33,7 @@ const packagesToTest = {
|
||||
|
||||
const wait = (ms) => new Promise(resolve => setTimeout(resolve, ms));
|
||||
|
||||
const whenEditorReady = function(editor) {
|
||||
const whenEditorReady = function (editor) {
|
||||
const languageMode = editor.getBuffer().getLanguageMode();
|
||||
if (!languageMode.constructor.name.includes('TreeSitter')) {
|
||||
return Promise.resolve();
|
||||
@ -105,8 +105,6 @@ describe("CSS property name and value autocompletions", async () => {
|
||||
await atom.workspace.open(packagesToTest[packageLabel].file);
|
||||
editor = atom.workspace.getActiveTextEditor();
|
||||
await whenEditorReady(editor);
|
||||
console.warn('USING TREE SITTER?!?', packageLabel, meta.useTreeSitter);
|
||||
atom.config.set('core.useExperimentalModernTreeSitter', meta.useTreeSitter ?? false);
|
||||
atom.config.set('core.useTreeSitterParsers', meta.useTreeSitter ?? false);
|
||||
});
|
||||
|
||||
@ -738,7 +736,7 @@ div:nth {
|
||||
})
|
||||
);
|
||||
|
||||
Object.keys(packagesToTest).forEach(function(packageLabel) {
|
||||
Object.keys(packagesToTest).forEach(function (packageLabel) {
|
||||
if (packagesToTest[packageLabel].name !== 'language-css') {
|
||||
describe(`${packageLabel} files`, async () => {
|
||||
beforeEach(async () => {
|
||||
|
12
packages/autocomplete-html/spec/.eslintrc.js
Normal file
12
packages/autocomplete-html/spec/.eslintrc.js
Normal file
@ -0,0 +1,12 @@
|
||||
module.exports = {
|
||||
env: { jasmine: true },
|
||||
globals: {
|
||||
"waitsForPromise": true
|
||||
},
|
||||
rules: {
|
||||
"node/no-unpublished-require": "off",
|
||||
"node/no-extraneous-require": "off",
|
||||
"no-unused-vars": "off",
|
||||
"no-empty": "off"
|
||||
}
|
||||
};
|
@ -26,10 +26,10 @@ module.exports = class GrammarListView {
|
||||
let badgeColor = 'badge-success';
|
||||
let badgeText = 'Tree-sitter';
|
||||
|
||||
if (isExperimentalTreeSitterMode()) {
|
||||
badgeColor = isModernTreeSitter(grammar) ?
|
||||
if (isLegacyTreeSitterMode()) {
|
||||
badgeColor = isLegacyTreeSitter(grammar) ?
|
||||
'badge-success' : 'badge-warning';
|
||||
badgeText = isModernTreeSitter(grammar) ?
|
||||
badgeText = isLegacyTreeSitter(grammar) ?
|
||||
'Tree-sitter' : 'Legacy Tree-sitter';
|
||||
}
|
||||
|
||||
@ -118,13 +118,15 @@ module.exports = class GrammarListView {
|
||||
return grammar !== atom.grammars.nullGrammar && grammar.name;
|
||||
});
|
||||
|
||||
// Don't show modern tree-sitter grammars in the selector unless the user
|
||||
// Don't show legacy Tree-sitter grammars in the selector unless the user
|
||||
// has opted into it.
|
||||
if (!isExperimentalTreeSitterMode()) {
|
||||
grammars = grammars.filter(grammar => !isModernTreeSitter(grammar));
|
||||
if (!isLegacyTreeSitterMode()) {
|
||||
grammars = grammars.filter(grammar => !isLegacyTreeSitter(grammar));
|
||||
}
|
||||
|
||||
if (atom.config.get('grammar-selector.hideDuplicateTextMateGrammars')) {
|
||||
// Filter out all TextMate grammars for which there is a Tree-sitter
|
||||
// grammar with the exact same name.
|
||||
const blacklist = new Set();
|
||||
grammars.forEach(grammar => {
|
||||
if (isTreeSitter(grammar)) {
|
||||
@ -155,24 +157,24 @@ module.exports = class GrammarListView {
|
||||
|
||||
function getLanguageModeConfig() {
|
||||
let isTreeSitterMode = atom.config.get('core.useTreeSitterParsers');
|
||||
let isExperimental = atom.config.get('core.useExperimentalModernTreeSitter');
|
||||
let isLegacy = atom.config.get('core.useLegacyTreeSitter');
|
||||
if (!isTreeSitterMode) return 'textmate';
|
||||
return isExperimental ? 'wasm-tree-sitter' : 'node-tree-sitter';
|
||||
return isLegacy ? 'node-tree-sitter' : 'wasm-tree-sitter';
|
||||
}
|
||||
|
||||
function isExperimentalTreeSitterMode() {
|
||||
return getLanguageModeConfig() === 'wasm-tree-sitter';
|
||||
function isLegacyTreeSitterMode() {
|
||||
return getLanguageModeConfig() === 'node-tree-sitter';
|
||||
}
|
||||
|
||||
function isTreeSitter(grammar) {
|
||||
return isOldTreeSitter(grammar) || isModernTreeSitter(grammar);
|
||||
return isLegacyTreeSitter(grammar) || isModernTreeSitter(grammar);
|
||||
}
|
||||
|
||||
function isModernTreeSitter(grammar) {
|
||||
return grammar.constructor.name === 'WASMTreeSitterGrammar';
|
||||
}
|
||||
|
||||
function isOldTreeSitter(grammar) {
|
||||
function isLegacyTreeSitter(grammar) {
|
||||
return grammar.constructor.name === 'TreeSitterGrammar';
|
||||
}
|
||||
|
||||
@ -183,6 +185,6 @@ function compareGrammarType(a, b) {
|
||||
function getGrammarScore(grammar) {
|
||||
let languageParser = getLanguageModeConfig();
|
||||
if (isModernTreeSitter(grammar)) { return -2; }
|
||||
if (isOldTreeSitter(grammar)) { return -1; }
|
||||
if (isLegacyTreeSitter(grammar)) { return -1; }
|
||||
return languageParser === 'textmate' ? -3 : 0;
|
||||
}
|
||||
|
8
packages/grammar-selector/spec/.eslintrc.js
Normal file
8
packages/grammar-selector/spec/.eslintrc.js
Normal file
@ -0,0 +1,8 @@
|
||||
module.exports = {
|
||||
env: { jasmine: true },
|
||||
rules: {
|
||||
"node/no-unpublished-require": "off",
|
||||
"no-unused-vars": "off",
|
||||
"no-empty": "off"
|
||||
}
|
||||
};
|
@ -3,9 +3,9 @@ const SelectListView = require('atom-select-list');
|
||||
|
||||
function setConfigForLanguageMode(mode) {
|
||||
let useTreeSitterParsers = mode !== 'textmate';
|
||||
let useExperimentalModernTreeSitter = mode === 'wasm-tree-sitter';
|
||||
let useLegacyTreeSitter = mode === 'node-tree-sitter';
|
||||
atom.config.set('core.useTreeSitterParsers', useTreeSitterParsers);
|
||||
atom.config.set('core.useExperimentalModernTreeSitter', useExperimentalModernTreeSitter);
|
||||
atom.config.set('core.useLegacyTreeSitter', useLegacyTreeSitter);
|
||||
}
|
||||
|
||||
describe('GrammarSelector', () => {
|
||||
|
9
packages/language-clojure/spec/.eslintrc.js
Normal file
9
packages/language-clojure/spec/.eslintrc.js
Normal file
@ -0,0 +1,9 @@
|
||||
module.exports = {
|
||||
env: { jasmine: true },
|
||||
rules: {
|
||||
"node/no-unpublished-require": "off",
|
||||
"node/no-extraneous-require": "off",
|
||||
"no-unused-vars": "off",
|
||||
"no-empty": "off"
|
||||
}
|
||||
};
|
@ -4,7 +4,6 @@ describe("Clojure grammar", function() {
|
||||
|
||||
beforeEach(function() {
|
||||
atom.config.set('core.useTreeSitterParsers', false);
|
||||
atom.config.set('core.useExperimentalModernTreeSitter', false);
|
||||
|
||||
waitsForPromise(() => atom.packages.activatePackage("language-clojure"));
|
||||
|
||||
|
@ -3,9 +3,7 @@ const path = require('path');
|
||||
|
||||
function setConfigForLanguageMode(mode) {
|
||||
let useTreeSitterParsers = mode !== 'textmate';
|
||||
let useExperimentalModernTreeSitter = mode === 'modern-tree-sitter';
|
||||
atom.config.set('core.useTreeSitterParsers', useTreeSitterParsers);
|
||||
atom.config.set('core.useExperimentalModernTreeSitter', useExperimentalModernTreeSitter);
|
||||
}
|
||||
|
||||
describe('Clojure grammars', () => {
|
||||
|
@ -3,7 +3,7 @@ const dedent = require('dedent');
|
||||
describe('Tree-sitter HTML grammar', () => {
|
||||
beforeEach(async () => {
|
||||
atom.config.set('core.useTreeSitterParsers', true);
|
||||
atom.config.set('core.useExperimentalModernTreeSitter', false);
|
||||
atom.config.set('core.useLegacyTreeSitter', true);
|
||||
await atom.packages.activatePackage('language-html');
|
||||
});
|
||||
|
||||
|
12
packages/language-java/spec/.eslintrc.js
Normal file
12
packages/language-java/spec/.eslintrc.js
Normal file
@ -0,0 +1,12 @@
|
||||
module.exports = {
|
||||
env: { jasmine: true },
|
||||
globals: {
|
||||
waitsForPromise: true
|
||||
},
|
||||
rules: {
|
||||
"node/no-unpublished-require": "off",
|
||||
"node/no-extraneous-require": "off",
|
||||
"no-unused-vars": "off",
|
||||
"no-empty": "off"
|
||||
}
|
||||
};
|
@ -8,7 +8,7 @@ describe('Tree-sitter based Java grammar', function() {
|
||||
|
||||
beforeEach(function() {
|
||||
atom.config.set('core.useTreeSitterParsers', true);
|
||||
atom.config.set('core.useExperimentalModernTreeSitter', false);
|
||||
atom.config.set('core.useLegacyTreeSitter', true);
|
||||
|
||||
waitsForPromise(() => atom.packages.activatePackage('language-java'));
|
||||
|
||||
|
15
packages/language-ruby/spec/.eslintrc.js
Normal file
15
packages/language-ruby/spec/.eslintrc.js
Normal file
@ -0,0 +1,15 @@
|
||||
module.exports = {
|
||||
env: { jasmine: true },
|
||||
globals: {
|
||||
waitsForPromise: true,
|
||||
runGrammarTests: true,
|
||||
runFoldsTests: true,
|
||||
normalizeTreeSitterTextData: true
|
||||
},
|
||||
rules: {
|
||||
"node/no-unpublished-require": "off",
|
||||
"node/no-extraneous-require": "off",
|
||||
"no-unused-vars": "off",
|
||||
"no-empty": "off"
|
||||
}
|
||||
};
|
@ -16,7 +16,7 @@ describe('Ruby grammars', () => {
|
||||
|
||||
xit('tokenizes the editor using node tree-sitter parser', async () => {
|
||||
atom.config.set('core.useTreeSitterParsers', true);
|
||||
atom.config.set('core.useExperimentalModernTreeSitter', false);
|
||||
atom.config.set('core.useLegacyTreeSitter', true);
|
||||
await runGrammarTests(path.join(__dirname, 'fixtures', 'textmate-grammar.rb'), /#/)
|
||||
});
|
||||
});
|
||||
|
@ -3,7 +3,7 @@ const dedent = require('dedent');
|
||||
describe('Tree-sitter Ruby grammar', () => {
|
||||
beforeEach(async () => {
|
||||
atom.config.set('core.useTreeSitterParsers', true);
|
||||
atom.config.set('core.useExperimentalModernTreeSitter', false);
|
||||
atom.config.set('core.useLegacyTreeSitter', true);
|
||||
await atom.packages.activatePackage('language-ruby');
|
||||
});
|
||||
|
||||
|
@ -7,7 +7,6 @@ xdescribe('WASM Tree-sitter Ruby grammar', () => {
|
||||
beforeEach(async () => {
|
||||
await atom.packages.activatePackage('language-ruby');
|
||||
atom.config.set('core.useTreeSitterParsers', true);
|
||||
atom.config.set('core.useExperimentalModernTreeSitter', true);
|
||||
});
|
||||
|
||||
it('tokenizes symbols', async () => {
|
||||
|
@ -11,9 +11,9 @@ const { OnigScanner } = SecondMate;
|
||||
// Expects one of `textmate`, `node-tree-sitter`, or `wasm-tree-sitter`.
|
||||
function setConfigForLanguageMode(mode, options = {}) {
|
||||
let useTreeSitterParsers = mode !== 'textmate';
|
||||
let useExperimentalModernTreeSitter = mode === 'wasm-tree-sitter';
|
||||
let useLegacyTreeSitter = mode === 'node-tree-sitter';
|
||||
atom.config.set('core.useTreeSitterParsers', useTreeSitterParsers, options);
|
||||
atom.config.set('core.useExperimentalModernTreeSitter', useExperimentalModernTreeSitter, options);
|
||||
atom.config.set('core.useLegacyTreeSitter', useLegacyTreeSitter, options);
|
||||
}
|
||||
|
||||
describe('GrammarRegistry', () => {
|
||||
|
@ -93,7 +93,7 @@ function rangeFromDescriptor(rawRange) {
|
||||
}
|
||||
|
||||
describe('ScopeResolver', () => {
|
||||
let editor, buffer, grammar, scopeResolver;
|
||||
let editor, buffer, grammar;
|
||||
|
||||
beforeEach(async () => {
|
||||
grammar = new WASMTreeSitterGrammar(atom.grammars, jsGrammarPath, jsConfig);
|
||||
@ -101,7 +101,6 @@ describe('ScopeResolver', () => {
|
||||
buffer = editor.getBuffer();
|
||||
atom.grammars.addGrammar(grammar);
|
||||
atom.config.set('core.useTreeSitterParsers', true);
|
||||
atom.config.set('core.useExperimentalModernTreeSitter', true);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
@ -126,7 +125,7 @@ describe('ScopeResolver', () => {
|
||||
let { scopeResolver, captures } = await getAllCaptures(grammar, languageMode);
|
||||
|
||||
for (let capture of captures) {
|
||||
let { node, name } = capture;
|
||||
let { node } = capture;
|
||||
let range = scopeResolver.store(capture);
|
||||
expect(stringForNodeRange(range))
|
||||
.toBe(stringForNodeRange(node));
|
||||
|
@ -42,7 +42,7 @@ describe('TreeSitterLanguageMode', () => {
|
||||
buffer = editor.getBuffer();
|
||||
editor.displayLayer.reset({ foldCharacter: '…' });
|
||||
atom.config.set('core.useTreeSitterParsers', true);
|
||||
atom.config.set('core.useExperimentalModernTreeSitter', false);
|
||||
atom.config.set('core.useLegacyTreeSitter', true);
|
||||
});
|
||||
|
||||
describe('highlighting', () => {
|
||||
|
@ -66,7 +66,6 @@ describe('WASMTreeSitterLanguageMode', () => {
|
||||
buffer = editor.getBuffer();
|
||||
editor.displayLayer.reset({ foldCharacter: '…' });
|
||||
atom.config.set('core.useTreeSitterParsers', true);
|
||||
atom.config.set('core.useExperimentalModernTreeSitter', true);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
|
@ -360,13 +360,14 @@ const configSchema = {
|
||||
useTreeSitterParsers: {
|
||||
type: 'boolean',
|
||||
default: true,
|
||||
title: 'Use Tree-sitter Parsers',
|
||||
description: 'Use Tree-sitter parsers for supported languages.'
|
||||
},
|
||||
useExperimentalModernTreeSitter: {
|
||||
useLegacyTreeSitter: {
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
title: 'Use Modern Tree-Sitter Implementation',
|
||||
description: 'Experimental: Use the new query-file-based Tree-sitter system instead of the legacy system from Atom. (This system will eventually replace the legacy system.) Has no effect unless "Use Tree Sitter Parsers" is also checked.'
|
||||
title: 'Use legacy Tree-sitter Implementation',
|
||||
description: 'Opt into the legacy Atom Tree-sitter system instead of the modern system added by Pulsar. (This legacy system will soon be removed.) Has no effect unless “Use Tree-sitter Parsers” is also checked.'
|
||||
},
|
||||
colorProfile: {
|
||||
description:
|
||||
|
@ -46,7 +46,7 @@ module.exports = class GrammarRegistry {
|
||||
this.textmateRegistry.onDidUpdateGrammar(grammarAddedOrUpdated);
|
||||
|
||||
let onLanguageModeChange = () => {
|
||||
this.grammarScoresByBuffer.forEach((score, buffer) => {
|
||||
this.grammarScoresByBuffer.forEach((_score, buffer) => {
|
||||
if (!this.languageOverridesByBufferId.has(buffer.id)) {
|
||||
this.autoAssignLanguageMode(buffer);
|
||||
}
|
||||
@ -55,7 +55,7 @@ module.exports = class GrammarRegistry {
|
||||
|
||||
this.subscriptions.add(
|
||||
this.config.onDidChange('core.useTreeSitterParsers', onLanguageModeChange),
|
||||
this.config.onDidChange('core.useExperimentalModernTreeSitter', onLanguageModeChange)
|
||||
this.config.onDidChange('core.useLegacyTreeSitter', onLanguageModeChange)
|
||||
);
|
||||
}
|
||||
|
||||
@ -253,10 +253,10 @@ module.exports = class GrammarRegistry {
|
||||
scope = new ScopeDescriptor({ scopes: [scope] })
|
||||
}
|
||||
let useTreeSitterParsers = this.config.get('core.useTreeSitterParsers', { scope });
|
||||
let useExperimentalModernTreeSitter = this.config.get('core.useExperimentalModernTreeSitter', { scope });
|
||||
let useLegacyTreeSitter = this.config.get('core.useLegacyTreeSitter', { scope });
|
||||
|
||||
if (!useTreeSitterParsers) return 'textmate';
|
||||
return useExperimentalModernTreeSitter ? 'wasm-tree-sitter' : 'node-tree-sitter';
|
||||
return useLegacyTreeSitter ? 'node-tree-sitter' : 'wasm-tree-sitter';
|
||||
}
|
||||
|
||||
// Extended: Returns a {Number} representing how well the grammar matches the
|
||||
@ -288,19 +288,15 @@ module.exports = class GrammarRegistry {
|
||||
if (isNewTreeSitter) {
|
||||
if (parserConfig === 'wasm-tree-sitter') {
|
||||
score += 0.1;
|
||||
} else {
|
||||
// Never automatically switch to a new Tree-sitter grammar unless the
|
||||
// user has opted into the experimental setting.
|
||||
score = -Infinity;
|
||||
}
|
||||
} else if (isOldTreeSitter) {
|
||||
if (parserConfig === 'node-tree-sitter') {
|
||||
score += 0.1;
|
||||
} else if (parserConfig === 'wasm-tree-sitter') {
|
||||
// In experimental new-tree-sitter mode, we still would rather fall
|
||||
// back to an old-tree-sitter grammar than a TM grammar. Bump the
|
||||
// If `useLegacyTreeSitter` isn't checked, we probably still prefer a
|
||||
// legacy Tree-sitter grammar over a TextMate-style grammar. Bump the
|
||||
// score, but just a bit less than we'd bump it if this were a
|
||||
// new-tree-sitter grammar.
|
||||
// modern Tree-sitter grammar.
|
||||
score += 0.09;
|
||||
}
|
||||
}
|
||||
@ -770,17 +766,17 @@ module.exports = class GrammarRegistry {
|
||||
let result = this.textmateRegistry.getGrammars();
|
||||
if (!(params && params.includeTreeSitter)) return result;
|
||||
|
||||
let includeModernTreeSitterGrammars =
|
||||
atom.config.get('core.useExperimentalModernTreeSitter') === true;
|
||||
let includeLegacyTreeSitterGrammars =
|
||||
atom.config.get('core.useLegacyTreeSitter') === true;
|
||||
|
||||
const tsGrammars = Object.values(this.treeSitterGrammarsById)
|
||||
let modernTsGrammars = Object.values(this.wasmTreeSitterGrammarsById)
|
||||
.filter(g => g.scopeName);
|
||||
result = result.concat(tsGrammars);
|
||||
result = result.concat(modernTsGrammars);
|
||||
|
||||
if (includeModernTreeSitterGrammars) {
|
||||
let modernTsGrammars = Object.values(this.wasmTreeSitterGrammarsById)
|
||||
if (includeLegacyTreeSitterGrammars) {
|
||||
const legacyTsGrammars = Object.values(this.treeSitterGrammarsById)
|
||||
.filter(g => g.scopeName);
|
||||
result = result.concat(modernTsGrammars);
|
||||
result = result.concat(legacyTsGrammars);
|
||||
}
|
||||
|
||||
return result;
|
||||
@ -790,8 +786,10 @@ module.exports = class GrammarRegistry {
|
||||
return this.textmateRegistry.scopeForId(id);
|
||||
}
|
||||
|
||||
// TODO: why is this being used? Can we remove it soon?
|
||||
treeSitterGrammarForLanguageString(languageString, type = 'original') {
|
||||
// Match up a language string (of the sort generated by an injection point)
|
||||
// with a grammar. Checks the `injectionRegex` property on grammars and
|
||||
// returns the one with the longest match.
|
||||
treeSitterGrammarForLanguageString(languageString, type = 'wasm') {
|
||||
let longestMatchLength = 0;
|
||||
let grammarWithLongestMatch = null;
|
||||
let table = type === 'original' ? this.treeSitterGrammarsById : this.wasmTreeSitterGrammarsById;
|
||||
|
Loading…
Reference in New Issue
Block a user