cursorless-dev/cursorless
1
1
Fork 0
mirror of https://github.com/cursorless-dev/cursorless.git synced 2024-10-04 04:47:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

Added block comments tokens to tokenizer (#479)

Browse Source
This commit is contained in:
Andreas Arvidsson 2022-01-15 15:14:06 +01:00 committed by GitHub
parent fc5057acd1
commit 81def893a6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 6 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/core/tokenizer.ts
View File

@ -34,6 +34,10 @@ const FIXED_TOKENS = [
"\\r", "\\r",
"\\n", "\\n",
"\\t", "\\t",
"/*",
"*/",
"<!--",
"-->",
]; ];
const REPEATABLE_SYMBOLS_REGEX = REPEATABLE_SYMBOLS.map(escapeRegExp) const REPEATABLE_SYMBOLS_REGEX = REPEATABLE_SYMBOLS.map(escapeRegExp)
.map((s) => `${s}+`) .map((s) => `${s}+`)

2
src/test/suite/tokenizer.test.ts
View File

@ -48,6 +48,8 @@ const tests: TestCase[] = [
["``", ["`", "`"]], ["``", ["`", "`"]],
// Comments // Comments
["// Hello world", ["//", "Hello", "world"]], ["// Hello world", ["//", "Hello", "world"]],
["/* Hello world */", ["/*", "Hello", "world", "*/"]],
["<!-- Hello world -->", ["<!--", "Hello", "world", "-->"]],
// Hex colors // Hex colors
["#aaaaaa", ["#", "aaaaaa"]], ["#aaaaaa", ["#", "aaaaaa"]],
["#11aaaa", ["#", "11aaaa"]], ["#11aaaa", ["#", "11aaaa"]],