oxalica/nil
1
1
Fork 0
mirror of https://github.com/oxalica/nil.git synced 2024-11-26 02:48:06 +03:00
Code Issues Projects Releases Wiki Activity

Defeated by regex

Browse Source 
Fixes #75
This commit is contained in:
oxalica 2023-04-20 02:04:36 +08:00
parent 1d5bc81023
commit c69e0fb5fc
1 changed files with 33 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
crates/syntax/src/lexer.rs
View File

@ -104,9 +104,10 @@ regex_dfa! {
// We can assume the input is already a valid UTF-8 string. // We can assume the input is already a valid UTF-8 string.
STRING_ESCAPE = r#"\\([\x00-\x7F]|[\x80-\xFF][\x80-\xBF]*)"#, STRING_ESCAPE = r#"\\([\x00-\x7F]|[\x80-\xFF][\x80-\xBF]*)"#,
DOLLAR_L_CURLY = r"\$\{", DOLLAR_L_CURLY = r"\$\{",
STRING_FRAGMENT = r#"([^"$\\]|\$[^{"\\])+"#, // `$$` makes the second `$` loses the special meaning.
// For '$' before ending. STRING_FRAGMENT = r"\$\$",
STRING_FRAGMENT = r#"\$"#, // Otherwise, treat it literally.
STRING_FRAGMENT = r"(?s).",
} }
} }
@ -117,9 +118,10 @@ regex_dfa! {
STRING_ESCAPE = r#"''\\([\x00-\x7F]|[\x80-\xFF][\x80-\xBF]*)|''\$|'''"#, STRING_ESCAPE = r#"''\\([\x00-\x7F]|[\x80-\xFF][\x80-\xBF]*)|''\$|'''"#,
QUOTE2 = r"''", QUOTE2 = r"''",
DOLLAR_L_CURLY = r"\$\{", DOLLAR_L_CURLY = r"\$\{",
STRING_FRAGMENT = r"([^'$]|\$[^{']|'[^'])+", // `$$` makes the second `$` loses the special meaning.
// For '$' before ending. STRING_FRAGMENT = r"\$\$",
STRING_FRAGMENT = r"\$", // Otherwise, treat it literally.
STRING_FRAGMENT = r"(?s).",
} }
} }
@ -202,6 +204,14 @@ pub fn lex(src: &[u8]) -> LexTokens {
out.push((PATH_START, TextRange::empty(offset))); out.push((PATH_START, TextRange::empty(offset)));
tok = PATH_FRAGMENT; tok = PATH_FRAGMENT;
} }
STRING_FRAGMENT => {
// Merge continuous fragments.
if let Some((STRING_FRAGMENT, range)) = out.last_mut() {
offset += len;
*range = TextRange::new(range.start(), offset);
continue;
}
}
_ => {} _ => {}
} }
@ -438,8 +448,7 @@ mod tests {
DQUOTE "\"" DQUOTE "\""
STRING_FRAGMENT "$${ $$" STRING_FRAGMENT "$${ $$"
STRING_ESCAPE "\\\"" STRING_ESCAPE "\\\""
STRING_FRAGMENT " $$x $ " STRING_FRAGMENT " $$x $ $"
STRING_FRAGMENT "$"
DQUOTE "\"" DQUOTE "\""
"#]], "#]],
); );
@ -491,8 +500,7 @@ mod tests {
QUOTE2 "''" QUOTE2 "''"
STRING_FRAGMENT "$${ $$" STRING_FRAGMENT "$${ $$"
STRING_ESCAPE "''$" STRING_ESCAPE "''$"
STRING_FRAGMENT " $$x $ " STRING_FRAGMENT " $$x $ $"
STRING_FRAGMENT "$"
QUOTE2 "''" QUOTE2 "''"
"#]], "#]],
); );
@ -558,4 +566,19 @@ mod tests {
"#]], "#]],
); );
} }
#[test]
fn indent_string_single_quote() {
check_lex(
"'''${x}''",
expect![[r#"
QUOTE2 "''"
STRING_FRAGMENT "'"
DOLLAR_L_CURLY "${"
IDENT "x"
R_CURLY "}"
QUOTE2 "''"
"#]],
);
}
} }