ProvableHQ/leo
1
1
Fork 0
mirror of https://github.com/ProvableHQ/leo.git synced 2024-11-24 07:48:04 +03:00
Code Issues Packages Projects Releases Wiki Activity

simplify tokenize_iter

Browse Source
This commit is contained in:
Mazdak Farrokhzad 2022-06-12 19:48:01 +02:00
parent 2c194b8e57
commit 1d92e07fdf
1 changed files with 4 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
compiler/parser/src/tokenizer/mod.rs
View File

@ -39,15 +39,14 @@ pub(crate) fn tokenize(input: &str, start_pos: BytePos) -> Result<Vec<SpannedTok
/// Yields spanned tokens from the given source code text. /// Yields spanned tokens from the given source code text.
/// ///
/// The `lo` byte position determines where spans will start. /// The `lo` byte position determines where spans will start.
pub(crate) fn tokenize_iter(input: &str, mut lo: BytePos) -> impl '_ + Iterator<Item = Result<SpannedToken>> { pub(crate) fn tokenize_iter(mut input: &str, mut lo: BytePos) -> impl '_ + Iterator<Item = Result<SpannedToken>> {
let mut index = 0usize;
iter::from_fn(move || { iter::from_fn(move || {
while input.len() > index { while !input.is_empty() {
let (token_len, token) = match Token::eat(&input[index..]) { let (token_len, token) = match Token::eat(input) {
Err(e) => return Some(Err(e)), Err(e) => return Some(Err(e)),
Ok(t) => t, Ok(t) => t,
}; };
index += token_len; input = &input[token_len..];
let span = Span::new(lo, lo + BytePos::from_usize(token_len)); let span = Span::new(lo, lo + BytePos::from_usize(token_len));
lo = span.hi; lo = span.hi;