ProvableHQ/leo
1
1
Fork 0
mirror of https://github.com/ProvableHQ/leo.git synced 2024-11-23 23:23:50 +03:00
Code Issues Packages Projects Releases Wiki Activity

simplify tokenize_iter

Browse Source
This commit is contained in:
Mazdak Farrokhzad 2022-06-12 19:48:01 +02:00
parent 2c194b8e57
commit 1d92e07fdf
1 changed files with 4 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
compiler/parser/src/tokenizer/mod.rs
View File

@ -39,15 +39,14 @@ pub(crate) fn tokenize(input: &str, start_pos: BytePos) -> Result<Vec<SpannedTok
/// Yields spanned tokens from the given source code text.
///
/// The `lo` byte position determines where spans will start.
pub(crate) fn tokenize_iter(input: &str, mut lo: BytePos) -> impl '_ + Iterator<Item = Result<SpannedToken>> {
let mut index = 0usize;
pub(crate) fn tokenize_iter(mut input: &str, mut lo: BytePos) -> impl '_ + Iterator<Item = Result<SpannedToken>> {
iter::from_fn(move || {
while input.len() > index {
let (token_len, token) = match Token::eat(&input[index..]) {
while !input.is_empty() {
let (token_len, token) = match Token::eat(input) {
Err(e) => return Some(Err(e)),
Ok(t) => t,
};
index += token_len;
input = &input[token_len..];
let span = Span::new(lo, lo + BytePos::from_usize(token_len));
lo = span.hi;