ProvableHQ/leo
1
1
Fork 0
mirror of https://github.com/ProvableHQ/leo.git synced 2024-12-25 03:04:13 +03:00
Code Issues Packages Projects Releases Wiki Activity

comment fixes

Browse Source
This commit is contained in:
gluax 2022-03-14 08:16:58 -07:00
parent 8e0a59dc5d
commit e484d5124f
1 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
compiler/parser/src/tokenizer/lexer.rs
View File

@ -180,7 +180,7 @@ impl Token {
Ok((i, Token::Int(input_tendril.subtendril(0, i as u32)))) Ok((i, Token::Int(input_tendril.subtendril(0, i as u32))))
} }
/// Returns the number of bytes in an emoji via a bit mask. /// Returns the number of bytes in an utf-8 encoding that starts with this byte.
fn utf8_byte_count(byte: u8) -> usize { fn utf8_byte_count(byte: u8) -> usize {
let mut mask = 0x80; let mut mask = 0x80;
let mut result = 0; let mut result = 0;
@ -220,7 +220,8 @@ impl Token {
let mut string = Vec::new(); let mut string = Vec::new();
while i < input.len() { while i < input.len() {
// If it's an emoji get the length. // Get the length of the utf-8 encoding here
// and position i at the last byte.
if input[i] & 0x80 > 0 { if input[i] & 0x80 > 0 {
len = Self::utf8_byte_count(input[i]); len = Self::utf8_byte_count(input[i]);
i += len; i += len;