ProvableHQ/leo
1
1
Fork 0
mirror of https://github.com/ProvableHQ/leo.git synced 2024-11-23 23:23:50 +03:00
Code Issues Packages Projects Releases Wiki Activity

preserve char scalar nonscalar

Browse Source
This commit is contained in:
collin 2022-05-12 13:16:25 -07:00
parent 8f17d6294a
commit c8de6826c2
3 changed files with 22 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
compiler/ast/src/chars/char_value.rs
View File

@ -31,24 +31,24 @@ where
D: ::serde::de::Deserializer<'de>,
{
let int = u32::deserialize(deserializer)?;
std::char::from_u32(int).ok_or_else(|| ::serde::de::Error::custom("Failed to convert u32 to primitive char."))
std::char::from_u32(int).ok_or_else(|| ::serde::de::Error::custom("Failed to convert u32 to scalar char."))
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub enum Char {
Primitive(
Scalar(
#[serde(deserialize_with = "char_from_u32")]
#[serde(serialize_with = "char_to_u32")]
char,
),
NonPrimitive(u32),
NonScalar(u32),
}
impl fmt::Display for Char {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Primitive(c) => write!(f, "{}", c),
Self::NonPrimitive(c) => write!(f, "{:X}", c),
Self::Scalar(c) => write!(f, "{}", c),
Self::NonScalar(c) => write!(f, "{:X}", c),
}
}
}

22
compiler/parser/src/tokenizer/lexer.rs
View File

@ -79,9 +79,9 @@ impl Token {
if let Ok(hex) = u32::from_str_radix(&unicode, 16) {
if let Some(character) = std::char::from_u32(hex) {
Ok((len, Char::Primitive(character)))
Ok((len, Char::Scalar(character)))
} else if hex <= 0x10FFFF {
Ok((len, Char::NonPrimitive(hex)))
Ok((len, Char::NonScalar(hex)))
} else {
Err(ParserError::lexer_invalid_character_exceeded_max_value(unicode).into())
}
@ -122,7 +122,7 @@ impl Token {
return Err(ParserError::lexer_expected_valid_hex_char(hex).into());
}
Ok((len, Char::Primitive(ascii_number as char)))
Ok((len, Char::Scalar(ascii_number as char)))
} else {
Err(ParserError::lexer_expected_valid_hex_char(hex).into())
}
@ -132,13 +132,13 @@ impl Token {
match input.next() {
None => Err(ParserError::lexer_empty_input_tendril().into()),
// Length of 2 to account the '\'.
Some('0') => Ok((2, Char::Primitive(0 as char))),
Some('t') => Ok((2, Char::Primitive(9 as char))),
Some('n') => Ok((2, Char::Primitive(10 as char))),
Some('r') => Ok((2, Char::Primitive(13 as char))),
Some('\"') => Ok((2, Char::Primitive(34 as char))),
Some('\'') => Ok((2, Char::Primitive(39 as char))),
Some('\\') => Ok((2, Char::Primitive(92 as char))),
Some('0') => Ok((2, Char::Scalar(0 as char))),
Some('t') => Ok((2, Char::Scalar(9 as char))),
Some('n') => Ok((2, Char::Scalar(10 as char))),
Some('r') => Ok((2, Char::Scalar(13 as char))),
Some('\"') => Ok((2, Char::Scalar(34 as char))),
Some('\'') => Ok((2, Char::Scalar(39 as char))),
Some('\\') => Ok((2, Char::Scalar(92 as char))),
Some('u') => Self::eat_unicode_char(input),
Some('x') => Self::eat_hex_char(input),
Some(c) => Err(ParserError::lexer_expected_valid_escaped_char(c).into()),
@ -150,7 +150,7 @@ impl Token {
match input.next() {
None => Err(ParserError::lexer_empty_input_tendril().into()),
Some('\\') => Self::eat_escaped_char(input),
Some(c) => Ok((c.len_utf8(), Char::Primitive(c))),
Some(c) => Ok((c.len_utf8(), Char::Scalar(c))),
}
}

12
compiler/parser/src/tokenizer/token.rs
View File

@ -21,16 +21,16 @@ use std::fmt;
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub enum Char {
Primitive(char),
NonPrimitive(u32),
Scalar(char),
NonScalar(u32),
}
#[allow(clippy::from_over_into)]
impl Into<leo_ast::Char> for Char {
fn into(self) -> leo_ast::Char {
match self {
Self::Primitive(c) => leo_ast::Char::Primitive(c),
Self::NonPrimitive(c) => leo_ast::Char::NonPrimitive(c),
Self::Scalar(c) => leo_ast::Char::Scalar(c),
Self::NonScalar(c) => leo_ast::Char::NonScalar(c),
}
}
}
@ -38,8 +38,8 @@ impl Into<leo_ast::Char> for Char {
impl fmt::Display for Char {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Primitive(c) => write!(f, "{}", c),
Self::NonPrimitive(c) => write!(f, "{:X}", c),
Self::Scalar(c) => write!(f, "{}", c),
Self::NonScalar(c) => write!(f, "{:X}", c),
}
}
}