AleoHQ/leo
1
1
Fork 0
mirror of https://github.com/AleoHQ/leo.git synced 2025-01-02 06:50:13 +03:00
Code Issues Packages Projects Releases Wiki Activity

preserve char scalar nonscalar

Browse Source
This commit is contained in:
collin 2022-05-12 13:16:25 -07:00
parent 8f17d6294a
commit c8de6826c2
3 changed files with 22 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
compiler/ast/src/chars/char_value.rs
View File

@ -31,24 +31,24 @@ where
D: ::serde::de::Deserializer<'de>, D: ::serde::de::Deserializer<'de>,
{ {
let int = u32::deserialize(deserializer)?; let int = u32::deserialize(deserializer)?;
std::char::from_u32(int).ok_or_else(|| ::serde::de::Error::custom("Failed to convert u32 to primitive char.")) std::char::from_u32(int).ok_or_else(|| ::serde::de::Error::custom("Failed to convert u32 to scalar char."))
} }
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub enum Char { pub enum Char {
Primitive( Scalar(
#[serde(deserialize_with = "char_from_u32")] #[serde(deserialize_with = "char_from_u32")]
#[serde(serialize_with = "char_to_u32")] #[serde(serialize_with = "char_to_u32")]
char, char,
), ),
NonPrimitive(u32), NonScalar(u32),
} }
impl fmt::Display for Char { impl fmt::Display for Char {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self { match self {
Self::Primitive(c) => write!(f, "{}", c), Self::Scalar(c) => write!(f, "{}", c),
Self::NonPrimitive(c) => write!(f, "{:X}", c), Self::NonScalar(c) => write!(f, "{:X}", c),
} }
} }
} }

22
compiler/parser/src/tokenizer/lexer.rs
View File

@ -79,9 +79,9 @@ impl Token {
if let Ok(hex) = u32::from_str_radix(&unicode, 16) { if let Ok(hex) = u32::from_str_radix(&unicode, 16) {
if let Some(character) = std::char::from_u32(hex) { if let Some(character) = std::char::from_u32(hex) {
Ok((len, Char::Primitive(character))) Ok((len, Char::Scalar(character)))
} else if hex <= 0x10FFFF { } else if hex <= 0x10FFFF {
Ok((len, Char::NonPrimitive(hex))) Ok((len, Char::NonScalar(hex)))
} else { } else {
Err(ParserError::lexer_invalid_character_exceeded_max_value(unicode).into()) Err(ParserError::lexer_invalid_character_exceeded_max_value(unicode).into())
} }
@ -122,7 +122,7 @@ impl Token {
return Err(ParserError::lexer_expected_valid_hex_char(hex).into()); return Err(ParserError::lexer_expected_valid_hex_char(hex).into());
} }
Ok((len, Char::Primitive(ascii_number as char))) Ok((len, Char::Scalar(ascii_number as char)))
} else { } else {
Err(ParserError::lexer_expected_valid_hex_char(hex).into()) Err(ParserError::lexer_expected_valid_hex_char(hex).into())
} }
@ -132,13 +132,13 @@ impl Token {
match input.next() { match input.next() {
None => Err(ParserError::lexer_empty_input_tendril().into()), None => Err(ParserError::lexer_empty_input_tendril().into()),
// Length of 2 to account the '\'. // Length of 2 to account the '\'.
Some('0') => Ok((2, Char::Primitive(0 as char))), Some('0') => Ok((2, Char::Scalar(0 as char))),
Some('t') => Ok((2, Char::Primitive(9 as char))), Some('t') => Ok((2, Char::Scalar(9 as char))),
Some('n') => Ok((2, Char::Primitive(10 as char))), Some('n') => Ok((2, Char::Scalar(10 as char))),
Some('r') => Ok((2, Char::Primitive(13 as char))), Some('r') => Ok((2, Char::Scalar(13 as char))),
Some('\"') => Ok((2, Char::Primitive(34 as char))), Some('\"') => Ok((2, Char::Scalar(34 as char))),
Some('\'') => Ok((2, Char::Primitive(39 as char))), Some('\'') => Ok((2, Char::Scalar(39 as char))),
Some('\\') => Ok((2, Char::Primitive(92 as char))), Some('\\') => Ok((2, Char::Scalar(92 as char))),
Some('u') => Self::eat_unicode_char(input), Some('u') => Self::eat_unicode_char(input),
Some('x') => Self::eat_hex_char(input), Some('x') => Self::eat_hex_char(input),
Some(c) => Err(ParserError::lexer_expected_valid_escaped_char(c).into()), Some(c) => Err(ParserError::lexer_expected_valid_escaped_char(c).into()),
@ -150,7 +150,7 @@ impl Token {
match input.next() { match input.next() {
None => Err(ParserError::lexer_empty_input_tendril().into()), None => Err(ParserError::lexer_empty_input_tendril().into()),
Some('\\') => Self::eat_escaped_char(input), Some('\\') => Self::eat_escaped_char(input),
Some(c) => Ok((c.len_utf8(), Char::Primitive(c))), Some(c) => Ok((c.len_utf8(), Char::Scalar(c))),
} }
} }

12
compiler/parser/src/tokenizer/token.rs
View File

@ -21,16 +21,16 @@ use std::fmt;
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub enum Char { pub enum Char {
Primitive(char), Scalar(char),
NonPrimitive(u32), NonScalar(u32),
} }
#[allow(clippy::from_over_into)] #[allow(clippy::from_over_into)]
impl Into<leo_ast::Char> for Char { impl Into<leo_ast::Char> for Char {
fn into(self) -> leo_ast::Char { fn into(self) -> leo_ast::Char {
match self { match self {
Self::Primitive(c) => leo_ast::Char::Primitive(c), Self::Scalar(c) => leo_ast::Char::Scalar(c),
Self::NonPrimitive(c) => leo_ast::Char::NonPrimitive(c), Self::NonScalar(c) => leo_ast::Char::NonScalar(c),
} }
} }
} }
@ -38,8 +38,8 @@ impl Into<leo_ast::Char> for Char {
impl fmt::Display for Char { impl fmt::Display for Char {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self { match self {
Self::Primitive(c) => write!(f, "{}", c), Self::Scalar(c) => write!(f, "{}", c),
Self::NonPrimitive(c) => write!(f, "{:X}", c), Self::NonScalar(c) => write!(f, "{:X}", c),
} }
} }
} }