mirror of
https://github.com/wez/wezterm.git
synced 2024-11-22 13:16:39 +03:00
bidi: move conformance tests to separate file
this is so that we can exclude it from the published crate
8b32ed483a
This commit is contained in:
Wez Furlong
parent
62cbcd691d
commit
84de038d5d
@ -5,7 +5,7 @@ edition = "2021"
|
||||
repository = "https://github.com/wez/wezterm"
|
||||
description = "The Unicode Bidi Algorithm (UBA)"
|
||||
license = "MIT AND Unicode-DFS-2016"
|
||||
exclude = ["/data"]
|
||||
exclude = ["/data", "/tests"]
|
||||
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
|
||||
@ -7,7 +7,7 @@ use crate::NO_LEVEL;
|
||||
pub const MAX_DEPTH: usize = 125;
|
||||
|
||||
#[derive(Default, Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)]
|
||||
pub struct Level(pub(crate) i8);
|
||||
pub struct Level(pub i8);
|
||||
|
||||
impl Level {
|
||||
pub fn direction(self) -> Direction {
|
||||
|
||||
288
bidi/src/lib.rs
288
bidi/src/lib.rs
@ -61,7 +61,7 @@ pub struct BidiContext {
|
||||
}
|
||||
|
||||
/// Represents a formatting character that has been removed by the X9 rule
|
||||
const NO_LEVEL: i8 = -1;
|
||||
pub const NO_LEVEL: i8 = -1;
|
||||
|
||||
/// A `BidiRun` represents a run which is a contiguous sequence of codepoints
|
||||
/// from the original paragraph that have been resolved to the same embedding
|
||||
@ -192,6 +192,10 @@ impl BidiContext {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
pub fn base_level(&self) -> Level {
|
||||
self.base_level
|
||||
}
|
||||
|
||||
/// When `reorder` is set to true, reordering will apply rule L3 to
|
||||
/// non-spacing marks. This is likely more desirable for terminal
|
||||
/// based applications than it is for more modern GUI applications
|
||||
@ -2080,286 +2084,4 @@ mod tests {
|
||||
];
|
||||
assert_eq!(reordered, explicit_ltr);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn it_works() {
|
||||
let result = 2 + 2;
|
||||
assert_eq!(result, 4);
|
||||
}
|
||||
|
||||
fn parse_codepoint(s: &str) -> u32 {
|
||||
u32::from_str_radix(s.trim(), 16).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bidi_character_test() {
|
||||
let _ = env_logger::Builder::new().is_test(true).try_init();
|
||||
|
||||
let data = include_str!("../data/BidiCharacterTest.txt");
|
||||
|
||||
// This helps to iterate on regressions by skipping over tests
|
||||
// until we reach the line number we're testing
|
||||
let first_line = 0;
|
||||
let mut levels: Vec<Level> = vec![];
|
||||
let mut reorder: Vec<usize> = vec![];
|
||||
let mut context = BidiContext::new();
|
||||
let mut level_passes = 0;
|
||||
let mut level_fails = 0;
|
||||
let mut para_passes = 0;
|
||||
let mut para_fails = 0;
|
||||
let mut reorder_passes = 0;
|
||||
let mut reorder_fails = 0;
|
||||
|
||||
for (line_number, line) in data.lines().enumerate() {
|
||||
if line_number + 1 < first_line {
|
||||
continue;
|
||||
}
|
||||
|
||||
if line.is_empty() || line.starts_with('#') {
|
||||
continue;
|
||||
}
|
||||
|
||||
let fields: Vec<&str> = line.split(';').collect();
|
||||
let codepoints: Vec<char> = fields[0]
|
||||
.split_whitespace()
|
||||
.map(parse_codepoint)
|
||||
.map(|cp| char::from_u32(cp).unwrap())
|
||||
.collect();
|
||||
|
||||
let direction: i8 = fields[1].parse().unwrap();
|
||||
let para_level: Level = Level(fields[2].parse().unwrap());
|
||||
|
||||
levels.clear();
|
||||
for field in fields[3].split_whitespace() {
|
||||
if field == "x" {
|
||||
levels.push(Level(NO_LEVEL));
|
||||
} else {
|
||||
levels.push(Level(field.parse().unwrap()));
|
||||
}
|
||||
}
|
||||
|
||||
reorder.clear();
|
||||
for field in fields[4].split_whitespace() {
|
||||
reorder.push(field.parse().unwrap());
|
||||
}
|
||||
|
||||
log::debug!("BidiCharacterTest.txt:{}", line_number + 1);
|
||||
log::debug!("{:?}", codepoints);
|
||||
|
||||
context.resolve_paragraph(
|
||||
&codepoints,
|
||||
match direction {
|
||||
0 => ParagraphDirectionHint::LeftToRight,
|
||||
1 => ParagraphDirectionHint::RightToLeft,
|
||||
2 => ParagraphDirectionHint::AutoLeftToRight,
|
||||
_ => panic!("invalid direction code {}", direction),
|
||||
},
|
||||
);
|
||||
|
||||
if context.base_level != para_level {
|
||||
log::error!(
|
||||
"\nBidiCharacterTest.txt:{}\n {:?}\n base_level={:?} expected={:?}",
|
||||
line_number + 1,
|
||||
codepoints,
|
||||
context.base_level,
|
||||
para_level,
|
||||
);
|
||||
para_fails += 1;
|
||||
} else {
|
||||
para_passes += 1;
|
||||
}
|
||||
|
||||
let (resolved_levels, actual_reordered) = context.reorder_line(0..codepoints.len());
|
||||
|
||||
if resolved_levels != levels {
|
||||
log::error!(
|
||||
"\nBidiCharacterTest.txt:{}\n {:?}\n expected={:?}",
|
||||
line_number + 1,
|
||||
codepoints,
|
||||
levels
|
||||
);
|
||||
log::error!(" levels={:?}", resolved_levels);
|
||||
level_fails += 1;
|
||||
} else {
|
||||
level_passes += 1;
|
||||
}
|
||||
|
||||
if actual_reordered != reorder {
|
||||
log::error!(
|
||||
"\nBidiCharacterTest.txt:{}\n visual={:?}\n expected={:?}",
|
||||
line_number + 1,
|
||||
actual_reordered,
|
||||
reorder
|
||||
);
|
||||
reorder_fails += 1;
|
||||
} else {
|
||||
reorder_passes += 1;
|
||||
}
|
||||
|
||||
if reorder_fails + level_fails + para_fails > 0 {
|
||||
log::error!("{:#?}", context);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
println!("level_passes={} level_fails={}", level_passes, level_fails);
|
||||
println!("para_passes={} para_fails={}", para_passes, para_fails);
|
||||
println!(
|
||||
"reorder_passes={} reorder_fails={}",
|
||||
reorder_passes, reorder_fails
|
||||
);
|
||||
assert_eq!(level_fails + para_fails + reorder_fails, 0);
|
||||
assert_eq!(level_passes, 91707);
|
||||
assert_eq!(reorder_passes, 91707);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bidi_test() {
|
||||
let _ = env_logger::Builder::new().is_test(true).try_init();
|
||||
let data = include_str!("../data/BidiTest.txt");
|
||||
|
||||
let mut levels: Vec<Level> = vec![];
|
||||
let mut reorder: Vec<usize> = vec![];
|
||||
let mut context = BidiContext::new();
|
||||
|
||||
let mut level_passes = 0;
|
||||
let mut level_fails = 0;
|
||||
let mut reorder_passes = 0;
|
||||
let mut reorder_fails = 0;
|
||||
|
||||
// This helps to iterate on regressions by skipping over tests
|
||||
// until we reach the line number we're testing
|
||||
let first_line = 0;
|
||||
|
||||
for (line_number, line) in data.lines().enumerate() {
|
||||
if line.is_empty() || line.starts_with('#') {
|
||||
continue;
|
||||
}
|
||||
if line.starts_with("@Levels:") {
|
||||
levels.clear();
|
||||
for field in line.split_whitespace().skip(1) {
|
||||
if field == "x" {
|
||||
levels.push(Level(NO_LEVEL));
|
||||
} else {
|
||||
levels.push(Level(field.parse().unwrap()));
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if line.starts_with("@Reorder:") {
|
||||
reorder.clear();
|
||||
for field in line.split_whitespace().skip(1) {
|
||||
reorder.push(field.parse().unwrap());
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if line.starts_with('@') {
|
||||
continue;
|
||||
}
|
||||
|
||||
let fields: Vec<&str> = line.split(';').collect();
|
||||
|
||||
let inputs: Vec<BidiClass> = fields[0].split_whitespace().map(class_by_name).collect();
|
||||
let bitset: u32 = fields[1].trim().parse().unwrap();
|
||||
|
||||
let mut directions: Vec<ParagraphDirectionHint> = vec![];
|
||||
if bitset & 1 == 1 {
|
||||
directions.push(ParagraphDirectionHint::AutoLeftToRight);
|
||||
}
|
||||
if bitset & 2 == 2 {
|
||||
directions.push(ParagraphDirectionHint::LeftToRight);
|
||||
}
|
||||
if bitset & 4 == 4 {
|
||||
directions.push(ParagraphDirectionHint::RightToLeft);
|
||||
}
|
||||
|
||||
let mut printed_summary = false;
|
||||
|
||||
if line_number < first_line {
|
||||
continue;
|
||||
}
|
||||
|
||||
for &dir in &directions {
|
||||
context.set_char_types(&inputs, dir);
|
||||
let (resolved_levels, actual_reordered) = context.reorder_line(0..inputs.len());
|
||||
if resolved_levels != levels {
|
||||
if !printed_summary {
|
||||
log::error!(
|
||||
"\nBidiTest.txt:{}: {:?}\n {:?}\n expected={:?}",
|
||||
line_number + 1,
|
||||
directions,
|
||||
inputs,
|
||||
levels
|
||||
);
|
||||
printed_summary = true;
|
||||
}
|
||||
log::error!(" {:?} levels={:?}", dir, resolved_levels);
|
||||
log::error!("{:#?}", context);
|
||||
level_fails += 1;
|
||||
} else {
|
||||
level_passes += 1;
|
||||
}
|
||||
|
||||
if actual_reordered != reorder {
|
||||
log::error!(
|
||||
"\nBidiTest.txt:{}: {:?}\n visual={:?}\n expected={:?}",
|
||||
line_number + 1,
|
||||
directions,
|
||||
actual_reordered,
|
||||
reorder
|
||||
);
|
||||
reorder_fails += 1;
|
||||
} else {
|
||||
reorder_passes += 1;
|
||||
}
|
||||
}
|
||||
|
||||
if level_fails + reorder_fails > 0 {
|
||||
log::error!("Stopping tests to limit output: too many failures");
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
println!("levels: {} passed, {} failed", level_passes, level_fails);
|
||||
println!(
|
||||
"reorders: {} passed, {} failed",
|
||||
reorder_passes, reorder_fails
|
||||
);
|
||||
|
||||
assert_eq!(level_fails, 0);
|
||||
assert_eq!(level_passes, 770241);
|
||||
|
||||
assert_eq!(reorder_fails, 0);
|
||||
assert_eq!(reorder_passes, 770241);
|
||||
}
|
||||
|
||||
fn class_by_name(s: &str) -> BidiClass {
|
||||
match s {
|
||||
"AL" => BidiClass::ArabicLetter,
|
||||
"AN" => BidiClass::ArabicNumber,
|
||||
"BN" => BidiClass::BoundaryNeutral,
|
||||
"CS" => BidiClass::CommonSeparator,
|
||||
"EN" => BidiClass::EuropeanNumber,
|
||||
"ES" => BidiClass::EuropeanSeparator,
|
||||
"ET" => BidiClass::EuropeanTerminator,
|
||||
"FSI" => BidiClass::FirstStrongIsolate,
|
||||
"L" => BidiClass::LeftToRight,
|
||||
"LRO" => BidiClass::LeftToRightOverride,
|
||||
"LRE" => BidiClass::LeftToRightEmbedding,
|
||||
"LRI" => BidiClass::LeftToRightIsolate,
|
||||
"NSM" => BidiClass::NonspacingMark,
|
||||
"ON" => BidiClass::OtherNeutral,
|
||||
"B" => BidiClass::ParagraphSeparator,
|
||||
"PDF" => BidiClass::PopDirectionalFormat,
|
||||
"PDI" => BidiClass::PopDirectionalIsolate,
|
||||
"R" => BidiClass::RightToLeft,
|
||||
"RLE" => BidiClass::RightToLeftEmbedding,
|
||||
"RLI" => BidiClass::RightToLeftIsolate,
|
||||
"RLO" => BidiClass::RightToLeftOverride,
|
||||
"S" => BidiClass::SegmentSeparator,
|
||||
"WS" => BidiClass::WhiteSpace,
|
||||
bad => panic!("invalid BidiClass {}", bad),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
277
bidi/tests/conformance.rs
Normal file
277
bidi/tests/conformance.rs
Normal file
@ -0,0 +1,277 @@
|
||||
use wezterm_bidi::*;
|
||||
|
||||
fn class_by_name(s: &str) -> BidiClass {
|
||||
match s {
|
||||
"AL" => BidiClass::ArabicLetter,
|
||||
"AN" => BidiClass::ArabicNumber,
|
||||
"BN" => BidiClass::BoundaryNeutral,
|
||||
"CS" => BidiClass::CommonSeparator,
|
||||
"EN" => BidiClass::EuropeanNumber,
|
||||
"ES" => BidiClass::EuropeanSeparator,
|
||||
"ET" => BidiClass::EuropeanTerminator,
|
||||
"FSI" => BidiClass::FirstStrongIsolate,
|
||||
"L" => BidiClass::LeftToRight,
|
||||
"LRO" => BidiClass::LeftToRightOverride,
|
||||
"LRE" => BidiClass::LeftToRightEmbedding,
|
||||
"LRI" => BidiClass::LeftToRightIsolate,
|
||||
"NSM" => BidiClass::NonspacingMark,
|
||||
"ON" => BidiClass::OtherNeutral,
|
||||
"B" => BidiClass::ParagraphSeparator,
|
||||
"PDF" => BidiClass::PopDirectionalFormat,
|
||||
"PDI" => BidiClass::PopDirectionalIsolate,
|
||||
"R" => BidiClass::RightToLeft,
|
||||
"RLE" => BidiClass::RightToLeftEmbedding,
|
||||
"RLI" => BidiClass::RightToLeftIsolate,
|
||||
"RLO" => BidiClass::RightToLeftOverride,
|
||||
"S" => BidiClass::SegmentSeparator,
|
||||
"WS" => BidiClass::WhiteSpace,
|
||||
bad => panic!("invalid BidiClass {}", bad),
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_codepoint(s: &str) -> u32 {
|
||||
u32::from_str_radix(s.trim(), 16).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bidi_character_test() {
|
||||
let _ = env_logger::Builder::new().is_test(true).try_init();
|
||||
|
||||
let data = include_str!("../data/BidiCharacterTest.txt");
|
||||
|
||||
// This helps to iterate on regressions by skipping over tests
|
||||
// until we reach the line number we're testing
|
||||
let first_line = 0;
|
||||
let mut levels: Vec<Level> = vec![];
|
||||
let mut reorder: Vec<usize> = vec![];
|
||||
let mut context = BidiContext::new();
|
||||
let mut level_passes = 0;
|
||||
let mut level_fails = 0;
|
||||
let mut para_passes = 0;
|
||||
let mut para_fails = 0;
|
||||
let mut reorder_passes = 0;
|
||||
let mut reorder_fails = 0;
|
||||
|
||||
for (line_number, line) in data.lines().enumerate() {
|
||||
if line_number + 1 < first_line {
|
||||
continue;
|
||||
}
|
||||
|
||||
if line.is_empty() || line.starts_with('#') {
|
||||
continue;
|
||||
}
|
||||
|
||||
let fields: Vec<&str> = line.split(';').collect();
|
||||
let codepoints: Vec<char> = fields[0]
|
||||
.split_whitespace()
|
||||
.map(parse_codepoint)
|
||||
.map(|cp| char::from_u32(cp).unwrap())
|
||||
.collect();
|
||||
|
||||
let direction: i8 = fields[1].parse().unwrap();
|
||||
let para_level: Level = Level(fields[2].parse().unwrap());
|
||||
|
||||
levels.clear();
|
||||
for field in fields[3].split_whitespace() {
|
||||
if field == "x" {
|
||||
levels.push(Level(NO_LEVEL));
|
||||
} else {
|
||||
levels.push(Level(field.parse().unwrap()));
|
||||
}
|
||||
}
|
||||
|
||||
reorder.clear();
|
||||
for field in fields[4].split_whitespace() {
|
||||
reorder.push(field.parse().unwrap());
|
||||
}
|
||||
|
||||
log::debug!("BidiCharacterTest.txt:{}", line_number + 1);
|
||||
log::debug!("{:?}", codepoints);
|
||||
|
||||
context.resolve_paragraph(
|
||||
&codepoints,
|
||||
match direction {
|
||||
0 => ParagraphDirectionHint::LeftToRight,
|
||||
1 => ParagraphDirectionHint::RightToLeft,
|
||||
2 => ParagraphDirectionHint::AutoLeftToRight,
|
||||
_ => panic!("invalid direction code {}", direction),
|
||||
},
|
||||
);
|
||||
|
||||
if context.base_level() != para_level {
|
||||
log::error!(
|
||||
"\nBidiCharacterTest.txt:{}\n {:?}\n base_level={:?} expected={:?}",
|
||||
line_number + 1,
|
||||
codepoints,
|
||||
context.base_level(),
|
||||
para_level,
|
||||
);
|
||||
para_fails += 1;
|
||||
} else {
|
||||
para_passes += 1;
|
||||
}
|
||||
|
||||
let (resolved_levels, actual_reordered) = context.reorder_line(0..codepoints.len());
|
||||
|
||||
if resolved_levels != levels {
|
||||
log::error!(
|
||||
"\nBidiCharacterTest.txt:{}\n {:?}\n expected={:?}",
|
||||
line_number + 1,
|
||||
codepoints,
|
||||
levels
|
||||
);
|
||||
log::error!(" levels={:?}", resolved_levels);
|
||||
level_fails += 1;
|
||||
} else {
|
||||
level_passes += 1;
|
||||
}
|
||||
|
||||
if actual_reordered != reorder {
|
||||
log::error!(
|
||||
"\nBidiCharacterTest.txt:{}\n visual={:?}\n expected={:?}",
|
||||
line_number + 1,
|
||||
actual_reordered,
|
||||
reorder
|
||||
);
|
||||
reorder_fails += 1;
|
||||
} else {
|
||||
reorder_passes += 1;
|
||||
}
|
||||
|
||||
if reorder_fails + level_fails + para_fails > 0 {
|
||||
log::error!("{:#?}", context);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
println!("level_passes={} level_fails={}", level_passes, level_fails);
|
||||
println!("para_passes={} para_fails={}", para_passes, para_fails);
|
||||
println!(
|
||||
"reorder_passes={} reorder_fails={}",
|
||||
reorder_passes, reorder_fails
|
||||
);
|
||||
assert_eq!(level_fails + para_fails + reorder_fails, 0);
|
||||
assert_eq!(level_passes, 91707);
|
||||
assert_eq!(reorder_passes, 91707);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bidi_test() {
|
||||
let _ = env_logger::Builder::new().is_test(true).try_init();
|
||||
let data = include_str!("../data/BidiTest.txt");
|
||||
|
||||
let mut levels: Vec<Level> = vec![];
|
||||
let mut reorder: Vec<usize> = vec![];
|
||||
let mut context = BidiContext::new();
|
||||
|
||||
let mut level_passes = 0;
|
||||
let mut level_fails = 0;
|
||||
let mut reorder_passes = 0;
|
||||
let mut reorder_fails = 0;
|
||||
|
||||
// This helps to iterate on regressions by skipping over tests
|
||||
// until we reach the line number we're testing
|
||||
let first_line = 0;
|
||||
|
||||
for (line_number, line) in data.lines().enumerate() {
|
||||
if line.is_empty() || line.starts_with('#') {
|
||||
continue;
|
||||
}
|
||||
if line.starts_with("@Levels:") {
|
||||
levels.clear();
|
||||
for field in line.split_whitespace().skip(1) {
|
||||
if field == "x" {
|
||||
levels.push(Level(NO_LEVEL));
|
||||
} else {
|
||||
levels.push(Level(field.parse().unwrap()));
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if line.starts_with("@Reorder:") {
|
||||
reorder.clear();
|
||||
for field in line.split_whitespace().skip(1) {
|
||||
reorder.push(field.parse().unwrap());
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if line.starts_with('@') {
|
||||
continue;
|
||||
}
|
||||
|
||||
let fields: Vec<&str> = line.split(';').collect();
|
||||
|
||||
let inputs: Vec<BidiClass> = fields[0].split_whitespace().map(class_by_name).collect();
|
||||
let bitset: u32 = fields[1].trim().parse().unwrap();
|
||||
|
||||
let mut directions: Vec<ParagraphDirectionHint> = vec![];
|
||||
if bitset & 1 == 1 {
|
||||
directions.push(ParagraphDirectionHint::AutoLeftToRight);
|
||||
}
|
||||
if bitset & 2 == 2 {
|
||||
directions.push(ParagraphDirectionHint::LeftToRight);
|
||||
}
|
||||
if bitset & 4 == 4 {
|
||||
directions.push(ParagraphDirectionHint::RightToLeft);
|
||||
}
|
||||
|
||||
let mut printed_summary = false;
|
||||
|
||||
if line_number < first_line {
|
||||
continue;
|
||||
}
|
||||
|
||||
for &dir in &directions {
|
||||
context.set_char_types(&inputs, dir);
|
||||
let (resolved_levels, actual_reordered) = context.reorder_line(0..inputs.len());
|
||||
if resolved_levels != levels {
|
||||
if !printed_summary {
|
||||
log::error!(
|
||||
"\nBidiTest.txt:{}: {:?}\n {:?}\n expected={:?}",
|
||||
line_number + 1,
|
||||
directions,
|
||||
inputs,
|
||||
levels
|
||||
);
|
||||
printed_summary = true;
|
||||
}
|
||||
log::error!(" {:?} levels={:?}", dir, resolved_levels);
|
||||
log::error!("{:#?}", context);
|
||||
level_fails += 1;
|
||||
} else {
|
||||
level_passes += 1;
|
||||
}
|
||||
|
||||
if actual_reordered != reorder {
|
||||
log::error!(
|
||||
"\nBidiTest.txt:{}: {:?}\n visual={:?}\n expected={:?}",
|
||||
line_number + 1,
|
||||
directions,
|
||||
actual_reordered,
|
||||
reorder
|
||||
);
|
||||
reorder_fails += 1;
|
||||
} else {
|
||||
reorder_passes += 1;
|
||||
}
|
||||
}
|
||||
|
||||
if level_fails + reorder_fails > 0 {
|
||||
log::error!("Stopping tests to limit output: too many failures");
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
println!("levels: {} passed, {} failed", level_passes, level_fails);
|
||||
println!(
|
||||
"reorders: {} passed, {} failed",
|
||||
reorder_passes, reorder_fails
|
||||
);
|
||||
|
||||
assert_eq!(level_fails, 0);
|
||||
assert_eq!(level_passes, 770241);
|
||||
|
||||
assert_eq!(reorder_fails, 0);
|
||||
assert_eq!(reorder_passes, 770241);
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user