git.iko.soy/helix
on master help sign in
Work on moving code over to LineEnding instead of assuming '\n'.
4efd671Nathan Vegdahl
Cargo.lock+1
compare with master
unifiedsplit
@@ -331,6 +331,7 @@ dependencies = [
331 "bitflags",
331 "bitflags",
332 "cassowary",
332 "cassowary",
333 "crossterm",
333 "crossterm",
334+ "helix-core",
334 "serde",
335 "serde",
335 "unicode-segmentation",
336 "unicode-segmentation",
336 "unicode-width",
337 "unicode-width",
helix-core/src/auto_pairs.rs+1−1
@@ -12,7 +12,7 @@ pub const PAIRS: &[(char, char)] = &[
12 ('`', '`'),
12 ('`', '`'),
13];
13];
14
14
15−const CLOSE_BEFORE: &str = ")]}'\":;> \n"; // includes space and newline
15+const CLOSE_BEFORE: &str = ")]}'\":;> \n\r\u{000B}\u{000C}\u{0085}\u{2028}\u{2029}"; // includes space and newlines
16
16
17// insert hook:
17// insert hook:
18// Fn(doc, selection, char) => Option<Transaction>
18// Fn(doc, selection, char) => Option<Transaction>
helix-core/src/chars.rs+107−15
@@ -1,25 +1,44 @@
1+use crate::LineEnding;
2+
3+#[derive(Debug, Eq, PartialEq)]
4+pub enum CharCategory {
5+ Whitespace,
6+ Eol,
7+ Word,
8+ Punctuation,
9+ Unknown,
10+}
11+
12+#[inline]
13+pub fn categorize_char(ch: char) -> CharCategory {
14+ if char_is_line_ending(ch) {
15+ CharCategory::Eol
16+ } else if ch.is_whitespace() {
17+ CharCategory::Whitespace
18+ } else if char_is_word(ch) {
19+ CharCategory::Word
20+ } else if char_is_punctuation(ch) {
21+ CharCategory::Punctuation
22+ } else {
23+ CharCategory::Unknown
24+ }
25+}
26+
1−/// Determine whether a character is a line break.
27+/// Determine whether a character is a line ending.
28+#[inline]
2−pub fn char_is_linebreak(c: char) -> bool {
29+pub fn char_is_line_ending(ch: char) -> bool {
3− matches!(
30+ LineEnding::from_char(ch).is_some()
4− c,
5− '\u{000A}' | // LineFeed
6− '\u{000B}' | // VerticalTab
7− '\u{000C}' | // FormFeed
8− '\u{000D}' | // CarriageReturn
9− '\u{0085}' | // NextLine
10− '\u{2028}' | // Line Separator
11− '\u{2029}' // ParagraphSeparator
12− )
13}
31}
14
32
15/// Determine whether a character qualifies as (non-line-break)
33/// Determine whether a character qualifies as (non-line-break)
16/// whitespace.
34/// whitespace.
35+#[inline]
17−pub fn char_is_whitespace(c: char) -> bool {
36+pub fn char_is_whitespace(ch: char) -> bool {
18 // TODO: this is a naive binary categorization of whitespace
37 // TODO: this is a naive binary categorization of whitespace
19 // characters. For display, word wrapping, etc. we'll need a better
38 // characters. For display, word wrapping, etc. we'll need a better
20 // categorization based on e.g. breaking vs non-breaking spaces
39 // categorization based on e.g. breaking vs non-breaking spaces
21 // and whether they're zero-width or not.
40 // and whether they're zero-width or not.
22− match c {
41+ match ch {
23 //'\u{1680}' | // Ogham Space Mark (here for completeness, but usually displayed as a dash, not as whitespace)
42 //'\u{1680}' | // Ogham Space Mark (here for completeness, but usually displayed as a dash, not as whitespace)
24 '\u{0009}' | // Character Tabulation
43 '\u{0009}' | // Character Tabulation
25 '\u{0020}' | // Space
44 '\u{0020}' | // Space
@@ -34,8 +53,81 @@ pub fn char_is_whitespace(c: char) -> bool {
34 // En Quad, Em Quad, En Space, Em Space, Three-per-em Space,
53 // En Quad, Em Quad, En Space, Em Space, Three-per-em Space,
35 // Four-per-em Space, Six-per-em Space, Figure Space,
54 // Four-per-em Space, Six-per-em Space, Figure Space,
36 // Punctuation Space, Thin Space, Hair Space, Zero Width Space.
55 // Punctuation Space, Thin Space, Hair Space, Zero Width Space.
37− c if ('\u{2000}' ..= '\u{200B}').contains(&c) => true,
56+ ch if ('\u{2000}' ..= '\u{200B}').contains(&ch) => true,
38
57
39 _ => false,
58 _ => false,
40 }
59 }
41}
60}
61+
62+#[inline]
63+pub fn char_is_punctuation(ch: char) -> bool {
64+ use unicode_general_category::{get_general_category, GeneralCategory};
65+
66+ matches!(
67+ get_general_category(ch),
68+ GeneralCategory::OtherPunctuation
69+ | GeneralCategory::OpenPunctuation
70+ | GeneralCategory::ClosePunctuation
71+ | GeneralCategory::InitialPunctuation
72+ | GeneralCategory::FinalPunctuation
73+ | GeneralCategory::ConnectorPunctuation
74+ | GeneralCategory::DashPunctuation
75+ | GeneralCategory::MathSymbol
76+ | GeneralCategory::CurrencySymbol
77+ | GeneralCategory::ModifierSymbol
78+ )
79+}
80+
81+#[inline]
82+pub fn char_is_word(ch: char) -> bool {
83+ ch.is_alphanumeric() || ch == '_'
84+}
85+
86+#[cfg(test)]
87+mod test {
88+ use super::*;
89+
90+ #[test]
91+ fn test_categorize() {
92+ const EOL_TEST_CASE: &'static str = "\n\r\u{000B}\u{000C}\u{0085}\u{2028}\u{2029}";
93+ const WORD_TEST_CASE: &'static str =
94+ "_hello_world_あいうえおー1234567890１２３４５６７８９０";
95+ const PUNCTUATION_TEST_CASE: &'static str =
96+ "!\"#$%&\'()*+,-./:;<=>?@[\\]^`{|}~！”＃＄％＆’（）＊＋、。：；＜＝＞？＠「」＾｀｛｜｝～";
97+ const WHITESPACE_TEST_CASE: &'static str = "  　   ";
98+
99+ for ch in EOL_TEST_CASE.chars() {
100+ assert_eq!(CharCategory::Eol, categorize_char(ch));
101+ }
102+
103+ for ch in WHITESPACE_TEST_CASE.chars() {
104+ assert_eq!(
105+ CharCategory::Whitespace,
106+ categorize_char(ch),
107+ "Testing '{}', but got `{:?}` instead of `Category::Whitespace`",
108+ ch,
109+ categorize_char(ch)
110+ );
111+ }
112+
113+ for ch in WORD_TEST_CASE.chars() {
114+ assert_eq!(
115+ CharCategory::Word,
116+ categorize_char(ch),
117+ "Testing '{}', but got `{:?}` instead of `Category::Word`",
118+ ch,
119+ categorize_char(ch)
120+ );
121+ }
122+
123+ for ch in PUNCTUATION_TEST_CASE.chars() {
124+ assert_eq!(
125+ CharCategory::Punctuation,
126+ categorize_char(ch),
127+ "Testing '{}', but got `{:?}` instead of `Category::Punctuation`",
128+ ch,
129+ categorize_char(ch)
130+ );
131+ }
132+ }
133+}
helix-core/src/lib.rs+1−1
@@ -113,6 +113,6 @@ pub use diagnostic::Diagnostic;
113pub use state::State;
113pub use state::State;
114
114
115pub use line_ending::{
115pub use line_ending::{
116− auto_detect_line_ending, get_line_ending, line_end, LineEnding, DEFAULT_LINE_ENDING,
116+ auto_detect_line_ending, get_line_ending, line_end_char_index, LineEnding, DEFAULT_LINE_ENDING,
117};
117};
118pub use transaction::{Assoc, Change, ChangeSet, Operation, Transaction};
118pub use transaction::{Assoc, Change, ChangeSet, Operation, Transaction};
helix-core/src/line_ending.rs+35−12
@@ -1,5 +1,10 @@
1use crate::{Rope, RopeGraphemes, RopeSlice};
1use crate::{Rope, RopeGraphemes, RopeSlice};
2
2
3+#[cfg(target_os = "windows")]
4+pub const DEFAULT_LINE_ENDING: LineEnding = LineEnding::Crlf;
5+#[cfg(not(target_os = "windows"))]
6+pub const DEFAULT_LINE_ENDING: LineEnding = LineEnding::LF;
7+
3/// Represents one of the valid Unicode line endings.
8/// Represents one of the valid Unicode line endings.
4#[derive(PartialEq, Copy, Clone, Debug)]
9#[derive(PartialEq, Copy, Clone, Debug)]
5pub enum LineEnding {
10pub enum LineEnding {
@@ -14,6 +19,7 @@ pub enum LineEnding {
14}
19}
15
20
16impl LineEnding {
21impl LineEnding {
22+ #[inline]
17 pub fn len_chars(&self) -> usize {
23 pub fn len_chars(&self) -> usize {
18 match self {
24 match self {
19 Self::Crlf => 2,
25 Self::Crlf => 2,
@@ -21,6 +27,7 @@ impl LineEnding {
21 }
27 }
22 }
28 }
23
29
30+ #[inline]
24 pub fn as_str(&self) -> &'static str {
31 pub fn as_str(&self) -> &'static str {
25 match self {
32 match self {
26 Self::Crlf => "\u{000D}\u{000A}",
33 Self::Crlf => "\u{000D}\u{000A}",
@@ -34,6 +41,22 @@ impl LineEnding {
34 }
41 }
35 }
42 }
36
43
44+ #[inline]
45+ pub fn from_char(ch: char) -> Option<LineEnding> {
46+ match ch {
47+ '\u{000A}' => Some(LineEnding::LF),
48+ '\u{000B}' => Some(LineEnding::VT),
49+ '\u{000C}' => Some(LineEnding::FF),
50+ '\u{000D}' => Some(LineEnding::CR),
51+ '\u{0085}' => Some(LineEnding::Nel),
52+ '\u{2028}' => Some(LineEnding::LS),
53+ '\u{2029}' => Some(LineEnding::PS),
54+ // Not a line ending
55+ _ => None,
56+ }
57+ }
58+
59+ #[inline]
37 pub fn from_str(g: &str) -> Option<LineEnding> {
60 pub fn from_str(g: &str) -> Option<LineEnding> {
38 match g {
61 match g {
39 "\u{000D}\u{000A}" => Some(LineEnding::Crlf),
62 "\u{000D}\u{000A}" => Some(LineEnding::Crlf),
@@ -49,6 +72,7 @@ impl LineEnding {
49 }
72 }
50 }
73 }
51
74
75+ #[inline]
52 pub fn from_rope_slice(g: &RopeSlice) -> Option<LineEnding> {
76 pub fn from_rope_slice(g: &RopeSlice) -> Option<LineEnding> {
53 if let Some(text) = g.as_str() {
77 if let Some(text) = g.as_str() {
54 LineEnding::from_str(text)
78 LineEnding::from_str(text)
@@ -62,6 +86,11 @@ impl LineEnding {
62 }
86 }
63}
87}
64
88
89+#[inline]
90+pub fn str_is_line_ending(s: &str) -> bool {
91+ LineEnding::from_str(s).is_some()
92+}
93+
65/// Attempts to detect what line ending the passed document uses.
94/// Attempts to detect what line ending the passed document uses.
66pub fn auto_detect_line_ending(doc: &Rope) -> Option<LineEnding> {
95pub fn auto_detect_line_ending(doc: &Rope) -> Option<LineEnding> {
67 // Return first matched line ending. Not all possible line endings
96 // Return first matched line ending. Not all possible line endings
@@ -96,19 +125,13 @@ pub fn get_line_ending(line: &RopeSlice) -> Option<LineEnding> {
96}
125}
97
126
98/// Returns the char index of the end of the given line, not including its line ending.
127/// Returns the char index of the end of the given line, not including its line ending.
99−pub fn line_end(slice: &RopeSlice, line: usize) -> usize {
128+pub fn line_end_char_index(slice: &RopeSlice, line: usize) -> usize {
100− slice.line_to_char(line + 1).saturating_sub(
129+ slice.line_to_char(line + 1)
101− get_line_ending(&slice.line(line))
130+ - get_line_ending(&slice.line(line))
102 .map(|le| le.len_chars())
131 .map(|le| le.len_chars())
103− .unwrap_or(0),
132+ .unwrap_or(0)
104− )
105}
133}
106
134
107−#[cfg(target_os = "windows")]
108−pub const DEFAULT_LINE_ENDING: LineEnding = LineEnding::Crlf;
109−#[cfg(not(target_os = "windows"))]
110−pub const DEFAULT_LINE_ENDING: LineEnding = LineEnding::LF;
111−
112#[cfg(test)]
135#[cfg(test)]
113mod line_ending_tests {
136mod line_ending_tests {
114 use super::*;
137 use super::*;
@@ -150,11 +173,11 @@ mod line_ending_tests {
150 fn test_rope_slice_to_line_ending() {
173 fn test_rope_slice_to_line_ending() {
151 let r = Rope::from_str("\r\n");
174 let r = Rope::from_str("\r\n");
152 assert_eq!(
175 assert_eq!(
153− rope_slice_to_line_ending(&r.slice(1..2)),
176+ LineEnding::from_rope_slice(&r.slice(1..2)),
154 Some(LineEnding::LF)
177 Some(LineEnding::LF)
155 );
178 );
156 assert_eq!(
179 assert_eq!(
157− rope_slice_to_line_ending(&r.slice(0..2)),
180+ LineEnding::from_rope_slice(&r.slice(0..2)),
158 Some(LineEnding::Crlf)
181 Some(LineEnding::Crlf)
159 );
182 );
160 }
183 }
helix-core/src/movement.rs+38−129
@@ -3,9 +3,13 @@ use std::iter::{self, from_fn, Peekable, SkipWhile};
3use ropey::iter::Chars;
3use ropey::iter::Chars;
4
4
5use crate::{
5use crate::{
6+ chars::{
7+ categorize_char, char_is_line_ending, char_is_punctuation, char_is_whitespace,
8+ char_is_word, CharCategory,
9+ },
6 coords_at_pos, get_line_ending,
10 coords_at_pos, get_line_ending,
7 graphemes::{nth_next_grapheme_boundary, nth_prev_grapheme_boundary},
11 graphemes::{nth_next_grapheme_boundary, nth_prev_grapheme_boundary},
8− line_end, pos_at_coords, Position, Range, RopeSlice,
12+ line_end_char_index, pos_at_coords, Position, Range, RopeSlice,
9};
13};
10
14
11#[derive(Debug, Copy, Clone, PartialEq, Eq)]
15#[derive(Debug, Copy, Clone, PartialEq, Eq)]
@@ -37,9 +41,8 @@ pub fn move_horizontally(
37 nth_prev_grapheme_boundary(slice, pos, count).max(start)
41 nth_prev_grapheme_boundary(slice, pos, count).max(start)
38 }
42 }
39 Direction::Forward => {
43 Direction::Forward => {
40− // Line end is pos at the start of next line - 1
44+ let end_char_idx = line_end_char_index(&slice, line);
41− let end = line_end(&slice, line);
42− nth_next_grapheme_boundary(slice, pos, count).min(end)
45+ nth_next_grapheme_boundary(slice, pos, count).min(end_char_idx)
43 }
46 }
44 };
47 };
45 let anchor = match behaviour {
48 let anchor = match behaviour {
@@ -68,8 +71,11 @@ pub fn move_vertically(
68 ),
71 ),
69 };
72 };
70
73
71− // convert to 0-indexed, subtract another 1 because len_chars() counts \n
74+ // Length of the line sans line-ending.
72− let new_line_len = slice.line(new_line).len_chars().saturating_sub(2);
75+ let new_line_len = {
76+ let line = slice.line(new_line);
77+ line.len_chars() - get_line_ending(&line).map(|le| le.len_chars()).unwrap_or(0)
78+ };
73
79
74 let new_col = std::cmp::min(horiz as usize, new_line_len);
80 let new_col = std::cmp::min(horiz as usize, new_line_len);
75
81
@@ -104,64 +110,6 @@ fn word_move(slice: RopeSlice, mut range: Range, count: usize, target: WordMotio
104}
110}
105
111
106// ---- util ------------
112// ---- util ------------
107−#[inline]
108−pub(crate) fn is_word(ch: char) -> bool {
109− ch.is_alphanumeric() || ch == '_'
110−}
111−
112−#[inline]
113−pub(crate) fn is_end_of_line(ch: char) -> bool {
114− ch == '\n'
115−}
116−
117−#[inline]
118−// Whitespace, but not end of line
119−pub(crate) fn is_strict_whitespace(ch: char) -> bool {
120− ch.is_whitespace() && !is_end_of_line(ch)
121−}
122−
123−#[inline]
124−pub(crate) fn is_punctuation(ch: char) -> bool {
125− use unicode_general_category::{get_general_category, GeneralCategory};
126−
127− matches!(
128− get_general_category(ch),
129− GeneralCategory::OtherPunctuation
130− | GeneralCategory::OpenPunctuation
131− | GeneralCategory::ClosePunctuation
132− | GeneralCategory::InitialPunctuation
133− | GeneralCategory::FinalPunctuation
134− | GeneralCategory::ConnectorPunctuation
135− | GeneralCategory::DashPunctuation
136− | GeneralCategory::MathSymbol
137− | GeneralCategory::CurrencySymbol
138− | GeneralCategory::ModifierSymbol
139− )
140−}
141−
142−#[derive(Debug, Eq, PartialEq)]
143−pub enum Category {
144− Whitespace,
145− Eol,
146− Word,
147− Punctuation,
148− Unknown,
149−}
150−
151−#[inline]
152−pub(crate) fn categorize(ch: char) -> Category {
153− if is_end_of_line(ch) {
154− Category::Eol
155− } else if ch.is_whitespace() {
156− Category::Whitespace
157− } else if is_word(ch) {
158− Category::Word
159− } else if is_punctuation(ch) {
160− Category::Punctuation
161− } else {
162− Category::Unknown
163− }
164−}
165
113
166#[inline]
114#[inline]
167/// Returns first index that doesn't satisfy a given predicate when
115/// Returns first index that doesn't satisfy a given predicate when
@@ -235,7 +183,8 @@ impl CharHelpers for Chars<'_> {
235 let mut phase = WordMotionPhase::Start;
183 let mut phase = WordMotionPhase::Start;
236 let mut head = origin.head;
184 let mut head = origin.head;
237 let mut anchor: Option<usize> = None;
185 let mut anchor: Option<usize> = None;
186+ let is_boundary =
238− let is_boundary = |a: char, b: Option<char>| categorize(a) != categorize(b.unwrap_or(a));
187+ |a: char, b: Option<char>| categorize_char(a) != categorize_char(b.unwrap_or(a));
239 while let Some(peek) = characters.peek().copied() {
188 while let Some(peek) = characters.peek().copied() {
240 phase = match phase {
189 phase = match phase {
241 WordMotionPhase::Start => {
190 WordMotionPhase::Start => {
@@ -244,7 +193,8 @@ impl CharHelpers for Chars<'_> {
244 break; // We're at the end, so there's nothing to do.
193 break; // We're at the end, so there's nothing to do.
245 }
194 }
246 // Anchor may remain here if the head wasn't at a boundary
195 // Anchor may remain here if the head wasn't at a boundary
247− if !is_boundary(peek, characters.peek().copied()) && !is_end_of_line(peek) {
196+ if !is_boundary(peek, characters.peek().copied()) && !char_is_line_ending(peek)
197+ {
248 anchor = Some(head);
198 anchor = Some(head);
249 }
199 }
250 // First character is always skipped by the head
200 // First character is always skipped by the head
@@ -252,7 +202,7 @@ impl CharHelpers for Chars<'_> {
252 WordMotionPhase::SkipNewlines
202 WordMotionPhase::SkipNewlines
253 }
203 }
254 WordMotionPhase::SkipNewlines => {
204 WordMotionPhase::SkipNewlines => {
255− if is_end_of_line(peek) {
205+ if char_is_line_ending(peek) {
256 characters.next();
206 characters.next();
257 if characters.peek().is_some() {
207 if characters.peek().is_some() {
258 advance(&mut head);
208 advance(&mut head);
@@ -286,12 +236,12 @@ fn reached_target(target: WordMotionTarget, peek: char, next_peek: Option<&char>
286
236
287 match target {
237 match target {
288 WordMotionTarget::NextWordStart => {
238 WordMotionTarget::NextWordStart => {
289− ((categorize(peek) != categorize(*next_peek))
239+ ((categorize_char(peek) != categorize_char(*next_peek))
290− && (is_end_of_line(*next_peek) || !next_peek.is_whitespace()))
240+ && (char_is_line_ending(*next_peek) || !next_peek.is_whitespace()))
291 }
241 }
292 WordMotionTarget::NextWordEnd | WordMotionTarget::PrevWordStart => {
242 WordMotionTarget::NextWordEnd | WordMotionTarget::PrevWordStart => {
293− ((categorize(peek) != categorize(*next_peek))
243+ ((categorize_char(peek) != categorize_char(*next_peek))
294− && (!peek.is_whitespace() || is_end_of_line(*next_peek)))
244+ && (!peek.is_whitespace() || char_is_line_ending(*next_peek)))
295 }
245 }
296 }
246 }
297}
247}
@@ -330,7 +280,7 @@ mod test {
330 slice,
280 slice,
331 move_vertically(slice, range, Direction::Forward, 1, Movement::Move).head
281 move_vertically(slice, range, Direction::Forward, 1, Movement::Move).head
332 ),
282 ),
333− (1, 2).into()
283+ (1, 3).into()
334 );
284 );
335 }
285 }
336
286
@@ -343,12 +293,12 @@ mod test {
343 let mut range = Range::point(position);
293 let mut range = Range::point(position);
344
294
345 let moves_and_expected_coordinates = [
295 let moves_and_expected_coordinates = [
346− ((Direction::Forward, 1usize), (0, 1)),
296+ ((Direction::Forward, 1usize), (0, 1)), // T|his is a simple alphabetic line
347− ((Direction::Forward, 2usize), (0, 3)),
297+ ((Direction::Forward, 2usize), (0, 3)), // Thi|s is a simple alphabetic line
348− ((Direction::Forward, 0usize), (0, 3)),
298+ ((Direction::Forward, 0usize), (0, 3)), // Thi|s is a simple alphabetic line
349− ((Direction::Forward, 999usize), (0, 31)),
299+ ((Direction::Forward, 999usize), (0, 32)), // This is a simple alphabetic line|
350− ((Direction::Forward, 999usize), (0, 31)),
300+ ((Direction::Forward, 999usize), (0, 32)), // This is a simple alphabetic line|
351− ((Direction::Backward, 999usize), (0, 0)),
301+ ((Direction::Backward, 999usize), (0, 0)), // |This is a simple alphabetic line
352 ];
302 ];
353
303
354 for ((direction, amount), coordinates) in IntoIter::new(moves_and_expected_coordinates) {
304 for ((direction, amount), coordinates) in IntoIter::new(moves_and_expected_coordinates) {
@@ -366,15 +316,15 @@ mod test {
366 let mut range = Range::point(position);
316 let mut range = Range::point(position);
367
317
368 let moves_and_expected_coordinates = IntoIter::new([
318 let moves_and_expected_coordinates = IntoIter::new([
369− ((Direction::Forward, 1usize), (0, 1)), // M_ltiline
319+ ((Direction::Forward, 1usize), (0, 1)), // M|ultiline\n
370− ((Direction::Forward, 2usize), (0, 3)), // Mul_iline
320+ ((Direction::Forward, 2usize), (0, 3)), // Mul|tiline\n
371− ((Direction::Backward, 6usize), (0, 0)), // _ultiline
321+ ((Direction::Backward, 6usize), (0, 0)), // |Multiline\n
372− ((Direction::Backward, 999usize), (0, 0)), // _ultiline
322+ ((Direction::Backward, 999usize), (0, 0)), // |Multiline\n
373− ((Direction::Forward, 3usize), (0, 3)), // Mul_iline
323+ ((Direction::Forward, 3usize), (0, 3)), // Mul|tiline\n
374− ((Direction::Forward, 0usize), (0, 3)), // Mul_iline
324+ ((Direction::Forward, 0usize), (0, 3)), // Mul|tiline\n
375− ((Direction::Backward, 0usize), (0, 3)), // Mul_iline
325+ ((Direction::Backward, 0usize), (0, 3)), // Mul|tiline\n
376− ((Direction::Forward, 999usize), (0, 9)), // Multilin_
326+ ((Direction::Forward, 999usize), (0, 9)), // Multiline|\n
377− ((Direction::Forward, 999usize), (0, 9)), // Multilin_
327+ ((Direction::Forward, 999usize), (0, 9)), // Multiline|\n
378 ]);
328 ]);
379
329
380 for ((direction, amount), coordinates) in moves_and_expected_coordinates {
330 for ((direction, amount), coordinates) in moves_and_expected_coordinates {
@@ -446,7 +396,7 @@ mod test {
446 // First descent preserves column as the target line is wider
396 // First descent preserves column as the target line is wider
447 ((Axis::V, Direction::Forward, 1usize), (1, 8)),
397 ((Axis::V, Direction::Forward, 1usize), (1, 8)),
448 // Second descent clamps column as the target line is shorter
398 // Second descent clamps column as the target line is shorter
449− ((Axis::V, Direction::Forward, 1usize), (2, 4)),
399+ ((Axis::V, Direction::Forward, 1usize), (2, 5)),
450 // Third descent restores the original column
400 // Third descent restores the original column
451 ((Axis::V, Direction::Forward, 1usize), (3, 8)),
401 ((Axis::V, Direction::Forward, 1usize), (3, 8)),
452 // Behaviour is preserved even through long jumps
402 // Behaviour is preserved even through long jumps
@@ -760,45 +710,4 @@ mod test {
760 }
710 }
761 }
711 }
762 }
712 }
763−
764− #[test]
765− fn test_categorize() {
766− const WORD_TEST_CASE: &'static str =
767− "_hello_world_あいうえおー1234567890１２３４５６７８９０";
768− const PUNCTUATION_TEST_CASE: &'static str =
769− "!\"#$%&\'()*+,-./:;<=>?@[\\]^`{|}~！”＃＄％＆’（）＊＋、。：；＜＝＞？＠「」＾｀｛｜｝～";
770− const WHITESPACE_TEST_CASE: &'static str = "  　   ";
771−
772− assert_eq!(Category::Eol, categorize('\n'));
773−
774− for ch in WHITESPACE_TEST_CASE.chars() {
775− assert_eq!(
776− Category::Whitespace,
777− categorize(ch),
778− "Testing '{}', but got `{:?}` instead of `Category::Whitespace`",
779− ch,
780− categorize(ch)
781− );
782− }
783−
784− for ch in WORD_TEST_CASE.chars() {
785− assert_eq!(
786− Category::Word,
787− categorize(ch),
788− "Testing '{}', but got `{:?}` instead of `Category::Word`",
789− ch,
790− categorize(ch)
791− );
792− }
793−
794− for ch in PUNCTUATION_TEST_CASE.chars() {
795− assert_eq!(
796− Category::Punctuation,
797− categorize(ch),
798− "Testing '{}', but got `{:?}` instead of `Category::Punctuation`",
799− ch,
800− categorize(ch)
801− );
802− }
803− }
804}
713}
helix-core/src/position.rs+4−2
@@ -1,4 +1,5 @@
1use crate::{
1use crate::{
2+ chars::char_is_line_ending,
2 graphemes::{nth_next_grapheme_boundary, RopeGraphemes},
3 graphemes::{nth_next_grapheme_boundary, RopeGraphemes},
3 Rope, RopeSlice,
4 Rope, RopeSlice,
4};
5};
@@ -23,8 +24,9 @@ impl Position {
23 pub fn traverse(self, text: &crate::Tendril) -> Self {
24 pub fn traverse(self, text: &crate::Tendril) -> Self {
24 let Self { mut row, mut col } = self;
25 let Self { mut row, mut col } = self;
25 // TODO: there should be a better way here
26 // TODO: there should be a better way here
26− for ch in text.chars() {
27+ let mut chars = text.chars().peekable();
27− if ch == '\n' {
28+ while let Some(ch) = chars.next() {
29+ if char_is_line_ending(ch) && !(ch == '\r' && chars.peek() == Some(&'\n')) {
28 row += 1;
30 row += 1;
29 col = 0;
31 col = 0;
30 } else {
32 } else {
helix-core/src/syntax.rs+5−4
@@ -1,4 +1,4 @@
1−use crate::{regex::Regex, Change, Rope, RopeSlice, Transaction};
1+use crate::{chars::char_is_line_ending, regex::Regex, Change, Rope, RopeSlice, Transaction};
2pub use helix_syntax::{get_language, get_language_name, Lang};
2pub use helix_syntax::{get_language, get_language_name, Lang};
3
3
4use std::{
4use std::{
@@ -579,9 +579,10 @@ impl LanguageLayer {
579 mut column,
579 mut column,
580 } = point;
580 } = point;
581
581
582− // TODO: there should be a better way here
582+ // TODO: there should be a better way here.
583− for ch in text.bytes() {
583+ let mut chars = text.chars().peekable();
584− if ch == b'\n' {
584+ while let Some(ch) = chars.next() {
585+ if char_is_line_ending(ch) && !(ch == '\r' && chars.peek() == Some(&'\n')) {
585 row += 1;
586 row += 1;
586 column = 0;
587 column = 0;
587 } else {
588 } else {
helix-lsp/src/client.rs+4−3
@@ -3,7 +3,7 @@ use crate::{
3 Call, Error, OffsetEncoding, Result,
3 Call, Error, OffsetEncoding, Result,
4};
4};
5
5
6−use helix_core::{find_root, ChangeSet, Rope};
6+use helix_core::{chars::char_is_line_ending, find_root, ChangeSet, Rope};
7use jsonrpc_core as jsonrpc;
7use jsonrpc_core as jsonrpc;
8use lsp_types as lsp;
8use lsp_types as lsp;
9use serde_json::Value;
9use serde_json::Value;
@@ -337,8 +337,9 @@ impl Client {
337 mut character,
337 mut character,
338 } = pos;
338 } = pos;
339
339
340− for ch in text.chars() {
340+ let mut chars = text.chars().peekable();
341− if ch == '\n' {
341+ while let Some(ch) = chars.next() {
342+ if char_is_line_ending(ch) && !(ch == '\r' && chars.peek() == Some(&'\n')) {
342 line += 1;
343 line += 1;
343 character = 0;
344 character = 0;
344 } else {
345 } else {
helix-term/src/commands.rs+7−5
@@ -1,6 +1,6 @@
1use helix_core::{
1use helix_core::{
2 comment, coords_at_pos, find_first_non_whitespace_char, find_root, get_line_ending, graphemes,
2 comment, coords_at_pos, find_first_non_whitespace_char, find_root, get_line_ending, graphemes,
3− indent, line_end, match_brackets,
3+ indent, line_end_char_index, match_brackets,
4 movement::{self, Direction},
4 movement::{self, Direction},
5 object, pos_at_coords,
5 object, pos_at_coords,
6 regex::{self, Regex},
6 regex::{self, Regex},
@@ -342,7 +342,7 @@ fn move_line_end(cx: &mut Context) {
342 let text = doc.text();
342 let text = doc.text();
343 let line = text.char_to_line(range.head);
343 let line = text.char_to_line(range.head);
344
344
345− let pos = line_end(&text.slice(..), line);
345+ let pos = line_end_char_index(&text.slice(..), line);
346
346
347 Range::new(pos, pos)
347 Range::new(pos, pos)
348 });
348 });
@@ -490,6 +490,8 @@ where
490 let count = cx.count();
490 let count = cx.count();
491
491
492 // need to wait for next key
492 // need to wait for next key
493+ // TODO: should this be done by grapheme rather than char? For example,
494+ // we can't properly handle the line-ending case here in terms of char.
493 cx.on_next_key(move |cx, event| {
495 cx.on_next_key(move |cx, event| {
494 let ch = match event {
496 let ch = match event {
495 KeyEvent {
497 KeyEvent {
@@ -623,7 +625,7 @@ fn replace(cx: &mut Context) {
623 KeyEvent {
625 KeyEvent {
624 code: KeyCode::Enter,
626 code: KeyCode::Enter,
625 ..
627 ..
626− } => Some('\n'), // TODO: replace this with DEFAULT_LINE_ENDING
628+ } => Some('\n'), // TODO: use the document's default line ending.
627 _ => None,
629 _ => None,
628 };
630 };
629
631
@@ -763,7 +765,7 @@ fn extend_line_end(cx: &mut Context) {
763 let text = doc.text();
765 let text = doc.text();
764 let line = text.char_to_line(range.head);
766 let line = text.char_to_line(range.head);
765
767
766− let pos = line_end(&text.slice(..), line);
768+ let pos = line_end_char_index(&text.slice(..), line);
767
769
768 Range::new(range.anchor, pos)
770 Range::new(range.anchor, pos)
769 });
771 });
@@ -1642,7 +1644,7 @@ fn append_to_line(cx: &mut Context) {
1642 let selection = doc.selection(view.id).transform(|range| {
1644 let selection = doc.selection(view.id).transform(|range| {
1643 let text = doc.text();
1645 let text = doc.text();
1644 let line = text.char_to_line(range.head);
1646 let line = text.char_to_line(range.head);
1645− let pos = line_end(&text.slice(..), line);
1647+ let pos = line_end_char_index(&text.slice(..), line);
1646 Range::new(pos, pos)
1648 Range::new(pos, pos)
1647 });
1649 });
1648 doc.set_selection(view.id, selection);
1650 doc.set_selection(view.id, selection);
helix-term/src/ui/markdown.rs+2
@@ -110,6 +110,8 @@ fn parse<'a>(contents: &'a str, theme: Option<&Theme>) -> tui::text::Text<'a> {
110 // TODO: replace tabs with indentation
110 // TODO: replace tabs with indentation
111
111
112 let mut slice = &text[start..end];
112 let mut slice = &text[start..end];
113+ // TODO: do we need to handle all unicode line endings
114+ // here, or is just '\n' okay?
113 while let Some(end) = slice.find('\n') {
115 while let Some(end) = slice.find('\n') {
114 // emit span up to newline
116 // emit span up to newline
115 let text = &slice[..end];
117 let text = &slice[..end];
helix-tui/Cargo.toml+1
@@ -22,3 +22,4 @@ unicode-segmentation = "1.2"
22unicode-width = "0.1"
22unicode-width = "0.1"
23crossterm = { version = "0.20", optional = true }
23crossterm = { version = "0.20", optional = true }
24serde = { version = "1", "optional" = true, features = ["derive"]}
24serde = { version = "1", "optional" = true, features = ["derive"]}
25+helix-core = { version = "0.2", path = "../helix-core" }
helix-tui/src/text.rs+2−1
@@ -47,6 +47,7 @@
47//! ]);
47//! ]);
48//! ```
48//! ```
49use crate::style::Style;
49use crate::style::Style;
50+use helix_core::line_ending::str_is_line_ending;
50use std::borrow::Cow;
51use std::borrow::Cow;
51use unicode_segmentation::UnicodeSegmentation;
52use unicode_segmentation::UnicodeSegmentation;
52use unicode_width::UnicodeWidthStr;
53use unicode_width::UnicodeWidthStr;
@@ -177,7 +178,7 @@ impl<'a> Span<'a> {
177 symbol: g,
178 symbol: g,
178 style: base_style.patch(self.style),
179 style: base_style.patch(self.style),
179 })
180 })
180− .filter(|s| s.symbol != "\n")
181+ .filter(|s| !str_is_line_ending(s.symbol))
181 }
182 }
182}
183}
183
184
helix-tui/src/widgets/reflow.rs+5−4
@@ -1,4 +1,5 @@
1use crate::text::StyledGrapheme;
1use crate::text::StyledGrapheme;
2+use helix_core::line_ending::str_is_line_ending;
2use unicode_segmentation::UnicodeSegmentation;
3use unicode_segmentation::UnicodeSegmentation;
3use unicode_width::UnicodeWidthStr;
4use unicode_width::UnicodeWidthStr;
4
5
@@ -62,13 +63,13 @@ impl<'a, 'b> LineComposer<'a> for WordWrapper<'a, 'b> {
62 // Ignore characters wider that the total max width.
63 // Ignore characters wider that the total max width.
63 if symbol.width() as u16 > self.max_line_width
64 if symbol.width() as u16 > self.max_line_width
64 // Skip leading whitespace when trim is enabled.
65 // Skip leading whitespace when trim is enabled.
65− || self.trim && symbol_whitespace && symbol != "\n" && current_line_width == 0
66+ || self.trim && symbol_whitespace && !str_is_line_ending(symbol) && current_line_width == 0
66 {
67 {
67 continue;
68 continue;
68 }
69 }
69
70
70 // Break on newline and discard it.
71 // Break on newline and discard it.
71− if symbol == "\n" {
72+ if str_is_line_ending(symbol) {
72 if prev_whitespace {
73 if prev_whitespace {
73 current_line_width = width_to_last_word_end;
74 current_line_width = width_to_last_word_end;
74 self.current_line.truncate(symbols_to_last_word_end);
75 self.current_line.truncate(symbols_to_last_word_end);
@@ -170,7 +171,7 @@ impl<'a, 'b> LineComposer<'a> for LineTruncator<'a, 'b> {
170 }
171 }
171
172
172 // Break on newline and discard it.
173 // Break on newline and discard it.
173− if symbol == "\n" {
174+ if str_is_line_ending(symbol) {
174 break;
175 break;
175 }
176 }
176
177
@@ -199,7 +200,7 @@ impl<'a, 'b> LineComposer<'a> for LineTruncator<'a, 'b> {
199
200
200 if skip_rest {
201 if skip_rest {
201 for StyledGrapheme { symbol, .. } in &mut self.symbols {
202 for StyledGrapheme { symbol, .. } in &mut self.symbols {
202− if symbol == "\n" {
203+ if str_is_line_ending(symbol) {
203 break;
204 break;
204 }
205 }
205 }
206 }
helix-view/src/document.rs+13−11
@@ -10,7 +10,7 @@ use std::sync::Arc;
10
10
11use helix_core::{
11use helix_core::{
12 auto_detect_line_ending,
12 auto_detect_line_ending,
13− chars::{char_is_linebreak, char_is_whitespace},
13+ chars::{char_is_line_ending, char_is_whitespace},
14 history::History,
14 history::History,
15 syntax::{LanguageConfiguration, LOADER},
15 syntax::{LanguageConfiguration, LOADER},
16 ChangeSet, Diagnostic, LineEnding, Rope, Selection, State, Syntax, Transaction,
16 ChangeSet, Diagnostic, LineEnding, Rope, Selection, State, Syntax, Transaction,
@@ -81,6 +81,9 @@ pub struct Document {
81 /// Current indent style.
81 /// Current indent style.
82 pub indent_style: IndentStyle,
82 pub indent_style: IndentStyle,
83
83
84+ /// The document's default line ending.
85+ pub line_ending: LineEnding,
86+
84 syntax: Option<Syntax>,
87 syntax: Option<Syntax>,
85 // /// Corresponding language scope name. Usually `source.<lang>`.
88 // /// Corresponding language scope name. Usually `source.<lang>`.
86 pub(crate) language: Option<Arc<LanguageConfiguration>>,
89 pub(crate) language: Option<Arc<LanguageConfiguration>>,
@@ -99,7 +102,6 @@ pub struct Document {
99
102
100 diagnostics: Vec<Diagnostic>,
103 diagnostics: Vec<Diagnostic>,
101 language_server: Option<Arc<helix_lsp::Client>>,
104 language_server: Option<Arc<helix_lsp::Client>>,
102− line_ending: LineEnding,
103}
105}
104
106
105use std::fmt;
107use std::fmt;
@@ -254,21 +256,21 @@ impl Document {
254 pub fn load(path: PathBuf) -> Result<Self, Error> {
256 pub fn load(path: PathBuf) -> Result<Self, Error> {
255 use std::{fs::File, io::BufReader};
257 use std::{fs::File, io::BufReader};
256
258
257− let doc = if !path.exists() {
259+ let mut doc = if !path.exists() {
258 Rope::from(DEFAULT_LINE_ENDING.as_str())
260 Rope::from(DEFAULT_LINE_ENDING.as_str())
259 } else {
261 } else {
260 let file = File::open(&path).context(format!("unable to open {:?}", path))?;
262 let file = File::open(&path).context(format!("unable to open {:?}", path))?;
261− let mut doc = Rope::from_reader(BufReader::new(file))?;
263+ Rope::from_reader(BufReader::new(file))?
262− // add missing newline at the end of file
263− if doc.len_bytes() == 0 || doc.byte(doc.len_bytes() - 1) != b'\n' {
264− doc.insert_char(doc.len_chars(), '\n');
265− }
266− doc
267 };
264 };
268
265
269 // search for line endings
266 // search for line endings
270 let line_ending = auto_detect_line_ending(&doc).unwrap_or(DEFAULT_LINE_ENDING);
267 let line_ending = auto_detect_line_ending(&doc).unwrap_or(DEFAULT_LINE_ENDING);
271
268
269+ // add missing newline at the end of file
270+ if doc.len_bytes() == 0 || char_is_line_ending(doc.char(doc.len_chars() - 1)) {
271+ doc.insert(doc.len_chars(), line_ending.as_str());
272+ }
273+
272 let mut doc = Self::new(doc);
274 let mut doc = Self::new(doc);
273 // set the path and try detecting the language
275 // set the path and try detecting the language
274 doc.set_path(&path)?;
276 doc.set_path(&path)?;
@@ -379,7 +381,7 @@ impl Document {
379 Some(' ') => false,
381 Some(' ') => false,
380
382
381 // Ignore blank lines.
383 // Ignore blank lines.
382− Some(c) if char_is_linebreak(c) => continue,
384+ Some(c) if char_is_line_ending(c) => continue,
383
385
384 _ => {
386 _ => {
385 prev_line_is_tabs = false;
387 prev_line_is_tabs = false;
@@ -403,7 +405,7 @@ impl Document {
403 c if char_is_whitespace(c) => count_is_done = true,
405 c if char_is_whitespace(c) => count_is_done = true,
404
406
405 // Ignore blank lines.
407 // Ignore blank lines.
406− c if char_is_linebreak(c) => continue 'outer,
408+ c if char_is_line_ending(c) => continue 'outer,
407
409
408 _ => break,
410 _ => break,
409 }
411 }
helix-view/src/editor.rs+2−2
@@ -12,7 +12,7 @@ use anyhow::Error;
12
12
13pub use helix_core::diagnostic::Severity;
13pub use helix_core::diagnostic::Severity;
14pub use helix_core::register::Registers;
14pub use helix_core::register::Registers;
15−use helix_core::Position;
15+use helix_core::{Position, DEFAULT_LINE_ENDING};
16
16
17#[derive(Debug)]
17#[derive(Debug)]
18pub struct Editor {
18pub struct Editor {
@@ -150,7 +150,7 @@ impl Editor {
150
150
151 pub fn new_file(&mut self, action: Action) -> DocumentId {
151 pub fn new_file(&mut self, action: Action) -> DocumentId {
152 use helix_core::Rope;
152 use helix_core::Rope;
153− let doc = Document::new(Rope::from("\n"));
153+ let doc = Document::new(Rope::from(DEFAULT_LINE_ENDING.as_str()));
154 let id = self.documents.insert(doc);
154 let id = self.documents.insert(doc);
155 self.documents[id].id = id;
155 self.documents[id].id = id;
156 self.switch(id, action);
156 self.switch(id, action);