git.iko.soy/helix
on master help sign in
Work on moving code over to LineEnding instead of assuming '\n'.

Also some general cleanup and some minor fixes along the way.

commit4efd6713c5b30b33c497a1f85b77a7b0a7fd17e0 authorNathan Vegdahl <cessen@cessen.com> dateSun 20 Jun 2021, 15:09 (5 years ago) parent5d22e3c
16 files changed+228−190
compare with master
unifiedsplit
Cargo.lock+1 helix-core/src/auto_pairs.rs+1−1 helix-core/src/chars.rs+107−15 helix-core/src/lib.rs+1−1 helix-core/src/line_ending.rs+35−12 helix-core/src/movement.rs+38−129 helix-core/src/position.rs+4−2 helix-core/src/syntax.rs+5−4 helix-lsp/src/client.rs+4−3 helix-term/src/commands.rs+7−5 helix-term/src/ui/markdown.rs+2 helix-tui/Cargo.toml+1 helix-tui/src/text.rs+2−1 helix-tui/src/widgets/reflow.rs+5−4 helix-view/src/document.rs+13−11 helix-view/src/editor.rs+2−2
Cargo.lock+1
@@ -331,6 +331,7 @@ dependencies = [
331331 "bitflags",
332332 "cassowary",
333333 "crossterm",
334+ "helix-core",
334335 "serde",
335336 "unicode-segmentation",
336337 "unicode-width",
helix-core/src/auto_pairs.rs+1−1
@@ -12,7 +12,7 @@ pub const PAIRS: &[(char, char)] = &[
1212 ('`', '`'),
1313];
1414
15−const CLOSE_BEFORE: &str = ")]}'\":;> \n"; // includes space and newline
15+const CLOSE_BEFORE: &str = ")]}'\":;> \n\r\u{000B}\u{000C}\u{0085}\u{2028}\u{2029}"; // includes space and newlines
1616
1717// insert hook:
1818// Fn(doc, selection, char) => Option<Transaction>
helix-core/src/chars.rs+107−15
@@ -1,25 +1,44 @@
1−/// Determine whether a character is a line break.
2−pub fn char_is_linebreak(c: char) -> bool {
3− matches!(
4− c,
5− '\u{000A}' | // LineFeed
6− '\u{000B}' | // VerticalTab
7− '\u{000C}' | // FormFeed
8− '\u{000D}' | // CarriageReturn
9− '\u{0085}' | // NextLine
10− '\u{2028}' | // Line Separator
11− '\u{2029}' // ParagraphSeparator
12− )
1+use crate::LineEnding;
2+
3+#[derive(Debug, Eq, PartialEq)]
4+pub enum CharCategory {
5+ Whitespace,
6+ Eol,
7+ Word,
8+ Punctuation,
9+ Unknown,
10+}
11+
12+#[inline]
13+pub fn categorize_char(ch: char) -> CharCategory {
14+ if char_is_line_ending(ch) {
15+ CharCategory::Eol
16+ } else if ch.is_whitespace() {
17+ CharCategory::Whitespace
18+ } else if char_is_word(ch) {
19+ CharCategory::Word
20+ } else if char_is_punctuation(ch) {
21+ CharCategory::Punctuation
22+ } else {
23+ CharCategory::Unknown
24+ }
25+}
26+
27+/// Determine whether a character is a line ending.
28+#[inline]
29+pub fn char_is_line_ending(ch: char) -> bool {
30+ LineEnding::from_char(ch).is_some()
1331}
1432
1533/// Determine whether a character qualifies as (non-line-break)
1634/// whitespace.
17−pub fn char_is_whitespace(c: char) -> bool {
35+#[inline]
36+pub fn char_is_whitespace(ch: char) -> bool {
1837 // TODO: this is a naive binary categorization of whitespace
1938 // characters. For display, word wrapping, etc. we'll need a better
2039 // categorization based on e.g. breaking vs non-breaking spaces
2140 // and whether they're zero-width or not.
22− match c {
41+ match ch {
2342 //'\u{1680}' | // Ogham Space Mark (here for completeness, but usually displayed as a dash, not as whitespace)
2443 '\u{0009}' | // Character Tabulation
2544 '\u{0020}' | // Space
@@ -34,8 +53,81 @@ pub fn char_is_whitespace(c: char) -> bool {
3453 // En Quad, Em Quad, En Space, Em Space, Three-per-em Space,
3554 // Four-per-em Space, Six-per-em Space, Figure Space,
3655 // Punctuation Space, Thin Space, Hair Space, Zero Width Space.
37− c if ('\u{2000}' ..= '\u{200B}').contains(&c) => true,
56+ ch if ('\u{2000}' ..= '\u{200B}').contains(&ch) => true,
3857
3958 _ => false,
4059 }
4160}
61+
62+#[inline]
63+pub fn char_is_punctuation(ch: char) -> bool {
64+ use unicode_general_category::{get_general_category, GeneralCategory};
65+
66+ matches!(
67+ get_general_category(ch),
68+ GeneralCategory::OtherPunctuation
69+ | GeneralCategory::OpenPunctuation
70+ | GeneralCategory::ClosePunctuation
71+ | GeneralCategory::InitialPunctuation
72+ | GeneralCategory::FinalPunctuation
73+ | GeneralCategory::ConnectorPunctuation
74+ | GeneralCategory::DashPunctuation
75+ | GeneralCategory::MathSymbol
76+ | GeneralCategory::CurrencySymbol
77+ | GeneralCategory::ModifierSymbol
78+ )
79+}
80+
81+#[inline]
82+pub fn char_is_word(ch: char) -> bool {
83+ ch.is_alphanumeric() || ch == '_'
84+}
85+
86+#[cfg(test)]
87+mod test {
88+ use super::*;
89+
90+ #[test]
91+ fn test_categorize() {
92+ const EOL_TEST_CASE: &'static str = "\n\r\u{000B}\u{000C}\u{0085}\u{2028}\u{2029}";
93+ const WORD_TEST_CASE: &'static str =
94+ "_hello_world_あいうえおー1234567890１２３４５６７８９０";
95+ const PUNCTUATION_TEST_CASE: &'static str =
96+ "!\"#$%&\'()*+,-./:;<=>?@[\\]^`{|}~！”＃＄％＆’（）＊＋、。：；＜＝＞？＠「」＾｀｛｜｝～";
97+ const WHITESPACE_TEST_CASE: &'static str = "  　   ";
98+
99+ for ch in EOL_TEST_CASE.chars() {
100+ assert_eq!(CharCategory::Eol, categorize_char(ch));
101+ }
102+
103+ for ch in WHITESPACE_TEST_CASE.chars() {
104+ assert_eq!(
105+ CharCategory::Whitespace,
106+ categorize_char(ch),
107+ "Testing '{}', but got `{:?}` instead of `Category::Whitespace`",
108+ ch,
109+ categorize_char(ch)
110+ );
111+ }
112+
113+ for ch in WORD_TEST_CASE.chars() {
114+ assert_eq!(
115+ CharCategory::Word,
116+ categorize_char(ch),
117+ "Testing '{}', but got `{:?}` instead of `Category::Word`",
118+ ch,
119+ categorize_char(ch)
120+ );
121+ }
122+
123+ for ch in PUNCTUATION_TEST_CASE.chars() {
124+ assert_eq!(
125+ CharCategory::Punctuation,
126+ categorize_char(ch),
127+ "Testing '{}', but got `{:?}` instead of `Category::Punctuation`",
128+ ch,
129+ categorize_char(ch)
130+ );
131+ }
132+ }
133+}
helix-core/src/lib.rs+1−1
@@ -113,6 +113,6 @@ pub use diagnostic::Diagnostic;
113113pub use state::State;
114114
115115pub use line_ending::{
116− auto_detect_line_ending, get_line_ending, line_end, LineEnding, DEFAULT_LINE_ENDING,
116+ auto_detect_line_ending, get_line_ending, line_end_char_index, LineEnding, DEFAULT_LINE_ENDING,
117117};
118118pub use transaction::{Assoc, Change, ChangeSet, Operation, Transaction};
helix-core/src/line_ending.rs+35−12
@@ -1,5 +1,10 @@
11use crate::{Rope, RopeGraphemes, RopeSlice};
22
3+#[cfg(target_os = "windows")]
4+pub const DEFAULT_LINE_ENDING: LineEnding = LineEnding::Crlf;
5+#[cfg(not(target_os = "windows"))]
6+pub const DEFAULT_LINE_ENDING: LineEnding = LineEnding::LF;
7+
38/// Represents one of the valid Unicode line endings.
49#[derive(PartialEq, Copy, Clone, Debug)]
510pub enum LineEnding {
@@ -14,6 +19,7 @@ pub enum LineEnding {
1419}
1520
1621impl LineEnding {
22+ #[inline]
1723 pub fn len_chars(&self) -> usize {
1824 match self {
1925 Self::Crlf => 2,
@@ -21,6 +27,7 @@ impl LineEnding {
2127 }
2228 }
2329
30+ #[inline]
2431 pub fn as_str(&self) -> &'static str {
2532 match self {
2633 Self::Crlf => "\u{000D}\u{000A}",
@@ -34,6 +41,22 @@ impl LineEnding {
3441 }
3542 }
3643
44+ #[inline]
45+ pub fn from_char(ch: char) -> Option<LineEnding> {
46+ match ch {
47+ '\u{000A}' => Some(LineEnding::LF),
48+ '\u{000B}' => Some(LineEnding::VT),
49+ '\u{000C}' => Some(LineEnding::FF),
50+ '\u{000D}' => Some(LineEnding::CR),
51+ '\u{0085}' => Some(LineEnding::Nel),
52+ '\u{2028}' => Some(LineEnding::LS),
53+ '\u{2029}' => Some(LineEnding::PS),
54+ // Not a line ending
55+ _ => None,
56+ }
57+ }
58+
59+ #[inline]
3760 pub fn from_str(g: &str) -> Option<LineEnding> {
3861 match g {
3962 "\u{000D}\u{000A}" => Some(LineEnding::Crlf),
@@ -49,6 +72,7 @@ impl LineEnding {
4972 }
5073 }
5174
75+ #[inline]
5276 pub fn from_rope_slice(g: &RopeSlice) -> Option<LineEnding> {
5377 if let Some(text) = g.as_str() {
5478 LineEnding::from_str(text)
@@ -62,6 +86,11 @@ impl LineEnding {
6286 }
6387}
6488
89+#[inline]
90+pub fn str_is_line_ending(s: &str) -> bool {
91+ LineEnding::from_str(s).is_some()
92+}
93+
6594/// Attempts to detect what line ending the passed document uses.
6695pub fn auto_detect_line_ending(doc: &Rope) -> Option<LineEnding> {
6796 // Return first matched line ending. Not all possible line endings
@@ -96,19 +125,13 @@ pub fn get_line_ending(line: &RopeSlice) -> Option<LineEnding> {
96125}
97126
98127/// Returns the char index of the end of the given line, not including its line ending.
99−pub fn line_end(slice: &RopeSlice, line: usize) -> usize {
100− slice.line_to_char(line + 1).saturating_sub(
101− get_line_ending(&slice.line(line))
128+pub fn line_end_char_index(slice: &RopeSlice, line: usize) -> usize {
129+ slice.line_to_char(line + 1)
130+ - get_line_ending(&slice.line(line))
102131 .map(|le| le.len_chars())
103− .unwrap_or(0),
104− )
132+ .unwrap_or(0)
105133}
106134
107−#[cfg(target_os = "windows")]
108−pub const DEFAULT_LINE_ENDING: LineEnding = LineEnding::Crlf;
109−#[cfg(not(target_os = "windows"))]
110−pub const DEFAULT_LINE_ENDING: LineEnding = LineEnding::LF;
111−
112135#[cfg(test)]
113136mod line_ending_tests {
114137 use super::*;
@@ -150,11 +173,11 @@ mod line_ending_tests {
150173 fn test_rope_slice_to_line_ending() {
151174 let r = Rope::from_str("\r\n");
152175 assert_eq!(
153− rope_slice_to_line_ending(&r.slice(1..2)),
176+ LineEnding::from_rope_slice(&r.slice(1..2)),
154177 Some(LineEnding::LF)
155178 );
156179 assert_eq!(
157− rope_slice_to_line_ending(&r.slice(0..2)),
180+ LineEnding::from_rope_slice(&r.slice(0..2)),
158181 Some(LineEnding::Crlf)
159182 );
160183 }
helix-core/src/movement.rs+38−129
@@ -3,9 +3,13 @@ use std::iter::{self, from_fn, Peekable, SkipWhile};
33use ropey::iter::Chars;
44
55use crate::{
6+ chars::{
7+ categorize_char, char_is_line_ending, char_is_punctuation, char_is_whitespace,
8+ char_is_word, CharCategory,
9+ },
610 coords_at_pos, get_line_ending,
711 graphemes::{nth_next_grapheme_boundary, nth_prev_grapheme_boundary},
8− line_end, pos_at_coords, Position, Range, RopeSlice,
12+ line_end_char_index, pos_at_coords, Position, Range, RopeSlice,
913};
1014
1115#[derive(Debug, Copy, Clone, PartialEq, Eq)]
@@ -37,9 +41,8 @@ pub fn move_horizontally(
3741 nth_prev_grapheme_boundary(slice, pos, count).max(start)
3842 }
3943 Direction::Forward => {
40− // Line end is pos at the start of next line - 1
41− let end = line_end(&slice, line);
42− nth_next_grapheme_boundary(slice, pos, count).min(end)
44+ let end_char_idx = line_end_char_index(&slice, line);
45+ nth_next_grapheme_boundary(slice, pos, count).min(end_char_idx)
4346 }
4447 };
4548 let anchor = match behaviour {
@@ -68,8 +71,11 @@ pub fn move_vertically(
6871 ),
6972 };
7073
71− // convert to 0-indexed, subtract another 1 because len_chars() counts \n
72− let new_line_len = slice.line(new_line).len_chars().saturating_sub(2);
74+ // Length of the line sans line-ending.
75+ let new_line_len = {
76+ let line = slice.line(new_line);
77+ line.len_chars() - get_line_ending(&line).map(|le| le.len_chars()).unwrap_or(0)
78+ };
7379
7480 let new_col = std::cmp::min(horiz as usize, new_line_len);
7581
@@ -104,64 +110,6 @@ fn word_move(slice: RopeSlice, mut range: Range, count: usize, target: WordMotio
104110}
105111
106112// ---- util ------------
107−#[inline]
108−pub(crate) fn is_word(ch: char) -> bool {
109− ch.is_alphanumeric() || ch == '_'
110−}
111−
112−#[inline]
113−pub(crate) fn is_end_of_line(ch: char) -> bool {
114− ch == '\n'
115−}
116−
117−#[inline]
118−// Whitespace, but not end of line
119−pub(crate) fn is_strict_whitespace(ch: char) -> bool {
120− ch.is_whitespace() && !is_end_of_line(ch)
121−}
122−
123−#[inline]
124−pub(crate) fn is_punctuation(ch: char) -> bool {
125− use unicode_general_category::{get_general_category, GeneralCategory};
126−
127− matches!(
128− get_general_category(ch),
129− GeneralCategory::OtherPunctuation
130− | GeneralCategory::OpenPunctuation
131− | GeneralCategory::ClosePunctuation
132− | GeneralCategory::InitialPunctuation
133− | GeneralCategory::FinalPunctuation
134− | GeneralCategory::ConnectorPunctuation
135− | GeneralCategory::DashPunctuation
136− | GeneralCategory::MathSymbol
137− | GeneralCategory::CurrencySymbol
138− | GeneralCategory::ModifierSymbol
139− )
140−}
141−
142−#[derive(Debug, Eq, PartialEq)]
143−pub enum Category {
144− Whitespace,
145− Eol,
146− Word,
147− Punctuation,
148− Unknown,
149−}
150−
151−#[inline]
152−pub(crate) fn categorize(ch: char) -> Category {
153− if is_end_of_line(ch) {
154− Category::Eol
155− } else if ch.is_whitespace() {
156− Category::Whitespace
157− } else if is_word(ch) {
158− Category::Word
159− } else if is_punctuation(ch) {
160− Category::Punctuation
161− } else {
162− Category::Unknown
163− }
164−}
165113
166114#[inline]
167115/// Returns first index that doesn't satisfy a given predicate when
@@ -235,7 +183,8 @@ impl CharHelpers for Chars<'_> {
235183 let mut phase = WordMotionPhase::Start;
236184 let mut head = origin.head;
237185 let mut anchor: Option<usize> = None;
238− let is_boundary = |a: char, b: Option<char>| categorize(a) != categorize(b.unwrap_or(a));
186+ let is_boundary =
187+ |a: char, b: Option<char>| categorize_char(a) != categorize_char(b.unwrap_or(a));
239188 while let Some(peek) = characters.peek().copied() {
240189 phase = match phase {
241190 WordMotionPhase::Start => {
@@ -244,7 +193,8 @@ impl CharHelpers for Chars<'_> {
244193 break; // We're at the end, so there's nothing to do.
245194 }
246195 // Anchor may remain here if the head wasn't at a boundary
247− if !is_boundary(peek, characters.peek().copied()) && !is_end_of_line(peek) {
196+ if !is_boundary(peek, characters.peek().copied()) && !char_is_line_ending(peek)
197+ {
248198 anchor = Some(head);
249199 }
250200 // First character is always skipped by the head
@@ -252,7 +202,7 @@ impl CharHelpers for Chars<'_> {
252202 WordMotionPhase::SkipNewlines
253203 }
254204 WordMotionPhase::SkipNewlines => {
255− if is_end_of_line(peek) {
205+ if char_is_line_ending(peek) {
256206 characters.next();
257207 if characters.peek().is_some() {
258208 advance(&mut head);
@@ -286,12 +236,12 @@ fn reached_target(target: WordMotionTarget, peek: char, next_peek: Option<&char>
286236
287237 match target {
288238 WordMotionTarget::NextWordStart => {
289− ((categorize(peek) != categorize(*next_peek))
290− && (is_end_of_line(*next_peek) || !next_peek.is_whitespace()))
239+ ((categorize_char(peek) != categorize_char(*next_peek))
240+ && (char_is_line_ending(*next_peek) || !next_peek.is_whitespace()))
291241 }
292242 WordMotionTarget::NextWordEnd | WordMotionTarget::PrevWordStart => {
293− ((categorize(peek) != categorize(*next_peek))
294− && (!peek.is_whitespace() || is_end_of_line(*next_peek)))
243+ ((categorize_char(peek) != categorize_char(*next_peek))
244+ && (!peek.is_whitespace() || char_is_line_ending(*next_peek)))
295245 }
296246 }
297247}
@@ -330,7 +280,7 @@ mod test {
330280 slice,
331281 move_vertically(slice, range, Direction::Forward, 1, Movement::Move).head
332282 ),
333− (1, 2).into()
283+ (1, 3).into()
334284 );
335285 }
336286
@@ -343,12 +293,12 @@ mod test {
343293 let mut range = Range::point(position);
344294
345295 let moves_and_expected_coordinates = [
346− ((Direction::Forward, 1usize), (0, 1)),
347− ((Direction::Forward, 2usize), (0, 3)),
348− ((Direction::Forward, 0usize), (0, 3)),
349− ((Direction::Forward, 999usize), (0, 31)),
350− ((Direction::Forward, 999usize), (0, 31)),
351− ((Direction::Backward, 999usize), (0, 0)),
296+ ((Direction::Forward, 1usize), (0, 1)), // T|his is a simple alphabetic line
297+ ((Direction::Forward, 2usize), (0, 3)), // Thi|s is a simple alphabetic line
298+ ((Direction::Forward, 0usize), (0, 3)), // Thi|s is a simple alphabetic line
299+ ((Direction::Forward, 999usize), (0, 32)), // This is a simple alphabetic line|
300+ ((Direction::Forward, 999usize), (0, 32)), // This is a simple alphabetic line|
301+ ((Direction::Backward, 999usize), (0, 0)), // |This is a simple alphabetic line
352302 ];
353303
354304 for ((direction, amount), coordinates) in IntoIter::new(moves_and_expected_coordinates) {
@@ -366,15 +316,15 @@ mod test {
366316 let mut range = Range::point(position);
367317
368318 let moves_and_expected_coordinates = IntoIter::new([
369− ((Direction::Forward, 1usize), (0, 1)), // M_ltiline
370− ((Direction::Forward, 2usize), (0, 3)), // Mul_iline
371− ((Direction::Backward, 6usize), (0, 0)), // _ultiline
372− ((Direction::Backward, 999usize), (0, 0)), // _ultiline
373− ((Direction::Forward, 3usize), (0, 3)), // Mul_iline
374− ((Direction::Forward, 0usize), (0, 3)), // Mul_iline
375− ((Direction::Backward, 0usize), (0, 3)), // Mul_iline
376− ((Direction::Forward, 999usize), (0, 9)), // Multilin_
377− ((Direction::Forward, 999usize), (0, 9)), // Multilin_
319+ ((Direction::Forward, 1usize), (0, 1)), // M|ultiline\n
320+ ((Direction::Forward, 2usize), (0, 3)), // Mul|tiline\n
321+ ((Direction::Backward, 6usize), (0, 0)), // |Multiline\n
322+ ((Direction::Backward, 999usize), (0, 0)), // |Multiline\n
323+ ((Direction::Forward, 3usize), (0, 3)), // Mul|tiline\n
324+ ((Direction::Forward, 0usize), (0, 3)), // Mul|tiline\n
325+ ((Direction::Backward, 0usize), (0, 3)), // Mul|tiline\n
326+ ((Direction::Forward, 999usize), (0, 9)), // Multiline|\n
327+ ((Direction::Forward, 999usize), (0, 9)), // Multiline|\n
378328 ]);
379329
380330 for ((direction, amount), coordinates) in moves_and_expected_coordinates {
@@ -446,7 +396,7 @@ mod test {
446396 // First descent preserves column as the target line is wider
447397 ((Axis::V, Direction::Forward, 1usize), (1, 8)),
448398 // Second descent clamps column as the target line is shorter
449− ((Axis::V, Direction::Forward, 1usize), (2, 4)),
399+ ((Axis::V, Direction::Forward, 1usize), (2, 5)),
450400 // Third descent restores the original column
451401 ((Axis::V, Direction::Forward, 1usize), (3, 8)),
452402 // Behaviour is preserved even through long jumps
@@ -760,45 +710,4 @@ mod test {
760710 }
761711 }
762712 }
763−
764− #[test]
765− fn test_categorize() {
766− const WORD_TEST_CASE: &'static str =
767− "_hello_world_あいうえおー1234567890１２３４５６７８９０";
768− const PUNCTUATION_TEST_CASE: &'static str =
769− "!\"#$%&\'()*+,-./:;<=>?@[\\]^`{|}~！”＃＄％＆’（）＊＋、。：；＜＝＞？＠「」＾｀｛｜｝～";
770− const WHITESPACE_TEST_CASE: &'static str = "  　   ";
771−
772− assert_eq!(Category::Eol, categorize('\n'));
773−
774− for ch in WHITESPACE_TEST_CASE.chars() {
775− assert_eq!(
776− Category::Whitespace,
777− categorize(ch),
778− "Testing '{}', but got `{:?}` instead of `Category::Whitespace`",
779− ch,
780− categorize(ch)
781− );
782− }
783−
784− for ch in WORD_TEST_CASE.chars() {
785− assert_eq!(
786− Category::Word,
787− categorize(ch),
788− "Testing '{}', but got `{:?}` instead of `Category::Word`",
789− ch,
790− categorize(ch)
791− );
792− }
793−
794− for ch in PUNCTUATION_TEST_CASE.chars() {
795− assert_eq!(
796− Category::Punctuation,
797− categorize(ch),
798− "Testing '{}', but got `{:?}` instead of `Category::Punctuation`",
799− ch,
800− categorize(ch)
801− );
802− }
803− }
804713}
helix-core/src/position.rs+4−2
@@ -1,4 +1,5 @@
11use crate::{
2+ chars::char_is_line_ending,
23 graphemes::{nth_next_grapheme_boundary, RopeGraphemes},
34 Rope, RopeSlice,
45};
@@ -23,8 +24,9 @@ impl Position {
2324 pub fn traverse(self, text: &crate::Tendril) -> Self {
2425 let Self { mut row, mut col } = self;
2526 // TODO: there should be a better way here
26− for ch in text.chars() {
27− if ch == '\n' {
27+ let mut chars = text.chars().peekable();
28+ while let Some(ch) = chars.next() {
29+ if char_is_line_ending(ch) && !(ch == '\r' && chars.peek() == Some(&'\n')) {
2830 row += 1;
2931 col = 0;
3032 } else {
helix-core/src/syntax.rs+5−4
@@ -1,4 +1,4 @@
1−use crate::{regex::Regex, Change, Rope, RopeSlice, Transaction};
1+use crate::{chars::char_is_line_ending, regex::Regex, Change, Rope, RopeSlice, Transaction};
22pub use helix_syntax::{get_language, get_language_name, Lang};
33
44use std::{
@@ -579,9 +579,10 @@ impl LanguageLayer {
579579 mut column,
580580 } = point;
581581
582− // TODO: there should be a better way here
583− for ch in text.bytes() {
584− if ch == b'\n' {
582+ // TODO: there should be a better way here.
583+ let mut chars = text.chars().peekable();
584+ while let Some(ch) = chars.next() {
585+ if char_is_line_ending(ch) && !(ch == '\r' && chars.peek() == Some(&'\n')) {
585586 row += 1;
586587 column = 0;
587588 } else {
helix-lsp/src/client.rs+4−3
@@ -3,7 +3,7 @@ use crate::{
33 Call, Error, OffsetEncoding, Result,
44};
55
6−use helix_core::{find_root, ChangeSet, Rope};
6+use helix_core::{chars::char_is_line_ending, find_root, ChangeSet, Rope};
77use jsonrpc_core as jsonrpc;
88use lsp_types as lsp;
99use serde_json::Value;
@@ -337,8 +337,9 @@ impl Client {
337337 mut character,
338338 } = pos;
339339
340− for ch in text.chars() {
341− if ch == '\n' {
340+ let mut chars = text.chars().peekable();
341+ while let Some(ch) = chars.next() {
342+ if char_is_line_ending(ch) && !(ch == '\r' && chars.peek() == Some(&'\n')) {
342343 line += 1;
343344 character = 0;
344345 } else {
helix-term/src/commands.rs+7−5
@@ -1,6 +1,6 @@
11use helix_core::{
22 comment, coords_at_pos, find_first_non_whitespace_char, find_root, get_line_ending, graphemes,
3− indent, line_end, match_brackets,
3+ indent, line_end_char_index, match_brackets,
44 movement::{self, Direction},
55 object, pos_at_coords,
66 regex::{self, Regex},
@@ -342,7 +342,7 @@ fn move_line_end(cx: &mut Context) {
342342 let text = doc.text();
343343 let line = text.char_to_line(range.head);
344344
345− let pos = line_end(&text.slice(..), line);
345+ let pos = line_end_char_index(&text.slice(..), line);
346346
347347 Range::new(pos, pos)
348348 });
@@ -490,6 +490,8 @@ where
490490 let count = cx.count();
491491
492492 // need to wait for next key
493+ // TODO: should this be done by grapheme rather than char? For example,
494+ // we can't properly handle the line-ending case here in terms of char.
493495 cx.on_next_key(move |cx, event| {
494496 let ch = match event {
495497 KeyEvent {
@@ -623,7 +625,7 @@ fn replace(cx: &mut Context) {
623625 KeyEvent {
624626 code: KeyCode::Enter,
625627 ..
626− } => Some('\n'), // TODO: replace this with DEFAULT_LINE_ENDING
628+ } => Some('\n'), // TODO: use the document's default line ending.
627629 _ => None,
628630 };
629631
@@ -763,7 +765,7 @@ fn extend_line_end(cx: &mut Context) {
763765 let text = doc.text();
764766 let line = text.char_to_line(range.head);
765767
766− let pos = line_end(&text.slice(..), line);
768+ let pos = line_end_char_index(&text.slice(..), line);
767769
768770 Range::new(range.anchor, pos)
769771 });
@@ -1642,7 +1644,7 @@ fn append_to_line(cx: &mut Context) {
16421644 let selection = doc.selection(view.id).transform(|range| {
16431645 let text = doc.text();
16441646 let line = text.char_to_line(range.head);
1645− let pos = line_end(&text.slice(..), line);
1647+ let pos = line_end_char_index(&text.slice(..), line);
16461648 Range::new(pos, pos)
16471649 });
16481650 doc.set_selection(view.id, selection);
helix-term/src/ui/markdown.rs+2
@@ -110,6 +110,8 @@ fn parse<'a>(contents: &'a str, theme: Option<&Theme>) -> tui::text::Text<'a> {
110110 // TODO: replace tabs with indentation
111111
112112 let mut slice = &text[start..end];
113+ // TODO: do we need to handle all unicode line endings
114+ // here, or is just '\n' okay?
113115 while let Some(end) = slice.find('\n') {
114116 // emit span up to newline
115117 let text = &slice[..end];
helix-tui/Cargo.toml+1
@@ -22,3 +22,4 @@ unicode-segmentation = "1.2"
2222unicode-width = "0.1"
2323crossterm = { version = "0.20", optional = true }
2424serde = { version = "1", "optional" = true, features = ["derive"]}
25+helix-core = { version = "0.2", path = "../helix-core" }
helix-tui/src/text.rs+2−1
@@ -47,6 +47,7 @@
4747//! ]);
4848//! ```
4949use crate::style::Style;
50+use helix_core::line_ending::str_is_line_ending;
5051use std::borrow::Cow;
5152use unicode_segmentation::UnicodeSegmentation;
5253use unicode_width::UnicodeWidthStr;
@@ -177,7 +178,7 @@ impl<'a> Span<'a> {
177178 symbol: g,
178179 style: base_style.patch(self.style),
179180 })
180− .filter(|s| s.symbol != "\n")
181+ .filter(|s| !str_is_line_ending(s.symbol))
181182 }
182183}
183184
helix-tui/src/widgets/reflow.rs+5−4
@@ -1,4 +1,5 @@
11use crate::text::StyledGrapheme;
2+use helix_core::line_ending::str_is_line_ending;
23use unicode_segmentation::UnicodeSegmentation;
34use unicode_width::UnicodeWidthStr;
45
@@ -62,13 +63,13 @@ impl<'a, 'b> LineComposer<'a> for WordWrapper<'a, 'b> {
6263 // Ignore characters wider that the total max width.
6364 if symbol.width() as u16 > self.max_line_width
6465 // Skip leading whitespace when trim is enabled.
65− || self.trim && symbol_whitespace && symbol != "\n" && current_line_width == 0
66+ || self.trim && symbol_whitespace && !str_is_line_ending(symbol) && current_line_width == 0
6667 {
6768 continue;
6869 }
6970
7071 // Break on newline and discard it.
71− if symbol == "\n" {
72+ if str_is_line_ending(symbol) {
7273 if prev_whitespace {
7374 current_line_width = width_to_last_word_end;
7475 self.current_line.truncate(symbols_to_last_word_end);
@@ -170,7 +171,7 @@ impl<'a, 'b> LineComposer<'a> for LineTruncator<'a, 'b> {
170171 }
171172
172173 // Break on newline and discard it.
173− if symbol == "\n" {
174+ if str_is_line_ending(symbol) {
174175 break;
175176 }
176177
@@ -199,7 +200,7 @@ impl<'a, 'b> LineComposer<'a> for LineTruncator<'a, 'b> {
199200
200201 if skip_rest {
201202 for StyledGrapheme { symbol, .. } in &mut self.symbols {
202− if symbol == "\n" {
203+ if str_is_line_ending(symbol) {
203204 break;
204205 }
205206 }
helix-view/src/document.rs+13−11
@@ -10,7 +10,7 @@ use std::sync::Arc;
1010
1111use helix_core::{
1212 auto_detect_line_ending,
13− chars::{char_is_linebreak, char_is_whitespace},
13+ chars::{char_is_line_ending, char_is_whitespace},
1414 history::History,
1515 syntax::{LanguageConfiguration, LOADER},
1616 ChangeSet, Diagnostic, LineEnding, Rope, Selection, State, Syntax, Transaction,
@@ -81,6 +81,9 @@ pub struct Document {
8181 /// Current indent style.
8282 pub indent_style: IndentStyle,
8383
84+ /// The document's default line ending.
85+ pub line_ending: LineEnding,
86+
8487 syntax: Option<Syntax>,
8588 // /// Corresponding language scope name. Usually `source.<lang>`.
8689 pub(crate) language: Option<Arc<LanguageConfiguration>>,
@@ -99,7 +102,6 @@ pub struct Document {
99102
100103 diagnostics: Vec<Diagnostic>,
101104 language_server: Option<Arc<helix_lsp::Client>>,
102− line_ending: LineEnding,
103105}
104106
105107use std::fmt;
@@ -254,21 +256,21 @@ impl Document {
254256 pub fn load(path: PathBuf) -> Result<Self, Error> {
255257 use std::{fs::File, io::BufReader};
256258
257− let doc = if !path.exists() {
259+ let mut doc = if !path.exists() {
258260 Rope::from(DEFAULT_LINE_ENDING.as_str())
259261 } else {
260262 let file = File::open(&path).context(format!("unable to open {:?}", path))?;
261− let mut doc = Rope::from_reader(BufReader::new(file))?;
262− // add missing newline at the end of file
263− if doc.len_bytes() == 0 || doc.byte(doc.len_bytes() - 1) != b'\n' {
264− doc.insert_char(doc.len_chars(), '\n');
265− }
266− doc
263+ Rope::from_reader(BufReader::new(file))?
267264 };
268265
269266 // search for line endings
270267 let line_ending = auto_detect_line_ending(&doc).unwrap_or(DEFAULT_LINE_ENDING);
271268
269+ // add missing newline at the end of file
270+ if doc.len_bytes() == 0 || char_is_line_ending(doc.char(doc.len_chars() - 1)) {
271+ doc.insert(doc.len_chars(), line_ending.as_str());
272+ }
273+
272274 let mut doc = Self::new(doc);
273275 // set the path and try detecting the language
274276 doc.set_path(&path)?;
@@ -379,7 +381,7 @@ impl Document {
379381 Some(' ') => false,
380382
381383 // Ignore blank lines.
382− Some(c) if char_is_linebreak(c) => continue,
384+ Some(c) if char_is_line_ending(c) => continue,
383385
384386 _ => {
385387 prev_line_is_tabs = false;
@@ -403,7 +405,7 @@ impl Document {
403405 c if char_is_whitespace(c) => count_is_done = true,
404406
405407 // Ignore blank lines.
406− c if char_is_linebreak(c) => continue 'outer,
408+ c if char_is_line_ending(c) => continue 'outer,
407409
408410 _ => break,
409411 }
helix-view/src/editor.rs+2−2
@@ -12,7 +12,7 @@ use anyhow::Error;
1212
1313pub use helix_core::diagnostic::Severity;
1414pub use helix_core::register::Registers;
15−use helix_core::Position;
15+use helix_core::{Position, DEFAULT_LINE_ENDING};
1616
1717#[derive(Debug)]
1818pub struct Editor {
@@ -150,7 +150,7 @@ impl Editor {
150150
151151 pub fn new_file(&mut self, action: Action) -> DocumentId {
152152 use helix_core::Rope;
153− let doc = Document::new(Rope::from("\n"));
153+ let doc = Document::new(Rope::from(DEFAULT_LINE_ENDING.as_str()));
154154 let id = self.documents.insert(doc);
155155 self.documents[id].id = id;
156156 self.switch(id, action);