git.iko.soy/helix
on master help sign in
Merge pull request #224 from helix-editor/line_ending_detection
a70de6eBlaž Hrastnik
Cargo.lock+1
compare with master
unifiedsplit
@@ -344,6 +344,7 @@ dependencies = [
344 "bitflags",
344 "bitflags",
345 "cassowary",
345 "cassowary",
346 "crossterm",
346 "crossterm",
347+ "helix-core",
347 "serde",
348 "serde",
348 "unicode-segmentation",
349 "unicode-segmentation",
349 "unicode-width",
350 "unicode-width",
helix-core/src/auto_pairs.rs+1−1
@@ -12,7 +12,7 @@ pub const PAIRS: &[(char, char)] = &[
12 ('`', '`'),
12 ('`', '`'),
13];
13];
14
14
15−const CLOSE_BEFORE: &str = ")]}'\":;> \n"; // includes space and newline
15+const CLOSE_BEFORE: &str = ")]}'\":;> \n\r\u{000B}\u{000C}\u{0085}\u{2028}\u{2029}"; // includes space and newlines
16
16
17// insert hook:
17// insert hook:
18// Fn(doc, selection, char) => Option<Transaction>
18// Fn(doc, selection, char) => Option<Transaction>
helix-core/src/chars.rs+107−15
@@ -1,25 +1,44 @@
1+use crate::LineEnding;
2+
3+#[derive(Debug, Eq, PartialEq)]
4+pub enum CharCategory {
5+ Whitespace,
6+ Eol,
7+ Word,
8+ Punctuation,
9+ Unknown,
10+}
11+
12+#[inline]
13+pub fn categorize_char(ch: char) -> CharCategory {
14+ if char_is_line_ending(ch) {
15+ CharCategory::Eol
16+ } else if ch.is_whitespace() {
17+ CharCategory::Whitespace
18+ } else if char_is_word(ch) {
19+ CharCategory::Word
20+ } else if char_is_punctuation(ch) {
21+ CharCategory::Punctuation
22+ } else {
23+ CharCategory::Unknown
24+ }
25+}
26+
1−/// Determine whether a character is a line break.
27+/// Determine whether a character is a line ending.
28+#[inline]
2−pub fn char_is_linebreak(c: char) -> bool {
29+pub fn char_is_line_ending(ch: char) -> bool {
3− matches!(
30+ LineEnding::from_char(ch).is_some()
4− c,
5− '\u{000A}' | // LineFeed
6− '\u{000B}' | // VerticalTab
7− '\u{000C}' | // FormFeed
8− '\u{000D}' | // CarriageReturn
9− '\u{0085}' | // NextLine
10− '\u{2028}' | // Line Separator
11− '\u{2029}' // ParagraphSeparator
12− )
13}
31}
14
32
15/// Determine whether a character qualifies as (non-line-break)
33/// Determine whether a character qualifies as (non-line-break)
16/// whitespace.
34/// whitespace.
35+#[inline]
17−pub fn char_is_whitespace(c: char) -> bool {
36+pub fn char_is_whitespace(ch: char) -> bool {
18 // TODO: this is a naive binary categorization of whitespace
37 // TODO: this is a naive binary categorization of whitespace
19 // characters. For display, word wrapping, etc. we'll need a better
38 // characters. For display, word wrapping, etc. we'll need a better
20 // categorization based on e.g. breaking vs non-breaking spaces
39 // categorization based on e.g. breaking vs non-breaking spaces
21 // and whether they're zero-width or not.
40 // and whether they're zero-width or not.
22− match c {
41+ match ch {
23 //'\u{1680}' | // Ogham Space Mark (here for completeness, but usually displayed as a dash, not as whitespace)
42 //'\u{1680}' | // Ogham Space Mark (here for completeness, but usually displayed as a dash, not as whitespace)
24 '\u{0009}' | // Character Tabulation
43 '\u{0009}' | // Character Tabulation
25 '\u{0020}' | // Space
44 '\u{0020}' | // Space
@@ -34,8 +53,81 @@ pub fn char_is_whitespace(c: char) -> bool {
34 // En Quad, Em Quad, En Space, Em Space, Three-per-em Space,
53 // En Quad, Em Quad, En Space, Em Space, Three-per-em Space,
35 // Four-per-em Space, Six-per-em Space, Figure Space,
54 // Four-per-em Space, Six-per-em Space, Figure Space,
36 // Punctuation Space, Thin Space, Hair Space, Zero Width Space.
55 // Punctuation Space, Thin Space, Hair Space, Zero Width Space.
37− c if ('\u{2000}' ..= '\u{200B}').contains(&c) => true,
56+ ch if ('\u{2000}' ..= '\u{200B}').contains(&ch) => true,
38
57
39 _ => false,
58 _ => false,
40 }
59 }
41}
60}
61+
62+#[inline]
63+pub fn char_is_punctuation(ch: char) -> bool {
64+ use unicode_general_category::{get_general_category, GeneralCategory};
65+
66+ matches!(
67+ get_general_category(ch),
68+ GeneralCategory::OtherPunctuation
69+ | GeneralCategory::OpenPunctuation
70+ | GeneralCategory::ClosePunctuation
71+ | GeneralCategory::InitialPunctuation
72+ | GeneralCategory::FinalPunctuation
73+ | GeneralCategory::ConnectorPunctuation
74+ | GeneralCategory::DashPunctuation
75+ | GeneralCategory::MathSymbol
76+ | GeneralCategory::CurrencySymbol
77+ | GeneralCategory::ModifierSymbol
78+ )
79+}
80+
81+#[inline]
82+pub fn char_is_word(ch: char) -> bool {
83+ ch.is_alphanumeric() || ch == '_'
84+}
85+
86+#[cfg(test)]
87+mod test {
88+ use super::*;
89+
90+ #[test]
91+ fn test_categorize() {
92+ const EOL_TEST_CASE: &'static str = "\n\r\u{000B}\u{000C}\u{0085}\u{2028}\u{2029}";
93+ const WORD_TEST_CASE: &'static str =
94+ "_hello_world_あいうえおー1234567890１２３４５６７８９０";
95+ const PUNCTUATION_TEST_CASE: &'static str =
96+ "!\"#$%&\'()*+,-./:;<=>?@[\\]^`{|}~！”＃＄％＆’（）＊＋、。：；＜＝＞？＠「」＾｀｛｜｝～";
97+ const WHITESPACE_TEST_CASE: &'static str = "  　   ";
98+
99+ for ch in EOL_TEST_CASE.chars() {
100+ assert_eq!(CharCategory::Eol, categorize_char(ch));
101+ }
102+
103+ for ch in WHITESPACE_TEST_CASE.chars() {
104+ assert_eq!(
105+ CharCategory::Whitespace,
106+ categorize_char(ch),
107+ "Testing '{}', but got `{:?}` instead of `Category::Whitespace`",
108+ ch,
109+ categorize_char(ch)
110+ );
111+ }
112+
113+ for ch in WORD_TEST_CASE.chars() {
114+ assert_eq!(
115+ CharCategory::Word,
116+ categorize_char(ch),
117+ "Testing '{}', but got `{:?}` instead of `Category::Word`",
118+ ch,
119+ categorize_char(ch)
120+ );
121+ }
122+
123+ for ch in PUNCTUATION_TEST_CASE.chars() {
124+ assert_eq!(
125+ CharCategory::Punctuation,
126+ categorize_char(ch),
127+ "Testing '{}', but got `{:?}` instead of `Category::Punctuation`",
128+ ch,
129+ categorize_char(ch)
130+ );
131+ }
132+ }
133+}
helix-core/src/lib.rs+3
@@ -6,6 +6,7 @@ pub mod diagnostic;
6pub mod graphemes;
6pub mod graphemes;
7pub mod history;
7pub mod history;
8pub mod indent;
8pub mod indent;
9+pub mod line_ending;
9pub mod macros;
10pub mod macros;
10pub mod match_brackets;
11pub mod match_brackets;
11pub mod movement;
12pub mod movement;
@@ -106,6 +107,7 @@ pub use tendril::StrTendril as Tendril;
106#[doc(inline)]
107#[doc(inline)]
107pub use {regex, tree_sitter};
108pub use {regex, tree_sitter};
108
109
110+pub use graphemes::RopeGraphemes;
109pub use position::{coords_at_pos, pos_at_coords, Position};
111pub use position::{coords_at_pos, pos_at_coords, Position};
110pub use selection::{Range, Selection};
112pub use selection::{Range, Selection};
111pub use smallvec::SmallVec;
113pub use smallvec::SmallVec;
@@ -114,4 +116,5 @@ pub use syntax::Syntax;
114pub use diagnostic::Diagnostic;
116pub use diagnostic::Diagnostic;
115pub use state::State;
117pub use state::State;
116
118
119+pub use line_ending::{LineEnding, DEFAULT_LINE_ENDING};
117pub use transaction::{Assoc, Change, ChangeSet, Operation, Transaction};
120pub use transaction::{Assoc, Change, ChangeSet, Operation, Transaction};
helix-core/src/line_ending.rsnew file+252
@@ -0,0 +1,252 @@
1+use crate::{Rope, RopeGraphemes, RopeSlice};
2+
3+#[cfg(target_os = "windows")]
4+pub const DEFAULT_LINE_ENDING: LineEnding = LineEnding::Crlf;
5+#[cfg(not(target_os = "windows"))]
6+pub const DEFAULT_LINE_ENDING: LineEnding = LineEnding::LF;
7+
8+/// Represents one of the valid Unicode line endings.
9+#[derive(PartialEq, Copy, Clone, Debug)]
10+pub enum LineEnding {
11+ Crlf, // CarriageReturn followed by LineFeed
12+ LF, // U+000A -- LineFeed
13+ VT, // U+000B -- VerticalTab
14+ FF, // U+000C -- FormFeed
15+ CR, // U+000D -- CarriageReturn
16+ Nel, // U+0085 -- NextLine
17+ LS, // U+2028 -- Line Separator
18+ PS, // U+2029 -- ParagraphSeparator
19+}
20+
21+impl LineEnding {
22+ #[inline]
23+ pub fn len_chars(&self) -> usize {
24+ match self {
25+ Self::Crlf => 2,
26+ _ => 1,
27+ }
28+ }
29+
30+ #[inline]
31+ pub fn as_str(&self) -> &'static str {
32+ match self {
33+ Self::Crlf => "\u{000D}\u{000A}",
34+ Self::LF => "\u{000A}",
35+ Self::VT => "\u{000B}",
36+ Self::FF => "\u{000C}",
37+ Self::CR => "\u{000D}",
38+ Self::Nel => "\u{0085}",
39+ Self::LS => "\u{2028}",
40+ Self::PS => "\u{2029}",
41+ }
42+ }
43+
44+ #[inline]
45+ pub fn from_char(ch: char) -> Option<LineEnding> {
46+ match ch {
47+ '\u{000A}' => Some(LineEnding::LF),
48+ '\u{000B}' => Some(LineEnding::VT),
49+ '\u{000C}' => Some(LineEnding::FF),
50+ '\u{000D}' => Some(LineEnding::CR),
51+ '\u{0085}' => Some(LineEnding::Nel),
52+ '\u{2028}' => Some(LineEnding::LS),
53+ '\u{2029}' => Some(LineEnding::PS),
54+ // Not a line ending
55+ _ => None,
56+ }
57+ }
58+
59+ // Normally we'd want to implement the FromStr trait, but in this case
60+ // that would force us into a different return type than from_char or
61+ // or from_rope_slice, which would be weird.
62+ #[allow(clippy::should_implement_trait)]
63+ #[inline]
64+ pub fn from_str(g: &str) -> Option<LineEnding> {
65+ match g {
66+ "\u{000D}\u{000A}" => Some(LineEnding::Crlf),
67+ "\u{000A}" => Some(LineEnding::LF),
68+ "\u{000B}" => Some(LineEnding::VT),
69+ "\u{000C}" => Some(LineEnding::FF),
70+ "\u{000D}" => Some(LineEnding::CR),
71+ "\u{0085}" => Some(LineEnding::Nel),
72+ "\u{2028}" => Some(LineEnding::LS),
73+ "\u{2029}" => Some(LineEnding::PS),
74+ // Not a line ending
75+ _ => None,
76+ }
77+ }
78+
79+ #[inline]
80+ pub fn from_rope_slice(g: &RopeSlice) -> Option<LineEnding> {
81+ if let Some(text) = g.as_str() {
82+ LineEnding::from_str(text)
83+ } else {
84+ // Non-contiguous, so it can't be a line ending.
85+ // Specifically, Ropey guarantees that CRLF is always
86+ // contiguous. And the remaining line endings are all
87+ // single `char`s, and therefore trivially contiguous.
88+ None
89+ }
90+ }
91+}
92+
93+#[inline]
94+pub fn str_is_line_ending(s: &str) -> bool {
95+ LineEnding::from_str(s).is_some()
96+}
97+
98+/// Attempts to detect what line ending the passed document uses.
99+pub fn auto_detect_line_ending(doc: &Rope) -> Option<LineEnding> {
100+ // Return first matched line ending. Not all possible line endings
101+ // are being matched, as they might be special-use only
102+ for line in doc.lines().take(100) {
103+ match get_line_ending(&line) {
104+ None | Some(LineEnding::VT) | Some(LineEnding::FF) | Some(LineEnding::PS) => {}
105+ ending => return ending,
106+ }
107+ }
108+ None
109+}
110+
111+/// Returns the passed line's line ending, if any.
112+pub fn get_line_ending(line: &RopeSlice) -> Option<LineEnding> {
113+ // Last character as str.
114+ let g1 = line
115+ .slice(line.len_chars().saturating_sub(1)..)
116+ .as_str()
117+ .unwrap();
118+
119+ // Last two characters as str, or empty str if they're not contiguous.
120+ // It's fine to punt on the non-contiguous case, because Ropey guarantees
121+ // that CRLF is always contiguous.
122+ let g2 = line
123+ .slice(line.len_chars().saturating_sub(2)..)
124+ .as_str()
125+ .unwrap_or("");
126+
127+ // First check the two-character case for CRLF, then check the single-character case.
128+ LineEnding::from_str(g2).or_else(|| LineEnding::from_str(g1))
129+}
130+
131+/// Returns the passed line's line ending, if any.
132+pub fn get_line_ending_of_str(line: &str) -> Option<LineEnding> {
133+ if line.ends_with("\u{000D}\u{000A}") {
134+ Some(LineEnding::Crlf)
135+ } else if line.ends_with('\u{000A}') {
136+ Some(LineEnding::LF)
137+ } else if line.ends_with('\u{000B}') {
138+ Some(LineEnding::VT)
139+ } else if line.ends_with('\u{000C}') {
140+ Some(LineEnding::FF)
141+ } else if line.ends_with('\u{000D}') {
142+ Some(LineEnding::CR)
143+ } else if line.ends_with('\u{0085}') {
144+ Some(LineEnding::Nel)
145+ } else if line.ends_with('\u{2028}') {
146+ Some(LineEnding::LS)
147+ } else if line.ends_with('\u{2029}') {
148+ Some(LineEnding::PS)
149+ } else {
150+ None
151+ }
152+}
153+
154+/// Returns the char index of the end of the given line, not including its line ending.
155+pub fn line_end_char_index(slice: &RopeSlice, line: usize) -> usize {
156+ slice.line_to_char(line + 1)
157+ - get_line_ending(&slice.line(line))
158+ .map(|le| le.len_chars())
159+ .unwrap_or(0)
160+}
161+
162+#[cfg(test)]
163+mod line_ending_tests {
164+ use super::*;
165+
166+ #[test]
167+ fn line_ending_autodetect() {
168+ assert_eq!(
169+ auto_detect_line_ending(&Rope::from_str("\n")),
170+ Some(LineEnding::LF)
171+ );
172+ assert_eq!(
173+ auto_detect_line_ending(&Rope::from_str("\r\n")),
174+ Some(LineEnding::Crlf)
175+ );
176+ assert_eq!(auto_detect_line_ending(&Rope::from_str("hello")), None);
177+ assert_eq!(auto_detect_line_ending(&Rope::from_str("")), None);
178+ assert_eq!(
179+ auto_detect_line_ending(&Rope::from_str("hello\nhelix\r\n")),
180+ Some(LineEnding::LF)
181+ );
182+ assert_eq!(
183+ auto_detect_line_ending(&Rope::from_str("a formfeed\u{000C}")),
184+ None
185+ );
186+ assert_eq!(
187+ auto_detect_line_ending(&Rope::from_str("\n\u{000A}\n \u{000A}")),
188+ Some(LineEnding::LF)
189+ );
190+ assert_eq!(
191+ auto_detect_line_ending(&Rope::from_str(
192+ "a formfeed\u{000C} with a\u{000C} linefeed\u{000A}"
193+ )),
194+ Some(LineEnding::LF)
195+ );
196+ assert_eq!(auto_detect_line_ending(&Rope::from_str("a formfeed\u{000C} with a\u{000C} carriage return linefeed\u{000D}\u{000A} and a linefeed\u{000A}")), Some(LineEnding::Crlf));
197+ }
198+
199+ #[test]
200+ fn str_to_line_ending() {
201+ assert_eq!(LineEnding::from_str("\r"), Some(LineEnding::CR));
202+ assert_eq!(LineEnding::from_str("\n"), Some(LineEnding::LF));
203+ assert_eq!(LineEnding::from_str("\r\n"), Some(LineEnding::Crlf));
204+ assert_eq!(LineEnding::from_str("hello\n"), None);
205+ }
206+
207+ #[test]
208+ fn rope_slice_to_line_ending() {
209+ let r = Rope::from_str("hello\r\n");
210+ assert_eq!(
211+ LineEnding::from_rope_slice(&r.slice(5..6)),
212+ Some(LineEnding::CR)
213+ );
214+ assert_eq!(
215+ LineEnding::from_rope_slice(&r.slice(6..7)),
216+ Some(LineEnding::LF)
217+ );
218+ assert_eq!(
219+ LineEnding::from_rope_slice(&r.slice(5..7)),
220+ Some(LineEnding::Crlf)
221+ );
222+ assert_eq!(LineEnding::from_rope_slice(&r.slice(..)), None);
223+ }
224+
225+ #[test]
226+ fn get_line_ending_rope_slice() {
227+ let r = Rope::from_str("Hello\rworld\nhow\r\nare you?");
228+ assert_eq!(get_line_ending(&r.slice(..6)), Some(LineEnding::CR));
229+ assert_eq!(get_line_ending(&r.slice(..12)), Some(LineEnding::LF));
230+ assert_eq!(get_line_ending(&r.slice(..17)), Some(LineEnding::Crlf));
231+ assert_eq!(get_line_ending(&r.slice(..)), None);
232+ }
233+
234+ #[test]
235+ fn get_line_ending_str() {
236+ let text = "Hello\rworld\nhow\r\nare you?";
237+ assert_eq!(get_line_ending_of_str(&text[..6]), Some(LineEnding::CR));
238+ assert_eq!(get_line_ending_of_str(&text[..12]), Some(LineEnding::LF));
239+ assert_eq!(get_line_ending_of_str(&text[..17]), Some(LineEnding::Crlf));
240+ assert_eq!(get_line_ending_of_str(&text[..]), None);
241+ }
242+
243+ #[test]
244+ fn line_end_char_index_rope_slice() {
245+ let r = Rope::from_str("Hello\rworld\nhow\r\nare you?");
246+ let s = &r.slice(..);
247+ assert_eq!(line_end_char_index(s, 0), 5);
248+ assert_eq!(line_end_char_index(s, 1), 11);
249+ assert_eq!(line_end_char_index(s, 2), 15);
250+ assert_eq!(line_end_char_index(s, 3), 25);
251+ }
252+}
helix-core/src/movement.rs+38−128
@@ -3,8 +3,13 @@ use std::iter::{self, from_fn, Peekable, SkipWhile};
3use ropey::iter::Chars;
3use ropey::iter::Chars;
4
4
5use crate::{
5use crate::{
6+ chars::{
7+ categorize_char, char_is_line_ending, char_is_punctuation, char_is_whitespace,
8+ char_is_word, CharCategory,
9+ },
6 coords_at_pos,
10 coords_at_pos,
7 graphemes::{nth_next_grapheme_boundary, nth_prev_grapheme_boundary},
11 graphemes::{nth_next_grapheme_boundary, nth_prev_grapheme_boundary},
12+ line_ending::{get_line_ending, line_end_char_index},
8 pos_at_coords, Position, Range, RopeSlice,
13 pos_at_coords, Position, Range, RopeSlice,
9};
14};
10
15
@@ -37,9 +42,8 @@ pub fn move_horizontally(
37 nth_prev_grapheme_boundary(slice, pos, count).max(start)
42 nth_prev_grapheme_boundary(slice, pos, count).max(start)
38 }
43 }
39 Direction::Forward => {
44 Direction::Forward => {
40− // Line end is pos at the start of next line - 1
45+ let end_char_idx = line_end_char_index(&slice, line);
41− let end = slice.line_to_char(line + 1).saturating_sub(1);
42− nth_next_grapheme_boundary(slice, pos, count).min(end)
46+ nth_next_grapheme_boundary(slice, pos, count).min(end_char_idx)
43 }
47 }
44 };
48 };
45 let anchor = match behaviour {
49 let anchor = match behaviour {
@@ -68,8 +72,11 @@ pub fn move_vertically(
68 ),
72 ),
69 };
73 };
70
74
71− // convert to 0-indexed, subtract another 1 because len_chars() counts \n
75+ // Length of the line sans line-ending.
72− let new_line_len = slice.line(new_line).len_chars().saturating_sub(2);
76+ let new_line_len = {
77+ let line = slice.line(new_line);
78+ line.len_chars() - get_line_ending(&line).map(|le| le.len_chars()).unwrap_or(0)
79+ };
73
80
74 let new_col = std::cmp::min(horiz as usize, new_line_len);
81 let new_col = std::cmp::min(horiz as usize, new_line_len);
75
82
@@ -104,64 +111,6 @@ fn word_move(slice: RopeSlice, mut range: Range, count: usize, target: WordMotio
104}
111}
105
112
106// ---- util ------------
113// ---- util ------------
107−#[inline]
108−pub(crate) fn is_word(ch: char) -> bool {
109− ch.is_alphanumeric() || ch == '_'
110−}
111−
112−#[inline]
113−pub(crate) fn is_end_of_line(ch: char) -> bool {
114− ch == '\n'
115−}
116−
117−#[inline]
118−// Whitespace, but not end of line
119−pub(crate) fn is_strict_whitespace(ch: char) -> bool {
120− ch.is_whitespace() && !is_end_of_line(ch)
121−}
122−
123−#[inline]
124−pub(crate) fn is_punctuation(ch: char) -> bool {
125− use unicode_general_category::{get_general_category, GeneralCategory};
126−
127− matches!(
128− get_general_category(ch),
129− GeneralCategory::OtherPunctuation
130− | GeneralCategory::OpenPunctuation
131− | GeneralCategory::ClosePunctuation
132− | GeneralCategory::InitialPunctuation
133− | GeneralCategory::FinalPunctuation
134− | GeneralCategory::ConnectorPunctuation
135− | GeneralCategory::DashPunctuation
136− | GeneralCategory::MathSymbol
137− | GeneralCategory::CurrencySymbol
138− | GeneralCategory::ModifierSymbol
139− )
140−}
141−
142−#[derive(Debug, Eq, PartialEq)]
143−pub enum Category {
144− Whitespace,
145− Eol,
146− Word,
147− Punctuation,
148− Unknown,
149−}
150−
151−#[inline]
152−pub(crate) fn categorize(ch: char) -> Category {
153− if is_end_of_line(ch) {
154− Category::Eol
155− } else if ch.is_whitespace() {
156− Category::Whitespace
157− } else if is_word(ch) {
158− Category::Word
159− } else if is_punctuation(ch) {
160− Category::Punctuation
161− } else {
162− Category::Unknown
163− }
164−}
165
114
166#[inline]
115#[inline]
167/// Returns first index that doesn't satisfy a given predicate when
116/// Returns first index that doesn't satisfy a given predicate when
@@ -235,7 +184,8 @@ impl CharHelpers for Chars<'_> {
235 let mut phase = WordMotionPhase::Start;
184 let mut phase = WordMotionPhase::Start;
236 let mut head = origin.head;
185 let mut head = origin.head;
237 let mut anchor: Option<usize> = None;
186 let mut anchor: Option<usize> = None;
187+ let is_boundary =
238− let is_boundary = |a: char, b: Option<char>| categorize(a) != categorize(b.unwrap_or(a));
188+ |a: char, b: Option<char>| categorize_char(a) != categorize_char(b.unwrap_or(a));
239 while let Some(peek) = characters.peek().copied() {
189 while let Some(peek) = characters.peek().copied() {
240 phase = match phase {
190 phase = match phase {
241 WordMotionPhase::Start => {
191 WordMotionPhase::Start => {
@@ -244,7 +194,8 @@ impl CharHelpers for Chars<'_> {
244 break; // We're at the end, so there's nothing to do.
194 break; // We're at the end, so there's nothing to do.
245 }
195 }
246 // Anchor may remain here if the head wasn't at a boundary
196 // Anchor may remain here if the head wasn't at a boundary
247− if !is_boundary(peek, characters.peek().copied()) && !is_end_of_line(peek) {
197+ if !is_boundary(peek, characters.peek().copied()) && !char_is_line_ending(peek)
198+ {
248 anchor = Some(head);
199 anchor = Some(head);
249 }
200 }
250 // First character is always skipped by the head
201 // First character is always skipped by the head
@@ -252,7 +203,7 @@ impl CharHelpers for Chars<'_> {
252 WordMotionPhase::SkipNewlines
203 WordMotionPhase::SkipNewlines
253 }
204 }
254 WordMotionPhase::SkipNewlines => {
205 WordMotionPhase::SkipNewlines => {
255− if is_end_of_line(peek) {
206+ if char_is_line_ending(peek) {
256 characters.next();
207 characters.next();
257 if characters.peek().is_some() {
208 if characters.peek().is_some() {
258 advance(&mut head);
209 advance(&mut head);
@@ -286,12 +237,12 @@ fn reached_target(target: WordMotionTarget, peek: char, next_peek: Option<&char>
286
237
287 match target {
238 match target {
288 WordMotionTarget::NextWordStart => {
239 WordMotionTarget::NextWordStart => {
289− ((categorize(peek) != categorize(*next_peek))
240+ ((categorize_char(peek) != categorize_char(*next_peek))
290− && (is_end_of_line(*next_peek) || !next_peek.is_whitespace()))
241+ && (char_is_line_ending(*next_peek) || !next_peek.is_whitespace()))
291 }
242 }
292 WordMotionTarget::NextWordEnd | WordMotionTarget::PrevWordStart => {
243 WordMotionTarget::NextWordEnd | WordMotionTarget::PrevWordStart => {
293− ((categorize(peek) != categorize(*next_peek))
244+ ((categorize_char(peek) != categorize_char(*next_peek))
294− && (!peek.is_whitespace() || is_end_of_line(*next_peek)))
245+ && (!peek.is_whitespace() || char_is_line_ending(*next_peek)))
295 }
246 }
296 }
247 }
297}
248}
@@ -330,7 +281,7 @@ mod test {
330 slice,
281 slice,
331 move_vertically(slice, range, Direction::Forward, 1, Movement::Move).head
282 move_vertically(slice, range, Direction::Forward, 1, Movement::Move).head
332 ),
283 ),
333− (1, 2).into()
284+ (1, 3).into()
334 );
285 );
335 }
286 }
336
287
@@ -343,12 +294,12 @@ mod test {
343 let mut range = Range::point(position);
294 let mut range = Range::point(position);
344
295
345 let moves_and_expected_coordinates = [
296 let moves_and_expected_coordinates = [
346− ((Direction::Forward, 1usize), (0, 1)),
297+ ((Direction::Forward, 1usize), (0, 1)), // T|his is a simple alphabetic line
347− ((Direction::Forward, 2usize), (0, 3)),
298+ ((Direction::Forward, 2usize), (0, 3)), // Thi|s is a simple alphabetic line
348− ((Direction::Forward, 0usize), (0, 3)),
299+ ((Direction::Forward, 0usize), (0, 3)), // Thi|s is a simple alphabetic line
349− ((Direction::Forward, 999usize), (0, 31)),
300+ ((Direction::Forward, 999usize), (0, 32)), // This is a simple alphabetic line|
350− ((Direction::Forward, 999usize), (0, 31)),
301+ ((Direction::Forward, 999usize), (0, 32)), // This is a simple alphabetic line|
351− ((Direction::Backward, 999usize), (0, 0)),
302+ ((Direction::Backward, 999usize), (0, 0)), // |This is a simple alphabetic line
352 ];
303 ];
353
304
354 for ((direction, amount), coordinates) in IntoIter::new(moves_and_expected_coordinates) {
305 for ((direction, amount), coordinates) in IntoIter::new(moves_and_expected_coordinates) {
@@ -366,15 +317,15 @@ mod test {
366 let mut range = Range::point(position);
317 let mut range = Range::point(position);
367
318
368 let moves_and_expected_coordinates = IntoIter::new([
319 let moves_and_expected_coordinates = IntoIter::new([
369− ((Direction::Forward, 1usize), (0, 1)), // M_ltiline
320+ ((Direction::Forward, 1usize), (0, 1)), // M|ultiline\n
370− ((Direction::Forward, 2usize), (0, 3)), // Mul_iline
321+ ((Direction::Forward, 2usize), (0, 3)), // Mul|tiline\n
371− ((Direction::Backward, 6usize), (0, 0)), // _ultiline
322+ ((Direction::Backward, 6usize), (0, 0)), // |Multiline\n
372− ((Direction::Backward, 999usize), (0, 0)), // _ultiline
323+ ((Direction::Backward, 999usize), (0, 0)), // |Multiline\n
373− ((Direction::Forward, 3usize), (0, 3)), // Mul_iline
324+ ((Direction::Forward, 3usize), (0, 3)), // Mul|tiline\n
374− ((Direction::Forward, 0usize), (0, 3)), // Mul_iline
325+ ((Direction::Forward, 0usize), (0, 3)), // Mul|tiline\n
375− ((Direction::Backward, 0usize), (0, 3)), // Mul_iline
326+ ((Direction::Backward, 0usize), (0, 3)), // Mul|tiline\n
376− ((Direction::Forward, 999usize), (0, 9)), // Multilin_
327+ ((Direction::Forward, 999usize), (0, 9)), // Multiline|\n
377− ((Direction::Forward, 999usize), (0, 9)), // Multilin_
328+ ((Direction::Forward, 999usize), (0, 9)), // Multiline|\n
378 ]);
329 ]);
379
330
380 for ((direction, amount), coordinates) in moves_and_expected_coordinates {
331 for ((direction, amount), coordinates) in moves_and_expected_coordinates {
@@ -446,7 +397,7 @@ mod test {
446 // First descent preserves column as the target line is wider
397 // First descent preserves column as the target line is wider
447 ((Axis::V, Direction::Forward, 1usize), (1, 8)),
398 ((Axis::V, Direction::Forward, 1usize), (1, 8)),
448 // Second descent clamps column as the target line is shorter
399 // Second descent clamps column as the target line is shorter
449− ((Axis::V, Direction::Forward, 1usize), (2, 4)),
400+ ((Axis::V, Direction::Forward, 1usize), (2, 5)),
450 // Third descent restores the original column
401 // Third descent restores the original column
451 ((Axis::V, Direction::Forward, 1usize), (3, 8)),
402 ((Axis::V, Direction::Forward, 1usize), (3, 8)),
452 // Behaviour is preserved even through long jumps
403 // Behaviour is preserved even through long jumps
@@ -760,45 +711,4 @@ mod test {
760 }
711 }
761 }
712 }
762 }
713 }
763−
764− #[test]
765− fn test_categorize() {
766− const WORD_TEST_CASE: &'static str =
767− "_hello_world_あいうえおー1234567890１２３４５６７８９０";
768− const PUNCTUATION_TEST_CASE: &'static str =
769− "!\"#$%&\'()*+,-./:;<=>?@[\\]^`{|}~！”＃＄％＆’（）＊＋、。：；＜＝＞？＠「」＾｀｛｜｝～";
770− const WHITESPACE_TEST_CASE: &'static str = "  　   ";
771−
772− assert_eq!(Category::Eol, categorize('\n'));
773−
774− for ch in WHITESPACE_TEST_CASE.chars() {
775− assert_eq!(
776− Category::Whitespace,
777− categorize(ch),
778− "Testing '{}', but got `{:?}` instead of `Category::Whitespace`",
779− ch,
780− categorize(ch)
781− );
782− }
783−
784− for ch in WORD_TEST_CASE.chars() {
785− assert_eq!(
786− Category::Word,
787− categorize(ch),
788− "Testing '{}', but got `{:?}` instead of `Category::Word`",
789− ch,
790− categorize(ch)
791− );
792− }
793−
794− for ch in PUNCTUATION_TEST_CASE.chars() {
795− assert_eq!(
796− Category::Punctuation,
797− categorize(ch),
798− "Testing '{}', but got `{:?}` instead of `Category::Punctuation`",
799− ch,
800− categorize(ch)
801− );
802− }
803− }
804}
714}
helix-core/src/position.rs+4−2
@@ -1,4 +1,5 @@
1use crate::{
1use crate::{
2+ chars::char_is_line_ending,
2 graphemes::{nth_next_grapheme_boundary, RopeGraphemes},
3 graphemes::{nth_next_grapheme_boundary, RopeGraphemes},
3 Rope, RopeSlice,
4 Rope, RopeSlice,
4};
5};
@@ -23,8 +24,9 @@ impl Position {
23 pub fn traverse(self, text: &crate::Tendril) -> Self {
24 pub fn traverse(self, text: &crate::Tendril) -> Self {
24 let Self { mut row, mut col } = self;
25 let Self { mut row, mut col } = self;
25 // TODO: there should be a better way here
26 // TODO: there should be a better way here
26− for ch in text.chars() {
27+ let mut chars = text.chars().peekable();
27− if ch == '\n' {
28+ while let Some(ch) = chars.next() {
29+ if char_is_line_ending(ch) && !(ch == '\r' && chars.peek() == Some(&'\n')) {
28 row += 1;
30 row += 1;
29 col = 0;
31 col = 0;
30 } else {
32 } else {
helix-core/src/syntax.rs+5−4
@@ -1,4 +1,4 @@
1−use crate::{regex::Regex, Change, Rope, RopeSlice, Transaction};
1+use crate::{chars::char_is_line_ending, regex::Regex, Change, Rope, RopeSlice, Transaction};
2pub use helix_syntax::{get_language, get_language_name, Lang};
2pub use helix_syntax::{get_language, get_language_name, Lang};
3
3
4use arc_swap::ArcSwap;
4use arc_swap::ArcSwap;
@@ -589,9 +589,10 @@ impl LanguageLayer {
589 mut column,
589 mut column,
590 } = point;
590 } = point;
591
591
592− // TODO: there should be a better way here
592+ // TODO: there should be a better way here.
593− for ch in text.bytes() {
593+ let mut chars = text.chars().peekable();
594− if ch == b'\n' {
594+ while let Some(ch) = chars.next() {
595+ if char_is_line_ending(ch) && !(ch == '\r' && chars.peek() == Some(&'\n')) {
595 row += 1;
596 row += 1;
596 column = 0;
597 column = 0;
597 } else {
598 } else {
helix-lsp/src/client.rs+4−3
@@ -3,7 +3,7 @@ use crate::{
3 Call, Error, OffsetEncoding, Result,
3 Call, Error, OffsetEncoding, Result,
4};
4};
5
5
6−use helix_core::{find_root, ChangeSet, Rope};
6+use helix_core::{chars::char_is_line_ending, find_root, ChangeSet, Rope};
7use jsonrpc_core as jsonrpc;
7use jsonrpc_core as jsonrpc;
8use lsp_types as lsp;
8use lsp_types as lsp;
9use serde_json::Value;
9use serde_json::Value;
@@ -337,8 +337,9 @@ impl Client {
337 mut character,
337 mut character,
338 } = pos;
338 } = pos;
339
339
340− for ch in text.chars() {
340+ let mut chars = text.chars().peekable();
341− if ch == '\n' {
341+ while let Some(ch) = chars.next() {
342+ if char_is_line_ending(ch) && !(ch == '\r' && chars.peek() == Some(&'\n')) {
342 line += 1;
343 line += 1;
343 character = 0;
344 character = 0;
344 } else {
345 } else {
helix-term/src/commands.rs+111−30
@@ -1,12 +1,15 @@
1use helix_core::{
1use helix_core::{
2 comment, coords_at_pos, find_first_non_whitespace_char, find_root, graphemes, indent,
2 comment, coords_at_pos, find_first_non_whitespace_char, find_root, graphemes, indent,
3+ line_ending::{
4+ get_line_ending, get_line_ending_of_str, line_end_char_index, str_is_line_ending,
5+ },
3 match_brackets,
6 match_brackets,
4 movement::{self, Direction},
7 movement::{self, Direction},
5 object, pos_at_coords,
8 object, pos_at_coords,
6 regex::{self, Regex},
9 regex::{self, Regex},
7 register::{self, Register, Registers},
10 register::{self, Register, Registers},
8− search, selection, Change, ChangeSet, Position, Range, Rope, RopeSlice, Selection, SmallVec,
11+ search, selection, Change, ChangeSet, LineEnding, Position, Range, Rope, RopeGraphemes,
9− Tendril, Transaction,
12+ RopeSlice, Selection, SmallVec, Tendril, Transaction, DEFAULT_LINE_ENDING,
10};
13};
11
14
12use helix_view::{
15use helix_view::{
@@ -303,9 +306,8 @@ fn move_line_end(cx: &mut Context) {
303 let text = doc.text();
306 let text = doc.text();
304 let line = text.char_to_line(range.head);
307 let line = text.char_to_line(range.head);
305
308
306− // Line end is pos at the start of next line - 1
309+ let pos = line_end_char_index(&text.slice(..), line);
307− // subtract another 1 because the line ends with \n
310+
308− let pos = text.line_to_char(line + 1).saturating_sub(2);
309 Range::new(pos, pos)
311 Range::new(pos, pos)
310 });
312 });
311
313
@@ -452,12 +454,28 @@ where
452 let count = cx.count();
454 let count = cx.count();
453
455
454 // need to wait for next key
456 // need to wait for next key
457+ // TODO: should this be done by grapheme rather than char? For example,
458+ // we can't properly handle the line-ending CRLF case here in terms of char.
455 cx.on_next_key(move |cx, event| {
459 cx.on_next_key(move |cx, event| {
456 let ch = match event {
460 let ch = match event {
457 KeyEvent {
461 KeyEvent {
458 code: KeyCode::Enter,
462 code: KeyCode::Enter,
459 ..
463 ..
460− } => '\n',
464+ } =>
465+ // TODO: this isn't quite correct when CRLF is involved.
466+ // This hack will work in most cases, since documents don't
467+ // usually mix line endings. But we should fix it eventually
468+ // anyway.
469+ {
470+ current!(cx.editor)
471+ .1
472+ .line_ending
473+ .as_str()
474+ .chars()
475+ .next()
476+ .unwrap()
477+ }
478+
461 KeyEvent {
479 KeyEvent {
462 code: KeyCode::Char(ch),
480 code: KeyCode::Char(ch),
463 ..
481 ..
@@ -575,32 +593,37 @@ fn extend_first_nonwhitespace(cx: &mut Context) {
575}
593}
576
594
577fn replace(cx: &mut Context) {
595fn replace(cx: &mut Context) {
596+ let mut buf = [0u8; 4]; // To hold utf8 encoded char.
597+
578 // need to wait for next key
598 // need to wait for next key
579 cx.on_next_key(move |cx, event| {
599 cx.on_next_key(move |cx, event| {
600+ let (view, doc) = current!(cx.editor);
580 let ch = match event {
601 let ch = match event {
581 KeyEvent {
602 KeyEvent {
582 code: KeyCode::Char(ch),
603 code: KeyCode::Char(ch),
583 ..
604 ..
584− } => Some(ch),
605+ } => Some(&ch.encode_utf8(&mut buf[..])[..]),
585 KeyEvent {
606 KeyEvent {
586 code: KeyCode::Enter,
607 code: KeyCode::Enter,
587 ..
608 ..
588− } => Some('\n'),
609+ } => Some(doc.line_ending.as_str()),
589 _ => None,
610 _ => None,
590 };
611 };
591
612
592 if let Some(ch) = ch {
613 if let Some(ch) = ch {
593− let (view, doc) = current!(cx.editor);
594−
595 let transaction =
614 let transaction =
596 Transaction::change_by_selection(doc.text(), doc.selection(view.id), |range| {
615 Transaction::change_by_selection(doc.text(), doc.selection(view.id), |range| {
597 let max_to = doc.text().len_chars().saturating_sub(1);
616 let max_to = doc.text().len_chars().saturating_sub(1);
598 let to = std::cmp::min(max_to, range.to() + 1);
617 let to = std::cmp::min(max_to, range.to() + 1);
599− let text: String = doc
618+ let text: String = RopeGraphemes::new(doc.text().slice(range.from()..to))
600− .text()
619+ .map(|g| {
601− .slice(range.from()..to)
620+ let cow: Cow<str> = g.into();
602− .chars()
621+ if str_is_line_ending(&cow) {
622+ cow
603− .map(|c| if c == '\n' { '\n' } else { ch })
623+ } else {
624+ ch.into()
625+ }
626+ })
604 .collect();
627 .collect();
605
628
606 (range.from(), to, Some(text.into()))
629 (range.from(), to, Some(text.into()))
@@ -725,9 +748,8 @@ fn extend_line_end(cx: &mut Context) {
725 let text = doc.text();
748 let text = doc.text();
726 let line = text.char_to_line(range.head);
749 let line = text.char_to_line(range.head);
727
750
728− // Line end is pos at the start of next line - 1
751+ let pos = line_end_char_index(&text.slice(..), line);
729− // subtract another 1 because the line ends with \n
752+
730− let pos = text.line_to_char(line + 1).saturating_sub(2);
731 Range::new(range.anchor, pos)
753 Range::new(range.anchor, pos)
732 });
754 });
733
755
@@ -783,7 +805,8 @@ fn split_selection_on_newline(cx: &mut Context) {
783 let text = doc.text().slice(..);
805 let text = doc.text().slice(..);
784 // only compile the regex once
806 // only compile the regex once
785 #[allow(clippy::trivial_regex)]
807 #[allow(clippy::trivial_regex)]
786− static REGEX: Lazy<Regex> = Lazy::new(|| Regex::new(r"\n").unwrap());
808+ static REGEX: Lazy<Regex> =
809+ Lazy::new(|| Regex::new(r"\r\n|[\n\r\u{000B}\u{000C}\u{0085}\u{2028}\u{2029}]").unwrap());
787 let selection = selection::split_on_matches(text, doc.selection(view.id), &REGEX);
810 let selection = selection::split_on_matches(text, doc.selection(view.id), &REGEX);
788 doc.set_selection(view.id, selection);
811 doc.set_selection(view.id, selection);
789}
812}
@@ -922,7 +945,13 @@ fn delete_selection_impl(reg: &mut Register, doc: &mut Document, view_id: ViewId
922 // then delete
945 // then delete
923 let transaction =
946 let transaction =
924 Transaction::change_by_selection(doc.text(), doc.selection(view_id), |range| {
947 Transaction::change_by_selection(doc.text(), doc.selection(view_id), |range| {
948+ let alltext = doc.text();
949+ let line = alltext.char_to_line(range.head);
925− let max_to = doc.text().len_chars().saturating_sub(1);
950+ let max_to = doc.text().len_chars().saturating_sub(
951+ get_line_ending(&alltext.line(line))
952+ .map(|le| le.len_chars())
953+ .unwrap_or(0),
954+ );
926 let to = std::cmp::min(max_to, range.to() + 1);
955 let to = std::cmp::min(max_to, range.to() + 1);
927 (range.from(), to, None)
956 (range.from(), to, None)
928 });
957 });
@@ -1003,7 +1032,7 @@ fn append_mode(cx: &mut Context) {
1003 if selection.iter().any(|range| range.head == end) {
1032 if selection.iter().any(|range| range.head == end) {
1004 let transaction = Transaction::change(
1033 let transaction = Transaction::change(
1005 doc.text(),
1034 doc.text(),
1006− std::array::IntoIter::new([(end, end, Some(Tendril::from_char('\n')))]),
1035+ std::array::IntoIter::new([(end, end, Some(doc.line_ending.as_str().into()))]),
1007 );
1036 );
1008 doc.apply(&transaction, view.id);
1037 doc.apply(&transaction, view.id);
1009 }
1038 }
@@ -1131,6 +1160,45 @@ mod cmd {
1131 }
1160 }
1132 }
1161 }
1133
1162
1163+ /// Sets or reports the current document's line ending setting.
1164+ fn set_line_ending(editor: &mut Editor, args: &[&str], event: PromptEvent) {
1165+ use LineEnding::*;
1166+
1167+ // If no argument, report current line ending setting.
1168+ if args.is_empty() {
1169+ let line_ending = current!(editor).1.line_ending;
1170+ editor.set_status(match line_ending {
1171+ Crlf => "crlf".into(),
1172+ LF => "line feed".into(),
1173+ FF => "form feed".into(),
1174+ CR => "carriage return".into(),
1175+ Nel => "next line".into(),
1176+
1177+ // These should never be a document's default line ending.
1178+ VT | LS | PS => "error".into(),
1179+ });
1180+ return;
1181+ }
1182+
1183+ // Attempt to parse argument as a line ending.
1184+ let line_ending = match args.get(0) {
1185+ // We check for CR first because it shares a common prefix with CRLF.
1186+ Some(arg) if "cr".starts_with(&arg.to_lowercase()) => Some(CR),
1187+ Some(arg) if "crlf".starts_with(&arg.to_lowercase()) => Some(Crlf),
1188+ Some(arg) if "lf".starts_with(&arg.to_lowercase()) => Some(LF),
1189+ Some(arg) if "ff".starts_with(&arg.to_lowercase()) => Some(FF),
1190+ Some(arg) if "nel".starts_with(&arg.to_lowercase()) => Some(Nel),
1191+ _ => None,
1192+ };
1193+
1194+ if let Some(le) = line_ending {
1195+ doc_mut!(editor).line_ending = le;
1196+ } else {
1197+ // Invalid argument.
1198+ editor.set_error(format!("invalid line ending '{}'", args[0],));
1199+ }
1200+ }
1201+
1134 fn earlier(editor: &mut Editor, args: &[&str], event: PromptEvent) {
1202 fn earlier(editor: &mut Editor, args: &[&str], event: PromptEvent) {
1135 let uk = match args.join(" ").parse::<helix_core::history::UndoKind>() {
1203 let uk = match args.join(" ").parse::<helix_core::history::UndoKind>() {
1136 Ok(uk) => uk,
1204 Ok(uk) => uk,
@@ -1274,7 +1342,11 @@ mod cmd {
1274 }
1342 }
1275
1343
1276 fn yank_joined_to_clipboard(editor: &mut Editor, args: &[&str], _: PromptEvent) {
1344 fn yank_joined_to_clipboard(editor: &mut Editor, args: &[&str], _: PromptEvent) {
1345+ let (_, doc) = current!(editor);
1277− let separator = args.first().copied().unwrap_or("\n");
1346+ let separator = args
1347+ .first()
1348+ .copied()
1349+ .unwrap_or_else(|| doc.line_ending.as_str());
1278 yank_joined_to_clipboard_impl(editor, separator);
1350 yank_joined_to_clipboard_impl(editor, separator);
1279 }
1351 }
1280
1352
@@ -1359,6 +1431,13 @@ mod cmd {
1359 fun: set_indent_style,
1431 fun: set_indent_style,
1360 completer: None,
1432 completer: None,
1361 },
1433 },
1434+ TypableCommand {
1435+ name: "line-ending",
1436+ alias: None,
1437+ doc: "Set the document's default line ending. Options: crlf, lf, cr, ff, nel.",
1438+ fun: set_line_ending,
1439+ completer: None,
1440+ },
1362 TypableCommand {
1441 TypableCommand {
1363 name: "earlier",
1442 name: "earlier",
1364 alias: Some("ear"),
1443 alias: Some("ear"),
@@ -1683,8 +1762,7 @@ fn append_to_line(cx: &mut Context) {
1683 let selection = doc.selection(view.id).transform(|range| {
1762 let selection = doc.selection(view.id).transform(|range| {
1684 let text = doc.text();
1763 let text = doc.text();
1685 let line = text.char_to_line(range.head);
1764 let line = text.char_to_line(range.head);
1686− // we can't use line_to_char(line + 1) - 2 because the last line might not contain \n
1765+ let pos = line_end_char_index(&text.slice(..), line);
1687− let pos = (text.line_to_char(line) + text.line(line).len_chars()).saturating_sub(1);
1688 Range::new(pos, pos)
1766 Range::new(pos, pos)
1689 });
1767 });
1690 doc.set_selection(view.id, selection);
1768 doc.set_selection(view.id, selection);
@@ -1731,7 +1809,7 @@ fn open(cx: &mut Context, open: Open) {
1731 let indent = doc.indent_unit().repeat(indent_level);
1809 let indent = doc.indent_unit().repeat(indent_level);
1732 let indent_len = indent.len();
1810 let indent_len = indent.len();
1733 let mut text = String::with_capacity(1 + indent_len);
1811 let mut text = String::with_capacity(1 + indent_len);
1734− text.push('\n');
1812+ text.push_str(doc.line_ending.as_str());
1735 text.push_str(&indent);
1813 text.push_str(&indent);
1736 let text = text.repeat(count);
1814 let text = text.repeat(count);
1737
1815
@@ -2344,7 +2422,7 @@ pub mod insert {
2344 );
2422 );
2345 let indent = doc.indent_unit().repeat(indent_level);
2423 let indent = doc.indent_unit().repeat(indent_level);
2346 let mut text = String::with_capacity(1 + indent.len());
2424 let mut text = String::with_capacity(1 + indent.len());
2347− text.push('\n');
2425+ text.push_str(doc.line_ending.as_str());
2348 text.push_str(&indent);
2426 text.push_str(&indent);
2349
2427
2350 let head = pos + offs + text.chars().count();
2428 let head = pos + offs + text.chars().count();
@@ -2365,7 +2443,7 @@ pub mod insert {
2365 if helix_core::auto_pairs::PAIRS.contains(&(prev, curr)) {
2443 if helix_core::auto_pairs::PAIRS.contains(&(prev, curr)) {
2366 // another newline, indent the end bracket one level less
2444 // another newline, indent the end bracket one level less
2367 let indent = doc.indent_unit().repeat(indent_level.saturating_sub(1));
2445 let indent = doc.indent_unit().repeat(indent_level.saturating_sub(1));
2368− text.push('\n');
2446+ text.push_str(doc.line_ending.as_str());
2369 text.push_str(&indent);
2447 text.push_str(&indent);
2370 }
2448 }
2371
2449
@@ -2488,7 +2566,8 @@ fn yank_joined_to_clipboard_impl(editor: &mut Editor, separator: &str) {
2488}
2566}
2489
2567
2490fn yank_joined_to_clipboard(cx: &mut Context) {
2568fn yank_joined_to_clipboard(cx: &mut Context) {
2569+ let line_ending = current!(cx.editor).1.line_ending;
2491− yank_joined_to_clipboard_impl(&mut cx.editor, "\n");
2570+ yank_joined_to_clipboard_impl(&mut cx.editor, line_ending.as_str());
2492}
2571}
2493
2572
2494fn yank_main_selection_to_clipboard_impl(editor: &mut Editor) {
2573fn yank_main_selection_to_clipboard_impl(editor: &mut Editor) {
@@ -2529,8 +2608,10 @@ fn paste_impl(
2529 .unwrap(),
2608 .unwrap(),
2530 );
2609 );
2531
2610
2532− // if any of values ends \n it's linewise paste
2611+ // if any of values ends with a line ending, it's linewise paste
2533− let linewise = values.iter().any(|value| value.ends_with('\n'));
2612+ let linewise = values
2613+ .iter()
2614+ .any(|value| get_line_ending_of_str(value).is_some());
2534
2615
2535 let mut values = values.iter().cloned().map(Tendril::from).chain(repeat);
2616 let mut values = values.iter().cloned().map(Tendril::from).chain(repeat);
2536
2617
helix-term/src/ui/editor.rs+2−2
@@ -9,7 +9,7 @@ use crate::{
9use helix_core::{
9use helix_core::{
10 coords_at_pos,
10 coords_at_pos,
11 syntax::{self, HighlightEvent},
11 syntax::{self, HighlightEvent},
12− Position, Range,
12+ LineEnding, Position, Range,
13};
13};
14use helix_lsp::LspProgressMap;
14use helix_lsp::LspProgressMap;
15use helix_view::{document::Mode, Document, Editor, Theme, View};
15use helix_view::{document::Mode, Document, Editor, Theme, View};
@@ -184,7 +184,7 @@ impl EditorView {
184
184
185 // iterate over range char by char
185 // iterate over range char by char
186 for grapheme in RopeGraphemes::new(text) {
186 for grapheme in RopeGraphemes::new(text) {
187− if grapheme == "\n" {
187+ if LineEnding::from_rope_slice(&grapheme).is_some() {
188 visual_x = 0;
188 visual_x = 0;
189 line += 1;
189 line += 1;
190
190
helix-term/src/ui/markdown.rs+2
@@ -115,6 +115,8 @@ fn parse<'a>(
115 // TODO: replace tabs with indentation
115 // TODO: replace tabs with indentation
116
116
117 let mut slice = &text[start..end];
117 let mut slice = &text[start..end];
118+ // TODO: do we need to handle all unicode line endings
119+ // here, or is just '\n' okay?
118 while let Some(end) = slice.find('\n') {
120 while let Some(end) = slice.find('\n') {
119 // emit span up to newline
121 // emit span up to newline
120 let text = &slice[..end];
122 let text = &slice[..end];
helix-tui/Cargo.toml+1
@@ -22,3 +22,4 @@ unicode-segmentation = "1.2"
22unicode-width = "0.1"
22unicode-width = "0.1"
23crossterm = { version = "0.20", optional = true }
23crossterm = { version = "0.20", optional = true }
24serde = { version = "1", "optional" = true, features = ["derive"]}
24serde = { version = "1", "optional" = true, features = ["derive"]}
25+helix-core = { version = "0.2", path = "../helix-core" }
helix-tui/src/text.rs+2−1
@@ -47,6 +47,7 @@
47//! ]);
47//! ]);
48//! ```
48//! ```
49use crate::style::Style;
49use crate::style::Style;
50+use helix_core::line_ending::str_is_line_ending;
50use std::borrow::Cow;
51use std::borrow::Cow;
51use unicode_segmentation::UnicodeSegmentation;
52use unicode_segmentation::UnicodeSegmentation;
52use unicode_width::UnicodeWidthStr;
53use unicode_width::UnicodeWidthStr;
@@ -177,7 +178,7 @@ impl<'a> Span<'a> {
177 symbol: g,
178 symbol: g,
178 style: base_style.patch(self.style),
179 style: base_style.patch(self.style),
179 })
180 })
180− .filter(|s| s.symbol != "\n")
181+ .filter(|s| !str_is_line_ending(s.symbol))
181 }
182 }
182}
183}
183
184
helix-tui/src/widgets/reflow.rs+5−4
@@ -1,4 +1,5 @@
1use crate::text::StyledGrapheme;
1use crate::text::StyledGrapheme;
2+use helix_core::line_ending::str_is_line_ending;
2use unicode_segmentation::UnicodeSegmentation;
3use unicode_segmentation::UnicodeSegmentation;
3use unicode_width::UnicodeWidthStr;
4use unicode_width::UnicodeWidthStr;
4
5
@@ -62,13 +63,13 @@ impl<'a, 'b> LineComposer<'a> for WordWrapper<'a, 'b> {
62 // Ignore characters wider that the total max width.
63 // Ignore characters wider that the total max width.
63 if symbol.width() as u16 > self.max_line_width
64 if symbol.width() as u16 > self.max_line_width
64 // Skip leading whitespace when trim is enabled.
65 // Skip leading whitespace when trim is enabled.
65− || self.trim && symbol_whitespace && symbol != "\n" && current_line_width == 0
66+ || self.trim && symbol_whitespace && !str_is_line_ending(symbol) && current_line_width == 0
66 {
67 {
67 continue;
68 continue;
68 }
69 }
69
70
70 // Break on newline and discard it.
71 // Break on newline and discard it.
71− if symbol == "\n" {
72+ if str_is_line_ending(symbol) {
72 if prev_whitespace {
73 if prev_whitespace {
73 current_line_width = width_to_last_word_end;
74 current_line_width = width_to_last_word_end;
74 self.current_line.truncate(symbols_to_last_word_end);
75 self.current_line.truncate(symbols_to_last_word_end);
@@ -170,7 +171,7 @@ impl<'a, 'b> LineComposer<'a> for LineTruncator<'a, 'b> {
170 }
171 }
171
172
172 // Break on newline and discard it.
173 // Break on newline and discard it.
173− if symbol == "\n" {
174+ if str_is_line_ending(symbol) {
174 break;
175 break;
175 }
176 }
176
177
@@ -199,7 +200,7 @@ impl<'a, 'b> LineComposer<'a> for LineTruncator<'a, 'b> {
199
200
200 if skip_rest {
201 if skip_rest {
201 for StyledGrapheme { symbol, .. } in &mut self.symbols {
202 for StyledGrapheme { symbol, .. } in &mut self.symbols {
202− if symbol == "\n" {
203+ if str_is_line_ending(symbol) {
203 break;
204 break;
204 }
205 }
205 }
206 }
helix-view/src/document.rs+22−12
@@ -7,10 +7,12 @@ use std::str::FromStr;
7use std::sync::Arc;
7use std::sync::Arc;
8
8
9use helix_core::{
9use helix_core::{
10− chars::{char_is_linebreak, char_is_whitespace},
10+ chars::{char_is_line_ending, char_is_whitespace},
11 history::History,
11 history::History,
12+ line_ending::auto_detect_line_ending,
12 syntax::{self, LanguageConfiguration},
13 syntax::{self, LanguageConfiguration},
13− ChangeSet, Diagnostic, Rope, Selection, State, Syntax, Transaction,
14+ ChangeSet, Diagnostic, LineEnding, Rope, Selection, State, Syntax, Transaction,
15+ DEFAULT_LINE_ENDING,
14};
16};
15
17
16use crate::{DocumentId, Theme, ViewId};
18use crate::{DocumentId, Theme, ViewId};
@@ -45,6 +47,9 @@ pub struct Document {
45 /// Current indent style.
47 /// Current indent style.
46 pub indent_style: IndentStyle,
48 pub indent_style: IndentStyle,
47
49
50+ /// The document's default line ending.
51+ pub line_ending: LineEnding,
52+
48 syntax: Option<Syntax>,
53 syntax: Option<Syntax>,
49 // /// Corresponding language scope name. Usually `source.<lang>`.
54 // /// Corresponding language scope name. Usually `source.<lang>`.
50 pub(crate) language: Option<Arc<LanguageConfiguration>>,
55 pub(crate) language: Option<Arc<LanguageConfiguration>>,
@@ -232,6 +237,7 @@ impl Document {
232 history: Cell::new(History::default()),
237 history: Cell::new(History::default()),
233 last_saved_revision: 0,
238 last_saved_revision: 0,
234 language_server: None,
239 language_server: None,
240+ line_ending: DEFAULT_LINE_ENDING,
235 }
241 }
236 }
242 }
237
243
@@ -243,22 +249,26 @@ impl Document {
243 ) -> Result<Self, Error> {
249 ) -> Result<Self, Error> {
244 use std::{fs::File, io::BufReader};
250 use std::{fs::File, io::BufReader};
245
251
246− let doc = if !path.exists() {
252+ let mut doc = if !path.exists() {
247− Rope::from("\n")
253+ Rope::from(DEFAULT_LINE_ENDING.as_str())
248 } else {
254 } else {
249 let file = File::open(&path).context(format!("unable to open {:?}", path))?;
255 let file = File::open(&path).context(format!("unable to open {:?}", path))?;
250− let mut doc = Rope::from_reader(BufReader::new(file))?;
256+ Rope::from_reader(BufReader::new(file))?
251− // add missing newline at the end of file
252− if doc.len_bytes() == 0 || doc.byte(doc.len_bytes() - 1) != b'\n' {
253− doc.insert_char(doc.len_chars(), '\n');
254− }
255− doc
256 };
257 };
257
258
259+ // search for line endings
260+ let line_ending = auto_detect_line_ending(&doc).unwrap_or(DEFAULT_LINE_ENDING);
261+
262+ // add missing newline at the end of file
263+ if doc.len_bytes() == 0 || char_is_line_ending(doc.char(doc.len_chars() - 1)) {
264+ doc.insert(doc.len_chars(), line_ending.as_str());
265+ }
266+
258 let mut doc = Self::new(doc);
267 let mut doc = Self::new(doc);
259 // set the path and try detecting the language
268 // set the path and try detecting the language
260 doc.set_path(&path)?;
269 doc.set_path(&path)?;
261 doc.detect_indent_style();
270 doc.detect_indent_style();
271+ doc.line_ending = line_ending;
262
272
263 if let Some(loader) = config_loader {
273 if let Some(loader) = config_loader {
264 doc.detect_language(theme, loader);
274 doc.detect_language(theme, loader);
@@ -366,7 +376,7 @@ impl Document {
366 Some(' ') => false,
376 Some(' ') => false,
367
377
368 // Ignore blank lines.
378 // Ignore blank lines.
369− Some(c) if char_is_linebreak(c) => continue,
379+ Some(c) if char_is_line_ending(c) => continue,
370
380
371 _ => {
381 _ => {
372 prev_line_is_tabs = false;
382 prev_line_is_tabs = false;
@@ -390,7 +400,7 @@ impl Document {
390 c if char_is_whitespace(c) => count_is_done = true,
400 c if char_is_whitespace(c) => count_is_done = true,
391
401
392 // Ignore blank lines.
402 // Ignore blank lines.
393− c if char_is_linebreak(c) => continue 'outer,
403+ c if char_is_line_ending(c) => continue 'outer,
394
404
395 _ => break,
405 _ => break,
396 }
406 }
helix-view/src/editor.rs+2−3
@@ -15,10 +15,9 @@ use slotmap::SlotMap;
15
15
16use anyhow::Error;
16use anyhow::Error;
17
17
18−use helix_core::Position;
19−
20pub use helix_core::diagnostic::Severity;
18pub use helix_core::diagnostic::Severity;
21pub use helix_core::register::Registers;
19pub use helix_core::register::Registers;
20+use helix_core::{Position, DEFAULT_LINE_ENDING};
22
21
23#[derive(Debug)]
22#[derive(Debug)]
24pub struct Editor {
23pub struct Editor {
@@ -173,7 +172,7 @@ impl Editor {
173
172
174 pub fn new_file(&mut self, action: Action) -> DocumentId {
173 pub fn new_file(&mut self, action: Action) -> DocumentId {
175 use helix_core::Rope;
174 use helix_core::Rope;
176− let doc = Document::new(Rope::from("\n"));
175+ let doc = Document::new(Rope::from(DEFAULT_LINE_ENDING.as_str()));
177 let id = self.documents.insert(doc);
176 let id = self.documents.insert(doc);
178 self.documents[id].id = id;
177 self.documents[id].id = id;
179 self.switch(id, action);
178 self.switch(id, action);