@@ -57,6 +57,7 @@ pub enum OffsetEncoding {
58pub mod util {
58pub mod util {
59 use super::*;
59 use super::*;
60+ use helix_core::line_ending::{line_end_byte_index, line_end_char_index};
60 use helix_core::{diagnostic::NumberOrString, Range, Rope, Selection, Tendril, Transaction};
61 use helix_core::{diagnostic::NumberOrString, Range, Rope, Selection, Tendril, Transaction};
62 /// Converts a diagnostic in the document to [`lsp::Diagnostic`].
63 /// Converts a diagnostic in the document to [`lsp::Diagnostic`].
@@ -117,7 +118,7 @@ pub mod util {
118 /// Converts [`lsp::Position`] to a position in the document.
119 /// Converts [`lsp::Position`] to a position in the document.
120− /// Returns `None` if position exceeds document length or an operation overflows.
121+ /// Returns `None` if position.line is out of bounds or an overflow occurs
121 pub fn lsp_pos_to_pos(
122 pub fn lsp_pos_to_pos(
122 doc: &Rope,
123 doc: &Rope,
123 pos: lsp::Position,
124 pos: lsp::Position,
@@ -128,22 +129,58 @@ pub mod util {
128 return None;
129 return None;
132+ // We need to be careful here to fully comply ith the LSP spec.
133+ // Two relevant quotes from the spec:
135+ // https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#position
136+ // > If the character value is greater than the line length it defaults back
137+ // > to the line length.
139+ // https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#textDocuments
140+ // > To ensure that both client and server split the string into the same
141+ // > line representation the protocol specifies the following end-of-line sequences:
142+ // > ‘\n’, ‘\r\n’ and ‘\r’. Positions are line end character agnostic.
143+ // > So you can not specify a position that denotes \r|\n or \n| where | represents the character offset.
145+ // This means that while the line must be in bounds the `charater`
146+ // must be capped to the end of the line.
147+ // Note that the end of the line here is **before** the line terminator
148+ // so we must use `line_end_char_index` istead of `doc.line_to_char(pos_line + 1)`
150+ // FIXME: Helix does not fully comply with the LSP spec for line terminators.
151+ // The LSP standard requires that line terminators are ['\n', '\r\n', '\r'].
152+ // Without the unicode-linebreak feature disabled, the `\r` terminator is not handled by helix.
153+ // With the unicode-linebreak feature, helix recognizes multiple extra line break chars
154+ // which means that positions will be decoded/encoded incorrectly in their presence
131− match offset_encoding {
156+ let line = match offset_encoding {
132 OffsetEncoding::Utf8 => {
157 OffsetEncoding::Utf8 => {
133− let line = doc.line_to_char(pos_line);
158+ let line_start = doc.line_to_byte(pos_line);
134− let pos = line.checked_add(pos.character as usize)?;
159+ let line_end = line_end_byte_index(&doc.slice(..), pos_line);
135− if pos <= doc.len_chars() {
160+ line_start..line_end
141 OffsetEncoding::Utf16 => {
162 OffsetEncoding::Utf16 => {
163+ // TODO directly translate line index to char-idx
164+ // ropey can do this just as easily as utf-8 byte translation
165+ // but the functions are just missing.
166+ // Translate to char first and then utf-16 as a workaround
142− let line = doc.line_to_char(pos_line);
167+ let line_start = doc.line_to_char(pos_line);
143− let line_start = doc.char_to_utf16_cu(line);
168+ let line_end = line_end_char_index(&doc.slice(..), pos_line);
144− let pos = line_start.checked_add(pos.character as usize)?;
169+ doc.char_to_utf16_cu(line_start)..doc.char_to_utf16_cu(line_end)
145− doc.try_utf16_cu_to_char(pos).ok()
173+ // The LSP spec demands that the offset is capped to the end of the line
176+ .checked_add(pos.character as usize)
177+ .unwrap_or(line.end)
180+ // TODO prefer UTF32/char indices to avoid this step
181+ match offset_encoding {
182+ OffsetEncoding::Utf8 => doc.try_byte_to_char(pos).ok(),
183+ OffsetEncoding::Utf16 => doc.try_utf16_cu_to_char(pos).ok(),
@@ -158,8 +195,8 @@ pub mod util {
158 match offset_encoding {
195 match offset_encoding {
159 OffsetEncoding::Utf8 => {
196 OffsetEncoding::Utf8 => {
160 let line = doc.char_to_line(pos);
197 let line = doc.char_to_line(pos);
161− let line_start = doc.line_to_char(line);
198+ let line_start = doc.line_to_byte(line);
162− let col = pos - line_start;
199+ let col = doc.char_to_byte(pos) - line_start;
164 lsp::Position::new(line as u32, col as u32)
201 lsp::Position::new(line as u32, col as u32)
@@ -606,16 +643,55 @@ mod tests {
608 test_case!("", (0, 0) => Some(0));
645 test_case!("", (0, 0) => Some(0));
609− test_case!("", (0, 1) => None);
646+ test_case!("", (0, 1) => Some(0));
610 test_case!("", (1, 0) => None);
647 test_case!("", (1, 0) => None);
611 test_case!("\n\n", (0, 0) => Some(0));
648 test_case!("\n\n", (0, 0) => Some(0));
612 test_case!("\n\n", (1, 0) => Some(1));
649 test_case!("\n\n", (1, 0) => Some(1));
613− test_case!("\n\n", (1, 1) => Some(2));
650+ test_case!("\n\n", (1, 1) => Some(1));
614 test_case!("\n\n", (2, 0) => Some(2));
651 test_case!("\n\n", (2, 0) => Some(2));
615 test_case!("\n\n", (3, 0) => None);
652 test_case!("\n\n", (3, 0) => None);
616 test_case!("test\n\n\n\ncase", (4, 3) => Some(11));
653 test_case!("test\n\n\n\ncase", (4, 3) => Some(11));
617 test_case!("test\n\n\n\ncase", (4, 4) => Some(12));
654 test_case!("test\n\n\n\ncase", (4, 4) => Some(12));
618− test_case!("test\n\n\n\ncase", (4, 5) => None);
655+ test_case!("test\n\n\n\ncase", (4, 5) => Some(12));
619 test_case!("", (u32::MAX, u32::MAX) => None);
656 test_case!("", (u32::MAX, u32::MAX) => None);
660+ fn emoji_format_gh_4791() {
661+ use lsp_types::{Position, Range, TextEdit};
675+ new_text: "\n ".to_string(),
688+ new_text: "\n ".to_string(),
692+ let mut source = Rope::from_str("[\n\"🇺🇸\",\n\"🎄\",\n]");
694+ let transaction = generate_transaction_from_edits(&source, edits, OffsetEncoding::Utf8);
695+ assert!(transaction.apply(&mut source));