on master help sign in
Fix LSP position encoding negotiation, UTF-16 incremental edits, formatting range, and code lens references
commit1e469f96758a65a4fbdc298d869e25790a98ec3c authoriko <mail@iko.soy> dateFri 25 Sep 2026, 16:30 (2 weeks ago) parent50119b1
3 files changed+141−50
editors/vscode/src/extension.ts+11−7 src/bin/htmlang_lsp/main.rs+76−17 src/bin/htmlang_lsp/state.rs+54−26
editors/vscode/src/extension.ts+11−7
@@ -1,7 +1,6 @@
11import {
22 commands,
33 ExtensionContext,
4− Location,
54 Position,
65 StatusBarAlignment,
76 StatusBarItem,
@@ -13,6 +12,7 @@ import {
1312import {
1413 LanguageClient,
1514 LanguageClientOptions,
15+ Location as ProtocolLocation,
1616 ServerOptions,
1717 State,
1818} from "vscode-languageclient/node";
@@ -51,19 +51,23 @@ async function startServer(context: ExtensionContext) {
5151 executeCommand: async (command, args, next) => {
5252 // The server emits `htmlang.showReferences` from code lenses; route
5353 // it through the built-in references viewer.
54+ // The server precomputes the reference locations, since a reference
55+ // query at the definition site would resolve the `@let` keyword.
5456 if (command === "htmlang.showReferences") {
55− const [uri, position] = args as [string, Position];
57+ const [uri, position, locations] = args as [
58+ string,
59+ Position,
60+ ProtocolLocation[] | undefined,
61+ ];
5662 const target = Uri.parse(uri);
57− const refs = await commands.executeCommand<Location[]>(
58− "vscode.executeReferenceProvider",
59− target,
60− new Position(position.line, position.character)
63+ const refs = (locations ?? []).map((l) =>
64+ client!.protocol2CodeConverter.asLocation(l)
6165 );
6266 await commands.executeCommand(
6367 "editor.action.showReferences",
6468 target,
6569 new Position(position.line, position.character),
66− refs ?? []
70+ refs
6771 );
6872 return;
6973 }
src/bin/htmlang_lsp/main.rs+76−17
@@ -1,5 +1,6 @@
11use std::collections::HashMap;
22use std::sync::Arc;
3+use std::sync::atomic::{AtomicBool, Ordering};
34use std::time::Duration;
45
56use serde_json::json;
@@ -36,6 +37,9 @@ struct Backend {
3637 /// Per-URI debounce handles. The value is a join handle for an in-flight
3738 /// diagnostic publish; replacing the entry cancels the prior handle.
3839 pending_diags: Arc<Mutex<HashMap<Url, JoinHandle<()>>>>,
40+ /// Whether the client accepted the `utf-8` position encoding. When false,
41+ /// positions are UTF-16 code units (the LSP default).
42+ utf8_positions: AtomicBool,
3943}
4044
4145const DIAG_DEBOUNCE: Duration = Duration::from_millis(150);
@@ -55,7 +59,11 @@ impl Backend {
5559 .write()
5660 .await
5761 .insert(uri.clone(), entry.clone());
62+ self.doc_updated(uri, entry).await;
63+ }
5864
65+ /// Follow-up work after a document entry has been stored.
66+ async fn doc_updated(&self, uri: Url, entry: Arc<DocumentEntry>) {
5967 // Refresh the workspace symbol index for this file synchronously —
6068 // symbol extraction is cheap (single pass over text) and keeps
6169 // workspace-wide queries consistent without waiting on the debounce.
@@ -152,9 +160,22 @@ impl LanguageServer for Backend {
152160 self.index.write().await.set_root(root);
153161 }
154162
163+ // Positions are byte offsets throughout this server, so prefer the
164+ // `utf-8` encoding — but only if the client offers it. Otherwise we
165+ // must stay on the UTF-16 default (VS Code refuses to start a server
166+ // that answers with an encoding it didn't offer).
167+ let client_supports_utf8 = params
168+ .capabilities
169+ .general
170+ .as_ref()
171+ .and_then(|g| g.position_encodings.as_ref())
172+ .is_some_and(|encs| encs.contains(&PositionEncodingKind::UTF8));
173+ self.utf8_positions
174+ .store(client_supports_utf8, Ordering::Relaxed);
175+
155176 Ok(InitializeResult {
156177 capabilities: ServerCapabilities {
157− position_encoding: Some(PositionEncodingKind::UTF8),
178+ position_encoding: client_supports_utf8.then_some(PositionEncodingKind::UTF8),
158179 text_document_sync: Some(TextDocumentSyncCapability::Kind(
159180 TextDocumentSyncKind::INCREMENTAL,
160181 )),
@@ -279,15 +300,22 @@ impl LanguageServer for Backend {
279300 let uri = params.text_document.uri.clone();
280301 let version = params.text_document.version;
281302
282− // Start from the current text and apply each change in order.
283− let mut text = match self.doc(&uri).await {
284− Some(d) => d.text.clone(),
285− None => String::new(),
303+ let utf8 = self.utf8_positions.load(Ordering::Relaxed);
304+
305+ // Hold the write lock across read-modify-write: tower-lsp may run
306+ // notification handlers concurrently, and two interleaved incremental
307+ // edits applied to the same base text would lose one of them.
308+ let entry = {
309+ let mut docs = self.documents.write().await;
310+ let mut text = docs.get(&uri).map(|d| d.text.clone()).unwrap_or_default();
311+ for change in &params.content_changes {
312+ apply_change(&mut text, change, utf8);
313+ }
314+ let entry = Arc::new(DocumentEntry::new(text, version));
315+ docs.insert(uri.clone(), entry.clone());
316+ entry
286317 };
287− for change in &params.content_changes {
288− apply_change(&mut text, change);
289− }
290− self.set_doc(uri, text, version).await;
318+ self.doc_updated(uri, entry).await;
291319 }
292320
293321 async fn did_close(&self, params: DidCloseTextDocumentParams) {
@@ -303,6 +331,16 @@ impl LanguageServer for Backend {
303331 {
304332 handle.abort();
305333 }
334+ // The index may hold symbols from unsaved buffer edits; resync it
335+ // with what's actually on disk.
336+ if let Ok(path) = params.text_document.uri.to_file_path() {
337+ let mut idx = self.index.write().await;
338+ if path.exists() {
339+ idx.update_from_disk(&path);
340+ } else {
341+ idx.remove(&path);
342+ }
343+ }
306344 self.client
307345 .publish_diagnostics(params.text_document.uri, vec![], None)
308346 .await;
@@ -613,8 +651,12 @@ impl LanguageServer for Backend {
613651 if formatted == doc.text {
614652 return Ok(None);
615653 }
616− let last_line = doc.text.lines().count().saturating_sub(1) as u32;
617− let last_col = doc.text.lines().last().map_or(0, |l| l.len()) as u32;
654+ // End the edit at the true end of the document. `str::lines` drops a
655+ // trailing newline, which would leave it outside the replaced range
656+ // and append an extra blank line after the (newline-terminated)
657+ // formatted text.
658+ let last_line = doc.text.matches('\n').count() as u32;
659+ let last_col = doc.text.rsplit('\n').next().map_or(0, |l| l.len()) as u32;
618660 Ok(Some(vec![TextEdit {
619661 range: Range::new(Position::new(0, 0), Position::new(last_line, last_col)),
620662 new_text: formatted,
@@ -738,6 +780,7 @@ impl LanguageServer for Backend {
738780 #[derive(Clone)]
739781 struct Def {
740782 line: u32,
783+ col: u32,
741784 name: String,
742785 kind: &'static str,
743786 }
@@ -746,6 +789,9 @@ impl LanguageServer for Backend {
746789 for (i, line) in lines.iter().enumerate() {
747790 let trimmed = line.trim_start();
748791 if let Some(rest) = trimmed.strip_prefix("@let ") {
792+ // `rest` is a suffix of `line`, so the name's column is
793+ // however much of the line precedes it.
794+ let col = (line.len() - rest.trim_start().len()) as u32;
749795 let rest = rest.trim();
750796 if let Some(name) = rest.split_whitespace().next() {
751797 let has_body = lines
@@ -758,18 +804,21 @@ impl LanguageServer for Backend {
758804 {
759805 defs.push(Def {
760806 line: i as u32,
807+ col,
761808 name: name.to_string(),
762809 kind: "fn",
763810 });
764811 } else if value_after_name.starts_with('[') {
765812 defs.push(Def {
766813 line: i as u32,
814+ col,
767815 name: name.to_string(),
768816 kind: "define",
769817 });
770818 } else {
771819 defs.push(Def {
772820 line: i as u32,
821+ col,
773822 name: name.to_string(),
774823 kind: "let",
775824 });
@@ -780,7 +829,7 @@ impl LanguageServer for Backend {
780829
781830 let mut lenses = Vec::with_capacity(defs.len());
782831 for def in &defs {
783− let mut count: usize = 0;
832+ let mut locations: Vec<Location> = Vec::new();
784833 let needle = match def.kind {
785834 "fn" => format!("@{}", def.name),
786835 _ => format!("${}", def.name),
@@ -798,27 +847,36 @@ impl LanguageServer for Backend {
798847 Some(c) => !(c.is_ascii_alphanumeric() || c == b'_' || c == b'-'),
799848 };
800849 if ok {
801− count += 1;
850+ locations.push(Location {
851+ uri: uri.clone(),
852+ range: Range::new(
853+ Position::new(i as u32, pos as u32),
854+ Position::new(i as u32, (pos + needle.len()) as u32),
855+ ),
856+ });
802857 }
803858 from = pos + needle.len();
804859 }
805860 }
806861
807− let title = if count == 1 {
862+ let title = if locations.len() == 1 {
808863 "1 reference".to_string()
809864 } else {
810− format!("{} references", count)
865+ format!("{} references", locations.len())
811866 };
812867 // A clickable lens that opens VS Code's references panel at this
813− // definition. The command name matches what VS Code's built-in
868+ // definition. The arguments mirror what VS Code's built-in
814869 // `editor.action.showReferences` accepts: (uri, position, locations).
870+ // The locations are sent precomputed because a reference query at
871+ // the definition site would resolve the `@let` keyword instead.
815872 let line_pos = Position::new(def.line, 0);
873+ let name_pos = Position::new(def.line, def.col);
816874 lenses.push(CodeLens {
817875 range: Range::new(line_pos, line_pos),
818876 command: Some(Command {
819877 title,
820878 command: "htmlang.showReferences".into(),
821− arguments: Some(vec![json!(uri), json!(line_pos)]),
879+ arguments: Some(vec![json!(uri), json!(name_pos), json!(locations)]),
822880 }),
823881 data: None,
824882 });
@@ -917,6 +975,7 @@ async fn main() {
917975 documents: Arc::new(RwLock::new(HashMap::new())),
918976 index: Arc::new(RwLock::new(WorkspaceIndex::new())),
919977 pending_diags: Arc::new(Mutex::new(HashMap::new())),
978+ utf8_positions: AtomicBool::new(false),
920979 });
921980 Server::new(stdin, stdout, socket).serve(service).await;
922981}
src/bin/htmlang_lsp/state.rs+54−26
@@ -43,11 +43,13 @@ impl DocumentEntry {
4343}
4444
4545/// Apply a single content change event to a string. Used when the LSP client
46−/// negotiates incremental sync.
47−pub fn apply_change(text: &mut String, change: &TextDocumentContentChangeEvent) {
46+/// negotiates incremental sync. `utf8` selects how `Position::character` is
47+/// interpreted: UTF-8 bytes when the client agreed to the `utf-8` position
48+/// encoding, UTF-16 code units (the LSP default) otherwise.
49+pub fn apply_change(text: &mut String, change: &TextDocumentContentChangeEvent, utf8: bool) {
4850 if let Some(range) = change.range {
49− let start = position_to_byte(text, range.start);
50− let end = position_to_byte(text, range.end);
51+ let start = position_to_byte(text, range.start, utf8);
52+ let end = position_to_byte(text, range.end, utf8).max(start);
5153 text.replace_range(start..end, &change.text);
5254 } else {
5355 text.clear();
@@ -57,29 +59,31 @@ pub fn apply_change(text: &mut String, change: &TextDocumentContentChangeEvent)
5759
5860/// Convert an LSP `Position` to a byte offset in `text`.
5961///
60−/// We treat `character` as a UTF-8 byte offset within the line. This matches
61−/// the convention used elsewhere in this crate (e.g. `(pos.character as usize).min(line.len())`),
62−/// which assumes UTF-8 positions. We negotiate the `utf-8` position encoding
63−/// during initialize, so this stays consistent for clients that respect it
64−/// (VS Code does).
65−fn position_to_byte(text: &str, pos: Position) -> usize {
66− let target_line = pos.line as usize;
62+/// Per the LSP spec, a `character` past the end of the line resolves to the
63+/// end of that line, and a line past the end of the document resolves to the
64+/// end of the document. The result always lies on a char boundary.
65+fn position_to_byte(text: &str, pos: Position, utf8: bool) -> usize {
6766 let mut line_start = 0;
68− let mut line = 0;
69− for (i, b) in text.bytes().enumerate() {
70− if line == target_line {
71− return (line_start + pos.character as usize).min(text.len());
72− }
73− if b == b'\n' {
74− line += 1;
75− line_start = i + 1;
67+ for _ in 0..pos.line {
68+ match text[line_start..].find('\n') {
69+ Some(nl) => line_start += nl + 1,
70+ None => return text.len(),
7671 }
7772 }
78− if line == target_line {
79− (line_start + pos.character as usize).min(text.len())
80− } else {
81− text.len()
73+ let line_end = text[line_start..]
74+ .find('\n')
75+ .map_or(text.len(), |nl| line_start + nl);
76+ let line = &text[line_start..line_end];
77+ let target = pos.character as usize;
78+
79+ let mut units = 0;
80+ for (byte_idx, ch) in line.char_indices() {
81+ if units >= target {
82+ return line_start + byte_idx;
83+ }
84+ units += if utf8 { ch.len_utf8() } else { ch.len_utf16() };
8285 }
86+ line_end
8387}
8488
8589// ---------------------------------------------------------------------------
@@ -220,6 +224,7 @@ mod tests {
220224 range_length: None,
221225 text: "replaced".into(),
222226 },
227+ true,
223228 );
224229 assert_eq!(s, "replaced");
225230 }
@@ -228,22 +233,45 @@ mod tests {
228233 fn apply_change_insert_in_middle() {
229234 let mut s = String::from("@row [width 200]\n@text hello");
230235 // Position 15 is just before the closing `]` on line 0.
231− apply_change(&mut s, &change((0, 15), (0, 15), ", padding 10"));
236+ apply_change(&mut s, &change((0, 15), (0, 15), ", padding 10"), true);
232237 assert_eq!(s, "@row [width 200, padding 10]\n@text hello");
233238 }
234239
235240 #[test]
236241 fn apply_change_delete_across_lines() {
237242 let mut s = String::from("line one\nline two\nline three");
238− apply_change(&mut s, &change((0, 4), (2, 4), ""));
243+ apply_change(&mut s, &change((0, 4), (2, 4), ""), true);
239244 assert_eq!(s, "line three");
240245 }
241246
242247 #[test]
243248 fn apply_change_replace_at_eof() {
244249 let mut s = String::from("abc");
245− apply_change(&mut s, &change((0, 3), (0, 3), "def"));
250+ apply_change(&mut s, &change((0, 3), (0, 3), "def"), true);
246251 assert_eq!(s, "abcdef");
247252 }
253+
254+ #[test]
255+ fn apply_change_utf16_positions() {
256+ // "é" is 2 UTF-8 bytes but 1 UTF-16 unit; "😀" is 4 bytes / 2 units.
257+ let mut s = String::from("é😀x\nnext");
258+ apply_change(&mut s, &change((0, 3), (0, 4), "y"), false);
259+ assert_eq!(s, "é😀y\nnext");
260+ }
261+
262+ #[test]
263+ fn apply_change_utf8_positions_with_multibyte() {
264+ let mut s = String::from("é😀x\nnext");
265+ apply_change(&mut s, &change((0, 6), (0, 7), "y"), true);
266+ assert_eq!(s, "é😀y\nnext");
267+ }
268+
269+ #[test]
270+ fn apply_change_clamps_character_to_line_end() {
271+ // A character past the end of line 0 must not spill into line 1.
272+ let mut s = String::from("ab\ncd");
273+ apply_change(&mut s, &change((0, 99), (0, 99), "!"), false);
274+ assert_eq!(s, "ab!\ncd");
275+ }
248276}
249277