1e469f9 · htmlang · git.iko.soy

git.iko.soy/htmlang

on master help sign in

Fix LSP position encoding negotiation, UTF-16 incremental edits, formatting range, and code lens references

1e469f9iko3 weeks ago

editors/vscode/src/extension.ts+11−7

compare with master

unifiedsplit

@@ -1,7 +1,6 @@

1import {

2 commands,

3 ExtensionContext,

4− Location,

5 Position,

4 Position,

6 StatusBarAlignment,

5 StatusBarAlignment,

7 StatusBarItem,

6 StatusBarItem,

@@ -13,6 +12,7 @@ import {

13import {

12import {

14 LanguageClient,

13 LanguageClient,

15 LanguageClientOptions,

14 LanguageClientOptions,

15+ Location as ProtocolLocation,

16 ServerOptions,

17 State,

18} from "vscode-languageclient/node";

@@ -51,19 +51,23 @@ async function startServer(context: ExtensionContext) {

51 executeCommand: async (command, args, next) => {

52 // The server emits `htmlang.showReferences` from code lenses; route

53 // it through the built-in references viewer.

54+ // The server precomputes the reference locations, since a reference

55+ // query at the definition site would resolve the `@let` keyword.

54 if (command === "htmlang.showReferences") {

56 if (command === "htmlang.showReferences") {

55− const [uri, position] = args as [string, Position];

57+ const [uri, position, locations] = args as [

58+ string,

59+ Position,

60+ ProtocolLocation[] | undefined,

61+ ];

56 const target = Uri.parse(uri);

62 const target = Uri.parse(uri);

57− const refs = await commands.executeCommand<Location[]>(

63+ const refs = (locations ?? []).map((l) =>

58− "vscode.executeReferenceProvider",

64+ client!.protocol2CodeConverter.asLocation(l)

59− target,

60− new Position(position.line, position.character)

61 );

65 );

62 await commands.executeCommand(

66 await commands.executeCommand(

63 "editor.action.showReferences",

67 "editor.action.showReferences",

64 target,

68 target,

65 new Position(position.line, position.character),

69 new Position(position.line, position.character),

66− refs ?? []

70+ refs

67 );

71 );

68 return;

72 return;

69 }

73 }

src/bin/htmlang_lsp/main.rs+76−17

@@ -1,5 +1,6 @@

1use std::collections::HashMap;

2use std::sync::Arc;

3+use std::sync::atomic::{AtomicBool, Ordering};

3use std::time::Duration;

4use std::time::Duration;

5use serde_json::json;

6use serde_json::json;

@@ -36,6 +37,9 @@ struct Backend {

36 /// Per-URI debounce handles. The value is a join handle for an in-flight

37 /// Per-URI debounce handles. The value is a join handle for an in-flight

37 /// diagnostic publish; replacing the entry cancels the prior handle.

38 /// diagnostic publish; replacing the entry cancels the prior handle.

38 pending_diags: Arc<Mutex<HashMap<Url, JoinHandle<()>>>>,

39 pending_diags: Arc<Mutex<HashMap<Url, JoinHandle<()>>>>,

40+ /// Whether the client accepted the `utf-8` position encoding. When false,

41+ /// positions are UTF-16 code units (the LSP default).

42+ utf8_positions: AtomicBool,

39}

43}

41const DIAG_DEBOUNCE: Duration = Duration::from_millis(150);

45const DIAG_DEBOUNCE: Duration = Duration::from_millis(150);

@@ -55,7 +59,11 @@ impl Backend {

55 .write()

59 .write()

56 .await

60 .await

57 .insert(uri.clone(), entry.clone());

61 .insert(uri.clone(), entry.clone());

62+ self.doc_updated(uri, entry).await;

63+ }

65+ /// Follow-up work after a document entry has been stored.

66+ async fn doc_updated(&self, uri: Url, entry: Arc<DocumentEntry>) {

59 // Refresh the workspace symbol index for this file synchronously —

67 // Refresh the workspace symbol index for this file synchronously —

60 // symbol extraction is cheap (single pass over text) and keeps

68 // symbol extraction is cheap (single pass over text) and keeps

61 // workspace-wide queries consistent without waiting on the debounce.

69 // workspace-wide queries consistent without waiting on the debounce.

@@ -152,9 +160,22 @@ impl LanguageServer for Backend {

152 self.index.write().await.set_root(root);

160 self.index.write().await.set_root(root);

153 }

161 }

154

162

163+ // Positions are byte offsets throughout this server, so prefer the

164+ // `utf-8` encoding — but only if the client offers it. Otherwise we

165+ // must stay on the UTF-16 default (VS Code refuses to start a server

166+ // that answers with an encoding it didn't offer).

167+ let client_supports_utf8 = params

168+ .capabilities

169+ .general

170+ .as_ref()

171+ .and_then(|g| g.position_encodings.as_ref())

172+ .is_some_and(|encs| encs.contains(&PositionEncodingKind::UTF8));

173+ self.utf8_positions

174+ .store(client_supports_utf8, Ordering::Relaxed);

175+

155 Ok(InitializeResult {

176 Ok(InitializeResult {

156 capabilities: ServerCapabilities {

177 capabilities: ServerCapabilities {

157− position_encoding: Some(PositionEncodingKind::UTF8),

178+ position_encoding: client_supports_utf8.then_some(PositionEncodingKind::UTF8),

158 text_document_sync: Some(TextDocumentSyncCapability::Kind(

179 text_document_sync: Some(TextDocumentSyncCapability::Kind(

159 TextDocumentSyncKind::INCREMENTAL,

180 TextDocumentSyncKind::INCREMENTAL,

160 )),

181 )),

@@ -279,15 +300,22 @@ impl LanguageServer for Backend {

279 let uri = params.text_document.uri.clone();

300 let uri = params.text_document.uri.clone();

280 let version = params.text_document.version;

301 let version = params.text_document.version;

281

302

282− // Start from the current text and apply each change in order.

303+ let utf8 = self.utf8_positions.load(Ordering::Relaxed);

283− let mut text = match self.doc(&uri).await {

304+

284− Some(d) => d.text.clone(),

305+ // Hold the write lock across read-modify-write: tower-lsp may run

285− None => String::new(),

306+ // notification handlers concurrently, and two interleaved incremental

307+ // edits applied to the same base text would lose one of them.

308+ let entry = {

309+ let mut docs = self.documents.write().await;

310+ let mut text = docs.get(&uri).map(|d| d.text.clone()).unwrap_or_default();

311+ for change in &params.content_changes {

312+ apply_change(&mut text, change, utf8);

313+ }

314+ let entry = Arc::new(DocumentEntry::new(text, version));

315+ docs.insert(uri.clone(), entry.clone());

316+ entry

286 };

317 };

287− for change in &params.content_changes {

288− apply_change(&mut text, change);

289− }

290− self.set_doc(uri, text, version).await;

318+ self.doc_updated(uri, entry).await;

291 }

319 }

292

320

293 async fn did_close(&self, params: DidCloseTextDocumentParams) {

321 async fn did_close(&self, params: DidCloseTextDocumentParams) {

@@ -303,6 +331,16 @@ impl LanguageServer for Backend {

303 {

331 {

304 handle.abort();

332 handle.abort();

305 }

333 }

334+ // The index may hold symbols from unsaved buffer edits; resync it

335+ // with what's actually on disk.

336+ if let Ok(path) = params.text_document.uri.to_file_path() {

337+ let mut idx = self.index.write().await;

338+ if path.exists() {

339+ idx.update_from_disk(&path);

340+ } else {

341+ idx.remove(&path);

342+ }

343+ }

306 self.client

344 self.client

307 .publish_diagnostics(params.text_document.uri, vec![], None)

345 .publish_diagnostics(params.text_document.uri, vec![], None)

308 .await;

346 .await;

@@ -613,8 +651,12 @@ impl LanguageServer for Backend {

613 if formatted == doc.text {

651 if formatted == doc.text {

614 return Ok(None);

652 return Ok(None);

615 }

653 }

654+ // End the edit at the true end of the document. `str::lines` drops a

655+ // trailing newline, which would leave it outside the replaced range

656+ // and append an extra blank line after the (newline-terminated)

657+ // formatted text.

616− let last_line = doc.text.lines().count().saturating_sub(1) as u32;

658+ let last_line = doc.text.matches('\n').count() as u32;

617− let last_col = doc.text.lines().last().map_or(0, |l| l.len()) as u32;

659+ let last_col = doc.text.rsplit('\n').next().map_or(0, |l| l.len()) as u32;

618 Ok(Some(vec![TextEdit {

660 Ok(Some(vec![TextEdit {

619 range: Range::new(Position::new(0, 0), Position::new(last_line, last_col)),

661 range: Range::new(Position::new(0, 0), Position::new(last_line, last_col)),

620 new_text: formatted,

662 new_text: formatted,

@@ -738,6 +780,7 @@ impl LanguageServer for Backend {

738 #[derive(Clone)]

780 #[derive(Clone)]

739 struct Def {

781 struct Def {

740 line: u32,

782 line: u32,

783+ col: u32,

741 name: String,

784 name: String,

742 kind: &'static str,

785 kind: &'static str,

743 }

786 }

@@ -746,6 +789,9 @@ impl LanguageServer for Backend {

746 for (i, line) in lines.iter().enumerate() {

789 for (i, line) in lines.iter().enumerate() {

747 let trimmed = line.trim_start();

790 let trimmed = line.trim_start();

748 if let Some(rest) = trimmed.strip_prefix("@let ") {

791 if let Some(rest) = trimmed.strip_prefix("@let ") {

792+ // `rest` is a suffix of `line`, so the name's column is

793+ // however much of the line precedes it.

794+ let col = (line.len() - rest.trim_start().len()) as u32;

749 let rest = rest.trim();

795 let rest = rest.trim();

750 if let Some(name) = rest.split_whitespace().next() {

796 if let Some(name) = rest.split_whitespace().next() {

751 let has_body = lines

797 let has_body = lines

@@ -758,18 +804,21 @@ impl LanguageServer for Backend {

758 {

804 {

759 defs.push(Def {

805 defs.push(Def {

760 line: i as u32,

806 line: i as u32,

807+ col,

761 name: name.to_string(),

808 name: name.to_string(),

762 kind: "fn",

809 kind: "fn",

763 });

810 });

764 } else if value_after_name.starts_with('[') {

811 } else if value_after_name.starts_with('[') {

765 defs.push(Def {

812 defs.push(Def {

766 line: i as u32,

813 line: i as u32,

814+ col,

767 name: name.to_string(),

815 name: name.to_string(),

768 kind: "define",

816 kind: "define",

769 });

817 });

770 } else {

818 } else {

771 defs.push(Def {

819 defs.push(Def {

772 line: i as u32,

820 line: i as u32,

821+ col,

773 name: name.to_string(),

822 name: name.to_string(),

774 kind: "let",

823 kind: "let",

775 });

824 });

@@ -780,7 +829,7 @@ impl LanguageServer for Backend {

780

829

781 let mut lenses = Vec::with_capacity(defs.len());

830 let mut lenses = Vec::with_capacity(defs.len());

782 for def in &defs {

831 for def in &defs {

783− let mut count: usize = 0;

832+ let mut locations: Vec<Location> = Vec::new();

784 let needle = match def.kind {

833 let needle = match def.kind {

785 "fn" => format!("@{}", def.name),

834 "fn" => format!("@{}", def.name),

786 _ => format!("${}", def.name),

835 _ => format!("${}", def.name),

@@ -798,27 +847,36 @@ impl LanguageServer for Backend {

798 Some(c) => !(c.is_ascii_alphanumeric() || c == b'_' || c == b'-'),

847 Some(c) => !(c.is_ascii_alphanumeric() || c == b'_' || c == b'-'),

799 };

848 };

800 if ok {

849 if ok {

801− count += 1;

850+ locations.push(Location {

851+ uri: uri.clone(),

852+ range: Range::new(

853+ Position::new(i as u32, pos as u32),

854+ Position::new(i as u32, (pos + needle.len()) as u32),

855+ ),

856+ });

802 }

857 }

803 from = pos + needle.len();

858 from = pos + needle.len();

804 }

859 }

805 }

860 }

806

861

807− let title = if count == 1 {

862+ let title = if locations.len() == 1 {

808 "1 reference".to_string()

863 "1 reference".to_string()

809 } else {

864 } else {

810− format!("{} references", count)

865+ format!("{} references", locations.len())

811 };

866 };

812 // A clickable lens that opens VS Code's references panel at this

867 // A clickable lens that opens VS Code's references panel at this

813− // definition. The command name matches what VS Code's built-in

868+ // definition. The arguments mirror what VS Code's built-in

814 // `editor.action.showReferences` accepts: (uri, position, locations).

869 // `editor.action.showReferences` accepts: (uri, position, locations).

870+ // The locations are sent precomputed because a reference query at

871+ // the definition site would resolve the `@let` keyword instead.

815 let line_pos = Position::new(def.line, 0);

872 let line_pos = Position::new(def.line, 0);

873+ let name_pos = Position::new(def.line, def.col);

816 lenses.push(CodeLens {

874 lenses.push(CodeLens {

817 range: Range::new(line_pos, line_pos),

875 range: Range::new(line_pos, line_pos),

818 command: Some(Command {

876 command: Some(Command {

819 title,

877 title,

820 command: "htmlang.showReferences".into(),

878 command: "htmlang.showReferences".into(),

821− arguments: Some(vec![json!(uri), json!(line_pos)]),

879+ arguments: Some(vec![json!(uri), json!(name_pos), json!(locations)]),

822 }),

880 }),

823 data: None,

881 data: None,

824 });

882 });

@@ -917,6 +975,7 @@ async fn main() {

917 documents: Arc::new(RwLock::new(HashMap::new())),

975 documents: Arc::new(RwLock::new(HashMap::new())),

918 index: Arc::new(RwLock::new(WorkspaceIndex::new())),

976 index: Arc::new(RwLock::new(WorkspaceIndex::new())),

919 pending_diags: Arc::new(Mutex::new(HashMap::new())),

977 pending_diags: Arc::new(Mutex::new(HashMap::new())),

978+ utf8_positions: AtomicBool::new(false),

920 });

979 });

921 Server::new(stdin, stdout, socket).serve(service).await;

980 Server::new(stdin, stdout, socket).serve(service).await;

922}

981}

src/bin/htmlang_lsp/state.rs+54−26

@@ -43,11 +43,13 @@ impl DocumentEntry {

43}

45/// Apply a single content change event to a string. Used when the LSP client

46−/// negotiates incremental sync.

46+/// negotiates incremental sync. `utf8` selects how `Position::character` is

47+/// interpreted: UTF-8 bytes when the client agreed to the `utf-8` position

48+/// encoding, UTF-16 code units (the LSP default) otherwise.

47−pub fn apply_change(text: &mut String, change: &TextDocumentContentChangeEvent) {

49+pub fn apply_change(text: &mut String, change: &TextDocumentContentChangeEvent, utf8: bool) {

48 if let Some(range) = change.range {

50 if let Some(range) = change.range {

49− let start = position_to_byte(text, range.start);

51+ let start = position_to_byte(text, range.start, utf8);

50− let end = position_to_byte(text, range.end);

52+ let end = position_to_byte(text, range.end, utf8).max(start);

51 text.replace_range(start..end, &change.text);

53 text.replace_range(start..end, &change.text);

52 } else {

54 } else {

53 text.clear();

55 text.clear();

@@ -57,29 +59,31 @@ pub fn apply_change(text: &mut String, change: &TextDocumentContentChangeEvent)

58/// Convert an LSP `Position` to a byte offset in `text`.

60/// Convert an LSP `Position` to a byte offset in `text`.

59///

61///

60−/// We treat `character` as a UTF-8 byte offset within the line. This matches

62+/// Per the LSP spec, a `character` past the end of the line resolves to the

61−/// the convention used elsewhere in this crate (e.g. `(pos.character as usize).min(line.len())`),

63+/// end of that line, and a line past the end of the document resolves to the

62−/// which assumes UTF-8 positions. We negotiate the `utf-8` position encoding

64+/// end of the document. The result always lies on a char boundary.

63−/// during initialize, so this stays consistent for clients that respect it

64−/// (VS Code does).

65−fn position_to_byte(text: &str, pos: Position) -> usize {

65+fn position_to_byte(text: &str, pos: Position, utf8: bool) -> usize {

66− let target_line = pos.line as usize;

67 let mut line_start = 0;

66 let mut line_start = 0;

68− let mut line = 0;

67+ for _ in 0..pos.line {

69− for (i, b) in text.bytes().enumerate() {

68+ match text[line_start..].find('\n') {

70− if line == target_line {

69+ Some(nl) => line_start += nl + 1,

71− return (line_start + pos.character as usize).min(text.len());

70+ None => return text.len(),

72− }

73− if b == b'\n' {

74− line += 1;

75− line_start = i + 1;

76 }

71 }

77 }

72 }

78− if line == target_line {

73+ let line_end = text[line_start..]

74+ .find('\n')

75+ .map_or(text.len(), |nl| line_start + nl);

76+ let line = &text[line_start..line_end];

79− (line_start + pos.character as usize).min(text.len())

77+ let target = pos.character as usize;

80− } else {

78+

81− text.len()

79+ let mut units = 0;

80+ for (byte_idx, ch) in line.char_indices() {

81+ if units >= target {

82+ return line_start + byte_idx;

83+ }

84+ units += if utf8 { ch.len_utf8() } else { ch.len_utf16() };

82 }

85 }

86+ line_end

83}

87}

85// ---------------------------------------------------------------------------

89// ---------------------------------------------------------------------------

@@ -220,6 +224,7 @@ mod tests {

220 range_length: None,

224 range_length: None,

221 text: "replaced".into(),

225 text: "replaced".into(),

222 },

226 },

227+ true,

223 );

228 );

224 assert_eq!(s, "replaced");

229 assert_eq!(s, "replaced");

225 }

230 }

@@ -228,22 +233,45 @@ mod tests {

228 fn apply_change_insert_in_middle() {

233 fn apply_change_insert_in_middle() {

229 let mut s = String::from("@row [width 200]\n@text hello");

234 let mut s = String::from("@row [width 200]\n@text hello");

230 // Position 15 is just before the closing `]` on line 0.

235 // Position 15 is just before the closing `]` on line 0.

231− apply_change(&mut s, &change((0, 15), (0, 15), ", padding 10"));

236+ apply_change(&mut s, &change((0, 15), (0, 15), ", padding 10"), true);

232 assert_eq!(s, "@row [width 200, padding 10]\n@text hello");

237 assert_eq!(s, "@row [width 200, padding 10]\n@text hello");

233 }

238 }

234

239

235 #[test]

240 #[test]

236 fn apply_change_delete_across_lines() {

241 fn apply_change_delete_across_lines() {

237 let mut s = String::from("line one\nline two\nline three");

242 let mut s = String::from("line one\nline two\nline three");

238− apply_change(&mut s, &change((0, 4), (2, 4), ""));

243+ apply_change(&mut s, &change((0, 4), (2, 4), ""), true);

239 assert_eq!(s, "line three");

244 assert_eq!(s, "line three");

240 }

245 }

241

246

242 #[test]

247 #[test]

243 fn apply_change_replace_at_eof() {

248 fn apply_change_replace_at_eof() {

244 let mut s = String::from("abc");

249 let mut s = String::from("abc");

245− apply_change(&mut s, &change((0, 3), (0, 3), "def"));

250+ apply_change(&mut s, &change((0, 3), (0, 3), "def"), true);

246 assert_eq!(s, "abcdef");

251 assert_eq!(s, "abcdef");

247 }

252 }

253+

254+ #[test]

255+ fn apply_change_utf16_positions() {

256+ // "é" is 2 UTF-8 bytes but 1 UTF-16 unit; "😀" is 4 bytes / 2 units.

257+ let mut s = String::from("é😀x\nnext");

258+ apply_change(&mut s, &change((0, 3), (0, 4), "y"), false);

259+ assert_eq!(s, "é😀y\nnext");

260+ }

261+

262+ #[test]

263+ fn apply_change_utf8_positions_with_multibyte() {

264+ let mut s = String::from("é😀x\nnext");

265+ apply_change(&mut s, &change((0, 6), (0, 7), "y"), true);

266+ assert_eq!(s, "é😀y\nnext");

267+ }

268+

269+ #[test]

270+ fn apply_change_clamps_character_to_line_end() {

271+ // A character past the end of line 0 must not spill into line 1.

272+ let mut s = String::from("ab\ncd");

273+ apply_change(&mut s, &change((0, 99), (0, 99), "!"), false);

274+ assert_eq!(s, "ab!\ncd");

275+ }

248}

276}

249

277