fae0f2e · crush · git.iko.soy

git.iko.soy/crush

on main help sign in

fix(lsp/edit): properly handle non-ascii chars (e.g. CJK) (#2325)

fae0f2eChristian Rocha7 months ago

go.mod+1−1

compare with main

unifiedsplit

@@ -31,7 +31,7 @@ require (

31 github.com/charmbracelet/x/exp/ordered v0.1.0

32 github.com/charmbracelet/x/exp/slice v0.0.0-20260209194814-eeb2896ac759

33 github.com/charmbracelet/x/exp/strings v0.1.0

34− github.com/charmbracelet/x/powernap v0.1.2

34+ github.com/charmbracelet/x/powernap v0.1.3

35 github.com/charmbracelet/x/term v0.2.2

36 github.com/clipperhouse/displaywidth v0.11.0

37 github.com/clipperhouse/uax29/v2 v2.7.0

go.sum+2−2

@@ -120,8 +120,8 @@ github.com/charmbracelet/x/exp/strings v0.1.0 h1:i69S2XI7uG1u4NLGeJPSYU++Nmjvpo9

120github.com/charmbracelet/x/exp/strings v0.1.0/go.mod h1:/ehtMPNh9K4odGFkqYJKpIYyePhdp1hLBRvyY4bWkH8=

121github.com/charmbracelet/x/json v0.2.0 h1:DqB+ZGx2h+Z+1s98HOuOyli+i97wsFQIxP2ZQANTPrQ=

122github.com/charmbracelet/x/json v0.2.0/go.mod h1:opFIflx2YgXgi49xVUu8gEQ21teFAxyMwvOiZhIvWNM=

123−github.com/charmbracelet/x/powernap v0.1.2 h1:5i0zWrqE5c9PY+H90DXg/5+jHNZ5FgXOyOU4dzOSIxs=

123+github.com/charmbracelet/x/powernap v0.1.3 h1:rmxdQelSPB1QAgLRNMLOrgCTq3q2RXoLOJ2ZTwKG17g=

124−github.com/charmbracelet/x/powernap v0.1.2/go.mod h1:cmdl5zlP5mR8TF2Y68UKc7hdGUDiSJ2+4hk0h04Hsx4=

124+github.com/charmbracelet/x/powernap v0.1.3/go.mod h1:cmdl5zlP5mR8TF2Y68UKc7hdGUDiSJ2+4hk0h04Hsx4=

125github.com/charmbracelet/x/term v0.2.2 h1:xVRT/S2ZcKdhhOuSP4t5cLi5o+JxklsoEObBSgfgZRk=

126github.com/charmbracelet/x/term v0.2.2/go.mod h1:kF8CY5RddLWrsgVwpw4kAa6TESp6EB5y3uxGLeCqzAI=

127github.com/charmbracelet/x/termios v0.1.1 h1:o3Q2bT8eqzGnGPOYheoYS8eEleT5ZVNYNy8JawjaNZY=

internal/lsp/client.go+1−1

@@ -194,7 +194,7 @@ func (c *Client) createPowernapClient() error {

194

195// registerHandlers registers the standard LSP notification and request handlers.

196func (c *Client) registerHandlers() {

197− c.RegisterServerRequestHandler("workspace/applyEdit", HandleApplyEdit)

197+ c.RegisterServerRequestHandler("workspace/applyEdit", HandleApplyEdit(c.client.GetOffsetEncoding()))

198 c.RegisterServerRequestHandler("workspace/configuration", HandleWorkspaceConfiguration)

199 c.RegisterServerRequestHandler("client/registerCapability", HandleRegisterCapability)

200 c.RegisterNotificationHandler("window/showMessage", func(ctx context.Context, method string, params json.RawMessage) {

internal/lsp/handlers.go+14−11

@@ -6,6 +6,7 @@ import (

6 "log/slog"

8 "github.com/charmbracelet/crush/internal/lsp/util"

9+ powernap "github.com/charmbracelet/x/powernap/pkg/lsp"

9 "github.com/charmbracelet/x/powernap/pkg/lsp/protocol"

10 "github.com/charmbracelet/x/powernap/pkg/lsp/protocol"

10)

11)

@@ -44,19 +45,21 @@ func HandleRegisterCapability(_ context.Context, _ string, params json.RawMessag

44}

45}

46// HandleApplyEdit handles workspace edit requests

47// HandleApplyEdit handles workspace edit requests

47−func HandleApplyEdit(_ context.Context, _ string, params json.RawMessage) (any, error) {

48+func HandleApplyEdit(encoding powernap.OffsetEncoding) func(_ context.Context, _ string, params json.RawMessage) (any, error) {

49+ return func(_ context.Context, _ string, params json.RawMessage) (any, error) {

48− var edit protocol.ApplyWorkspaceEditParams

50+ var edit protocol.ApplyWorkspaceEditParams

49− if err := json.Unmarshal(params, &edit); err != nil {

51+ if err := json.Unmarshal(params, &edit); err != nil {

50− return nil, err

52+ return nil, err

51− }

53+ }

53− err := util.ApplyWorkspaceEdit(edit.Edit)

55+ err := util.ApplyWorkspaceEdit(edit.Edit, encoding)

54− if err != nil {

56+ if err != nil {

55− slog.Error("Error applying workspace edit", "error", err)

57+ slog.Error("Error applying workspace edit", "error", err)

56− return protocol.ApplyWorkspaceEditResult{Applied: false, FailureReason: err.Error()}, nil

58+ return protocol.ApplyWorkspaceEditResult{Applied: false, FailureReason: err.Error()}, nil

57− }

59+ }

59− return protocol.ApplyWorkspaceEditResult{Applied: true}, nil

61+ return protocol.ApplyWorkspaceEditResult{Applied: true}, nil

62+ }

60}

63}

62// FileWatchRegistrationHandler is a function that will be called when file watch registrations are received

65// FileWatchRegistrationHandler is a function that will be called when file watch registrations are received

internal/lsp/util/edit.go+49−12

@@ -7,10 +7,11 @@ import (

7 "sort"

8 "strings"

10+ powernap "github.com/charmbracelet/x/powernap/pkg/lsp"

10 "github.com/charmbracelet/x/powernap/pkg/lsp/protocol"

11 "github.com/charmbracelet/x/powernap/pkg/lsp/protocol"

11)

12)

13−func applyTextEdits(uri protocol.DocumentURI, edits []protocol.TextEdit) error {

14+func applyTextEdits(uri protocol.DocumentURI, edits []protocol.TextEdit, encoding powernap.OffsetEncoding) error {

14 path, err := uri.Path()

15 path, err := uri.Path()

15 if err != nil {

16 if err != nil {

16 return fmt.Errorf("invalid URI: %w", err)

17 return fmt.Errorf("invalid URI: %w", err)

@@ -57,7 +58,7 @@ func applyTextEdits(uri protocol.DocumentURI, edits []protocol.TextEdit) error {

58 // Apply each edit

59 // Apply each edit

59 for _, edit := range sortedEdits {

60 for _, edit := range sortedEdits {

60− newLines, err := applyTextEdit(lines, edit)

61+ newLines, err := applyTextEdit(lines, edit, encoding)

61 if err != nil {

62 if err != nil {

62 return fmt.Errorf("failed to apply edit: %w", err)

63 return fmt.Errorf("failed to apply edit: %w", err)

63 }

64 }

@@ -85,13 +86,11 @@ func applyTextEdits(uri protocol.DocumentURI, edits []protocol.TextEdit) error {

85 return nil

86 return nil

86}

87}

88−func applyTextEdit(lines []string, edit protocol.TextEdit) ([]string, error) {

89+func applyTextEdit(lines []string, edit protocol.TextEdit, encoding powernap.OffsetEncoding) ([]string, error) {

89 startLine := int(edit.Range.Start.Line)

90 startLine := int(edit.Range.Start.Line)

90 endLine := int(edit.Range.End.Line)

91 endLine := int(edit.Range.End.Line)

91− startChar := int(edit.Range.Start.Character)

92− endChar := int(edit.Range.End.Character)

94− // Validate positions

93+ // Validate positions before accessing lines.

95 if startLine < 0 || startLine >= len(lines) {

94 if startLine < 0 || startLine >= len(lines) {

96 return nil, fmt.Errorf("invalid start line: %d", startLine)

95 return nil, fmt.Errorf("invalid start line: %d", startLine)

97 }

96 }

@@ -99,6 +98,26 @@ func applyTextEdit(lines []string, edit protocol.TextEdit) ([]string, error) {

99 endLine = len(lines) - 1

98 endLine = len(lines) - 1

100 }

99 }

101

100

101+ var startChar, endChar int

102+ switch encoding {

103+ case powernap.UTF8:

104+ // UTF-8: Character offset is already a byte offset

105+ startChar = int(edit.Range.Start.Character)

106+ endChar = int(edit.Range.End.Character)

107+ case powernap.UTF16:

108+ // UTF-16 (default): Convert to byte offset

109+ startLineContent := lines[startLine]

110+ endLineContent := lines[endLine]

111+ startChar = powernap.PositionToByteOffset(startLineContent, edit.Range.Start.Character)

112+ endChar = powernap.PositionToByteOffset(endLineContent, edit.Range.End.Character)

113+ default:

114+ // UTF-32: Character offset is codepoint count, convert to byte offset

115+ startLineContent := lines[startLine]

116+ endLineContent := lines[endLine]

117+ startChar = utf32ToByteOffset(startLineContent, edit.Range.Start.Character)

118+ endChar = utf32ToByteOffset(endLineContent, edit.Range.End.Character)

119+ }

120+

102 // Create result slice with initial capacity

121 // Create result slice with initial capacity

103 result := make([]string, 0, len(lines))

122 result := make([]string, 0, len(lines))

104

123

@@ -149,7 +168,7 @@ func applyTextEdit(lines []string, edit protocol.TextEdit) ([]string, error) {

149}

168}

150

169

151// applyDocumentChange applies a DocumentChange (create/rename/delete operations)

170// applyDocumentChange applies a DocumentChange (create/rename/delete operations)

152−func applyDocumentChange(change protocol.DocumentChange) error {

171+func applyDocumentChange(change protocol.DocumentChange, encoding powernap.OffsetEncoding) error {

153 if change.CreateFile != nil {

172 if change.CreateFile != nil {

154 path, err := change.CreateFile.URI.Path()

173 path, err := change.CreateFile.URI.Path()

155 if err != nil {

174 if err != nil {

@@ -222,24 +241,42 @@ func applyDocumentChange(change protocol.DocumentChange) error {

222 return fmt.Errorf("invalid edit type: %w", err)

241 return fmt.Errorf("invalid edit type: %w", err)

223 }

242 }

224 }

243 }

225− return applyTextEdits(change.TextDocumentEdit.TextDocument.URI, textEdits)

244+ return applyTextEdits(change.TextDocumentEdit.TextDocument.URI, textEdits, encoding)

226 }

245 }

227

246

228 return nil

247 return nil

229}

248}

230

249

250+// utf32ToByteOffset converts a UTF-32 codepoint offset to a byte offset.

251+func utf32ToByteOffset(lineText string, codepointOffset uint32) int {

252+ if codepointOffset == 0 {

253+ return 0

254+ }

255+

256+ var codepointCount uint32

257+ for byteOffset := range lineText {

258+ if codepointCount >= codepointOffset {

259+ return byteOffset

260+ }

261+ codepointCount++

262+ }

263+ return len(lineText)

264+}

265+

231−// ApplyWorkspaceEdit applies the given WorkspaceEdit to the filesystem

266+// ApplyWorkspaceEdit applies the given WorkspaceEdit to the filesystem.

267+// The encoding parameter specifies the position encoding used by the LSP server

268+// (UTF8, UTF16, or UTF32). This affects how character offsets are interpreted.

232−func ApplyWorkspaceEdit(edit protocol.WorkspaceEdit) error {

269+func ApplyWorkspaceEdit(edit protocol.WorkspaceEdit, encoding powernap.OffsetEncoding) error {

233 // Handle Changes field

270 // Handle Changes field

234 for uri, textEdits := range edit.Changes {

271 for uri, textEdits := range edit.Changes {

235− if err := applyTextEdits(uri, textEdits); err != nil {

272+ if err := applyTextEdits(uri, textEdits, encoding); err != nil {

236 return fmt.Errorf("failed to apply text edits: %w", err)

273 return fmt.Errorf("failed to apply text edits: %w", err)

237 }

274 }

238 }

275 }

239

276

240 // Handle DocumentChanges field

277 // Handle DocumentChanges field

241 for _, change := range edit.DocumentChanges {

278 for _, change := range edit.DocumentChanges {

242− if err := applyDocumentChange(change); err != nil {

279+ if err := applyDocumentChange(change, encoding); err != nil {

243 return fmt.Errorf("failed to apply document change: %w", err)

280 return fmt.Errorf("failed to apply document change: %w", err)

244 }

281 }

245 }

282 }

internal/lsp/util/edit_test.go+310−2

@@ -3,11 +3,319 @@ package util

3import (

4 "testing"

6− "github.com/stretchr/testify/require"

6+ powernap "github.com/charmbracelet/x/powernap/pkg/lsp"

7−

8 "github.com/charmbracelet/x/powernap/pkg/lsp/protocol"

7 "github.com/charmbracelet/x/powernap/pkg/lsp/protocol"

8+ "github.com/stretchr/testify/require"

11+func TestPositionToByteOffset(t *testing.T) {

12+ tests := []struct {

13+ name string

14+ lineText string

15+ utf16Char uint32

16+ expected int

17+ }{

18+ {

19+ name: "ASCII only",

20+ lineText: "hello world",

21+ utf16Char: 6,

22+ expected: 6,

23+ },

24+ {

25+ name: "CJK characters (3 bytes each in UTF-8, 1 UTF-16 unit)",

26+ lineText: "你好world",

27+ utf16Char: 2,

28+ expected: 6,

29+ },

30+ {

31+ name: "CJK - position after CJK",

32+ lineText: "var x = \"你好world\"",

33+ utf16Char: 11,

34+ expected: 15,

35+ },

36+ {

37+ name: "Emoji (4 bytes in UTF-8, 2 UTF-16 units)",

38+ lineText: "👋hello",

39+ utf16Char: 2,

40+ expected: 4,

41+ },

42+ {

43+ name: "Multiple emoji",

44+ lineText: "👋👋world",

45+ utf16Char: 4,

46+ expected: 8,

47+ },

48+ {

49+ name: "Mixed content",

50+ lineText: "Hello👋你好",

51+ utf16Char: 8,

52+ expected: 12,

53+ },

54+ {

55+ name: "Position 0",

56+ lineText: "hello",

57+ utf16Char: 0,

58+ expected: 0,

59+ },

60+ {

61+ name: "Position beyond end",

62+ lineText: "hi",

63+ utf16Char: 100,

64+ expected: 2,

65+ },

66+ {

67+ name: "Empty string",

68+ lineText: "",

69+ utf16Char: 0,

70+ expected: 0,

71+ },

72+ {

73+ name: "Surrogate pair at start",

74+ lineText: "𐐷hello",

75+ utf16Char: 2,

76+ expected: 4,

77+ },

78+ {

79+ name: "ZWJ family emoji (1 grapheme, 7 runes, 11 UTF-16 units)",

80+ lineText: "hello👨\u200d👩\u200d👧\u200d👦world",

81+ utf16Char: 16,

82+ expected: 30,

83+ },

84+ {

85+ name: "ZWJ family emoji - offset into middle of grapheme cluster",

86+ lineText: "hello👨\u200d👩\u200d👧\u200d👦world",

87+ utf16Char: 8,

88+ expected: 12,

89+ },

90+ {

91+ name: "Flag emoji (1 grapheme, 2 runes, 4 UTF-16 units)",

92+ lineText: "hello🇺🇸world",

93+ utf16Char: 9,

94+ expected: 13,

95+ },

96+ {

97+ name: "Combining character (1 grapheme, 2 runes, 2 UTF-16 units)",

98+ lineText: "caf\u0065\u0301!",

99+ utf16Char: 5,

100+ expected: 6,

101+ },

102+ {

103+ name: "Skin tone modifier (1 grapheme, 2 runes, 4 UTF-16 units)",

104+ lineText: "hi👋🏽bye",

105+ utf16Char: 6,

106+ expected: 10,

107+ },

108+ }

109+

110+ for _, tt := range tests {

111+ t.Run(tt.name, func(t *testing.T) {

112+ result := powernap.PositionToByteOffset(tt.lineText, tt.utf16Char)

113+ if result != tt.expected {

114+ t.Errorf("PositionToByteOffset(%q, %d) = %d, want %d",

115+ tt.lineText, tt.utf16Char, result, tt.expected)

116+ }

117+ })

118+ }

119+}

120+

121+func TestApplyTextEdit_UTF16(t *testing.T) {

122+ // Test that UTF-16 offsets are correctly converted to byte offsets

123+ tests := []struct {

124+ name string

125+ lines []string

126+ edit protocol.TextEdit

127+ expected []string

128+ }{

129+ {

130+ name: "ASCII only - no conversion needed",

131+ lines: []string{"hello world"},

132+ edit: protocol.TextEdit{

133+ Range: protocol.Range{

134+ Start: protocol.Position{Line: 0, Character: 6},

135+ End: protocol.Position{Line: 0, Character: 11},

136+ },

137+ NewText: "universe",

138+ },

139+ expected: []string{"hello universe"},

140+ },

141+ {

142+ name: "CJK characters - edit after Chinese characters",

143+ lines: []string{`var x = "你好world"`},

144+ edit: protocol.TextEdit{

145+ Range: protocol.Range{

146+ // "你好" = 2 UTF-16 units, but 6 bytes in UTF-8

147+ // Position 11 is where "world" starts in UTF-16

148+ Start: protocol.Position{Line: 0, Character: 11},

149+ End: protocol.Position{Line: 0, Character: 16},

150+ },

151+ NewText: "universe",

152+ },

153+ expected: []string{`var x = "你好universe"`},

154+ },

155+ {

156+ name: "Emoji - edit after emoji (2 UTF-16 units)",

157+ lines: []string{`fmt.Println("👋hello")`},

158+ edit: protocol.TextEdit{

159+ Range: protocol.Range{

160+ // 👋 = 2 UTF-16 units, 4 bytes in UTF-8

161+ // Position 15 is where "hello" starts in UTF-16

162+ Start: protocol.Position{Line: 0, Character: 15},

163+ End: protocol.Position{Line: 0, Character: 20},

164+ },

165+ NewText: "world",

166+ },

167+ expected: []string{`fmt.Println("👋world")`},

168+ },

169+ {

170+ name: "ZWJ family emoji - edit after grapheme cluster",

171+ // "hello👨‍👩‍👧‍👦world" — family is 1 grapheme but 11 UTF-16 units

172+ lines: []string{"hello\U0001F468\u200d\U0001F469\u200d\U0001F467\u200d\U0001F466world"},

173+ edit: protocol.TextEdit{

174+ Range: protocol.Range{

175+ // "hello" = 5 UTF-16 units, family = 11 UTF-16 units

176+ // "world" starts at UTF-16 offset 16

177+ Start: protocol.Position{Line: 0, Character: 16},

178+ End: protocol.Position{Line: 0, Character: 21},

179+ },

180+ NewText: "earth",

181+ },

182+ expected: []string{"hello\U0001F468\u200d\U0001F469\u200d\U0001F467\u200d\U0001F466earth"},

183+ },

184+ {

185+ name: "ZWJ family emoji - edit splits grapheme cluster in half",

186+ // LSP servers can position into the middle of a grapheme cluster.

187+ // After "hello" (5 UTF-16 units), the ZWJ family emoji starts.

188+ // UTF-16 offset 7 lands between 👨 (2 units) and ZWJ, inside

189+ // the grapheme cluster. The byte offset for position 7 is 9

190+ // (5 bytes for "hello" + 4 bytes for 👨).

191+ lines: []string{"hello\U0001F468\u200d\U0001F469\u200d\U0001F467\u200d\U0001F466world"},

192+ edit: protocol.TextEdit{

193+ Range: protocol.Range{

194+ Start: protocol.Position{Line: 0, Character: 7},

195+ End: protocol.Position{Line: 0, Character: 16},

196+ },

197+ NewText: "",

198+ },

199+ // Keeps "hello" + 👨 (first rune of cluster) then removes

200+ // the rest of the cluster, leaving "hello👨world".

201+ expected: []string{"hello\U0001F468world"},

202+ },

203+ {

204+ name: "Flag emoji - edit after flag",

205+ // 🇺🇸 = 2 regional indicator runes, 4 UTF-16 units, 8 bytes

206+ lines: []string{"hello🇺🇸world"},

207+ edit: protocol.TextEdit{

208+ Range: protocol.Range{

209+ Start: protocol.Position{Line: 0, Character: 9},

210+ End: protocol.Position{Line: 0, Character: 14},

211+ },

212+ NewText: "earth",

213+ },

214+ expected: []string{"hello🇺🇸earth"},

215+ },

216+ {

217+ name: "Combining accent - edit after composed character",

218+ // "café!" where é = e + U+0301 (2 code points, 2 UTF-16 units)

219+ lines: []string{"caf\u0065\u0301!"},

220+ edit: protocol.TextEdit{

221+ Range: protocol.Range{

222+ // "caf" = 3, "e" = 1, U+0301 = 1, total = 5 UTF-16 units

223+ Start: protocol.Position{Line: 0, Character: 5},

224+ End: protocol.Position{Line: 0, Character: 6},

225+ },

226+ NewText: "?",

227+ },

228+ expected: []string{"caf\u0065\u0301?"},

229+ },

230+ {

231+ name: "Skin tone modifier - edit after modified emoji",

232+ // 👋🏽 = U+1F44B U+1F3FD = 2 runes, 4 UTF-16 units, 8 bytes

233+ lines: []string{"hi👋🏽bye"},

234+ edit: protocol.TextEdit{

235+ Range: protocol.Range{

236+ // "hi" = 2, 👋🏽 = 4, total = 6 UTF-16 units

237+ Start: protocol.Position{Line: 0, Character: 6},

238+ End: protocol.Position{Line: 0, Character: 9},

239+ },

240+ NewText: "later",

241+ },

242+ expected: []string{"hi👋🏽later"},

243+ },

244+ }

245+

246+ for _, tt := range tests {

247+ t.Run(tt.name, func(t *testing.T) {

248+ result, err := applyTextEdit(tt.lines, tt.edit, powernap.UTF16)

249+ if err != nil {

250+ t.Fatalf("applyTextEdit failed: %v", err)

251+ }

252+ if len(result) != len(tt.expected) {

253+ t.Errorf("expected %d lines, got %d: %v", len(tt.expected), len(result), result)

254+ return

255+ }

256+ for i := range result {

257+ if result[i] != tt.expected[i] {

258+ t.Errorf("line %d: expected %q, got %q", i, tt.expected[i], result[i])

259+ }

260+ }

261+ })

262+ }

263+}

264+

265+func TestApplyTextEdit_UTF8(t *testing.T) {

266+ // Test that UTF-8 offsets are used directly without conversion

267+ tests := []struct {

268+ name string

269+ lines []string

270+ edit protocol.TextEdit

271+ expected []string

272+ }{

273+ {

274+ name: "ASCII only - direct byte offset",

275+ lines: []string{"hello world"},

276+ edit: protocol.TextEdit{

277+ Range: protocol.Range{

278+ Start: protocol.Position{Line: 0, Character: 6},

279+ End: protocol.Position{Line: 0, Character: 11},

280+ },

281+ NewText: "universe",

282+ },

283+ expected: []string{"hello universe"},

284+ },

285+ {

286+ name: "CJK characters - byte offset used directly",

287+ lines: []string{`var x = "你好world"`},

288+ edit: protocol.TextEdit{

289+ Range: protocol.Range{

290+ // With UTF-8 encoding, position 15 is the byte offset

291+ Start: protocol.Position{Line: 0, Character: 15},

292+ End: protocol.Position{Line: 0, Character: 20},

293+ },

294+ NewText: "universe",

295+ },

296+ expected: []string{`var x = "你好universe"`},

297+ },

298+ }

299+

300+ for _, tt := range tests {

301+ t.Run(tt.name, func(t *testing.T) {

302+ result, err := applyTextEdit(tt.lines, tt.edit, powernap.UTF8)

303+ if err != nil {

304+ t.Fatalf("applyTextEdit failed: %v", err)

305+ }

306+ if len(result) != len(tt.expected) {

307+ t.Errorf("expected %d lines, got %d: %v", len(tt.expected), len(result), result)

308+ return

309+ }

310+ for i := range result {

311+ if result[i] != tt.expected[i] {

312+ t.Errorf("line %d: expected %q, got %q", i, tt.expected[i], result[i])

313+ }

314+ }

315+ })

316+ }

317+}

318+

11func TestRangesOverlap(t *testing.T) {

319func TestRangesOverlap(t *testing.T) {

12 t.Parallel()

320 t.Parallel()

321