on main help sign in
perf(message): eliminate O(n²) streaming accumulation

Accumulate streaming text/reasoning deltas into a per-message strings.Builder (amortized O(1)) instead of rebuilding the whole accumulated string on every token, which was O(n²) over a turn. The builders are unexported, dropped by Clone, and only touched by the owning stream, so snapshots read an immutable append-only prefix (verified race-free under -race). Also make shouldFlushNow allocation-free via a single-pass tool-call stats helper instead of allocating ToolCalls slices per delta. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

commit9492d7d0966b866de29576af8d3e4d038f7ecbe4 authoriko <mail@iko.soy> dateThu 2 Jul 2026, 03:17 (3 months ago) parent0d76633
2 files changed+74−34
internal/message/content.go+66−21 internal/message/message.go+8−13
internal/message/content.go+66−21
@@ -171,6 +171,18 @@ type Message struct {
171171 CreatedAt int64
172172 UpdatedAt int64
173173 IsSummaryMessage bool
174+
175+ // textBuf and reasoningBuf accumulate streaming deltas in amortized
176+ // O(1) so building a long response is O(n) rather than O(n²) (the
177+ // naive `existing + delta` reallocated and copied the whole
178+ // accumulated string on every token). They are nil until the first
179+ // delta, unexported (never persisted), and deliberately dropped by
180+ // Clone: only the streaming message that owns them ever appends, so
181+ // no other goroutine touches the builder. Snapshots observe an
182+ // immutable prefix of the builder's append-only buffer, which is
183+ // race-free to read concurrently.
184+ textBuf *strings.Builder
185+ reasoningBuf *strings.Builder
174186}
175187
176188func (m *Message) Content() TextContent {
@@ -221,6 +233,21 @@ func (m *Message) ToolCalls() []ToolCall {
221233 return toolCalls
222234}
223235
236+// toolCallStats returns the number of tool-call parts and how many of
237+// them are finished, in a single pass without allocating (unlike
238+// ToolCalls, which builds a slice). Used on the streaming hot path.
239+func (m *Message) toolCallStats() (count, finished int) {
240+ for _, part := range m.Parts {
241+ if c, ok := part.(ToolCall); ok {
242+ count++
243+ if c.Finished {
244+ finished++
245+ }
246+ }
247+ }
248+ return count, finished
249+}
250+
224251func (m *Message) ToolResults() []ToolResult {
225252 toolResults := make([]ToolResult, 0)
226253 for _, part := range m.Parts {
@@ -266,37 +293,51 @@ func (m *Message) IsThinking() bool {
266293}
267294
268295func (m *Message) AppendContent(delta string) {
269− found := false
270− for i, part := range m.Parts {
271− if c, ok := part.(TextContent); ok {
272− m.Parts[i] = TextContent{Text: c.Text + delta}
273− found = true
296+ if m.textBuf == nil {
297+ m.textBuf = &strings.Builder{}
298+ // Seed from any pre-existing text part (e.g. a restored message);
299+ // streaming messages start empty so this is normally a no-op.
300+ for _, part := range m.Parts {
301+ if c, ok := part.(TextContent); ok {
302+ m.textBuf.WriteString(c.Text)
303+ break
304+ }
274305 }
275306 }
276− if !found {
277− m.Parts = append(m.Parts, TextContent{Text: delta})
307+ m.textBuf.WriteString(delta)
308+ text := m.textBuf.String()
309+ for i, part := range m.Parts {
310+ if _, ok := part.(TextContent); ok {
311+ m.Parts[i] = TextContent{Text: text}
312+ return
313+ }
278314 }
315+ m.Parts = append(m.Parts, TextContent{Text: text})
279316}
280317
281318func (m *Message) AppendReasoningContent(delta string) {
282− found := false
283− for i, part := range m.Parts {
284− if c, ok := part.(ReasoningContent); ok {
285− m.Parts[i] = ReasoningContent{
286− Thinking: c.Thinking + delta,
287− Signature: c.Signature,
288− StartedAt: c.StartedAt,
289− FinishedAt: c.FinishedAt,
319+ if m.reasoningBuf == nil {
320+ m.reasoningBuf = &strings.Builder{}
321+ for _, part := range m.Parts {
322+ if c, ok := part.(ReasoningContent); ok {
323+ m.reasoningBuf.WriteString(c.Thinking)
324+ break
290325 }
291− found = true
292326 }
293327 }
294− if !found {
295− m.Parts = append(m.Parts, ReasoningContent{
296− Thinking: delta,
297− StartedAt: time.Now().Unix(),
298− })
328+ m.reasoningBuf.WriteString(delta)
329+ thinking := m.reasoningBuf.String()
330+ for i, part := range m.Parts {
331+ if c, ok := part.(ReasoningContent); ok {
332+ c.Thinking = thinking
333+ m.Parts[i] = c
334+ return
335+ }
299336 }
337+ m.Parts = append(m.Parts, ReasoningContent{
338+ Thinking: thinking,
339+ StartedAt: time.Now().Unix(),
340+ })
300341}
301342
302343func (m *Message) AppendThoughtSignature(signature string, toolCallID string) {
@@ -450,6 +491,10 @@ func (m *Message) Clone() Message {
450491 clone := *m
451492 clone.Parts = make([]ContentPart, len(m.Parts))
452493 copy(clone.Parts, m.Parts)
494+ // Snapshots never stream; drop the accumulation builders so only the
495+ // owning message ever appends to them (keeps them single-goroutine).
496+ clone.textBuf = nil
497+ clone.reasoningBuf = nil
453498 return clone
454499}
455500
internal/message/message.go+8−13
@@ -420,25 +420,20 @@ func shouldFlushNow(prev, next *Message) bool {
420420 return true
421421 }
422422
423− var prevCalls []ToolCall
423+ var prevCount, prevFinished int
424424 var prevReasoningFinishedAt int64
425425 if prev != nil {
426− prevCalls = prev.ToolCalls()
426+ prevCount, prevFinished = prev.toolCallStats()
427427 prevReasoningFinishedAt = prev.ReasoningContent().FinishedAt
428428 }
429− nextCalls := next.ToolCalls()
430− if len(nextCalls) != len(prevCalls) {
429+ nextCount, nextFinished := next.toolCallStats()
430+ // The tool-call set grew, or a call transitioned to finished. Tool
431+ // calls never un-finish and the count only grows, so comparing the
432+ // finished counts is equivalent to the old per-index Finished check
433+ // but allocation-free (a call's Input only matters once Finished).
434+ if nextCount != prevCount || nextFinished != prevFinished {
431435 return true
432436 }
433− for i := range nextCalls {
434− // Bounds-safe: lengths are equal here.
435− if nextCalls[i].Finished != prevCalls[i].Finished {
436− return true
437− }
438− // A tool call's input only matters once it has landed (Finished
439− // flips true). Earlier deltas to Input are debounced with the
440− // rest of the streaming state.
441− }
442437 if next.ReasoningContent().FinishedAt > 0 && prevReasoningFinishedAt == 0 {
443438 return true
444439 }