@@ -123,9 +123,26 @@ default(Int)
123123infixl 2 :!:
124124data PairS a b = !a :!: !b
125125
126+-- | Allow a function over a stream to switch between two states.
126127data Switch = S1 | S2
127128
128−data Stream a = forall s. Stream (s -> Step s a) !s {-# UNPACK #-}!Int
129+data Stream a =
130+ forall s. Stream
131+ (s -> Step s a) -- stepper function
132+ !s -- current state
133+ {-# UNPACK #-}!Int -- length hint
134+
135+-- The length hint in a Stream has two roles. If its value is zero,
136+-- we trust it, and treat the stream as empty. Otherwise, we treat it
137+-- as a hint: it should usually be accurate, so we use it when
138+-- unstreaming to decide what size array to allocate. However, the
139+-- unstreaming functions must be able to cope with the hint being too
140+-- small or too large.
141+--
142+-- The size hint tries to track the UTF-16 code points in a stream,
143+-- but often counts the number of characters instead. It can easily
144+-- undercount if, for instance, a transformed stream contains astral
145+-- plane characters (those above 0x10000).
129146
130147data Step s a = Done
131148 | Skip !s