@@ -123,9 +123,26 @@ default(Int)
123infixl 2 :!:
123infixl 2 :!:
124data PairS a b = !a :!: !b
124data PairS a b = !a :!: !b
126+-- | Allow a function over a stream to switch between two states.
126data Switch = S1 | S2
127data Switch = S1 | S2
128−data Stream a = forall s. Stream (s -> Step s a) !s {-# UNPACK #-}!Int
130+ forall s. Stream
131+ (s -> Step s a) -- stepper function
133+ {-# UNPACK #-}!Int -- length hint
135+-- The length hint in a Stream has two roles. If its value is zero,
136+-- we trust it, and treat the stream as empty. Otherwise, we treat it
137+-- as a hint: it should usually be accurate, so we use it when
138+-- unstreaming to decide what size array to allocate. However, the
139+-- unstreaming functions must be able to cope with the hint being too
140+-- small or too large.
142+-- The size hint tries to track the UTF-16 code points in a stream,
143+-- but often counts the number of characters instead. It can easily
144+-- undercount if, for instance, a transformed stream contains astral
145+-- plane characters (those above 0x10000).
130data Step s a = Done
147data Step s a = Done
131 | Skip !s
148 | Skip !s