@@ -56,7 +56,14 @@ import qualified Data.Text.Utf16 as U16
5656import qualified Data.Text.Utf32 as U32
5757import qualified Data.Text.Utf8 as U8
5858
59−data T4 a b c d = T4 {-# UNPACK #-} !a {-# UNPACK #-} !b {-# UNPACK #-} !c {-# UNPACK #-} !d
59+-- Specialised, strict Maybe-like type.
60+data M = N
61+ | J {-# UNPACK #-} !Word8
62+ deriving (Eq, Ord, Show)
63+
64+-- Restreaming state.
65+data S s = S {-# UNPACK #-} !s
66+ {-# UNPACK #-} !M {-# UNPACK #-} !M {-# UNPACK #-} !M
6067
6168streamASCII :: ByteString -> Stream Char
6269streamASCII bs = Stream next 0 l
@@ -70,7 +77,8 @@ streamASCII bs = Stream next 0 l
7077 x1 = B.unsafeIndex bs i
7178{-# INLINE [0] streamASCII #-}
7279
73−-- | /O(n) Convert a ByteString into a Stream Char, using the specified encoding standard.
80+-- | /O(n) Convert a 'ByteString' into a 'Stream Char', using UTF-8
81+-- encoding.
7482streamUtf8 :: ByteString -> Stream Char
7583streamUtf8 bs = Stream next 0 l
7684 where
@@ -91,6 +99,8 @@ streamUtf8 bs = Stream next 0 l
9199 idx = B.unsafeIndex bs
92100{-# INLINE [0] streamUtf8 #-}
93101
102+-- | /O(n) Convert a 'ByteString' into a 'Stream Char', using little
103+-- endian UTF-16 encoding.
94104streamUtf16LE :: ByteString -> Stream Char
95105streamUtf16LE bs = Stream next 0 l
96106 where
@@ -102,11 +112,13 @@ streamUtf16LE bs = Stream next 0 l
102112 | i+3 < l && U16.validate2 x1 x2 = Yield (U16.chr2 x1 x2) (i+4)
103113 | otherwise = encodingError "UTF-16LE"
104114 where
105− x1 = (idx (i + 1) `shiftL` 8) + idx i
106− x2 = (idx (i + 3) `shiftL` 8) + idx (i + 2)
115+ x1 = idx i + (idx (i + 1) `shiftL` 8)
116+ x2 = idx (i + 2) + (idx (i + 3) `shiftL` 8)
107117 idx = fromIntegral . B.unsafeIndex bs :: Int -> Word16
108118{-# INLINE [0] streamUtf16LE #-}
109119
120+-- | /O(n) Convert a 'ByteString' into a 'Stream Char', using big
121+-- endian UTF-16 encoding.
110122streamUtf16BE :: ByteString -> Stream Char
111123streamUtf16BE bs = Stream next 0 l
112124 where
@@ -118,11 +130,13 @@ streamUtf16BE bs = Stream next 0 l
118130 | i+3 < l && U16.validate2 x1 x2 = Yield (U16.chr2 x1 x2) (i+4)
119131 | otherwise = encodingError "UTF16-BE"
120132 where
121− x1 = (shiftL (idx i) 8) + (idx (i + 1))
122− x2 = (shiftL (idx (i + 2)) 8) + (idx (i + 3))
133+ x1 = (idx i `shiftL` 8) + idx (i + 1)
134+ x2 = (idx (i + 2) `shiftL` 8) + idx (i + 3)
123135 idx = fromIntegral . B.unsafeIndex bs :: Int -> Word16
124136{-# INLINE [0] streamUtf16BE #-}
125137
138+-- | /O(n) Convert a 'ByteString' into a 'Stream Char', using big
139+-- endian UTF-32 encoding.
126140streamUtf32BE :: ByteString -> Stream Char
127141streamUtf32BE bs = Stream next 0 l
128142 where
@@ -141,6 +155,8 @@ streamUtf32BE bs = Stream next 0 l
141155 idx = fromIntegral . B.unsafeIndex bs :: Int -> Word32
142156{-# INLINE [0] streamUtf32BE #-}
143157
158+-- | /O(n) Convert a 'ByteString' into a 'Stream Char', using little
159+-- endian UTF-32 encoding.
144160streamUtf32LE :: ByteString -> Stream Char
145161streamUtf32LE bs = Stream next 0 l
146162 where
@@ -169,73 +185,72 @@ restreamASCII (Stream next0 s0 len) = Stream next s0 (len*2)
169185 where x' = fromIntegral (ord x) :: Word8
170186{-# INLINE restreamASCII #-}
171187
172−data M = N | J {-# UNPACK #-} !Word8
173− deriving (Eq, Ord, Show)
174−
175188-- | /O(n)/ Convert a Stream Char into a UTF-8 encoded Stream Word8.
176189restreamUtf8 :: Stream Char -> Stream Word8
177190restreamUtf8 (Stream next0 s0 len) =
178− Stream next (T4 s0 N N N) (len*2)
191+ Stream next (S s0 N N N) (len*2)
179192 where
180193 {-# INLINE next #-}
181− next (T4 s N N N) = case next0 s of
194+ next (S s N N N) = case next0 s of
182195 Done -> Done
183− Skip s' -> Skip (T4 s' N N N)
196+ Skip s' -> Skip (S s' N N N)
184197 Yield x xs
185− | n <= 0x7F -> Yield c (T4 xs N N N)
186− | n <= 0x07FF -> Yield a2 (T4 xs (J b2) N N)
187− | n <= 0xFFFF -> Yield a3 (T4 xs (J b3) (J c3) N)
188− | otherwise -> Yield a4 (T4 xs (J b4) (J c4) (J d4))
198+ | n <= 0x7F -> Yield c (S xs N N N)
199+ | n <= 0x07FF -> Yield a2 (S xs (J b2) N N)
200+ | n <= 0xFFFF -> Yield a3 (S xs (J b3) (J c3) N)
201+ | otherwise -> Yield a4 (S xs (J b4) (J c4) (J d4))
189202 where
190203 n = ord x
191204 c = fromIntegral n
192205 (a2,b2) = U8.ord2 x
193206 (a3,b3,c3) = U8.ord3 x
194207 (a4,b4,c4,d4) = U8.ord4 x
195− next (T4 s (J x2) N N) = Yield x2 (T4 s N N N)
196− next (T4 s (J x2) x3 N) = Yield x2 (T4 s x3 N N)
197− next (T4 s (J x2) x3 x4) = Yield x2 (T4 s x3 x4 N)
208+ next (S s (J x2) N N) = Yield x2 (S s N N N)
209+ next (S s (J x2) x3 N) = Yield x2 (S s x3 N N)
210+ next (S s (J x2) x3 x4) = Yield x2 (S s x3 x4 N)
198211 next _ = internalError "restreamUtf8"
199212{-# INLINE restreamUtf8 #-}
200213
201214restreamUtf16BE :: Stream Char -> Stream Word8
202215restreamUtf16BE (Stream next0 s0 len) =
203− Stream next (T4 s0 N N N) (len*2)
216+ Stream next (S s0 N N N) (len*2)
204217 where
205218 {-# INLINE next #-}
206− next (T4 s N N N) = case next0 s of
219+ next (S s N N N) = case next0 s of
207220 Done -> Done
208− Skip s' -> Skip (T4 s' N N N)
221+ Skip s' -> Skip (S s' N N N)
209222 Yield x xs
210− | n < 0x10000 -> Yield (fromIntegral $ shiftR n 8) (T4 xs (J (fromIntegral n)) N N)
211− | otherwise -> Yield c1 (T4 xs (J c2) (J c3) (J c4))
223+ | n < 0x10000 -> Yield (fromIntegral $ n `shiftR` 8) $
224+ S xs (J $ fromIntegral n) N N
225+ | otherwise -> Yield c1 $
226+ S xs (J c2) (J c3) (J c4)
212227 where
213228 n = ord x
214229 n1 = n - 0x10000
215− c1 = fromIntegral (shiftR n1 18 + 0xD8)
216− c2 = fromIntegral (shiftR n1 10)
230+ c1 = fromIntegral (n1 `shiftR` 18 + 0xD8)
231+ c2 = fromIntegral (n1 `shiftR` 10)
217232 n2 = n1 .&. 0x3FF
218− c3 = fromIntegral (shiftR n2 8 + 0xDC)
233+ c3 = fromIntegral (n2 `shiftR` 8 + 0xDC)
219234 c4 = fromIntegral n2
220− next (T4 s (J x2) N N) = Yield x2 (T4 s N N N)
221− next (T4 s (J x2) x3 N) = Yield x2 (T4 s x3 N N)
222− next (T4 s (J x2) x3 x4) = Yield x2 (T4 s x3 x4 N)
235+ next (S s (J x2) N N) = Yield x2 (S s N N N)
236+ next (S s (J x2) x3 N) = Yield x2 (S s x3 N N)
237+ next (S s (J x2) x3 x4) = Yield x2 (S s x3 x4 N)
223238 next _ = internalError "restreamUtf16BE"
224239{-# INLINE restreamUtf16BE #-}
225240
226241restreamUtf16LE :: Stream Char -> Stream Word8
227242restreamUtf16LE (Stream next0 s0 len) =
228− Stream next (T4 s0 N N N) (len*2)
243+ Stream next (S s0 N N N) (len*2)
229244 where
230245 {-# INLINE next #-}
231− next (T4 s N N N) = case next0 s of
246+ next (S s N N N) = case next0 s of
232247 Done -> Done
233− Skip s' -> Skip (T4 s' N N N)
248+ Skip s' -> Skip (S s' N N N)
234249 Yield x xs
235250 | n < 0x10000 -> Yield (fromIntegral n) $
236− T4 xs (J (fromIntegral $ shiftR n 8)) N N
251+ S xs (J (fromIntegral $ shiftR n 8)) N N
237252 | otherwise -> Yield c1 $
238− T4 xs (J c2) (J c3) (J c4)
253+ S xs (J c2) (J c3) (J c4)
239254 where
240255 n = ord x
241256 n1 = n - 0x10000
@@ -244,51 +259,51 @@ restreamUtf16LE (Stream next0 s0 len) =
244259 n2 = n1 .&. 0x3FF
245260 c4 = fromIntegral (shiftR n2 8 + 0xDC)
246261 c3 = fromIntegral n2
247− next (T4 s (J x2) N N) = Yield x2 (T4 s N N N)
248− next (T4 s (J x2) x3 N) = Yield x2 (T4 s x3 N N)
249− next (T4 s (J x2) x3 x4) = Yield x2 (T4 s x3 x4 N)
262+ next (S s (J x2) N N) = Yield x2 (S s N N N)
263+ next (S s (J x2) x3 N) = Yield x2 (S s x3 N N)
264+ next (S s (J x2) x3 x4) = Yield x2 (S s x3 x4 N)
250265 next _ = internalError "restreamUtf16LE"
251266{-# INLINE restreamUtf16LE #-}
252267
253268restreamUtf32BE :: Stream Char -> Stream Word8
254269restreamUtf32BE (Stream next0 s0 len) =
255− Stream next (T4 (Just s0) Nothing Nothing Nothing) (len*2)
270+ Stream next (S s0 N N N) (len*2)
256271 where
257272 {-# INLINE next #-}
258− next (T4 (Just s) Nothing Nothing Nothing) = case next0 s of
273+ next (S s N N N) = case next0 s of
259274 Done -> Done
260− Skip s' -> Skip (T4 (Just s') Nothing Nothing Nothing)
261− Yield x xs -> Yield c1 (T4 (Just xs) (Just c2) (Just c3) (Just c4))
275+ Skip s' -> Skip (S s' N N N)
276+ Yield x xs -> Yield c1 (S xs (J c2) (J c3) (J c4))
262277 where
263278 n = ord x
264279 c1 = fromIntegral $ shiftR n 24
265280 c2 = fromIntegral $ shiftR n 16
266281 c3 = fromIntegral $ shiftR n 8
267282 c4 = fromIntegral n
268− next (T4 (Just s) (Just x2) Nothing Nothing) = Yield x2 (T4 (Just s) Nothing Nothing Nothing)
269− next (T4 (Just s) (Just x2) x3 Nothing) = Yield x2 (T4 (Just s) x3 Nothing Nothing)
270− next (T4 (Just s) (Just x2) x3 x4) = Yield x2 (T4 (Just s) x3 x4 Nothing)
283+ next (S s (J x2) N N) = Yield x2 (S s N N N)
284+ next (S s (J x2) x3 N) = Yield x2 (S s x3 N N)
285+ next (S s (J x2) x3 x4) = Yield x2 (S s x3 x4 N)
271286 next _ = internalError "restreamUtf32BE"
272287{-# INLINE restreamUtf32BE #-}
273288
274289restreamUtf32LE :: Stream Char -> Stream Word8
275290restreamUtf32LE (Stream next0 s0 len) =
276− Stream next (T4 (Just s0) Nothing Nothing Nothing) (len*2)
291+ Stream next (S s0 N N N) (len*2)
277292 where
278293 {-# INLINE next #-}
279− next (T4 (Just s) Nothing Nothing Nothing) = case next0 s of
294+ next (S s N N N) = case next0 s of
280295 Done -> Done
281− Skip s' -> Skip (T4 (Just s') Nothing Nothing Nothing)
282− Yield x xs -> Yield c1 (T4 (Just xs) (Just c2) (Just c3) (Just c4))
296+ Skip s' -> Skip (S s' N N N)
297+ Yield x xs -> Yield c1 (S xs (J c2) (J c3) (J c4))
283298 where
284299 n = ord x
285300 c4 = fromIntegral $ shiftR n 24
286301 c3 = fromIntegral $ shiftR n 16
287302 c2 = fromIntegral $ shiftR n 8
288303 c1 = fromIntegral n
289− next (T4 (Just s) (Just x2) Nothing Nothing) = Yield x2 (T4 (Just s) Nothing Nothing Nothing)
290− next (T4 (Just s) (Just x2) x3 Nothing) = Yield x2 (T4 (Just s) x3 Nothing Nothing)
291− next (T4 (Just s) (Just x2) x3 x4) = Yield x2 (T4 (Just s) x3 x4 Nothing)
304+ next (S s (J x2) N N) = Yield x2 (S s N N N)
305+ next (S s (J x2) x3 N) = Yield x2 (S s x3 N N)
306+ next (S s (J x2) x3 x4) = Yield x2 (S s x3 x4 N)
292307 next _ = internalError "restreamUtf32LE"
293308{-# INLINE restreamUtf32LE #-}
294309