94df678 · text · git.iko.soy

git.iko.soy/text

on master help sign in

Tidy up and test UTF-16BE

94df678Bryan O'Sullivan18 years ago

Data/Text/Encoding/Fusion.hs+8−8

compare with master

unifiedsplit

@@ -200,15 +200,15 @@ restreamUtf8 (Stream next0 s0 len) =

200

201restreamUtf16BE :: Stream Char -> Stream Word8

202restreamUtf16BE (Stream next0 s0 len) =

203− Stream next (T4 (Just s0) Nothing Nothing Nothing) (len*2)

203+ Stream next (T4 s0 N N N) (len*2)

204 where

205 {-# INLINE next #-}

206− next (T4 (Just s) Nothing Nothing Nothing) = case next0 s of

206+ next (T4 s N N N) = case next0 s of

207 Done -> Done

208− Skip s' -> Skip (T4 (Just s') Nothing Nothing Nothing)

208+ Skip s' -> Skip (T4 s' N N N)

209 Yield x xs

210− | n < 0x10000 -> Yield (fromIntegral $ shiftR n 8) (T4 (Just xs) (Just (fromIntegral n)) Nothing Nothing)

210+ | n < 0x10000 -> Yield (fromIntegral $ shiftR n 8) (T4 xs (J (fromIntegral n)) N N)

211− | otherwise -> Yield c1 (T4 (Just xs) (Just c2) (Just c3) (Just c4))

211+ | otherwise -> Yield c1 (T4 xs (J c2) (J c3) (J c4))

212 where

213 n = ord x

214 n1 = n - 0x10000

@@ -217,9 +217,9 @@ restreamUtf16BE (Stream next0 s0 len) =

217 n2 = n1 .&. 0x3FF

218 c3 = fromIntegral (shiftR n2 8 + 0xDC)

219 c4 = fromIntegral n2

220− next (T4 (Just s) (Just x2) Nothing Nothing) = Yield x2 (T4 (Just s) Nothing Nothing Nothing)

220+ next (T4 s (J x2) N N) = Yield x2 (T4 s N N N)

221− next (T4 (Just s) (Just x2) x3 Nothing) = Yield x2 (T4 (Just s) x3 Nothing Nothing)

221+ next (T4 s (J x2) x3 N) = Yield x2 (T4 s x3 N N)

222− next (T4 (Just s) (Just x2) x3 x4) = Yield x2 (T4 (Just s) x3 x4 Nothing)

222+ next (T4 s (J x2) x3 x4) = Yield x2 (T4 s x3 x4 N)

223 next _ = internalError "restreamUtf16BE"

224{-# INLINE restreamUtf16BE #-}

225

tests/Properties.hs+2

@@ -42,6 +42,7 @@ prop_ascii t = E.decodeASCII (E.encodeASCII a) == a

42 where a = T.map (\c -> chr (ord c `mod` 128)) t

43prop_utf8 = (E.decodeUtf8 . E.encodeUtf8) `eq` id

44prop_utf16LE = (E.decodeUtf16LE . E.encodeUtf16LE) `eq` id

45+prop_utf16BE = (E.decodeUtf16BE . E.encodeUtf16BE) `eq` id

46-- Do two functions give the same answer?

47-- Do two functions give the same answer?

47eq :: (Eq a) => (t -> a) -> (t -> a) -> t -> Bool

48eq :: (Eq a) => (t -> a) -> (t -> a) -> t -> Bool

@@ -233,6 +234,7 @@ tests = [

233 ("prop_ascii", mytest prop_ascii),

234 ("prop_ascii", mytest prop_ascii),

234 ("prop_utf8", mytest prop_utf8),

235 ("prop_utf8", mytest prop_utf8),

235 ("prop_utf16LE", mytest prop_utf16LE),

236 ("prop_utf16LE", mytest prop_utf16LE),

237+ ("prop_utf16BE", mytest prop_utf16BE),

236

238

237 ("prop_cons", mytest prop_cons),

239 ("prop_cons", mytest prop_cons),

238 ("prop_snoc", mytest prop_snoc),

240 ("prop_snoc", mytest prop_snoc),