1175a74 · text · git.iko.soy

git.iko.soy/text

First stab at lazy text support

--HG-- extra : convert_revision : afe709f5182304af650e67bc669334531ee12016

commit1175a7401efda26dd64ab675cd6f17ff23f75350 authorBryan O'Sullivan <bos@serpentine.com> dateSat 28 Feb 2009, 01:24 (18 years ago) parent9f53a93

7 files changed+330−274

compare with master

unifiedsplit

Data/Text.hs+5−23 Data/Text/Fusion.hs+5−51 Data/Text/Internal.hs+8 Data/Text/Lazy/Internal.hs+95 tests/Properties.hs+204−200 tests/QuickCheckUtils.hs+9 text.cabal+4

Data/Text.hs+5−23

@@ -158,8 +158,10 @@ import Data.Word (Word16)

158158import Data.String (IsString(..))

159159

160160import qualified Data.Text.Fusion as S

161+import qualified Data.Text.Fusion.Internal as S

161162import Data.Text.Fusion (Stream(..), Step(..), stream, reverseStream, unstream)

162−import Data.Text.Internal (Text(..), empty, text)

163+

164+import Data.Text.Internal (Text(..), empty, text, textP)

163165import qualified Prelude as P

164166import Data.Text.Unsafe (iter, iter_, unsafeHead, unsafeTail)

165167import Data.Text.UnsafeChar (unsafeChr)

@@ -195,26 +197,13 @@ instance IsString Text where

195197--

196198-- This function is subject to array fusion.

197199pack :: String -> Text

198−pack str = (unstream (stream_list str))

199− where

200− stream_list s0 = S.Stream next s0 (P.length s0) -- total guess

201− where

202− next [] = S.Done

203− next (x:xs) = S.Yield x xs

200+pack = unstream . S.streamList

204201{-# INLINE [1] pack #-}

205−-- TODO: Has to do validation! -- No, it doesn't, the

206202

207203-- | /O(n)/ Convert a Text into a String.

208204-- Subject to array fusion.

209205unpack :: Text -> String

210−unpack txt = (unstream_list (stream txt))

211− where

212− unstream_list (S.Stream next s0 _len) = unfold s0

213− where

214− unfold !s = case next s of

215− S.Done -> []

216− S.Skip s' -> unfold s'

217− S.Yield x s' -> x : unfold s'

206+unpack = S.unstreamList . stream

218207{-# INLINE [1] unpack #-}

219208

220209-- | /O(1)/ Convert a character into a Text.

@@ -312,13 +301,6 @@ last (Text arr off len)

312301 S.last (stream t) = last t

313302 #-}

314303

315−-- | Construct a 'Text' without invisibly pinning its byte array in

316−-- memory if its length has dwindled to zero.

317−textP :: A.Array Word16 -> Int -> Int -> Text

318−textP arr off len | len == 0 = empty

319− | otherwise = text arr off len

320−{-# INLINE textP #-}

321−

322304-- | /O(1)/ Returns all characters after the head of a 'Text', which

323305-- must be non-empty. Subject to array fusion.

324306tail :: Text -> Text

Data/Text/Fusion.hs+5−51

@@ -1,4 +1,4 @@

1−{-# LANGUAGE ExistentialQuantification, BangPatterns, MagicHash #-}

1+{-# LANGUAGE BangPatterns, MagicHash #-}

33-- |

44-- Module : Data.Text.Fusion

@@ -22,6 +22,7 @@ module Data.Text.Fusion

2222

2323 -- * Creation and elimination

2424 , stream

25+ , streamList

2526 , unstream

2627 , reverseStream

2728 , empty

@@ -114,40 +115,13 @@ import Data.Char (ord)

114115import Data.Text.Internal (Text(..))

115116import Data.Text.UnsafeChar (unsafeChr, unsafeWrite, unsafeWriteRev)

116117import qualified Data.Text.Array as A

118+import Data.Text.Fusion.Internal

117119import qualified Data.Text.Internal as I

118120import qualified Data.Text.Encoding.Utf16 as U16

119121import qualified Prelude as P

120122

121123default(Int)

122124

123−infixl 2 :!:

124−data PairS a b = !a :!: !b

125−

126−-- | Allow a function over a stream to switch between two states.

127−data Switch = S1 | S2

128−

129−data Stream a =

130− forall s. Stream

131− (s -> Step s a) -- stepper function

132− !s -- current state

133− {-# UNPACK #-}!Int -- length hint

134−

135−-- The length hint in a Stream has two roles. If its value is zero,

136−-- we trust it, and treat the stream as empty. Otherwise, we treat it

137−-- as a hint: it should usually be accurate, so we use it when

138−-- unstreaming to decide what size array to allocate. However, the

139−-- unstreaming functions must be able to cope with the hint being too

140−-- small or too large.

141−--

142−-- The size hint tries to track the UTF-16 code points in a stream,

143−-- but often counts the number of characters instead. It can easily

144−-- undercount if, for instance, a transformed stream contains astral

145−-- plane characters (those above 0x10000).

146−

147−data Step s a = Done

148− | Skip !s

149− | Yield !a !s

150−

151125-- | /O(n)/ Convert a 'Text' into a 'Stream Char'.

152126stream :: Text -> Stream Char

153127stream (Text arr off len) = Stream next off len

@@ -182,7 +156,7 @@ reverseStream (Text arr off len) = Stream next (off+len-1) len

182156unstream :: Stream Char -> Text

183157unstream (Stream next0 s0 len)

184158 | len == 0 = I.empty

185− | otherwise = Text (P.fst a) 0 (P.snd a)

159+ | otherwise = I.textP (P.fst a) 0 (P.snd a)

186160 where

187161 a = runST (A.unsafeNew len >>= (\arr -> loop arr len s0 0))

188162 loop arr !top !s !i

@@ -198,26 +172,6 @@ unstream (Stream next0 s0 len)

198172{-# INLINE [0] unstream #-}

199173{-# RULES "STREAM stream/unstream fusion" forall s. stream (unstream s) = s #-}

200174

201−-- | The empty stream.

202−empty :: Stream Char

203−empty = Stream next () 0

204− where next _ = Done

205−{-# INLINE [0] empty #-}

206−

207−-- | /O(n)/ Determines if two streams are equal.

208−eq :: Ord a => Stream a -> Stream a -> Bool

209−eq (Stream next1 s1 _) (Stream next2 s2 _) = cmp (next1 s1) (next2 s2)

210− where

211− cmp Done Done = True

212− cmp Done _ = False

213− cmp _ Done = False

214− cmp (Skip s1') (Skip s2') = cmp (next1 s1') (next2 s2')

215− cmp (Skip s1') x2 = cmp (next1 s1') x2

216− cmp x1 (Skip s2') = cmp x1 (next2 s2')

217− cmp (Yield x1 s1') (Yield x2 s2') = x1 == x2 &&

218− cmp (next1 s1') (next2 s2')

219−{-# SPECIALISE eq :: Stream Char -> Stream Char -> Bool #-}

220−

221175streamError :: String -> String -> a

222176streamError func msg = P.error $ "Data.Text.Fusion." ++ func ++ ": " ++ msg

223177

@@ -403,7 +357,7 @@ intersperse c (Stream next0 s0 len) = Stream next (s0 :!: Nothing :!: S1) len

403357reverse :: Stream Char -> Text

404358reverse (Stream next s len0)

405359 | len0 == 0 = I.empty

406− | otherwise = Text arr off' len'

360+ | otherwise = I.textP arr off' len'

407361 where

408362 len0' = max len0 4

409363 (arr, (off', len')) = A.run2 (A.unsafeNew len0' >>= loop s (len0'-1) len0')

Data/Text/Internal.hs+8

@@ -21,6 +21,7 @@ module Data.Text.Internal

2121 Text(..)

2222 -- * Construction

2323 , text

24+ , textP

2425 -- * Code that must be here for accessibility

2526 , empty

2627 -- * Debugging

@@ -56,6 +57,13 @@ empty :: Text

5657empty = Text A.empty 0 0

5758{-# INLINE [1] empty #-}

5859

60+-- | Construct a 'Text' without invisibly pinning its byte array in

61+-- memory if its length has dwindled to zero.

62+textP :: A.Array Word16 -> Int -> Int -> Text

63+textP arr off len | len == 0 = empty

64+ | otherwise = text arr off len

65+{-# INLINE textP #-}

66+

5967-- | A useful 'show'-like function for debugging purposes.

6068showText :: Text -> String

6169showText (Text arr off len) =

Data/Text/Lazy/Internal.hsnew file+95

@@ -0,0 +1,95 @@

1+{-# LANGUAGE BangPatterns, DeriveDataTypeable #-}

2+-- |

3+-- Module : Data.Text.Lazy.Internal

4+-- Copyright : (c) Bryan O'Sullivan 2009

5+--

6+-- License : BSD-style

7+-- Maintainer : bos@serpentine.com

8+-- Stability : experimental

9+-- Portability : GHC

10+--

11+-- A module containing semi-public 'Text' internals. This exposes the

12+-- 'Text' representation and low level construction functions.

13+-- Modules which extend the 'Text' system may need to use this module.

14+-- Regular users should not.

15+module Data.Text.Lazy.Internal

16+ (

17+ Text(..)

18+ , chunk

19+ , foldrChunks

20+ , foldlChunks

21+ -- * Data type invariant and abstraction functions

22+ , invariant

23+ , checkInvariant

24+ , showStructure

25+ -- * Chunk allocation sizes

26+ , defaultChunkSize

27+ , smallChunkSize

28+ , chunkOverhead

29+ ) where

30+

31+import qualified Data.Text.Internal as T

32+import qualified Data.Text as T

33+import Data.Typeable (Typeable)

34+import Data.Word (Word16)

35+import Foreign.Storable (sizeOf)

36+

37+data Text = Empty

38+ | Chunk {-# UNPACK #-} !T.Text Text

39+ deriving (Typeable)

40+

41+-- | The data type invariant: Every 'Text' is either 'Empty' or

42+-- consists of non-null 'T.Text's. All functions must preserve this,

43+-- and the QC properties must check this.

44+invariant :: Text -> Bool

45+invariant Empty = True

46+invariant (Chunk (T.Text _ _ len) cs) = len > 0 && invariant cs

47+

48+showStructure :: Text -> String

49+showStructure Empty = "Empty"

50+showStructure (Chunk t Empty) = "Chunk " ++ show t ++ " Empty"

51+showStructure (Chunk t ts) =

52+ "Chunk " ++ show t ++ " (" ++ showStructure ts ++ ")"

53+

54+-- | In a form that checks the invariant lazily.

55+checkInvariant :: Text -> Text

56+checkInvariant Empty = Empty

57+checkInvariant (Chunk c@(T.Text _ _ len) cs)

58+ | len > 0 = Chunk c (checkInvariant cs)

59+ | otherwise = error $ "Data.Text.Lazy: invariant violation: "

60+ ++ showStructure (Chunk c cs)

61+

62+-- | Smart constructor for 'Chunk'. Guarantees the data type invariant.

63+chunk :: T.Text -> Text -> Text

64+{-# INLINE chunk #-}

65+chunk t@(T.Text _ _ len) ts | len == 0 = ts

66+ | otherwise = Chunk t ts

67+

68+-- | Consume the chunks of a lazy 'Text' with a natural right fold.

69+foldrChunks :: (T.Text -> a -> a) -> a -> Text -> a

70+foldrChunks f z = go

71+ where go Empty = z

72+ go (Chunk c cs) = f c (go cs)

73+{-# INLINE foldrChunks #-}

74+

75+-- | Consume the chunks of a lazy 'Text' with a strict, tail-recursive,

76+-- accumulating left fold.

77+foldlChunks :: (a -> T.Text -> a) -> a -> Text -> a

78+foldlChunks f z = go z

79+ where go !a Empty = a

80+ go !a (Chunk c cs) = go (f a c) cs

81+{-# INLINE foldlChunks #-}

82+

83+-- | Currently set to 32k, less the memory management overhead.

84+defaultChunkSize :: Int

85+defaultChunkSize = 32 * k - chunkOverhead

86+ where k = 1024 `div` sizeOf (undefined :: Word16)

87+

88+-- | Currently set to 4k, less the memory management overhead.

89+smallChunkSize :: Int

90+smallChunkSize = 4 * k - chunkOverhead

91+ where k = 1024 `div` sizeOf (undefined :: Word16)

92+

93+-- | The memory management overhead. Currently this is tuned for GHC only.

94+chunkOverhead :: Int

95+chunkOverhead = 2 * sizeOf (undefined :: Int)

tests/Properties.hs+204−200

@@ -13,11 +13,11 @@ import Control.Monad

1313import Data.Word

1414import qualified Data.ByteString as B

1515import qualified Data.Text as T

16−import Data.Text (pack,unpack)

16+import qualified Data.Text.Lazy as TL

1717import qualified Data.Text.Encoding as E

1818import Control.Exception

1919import qualified Data.Text.Fusion as S

20−import Data.Text.Fusion (stream, unstream)

20+import qualified Data.Text.Lazy.Fusion as SL

2121import qualified Data.List as L

2222import System.IO.Unsafe

2323import Prelude hiding (catch)

@@ -33,18 +33,20 @@ crashy onException p = unsafePerformIO $

3333 let types = e :: SomeException

3434 in trace ("*** Exception: " ++ show e) return onException

3535

36−prop_pack_unpack s = (unpack . pack) s == s

37−prop_stream_unstream t = (unstream . stream) t == t

38−prop_reverse_stream t = (S.reverse . S.reverseStream) t == t

39−prop_singleton c = [c] == (unpack . T.singleton) c

36+prop_T_pack_unpack = (T.unpack . T.pack) `eq` id

37+prop_TL_pack_unpack = (TL.unpack . TL.pack) `eq` id

38+prop_T_stream_unstream = (S.unstream . S.stream) `eq` id

39+prop_TL_stream_unstream = (SL.unstream . SL.stream) `eq` id

40+prop_T_reverse_stream t = (S.reverse . S.reverseStream) t == t

41+prop_T_singleton c = [c] == (T.unpack . T.singleton) c

4042

41−prop_ascii t = E.decodeASCII (E.encodeUtf8 a) == a

43+prop_T_ascii t = E.decodeASCII (E.encodeUtf8 a) == a

4244 where a = T.map (\c -> chr (ord c `mod` 128)) t

43−prop_utf8 = (E.decodeUtf8 . E.encodeUtf8) `eq` id

44−prop_utf16LE = (E.decodeUtf16LE . E.encodeUtf16LE) `eq` id

45−prop_utf16BE = (E.decodeUtf16BE . E.encodeUtf16BE) `eq` id

46−prop_utf32LE = (E.decodeUtf32LE . E.encodeUtf32LE) `eq` id

47−prop_utf32BE = (E.decodeUtf32BE . E.encodeUtf32BE) `eq` id

45+prop_T_utf8 = (E.decodeUtf8 . E.encodeUtf8) `eq` id

46+prop_T_utf16LE = (E.decodeUtf16LE . E.encodeUtf16LE) `eq` id

47+prop_T_utf16BE = (E.decodeUtf16BE . E.encodeUtf16BE) `eq` id

48+prop_T_utf32LE = (E.decodeUtf32LE . E.encodeUtf32LE) `eq` id

49+prop_T_utf32BE = (E.decodeUtf32BE . E.encodeUtf32BE) `eq` id

4850

4951-- Do two functions give the same answer?

5052eq :: (Eq a) => (t -> a) -> (t -> a) -> t -> Bool

@@ -54,7 +56,7 @@ eqP :: (Eq a, Show a) => (String -> a) -> (T.Text -> a) -> String -> Word8 -> Bo

5456eqP a b s w = eq "orig" (a s) (b t) &&

5557 eq "head" (a sa) (b ta) &&

5658 eq "tail" (a sb) (b tb)

57− where t = pack s

59+ where t = T.pack s

5860 (sa,sb) = splitAt m s

5961 (ta,tb) = T.splitAt m t

6062 l = length s

@@ -71,92 +73,92 @@ eqEP a b e w = a s == b t &&

7173 (null sb || a sb == b tb)

7274 where (sa,sb) = splitAt m s

7375 (ta,tb) = T.splitAt m t

74− t = pack s

76+ t = T.pack s

7577 l = length s

7678 m | l == 0 = n

7779 | otherwise = n `mod` l

7880 n = fromIntegral w

7981 s = notEmpty e

8082

81−prop_cons x = (x:) `eqP` (unpack . T.cons x)

82−prop_snoc x = (++ [x]) `eqP` (unpack . (flip T.snoc) x)

83−prop_append s = (s++) `eqP` (unpack . T.append (pack s))

84−prop_appendS s = (s++) `eqP` (unpack . unstream . S.append (stream (pack s)) . stream)

85−prop_uncons s = uncons `eqP` (fmap (second unpack) . T.uncons)

83+prop_T_cons x = (x:) `eqP` (T.unpack . T.cons x)

84+prop_T_snoc x = (++ [x]) `eqP` (T.unpack . (flip T.snoc) x)

85+prop_T_append s = (s++) `eqP` (T.unpack . T.append (T.pack s))

86+prop_T_appendS s = (s++) `eqP` (T.unpack . S.unstream . S.append (S.stream (T.pack s)) . S.stream)

87+prop_T_uncons s = uncons `eqP` (fmap (second T.unpack) . T.uncons)

8688 where uncons (x:xs) = Just (x,xs)

8789 uncons _ = Nothing

8890 types = s :: String

89−prop_head = head `eqEP` T.head

90−prop_last = last `eqEP` T.last

91−prop_lastS = last `eqEP` (S.last . stream)

92−prop_tail = tail `eqEP` (unpack . T.tail)

93−prop_tailS = tail `eqEP` (unpack . unstream . S.tail . stream)

94−prop_init = init `eqEP` (unpack . T.init)

95−prop_initS = init `eqEP` (unpack . unstream . S.init . stream)

96−prop_null = null `eqP` T.null

97−prop_length = length `eqP` T.length

98−prop_map f = map f `eqP` (unpack . T.map f)

99−prop_intercalate c = L.intercalate c `eq` (unpack . T.intercalate (pack c) . map pack)

100−prop_intersperse c = L.intersperse c `eqP` (unpack . T.intersperse c)

101−prop_transpose = L.transpose `eq` (map unpack . T.transpose . map pack)

102−prop_reverse = L.reverse `eqP` (unpack . T.reverse)

103−prop_reverse_short n = L.reverse `eqP` (unpack . S.reverse . shorten n . stream)

104−

105−prop_foldl f z = L.foldl f z `eqP` (T.foldl f z)

91+prop_T_head = head `eqEP` T.head

92+prop_T_last = last `eqEP` T.last

93+prop_T_lastS = last `eqEP` (S.last . S.stream)

94+prop_T_tail = tail `eqEP` (T.unpack . T.tail)

95+prop_T_tailS = tail `eqEP` (T.unpack . S.unstream . S.tail . S.stream)

96+prop_T_init = init `eqEP` (T.unpack . T.init)

97+prop_T_initS = init `eqEP` (T.unpack . S.unstream . S.init . S.stream)

98+prop_T_null = null `eqP` T.null

99+prop_T_length = length `eqP` T.length

100+prop_T_map f = map f `eqP` (T.unpack . T.map f)

101+prop_T_intercalate c = L.intercalate c `eq` (T.unpack . T.intercalate (T.pack c) . map T.pack)

102+prop_T_intersperse c = L.intersperse c `eqP` (T.unpack . T.intersperse c)

103+prop_T_transpose = L.transpose `eq` (map T.unpack . T.transpose . map T.pack)

104+prop_T_reverse = L.reverse `eqP` (T.unpack . T.reverse)

105+prop_T_reverse_short n = L.reverse `eqP` (T.unpack . S.reverse . shorten n . S.stream)

106+

107+prop_T_foldl f z = L.foldl f z `eqP` (T.foldl f z)

106108 where types = f :: Char -> Char -> Char

107−prop_foldl' f z = L.foldl' f z `eqP` T.foldl' f z

109+prop_T_foldl' f z = L.foldl' f z `eqP` T.foldl' f z

108110 where types = f :: Char -> Char -> Char

109−prop_foldl1 f = L.foldl1 f `eqEP` T.foldl1 f

110−prop_foldl1' f = L.foldl1' f `eqEP` T.foldl1' f

111−prop_foldr f z = L.foldr f z `eqP` T.foldr f z

111+prop_T_foldl1 f = L.foldl1 f `eqEP` T.foldl1 f

112+prop_T_foldl1' f = L.foldl1' f `eqEP` T.foldl1' f

113+prop_T_foldr f z = L.foldr f z `eqP` T.foldr f z

112114 where types = f :: Char -> Char -> Char

113−prop_foldr1 f = L.foldr1 f `eqEP` T.foldr1 f

115+prop_T_foldr1 f = L.foldr1 f `eqEP` T.foldr1 f

114116

115−prop_concat = L.concat `eq` (unpack . T.concat . map pack)

116−prop_concatMap f = L.concatMap f `eqP` (unpack . T.concatMap (pack . f))

117−prop_any p = L.any p `eqP` T.any p

118−prop_all p = L.all p `eqP` T.all p

119−prop_maximum = L.maximum `eqEP` T.maximum

120−prop_minimum = L.minimum `eqEP` T.minimum

117+prop_T_concat = L.concat `eq` (T.unpack . T.concat . map T.pack)

118+prop_T_concatMap f = L.concatMap f `eqP` (T.unpack . T.concatMap (T.pack . f))

119+prop_T_any p = L.any p `eqP` T.any p

120+prop_T_all p = L.all p `eqP` T.all p

121+prop_T_maximum = L.maximum `eqEP` T.maximum

122+prop_T_minimum = L.minimum `eqEP` T.minimum

121123

122−prop_scanl f z = L.scanl f z `eqP` (unpack . T.scanl f z)

123−prop_scanl1 f = L.scanl1 f `eqP` (unpack . T.scanl1 f)

124−prop_scanr f z = L.scanr f z `eqP` (unpack . T.scanr f z)

125−prop_scanr1 f = L.scanr1 f `eqP` (unpack . T.scanr1 f)

124+prop_T_scanl f z = L.scanl f z `eqP` (T.unpack . T.scanl f z)

125+prop_T_scanl1 f = L.scanl1 f `eqP` (T.unpack . T.scanl1 f)

126+prop_T_scanr f z = L.scanr f z `eqP` (T.unpack . T.scanr f z)

127+prop_T_scanr1 f = L.scanr1 f `eqP` (T.unpack . T.scanr1 f)

126128

127−prop_mapAccumL f z = L.mapAccumL f z `eqP` (second unpack . T.mapAccumL f z)

129+prop_T_mapAccumL f z = L.mapAccumL f z `eqP` (second T.unpack . T.mapAccumL f z)

128130 where types = f :: Int -> Char -> (Int,Char)

129−prop_mapAccumR f z = L.mapAccumR f z `eqP` (second unpack . T.mapAccumR f z)

131+prop_T_mapAccumR f z = L.mapAccumR f z `eqP` (second T.unpack . T.mapAccumR f z)

130132 where types = f :: Int -> Char -> (Int,Char)

131133

132−prop_replicate n = L.replicate n `eq` (unpack . T.replicate n)

133−prop_unfoldr n = L.unfoldr f `eq` (unpack . T.unfoldr f)

134+prop_T_replicate n = L.replicate n `eq` (T.unpack . T.replicate n)

135+prop_T_unfoldr n = L.unfoldr f `eq` (T.unpack . T.unfoldr f)

134136 where f c | fromEnum c * 100 > n = Nothing

135137 | otherwise = Just (c, succ c)

136138

137−prop_unfoldrN n m = (L.take n . L.unfoldr f) `eq` (unpack . T.unfoldrN n f)

139+prop_T_unfoldrN n m = (L.take n . L.unfoldr f) `eq` (T.unpack . T.unfoldrN n f)

138140 where f c | fromEnum c * 100 > m = Nothing

139141 | otherwise = Just (c, succ c)

140142

141−unpack2 = unpack *** unpack

143+unpack2 = T.unpack *** T.unpack

142144

143−prop_take n = L.take n `eqP` (unpack . T.take n)

144−prop_drop n = L.drop n `eqP` (unpack . T.drop n)

145−prop_takeWhile p = L.takeWhile p `eqP` (unpack . T.takeWhile p)

146−prop_takeWhileS p = L.takeWhile p `eqP` (unpack . unstream . S.takeWhile p . stream)

147−prop_dropWhile p = L.dropWhile p `eqP` (unpack . T.dropWhile p)

148−prop_dropWhileS p = L.dropWhile p `eqP` (unpack . unstream . S.dropWhile p . stream)

149−prop_splitAt n = L.splitAt n `eqP` (unpack2 . T.splitAt n)

150−prop_span p = L.span p `eqP` (unpack2 . T.span p)

151−prop_break p = L.break p `eqP` (unpack2 . T.break p)

152−prop_group = L.group `eqP` (map unpack . T.group)

153−prop_groupBy p = L.groupBy p `eqP` (map unpack . T.groupBy p)

154−prop_inits = L.inits `eqP` (map unpack . T.inits)

155−prop_tails = L.tails `eqP` (map unpack . T.tails)

145+prop_T_take n = L.take n `eqP` (T.unpack . T.take n)

146+prop_T_drop n = L.drop n `eqP` (T.unpack . T.drop n)

147+prop_T_takeWhile p = L.takeWhile p `eqP` (T.unpack . T.takeWhile p)

148+prop_T_takeWhileS p = L.takeWhile p `eqP` (T.unpack . S.unstream . S.takeWhile p . S.stream)

149+prop_T_dropWhile p = L.dropWhile p `eqP` (T.unpack . T.dropWhile p)

150+prop_T_dropWhileS p = L.dropWhile p `eqP` (T.unpack . S.unstream . S.dropWhile p . S.stream)

151+prop_T_splitAt n = L.splitAt n `eqP` (unpack2 . T.splitAt n)

152+prop_T_span p = L.span p `eqP` (unpack2 . T.span p)

153+prop_T_break p = L.break p `eqP` (unpack2 . T.break p)

154+prop_T_group = L.group `eqP` (map T.unpack . T.group)

155+prop_T_groupBy p = L.groupBy p `eqP` (map T.unpack . T.groupBy p)

156+prop_T_inits = L.inits `eqP` (map T.unpack . T.inits)

157+prop_T_tails = L.tails `eqP` (map T.unpack . T.tails)

156158

157−prop_split_i c = id `eq` (T.intercalate (T.singleton c) . T.split c)

159+prop_T_split_i c = id `eq` (T.intercalate (T.singleton c) . T.split c)

158160

159−prop_splitWith p = splitWith p `eqP` (map unpack . T.splitWith p)

161+prop_T_splitWith p = splitWith p `eqP` (map T.unpack . T.splitWith p)

160162

161163splitWith _ "" = []

162164splitWith p s = if null s'

@@ -164,16 +166,16 @@ splitWith p s = if null s'

164166 else l : splitWith p (tail s')

165167 where (l, s') = break p s

166168

167−prop_breakSubstring_isInfixOf s l

169+prop_T_breakSubstring_isInfixOf s l

168170 = T.isInfixOf s l ==

169171 T.null s || (not . T.null . snd $ T.breakSubstring s l)

170−prop_breakSubstringC c

172+prop_T_breakSubstringC c

171173 = L.break (==c) `eqP`

172174 (unpack2 . T.breakSubstring (T.singleton c))

173175

174−prop_lines = L.lines `eqP` (map unpack . T.lines)

176+prop_T_lines = L.lines `eqP` (map T.unpack . T.lines)

175177{-

176−prop_lines' = lines' `eqP` (map unpack . T.lines')

178+prop_T_lines' = lines' `eqP` (map T.unpack . T.lines')

177179 where lines' "" = []

178180 lines' s = let (l, s') = break eol s

179181 in l : case s' of

@@ -182,28 +184,28 @@ prop_lines' = lines' `eqP` (map unpack . T.lines')

182184 (_:s'') -> lines' s''

183185 eol c = c == '\r' || c == '\n'

184186-}

185−prop_words = L.words `eqP` (map unpack . T.words)

186−prop_unlines = L.unlines `eq` (unpack . T.unlines . map pack)

187−prop_unwords = L.unwords `eq` (unpack . T.unwords . map pack)

188−

189−prop_isPrefixOf s = L.isPrefixOf s`eqP` T.isPrefixOf (pack s)

190−prop_isPrefixOfS s = L.isPrefixOf s`eqP` (S.isPrefixOf (stream $ pack s) . stream)

191−prop_isSuffixOf s = L.isSuffixOf s`eqP` T.isSuffixOf (pack s)

192−prop_isInfixOf s = L.isInfixOf s `eqP` T.isInfixOf (pack s)

193−

194−prop_elem c = L.elem c `eqP` T.elem c

195−prop_filter p = L.filter p `eqP` (unpack . T.filter p)

196−prop_find p = L.find p `eqP` T.find p

197−prop_partition p = L.partition p `eqP` (unpack2 . T.partition p)

198−

199−prop_index x s = x < L.length s && x >= 0 ==>

200− (L.!!) s x == T.index (pack s) x

201−prop_findIndex p = L.findIndex p `eqP` T.findIndex p

202−prop_findIndices p = L.findIndices p`eqP` T.findIndices p

203−prop_elemIndex c = L.elemIndex c `eqP` T.elemIndex c

204−prop_elemIndices c = L.elemIndices c`eqP` T.elemIndices c

205−prop_count c = (L.length . L.elemIndices c) `eqP` T.count c

206−prop_zipWith c s = L.zipWith c s `eqP` (unpack . T.zipWith c (pack s))

187+prop_T_words = L.words `eqP` (map T.unpack . T.words)

188+prop_T_unlines = L.unlines `eq` (T.unpack . T.unlines . map T.pack)

189+prop_T_unwords = L.unwords `eq` (T.unpack . T.unwords . map T.pack)

190+

191+prop_T_isPrefixOf s = L.isPrefixOf s`eqP` T.isPrefixOf (T.pack s)

192+prop_T_isPrefixOfS s = L.isPrefixOf s`eqP` (S.isPrefixOf (S.stream $ T.pack s) . S.stream)

193+prop_T_isSuffixOf s = L.isSuffixOf s`eqP` T.isSuffixOf (T.pack s)

194+prop_T_isInfixOf s = L.isInfixOf s `eqP` T.isInfixOf (T.pack s)

195+

196+prop_T_elem c = L.elem c `eqP` T.elem c

197+prop_T_filter p = L.filter p `eqP` (T.unpack . T.filter p)

198+prop_T_find p = L.find p `eqP` T.find p

199+prop_T_partition p = L.partition p `eqP` (unpack2 . T.partition p)

200+

201+prop_T_index x s = x < L.length s && x >= 0 ==>

202+ (L.!!) s x == T.index (T.pack s) x

203+prop_T_findIndex p = L.findIndex p `eqP` T.findIndex p

204+prop_T_findIndices p = L.findIndices p`eqP` T.findIndices p

205+prop_T_elemIndex c = L.elemIndex c `eqP` T.elemIndex c

206+prop_T_elemIndices c = L.elemIndices c`eqP` T.elemIndices c

207+prop_T_count c = (L.length . L.elemIndices c) `eqP` T.count c

208+prop_T_zipWith c s = L.zipWith c s `eqP` (T.unpack . T.zipWith c (T.pack s))

207209

208210-- Make a stream appear shorter than it really is, to ensure that

209211-- functions that consume inaccurately sized streams behave

@@ -228,106 +230,108 @@ run tests args = do

228230

229231tests :: [(String, Int -> IO (Bool, Int))]

230232tests = [

231− ("prop_pack_unpack", mytest prop_pack_unpack),

232− ("prop_stream_unstream", mytest prop_stream_unstream),

233− ("prop_reverse_stream", mytest prop_reverse_stream),

234− ("prop_singleton", mytest prop_singleton),

235−

236− ("prop_ascii", mytest prop_ascii),

237− ("prop_utf8", mytest prop_utf8),

238− ("prop_utf16LE", mytest prop_utf16LE),

239− ("prop_utf16BE", mytest prop_utf16BE),

240− ("prop_utf32LE", mytest prop_utf32LE),

241− ("prop_utf32BE", mytest prop_utf32BE),

242−

243− ("prop_cons", mytest prop_cons),

244− ("prop_snoc", mytest prop_snoc),

245− ("prop_append", mytest prop_append),

246− ("prop_appendS", mytest prop_appendS),

247− ("prop_uncons", mytest prop_uncons),

248− ("prop_head", mytest prop_head),

249− ("prop_last", mytest prop_last),

250− ("prop_lastS", mytest prop_lastS),

251− ("prop_tail", mytest prop_tail),

252− ("prop_tailS", mytest prop_tailS),

253− ("prop_init", mytest prop_init),

254− ("prop_initS", mytest prop_initS),

255− ("prop_null", mytest prop_null),

256− ("prop_length", mytest prop_length),

257−

258− ("prop_map", mytest prop_map),

259− ("prop_intercalate", mytest prop_intercalate),

260− ("prop_intersperse", mytest prop_intersperse),

261− ("prop_transpose", mytest prop_transpose),

262− ("prop_reverse", mytest prop_reverse),

263− ("prop_reverse_short", mytest prop_reverse_short),

264−

265− ("prop_foldl", mytest prop_foldl),

266− ("prop_foldl'", mytest prop_foldl'),

267− ("prop_foldl1", mytest prop_foldl1),

268− ("prop_foldl1'", mytest prop_foldl1'),

269− ("prop_foldr", mytest prop_foldr),

270− ("prop_foldr1", mytest prop_foldr1),

271−

272− ("prop_concat", mytest prop_concat),

273− ("prop_concatMap", mytest prop_concatMap),

274− ("prop_any", mytest prop_any),

275− ("prop_all", mytest prop_all),

276− ("prop_maximum", mytest prop_maximum),

277− ("prop_minimum", mytest prop_minimum),

278−

279− ("prop_scanl", mytest prop_scanl),

280− ("prop_scanl1", mytest prop_scanl1),

281− ("prop_scanr", mytest prop_scanr),

282− ("prop_scanr1", mytest prop_scanr1),

283−

284− ("prop_mapAccumL", mytest prop_mapAccumL),

285− ("prop_mapAccumR", mytest prop_mapAccumR),

286−

287− ("prop_replicate", mytest prop_replicate),

288− ("prop_unfoldr", mytest prop_unfoldr),

289− ("prop_unfoldrN", mytest prop_unfoldrN),

290−

291− ("prop_take", mytest prop_take),

292− ("prop_drop", mytest prop_drop),

293− ("prop_takeWhile", mytest prop_takeWhile),

294− ("prop_takeWhileS", mytest prop_takeWhileS),

295− ("prop_dropWhile", mytest prop_dropWhile),

296− ("prop_dropWhileS", mytest prop_dropWhileS),

297− ("prop_splitAt", mytest prop_splitAt),

298− ("prop_span", mytest prop_span),

299− ("prop_break", mytest prop_break),

300− ("prop_group", mytest prop_group),

301− ("prop_groupBy", mytest prop_groupBy),

302− ("prop_inits", mytest prop_inits),

303− ("prop_tails", mytest prop_tails),

304−

305− ("prop_split_i", mytest prop_split_i),

306− ("prop_splitWith", mytest prop_splitWith),

307− ("prop_breakSubstringC", mytest prop_breakSubstringC),

308− ("prop_breakSubstring_isInfixOf", mytest prop_breakSubstring_isInfixOf),

309−

310− ("prop_lines", mytest prop_lines),

311−--("prop_lines'", mytest prop_lines'),

312− ("prop_words", mytest prop_words),

313− ("prop_unlines", mytest prop_unlines),

314− ("prop_unwords", mytest prop_unwords),

315−

316− ("prop_isPrefixOf", mytest prop_isPrefixOf),

317− ("prop_isPrefixOfS", mytest prop_isPrefixOfS),

318− ("prop_isSuffixOf", mytest prop_isSuffixOf),

319− ("prop_isInfixOf", mytest prop_isInfixOf),

320−

321− ("prop_elem", mytest prop_elem),

322− ("prop_filter", mytest prop_filter),

323− ("prop_find", mytest prop_find),

324− ("prop_partition", mytest prop_partition),

325−

326− ("prop_index", mytest prop_index),

327− ("prop_findIndex", mytest prop_findIndex),

328− ("prop_findIndices", mytest prop_findIndices),

329− ("prop_elemIndex", mytest prop_elemIndex),

330− ("prop_elemIndices", mytest prop_elemIndices),

331− ("prop_count", mytest prop_count),

332− ("prop_zipWith", mytest prop_zipWith)

233+ ("prop_T_pack_unpack", mytest prop_T_pack_unpack),

234+ ("prop_TL_pack_unpack", mytest prop_TL_pack_unpack),

235+ ("prop_T_stream_unstream", mytest prop_T_stream_unstream),

236+ ("prop_TL_stream_unstream", mytest prop_TL_stream_unstream),

237+ ("prop_T_reverse_stream", mytest prop_T_reverse_stream),

238+ ("prop_T_singleton", mytest prop_T_singleton),

239+

240+ ("prop_T_ascii", mytest prop_T_ascii),

241+ ("prop_T_utf8", mytest prop_T_utf8),

242+ ("prop_T_utf16LE", mytest prop_T_utf16LE),

243+ ("prop_T_utf16BE", mytest prop_T_utf16BE),

244+ ("prop_T_utf32LE", mytest prop_T_utf32LE),

245+ ("prop_T_utf32BE", mytest prop_T_utf32BE),

246+

247+ ("prop_T_cons", mytest prop_T_cons),

248+ ("prop_T_snoc", mytest prop_T_snoc),

249+ ("prop_T_append", mytest prop_T_append),

250+ ("prop_T_appendS", mytest prop_T_appendS),

251+ ("prop_T_uncons", mytest prop_T_uncons),

252+ ("prop_T_head", mytest prop_T_head),

253+ ("prop_T_last", mytest prop_T_last),

254+ ("prop_T_lastS", mytest prop_T_lastS),

255+ ("prop_T_tail", mytest prop_T_tail),

256+ ("prop_T_tailS", mytest prop_T_tailS),

257+ ("prop_T_init", mytest prop_T_init),

258+ ("prop_T_initS", mytest prop_T_initS),

259+ ("prop_T_null", mytest prop_T_null),

260+ ("prop_T_length", mytest prop_T_length),

261+

262+ ("prop_T_map", mytest prop_T_map),

263+ ("prop_T_intercalate", mytest prop_T_intercalate),

264+ ("prop_T_intersperse", mytest prop_T_intersperse),

265+ ("prop_T_transpose", mytest prop_T_transpose),

266+ ("prop_T_reverse", mytest prop_T_reverse),

267+ ("prop_T_reverse_short", mytest prop_T_reverse_short),

268+

269+ ("prop_T_foldl", mytest prop_T_foldl),

270+ ("prop_T_foldl'", mytest prop_T_foldl'),

271+ ("prop_T_foldl1", mytest prop_T_foldl1),

272+ ("prop_T_foldl1'", mytest prop_T_foldl1'),

273+ ("prop_T_foldr", mytest prop_T_foldr),

274+ ("prop_T_foldr1", mytest prop_T_foldr1),

275+

276+ ("prop_T_concat", mytest prop_T_concat),

277+ ("prop_T_concatMap", mytest prop_T_concatMap),

278+ ("prop_T_any", mytest prop_T_any),

279+ ("prop_T_all", mytest prop_T_all),

280+ ("prop_T_maximum", mytest prop_T_maximum),

281+ ("prop_T_minimum", mytest prop_T_minimum),

282+

283+ ("prop_T_scanl", mytest prop_T_scanl),

284+ ("prop_T_scanl1", mytest prop_T_scanl1),

285+ ("prop_T_scanr", mytest prop_T_scanr),

286+ ("prop_T_scanr1", mytest prop_T_scanr1),

287+

288+ ("prop_T_mapAccumL", mytest prop_T_mapAccumL),

289+ ("prop_T_mapAccumR", mytest prop_T_mapAccumR),

290+

291+ ("prop_T_replicate", mytest prop_T_replicate),

292+ ("prop_T_unfoldr", mytest prop_T_unfoldr),

293+ ("prop_T_unfoldrN", mytest prop_T_unfoldrN),

294+

295+ ("prop_T_take", mytest prop_T_take),

296+ ("prop_T_drop", mytest prop_T_drop),

297+ ("prop_T_takeWhile", mytest prop_T_takeWhile),

298+ ("prop_T_takeWhileS", mytest prop_T_takeWhileS),

299+ ("prop_T_dropWhile", mytest prop_T_dropWhile),

300+ ("prop_T_dropWhileS", mytest prop_T_dropWhileS),

301+ ("prop_T_splitAt", mytest prop_T_splitAt),

302+ ("prop_T_span", mytest prop_T_span),

303+ ("prop_T_break", mytest prop_T_break),

304+ ("prop_T_group", mytest prop_T_group),

305+ ("prop_T_groupBy", mytest prop_T_groupBy),

306+ ("prop_T_inits", mytest prop_T_inits),

307+ ("prop_T_tails", mytest prop_T_tails),

308+

309+ ("prop_T_split_i", mytest prop_T_split_i),

310+ ("prop_T_splitWith", mytest prop_T_splitWith),

311+ ("prop_T_breakSubstringC", mytest prop_T_breakSubstringC),

312+ ("prop_T_breakSubstring_isInfixOf", mytest prop_T_breakSubstring_isInfixOf),

313+

314+ ("prop_T_lines", mytest prop_T_lines),

315+--("prop_T_lines'", mytest prop_T_lines'),

316+ ("prop_T_words", mytest prop_T_words),

317+ ("prop_T_unlines", mytest prop_T_unlines),

318+ ("prop_T_unwords", mytest prop_T_unwords),

319+

320+ ("prop_T_isPrefixOf", mytest prop_T_isPrefixOf),

321+ ("prop_T_isPrefixOfS", mytest prop_T_isPrefixOfS),

322+ ("prop_T_isSuffixOf", mytest prop_T_isSuffixOf),

323+ ("prop_T_isInfixOf", mytest prop_T_isInfixOf),

324+

325+ ("prop_T_elem", mytest prop_T_elem),

326+ ("prop_T_filter", mytest prop_T_filter),

327+ ("prop_T_find", mytest prop_T_find),

328+ ("prop_T_partition", mytest prop_T_partition),

329+

330+ ("prop_T_index", mytest prop_T_index),

331+ ("prop_T_findIndex", mytest prop_T_findIndex),

332+ ("prop_T_findIndices", mytest prop_T_findIndices),

333+ ("prop_T_elemIndex", mytest prop_T_elemIndex),

334+ ("prop_T_elemIndices", mytest prop_T_elemIndices),

335+ ("prop_T_count", mytest prop_T_count),

336+ ("prop_T_zipWith", mytest prop_T_zipWith)

333337 ]

tests/QuickCheckUtils.hs+9

@@ -5,6 +5,7 @@ module QuickCheckUtils where

55import Data.List

66import Data.Word

77import qualified Data.Text as T

8+import qualified Data.Text.Lazy as TL

89import System.IO

910import System.Random

1011import Test.QuickCheck

@@ -43,6 +44,10 @@ instance Arbitrary T.Text where

4344 arbitrary = T.pack `fmap` arbitrary

4445 coarbitrary s = coarbitrary (T.unpack s)

4546

47+instance Arbitrary TL.Text where

48+ arbitrary = TL.pack `fmap` arbitrary

49+ coarbitrary s = coarbitrary (TL.unpack s)

50+

4651newtype NotEmpty a = NotEmpty { notEmpty :: a }

4752 deriving (Eq, Ord, Show)

4853

@@ -57,6 +62,10 @@ instance Arbitrary (NotEmpty T.Text) where

5762 arbitrary = (fmap T.pack) `fmap` arbitrary

5863 coarbitrary = coarbitrary . notEmpty

5964

65+instance Arbitrary (NotEmpty TL.Text) where

66+ arbitrary = (fmap TL.pack) `fmap` arbitrary

67+ coarbitrary = coarbitrary . notEmpty

68+

6069instance Arbitrary (NotEmpty B.ByteString) where

6170 arbitrary = (fmap B.pack) `fmap` arbitrary

6271 coarbitrary = coarbitrary . notEmpty

text.cabal+4

@@ -21,9 +21,13 @@ library

2121 Data.Text.Encoding.Fusion

2222 Data.Text.Foreign

2323 Data.Text.Fusion

24+ Data.Text.Lazy

25+ Data.Text.Lazy.Fusion

2426 other-modules:

2527 Data.Text.Array

2628 Data.Text.Internal

29+ Data.Text.Fusion.Internal

30+ Data.Text.Lazy.Internal

2731 Data.Text.Unsafe

2832 Data.Text.UnsafeChar

2933 Data.Text.Encoding.Utf8