bdbd24b · text · git.iko.soy

git.iko.soy/text

Merge

commitbdbd24b7dcd405658c4ad68f6a1ecc6cfce2a350 authorBryan O'Sullivan <bos@serpentine.com> dateThu 2 Dec 2010, 21:01 (16 years ago) parentdc14c00 3f884e1

15 files changed+375−218

compare with master

unifiedsplit

.hgignore+2−2 .hgtags+2 Data/Text.hs+69−67 Data/Text/Encoding.hs+38−2 Data/Text/IO.hs+22 Data/Text/Lazy.hs+66−66 Data/Text/Lazy/Encoding.hs+36−2 Data/Text/Lazy/IO.hs+21 Data/Text/Lazy/Read.hs+6−4 Data/Text/Read.hs+7−5 tests/Properties.hs+62−62 tests/SlowFunctions.hs+7−7 tests/benchmarks/CaseMap.hs+21 tests/benchmarks/casemap.py+15 text.cabal+1−1

.hgignore+2−2

@@ -1,7 +1,7 @@

11^dist$

22^tests/benchmarks/.*\.txt$

3−^tests/(?:qc|text/test)$

4−\.(?:aux|eventlog|h[ip]|log|[oa]|orig|prof|ps|swp)$

4+\.(?:aux|eventlog|h[ip]|log|[oa]|orig|prof|ps|rej|swp)$

55~$

66syntax: glob

77.\#*

.hgtags+2

@@ -11,4 +11,6 @@ b4fdbef1c1027a70b95c1a81747d968890acc3f6 0.3

1111de7cb0cb2a72606126c357b860bb0dbf7f145ad2 0.9.0.1

1212f2c9b1c019e3277926378d7b5a604f1c995f0202 0.7

1313f3d625f2d0730cb6adf8506a1f37762178c9aea6 0.10.0.0

14+99ea36957b1d47145d70f2ee8b13f1ddfd09245c 0.10.0.1

14151b7889216e78b00d69b32c48782111bcd935978d 0.10.0.2

16+75157eaec6eb38574ad5c347e75c48dcb7262dee 0.11.0.0

Data/Text.hs+69−67

@@ -128,10 +128,10 @@ module Data.Text

128128 , stripStart

129129 , stripEnd

130130 , splitAt

131− , spanBy

131+ , breakOn

132+ , breakOnEnd

132133 , break

133− , breakEnd

134− , breakBy

134+ , span

135135 , group

136136 , groupBy

137137 , inits

@@ -139,8 +139,8 @@ module Data.Text

139139

140140 -- ** Breaking into many substrings

141141 -- $split

142+ , splitOn

142143 , split

143− , splitBy

144144 , chunksOf

145145

146146 -- ** Breaking into lines and words

@@ -161,9 +161,9 @@ module Data.Text

161161

162162 -- * Searching

163163 , filter

164+ , breakOnAll

164165 , find

165− , findBy

166− , partitionBy

166+ , partition

167167

168168 -- , findSubstring

169169

@@ -598,7 +598,7 @@ replace :: Text -- ^ Text to search for

598598 -> Text -- ^ Replacement text

599599 -> Text -- ^ Input text

600600 -> Text

601−replace s d = intercalate d . split s

601+replace s d = intercalate d . splitOn s

602602{-# INLINE replace #-}

603603

604604-- ----------------------------------------------------------------------------

@@ -619,7 +619,8 @@ replace s d = intercalate d . split s

619619-- context-dependent operation. The case conversion functions in this

620620-- module are /not/ locale sensitive. Programs that require locale

621621-- sensitivity should use appropriate versions of the case mapping

622−-- functions from the @text-icu@ package.

622+-- functions from the @text-icu@ package:

623+-- <http://hackage.haskell.org/package/text-icu>

623624

624625-- | /O(n)/ Convert a string to folded case. This function is mainly

625626-- useful for performing caseless (also known as case insensitive)

@@ -1072,23 +1073,23 @@ splitAt n t@(Text arr off len)

10721073 where d = iter_ t i

10731074{-# INLINE splitAt #-}

10741075

1075−-- | /O(n)/ 'spanBy', applied to a predicate @p@ and text @t@, returns

1076+-- | /O(n)/ 'span', applied to a predicate @p@ and text @t@, returns

10761077-- a pair whose first element is the longest prefix (possibly empty)

10771078-- of @t@ of elements that satisfy @p@, and whose second is the

10781079-- remainder of the list.

1079−spanBy :: (Char -> Bool) -> Text -> (Text, Text)

1080−spanBy p t@(Text arr off len) = (textP arr off k, textP arr (off+k) (len-k))

1080+span :: (Char -> Bool) -> Text -> (Text, Text)

1081+span p t@(Text arr off len) = (textP arr off k, textP arr (off+k) (len-k))

10811082 where k = loop 0

10821083 loop !i | i >= len || not (p c) = i

10831084 | otherwise = loop (i+d)

10841085 where Iter c d = iter t i

1085−{-# INLINE spanBy #-}

1086+{-# INLINE span #-}

10861087

1087−-- | /O(n)/ 'breakBy' is like 'spanBy', but the prefix returned is

1088+-- | /O(n)/ 'break' is like 'span', but the prefix returned is

10881089-- over elements that fail the predicate @p@.

1089−breakBy :: (Char -> Bool) -> Text -> (Text, Text)

1090−breakBy p = spanBy (not . p)

1091−{-# INLINE breakBy #-}

1090+break :: (Char -> Bool) -> Text -> (Text, Text)

1091+break p = span (not . p)

1092+{-# INLINE break #-}

10921093

10931094-- | /O(n)/ Group characters in a string according to a predicate.

10941095groupBy :: (Char -> Char -> Bool) -> Text -> [Text]

@@ -1138,30 +1139,30 @@ tails t | null t = [empty]

11381139--

11391140-- Examples:

11401141--

1141−-- > split "\r\n" "a\r\nb\r\nd\r\ne" == ["a","b","d","e"]

1142−-- > split "aaa" "aaaXaaaXaaaXaaa" == ["","X","X","X",""]

1143−-- > split "x" "x" == ["",""]

1142+-- > splitOn "\r\n" "a\r\nb\r\nd\r\ne" == ["a","b","d","e"]

1143+-- > splitOn "aaa" "aaaXaaaXaaaXaaa" == ["","X","X","X",""]

1144+-- > splitOn "x" "x" == ["",""]

11441145--

11451146-- and

11461147--

1147−-- > intercalate s . split s == id

1148−-- > split (singleton c) == splitBy (==c)

1148+-- > intercalate s . splitOn s == id

1149+-- > splitOn (singleton c) == split (==c)

11491150--

11501151-- In (unlikely) bad cases, this function's time complexity degrades

11511152-- towards /O(n*m)/.

1152−split :: Text -> Text -> [Text]

1153−split pat@(Text _ _ l) src@(Text arr off len)

1154− | l <= 0 = emptyError "split"

1155− | isSingleton pat = splitBy (== unsafeHead pat) src

1153+splitOn :: Text -> Text -> [Text]

1154+splitOn pat@(Text _ _ l) src@(Text arr off len)

1155+ | l <= 0 = emptyError "splitOn"

1156+ | isSingleton pat = split (== unsafeHead pat) src

11561157 | otherwise = go 0 (indices pat src)

11571158 where

11581159 go !s (x:xs) = textP arr (s+off) (x-s) : go (x+l) xs

11591160 go s _ = [textP arr (s+off) (len-s)]

1160−{-# INLINE [1] split #-}

1161+{-# INLINE [1] splitOn #-}

11611162

11621163{-# RULES

1163−"TEXT split/singleton -> splitBy/==" [~1] forall c t.

1164− split (singleton c) t = splitBy (==c) t

1164+"TEXT splitOn/singleton -> split/==" [~1] forall c t.

1165+ splitOn (singleton c) t = split (==c) t

11651166 #-}

11661167

11671168-- | /O(n)/ Splits a 'Text' into components delimited by separators,

@@ -1169,15 +1170,15 @@ split pat@(Text _ _ l) src@(Text arr off len)

11691170-- resulting components do not contain the separators. Two adjacent

11701171-- separators result in an empty component in the output. eg.

11711172--

1172−-- > splitBy (=='a') "aabbaca" == ["","","bb","c",""]

1173−-- > splitBy (=='a') "" == [""]

1174−splitBy :: (Char -> Bool) -> Text -> [Text]

1175−splitBy _ t@(Text _off _arr 0) = [t]

1176−splitBy p t = loop t

1173+-- > split (=='a') "aabbaca" == ["","","bb","c",""]

1174+-- > split (=='a') "" == [""]

1175+split :: (Char -> Bool) -> Text -> [Text]

1176+split _ t@(Text _off _arr 0) = [t]

1177+split p t = loop t

11771178 where loop s | null s' = [l]

11781179 | otherwise = l : loop (unsafeTail s')

1179− where (l, s') = breakBy p s

1180−{-# INLINE splitBy #-}

1180+ where (l, s') = break p s

1181+{-# INLINE split #-}

11811182

11821183-- | /O(n)/ Splits a 'Text' into components of length @k@. The last

11831184-- element may be shorter than the other chunks, depending on the

@@ -1199,21 +1200,21 @@ chunksOf k = go

11991200-------------------------------------------------------------------------------

12001201-- ** Searching with a predicate

12011202

1202−-- | /O(n)/ The 'findBy' function takes a predicate and a 'Text', and

1203−-- returns the first element in matching the predicate, or 'Nothing'

1204−-- if there is no such element.

1205−findBy :: (Char -> Bool) -> Text -> Maybe Char

1206−findBy p t = S.findBy p (stream t)

1207−{-# INLINE findBy #-}

1203+-- | /O(n)/ The 'find' function takes a predicate and a 'Text', and

1204+-- returns the first element matching the predicate, or 'Nothing' if

1205+-- there is no such element.

1206+find :: (Char -> Bool) -> Text -> Maybe Char

1207+find p t = S.findBy p (stream t)

1208+{-# INLINE find #-}

12081209

1209−-- | /O(n)/ The 'partitionBy' function takes a predicate and a 'Text',

1210+-- | /O(n)/ The 'partition' function takes a predicate and a 'Text',

12101211-- and returns the pair of 'Text's with elements which do and do not

12111212-- satisfy the predicate, respectively; i.e.

12121213--

1213−-- > partitionBy p t == (filter p t, filter (not . p) t)

1214−partitionBy :: (Char -> Bool) -> Text -> (Text, Text)

1215−partitionBy p t = (filter p t, filter (not . p) t)

1216−{-# INLINE partitionBy #-}

1214+-- > partition p t == (filter p t, filter (not . p) t)

1215+partition :: (Char -> Bool) -> Text -> (Text, Text)

1216+partition p t = (filter p t, filter (not . p) t)

1217+{-# INLINE partition #-}

12171218

12181219-- | /O(n)/ 'filter', applied to a predicate and a 'Text',

12191220-- returns a 'Text' containing those characters that satisfy the

@@ -1229,39 +1230,40 @@ filter p t = unstream (S.filter p (stream t))

12291230--

12301231-- Examples:

12311232--

1232−-- > break "::" "a::b::c" ==> ("a", "::b::c")

1233−-- > break "/" "foobar" ==> ("foobar", "")

1233+-- > breakOn "::" "a::b::c" ==> ("a", "::b::c")

1234+-- > breakOn "/" "foobar" ==> ("foobar", "")

12341235--

12351236-- Laws:

12361237--

12371238-- > append prefix match == haystack

1238−-- > where (prefix, match) = break needle haystack

1239+-- > where (prefix, match) = breakOn needle haystack

12391240--

12401241-- If you need to break a string by a substring repeatedly (e.g. you

1241−-- want to break on every instance of a substring), use 'find'

1242+-- want to break on every instance of a substring), use 'breakOnAll'

12421243-- instead, as it has lower startup overhead.

12431244--

12441245-- In (unlikely) bad cases, this function's time complexity degrades

12451246-- towards /O(n*m)/.

1246−break :: Text -> Text -> (Text, Text)

1247−break pat src@(Text arr off len)

1248− | null pat = emptyError "break"

1247+breakOn :: Text -> Text -> (Text, Text)

1248+breakOn pat src@(Text arr off len)

1249+ | null pat = emptyError "breakOn"

12491250 | otherwise = case indices pat src of

12501251 [] -> (src, empty)

12511252 (x:_) -> (textP arr off x, textP arr (off+x) (len-x))

1252−{-# INLINE break #-}

1253+{-# INLINE breakOn #-}

12531254

1254−-- | /O(n+m)/ Similar to 'break', but searches from the end of the string.

1255+-- | /O(n+m)/ Similar to 'breakOn', but searches from the end of the

1256+-- string.

12551257--

12561258-- The first element of the returned tuple is the prefix of @haystack@

12571259-- up to and including the last match of @needle@. The second is the

12581260-- remainder of @haystack@, following the match.

12591261--

1260−-- > breakEnd "::" "a::b::c" ==> ("a::b::", "c")

1261−breakEnd :: Text -> Text -> (Text, Text)

1262−breakEnd pat src = let (a,b) = break (reverse pat) (reverse src)

1263− in (reverse b, reverse a)

1264−{-# INLINE breakEnd #-}

1262+-- > breakOnEnd "::" "a::b::c" ==> ("a::b::", "c")

1263+breakOnEnd :: Text -> Text -> (Text, Text)

1264+breakOnEnd pat src = (reverse b, reverse a)

1265+ where (a,b) = breakOn (reverse pat) (reverse src)

1266+{-# INLINE breakOnEnd #-}

12651267

12661268-- | /O(n+m)/ Find all non-overlapping instances of @needle@ in

12671269-- @haystack@. Each element of the returned list consists of a pair:

@@ -1281,16 +1283,16 @@ breakEnd pat src = let (a,b) = break (reverse pat) (reverse src)

12811283-- towards /O(n*m)/.

12821284--

12831285-- The @needle@ parameter may not be empty.

1284−find :: Text -- ^ @needle@ to search for

1285− -> Text -- ^ @haystack@ in which to search

1286− -> [(Text, Text)]

1287−find pat src@(Text arr off slen)

1288− | null pat = emptyError "find"

1286+breakOnAll :: Text -- ^ @needle@ to search for

1287+ -> Text -- ^ @haystack@ in which to search

1288+ -> [(Text, Text)]

1289+breakOnAll pat src@(Text arr off slen)

1290+ | null pat = emptyError "breakOnAll"

12891291 | otherwise = L.map step (indices pat src)

12901292 where

12911293 step x = (chunk 0 x, chunk x (slen-x))

12921294 chunk !n !l = textP arr (n+off) l

1293−{-# INLINE find #-}

1295+{-# INLINE breakOnAll #-}

12941296

12951297-------------------------------------------------------------------------------

12961298-- ** Indexing 'Text's

@@ -1390,7 +1392,7 @@ lines ps | null ps = []

13901392 | otherwise = h : if null t

13911393 then []

13921394 else lines (unsafeTail t)

1393− where (h,t) = spanBy (/= '\n') ps

1395+ where (h,t) = span (/= '\n') ps

13941396{-# INLINE lines #-}

13951397

13961398{-

Data/Text/Encoding.hs+38−2

@@ -14,18 +14,20 @@

1414-- Functions for converting 'Text' values to and from 'ByteString',

1515-- using several standard encodings.

1616--

17−-- To make use of a much larger variety of encodings, use the @text-icu@

18−-- package.

17+-- To gain access to a much larger family of encodings, use the

18+-- @text-icu@ package: <http://hackage.haskell.org/package/text-icu>

1919

2020module Data.Text.Encoding

2121 (

2222 -- * Decoding ByteStrings to Text

23+ -- $strict

2324 decodeASCII

2425 , decodeUtf8

2526 , decodeUtf16LE

2627 , decodeUtf16BE

2728 , decodeUtf32LE

2829 , decodeUtf32BE

30+

2931 -- ** Controllable error handling

3032 , decodeUtf8With

3133 , decodeUtf16LEWith

@@ -60,11 +62,25 @@ import qualified Data.Text.Encoding.Utf16 as U16

6062import qualified Data.Text.Encoding.Utf8 as U8

6163import qualified Data.Text.Fusion as F

6264

65+-- $strict

66+--

67+-- All of the single-parameter functions for decoding bytestrings

68+-- encoded in one of the Unicode Transformation Formats (UTF) operate

69+-- in a /strict/ mode: each will throw an exception if given invalid

70+-- input.

71+--

72+-- Each function has a variant, whose name is suffixed with -'With',

73+-- that gives greater control over the handling of decoding errors.

74+-- For instance, 'decodeUtf8' will throw an exception, but

75+-- 'decodeUtf8With' allows the programmer to determine what to do on a

76+-- decoding error.

77+

6378-- | Decode a 'ByteString' containing 7-bit ASCII encoded text.

6479decodeASCII :: ByteString -> Text

6580decodeASCII bs = F.unstream (E.streamASCII bs)

6681{-# INLINE decodeASCII #-}

6782

83+-- | Decode a 'ByteString' containing UTF-8 encoded text.

6884decodeUtf8With :: OnDecodeError -> ByteString -> Text

6985decodeUtf8With onErr bs = textP (fst a) 0 (snd a)

7086 where

@@ -101,6 +117,10 @@ decodeUtf8With onErr bs = textP (fst a) 0 (snd a)

101117{-# INLINE[0] decodeUtf8With #-}

102118

103119-- | Decode a 'ByteString' containing UTF-8 encoded text.

120+--

121+-- If the input contains any invalid UTF-8 data, an exception will be

122+-- thrown. For more control over the handling of invalid data, use

123+-- 'decodeUtf8With'.

104124decodeUtf8 :: ByteString -> Text

105125decodeUtf8 = decodeUtf8With strictDecode

106126{-# INLINE[0] decodeUtf8 #-}

@@ -156,6 +176,10 @@ decodeUtf16LEWith onErr bs = F.unstream (E.streamUtf16LE onErr bs)

156176{-# INLINE decodeUtf16LEWith #-}

157177

158178-- | Decode text from little endian UTF-16 encoding.

179+--

180+-- If the input contains any invalid little endian UTF-16 data, an

181+-- exception will be thrown. For more control over the handling of

182+-- invalid data, use 'decodeUtf16LEWith'.

159183decodeUtf16LE :: ByteString -> Text

160184decodeUtf16LE = decodeUtf16LEWith strictDecode

161185{-# INLINE decodeUtf16LE #-}

@@ -166,6 +190,10 @@ decodeUtf16BEWith onErr bs = F.unstream (E.streamUtf16BE onErr bs)

166190{-# INLINE decodeUtf16BEWith #-}

167191

168192-- | Decode text from big endian UTF-16 encoding.

193+--

194+-- If the input contains any invalid big endian UTF-16 data, an

195+-- exception will be thrown. For more control over the handling of

196+-- invalid data, use 'decodeUtf16BEWith'.

169197decodeUtf16BE :: ByteString -> Text

170198decodeUtf16BE = decodeUtf16BEWith strictDecode

171199{-# INLINE decodeUtf16BE #-}

@@ -186,6 +214,10 @@ decodeUtf32LEWith onErr bs = F.unstream (E.streamUtf32LE onErr bs)

186214{-# INLINE decodeUtf32LEWith #-}

187215

188216-- | Decode text from little endian UTF-32 encoding.

217+--

218+-- If the input contains any invalid little endian UTF-32 data, an

219+-- exception will be thrown. For more control over the handling of

220+-- invalid data, use 'decodeUtf32LEWith'.

189221decodeUtf32LE :: ByteString -> Text

190222decodeUtf32LE = decodeUtf32LEWith strictDecode

191223{-# INLINE decodeUtf32LE #-}

@@ -196,6 +228,10 @@ decodeUtf32BEWith onErr bs = F.unstream (E.streamUtf32BE onErr bs)

196228{-# INLINE decodeUtf32BEWith #-}

197229

198230-- | Decode text from big endian UTF-32 encoding.

231+--

232+-- If the input contains any invalid big endian UTF-32 data, an

233+-- exception will be thrown. For more control over the handling of

234+-- invalid data, use 'decodeUtf32BEWith'.

199235decodeUtf32BE :: ByteString -> Text

200236decodeUtf32BE = decodeUtf32BEWith strictDecode

201237{-# INLINE decodeUtf32BE #-}

Data/Text/IO.hs+22

@@ -9,9 +9,15 @@

99-- Portability : GHC

1010--

1111-- Efficient locale-sensitive support for text I\/O.

12+--

13+-- Skip past the synopsis for some important notes on performance and

14+-- portability across different versions of GHC.

1215

1316module Data.Text.IO

1417 (

18+ -- * Performance

19+ -- $performance

20+

1521 -- * Locale support

1622 -- $locale

1723 -- * File-at-a-time operations

@@ -60,6 +66,22 @@ import System.IO (hGetBuffering, hFileSize, hSetBuffering, hTell)

6066import System.IO.Error (isEOFError)

6167#endif

6268

69+-- $performance

70+-- #performance#

71+--

72+-- The functions in this module obey the runtime system's locale,

73+-- character set encoding, and line ending conversion settings.

74+--

75+-- If you know in advance that you will be working with data that has

76+-- a specific encoding (e.g. UTF-8), and your application is highly

77+-- performance sensitive, you may find that it is faster to perform

78+-- I\/O with bytestrings and to encode and decode yourself than to use

79+-- the functions in this module.

80+--

81+-- Whether this will hold depends on the version of GHC you are using,

82+-- the platform you are working on, the data you are working with, and

83+-- the encodings you are using, so be sure to test for yourself.

84+

6385-- | The 'readFile' function reads a file and returns the contents of

6486-- the file as a string. The entire file is read strictly, as with

6587-- 'getContents'.

Data/Text/Lazy.hs+66−66

@@ -135,10 +135,10 @@ module Data.Text.Lazy

135135 , stripStart

136136 , stripEnd

137137 , splitAt

138− , spanBy

138+ , span

139+ , breakOn

140+ , breakOnEnd

139141 , break

140− , breakEnd

141− , breakBy

142142 , group

143143 , groupBy

144144 , inits

@@ -146,8 +146,8 @@ module Data.Text.Lazy

146146

147147 -- ** Breaking into many substrings

148148 -- $split

149+ , splitOn

149150 , split

150− , splitBy

151151 , chunksOf

152152 -- , breakSubstring

153153

@@ -169,8 +169,8 @@ module Data.Text.Lazy

169169 -- * Searching

170170 , filter

171171 , find

172− , findBy

173− , partitionBy

172+ , breakOnAll

173+ , partition

174174

175175 -- , findSubstring

176176

@@ -630,7 +630,7 @@ replace :: Text -- ^ Text to search for

630630 -> Text -- ^ Replacement text

631631 -> Text -- ^ Input text

632632 -> Text

633−replace s d = intercalate d . split s

633+replace s d = intercalate d . splitOn s

634634{-# INLINE replace #-}

635635

636636-- ----------------------------------------------------------------------------

@@ -1044,13 +1044,13 @@ splitAtWord x (Chunk c@(T.Text arr off len) cs)

10441044--

10451045-- Examples:

10461046--

1047−-- > break "::" "a::b::c" ==> ("a", "::b::c")

1048−-- > break "/" "foobar" ==> ("foobar", "")

1047+-- > breakOn "::" "a::b::c" ==> ("a", "::b::c")

1048+-- > breakOn "/" "foobar" ==> ("foobar", "")

10491049--

10501050-- Laws:

10511051--

10521052-- > append prefix match == haystack

1053−-- > where (prefix, match) = break needle haystack

1053+-- > where (prefix, match) = breakOn needle haystack

10541054--

10551055-- If you need to break a string by a substring repeatedly (e.g. you

10561056-- want to break on every instance of a substring), use 'find'

@@ -1061,25 +1061,25 @@ splitAtWord x (Chunk c@(T.Text arr off len) cs)

10611061--

10621062-- In (unlikely) bad cases, this function's time complexity degrades

10631063-- towards /O(n*m)/.

1064−break :: Text -> Text -> (Text, Text)

1065−break pat src

1066− | null pat = emptyError "break"

1064+breakOn :: Text -> Text -> (Text, Text)

1065+breakOn pat src

1066+ | null pat = emptyError "breakOn"

10671067 | otherwise = case indices pat src of

10681068 [] -> (src, empty)

10691069 (x:_) -> let h :*: t = splitAtWord x src

10701070 in (h, t)

10711071

1072−-- | /O(n+m)/ Similar to 'break', but searches from the end of the string.

1072+-- | /O(n+m)/ Similar to 'breakOn', but searches from the end of the string.

10731073--

10741074-- The first element of the returned tuple is the prefix of @haystack@

10751075-- up to and including the last match of @needle@. The second is the

10761076-- remainder of @haystack@, following the match.

10771077--

1078−-- > breakEnd "::" "a::b::c" ==> ("a::b::", "c")

1079−breakEnd :: Text -> Text -> (Text, Text)

1080−breakEnd pat src = let (a,b) = break (reverse pat) (reverse src)

1078+-- > breakOnEnd "::" "a::b::c" ==> ("a::b::", "c")

1079+breakOnEnd :: Text -> Text -> (Text, Text)

1080+breakOnEnd pat src = let (a,b) = breakOn (reverse pat) (reverse src)

10811081 in (reverse b, reverse a)

1082−{-# INLINE breakEnd #-}

1082+{-# INLINE breakOnEnd #-}

10831083

10841084-- | /O(n+m)/ Find all non-overlapping instances of @needle@ in

10851085-- @haystack@. Each element of the returned list consists of a pair:

@@ -1090,9 +1090,9 @@ breakEnd pat src = let (a,b) = break (reverse pat) (reverse src)

10901090--

10911091-- Examples:

10921092--

1093−-- > find "::" ""

1093+-- > breakOnAll "::" ""

10941094-- > ==> []

1095−-- > find "/" "a/b/c/"

1095+-- > breakOnAll "/" "a/b/c/"

10961096-- > ==> [("a", "/b/c/"), ("a/b", "/c/"), ("a/b/c", "/")]

10971097--

10981098-- This function is strict in its first argument, and lazy in its

@@ -1102,11 +1102,11 @@ breakEnd pat src = let (a,b) = break (reverse pat) (reverse src)

11021102-- towards /O(n*m)/.

11031103--

11041104-- The @needle@ parameter may not be empty.

1105−find :: Text -- ^ @needle@ to search for

1106− -> Text -- ^ @haystack@ in which to search

1107− -> [(Text, Text)]

1108−find pat src

1109− | null pat = emptyError "find"

1105+breakOnAll :: Text -- ^ @needle@ to search for

1106+ -> Text -- ^ @haystack@ in which to search

1107+ -> [(Text, Text)]

1108+breakOnAll pat src

1109+ | null pat = emptyError "breakOnAll"

11101110 | otherwise = go 0 empty src (indices pat src)

11111111 where

11121112 go !n p s (x:xs) = let h :*: t = splitAtWord (x-n) s

@@ -1114,10 +1114,10 @@ find pat src

11141114 in (h',t) : go x h' t xs

11151115 go _ _ _ _ = []

11161116

1117−-- | /O(n)/ 'breakBy' is like 'spanBy', but the prefix returned is over

1117+-- | /O(n)/ 'break' is like 'span', but the prefix returned is over

11181118-- elements that fail the predicate @p@.

1119−breakBy :: (Char -> Bool) -> Text -> (Text, Text)

1120−breakBy p t0 = break' t0

1119+break :: (Char -> Bool) -> Text -> (Text, Text)

1120+break p t0 = break' t0

11211121 where break' Empty = (empty, empty)

11221122 break' c@(Chunk t ts) =

11231123 case T.findIndex p t of

@@ -1127,13 +1127,13 @@ breakBy p t0 = break' t0

11271127 | otherwise -> let (a,b) = T.splitAt n t

11281128 in (Chunk a Empty, Chunk b ts)

11291129

1130−-- | /O(n)/ 'spanBy', applied to a predicate @p@ and text @t@, returns

1130+-- | /O(n)/ 'span', applied to a predicate @p@ and text @t@, returns

11311131-- a pair whose first element is the longest prefix (possibly empty)

11321132-- of @t@ of elements that satisfy @p@, and whose second is the

11331133-- remainder of the list.

1134−spanBy :: (Char -> Bool) -> Text -> (Text, Text)

1135−spanBy p = breakBy (not . p)

1136−{-# INLINE spanBy #-}

1134+span :: (Char -> Bool) -> Text -> (Text, Text)

1135+span p = break (not . p)

1136+{-# INLINE span #-}

11371137

11381138-- | The 'group' function takes a 'Text' and returns a list of 'Text's

11391139-- such that the concatenation of the result is equal to the argument.

@@ -1152,7 +1152,7 @@ group = groupBy (==)

11521152groupBy :: (Char -> Char -> Bool) -> Text -> [Text]

11531153groupBy _ Empty = []

11541154groupBy eq (Chunk t ts) = cons x ys : groupBy eq zs

1155− where (ys,zs) = spanBy (eq x) xs

1155+ where (ys,zs) = span (eq x) xs

11561156 x = T.unsafeHead t

11571157 xs = chunk (T.unsafeTail t) ts

11581158

@@ -1184,37 +1184,37 @@ tails ts@(Chunk t ts')

11841184--

11851185-- Examples:

11861186--

1187−-- > split "\r\n" "a\r\nb\r\nd\r\ne" == ["a","b","d","e"]

1188−-- > split "aaa" "aaaXaaaXaaaXaaa" == ["","X","X","X",""]

1189−-- > split "x" "x" == ["",""]

1187+-- > splitOn "\r\n" "a\r\nb\r\nd\r\ne" == ["a","b","d","e"]

1188+-- > splitOn "aaa" "aaaXaaaXaaaXaaa" == ["","X","X","X",""]

1189+-- > splitOn "x" "x" == ["",""]

11901190--

11911191-- and

11921192--

1193−-- > intercalate s . split s == id

1194−-- > split (singleton c) == splitBy (==c)

1193+-- > intercalate s . splitOn s == id

1194+-- > splitOn (singleton c) == split (==c)

11951195--

11961196-- This function is strict in its first argument, and lazy in its

11971197-- second.

11981198--

11991199-- In (unlikely) bad cases, this function's time complexity degrades

12001200-- towards /O(n*m)/.

1201−split :: Text -- ^ Text to split on

1202− -> Text -- ^ Input text

1203− -> [Text]

1204−split pat src

1205− | null pat = emptyError "split"

1206− | isSingleton pat = splitBy (== head pat) src

1201+splitOn :: Text -- ^ Text to split on

1202+ -> Text -- ^ Input text

1203+ -> [Text]

1204+splitOn pat src

1205+ | null pat = emptyError "splitOn"

1206+ | isSingleton pat = split (== head pat) src

12071207 | otherwise = go 0 (indices pat src) src

12081208 where

12091209 go _ [] cs = [cs]

12101210 go !i (x:xs) cs = let h :*: t = splitAtWord (x-i) cs

12111211 in h : go (x+l) xs (dropWords l t)

12121212 l = foldlChunks (\a (T.Text _ _ b) -> a + fromIntegral b) 0 pat

1213−{-# INLINE [1] split #-}

1213+{-# INLINE [1] splitOn #-}

12141214

12151215{-# RULES

1216−"LAZY TEXT split/singleton -> splitBy/==" [~1] forall c t.

1217− split (singleton c) t = splitBy (==c) t

1216+"LAZY TEXT splitOn/singleton -> split/==" [~1] forall c t.

1217+ splitOn (singleton c) t = split (==c) t

12181218 #-}

12191219

12201220-- | /O(n)/ Splits a 'Text' into components delimited by separators,

@@ -1222,16 +1222,16 @@ split pat src

12221222-- resulting components do not contain the separators. Two adjacent

12231223-- separators result in an empty component in the output. eg.

12241224--

1225−-- > splitBy (=='a') "aabbaca" == ["","","bb","c",""]

1226−-- > splitBy (=='a') [] == [""]

1227−splitBy :: (Char -> Bool) -> Text -> [Text]

1228−splitBy _ Empty = [Empty]

1229−splitBy p (Chunk t0 ts0) = comb [] (T.splitBy p t0) ts0

1225+-- > split (=='a') "aabbaca" == ["","","bb","c",""]

1226+-- > split (=='a') [] == [""]

1227+split :: (Char -> Bool) -> Text -> [Text]

1228+split _ Empty = [Empty]

1229+split p (Chunk t0 ts0) = comb [] (T.split p t0) ts0

12301230 where comb acc (s:[]) Empty = revChunks (s:acc) : []

1231− comb acc (s:[]) (Chunk t ts) = comb (s:acc) (T.splitBy p t) ts

1231+ comb acc (s:[]) (Chunk t ts) = comb (s:acc) (T.split p t) ts

12321232 comb acc (s:ss) ts = revChunks (s:acc) : comb [] ss ts

1233− comb _ [] _ = impossibleError "splitBy"

1234−{-# INLINE splitBy #-}

1233+ comb _ [] _ = impossibleError "split"

1234+{-# INLINE split #-}

12351235

12361236-- | /O(n)/ Splits a 'Text' into components of length @k@. The last

12371237-- element may be shorter than the other chunks, depending on the

@@ -1251,14 +1251,14 @@ chunksOf k = go

12511251-- newline 'Char's. The resulting strings do not contain newlines.

12521252lines :: Text -> [Text]

12531253lines Empty = []

1254−lines t = let (l,t') = breakBy ((==) '\n') t

1254+lines t = let (l,t') = break ((==) '\n') t

12551255 in l : if null t' then []

12561256 else lines (tail t')

12571257

12581258-- | /O(n)/ Breaks a 'Text' up into a list of words, delimited by 'Char's

12591259-- representing white space.

12601260words :: Text -> [Text]

1261−words = L.filter (not . null) . splitBy isSpace

1261+words = L.filter (not . null) . split isSpace

12621262{-# INLINE words #-}

12631263

12641264-- | /O(n)/ Joins lines, after appending a terminating newline to

@@ -1378,21 +1378,21 @@ filter :: (Char -> Bool) -> Text -> Text

13781378filter p t = unstream (S.filter p (stream t))

13791379{-# INLINE filter #-}

13801380

1381−-- | /O(n)/ The 'findBy' function takes a predicate and a 'Text', and

1381+-- | /O(n)/ The 'find' function takes a predicate and a 'Text', and

13821382-- returns the first element in matching the predicate, or 'Nothing'

13831383-- if there is no such element.

1384−findBy :: (Char -> Bool) -> Text -> Maybe Char

1385−findBy p t = S.findBy p (stream t)

1386−{-# INLINE findBy #-}

1384+find :: (Char -> Bool) -> Text -> Maybe Char

1385+find p t = S.findBy p (stream t)

1386+{-# INLINE find #-}

13871387

1388−-- | /O(n)/ The 'partitionBy' function takes a predicate and a 'Text',

1388+-- | /O(n)/ The 'partition' function takes a predicate and a 'Text',

13891389-- and returns the pair of 'Text's with elements which do and do not

13901390-- satisfy the predicate, respectively; i.e.

13911391--

1392−-- > partitionBy p t == (filter p t, filter (not . p) t)

1393−partitionBy :: (Char -> Bool) -> Text -> (Text, Text)

1394−partitionBy p t = (filter p t, filter (not . p) t)

1395−{-# INLINE partitionBy #-}

1392+-- > partition p t == (filter p t, filter (not . p) t)

1393+partition :: (Char -> Bool) -> Text -> (Text, Text)

1394+partition p t = (filter p t, filter (not . p) t)

1395+{-# INLINE partition #-}

13961396

13971397-- | /O(n)/ 'Text' index (subscript) operator, starting from 0.

13981398index :: Text -> Int64 -> Char

Data/Text/Lazy/Encoding.hs+36−2

@@ -12,12 +12,13 @@

1212-- Functions for converting lazy 'Text' values to and from lazy

1313-- 'ByteString', using several standard encodings.

1414--

15−-- To make use of a much larger variety of encodings, use the @text-icu@

16−-- package.

15+-- To gain access to a much larger variety of encodings, use the

16+-- @text-icu@ package: <http://hackage.haskell.org/package/text-icu>

1717

1818module Data.Text.Lazy.Encoding

1919 (

2020 -- * Decoding ByteStrings to Text

21+ -- $strict

2122 decodeASCII

2223 , decodeUtf8

2324 , decodeUtf16LE

@@ -51,6 +52,19 @@ import qualified Data.Text.Encoding as TE

5152import qualified Data.Text.Lazy.Encoding.Fusion as E

5253import qualified Data.Text.Lazy.Fusion as F

5354

55+-- $strict

56+--

57+-- All of the single-parameter functions for decoding bytestrings

58+-- encoded in one of the Unicode Transformation Formats (UTF) operate

59+-- in a /strict/ mode: each will throw an exception if given invalid

60+-- input.

61+--

62+-- Each function has a variant, whose name is suffixed with -'With',

63+-- that gives greater control over the handling of decoding errors.

64+-- For instance, 'decodeUtf8' will throw an exception, but

65+-- 'decodeUtf8With' allows the programmer to determine what to do on a

66+-- decoding error.

67+

5468-- | Decode a 'ByteString' containing 7-bit ASCII encoded text.

5569decodeASCII :: B.ByteString -> Text

5670decodeASCII bs = foldr (chunk . TE.decodeASCII) empty (B.toChunks bs)

@@ -95,6 +109,10 @@ decodeUtf8With onErr bs0 = fast bs0

95109{-# INLINE[0] decodeUtf8With #-}

96110

97111-- | Decode a 'ByteString' containing UTF-8 encoded text.

112+--

113+-- If the input contains any invalid UTF-8 data, an exception will be

114+-- thrown. For more control over the handling of invalid data, use

115+-- 'decodeUtf8With'.

98116decodeUtf8 :: B.ByteString -> Text

99117decodeUtf8 = decodeUtf8With strictDecode

100118{-# INLINE[0] decodeUtf8 #-}

@@ -113,6 +131,10 @@ decodeUtf16LEWith onErr bs = F.unstream (E.streamUtf16LE onErr bs)

113131{-# INLINE decodeUtf16LEWith #-}

114132

115133-- | Decode text from little endian UTF-16 encoding.

134+--

135+-- If the input contains any invalid little endian UTF-16 data, an

136+-- exception will be thrown. For more control over the handling of

137+-- invalid data, use 'decodeUtf16LEWith'.

116138decodeUtf16LE :: B.ByteString -> Text

117139decodeUtf16LE = decodeUtf16LEWith strictDecode

118140{-# INLINE decodeUtf16LE #-}

@@ -123,6 +145,10 @@ decodeUtf16BEWith onErr bs = F.unstream (E.streamUtf16BE onErr bs)

123145{-# INLINE decodeUtf16BEWith #-}

124146

125147-- | Decode text from big endian UTF-16 encoding.

148+--

149+-- If the input contains any invalid big endian UTF-16 data, an

150+-- exception will be thrown. For more control over the handling of

151+-- invalid data, use 'decodeUtf16BEWith'.

126152decodeUtf16BE :: B.ByteString -> Text

127153decodeUtf16BE = decodeUtf16BEWith strictDecode

128154{-# INLINE decodeUtf16BE #-}

@@ -143,6 +169,10 @@ decodeUtf32LEWith onErr bs = F.unstream (E.streamUtf32LE onErr bs)

143169{-# INLINE decodeUtf32LEWith #-}

144170

145171-- | Decode text from little endian UTF-32 encoding.

172+--

173+-- If the input contains any invalid little endian UTF-32 data, an

174+-- exception will be thrown. For more control over the handling of

175+-- invalid data, use 'decodeUtf32LEWith'.

146176decodeUtf32LE :: B.ByteString -> Text

147177decodeUtf32LE = decodeUtf32LEWith strictDecode

148178{-# INLINE decodeUtf32LE #-}

@@ -153,6 +183,10 @@ decodeUtf32BEWith onErr bs = F.unstream (E.streamUtf32BE onErr bs)

153183{-# INLINE decodeUtf32BEWith #-}

154184

155185-- | Decode text from big endian UTF-32 encoding.

186+--

187+-- If the input contains any invalid big endian UTF-32 data, an

188+-- exception will be thrown. For more control over the handling of

189+-- invalid data, use 'decodeUtf32BEWith'.

156190decodeUtf32BE :: B.ByteString -> Text

157191decodeUtf32BE = decodeUtf32BEWith strictDecode

158192{-# INLINE decodeUtf32BE #-}

Data/Text/Lazy/IO.hs+21

@@ -9,9 +9,15 @@

99-- Portability : GHC

1010--

1111-- Efficient locale-sensitive support for lazy text I\/O.

12+--

13+-- Skip past the synopsis for some important notes on performance and

14+-- portability across different versions of GHC.

1215

1316module Data.Text.Lazy.IO

1417 (

18+ -- * Performance

19+ -- $performance

20+

1521 -- * Locale support

1622 -- $locale

1723 -- * File-at-a-time operations

@@ -58,6 +64,21 @@ import System.IO.Error (isEOFError)

5864import System.IO.Unsafe (unsafeInterleaveIO)

5965#endif

6066

67+-- $performance

68+--

69+-- The functions in this module obey the runtime system's locale,

70+-- character set encoding, and line ending conversion settings.

71+--

72+-- If you know in advance that you will be working with data that has

73+-- a specific encoding (e.g. UTF-8), and your application is highly

74+-- performance sensitive, you may find that it is faster to perform

75+-- I\/O with bytestrings and to encode and decode yourself than to use

76+-- the functions in this module.

77+--

78+-- Whether this will hold depends on the version of GHC you are using,

79+-- the platform you are working on, the data you are working with, and

80+-- the encodings you are using, so be sure to test for yourself.

81+

6182-- | Read a file and return its contents as a string. The file is

6283-- read lazily, as with 'getContents'.

6384readFile :: FilePath -> IO Text

Data/Text/Lazy/Read.hs+6−4

@@ -39,14 +39,15 @@ type Reader a = Text -> Either String (a,Text)

3939--

4040-- /Note/: For fixed-width integer types, this function does not

4141-- attempt to detect overflow, so a sufficiently long input may give

42−-- incorrect results.

42+-- incorrect results. If you are worried about overflow, use

43+-- 'Integer' for your result type.

4344decimal :: Integral a => Reader a

4445{-# SPECIALIZE decimal :: Reader Int #-}

4546{-# SPECIALIZE decimal :: Reader Integer #-}

4647decimal txt

4748 | T.null h = Left "input does not start with a digit"

4849 | otherwise = Right (T.foldl' go 0 h, t)

49− where (h,t) = T.spanBy isDigit txt

50+ where (h,t) = T.span isDigit txt

5051 go n d = (n * 10 + fromIntegral (digitToInt d))

5152

5253-- | Read a hexadecimal integer, consisting of an optional leading

@@ -59,7 +60,8 @@ decimal txt

5960--

6061-- /Note/: For fixed-width integer types, this function does not

6162-- attempt to detect overflow, so a sufficiently long input may give

62−-- incorrect results.

63+-- incorrect results. If you are worried about overflow, use

64+-- 'Integer' for your result type.

6365hexadecimal :: Integral a => Reader a

6466{-# SPECIALIZE hexadecimal :: Reader Int #-}

6567{-# SPECIALIZE hexadecimal :: Reader Integer #-}

@@ -74,7 +76,7 @@ hex :: Integral a => Reader a

7476hex txt

7577 | T.null h = Left "input does not start with a hexadecimal digit"

7678 | otherwise = Right (T.foldl' go 0 h, t)

77− where (h,t) = T.spanBy isHexDigit txt

79+ where (h,t) = T.span isHexDigit txt

7880 go n d = (n * 16 + fromIntegral (hexDigitToInt d))

7981

8082hexDigitToInt :: Char -> Int

Data/Text/Read.hs+7−5

@@ -39,14 +39,15 @@ type Reader a = Text -> Either String (a,Text)

3939--

4040-- /Note/: For fixed-width integer types, this function does not

4141-- attempt to detect overflow, so a sufficiently long input may give

42−-- incorrect results.

42+-- incorrect results. If you are worried about overflow, use

43+-- 'Integer' for your result type.

4344decimal :: Integral a => Reader a

4445{-# SPECIALIZE decimal :: Reader Int #-}

4546{-# SPECIALIZE decimal :: Reader Integer #-}

4647decimal txt

4748 | T.null h = Left "input does not start with a digit"

4849 | otherwise = Right (T.foldl' go 0 h, t)

49− where (h,t) = T.spanBy isDigit txt

50+ where (h,t) = T.span isDigit txt

5051 go n d = (n * 10 + fromIntegral (digitToInt d))

5152

5253-- | Read a hexadecimal integer, consisting of an optional leading

@@ -59,7 +60,8 @@ decimal txt

5960--

6061-- /Note/: For fixed-width integer types, this function does not

6162-- attempt to detect overflow, so a sufficiently long input may give

62−-- incorrect results.

63+-- incorrect results. If you are worried about overflow, use

64+-- 'Integer' for your result type.

6365hexadecimal :: Integral a => Reader a

6466{-# SPECIALIZE hexadecimal :: Reader Int #-}

6567{-# SPECIALIZE hexadecimal :: Reader Integer #-}

@@ -74,7 +76,7 @@ hex :: Integral a => Reader a

7476hex txt

7577 | T.null h = Left "input does not start with a hexadecimal digit"

7678 | otherwise = Right (T.foldl' go 0 h, t)

77− where (h,t) = T.spanBy isHexDigit txt

79+ where (h,t) = T.span isHexDigit txt

7880 go n d = (n * 16 + fromIntegral (hexDigitToInt d))

7981

8082hexDigitToInt :: Char -> Int

@@ -99,7 +101,7 @@ signed f = runP (signa (P f))

99101-- by the 'read' function, with the exception that a trailing @\'.\'@

100102-- or @\'e\'@ /not/ followed by a number is not consumed.

101103--

102−-- Examples:

104+-- Examples (with behaviour identical to 'read'):

103105--

104106-- >rational "3" == Right (3.0, "")

105107-- >rational "3.1" == Right (3.1, "")

tests/Properties.hs+62−62

@@ -265,13 +265,13 @@ t_reverse = L.reverse `eqP` (unpackS . T.reverse)

265265tl_reverse = L.reverse `eqP` (unpackS . TL.reverse)

266266t_reverse_short n = L.reverse `eqP` (unpackS . S.reverse . shorten n . S.stream)

267267

268−t_replace s d = (L.intercalate d . split s) `eqP`

268+t_replace s d = (L.intercalate d . splitOn s) `eqP`

269269 (unpackS . T.replace (T.pack s) (T.pack d))

270−tl_replace s d = (L.intercalate d . split s) `eqP`

270+tl_replace s d = (L.intercalate d . splitOn s) `eqP`

271271 (unpackS . TL.replace (TL.pack s) (TL.pack d))

272272

273−split :: (Eq a) => [a] -> [a] -> [[a]]

274−split pat src0

273+splitOn :: (Eq a) => [a] -> [a] -> [[a]]

274+splitOn pat src0

275275 | l == 0 = error "empty"

276276 | otherwise = go src0

277277 where

@@ -482,25 +482,25 @@ t_strip = T.dropAround isSpace `eq` T.strip

482482tl_strip = TL.dropAround isSpace `eq` TL.strip

483483t_splitAt n = L.splitAt n `eqP` (unpack2 . T.splitAt n)

484484tl_splitAt n = L.splitAt n `eqP` (unpack2 . TL.splitAt (fromIntegral n))

485−t_spanBy p = L.span p `eqP` (unpack2 . T.spanBy p)

486−tl_spanBy p = L.span p `eqP` (unpack2 . TL.spanBy p)

485+t_span p = L.span p `eqP` (unpack2 . T.span p)

486+tl_span p = L.span p `eqP` (unpack2 . TL.span p)

487487

488−t_break_id s = squid `eq` (uncurry T.append . T.break s)

488+t_breakOn_id s = squid `eq` (uncurry T.append . T.breakOn s)

489489 where squid t | T.null s = error "empty"

490490 | otherwise = t

491−tl_break_id s = squid `eq` (uncurry TL.append . TL.break s)

491+tl_breakOn_id s = squid `eq` (uncurry TL.append . TL.breakOn s)

492492 where squid t | TL.null s = error "empty"

493493 | otherwise = t

494−t_break_start (NotEmpty s) t = let (_,m) = T.break s t

494+t_breakOn_start (NotEmpty s) t = let (_,m) = T.breakOn s t

495495 in T.null m || s `T.isPrefixOf` m

496−tl_break_start (NotEmpty s) t = let (_,m) = TL.break s t

496+tl_breakOn_start (NotEmpty s) t = let (_,m) = TL.breakOn s t

497497 in TL.null m || s `TL.isPrefixOf` m

498−t_breakEnd_end (NotEmpty s) t = let (m,_) = T.breakEnd s t

498+t_breakOnEnd_end (NotEmpty s) t = let (m,_) = T.breakOnEnd s t

499499 in T.null m || s `T.isSuffixOf` m

500−tl_breakEnd_end (NotEmpty s) t = let (m,_) = TL.breakEnd s t

500+tl_breakOnEnd_end (NotEmpty s) t = let (m,_) = TL.breakOnEnd s t

501501 in TL.null m || s `TL.isSuffixOf` m

502−t_breakBy p = L.break p `eqP` (unpack2 . T.breakBy p)

503−tl_breakBy p = L.break p `eqP` (unpack2 . TL.breakBy p)

502+t_break p = L.break p `eqP` (unpack2 . T.break p)

503+tl_break p = L.break p `eqP` (unpack2 . TL.break p)

504504t_group = L.group `eqP` (map unpackS . T.group)

505505tl_group = L.group `eqP` (map unpackS . TL.group)

506506t_groupBy p = L.groupBy p `eqP` (map unpackS . T.groupBy p)

@@ -511,33 +511,33 @@ t_tails = L.tails `eqP` (map unpackS . T.tails)

511511tl_tails = L.tails `eqP` (map unpackS . TL.tails)

512512t_findAppendId (NotEmpty s) = unsquare $ \ts ->

513513 let t = T.intercalate s ts

514− in all (==t) $ map (uncurry T.append) (T.find s t)

514+ in all (==t) $ map (uncurry T.append) (T.breakOnAll s t)

515515tl_findAppendId (NotEmpty s) = unsquare $ \ts ->

516516 let t = TL.intercalate s ts

517− in all (==t) $ map (uncurry TL.append) (TL.find s t)

518−t_findContains (NotEmpty s) = all (T.isPrefixOf s . snd) . T.find s .

517+ in all (==t) $ map (uncurry TL.append) (TL.breakOnAll s t)

518+t_findContains (NotEmpty s) = all (T.isPrefixOf s . snd) . T.breakOnAll s .

519519 T.intercalate s

520520tl_findContains (NotEmpty s) = all (TL.isPrefixOf s . snd) .

521− TL.find s . TL.intercalate s

521+ TL.breakOnAll s . TL.intercalate s

522522sl_filterCount c = (L.genericLength . L.filter (==c)) `eqP` SL.countChar c

523−t_findCount s = (L.length . T.find s) `eq` T.count s

524−tl_findCount s = (L.genericLength . TL.find s) `eq` TL.count s

525−

526−t_split_split s = (T.split s `eq` Slow.split s) . T.intercalate s

527−tl_split_split s = ((TL.split (TL.fromStrict s) . TL.fromStrict) `eq`

528− (map TL.fromStrict . T.split s)) . T.intercalate s

529−t_split_i (NotEmpty t) = id `eq` (T.intercalate t . T.split t)

530−tl_split_i (NotEmpty t) = id `eq` (TL.intercalate t . TL.split t)

531−

532−t_splitBy p = splitBy p `eqP` (map unpackS . T.splitBy p)

533−t_splitBy_count c = (L.length . T.splitBy (==c)) `eq`

534− ((1+) . T.count (T.singleton c))

535−t_splitBy_split c = T.splitBy (==c) `eq` T.split (T.singleton c)

536−tl_splitBy p = splitBy p `eqP` (map unpackS . TL.splitBy p)

537−

538−splitBy :: (a -> Bool) -> [a] -> [[a]]

539−splitBy _ [] = [[]]

540−splitBy p xs = loop xs

523+t_findCount s = (L.length . T.breakOnAll s) `eq` T.count s

524+tl_findCount s = (L.genericLength . TL.breakOnAll s) `eq` TL.count s

525+

526+t_splitOn_split s = (T.splitOn s `eq` Slow.splitOn s) . T.intercalate s

527+tl_splitOn_split s = ((TL.splitOn (TL.fromStrict s) . TL.fromStrict) `eq`

528+ (map TL.fromStrict . T.splitOn s)) . T.intercalate s

529+t_splitOn_i (NotEmpty t) = id `eq` (T.intercalate t . T.splitOn t)

530+tl_splitOn_i (NotEmpty t) = id `eq` (TL.intercalate t . TL.splitOn t)

531+

532+t_split p = split p `eqP` (map unpackS . T.split p)

533+t_split_count c = (L.length . T.split (==c)) `eq`

534+ ((1+) . T.count (T.singleton c))

535+t_split_splitOn c = T.split (==c) `eq` T.splitOn (T.singleton c)

536+tl_split p = split p `eqP` (map unpackS . TL.split p)

537+

538+split :: (a -> Bool) -> [a] -> [[a]]

539+split _ [] = [[]]

540+split p xs = loop xs

541541 where loop s | null s' = [l]

542542 | otherwise = l : loop (tail s')

543543 where (l, s') = break p s

@@ -597,10 +597,10 @@ sf_filter q p = (L.filter p . L.filter q) `eqP`

597597t_filter p = L.filter p `eqP` (unpackS . T.filter p)

598598tl_filter p = L.filter p `eqP` (unpackS . TL.filter p)

599599sf_findBy q p = (L.find p . L.filter q) `eqP` (S.findBy p . S.filter q)

600−t_findBy p = L.find p `eqP` T.findBy p

601−tl_findBy p = L.find p `eqP` TL.findBy p

602−t_partition p = L.partition p `eqP` (unpack2 . T.partitionBy p)

603−tl_partition p = L.partition p `eqP` (unpack2 . TL.partitionBy p)

600+t_find p = L.find p `eqP` T.find p

601+tl_find p = L.find p `eqP` TL.find p

602+t_partition p = L.partition p `eqP` (unpack2 . T.partition p)

603+tl_partition p = L.partition p `eqP` (unpack2 . TL.partition p)

604604

605605sf_index p s = forAll (choose (-l,l*2))

606606 ((L.filter p s L.!!) `eq` S.index (S.filter p $ packS s))

@@ -613,8 +613,8 @@ tl_index s = forAll (choose (-l,l*2))

613613 where l = L.length s

614614

615615t_findIndex p = L.findIndex p `eqP` T.findIndex p

616−t_count (NotEmpty t) = (subtract 1 . L.length . T.split t) `eq` T.count t

617−tl_count (NotEmpty t) = (subtract 1 . L.genericLength . TL.split t) `eq`

616+t_count (NotEmpty t) = (subtract 1 . L.length . T.splitOn t) `eq` T.count t

617+tl_count (NotEmpty t) = (subtract 1 . L.genericLength . TL.splitOn t) `eq`

618618 TL.count t

619619t_zip s = L.zip s `eqP` T.zip (packS s)

620620tl_zip s = L.zip s `eqP` TL.zip (packS s)

@@ -1052,16 +1052,16 @@ tests = [

10521052 testProperty "tl_strip" tl_strip,

10531053 testProperty "t_splitAt" t_splitAt,

10541054 testProperty "tl_splitAt" tl_splitAt,

1055− testProperty "t_spanBy" t_spanBy,

1056− testProperty "tl_spanBy" tl_spanBy,

1057− testProperty "t_break_id" t_break_id,

1058− testProperty "tl_break_id" tl_break_id,

1059− testProperty "t_break_start" t_break_start,

1060− testProperty "tl_break_start" tl_break_start,

1061− testProperty "t_breakEnd_end" t_breakEnd_end,

1062− testProperty "tl_breakEnd_end" tl_breakEnd_end,

1063− testProperty "t_breakBy" t_breakBy,

1064− testProperty "tl_breakBy" tl_breakBy,

1055+ testProperty "t_span" t_span,

1056+ testProperty "tl_span" tl_span,

1057+ testProperty "t_breakOn_id" t_breakOn_id,

1058+ testProperty "tl_breakOn_id" tl_breakOn_id,

1059+ testProperty "t_breakOn_start" t_breakOn_start,

1060+ testProperty "tl_breakOn_start" tl_breakOn_start,

1061+ testProperty "t_breakOnEnd_end" t_breakOnEnd_end,

1062+ testProperty "tl_breakOnEnd_end" tl_breakOnEnd_end,

1063+ testProperty "t_break" t_break,

1064+ testProperty "tl_break" tl_break,

10651065 testProperty "t_group" t_group,

10661066 testProperty "tl_group" tl_group,

10671067 testProperty "t_groupBy" t_groupBy,

@@ -1080,14 +1080,14 @@ tests = [

10801080 testProperty "sl_filterCount" sl_filterCount,

10811081 testProperty "t_findCount" t_findCount,

10821082 testProperty "tl_findCount" tl_findCount,

1083− testProperty "t_split_split" t_split_split,

1084− testProperty "tl_split_split" tl_split_split,

1085− testProperty "t_split_i" t_split_i,

1086− testProperty "tl_split_i" tl_split_i,

1087− testProperty "t_splitBy" t_splitBy,

1088− testProperty "t_splitBy_count" t_splitBy_count,

1089− testProperty "t_splitBy_split" t_splitBy_split,

1090− testProperty "tl_splitBy" tl_splitBy,

1083+ testProperty "t_splitOn_split" t_splitOn_split,

1084+ testProperty "tl_splitOn_split" tl_splitOn_split,

1085+ testProperty "t_splitOn_i" t_splitOn_i,

1086+ testProperty "tl_splitOn_i" tl_splitOn_i,

1087+ testProperty "t_split" t_split,

1088+ testProperty "t_split_count" t_split_count,

1089+ testProperty "t_split_splitOn" t_split_splitOn,

1090+ testProperty "tl_split" tl_split,

10911091 testProperty "t_chunksOf_same_lengths" t_chunksOf_same_lengths,

10921092 testProperty "t_chunksOf_length" t_chunksOf_length,

10931093 testProperty "tl_chunksOf" tl_chunksOf

@@ -1130,8 +1130,8 @@ tests = [

11301130 testProperty "t_filter" t_filter,

11311131 testProperty "tl_filter" tl_filter,

11321132 testProperty "sf_findBy" sf_findBy,

1133− testProperty "t_findBy" t_findBy,

1134− testProperty "tl_findBy" tl_findBy,

1133+ testProperty "t_find" t_find,

1134+ testProperty "tl_find" tl_find,

11351135 testProperty "t_partition" t_partition,

11361136 testProperty "tl_partition" tl_partition

11371137 ],

tests/SlowFunctions.hs+7−7

@@ -3,7 +3,7 @@

33module SlowFunctions

44 (

55 indices

6− , split

6+ , splitOn

77 ) where

99import qualified Data.Text as T

@@ -23,12 +23,12 @@ indices needle@(Text _narr _noff nlen) haystack@(Text harr hoff hlen)

2323 where t = Text harr (hoff+i) (hlen-i)

2424 d = iter_ haystack i

2525

26−split :: T.Text -- ^ Text to split on

27− -> T.Text -- ^ Input text

28− -> [T.Text]

29−split pat src0

30− | T.null pat = error "split: empty"

31− | l == 1 = T.splitBy (== (unsafeHead pat)) src0

26+splitOn :: T.Text -- ^ Text to split on

27+ -> T.Text -- ^ Input text

28+ -> [T.Text]

29+splitOn pat src0

30+ | T.null pat = error "splitOn: empty"

31+ | l == 1 = T.split (== (unsafeHead pat)) src0

3232 | otherwise = go src0

3333 where

3434 l = T.length pat

tests/benchmarks/CaseMap.hsnew file+21

@@ -0,0 +1,21 @@

1+import Control.Exception

2+import Control.Monad

3+import Data.Time.Clock

4+import Data.ByteString as B

5+import Data.Text.Encoding as T

6+import Data.Text as T

7+import System.Environment

9+time act = do

10+ start <- getCurrentTime

11+ act

12+ end <- getCurrentTime

13+ let d = diffUTCTime end start

14+ print d

15+

16+main = do

17+ args <- getArgs

18+ forM_ args $ \f -> do

19+ t <- T.decodeUtf8 `fmap` B.readFile f

20+ evaluate t

21+ time $ evaluate (T.toUpper t)

tests/benchmarks/casemap.pynew file+15

@@ -0,0 +1,15 @@

1+#!/usr/bin/env python

3+import sys, time

5+def timeit(f):

6+ start = time.time()

7+ f()

8+ end = time.time()

9+ print end - start

10+

11+for f in sys.argv[1:]:

12+ s = open(f).read()

13+ u = s.decode('utf8')

14+ timeit(lambda: s.upper())

15+ timeit(lambda: u.upper())

text.cabal+1−1

@@ -1,5 +1,5 @@

11name: text

2−version: 0.10.0.2

2+version: 0.11.0.0

33homepage: http://bitbucket.org/bos/text

44bug-reports: http://bitbucket.org/bos/text/issues

55synopsis: An efficient packed Unicode text type.