bdbd24b · text · git.iko.soy

git.iko.soy/text

on master help sign in

Merge

bdbd24bBryan O'Sullivan16 years ago

.hgignore+2−2

compare with master

unifiedsplit

@@ -1,7 +1,7 @@

1^dist$

2^tests/benchmarks/.*\.txt$

3−^tests/(?:qc|text/test)$

4−\.(?:aux|eventlog|h[ip]|log|[oa]|orig|prof|ps|swp)$

4+\.(?:aux|eventlog|h[ip]|log|[oa]|orig|prof|ps|rej|swp)$

5~$

6syntax: glob

7.\#*

.hgtags+2

@@ -11,4 +11,6 @@ b4fdbef1c1027a70b95c1a81747d968890acc3f6 0.3

11de7cb0cb2a72606126c357b860bb0dbf7f145ad2 0.9.0.1

12f2c9b1c019e3277926378d7b5a604f1c995f0202 0.7

13f3d625f2d0730cb6adf8506a1f37762178c9aea6 0.10.0.0

14+99ea36957b1d47145d70f2ee8b13f1ddfd09245c 0.10.0.1

141b7889216e78b00d69b32c48782111bcd935978d 0.10.0.2

151b7889216e78b00d69b32c48782111bcd935978d 0.10.0.2

16+75157eaec6eb38574ad5c347e75c48dcb7262dee 0.11.0.0

Data/Text.hs+69−67

@@ -128,10 +128,10 @@ module Data.Text

128 , stripStart

129 , stripEnd

130 , splitAt

131− , spanBy

131+ , breakOn

132+ , breakOnEnd

132 , break

133 , break

133− , breakEnd

134+ , span

134− , breakBy

135 , group

136 , groupBy

137 , inits

@@ -139,8 +139,8 @@ module Data.Text

139

140 -- ** Breaking into many substrings

141 -- $split

142+ , splitOn

142 , split

143 , split

143− , splitBy

144 , chunksOf

145

146 -- ** Breaking into lines and words

@@ -161,9 +161,9 @@ module Data.Text

161

162 -- * Searching

163 , filter

164+ , breakOnAll

164 , find

165 , find

165− , findBy

166+ , partition

166− , partitionBy

167

168 -- , findSubstring

169

@@ -598,7 +598,7 @@ replace :: Text -- ^ Text to search for

598 -> Text -- ^ Replacement text

599 -> Text -- ^ Input text

600 -> Text

601−replace s d = intercalate d . split s

601+replace s d = intercalate d . splitOn s

602{-# INLINE replace #-}

603

604-- ----------------------------------------------------------------------------

@@ -619,7 +619,8 @@ replace s d = intercalate d . split s

619-- context-dependent operation. The case conversion functions in this

620-- module are /not/ locale sensitive. Programs that require locale

621-- sensitivity should use appropriate versions of the case mapping

622−-- functions from the @text-icu@ package.

622+-- functions from the @text-icu@ package:

623+-- <http://hackage.haskell.org/package/text-icu>

623

624

624-- | /O(n)/ Convert a string to folded case. This function is mainly

625-- | /O(n)/ Convert a string to folded case. This function is mainly

625-- useful for performing caseless (also known as case insensitive)

626-- useful for performing caseless (also known as case insensitive)

@@ -1072,23 +1073,23 @@ splitAt n t@(Text arr off len)

1072 where d = iter_ t i

1073 where d = iter_ t i

1073{-# INLINE splitAt #-}

1074{-# INLINE splitAt #-}

1074

1075

1075−-- | /O(n)/ 'spanBy', applied to a predicate @p@ and text @t@, returns

1076+-- | /O(n)/ 'span', applied to a predicate @p@ and text @t@, returns

1076-- a pair whose first element is the longest prefix (possibly empty)

1077-- a pair whose first element is the longest prefix (possibly empty)

1077-- of @t@ of elements that satisfy @p@, and whose second is the

1078-- of @t@ of elements that satisfy @p@, and whose second is the

1078-- remainder of the list.

1079-- remainder of the list.

1079−spanBy :: (Char -> Bool) -> Text -> (Text, Text)

1080+span :: (Char -> Bool) -> Text -> (Text, Text)

1080−spanBy p t@(Text arr off len) = (textP arr off k, textP arr (off+k) (len-k))

1081+span p t@(Text arr off len) = (textP arr off k, textP arr (off+k) (len-k))

1081 where k = loop 0

1082 where k = loop 0

1082 loop !i | i >= len || not (p c) = i

1083 loop !i | i >= len || not (p c) = i

1083 | otherwise = loop (i+d)

1084 | otherwise = loop (i+d)

1084 where Iter c d = iter t i

1085 where Iter c d = iter t i

1085−{-# INLINE spanBy #-}

1086+{-# INLINE span #-}

1086

1087

1087−-- | /O(n)/ 'breakBy' is like 'spanBy', but the prefix returned is

1088+-- | /O(n)/ 'break' is like 'span', but the prefix returned is

1088-- over elements that fail the predicate @p@.

1089-- over elements that fail the predicate @p@.

1089−breakBy :: (Char -> Bool) -> Text -> (Text, Text)

1090+break :: (Char -> Bool) -> Text -> (Text, Text)

1090−breakBy p = spanBy (not . p)

1091+break p = span (not . p)

1091−{-# INLINE breakBy #-}

1092+{-# INLINE break #-}

1092

1093

1093-- | /O(n)/ Group characters in a string according to a predicate.

1094-- | /O(n)/ Group characters in a string according to a predicate.

1094groupBy :: (Char -> Char -> Bool) -> Text -> [Text]

1095groupBy :: (Char -> Char -> Bool) -> Text -> [Text]

@@ -1138,30 +1139,30 @@ tails t | null t = [empty]

1138--

1139--

1139-- Examples:

1140-- Examples:

1140--

1141--

1141−-- > split "\r\n" "a\r\nb\r\nd\r\ne" == ["a","b","d","e"]

1142+-- > splitOn "\r\n" "a\r\nb\r\nd\r\ne" == ["a","b","d","e"]

1142−-- > split "aaa" "aaaXaaaXaaaXaaa" == ["","X","X","X",""]

1143+-- > splitOn "aaa" "aaaXaaaXaaaXaaa" == ["","X","X","X",""]

1143−-- > split "x" "x" == ["",""]

1144+-- > splitOn "x" "x" == ["",""]

1144--

1145--

1145-- and

1146-- and

1146--

1147--

1147−-- > intercalate s . split s == id

1148+-- > intercalate s . splitOn s == id

1148−-- > split (singleton c) == splitBy (==c)

1149+-- > splitOn (singleton c) == split (==c)

1149--

1150--

1150-- In (unlikely) bad cases, this function's time complexity degrades

1151-- In (unlikely) bad cases, this function's time complexity degrades

1151-- towards /O(n*m)/.

1152-- towards /O(n*m)/.

1152−split :: Text -> Text -> [Text]

1153+splitOn :: Text -> Text -> [Text]

1153−split pat@(Text _ _ l) src@(Text arr off len)

1154+splitOn pat@(Text _ _ l) src@(Text arr off len)

1154− | l <= 0 = emptyError "split"

1155+ | l <= 0 = emptyError "splitOn"

1155− | isSingleton pat = splitBy (== unsafeHead pat) src

1156+ | isSingleton pat = split (== unsafeHead pat) src

1156 | otherwise = go 0 (indices pat src)

1157 | otherwise = go 0 (indices pat src)

1157 where

1158 where

1158 go !s (x:xs) = textP arr (s+off) (x-s) : go (x+l) xs

1159 go !s (x:xs) = textP arr (s+off) (x-s) : go (x+l) xs

1159 go s _ = [textP arr (s+off) (len-s)]

1160 go s _ = [textP arr (s+off) (len-s)]

1160−{-# INLINE [1] split #-}

1161+{-# INLINE [1] splitOn #-}

1161

1162

1162{-# RULES

1163{-# RULES

1163−"TEXT split/singleton -> splitBy/==" [~1] forall c t.

1164+"TEXT splitOn/singleton -> split/==" [~1] forall c t.

1164− split (singleton c) t = splitBy (==c) t

1165+ splitOn (singleton c) t = split (==c) t

1165 #-}

1166 #-}

1166

1167

1167-- | /O(n)/ Splits a 'Text' into components delimited by separators,

1168-- | /O(n)/ Splits a 'Text' into components delimited by separators,

@@ -1169,15 +1170,15 @@ split pat@(Text _ _ l) src@(Text arr off len)

1169-- resulting components do not contain the separators. Two adjacent

1170-- resulting components do not contain the separators. Two adjacent

1170-- separators result in an empty component in the output. eg.

1171-- separators result in an empty component in the output. eg.

1171--

1172--

1172−-- > splitBy (=='a') "aabbaca" == ["","","bb","c",""]

1173+-- > split (=='a') "aabbaca" == ["","","bb","c",""]

1173−-- > splitBy (=='a') "" == [""]

1174+-- > split (=='a') "" == [""]

1174−splitBy :: (Char -> Bool) -> Text -> [Text]

1175+split :: (Char -> Bool) -> Text -> [Text]

1175−splitBy _ t@(Text _off _arr 0) = [t]

1176+split _ t@(Text _off _arr 0) = [t]

1176−splitBy p t = loop t

1177+split p t = loop t

1177 where loop s | null s' = [l]

1178 where loop s | null s' = [l]

1178 | otherwise = l : loop (unsafeTail s')

1179 | otherwise = l : loop (unsafeTail s')

1179− where (l, s') = breakBy p s

1180+ where (l, s') = break p s

1180−{-# INLINE splitBy #-}

1181+{-# INLINE split #-}

1181

1182

1182-- | /O(n)/ Splits a 'Text' into components of length @k@. The last

1183-- | /O(n)/ Splits a 'Text' into components of length @k@. The last

1183-- element may be shorter than the other chunks, depending on the

1184-- element may be shorter than the other chunks, depending on the

@@ -1199,21 +1200,21 @@ chunksOf k = go

1199-------------------------------------------------------------------------------

1200-------------------------------------------------------------------------------

1200-- ** Searching with a predicate

1201-- ** Searching with a predicate

1201

1202

1202−-- | /O(n)/ The 'findBy' function takes a predicate and a 'Text', and

1203+-- | /O(n)/ The 'find' function takes a predicate and a 'Text', and

1203−-- returns the first element in matching the predicate, or 'Nothing'

1204+-- returns the first element matching the predicate, or 'Nothing' if

1204−-- if there is no such element.

1205+-- there is no such element.

1205−findBy :: (Char -> Bool) -> Text -> Maybe Char

1206+find :: (Char -> Bool) -> Text -> Maybe Char

1206−findBy p t = S.findBy p (stream t)

1207+find p t = S.findBy p (stream t)

1207−{-# INLINE findBy #-}

1208+{-# INLINE find #-}

1208

1209

1209−-- | /O(n)/ The 'partitionBy' function takes a predicate and a 'Text',

1210+-- | /O(n)/ The 'partition' function takes a predicate and a 'Text',

1210-- and returns the pair of 'Text's with elements which do and do not

1211-- and returns the pair of 'Text's with elements which do and do not

1211-- satisfy the predicate, respectively; i.e.

1212-- satisfy the predicate, respectively; i.e.

1212--

1213--

1213−-- > partitionBy p t == (filter p t, filter (not . p) t)

1214+-- > partition p t == (filter p t, filter (not . p) t)

1214−partitionBy :: (Char -> Bool) -> Text -> (Text, Text)

1215+partition :: (Char -> Bool) -> Text -> (Text, Text)

1215−partitionBy p t = (filter p t, filter (not . p) t)

1216+partition p t = (filter p t, filter (not . p) t)

1216−{-# INLINE partitionBy #-}

1217+{-# INLINE partition #-}

1217

1218

1218-- | /O(n)/ 'filter', applied to a predicate and a 'Text',

1219-- | /O(n)/ 'filter', applied to a predicate and a 'Text',

1219-- returns a 'Text' containing those characters that satisfy the

1220-- returns a 'Text' containing those characters that satisfy the

@@ -1229,39 +1230,40 @@ filter p t = unstream (S.filter p (stream t))

1229--

1230--

1230-- Examples:

1231-- Examples:

1231--

1232--

1232−-- > break "::" "a::b::c" ==> ("a", "::b::c")

1233+-- > breakOn "::" "a::b::c" ==> ("a", "::b::c")

1233−-- > break "/" "foobar" ==> ("foobar", "")

1234+-- > breakOn "/" "foobar" ==> ("foobar", "")

1234--

1235--

1235-- Laws:

1236-- Laws:

1236--

1237--

1237-- > append prefix match == haystack

1238-- > append prefix match == haystack

1238−-- > where (prefix, match) = break needle haystack

1239+-- > where (prefix, match) = breakOn needle haystack

1239--

1240--

1240-- If you need to break a string by a substring repeatedly (e.g. you

1241-- If you need to break a string by a substring repeatedly (e.g. you

1241−-- want to break on every instance of a substring), use 'find'

1242+-- want to break on every instance of a substring), use 'breakOnAll'

1242-- instead, as it has lower startup overhead.

1243-- instead, as it has lower startup overhead.

1243--

1244--

1244-- In (unlikely) bad cases, this function's time complexity degrades

1245-- In (unlikely) bad cases, this function's time complexity degrades

1245-- towards /O(n*m)/.

1246-- towards /O(n*m)/.

1246−break :: Text -> Text -> (Text, Text)

1247+breakOn :: Text -> Text -> (Text, Text)

1247−break pat src@(Text arr off len)

1248+breakOn pat src@(Text arr off len)

1248− | null pat = emptyError "break"

1249+ | null pat = emptyError "breakOn"

1249 | otherwise = case indices pat src of

1250 | otherwise = case indices pat src of

1250 [] -> (src, empty)

1251 [] -> (src, empty)

1251 (x:_) -> (textP arr off x, textP arr (off+x) (len-x))

1252 (x:_) -> (textP arr off x, textP arr (off+x) (len-x))

1252−{-# INLINE break #-}

1253+{-# INLINE breakOn #-}

1253

1254

1254−-- | /O(n+m)/ Similar to 'break', but searches from the end of the string.

1255+-- | /O(n+m)/ Similar to 'breakOn', but searches from the end of the

1256+-- string.

1255--

1257--

1256-- The first element of the returned tuple is the prefix of @haystack@

1258-- The first element of the returned tuple is the prefix of @haystack@

1257-- up to and including the last match of @needle@. The second is the

1259-- up to and including the last match of @needle@. The second is the

1258-- remainder of @haystack@, following the match.

1260-- remainder of @haystack@, following the match.

1259--

1261--

1260−-- > breakEnd "::" "a::b::c" ==> ("a::b::", "c")

1262+-- > breakOnEnd "::" "a::b::c" ==> ("a::b::", "c")

1261−breakEnd :: Text -> Text -> (Text, Text)

1263+breakOnEnd :: Text -> Text -> (Text, Text)

1264+breakOnEnd pat src = (reverse b, reverse a)

1262−breakEnd pat src = let (a,b) = break (reverse pat) (reverse src)

1265+ where (a,b) = breakOn (reverse pat) (reverse src)

1263− in (reverse b, reverse a)

1264−{-# INLINE breakEnd #-}

1266+{-# INLINE breakOnEnd #-}

1265

1267

1266-- | /O(n+m)/ Find all non-overlapping instances of @needle@ in

1268-- | /O(n+m)/ Find all non-overlapping instances of @needle@ in

1267-- @haystack@. Each element of the returned list consists of a pair:

1269-- @haystack@. Each element of the returned list consists of a pair:

@@ -1281,16 +1283,16 @@ breakEnd pat src = let (a,b) = break (reverse pat) (reverse src)

1281-- towards /O(n*m)/.

1283-- towards /O(n*m)/.

1282--

1284--

1283-- The @needle@ parameter may not be empty.

1285-- The @needle@ parameter may not be empty.

1284−find :: Text -- ^ @needle@ to search for

1286+breakOnAll :: Text -- ^ @needle@ to search for

1285− -> Text -- ^ @haystack@ in which to search

1287+ -> Text -- ^ @haystack@ in which to search

1286− -> [(Text, Text)]

1288+ -> [(Text, Text)]

1287−find pat src@(Text arr off slen)

1289+breakOnAll pat src@(Text arr off slen)

1288− | null pat = emptyError "find"

1290+ | null pat = emptyError "breakOnAll"

1289 | otherwise = L.map step (indices pat src)

1291 | otherwise = L.map step (indices pat src)

1290 where

1292 where

1291 step x = (chunk 0 x, chunk x (slen-x))

1293 step x = (chunk 0 x, chunk x (slen-x))

1292 chunk !n !l = textP arr (n+off) l

1294 chunk !n !l = textP arr (n+off) l

1293−{-# INLINE find #-}

1295+{-# INLINE breakOnAll #-}

1294

1296

1295-------------------------------------------------------------------------------

1297-------------------------------------------------------------------------------

1296-- ** Indexing 'Text's

1298-- ** Indexing 'Text's

@@ -1390,7 +1392,7 @@ lines ps | null ps = []

1390 | otherwise = h : if null t

1392 | otherwise = h : if null t

1391 then []

1393 then []

1392 else lines (unsafeTail t)

1394 else lines (unsafeTail t)

1393− where (h,t) = spanBy (/= '\n') ps

1395+ where (h,t) = span (/= '\n') ps

1394{-# INLINE lines #-}

1396{-# INLINE lines #-}

1395

1397

1396{-

1398{-

Data/Text/Encoding.hs+38−2

@@ -14,18 +14,20 @@

14-- Functions for converting 'Text' values to and from 'ByteString',

15-- using several standard encodings.

16--

17−-- To make use of a much larger variety of encodings, use the @text-icu@

17+-- To gain access to a much larger family of encodings, use the

18−-- package.

18+-- @text-icu@ package: <http://hackage.haskell.org/package/text-icu>

20module Data.Text.Encoding

21 (

22 -- * Decoding ByteStrings to Text

23+ -- $strict

23 decodeASCII

24 decodeASCII

24 , decodeUtf8

25 , decodeUtf8

25 , decodeUtf16LE

26 , decodeUtf16LE

26 , decodeUtf16BE

27 , decodeUtf16BE

27 , decodeUtf32LE

28 , decodeUtf32LE

28 , decodeUtf32BE

29 , decodeUtf32BE

30+

29 -- ** Controllable error handling

31 -- ** Controllable error handling

30 , decodeUtf8With

32 , decodeUtf8With

31 , decodeUtf16LEWith

33 , decodeUtf16LEWith

@@ -60,11 +62,25 @@ import qualified Data.Text.Encoding.Utf16 as U16

60import qualified Data.Text.Encoding.Utf8 as U8

62import qualified Data.Text.Encoding.Utf8 as U8

61import qualified Data.Text.Fusion as F

63import qualified Data.Text.Fusion as F

65+-- $strict

66+--

67+-- All of the single-parameter functions for decoding bytestrings

68+-- encoded in one of the Unicode Transformation Formats (UTF) operate

69+-- in a /strict/ mode: each will throw an exception if given invalid

70+-- input.

71+--

72+-- Each function has a variant, whose name is suffixed with -'With',

73+-- that gives greater control over the handling of decoding errors.

74+-- For instance, 'decodeUtf8' will throw an exception, but

75+-- 'decodeUtf8With' allows the programmer to determine what to do on a

76+-- decoding error.

77+

63-- | Decode a 'ByteString' containing 7-bit ASCII encoded text.

78-- | Decode a 'ByteString' containing 7-bit ASCII encoded text.

64decodeASCII :: ByteString -> Text

79decodeASCII :: ByteString -> Text

65decodeASCII bs = F.unstream (E.streamASCII bs)

80decodeASCII bs = F.unstream (E.streamASCII bs)

66{-# INLINE decodeASCII #-}

81{-# INLINE decodeASCII #-}

83+-- | Decode a 'ByteString' containing UTF-8 encoded text.

68decodeUtf8With :: OnDecodeError -> ByteString -> Text

84decodeUtf8With :: OnDecodeError -> ByteString -> Text

69decodeUtf8With onErr bs = textP (fst a) 0 (snd a)

85decodeUtf8With onErr bs = textP (fst a) 0 (snd a)

70 where

86 where

@@ -101,6 +117,10 @@ decodeUtf8With onErr bs = textP (fst a) 0 (snd a)

101{-# INLINE[0] decodeUtf8With #-}

117{-# INLINE[0] decodeUtf8With #-}

102

118

103-- | Decode a 'ByteString' containing UTF-8 encoded text.

119-- | Decode a 'ByteString' containing UTF-8 encoded text.

120+--

121+-- If the input contains any invalid UTF-8 data, an exception will be

122+-- thrown. For more control over the handling of invalid data, use

123+-- 'decodeUtf8With'.

104decodeUtf8 :: ByteString -> Text

124decodeUtf8 :: ByteString -> Text

105decodeUtf8 = decodeUtf8With strictDecode

125decodeUtf8 = decodeUtf8With strictDecode

106{-# INLINE[0] decodeUtf8 #-}

126{-# INLINE[0] decodeUtf8 #-}

@@ -156,6 +176,10 @@ decodeUtf16LEWith onErr bs = F.unstream (E.streamUtf16LE onErr bs)

156{-# INLINE decodeUtf16LEWith #-}

176{-# INLINE decodeUtf16LEWith #-}

157

177

158-- | Decode text from little endian UTF-16 encoding.

178-- | Decode text from little endian UTF-16 encoding.

179+--

180+-- If the input contains any invalid little endian UTF-16 data, an

181+-- exception will be thrown. For more control over the handling of

182+-- invalid data, use 'decodeUtf16LEWith'.

159decodeUtf16LE :: ByteString -> Text

183decodeUtf16LE :: ByteString -> Text

160decodeUtf16LE = decodeUtf16LEWith strictDecode

184decodeUtf16LE = decodeUtf16LEWith strictDecode

161{-# INLINE decodeUtf16LE #-}

185{-# INLINE decodeUtf16LE #-}

@@ -166,6 +190,10 @@ decodeUtf16BEWith onErr bs = F.unstream (E.streamUtf16BE onErr bs)

166{-# INLINE decodeUtf16BEWith #-}

190{-# INLINE decodeUtf16BEWith #-}

167

191

168-- | Decode text from big endian UTF-16 encoding.

192-- | Decode text from big endian UTF-16 encoding.

193+--

194+-- If the input contains any invalid big endian UTF-16 data, an

195+-- exception will be thrown. For more control over the handling of

196+-- invalid data, use 'decodeUtf16BEWith'.

169decodeUtf16BE :: ByteString -> Text

197decodeUtf16BE :: ByteString -> Text

170decodeUtf16BE = decodeUtf16BEWith strictDecode

198decodeUtf16BE = decodeUtf16BEWith strictDecode

171{-# INLINE decodeUtf16BE #-}

199{-# INLINE decodeUtf16BE #-}

@@ -186,6 +214,10 @@ decodeUtf32LEWith onErr bs = F.unstream (E.streamUtf32LE onErr bs)

186{-# INLINE decodeUtf32LEWith #-}

214{-# INLINE decodeUtf32LEWith #-}

187

215

188-- | Decode text from little endian UTF-32 encoding.

216-- | Decode text from little endian UTF-32 encoding.

217+--

218+-- If the input contains any invalid little endian UTF-32 data, an

219+-- exception will be thrown. For more control over the handling of

220+-- invalid data, use 'decodeUtf32LEWith'.

189decodeUtf32LE :: ByteString -> Text

221decodeUtf32LE :: ByteString -> Text

190decodeUtf32LE = decodeUtf32LEWith strictDecode

222decodeUtf32LE = decodeUtf32LEWith strictDecode

191{-# INLINE decodeUtf32LE #-}

223{-# INLINE decodeUtf32LE #-}

@@ -196,6 +228,10 @@ decodeUtf32BEWith onErr bs = F.unstream (E.streamUtf32BE onErr bs)

196{-# INLINE decodeUtf32BEWith #-}

228{-# INLINE decodeUtf32BEWith #-}

197

229

198-- | Decode text from big endian UTF-32 encoding.

230-- | Decode text from big endian UTF-32 encoding.

231+--

232+-- If the input contains any invalid big endian UTF-32 data, an

233+-- exception will be thrown. For more control over the handling of

234+-- invalid data, use 'decodeUtf32BEWith'.

199decodeUtf32BE :: ByteString -> Text

235decodeUtf32BE :: ByteString -> Text

200decodeUtf32BE = decodeUtf32BEWith strictDecode

236decodeUtf32BE = decodeUtf32BEWith strictDecode

201{-# INLINE decodeUtf32BE #-}

237{-# INLINE decodeUtf32BE #-}

Data/Text/IO.hs+22

@@ -9,9 +9,15 @@

9-- Portability : GHC

10--

11-- Efficient locale-sensitive support for text I\/O.

12+--

13+-- Skip past the synopsis for some important notes on performance and

14+-- portability across different versions of GHC.

13module Data.Text.IO

16module Data.Text.IO

14 (

17 (

18+ -- * Performance

19+ -- $performance

20+

15 -- * Locale support

21 -- * Locale support

16 -- $locale

22 -- $locale

17 -- * File-at-a-time operations

23 -- * File-at-a-time operations

@@ -60,6 +66,22 @@ import System.IO (hGetBuffering, hFileSize, hSetBuffering, hTell)

60import System.IO.Error (isEOFError)

66import System.IO.Error (isEOFError)

61#endif

67#endif

69+-- $performance

70+-- #performance#

71+--

72+-- The functions in this module obey the runtime system's locale,

73+-- character set encoding, and line ending conversion settings.

74+--

75+-- If you know in advance that you will be working with data that has

76+-- a specific encoding (e.g. UTF-8), and your application is highly

77+-- performance sensitive, you may find that it is faster to perform

78+-- I\/O with bytestrings and to encode and decode yourself than to use

79+-- the functions in this module.

80+--

81+-- Whether this will hold depends on the version of GHC you are using,

82+-- the platform you are working on, the data you are working with, and

83+-- the encodings you are using, so be sure to test for yourself.

84+

63-- | The 'readFile' function reads a file and returns the contents of

85-- | The 'readFile' function reads a file and returns the contents of

64-- the file as a string. The entire file is read strictly, as with

86-- the file as a string. The entire file is read strictly, as with

65-- 'getContents'.

87-- 'getContents'.

Data/Text/Lazy.hs+66−66

@@ -135,10 +135,10 @@ module Data.Text.Lazy

135 , stripStart

136 , stripEnd

137 , splitAt

138− , spanBy

138+ , span

139+ , breakOn

140+ , breakOnEnd

139 , break

141 , break

140− , breakEnd

141− , breakBy

142 , group

143 , groupBy

144 , inits

@@ -146,8 +146,8 @@ module Data.Text.Lazy

146

147 -- ** Breaking into many substrings

148 -- $split

149+ , splitOn

149 , split

150 , split

150− , splitBy

151 , chunksOf

152 -- , breakSubstring

153

@@ -169,8 +169,8 @@ module Data.Text.Lazy

169 -- * Searching

170 , filter

171 , find

172− , findBy

172+ , breakOnAll

173− , partitionBy

173+ , partition

174

175 -- , findSubstring

176

@@ -630,7 +630,7 @@ replace :: Text -- ^ Text to search for

630 -> Text -- ^ Replacement text

631 -> Text -- ^ Input text

632 -> Text

633−replace s d = intercalate d . split s

633+replace s d = intercalate d . splitOn s

634{-# INLINE replace #-}

635

636-- ----------------------------------------------------------------------------

@@ -1044,13 +1044,13 @@ splitAtWord x (Chunk c@(T.Text arr off len) cs)

1044--

1045-- Examples:

1046--

1047−-- > break "::" "a::b::c" ==> ("a", "::b::c")

1047+-- > breakOn "::" "a::b::c" ==> ("a", "::b::c")

1048−-- > break "/" "foobar" ==> ("foobar", "")

1048+-- > breakOn "/" "foobar" ==> ("foobar", "")

1049--

1050-- Laws:

1051--

1052-- > append prefix match == haystack

1053−-- > where (prefix, match) = break needle haystack

1053+-- > where (prefix, match) = breakOn needle haystack

1054--

1055-- If you need to break a string by a substring repeatedly (e.g. you

1056-- want to break on every instance of a substring), use 'find'

@@ -1061,25 +1061,25 @@ splitAtWord x (Chunk c@(T.Text arr off len) cs)

1061--

1062-- In (unlikely) bad cases, this function's time complexity degrades

1063-- towards /O(n*m)/.

1064−break :: Text -> Text -> (Text, Text)

1064+breakOn :: Text -> Text -> (Text, Text)

1065−break pat src

1065+breakOn pat src

1066− | null pat = emptyError "break"

1066+ | null pat = emptyError "breakOn"

1067 | otherwise = case indices pat src of

1068 [] -> (src, empty)

1069 (x:_) -> let h :*: t = splitAtWord x src

1070 in (h, t)

1071

1072−-- | /O(n+m)/ Similar to 'break', but searches from the end of the string.

1072+-- | /O(n+m)/ Similar to 'breakOn', but searches from the end of the string.

1073--

1074-- The first element of the returned tuple is the prefix of @haystack@

1075-- up to and including the last match of @needle@. The second is the

1076-- remainder of @haystack@, following the match.

1077--

1078−-- > breakEnd "::" "a::b::c" ==> ("a::b::", "c")

1078+-- > breakOnEnd "::" "a::b::c" ==> ("a::b::", "c")

1079−breakEnd :: Text -> Text -> (Text, Text)

1079+breakOnEnd :: Text -> Text -> (Text, Text)

1080−breakEnd pat src = let (a,b) = break (reverse pat) (reverse src)

1080+breakOnEnd pat src = let (a,b) = breakOn (reverse pat) (reverse src)

1081 in (reverse b, reverse a)

1082−{-# INLINE breakEnd #-}

1082+{-# INLINE breakOnEnd #-}

1083

1084-- | /O(n+m)/ Find all non-overlapping instances of @needle@ in

1085-- @haystack@. Each element of the returned list consists of a pair:

@@ -1090,9 +1090,9 @@ breakEnd pat src = let (a,b) = break (reverse pat) (reverse src)

1090--

1091-- Examples:

1092--

1093−-- > find "::" ""

1093+-- > breakOnAll "::" ""

1094-- > ==> []

1095−-- > find "/" "a/b/c/"

1095+-- > breakOnAll "/" "a/b/c/"

1096-- > ==> [("a", "/b/c/"), ("a/b", "/c/"), ("a/b/c", "/")]

1097--

1098-- This function is strict in its first argument, and lazy in its

@@ -1102,11 +1102,11 @@ breakEnd pat src = let (a,b) = break (reverse pat) (reverse src)

1102-- towards /O(n*m)/.

1103--

1104-- The @needle@ parameter may not be empty.

1105−find :: Text -- ^ @needle@ to search for

1105+breakOnAll :: Text -- ^ @needle@ to search for

1106− -> Text -- ^ @haystack@ in which to search

1106+ -> Text -- ^ @haystack@ in which to search

1107− -> [(Text, Text)]

1107+ -> [(Text, Text)]

1108−find pat src

1108+breakOnAll pat src

1109− | null pat = emptyError "find"

1109+ | null pat = emptyError "breakOnAll"

1110 | otherwise = go 0 empty src (indices pat src)

1111 where

1112 go !n p s (x:xs) = let h :*: t = splitAtWord (x-n) s

@@ -1114,10 +1114,10 @@ find pat src

1114 in (h',t) : go x h' t xs

1115 go _ _ _ _ = []

1116

1117−-- | /O(n)/ 'breakBy' is like 'spanBy', but the prefix returned is over

1117+-- | /O(n)/ 'break' is like 'span', but the prefix returned is over

1118-- elements that fail the predicate @p@.

1119−breakBy :: (Char -> Bool) -> Text -> (Text, Text)

1119+break :: (Char -> Bool) -> Text -> (Text, Text)

1120−breakBy p t0 = break' t0

1120+break p t0 = break' t0

1121 where break' Empty = (empty, empty)

1122 break' c@(Chunk t ts) =

1123 case T.findIndex p t of

@@ -1127,13 +1127,13 @@ breakBy p t0 = break' t0

1127 | otherwise -> let (a,b) = T.splitAt n t

1128 in (Chunk a Empty, Chunk b ts)

1129

1130−-- | /O(n)/ 'spanBy', applied to a predicate @p@ and text @t@, returns

1130+-- | /O(n)/ 'span', applied to a predicate @p@ and text @t@, returns

1131-- a pair whose first element is the longest prefix (possibly empty)

1132-- of @t@ of elements that satisfy @p@, and whose second is the

1133-- remainder of the list.

1134−spanBy :: (Char -> Bool) -> Text -> (Text, Text)

1134+span :: (Char -> Bool) -> Text -> (Text, Text)

1135−spanBy p = breakBy (not . p)

1135+span p = break (not . p)

1136−{-# INLINE spanBy #-}

1136+{-# INLINE span #-}

1137

1138-- | The 'group' function takes a 'Text' and returns a list of 'Text's

1139-- such that the concatenation of the result is equal to the argument.

@@ -1152,7 +1152,7 @@ group = groupBy (==)

1152groupBy :: (Char -> Char -> Bool) -> Text -> [Text]

1153groupBy _ Empty = []

1154groupBy eq (Chunk t ts) = cons x ys : groupBy eq zs

1155− where (ys,zs) = spanBy (eq x) xs

1155+ where (ys,zs) = span (eq x) xs

1156 x = T.unsafeHead t

1157 xs = chunk (T.unsafeTail t) ts

1158

@@ -1184,37 +1184,37 @@ tails ts@(Chunk t ts')

1184--

1185-- Examples:

1186--

1187−-- > split "\r\n" "a\r\nb\r\nd\r\ne" == ["a","b","d","e"]

1187+-- > splitOn "\r\n" "a\r\nb\r\nd\r\ne" == ["a","b","d","e"]

1188−-- > split "aaa" "aaaXaaaXaaaXaaa" == ["","X","X","X",""]

1188+-- > splitOn "aaa" "aaaXaaaXaaaXaaa" == ["","X","X","X",""]

1189−-- > split "x" "x" == ["",""]

1189+-- > splitOn "x" "x" == ["",""]

1190--

1191-- and

1192--

1193−-- > intercalate s . split s == id

1193+-- > intercalate s . splitOn s == id

1194−-- > split (singleton c) == splitBy (==c)

1194+-- > splitOn (singleton c) == split (==c)

1195--

1196-- This function is strict in its first argument, and lazy in its

1197-- second.

1198--

1199-- In (unlikely) bad cases, this function's time complexity degrades

1200-- towards /O(n*m)/.

1201−split :: Text -- ^ Text to split on

1201+splitOn :: Text -- ^ Text to split on

1202− -> Text -- ^ Input text

1202+ -> Text -- ^ Input text

1203− -> [Text]

1203+ -> [Text]

1204−split pat src

1204+splitOn pat src

1205− | null pat = emptyError "split"

1205+ | null pat = emptyError "splitOn"

1206− | isSingleton pat = splitBy (== head pat) src

1206+ | isSingleton pat = split (== head pat) src

1207 | otherwise = go 0 (indices pat src) src

1208 where

1209 go _ [] cs = [cs]

1210 go !i (x:xs) cs = let h :*: t = splitAtWord (x-i) cs

1211 in h : go (x+l) xs (dropWords l t)

1212 l = foldlChunks (\a (T.Text _ _ b) -> a + fromIntegral b) 0 pat

1213−{-# INLINE [1] split #-}

1213+{-# INLINE [1] splitOn #-}

1214

1215{-# RULES

1216−"LAZY TEXT split/singleton -> splitBy/==" [~1] forall c t.

1216+"LAZY TEXT splitOn/singleton -> split/==" [~1] forall c t.

1217− split (singleton c) t = splitBy (==c) t

1217+ splitOn (singleton c) t = split (==c) t

1218 #-}

1219

1220-- | /O(n)/ Splits a 'Text' into components delimited by separators,

@@ -1222,16 +1222,16 @@ split pat src

1222-- resulting components do not contain the separators. Two adjacent

1223-- separators result in an empty component in the output. eg.

1224--

1225−-- > splitBy (=='a') "aabbaca" == ["","","bb","c",""]

1225+-- > split (=='a') "aabbaca" == ["","","bb","c",""]

1226−-- > splitBy (=='a') [] == [""]

1226+-- > split (=='a') [] == [""]

1227−splitBy :: (Char -> Bool) -> Text -> [Text]

1227+split :: (Char -> Bool) -> Text -> [Text]

1228−splitBy _ Empty = [Empty]

1228+split _ Empty = [Empty]

1229−splitBy p (Chunk t0 ts0) = comb [] (T.splitBy p t0) ts0

1229+split p (Chunk t0 ts0) = comb [] (T.split p t0) ts0

1230 where comb acc (s:[]) Empty = revChunks (s:acc) : []

1231− comb acc (s:[]) (Chunk t ts) = comb (s:acc) (T.splitBy p t) ts

1231+ comb acc (s:[]) (Chunk t ts) = comb (s:acc) (T.split p t) ts

1232 comb acc (s:ss) ts = revChunks (s:acc) : comb [] ss ts

1233− comb _ [] _ = impossibleError "splitBy"

1233+ comb _ [] _ = impossibleError "split"

1234−{-# INLINE splitBy #-}

1234+{-# INLINE split #-}

1235

1236-- | /O(n)/ Splits a 'Text' into components of length @k@. The last

1237-- element may be shorter than the other chunks, depending on the

@@ -1251,14 +1251,14 @@ chunksOf k = go

1251-- newline 'Char's. The resulting strings do not contain newlines.

1252lines :: Text -> [Text]

1253lines Empty = []

1254−lines t = let (l,t') = breakBy ((==) '\n') t

1254+lines t = let (l,t') = break ((==) '\n') t

1255 in l : if null t' then []

1256 else lines (tail t')

1257

1258-- | /O(n)/ Breaks a 'Text' up into a list of words, delimited by 'Char's

1259-- representing white space.

1260words :: Text -> [Text]

1261−words = L.filter (not . null) . splitBy isSpace

1261+words = L.filter (not . null) . split isSpace

1262{-# INLINE words #-}

1263

1264-- | /O(n)/ Joins lines, after appending a terminating newline to

@@ -1378,21 +1378,21 @@ filter :: (Char -> Bool) -> Text -> Text

1378filter p t = unstream (S.filter p (stream t))

1379{-# INLINE filter #-}

1380

1381−-- | /O(n)/ The 'findBy' function takes a predicate and a 'Text', and

1381+-- | /O(n)/ The 'find' function takes a predicate and a 'Text', and

1382-- returns the first element in matching the predicate, or 'Nothing'

1383-- if there is no such element.

1384−findBy :: (Char -> Bool) -> Text -> Maybe Char

1384+find :: (Char -> Bool) -> Text -> Maybe Char

1385−findBy p t = S.findBy p (stream t)

1385+find p t = S.findBy p (stream t)

1386−{-# INLINE findBy #-}

1386+{-# INLINE find #-}

1387

1388−-- | /O(n)/ The 'partitionBy' function takes a predicate and a 'Text',

1388+-- | /O(n)/ The 'partition' function takes a predicate and a 'Text',

1389-- and returns the pair of 'Text's with elements which do and do not

1390-- satisfy the predicate, respectively; i.e.

1391--

1392−-- > partitionBy p t == (filter p t, filter (not . p) t)

1392+-- > partition p t == (filter p t, filter (not . p) t)

1393−partitionBy :: (Char -> Bool) -> Text -> (Text, Text)

1393+partition :: (Char -> Bool) -> Text -> (Text, Text)

1394−partitionBy p t = (filter p t, filter (not . p) t)

1394+partition p t = (filter p t, filter (not . p) t)

1395−{-# INLINE partitionBy #-}

1395+{-# INLINE partition #-}

1396

1397-- | /O(n)/ 'Text' index (subscript) operator, starting from 0.

1398index :: Text -> Int64 -> Char

Data/Text/Lazy/Encoding.hs+36−2

@@ -12,12 +12,13 @@

12-- Functions for converting lazy 'Text' values to and from lazy

13-- 'ByteString', using several standard encodings.

14--

15−-- To make use of a much larger variety of encodings, use the @text-icu@

15+-- To gain access to a much larger variety of encodings, use the

16−-- package.

16+-- @text-icu@ package: <http://hackage.haskell.org/package/text-icu>

18module Data.Text.Lazy.Encoding

19 (

20 -- * Decoding ByteStrings to Text

21+ -- $strict

21 decodeASCII

22 decodeASCII

22 , decodeUtf8

23 , decodeUtf8

23 , decodeUtf16LE

24 , decodeUtf16LE

@@ -51,6 +52,19 @@ import qualified Data.Text.Encoding as TE

51import qualified Data.Text.Lazy.Encoding.Fusion as E

52import qualified Data.Text.Lazy.Encoding.Fusion as E

52import qualified Data.Text.Lazy.Fusion as F

53import qualified Data.Text.Lazy.Fusion as F

55+-- $strict

56+--

57+-- All of the single-parameter functions for decoding bytestrings

58+-- encoded in one of the Unicode Transformation Formats (UTF) operate

59+-- in a /strict/ mode: each will throw an exception if given invalid

60+-- input.

61+--

62+-- Each function has a variant, whose name is suffixed with -'With',

63+-- that gives greater control over the handling of decoding errors.

64+-- For instance, 'decodeUtf8' will throw an exception, but

65+-- 'decodeUtf8With' allows the programmer to determine what to do on a

66+-- decoding error.

67+

54-- | Decode a 'ByteString' containing 7-bit ASCII encoded text.

68-- | Decode a 'ByteString' containing 7-bit ASCII encoded text.

55decodeASCII :: B.ByteString -> Text

69decodeASCII :: B.ByteString -> Text

56decodeASCII bs = foldr (chunk . TE.decodeASCII) empty (B.toChunks bs)

70decodeASCII bs = foldr (chunk . TE.decodeASCII) empty (B.toChunks bs)

@@ -95,6 +109,10 @@ decodeUtf8With onErr bs0 = fast bs0

95{-# INLINE[0] decodeUtf8With #-}

109{-# INLINE[0] decodeUtf8With #-}

110

97-- | Decode a 'ByteString' containing UTF-8 encoded text.

111-- | Decode a 'ByteString' containing UTF-8 encoded text.

112+--

113+-- If the input contains any invalid UTF-8 data, an exception will be

114+-- thrown. For more control over the handling of invalid data, use

115+-- 'decodeUtf8With'.

98decodeUtf8 :: B.ByteString -> Text

116decodeUtf8 :: B.ByteString -> Text

99decodeUtf8 = decodeUtf8With strictDecode

117decodeUtf8 = decodeUtf8With strictDecode

100{-# INLINE[0] decodeUtf8 #-}

118{-# INLINE[0] decodeUtf8 #-}

@@ -113,6 +131,10 @@ decodeUtf16LEWith onErr bs = F.unstream (E.streamUtf16LE onErr bs)

113{-# INLINE decodeUtf16LEWith #-}

131{-# INLINE decodeUtf16LEWith #-}

114

132

115-- | Decode text from little endian UTF-16 encoding.

133-- | Decode text from little endian UTF-16 encoding.

134+--

135+-- If the input contains any invalid little endian UTF-16 data, an

136+-- exception will be thrown. For more control over the handling of

137+-- invalid data, use 'decodeUtf16LEWith'.

116decodeUtf16LE :: B.ByteString -> Text

138decodeUtf16LE :: B.ByteString -> Text

117decodeUtf16LE = decodeUtf16LEWith strictDecode

139decodeUtf16LE = decodeUtf16LEWith strictDecode

118{-# INLINE decodeUtf16LE #-}

140{-# INLINE decodeUtf16LE #-}

@@ -123,6 +145,10 @@ decodeUtf16BEWith onErr bs = F.unstream (E.streamUtf16BE onErr bs)

123{-# INLINE decodeUtf16BEWith #-}

145{-# INLINE decodeUtf16BEWith #-}

124

146

125-- | Decode text from big endian UTF-16 encoding.

147-- | Decode text from big endian UTF-16 encoding.

148+--

149+-- If the input contains any invalid big endian UTF-16 data, an

150+-- exception will be thrown. For more control over the handling of

151+-- invalid data, use 'decodeUtf16BEWith'.

126decodeUtf16BE :: B.ByteString -> Text

152decodeUtf16BE :: B.ByteString -> Text

127decodeUtf16BE = decodeUtf16BEWith strictDecode

153decodeUtf16BE = decodeUtf16BEWith strictDecode

128{-# INLINE decodeUtf16BE #-}

154{-# INLINE decodeUtf16BE #-}

@@ -143,6 +169,10 @@ decodeUtf32LEWith onErr bs = F.unstream (E.streamUtf32LE onErr bs)

143{-# INLINE decodeUtf32LEWith #-}

169{-# INLINE decodeUtf32LEWith #-}

144

170

145-- | Decode text from little endian UTF-32 encoding.

171-- | Decode text from little endian UTF-32 encoding.

172+--

173+-- If the input contains any invalid little endian UTF-32 data, an

174+-- exception will be thrown. For more control over the handling of

175+-- invalid data, use 'decodeUtf32LEWith'.

146decodeUtf32LE :: B.ByteString -> Text

176decodeUtf32LE :: B.ByteString -> Text

147decodeUtf32LE = decodeUtf32LEWith strictDecode

177decodeUtf32LE = decodeUtf32LEWith strictDecode

148{-# INLINE decodeUtf32LE #-}

178{-# INLINE decodeUtf32LE #-}

@@ -153,6 +183,10 @@ decodeUtf32BEWith onErr bs = F.unstream (E.streamUtf32BE onErr bs)

153{-# INLINE decodeUtf32BEWith #-}

183{-# INLINE decodeUtf32BEWith #-}

154

184

155-- | Decode text from big endian UTF-32 encoding.

185-- | Decode text from big endian UTF-32 encoding.

186+--

187+-- If the input contains any invalid big endian UTF-32 data, an

188+-- exception will be thrown. For more control over the handling of

189+-- invalid data, use 'decodeUtf32BEWith'.

156decodeUtf32BE :: B.ByteString -> Text

190decodeUtf32BE :: B.ByteString -> Text

157decodeUtf32BE = decodeUtf32BEWith strictDecode

191decodeUtf32BE = decodeUtf32BEWith strictDecode

158{-# INLINE decodeUtf32BE #-}

192{-# INLINE decodeUtf32BE #-}

Data/Text/Lazy/IO.hs+21

@@ -9,9 +9,15 @@

9-- Portability : GHC

10--

11-- Efficient locale-sensitive support for lazy text I\/O.

12+--

13+-- Skip past the synopsis for some important notes on performance and

14+-- portability across different versions of GHC.

13module Data.Text.Lazy.IO

16module Data.Text.Lazy.IO

14 (

17 (

18+ -- * Performance

19+ -- $performance

20+

15 -- * Locale support

21 -- * Locale support

16 -- $locale

22 -- $locale

17 -- * File-at-a-time operations

23 -- * File-at-a-time operations

@@ -58,6 +64,21 @@ import System.IO.Error (isEOFError)

58import System.IO.Unsafe (unsafeInterleaveIO)

64import System.IO.Unsafe (unsafeInterleaveIO)

59#endif

65#endif

67+-- $performance

68+--

69+-- The functions in this module obey the runtime system's locale,

70+-- character set encoding, and line ending conversion settings.

71+--

72+-- If you know in advance that you will be working with data that has

73+-- a specific encoding (e.g. UTF-8), and your application is highly

74+-- performance sensitive, you may find that it is faster to perform

75+-- I\/O with bytestrings and to encode and decode yourself than to use

76+-- the functions in this module.

77+--

78+-- Whether this will hold depends on the version of GHC you are using,

79+-- the platform you are working on, the data you are working with, and

80+-- the encodings you are using, so be sure to test for yourself.

81+

61-- | Read a file and return its contents as a string. The file is

82-- | Read a file and return its contents as a string. The file is

62-- read lazily, as with 'getContents'.

83-- read lazily, as with 'getContents'.

63readFile :: FilePath -> IO Text

84readFile :: FilePath -> IO Text

Data/Text/Lazy/Read.hs+6−4

@@ -39,14 +39,15 @@ type Reader a = Text -> Either String (a,Text)

39--

40-- /Note/: For fixed-width integer types, this function does not

41-- attempt to detect overflow, so a sufficiently long input may give

42−-- incorrect results.

42+-- incorrect results. If you are worried about overflow, use

43+-- 'Integer' for your result type.

43decimal :: Integral a => Reader a

44decimal :: Integral a => Reader a

44{-# SPECIALIZE decimal :: Reader Int #-}

45{-# SPECIALIZE decimal :: Reader Int #-}

45{-# SPECIALIZE decimal :: Reader Integer #-}

46{-# SPECIALIZE decimal :: Reader Integer #-}

46decimal txt

47decimal txt

47 | T.null h = Left "input does not start with a digit"

48 | T.null h = Left "input does not start with a digit"

48 | otherwise = Right (T.foldl' go 0 h, t)

49 | otherwise = Right (T.foldl' go 0 h, t)

49− where (h,t) = T.spanBy isDigit txt

50+ where (h,t) = T.span isDigit txt

50 go n d = (n * 10 + fromIntegral (digitToInt d))

51 go n d = (n * 10 + fromIntegral (digitToInt d))

52-- | Read a hexadecimal integer, consisting of an optional leading

53-- | Read a hexadecimal integer, consisting of an optional leading

@@ -59,7 +60,8 @@ decimal txt

59--

60--

60-- /Note/: For fixed-width integer types, this function does not

61-- /Note/: For fixed-width integer types, this function does not

61-- attempt to detect overflow, so a sufficiently long input may give

62-- attempt to detect overflow, so a sufficiently long input may give

62−-- incorrect results.

63+-- incorrect results. If you are worried about overflow, use

64+-- 'Integer' for your result type.

63hexadecimal :: Integral a => Reader a

65hexadecimal :: Integral a => Reader a

64{-# SPECIALIZE hexadecimal :: Reader Int #-}

66{-# SPECIALIZE hexadecimal :: Reader Int #-}

65{-# SPECIALIZE hexadecimal :: Reader Integer #-}

67{-# SPECIALIZE hexadecimal :: Reader Integer #-}

@@ -74,7 +76,7 @@ hex :: Integral a => Reader a

74hex txt

76hex txt

75 | T.null h = Left "input does not start with a hexadecimal digit"

77 | T.null h = Left "input does not start with a hexadecimal digit"

76 | otherwise = Right (T.foldl' go 0 h, t)

78 | otherwise = Right (T.foldl' go 0 h, t)

77− where (h,t) = T.spanBy isHexDigit txt

79+ where (h,t) = T.span isHexDigit txt

78 go n d = (n * 16 + fromIntegral (hexDigitToInt d))

80 go n d = (n * 16 + fromIntegral (hexDigitToInt d))

80hexDigitToInt :: Char -> Int

82hexDigitToInt :: Char -> Int

Data/Text/Read.hs+7−5

@@ -39,14 +39,15 @@ type Reader a = Text -> Either String (a,Text)

39--

40-- /Note/: For fixed-width integer types, this function does not

41-- attempt to detect overflow, so a sufficiently long input may give

42−-- incorrect results.

42+-- incorrect results. If you are worried about overflow, use

43+-- 'Integer' for your result type.

43decimal :: Integral a => Reader a

44decimal :: Integral a => Reader a

44{-# SPECIALIZE decimal :: Reader Int #-}

45{-# SPECIALIZE decimal :: Reader Int #-}

45{-# SPECIALIZE decimal :: Reader Integer #-}

46{-# SPECIALIZE decimal :: Reader Integer #-}

46decimal txt

47decimal txt

47 | T.null h = Left "input does not start with a digit"

48 | T.null h = Left "input does not start with a digit"

48 | otherwise = Right (T.foldl' go 0 h, t)

49 | otherwise = Right (T.foldl' go 0 h, t)

49− where (h,t) = T.spanBy isDigit txt

50+ where (h,t) = T.span isDigit txt

50 go n d = (n * 10 + fromIntegral (digitToInt d))

51 go n d = (n * 10 + fromIntegral (digitToInt d))

52-- | Read a hexadecimal integer, consisting of an optional leading

53-- | Read a hexadecimal integer, consisting of an optional leading

@@ -59,7 +60,8 @@ decimal txt

59--

60--

60-- /Note/: For fixed-width integer types, this function does not

61-- /Note/: For fixed-width integer types, this function does not

61-- attempt to detect overflow, so a sufficiently long input may give

62-- attempt to detect overflow, so a sufficiently long input may give

62−-- incorrect results.

63+-- incorrect results. If you are worried about overflow, use

64+-- 'Integer' for your result type.

63hexadecimal :: Integral a => Reader a

65hexadecimal :: Integral a => Reader a

64{-# SPECIALIZE hexadecimal :: Reader Int #-}

66{-# SPECIALIZE hexadecimal :: Reader Int #-}

65{-# SPECIALIZE hexadecimal :: Reader Integer #-}

67{-# SPECIALIZE hexadecimal :: Reader Integer #-}

@@ -74,7 +76,7 @@ hex :: Integral a => Reader a

74hex txt

76hex txt

75 | T.null h = Left "input does not start with a hexadecimal digit"

77 | T.null h = Left "input does not start with a hexadecimal digit"

76 | otherwise = Right (T.foldl' go 0 h, t)

78 | otherwise = Right (T.foldl' go 0 h, t)

77− where (h,t) = T.spanBy isHexDigit txt

79+ where (h,t) = T.span isHexDigit txt

78 go n d = (n * 16 + fromIntegral (hexDigitToInt d))

80 go n d = (n * 16 + fromIntegral (hexDigitToInt d))

80hexDigitToInt :: Char -> Int

82hexDigitToInt :: Char -> Int

@@ -99,7 +101,7 @@ signed f = runP (signa (P f))

99-- by the 'read' function, with the exception that a trailing @\'.\'@

101-- by the 'read' function, with the exception that a trailing @\'.\'@

100-- or @\'e\'@ /not/ followed by a number is not consumed.

102-- or @\'e\'@ /not/ followed by a number is not consumed.

101--

103--

102−-- Examples:

104+-- Examples (with behaviour identical to 'read'):

103--

105--

104-- >rational "3" == Right (3.0, "")

106-- >rational "3" == Right (3.0, "")

105-- >rational "3.1" == Right (3.1, "")

107-- >rational "3.1" == Right (3.1, "")

tests/Properties.hs+62−62

@@ -265,13 +265,13 @@ t_reverse = L.reverse `eqP` (unpackS . T.reverse)

265tl_reverse = L.reverse `eqP` (unpackS . TL.reverse)

266t_reverse_short n = L.reverse `eqP` (unpackS . S.reverse . shorten n . S.stream)

267

268−t_replace s d = (L.intercalate d . split s) `eqP`

268+t_replace s d = (L.intercalate d . splitOn s) `eqP`

269 (unpackS . T.replace (T.pack s) (T.pack d))

270−tl_replace s d = (L.intercalate d . split s) `eqP`

270+tl_replace s d = (L.intercalate d . splitOn s) `eqP`

271 (unpackS . TL.replace (TL.pack s) (TL.pack d))

272

273−split :: (Eq a) => [a] -> [a] -> [[a]]

273+splitOn :: (Eq a) => [a] -> [a] -> [[a]]

274−split pat src0

274+splitOn pat src0

275 | l == 0 = error "empty"

276 | otherwise = go src0

277 where

@@ -482,25 +482,25 @@ t_strip = T.dropAround isSpace `eq` T.strip

482tl_strip = TL.dropAround isSpace `eq` TL.strip

483t_splitAt n = L.splitAt n `eqP` (unpack2 . T.splitAt n)

484tl_splitAt n = L.splitAt n `eqP` (unpack2 . TL.splitAt (fromIntegral n))

485−t_spanBy p = L.span p `eqP` (unpack2 . T.spanBy p)

485+t_span p = L.span p `eqP` (unpack2 . T.span p)

486−tl_spanBy p = L.span p `eqP` (unpack2 . TL.spanBy p)

486+tl_span p = L.span p `eqP` (unpack2 . TL.span p)

487

488−t_break_id s = squid `eq` (uncurry T.append . T.break s)

488+t_breakOn_id s = squid `eq` (uncurry T.append . T.breakOn s)

489 where squid t | T.null s = error "empty"

490 | otherwise = t

491−tl_break_id s = squid `eq` (uncurry TL.append . TL.break s)

491+tl_breakOn_id s = squid `eq` (uncurry TL.append . TL.breakOn s)

492 where squid t | TL.null s = error "empty"

493 | otherwise = t

494−t_break_start (NotEmpty s) t = let (_,m) = T.break s t

494+t_breakOn_start (NotEmpty s) t = let (_,m) = T.breakOn s t

495 in T.null m || s `T.isPrefixOf` m

496−tl_break_start (NotEmpty s) t = let (_,m) = TL.break s t

496+tl_breakOn_start (NotEmpty s) t = let (_,m) = TL.breakOn s t

497 in TL.null m || s `TL.isPrefixOf` m

498−t_breakEnd_end (NotEmpty s) t = let (m,_) = T.breakEnd s t

498+t_breakOnEnd_end (NotEmpty s) t = let (m,_) = T.breakOnEnd s t

499 in T.null m || s `T.isSuffixOf` m

500−tl_breakEnd_end (NotEmpty s) t = let (m,_) = TL.breakEnd s t

500+tl_breakOnEnd_end (NotEmpty s) t = let (m,_) = TL.breakOnEnd s t

501 in TL.null m || s `TL.isSuffixOf` m

502−t_breakBy p = L.break p `eqP` (unpack2 . T.breakBy p)

502+t_break p = L.break p `eqP` (unpack2 . T.break p)

503−tl_breakBy p = L.break p `eqP` (unpack2 . TL.breakBy p)

503+tl_break p = L.break p `eqP` (unpack2 . TL.break p)

504t_group = L.group `eqP` (map unpackS . T.group)

505tl_group = L.group `eqP` (map unpackS . TL.group)

506t_groupBy p = L.groupBy p `eqP` (map unpackS . T.groupBy p)

@@ -511,33 +511,33 @@ t_tails = L.tails `eqP` (map unpackS . T.tails)

511tl_tails = L.tails `eqP` (map unpackS . TL.tails)

512t_findAppendId (NotEmpty s) = unsquare $ \ts ->

513 let t = T.intercalate s ts

514− in all (==t) $ map (uncurry T.append) (T.find s t)

514+ in all (==t) $ map (uncurry T.append) (T.breakOnAll s t)

515tl_findAppendId (NotEmpty s) = unsquare $ \ts ->

516 let t = TL.intercalate s ts

517− in all (==t) $ map (uncurry TL.append) (TL.find s t)

517+ in all (==t) $ map (uncurry TL.append) (TL.breakOnAll s t)

518−t_findContains (NotEmpty s) = all (T.isPrefixOf s . snd) . T.find s .

518+t_findContains (NotEmpty s) = all (T.isPrefixOf s . snd) . T.breakOnAll s .

519 T.intercalate s

520tl_findContains (NotEmpty s) = all (TL.isPrefixOf s . snd) .

521− TL.find s . TL.intercalate s

521+ TL.breakOnAll s . TL.intercalate s

522sl_filterCount c = (L.genericLength . L.filter (==c)) `eqP` SL.countChar c

523−t_findCount s = (L.length . T.find s) `eq` T.count s

523+t_findCount s = (L.length . T.breakOnAll s) `eq` T.count s

524−tl_findCount s = (L.genericLength . TL.find s) `eq` TL.count s

524+tl_findCount s = (L.genericLength . TL.breakOnAll s) `eq` TL.count s

525−

525+

526−t_split_split s = (T.split s `eq` Slow.split s) . T.intercalate s

526+t_splitOn_split s = (T.splitOn s `eq` Slow.splitOn s) . T.intercalate s

527−tl_split_split s = ((TL.split (TL.fromStrict s) . TL.fromStrict) `eq`

527+tl_splitOn_split s = ((TL.splitOn (TL.fromStrict s) . TL.fromStrict) `eq`

528− (map TL.fromStrict . T.split s)) . T.intercalate s

528+ (map TL.fromStrict . T.splitOn s)) . T.intercalate s

529−t_split_i (NotEmpty t) = id `eq` (T.intercalate t . T.split t)

529+t_splitOn_i (NotEmpty t) = id `eq` (T.intercalate t . T.splitOn t)

530−tl_split_i (NotEmpty t) = id `eq` (TL.intercalate t . TL.split t)

530+tl_splitOn_i (NotEmpty t) = id `eq` (TL.intercalate t . TL.splitOn t)

531−

531+

532−t_splitBy p = splitBy p `eqP` (map unpackS . T.splitBy p)

532+t_split p = split p `eqP` (map unpackS . T.split p)

533−t_splitBy_count c = (L.length . T.splitBy (==c)) `eq`

533+t_split_count c = (L.length . T.split (==c)) `eq`

534− ((1+) . T.count (T.singleton c))

534+ ((1+) . T.count (T.singleton c))

535−t_splitBy_split c = T.splitBy (==c) `eq` T.split (T.singleton c)

535+t_split_splitOn c = T.split (==c) `eq` T.splitOn (T.singleton c)

536−tl_splitBy p = splitBy p `eqP` (map unpackS . TL.splitBy p)

536+tl_split p = split p `eqP` (map unpackS . TL.split p)

537−

537+

538−splitBy :: (a -> Bool) -> [a] -> [[a]]

538+split :: (a -> Bool) -> [a] -> [[a]]

539−splitBy _ [] = [[]]

539+split _ [] = [[]]

540−splitBy p xs = loop xs

540+split p xs = loop xs

541 where loop s | null s' = [l]

542 | otherwise = l : loop (tail s')

543 where (l, s') = break p s

@@ -597,10 +597,10 @@ sf_filter q p = (L.filter p . L.filter q) `eqP`

597t_filter p = L.filter p `eqP` (unpackS . T.filter p)

598tl_filter p = L.filter p `eqP` (unpackS . TL.filter p)

599sf_findBy q p = (L.find p . L.filter q) `eqP` (S.findBy p . S.filter q)

600−t_findBy p = L.find p `eqP` T.findBy p

600+t_find p = L.find p `eqP` T.find p

601−tl_findBy p = L.find p `eqP` TL.findBy p

601+tl_find p = L.find p `eqP` TL.find p

602−t_partition p = L.partition p `eqP` (unpack2 . T.partitionBy p)

602+t_partition p = L.partition p `eqP` (unpack2 . T.partition p)

603−tl_partition p = L.partition p `eqP` (unpack2 . TL.partitionBy p)

603+tl_partition p = L.partition p `eqP` (unpack2 . TL.partition p)

604

605sf_index p s = forAll (choose (-l,l*2))

606 ((L.filter p s L.!!) `eq` S.index (S.filter p $ packS s))

@@ -613,8 +613,8 @@ tl_index s = forAll (choose (-l,l*2))

613 where l = L.length s

614

615t_findIndex p = L.findIndex p `eqP` T.findIndex p

616−t_count (NotEmpty t) = (subtract 1 . L.length . T.split t) `eq` T.count t

616+t_count (NotEmpty t) = (subtract 1 . L.length . T.splitOn t) `eq` T.count t

617−tl_count (NotEmpty t) = (subtract 1 . L.genericLength . TL.split t) `eq`

617+tl_count (NotEmpty t) = (subtract 1 . L.genericLength . TL.splitOn t) `eq`

618 TL.count t

619t_zip s = L.zip s `eqP` T.zip (packS s)

620tl_zip s = L.zip s `eqP` TL.zip (packS s)

@@ -1052,16 +1052,16 @@ tests = [

1052 testProperty "tl_strip" tl_strip,

1053 testProperty "t_splitAt" t_splitAt,

1054 testProperty "tl_splitAt" tl_splitAt,

1055− testProperty "t_spanBy" t_spanBy,

1055+ testProperty "t_span" t_span,

1056− testProperty "tl_spanBy" tl_spanBy,

1056+ testProperty "tl_span" tl_span,

1057− testProperty "t_break_id" t_break_id,

1057+ testProperty "t_breakOn_id" t_breakOn_id,

1058− testProperty "tl_break_id" tl_break_id,

1058+ testProperty "tl_breakOn_id" tl_breakOn_id,

1059− testProperty "t_break_start" t_break_start,

1059+ testProperty "t_breakOn_start" t_breakOn_start,

1060− testProperty "tl_break_start" tl_break_start,

1060+ testProperty "tl_breakOn_start" tl_breakOn_start,

1061− testProperty "t_breakEnd_end" t_breakEnd_end,

1061+ testProperty "t_breakOnEnd_end" t_breakOnEnd_end,

1062− testProperty "tl_breakEnd_end" tl_breakEnd_end,

1062+ testProperty "tl_breakOnEnd_end" tl_breakOnEnd_end,

1063− testProperty "t_breakBy" t_breakBy,

1063+ testProperty "t_break" t_break,

1064− testProperty "tl_breakBy" tl_breakBy,

1064+ testProperty "tl_break" tl_break,

1065 testProperty "t_group" t_group,

1066 testProperty "tl_group" tl_group,

1067 testProperty "t_groupBy" t_groupBy,

@@ -1080,14 +1080,14 @@ tests = [

1080 testProperty "sl_filterCount" sl_filterCount,

1081 testProperty "t_findCount" t_findCount,

1082 testProperty "tl_findCount" tl_findCount,

1083− testProperty "t_split_split" t_split_split,

1083+ testProperty "t_splitOn_split" t_splitOn_split,

1084− testProperty "tl_split_split" tl_split_split,

1084+ testProperty "tl_splitOn_split" tl_splitOn_split,

1085− testProperty "t_split_i" t_split_i,

1085+ testProperty "t_splitOn_i" t_splitOn_i,

1086− testProperty "tl_split_i" tl_split_i,

1086+ testProperty "tl_splitOn_i" tl_splitOn_i,

1087− testProperty "t_splitBy" t_splitBy,

1087+ testProperty "t_split" t_split,

1088− testProperty "t_splitBy_count" t_splitBy_count,

1088+ testProperty "t_split_count" t_split_count,

1089− testProperty "t_splitBy_split" t_splitBy_split,

1089+ testProperty "t_split_splitOn" t_split_splitOn,

1090− testProperty "tl_splitBy" tl_splitBy,

1090+ testProperty "tl_split" tl_split,

1091 testProperty "t_chunksOf_same_lengths" t_chunksOf_same_lengths,

1092 testProperty "t_chunksOf_length" t_chunksOf_length,

1093 testProperty "tl_chunksOf" tl_chunksOf

@@ -1130,8 +1130,8 @@ tests = [

1130 testProperty "t_filter" t_filter,

1131 testProperty "tl_filter" tl_filter,

1132 testProperty "sf_findBy" sf_findBy,

1133− testProperty "t_findBy" t_findBy,

1133+ testProperty "t_find" t_find,

1134− testProperty "tl_findBy" tl_findBy,

1134+ testProperty "tl_find" tl_find,

1135 testProperty "t_partition" t_partition,

1136 testProperty "tl_partition" tl_partition

1137 ],

tests/SlowFunctions.hs+7−7

@@ -3,7 +3,7 @@

3module SlowFunctions

4 (

5 indices

6− , split

6+ , splitOn

7 ) where

9import qualified Data.Text as T

@@ -23,12 +23,12 @@ indices needle@(Text _narr _noff nlen) haystack@(Text harr hoff hlen)

23 where t = Text harr (hoff+i) (hlen-i)

24 d = iter_ haystack i

26−split :: T.Text -- ^ Text to split on

26+splitOn :: T.Text -- ^ Text to split on

27− -> T.Text -- ^ Input text

27+ -> T.Text -- ^ Input text

28− -> [T.Text]

28+ -> [T.Text]

29−split pat src0

29+splitOn pat src0

30− | T.null pat = error "split: empty"

30+ | T.null pat = error "splitOn: empty"

31− | l == 1 = T.splitBy (== (unsafeHead pat)) src0

31+ | l == 1 = T.split (== (unsafeHead pat)) src0

32 | otherwise = go src0

33 where

34 l = T.length pat

tests/benchmarks/CaseMap.hsnew file+21

@@ -0,0 +1,21 @@

1+import Control.Exception

2+import Control.Monad

3+import Data.Time.Clock

4+import Data.ByteString as B

5+import Data.Text.Encoding as T

6+import Data.Text as T

7+import System.Environment

9+time act = do

10+ start <- getCurrentTime

11+ act

12+ end <- getCurrentTime

13+ let d = diffUTCTime end start

14+ print d

15+

16+main = do

17+ args <- getArgs

18+ forM_ args $ \f -> do

19+ t <- T.decodeUtf8 `fmap` B.readFile f

20+ evaluate t

21+ time $ evaluate (T.toUpper t)

tests/benchmarks/casemap.pynew file+15

@@ -0,0 +1,15 @@

1+#!/usr/bin/env python

3+import sys, time

5+def timeit(f):

6+ start = time.time()

7+ f()

8+ end = time.time()

9+ print end - start

10+

11+for f in sys.argv[1:]:

12+ s = open(f).read()

13+ u = s.decode('utf8')

14+ timeit(lambda: s.upper())

15+ timeit(lambda: u.upper())

text.cabal+1−1

@@ -1,5 +1,5 @@

1name: text

2−version: 0.10.0.2

2+version: 0.11.0.0

3homepage: http://bitbucket.org/bos/text

4bug-reports: http://bitbucket.org/bos/text/issues

5synopsis: An efficient packed Unicode text type.